mindstudio-probe 8.3.2__py3-none-any.whl → 26.0.0a1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {mindstudio_probe-8.3.2.dist-info → mindstudio_probe-26.0.0a1.dist-info}/METADATA +26 -14
- mindstudio_probe-26.0.0a1.dist-info/RECORD +498 -0
- {mindstudio_probe-8.3.2.dist-info → mindstudio_probe-26.0.0a1.dist-info}/WHEEL +1 -1
- mindstudio_probe-26.0.0a1.dist-info/entry_points.txt +5 -0
- mindstudio_probe-26.0.0a1.dist-info/licenses/LICENSE +124 -0
- mindstudio_probe-26.0.0a1.dist-info/top_level.txt +2 -0
- msprobe/__init__.py +12 -13
- msprobe/config.json +9 -31
- msprobe/core/__init__.py +12 -11
- msprobe/core/acc_check/acc_check_cli.py +145 -0
- msprobe/core/common/const.py +97 -38
- msprobe/core/common/db_manager.py +133 -12
- msprobe/core/common/decorator.py +12 -11
- msprobe/core/common/exceptions.py +12 -11
- msprobe/core/common/file_utils.py +101 -25
- msprobe/core/common/framework_adapter.py +36 -25
- msprobe/core/common/global_lock.py +12 -11
- msprobe/core/common/inplace_op_checker.py +12 -11
- msprobe/core/common/log.py +22 -11
- msprobe/core/common/megatron_utils.py +566 -11
- msprobe/core/common/parallel_state.py +12 -11
- msprobe/core/common/runtime.py +12 -11
- msprobe/core/common/utils.py +41 -41
- msprobe/core/compare/acc_compare.py +361 -104
- msprobe/core/compare/atb_data_compare.py +422 -0
- msprobe/core/compare/auto_compare.py +134 -0
- msprobe/core/compare/check.py +14 -17
- msprobe/core/compare/compare_cli.py +72 -149
- msprobe/core/compare/config.py +12 -13
- msprobe/core/compare/diff_analyze/first_diff_analyze.py +28 -15
- msprobe/core/compare/diff_analyze/ignore_op_list.yaml +3 -0
- msprobe/core/compare/find_first/analyzer.py +18 -18
- msprobe/core/compare/find_first/graph.py +12 -11
- msprobe/core/compare/find_first/utils.py +13 -12
- msprobe/core/compare/indicator_analysis/__init__.py +15 -0
- msprobe/core/compare/indicator_analysis/algorithm.py +363 -0
- msprobe/core/compare/indicator_analysis/api_data.py +141 -0
- msprobe/core/compare/indicator_analysis/calculator.py +181 -0
- msprobe/core/compare/indicator_analysis/utils.py +116 -0
- msprobe/core/compare/layer_mapping/__init__.py +12 -11
- msprobe/core/compare/layer_mapping/data_scope_parser.py +20 -11
- msprobe/core/compare/layer_mapping/layer_mapping.py +14 -13
- msprobe/core/compare/layer_mapping/postprocess_pass.py +13 -11
- msprobe/core/compare/merge_result/merge_result.py +12 -11
- msprobe/core/compare/merge_result/merge_result_cli.py +12 -11
- msprobe/core/compare/merge_result/utils.py +12 -11
- msprobe/core/compare/multiprocessing_compute.py +13 -14
- msprobe/core/compare/npy_compare.py +13 -11
- msprobe/core/compare/offline_data_compare.py +160 -0
- msprobe/core/compare/stats_diff_calc.py +39 -0
- msprobe/core/compare/torchair_acc_cmp.py +764 -0
- msprobe/core/compare/torchair_cmp_utils.py +338 -0
- msprobe/core/compare/utils.py +140 -49
- msprobe/core/config_check/__init__.py +12 -11
- msprobe/core/config_check/checkers/__init__.py +12 -11
- msprobe/core/config_check/checkers/base_checker.py +15 -14
- msprobe/core/config_check/checkers/dataset_checker.py +13 -12
- msprobe/core/config_check/checkers/env_args_checker.py +13 -12
- msprobe/core/config_check/checkers/hyperparameter_checker.py +16 -15
- msprobe/core/config_check/checkers/pip_checker.py +15 -15
- msprobe/core/config_check/checkers/random_checker.py +13 -12
- msprobe/core/config_check/checkers/weights_checker.py +14 -12
- msprobe/core/config_check/ckpt_compare/ckpt_comparator.py +13 -17
- msprobe/core/config_check/ckpt_compare/megatron_loader.py +13 -12
- msprobe/core/config_check/ckpt_compare/metrics.py +12 -11
- msprobe/core/config_check/config_check_cli.py +18 -17
- msprobe/core/config_check/config_checker.py +16 -14
- msprobe/core/config_check/resource/dependency.yaml +15 -12
- msprobe/core/config_check/resource/env.yaml +12 -11
- msprobe/core/config_check/utils/hyperparameter_parser.py +12 -11
- msprobe/core/config_check/utils/utils.py +12 -11
- msprobe/core/{data_dump → dump/api_dump}/api_registry.py +12 -11
- msprobe/core/{common_config.py → dump/common_config.py} +13 -24
- msprobe/core/dump/data_dump/data_collector.py +257 -0
- msprobe/core/{data_dump → dump/data_dump}/data_processor/base.py +45 -36
- msprobe/core/{data_dump → dump/data_dump}/data_processor/factory.py +33 -25
- msprobe/core/{data_dump → dump/data_dump}/data_processor/mindspore_processor.py +37 -113
- msprobe/core/{data_dump → dump/data_dump}/data_processor/pytorch_processor.py +364 -131
- msprobe/core/{data_dump → dump/data_dump}/json_writer.py +24 -31
- msprobe/core/{data_dump → dump/data_dump}/scope.py +12 -13
- msprobe/core/{debugger → dump/debugger}/precision_debugger.py +15 -23
- msprobe/core/dump/dump2db/db_utils.py +215 -0
- msprobe/core/dump/dump2db/dump2db.py +409 -0
- msprobe/core/{hook_manager.py → dump/hook_manager.py} +38 -87
- msprobe/core/dump/kernel_dump/kernel_config.py +34 -0
- msprobe/core/{service.py → dump/service.py} +43 -27
- msprobe/core/install_deps/install_deps.py +51 -0
- msprobe/core/monitor/anomaly_processor.py +13 -11
- msprobe/core/monitor/csv2db.py +73 -93
- msprobe/core/monitor/db_utils.py +140 -205
- msprobe/core/monitor/utils.py +18 -17
- msprobe/core/monitor_v2/__init__.py +20 -0
- msprobe/core/monitor_v2/base.py +83 -0
- msprobe/core/monitor_v2/cc.py +287 -0
- msprobe/core/monitor_v2/factory.py +81 -0
- msprobe/core/monitor_v2/module.py +201 -0
- msprobe/core/monitor_v2/optimizer.py +245 -0
- msprobe/core/monitor_v2/param.py +154 -0
- msprobe/core/monitor_v2/trainer.py +326 -0
- msprobe/core/monitor_v2/utils.py +122 -0
- msprobe/core/monitor_v2/weight_grad.py +419 -0
- msprobe/core/monitor_v2/writer.py +162 -0
- msprobe/core/overflow_check/abnormal_scene.py +12 -11
- msprobe/core/overflow_check/api_info.py +12 -11
- msprobe/core/overflow_check/checker.py +12 -11
- msprobe/core/overflow_check/filter.py +13 -11
- msprobe/core/overflow_check/level.py +12 -11
- msprobe/core/overflow_check/utils.py +12 -11
- msprobe/core/single_save/single_comparator.py +12 -11
- msprobe/core/single_save/single_saver.py +12 -11
- msprobe/infer/__init__.py +16 -0
- msprobe/infer/offline/__init__.py +16 -0
- msprobe/infer/offline/compare/__init__.py +16 -0
- msprobe/infer/offline/compare/msquickcmp/__init__.py +16 -0
- msprobe/infer/offline/compare/msquickcmp/adapter_cli/__init__.py +16 -0
- msprobe/infer/offline/compare/msquickcmp/adapter_cli/args_adapter.py +46 -0
- msprobe/infer/offline/compare/msquickcmp/atc/__init__.py +16 -0
- msprobe/infer/offline/compare/msquickcmp/atc/atc_utils.py +98 -0
- msprobe/infer/offline/compare/msquickcmp/cmp_process.py +328 -0
- msprobe/infer/offline/compare/msquickcmp/common/__init__.py +16 -0
- msprobe/infer/offline/compare/msquickcmp/common/args_check.py +112 -0
- msprobe/infer/offline/compare/msquickcmp/common/convert.py +74 -0
- msprobe/infer/offline/compare/msquickcmp/common/dump_data.py +121 -0
- msprobe/infer/offline/compare/msquickcmp/common/dynamic_argument_bean.py +39 -0
- msprobe/infer/offline/compare/msquickcmp/common/utils.py +669 -0
- msprobe/infer/offline/compare/msquickcmp/config.ini +6 -0
- msprobe/infer/offline/compare/msquickcmp/dump/__init__.py +16 -0
- msprobe/infer/offline/compare/msquickcmp/dump/args_adapter.py +50 -0
- msprobe/infer/offline/compare/msquickcmp/dump/dump_process.py +91 -0
- msprobe/infer/offline/compare/msquickcmp/install_aclruntime_aisbench.sh +180 -0
- msprobe/infer/offline/compare/msquickcmp/main.py +199 -0
- msprobe/infer/offline/compare/msquickcmp/net_compare/__init__.py +16 -0
- msprobe/infer/offline/compare/msquickcmp/net_compare/net_compare.py +277 -0
- msprobe/infer/offline/compare/msquickcmp/npu/__init__.py +16 -0
- msprobe/infer/offline/compare/msquickcmp/npu/npu_dump_data.py +558 -0
- msprobe/infer/offline/compare/msquickcmp/npu/om_parser.py +416 -0
- msprobe/infer/offline/compare/msquickcmp/onnx_model/__init__.py +16 -0
- msprobe/infer/offline/compare/msquickcmp/onnx_model/onnx_dump_data.py +374 -0
- msprobe/infer/utils/__init__.py +15 -0
- msprobe/infer/utils/acc_cmp.py +94 -0
- msprobe/infer/utils/check/__init__.py +37 -0
- msprobe/infer/utils/check/args_checker.py +35 -0
- msprobe/infer/utils/check/checker.py +227 -0
- msprobe/infer/utils/check/dict_checker.py +78 -0
- msprobe/infer/utils/check/func_wrapper.py +96 -0
- msprobe/infer/utils/check/list_checker.py +56 -0
- msprobe/infer/utils/check/number_checker.py +64 -0
- msprobe/infer/utils/check/obj_checker.py +41 -0
- msprobe/infer/utils/check/path_checker.py +249 -0
- msprobe/infer/utils/check/rule.py +126 -0
- msprobe/infer/utils/check/string_checker.py +66 -0
- msprobe/infer/utils/cmp_algorithm.py +261 -0
- msprobe/infer/utils/constants.py +112 -0
- msprobe/infer/utils/file_open_check.py +337 -0
- msprobe/infer/utils/util.py +177 -0
- msprobe/mindspore/__init__.py +14 -13
- msprobe/mindspore/api_accuracy_checker/api_accuracy_checker.py +14 -13
- msprobe/mindspore/api_accuracy_checker/api_info.py +12 -11
- msprobe/mindspore/api_accuracy_checker/api_runner.py +12 -11
- msprobe/mindspore/api_accuracy_checker/base_compare_algorithm.py +12 -11
- msprobe/mindspore/api_accuracy_checker/bench_functions/flash_attention_score.py +12 -11
- msprobe/mindspore/api_accuracy_checker/bench_functions/fusion_operator.py +12 -11
- msprobe/mindspore/api_accuracy_checker/checker_support_api.yaml +12 -11
- msprobe/mindspore/api_accuracy_checker/cmd_parser.py +15 -14
- msprobe/mindspore/api_accuracy_checker/compute_element.py +12 -11
- msprobe/mindspore/api_accuracy_checker/data_manager.py +13 -11
- msprobe/mindspore/api_accuracy_checker/main.py +12 -11
- msprobe/mindspore/api_accuracy_checker/multi_api_accuracy_checker.py +14 -12
- msprobe/mindspore/api_accuracy_checker/multi_data_manager.py +13 -11
- msprobe/mindspore/api_accuracy_checker/torch_mindtorch_importer.py +12 -11
- msprobe/mindspore/api_accuracy_checker/type_mapping.py +12 -11
- msprobe/mindspore/api_accuracy_checker/utils.py +12 -11
- msprobe/mindspore/common/const.py +15 -74
- msprobe/mindspore/common/log.py +12 -11
- msprobe/mindspore/common/utils.py +30 -15
- msprobe/mindspore/compare/common_dir_compare.py +21 -23
- msprobe/mindspore/compare/distributed_compare.py +18 -16
- msprobe/mindspore/compare/ms_compare.py +14 -14
- msprobe/mindspore/compare/ms_graph_compare.py +26 -20
- msprobe/mindspore/compare/utils.py +14 -12
- msprobe/mindspore/{cell_processor.py → dump/cell_processor.py} +15 -14
- msprobe/mindspore/{debugger → dump/debugger}/debugger_config.py +12 -30
- msprobe/mindspore/{debugger → dump/debugger}/precision_debugger.py +43 -45
- msprobe/mindspore/dump/{cell_dump_process.py → dump_processor/cell_dump_process.py} +31 -17
- msprobe/mindspore/dump/{cell_dump_with_insert_gradient.py → dump_processor/cell_dump_with_insert_gradient.py} +18 -14
- msprobe/mindspore/dump/{dump_tool_factory.py → dump_processor/dump_tool_factory.py} +16 -15
- msprobe/mindspore/dump/{graph_mode_cell_dump.py → dump_processor/graph_mode_cell_dump.py} +16 -15
- msprobe/mindspore/dump/{graph_tensor_dump.py → dump_processor/graph_tensor_dump.py} +134 -133
- msprobe/mindspore/dump/{hook_cell → dump_processor/hook_cell}/api_register.py +15 -14
- msprobe/mindspore/dump/{hook_cell → dump_processor/hook_cell}/hook_cell.py +12 -11
- msprobe/mindspore/dump/{hook_cell → dump_processor/hook_cell}/ms_hook_manager.py +47 -20
- msprobe/mindspore/dump/{hook_cell → dump_processor/hook_cell}/primitive_hooks.py +14 -13
- msprobe/mindspore/dump/{hook_cell → dump_processor/hook_cell}/support_wrap_ops.yaml +13 -11
- msprobe/mindspore/dump/{jit_dump.py → dump_processor/jit_dump.py} +14 -13
- msprobe/mindspore/dump/{kernel_graph_dump.py → dump_processor/kernel_graph_dump.py} +13 -12
- msprobe/mindspore/dump/{kernel_kbyk_dump.py → dump_processor/kernel_kbyk_dump.py} +13 -12
- msprobe/mindspore/{exception_dump → dump/exception_dump}/exception_dump_tool_factory.py +14 -13
- msprobe/mindspore/{exception_dump → dump/exception_dump}/kernel_graph_exception_dump.py +13 -12
- msprobe/mindspore/{mindspore_service.py → dump/mindspore_service.py} +18 -17
- msprobe/mindspore/dump/mindtorch/__init__.py +19 -0
- msprobe/mindspore/dump/ms_config.py +105 -0
- msprobe/mindspore/{overflow_check → dump/overflow_check}/kernel_graph_overflow_check.py +13 -12
- msprobe/mindspore/{overflow_check → dump/overflow_check}/overflow_check_tool_factory.py +14 -13
- msprobe/mindspore/dump/task_handler_factory.py +43 -0
- msprobe/mindspore/monitor/common_func.py +12 -11
- msprobe/mindspore/monitor/data_writers.py +12 -11
- msprobe/mindspore/monitor/distributed/wrap_distributed.py +93 -39
- msprobe/mindspore/monitor/features.py +12 -11
- msprobe/mindspore/monitor/module_hook.py +19 -22
- msprobe/mindspore/monitor/optimizer_collect.py +29 -25
- msprobe/mindspore/monitor/utils.py +13 -11
- msprobe/msaccucmp/advisor/__init__.py +16 -0
- msprobe/msaccucmp/advisor/advisor_const.py +65 -0
- msprobe/msaccucmp/advisor/advisor_result.py +73 -0
- msprobe/msaccucmp/advisor/compare_advisor.py +99 -0
- msprobe/msaccucmp/advisor/input_advisor.py +66 -0
- msprobe/msaccucmp/advisor/node_advisor.py +68 -0
- msprobe/msaccucmp/advisor/overflow_advisor.py +58 -0
- msprobe/msaccucmp/algorithm_manager/__init__.py +16 -0
- msprobe/msaccucmp/algorithm_manager/algorithm_manager.py +464 -0
- msprobe/msaccucmp/algorithm_manager/algorithm_parameter.py +42 -0
- msprobe/msaccucmp/algorithm_manager/builtin_algorithm/alg_AccumulatedRelativeError.py +46 -0
- msprobe/msaccucmp/algorithm_manager/builtin_algorithm/alg_CosineSimilarity.py +58 -0
- msprobe/msaccucmp/algorithm_manager/builtin_algorithm/alg_KullbackLeiblerDivergence.py +84 -0
- msprobe/msaccucmp/algorithm_manager/builtin_algorithm/alg_MaxAbsoluteError.py +41 -0
- msprobe/msaccucmp/algorithm_manager/builtin_algorithm/alg_MaxRelativeError.py +46 -0
- msprobe/msaccucmp/algorithm_manager/builtin_algorithm/alg_MeanAbsoluteError.py +41 -0
- msprobe/msaccucmp/algorithm_manager/builtin_algorithm/alg_MeanRelativeError.py +46 -0
- msprobe/msaccucmp/algorithm_manager/builtin_algorithm/alg_RelativeEuclideanDistance.py +46 -0
- msprobe/msaccucmp/algorithm_manager/builtin_algorithm/alg_RootMeanSquareError.py +40 -0
- msprobe/msaccucmp/algorithm_manager/builtin_algorithm/alg_StandardDeviation.py +47 -0
- msprobe/msaccucmp/cmp_utils/__init__.py +16 -0
- msprobe/msaccucmp/cmp_utils/common.py +113 -0
- msprobe/msaccucmp/cmp_utils/constant/__init__.py +16 -0
- msprobe/msaccucmp/cmp_utils/constant/compare_error.py +81 -0
- msprobe/msaccucmp/cmp_utils/constant/const_manager.py +530 -0
- msprobe/msaccucmp/cmp_utils/file_utils.py +497 -0
- msprobe/msaccucmp/cmp_utils/log.py +257 -0
- msprobe/msaccucmp/cmp_utils/multi_process/__init__.py +16 -0
- msprobe/msaccucmp/cmp_utils/multi_process/multi_convert_process.py +140 -0
- msprobe/msaccucmp/cmp_utils/multi_process/progress.py +78 -0
- msprobe/msaccucmp/cmp_utils/path_check.py +274 -0
- msprobe/msaccucmp/cmp_utils/reg_manager.py +98 -0
- msprobe/msaccucmp/cmp_utils/tlv_parse.py +279 -0
- msprobe/msaccucmp/cmp_utils/utils.py +356 -0
- msprobe/msaccucmp/cmp_utils/utils_type.py +63 -0
- msprobe/msaccucmp/compare_vector.py +48 -0
- msprobe/msaccucmp/conversion/__init__.py +16 -0
- msprobe/msaccucmp/conversion/data_conversion.py +277 -0
- msprobe/msaccucmp/conversion/dtype_conversion.py +99 -0
- msprobe/msaccucmp/conversion/shape_format_conversion.py +477 -0
- msprobe/msaccucmp/conversion/tensor_conversion.py +369 -0
- msprobe/msaccucmp/dump_data_conversion.py +46 -0
- msprobe/msaccucmp/dump_parse/__init__.py +16 -0
- msprobe/msaccucmp/dump_parse/big_dump_data.py +317 -0
- msprobe/msaccucmp/dump_parse/dump.py +423 -0
- msprobe/msaccucmp/dump_parse/dump_data_object.py +322 -0
- msprobe/msaccucmp/dump_parse/dump_data_parser.py +436 -0
- msprobe/msaccucmp/dump_parse/dump_utils.py +246 -0
- msprobe/msaccucmp/dump_parse/ffts_parser.py +137 -0
- msprobe/msaccucmp/dump_parse/mapping.py +62 -0
- msprobe/msaccucmp/dump_parse/nano_dump_data.py +392 -0
- msprobe/msaccucmp/dump_parse/proto_dump_data.py +308 -0
- msprobe/msaccucmp/dump_parser.py +90 -0
- msprobe/msaccucmp/format_manager/__init__.py +16 -0
- msprobe/msaccucmp/format_manager/builtin_format_convert/convert_FRACTAL_NZ_to_NCHW.py +53 -0
- msprobe/msaccucmp/format_manager/builtin_format_convert/convert_FRACTAL_NZ_to_ND.py +52 -0
- msprobe/msaccucmp/format_manager/builtin_format_convert/convert_FRACTAL_NZ_to_NHWC.py +53 -0
- msprobe/msaccucmp/format_manager/builtin_format_convert/convert_FRACTAL_Z_to_HWCN.py +47 -0
- msprobe/msaccucmp/format_manager/builtin_format_convert/convert_FRACTAL_Z_to_NCHW.py +47 -0
- msprobe/msaccucmp/format_manager/builtin_format_convert/convert_HWCN_to_FRACTAL_Z.py +89 -0
- msprobe/msaccucmp/format_manager/builtin_format_convert/convert_HWCN_to_NCHW.py +37 -0
- msprobe/msaccucmp/format_manager/builtin_format_convert/convert_HWCN_to_NHWC.py +37 -0
- msprobe/msaccucmp/format_manager/builtin_format_convert/convert_NC1HWC0_to_HWCN.py +43 -0
- msprobe/msaccucmp/format_manager/builtin_format_convert/convert_NC1HWC0_to_NCHW.py +48 -0
- msprobe/msaccucmp/format_manager/builtin_format_convert/convert_NC1HWC0_to_NHWC.py +43 -0
- msprobe/msaccucmp/format_manager/builtin_format_convert/convert_NCHW_to_FRACTAL_Z.py +87 -0
- msprobe/msaccucmp/format_manager/builtin_format_convert/convert_NCHW_to_NHWC.py +37 -0
- msprobe/msaccucmp/format_manager/builtin_format_convert/convert_NDC1HWC0_to_NCDHW.py +48 -0
- msprobe/msaccucmp/format_manager/builtin_format_convert/convert_NDC1HWC0_to_ND.py +44 -0
- msprobe/msaccucmp/format_manager/builtin_format_convert/convert_NHWC_to_FRACTAL_Z.py +87 -0
- msprobe/msaccucmp/format_manager/builtin_format_convert/convert_NHWC_to_HWCN.py +37 -0
- msprobe/msaccucmp/format_manager/builtin_format_convert/convert_NHWC_to_NCHW.py +37 -0
- msprobe/msaccucmp/format_manager/format_manager.py +307 -0
- msprobe/msaccucmp/inplace_layer_process.py +186 -0
- msprobe/msaccucmp/msaccucmp.py +532 -0
- msprobe/msaccucmp/mscmp_advisor.py +128 -0
- msprobe/msaccucmp/overflow/__init__.py +16 -0
- msprobe/msaccucmp/overflow/overflow_analyse.py +305 -0
- msprobe/msaccucmp/overflow/overflow_detection.py +143 -0
- msprobe/msaccucmp/pytorch_cmp/__init__.py +16 -0
- msprobe/msaccucmp/pytorch_cmp/compare_pytorch.py +389 -0
- msprobe/msaccucmp/pytorch_cmp/hdf5_parser.py +377 -0
- msprobe/msaccucmp/pytorch_cmp/pytorch_dump_data.py +461 -0
- msprobe/msaccucmp/shape_conversion.py +41 -0
- msprobe/msaccucmp/vector_cmp/__init__.py +16 -0
- msprobe/msaccucmp/vector_cmp/batch_compare.py +197 -0
- msprobe/msaccucmp/vector_cmp/compare_detail/__init__.py +16 -0
- msprobe/msaccucmp/vector_cmp/compare_detail/compare_detail.py +245 -0
- msprobe/msaccucmp/vector_cmp/compare_detail/detail.py +182 -0
- msprobe/msaccucmp/vector_cmp/compare_detail/detail_writer.py +580 -0
- msprobe/msaccucmp/vector_cmp/fusion_manager/__init__.py +16 -0
- msprobe/msaccucmp/vector_cmp/fusion_manager/compare_fusion_op.py +588 -0
- msprobe/msaccucmp/vector_cmp/fusion_manager/compare_npu_vs_npu.py +339 -0
- msprobe/msaccucmp/vector_cmp/fusion_manager/compare_result.py +326 -0
- msprobe/msaccucmp/vector_cmp/fusion_manager/compare_rule.py +156 -0
- msprobe/msaccucmp/vector_cmp/fusion_manager/fusion_op.py +204 -0
- msprobe/msaccucmp/vector_cmp/fusion_manager/fusion_rule_parser.py +635 -0
- msprobe/msaccucmp/vector_cmp/fusion_manager/quant_filter.py +187 -0
- msprobe/msaccucmp/vector_cmp/range_manager/__init__.py +16 -0
- msprobe/msaccucmp/vector_cmp/range_manager/range_manager.py +100 -0
- msprobe/msaccucmp/vector_cmp/range_manager/range_mode.py +94 -0
- msprobe/msaccucmp/vector_cmp/range_manager/select_mode.py +86 -0
- msprobe/msaccucmp/vector_cmp/vector_comparison.py +535 -0
- msprobe/msprobe.py +101 -130
- msprobe/overflow_check/__init__.py +15 -0
- msprobe/{nan_analyze → overflow_check}/analyzer.py +38 -27
- msprobe/{nan_analyze → overflow_check}/graph.py +30 -27
- msprobe/{nan_analyze → overflow_check}/utils.py +15 -14
- msprobe/pytorch/__init__.py +20 -14
- msprobe/pytorch/aclgraph_dump/__init__.py +45 -0
- msprobe/pytorch/aclgraph_dump/_meta.py +26 -0
- msprobe/pytorch/api_accuracy_checker/{run_ut/run_ut.py → acc_check/acc_check.py} +50 -45
- msprobe/pytorch/api_accuracy_checker/{run_ut/run_ut_utils.py → acc_check/acc_check_utils.py} +201 -30
- msprobe/pytorch/api_accuracy_checker/{run_ut → acc_check}/data_generate.py +56 -16
- msprobe/pytorch/api_accuracy_checker/{run_ut/multi_run_ut.py → acc_check/multi_acc_check.py} +32 -47
- msprobe/pytorch/api_accuracy_checker/{run_ut → acc_check}/run_overflow_check.py +19 -18
- msprobe/pytorch/api_accuracy_checker/common/config.py +22 -20
- msprobe/pytorch/api_accuracy_checker/common/utils.py +72 -13
- msprobe/pytorch/api_accuracy_checker/compare/algorithm.py +41 -11
- msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py +23 -14
- msprobe/pytorch/api_accuracy_checker/compare/compare.py +45 -32
- msprobe/pytorch/api_accuracy_checker/compare/compare_column.py +12 -11
- msprobe/pytorch/api_accuracy_checker/compare/compare_input.py +14 -12
- msprobe/pytorch/api_accuracy_checker/compare/compare_utils.py +14 -12
- msprobe/pytorch/api_accuracy_checker/precision_standard/absolute_threshold.py +12 -11
- msprobe/pytorch/api_accuracy_checker/precision_standard/accumulative_error_compare.py +12 -11
- msprobe/pytorch/api_accuracy_checker/precision_standard/base_standard.py +21 -19
- msprobe/pytorch/api_accuracy_checker/precision_standard/benchmark_compare.py +14 -13
- msprobe/pytorch/api_accuracy_checker/precision_standard/binary_consistency.py +12 -11
- msprobe/pytorch/api_accuracy_checker/precision_standard/standard_config.py +60 -11
- msprobe/pytorch/api_accuracy_checker/precision_standard/standard_register.py +27 -16
- msprobe/pytorch/api_accuracy_checker/precision_standard/thousandth_standard.py +13 -11
- msprobe/pytorch/api_accuracy_checker/precision_standard/ulp_compare.py +39 -18
- msprobe/pytorch/bench_functions/__init__.py +12 -11
- msprobe/pytorch/bench_functions/apply_adam.py +12 -11
- msprobe/pytorch/bench_functions/apply_adam_w.py +12 -11
- msprobe/pytorch/bench_functions/confusion_transpose.py +12 -11
- msprobe/pytorch/bench_functions/fast_gelu.py +12 -11
- msprobe/pytorch/bench_functions/group_norm_silu.py +12 -11
- msprobe/pytorch/bench_functions/layer_norm_eval.py +12 -11
- msprobe/pytorch/bench_functions/linear.py +12 -11
- msprobe/pytorch/bench_functions/matmul_backward.py +12 -11
- msprobe/pytorch/bench_functions/mish.py +12 -11
- msprobe/pytorch/bench_functions/moe_gating_top_k_softmax.py +12 -11
- msprobe/pytorch/bench_functions/npu_fusion_attention.py +12 -11
- msprobe/pytorch/bench_functions/rms_norm.py +12 -11
- msprobe/pytorch/bench_functions/rotary_mul.py +12 -11
- msprobe/pytorch/bench_functions/scaled_mask_softmax.py +12 -11
- msprobe/pytorch/bench_functions/sort_v2.py +12 -11
- msprobe/pytorch/bench_functions/swiglu.py +12 -11
- msprobe/pytorch/common/__init__.py +12 -11
- msprobe/pytorch/common/log.py +12 -11
- msprobe/pytorch/common/parse_json.py +12 -11
- msprobe/pytorch/common/utils.py +52 -19
- msprobe/pytorch/compare/distributed_compare.py +13 -13
- msprobe/pytorch/compare/match.py +12 -11
- msprobe/pytorch/compare/pt_compare.py +14 -20
- msprobe/pytorch/compare/pt_diff_analyze.py +12 -11
- msprobe/pytorch/compare/utils.py +12 -11
- msprobe/pytorch/{hook_module → dump/api_dump}/api_register.py +18 -16
- msprobe/pytorch/{hook_module → dump/api_dump}/hook_module.py +14 -13
- msprobe/pytorch/{hook_module → dump/api_dump}/pt_hook_manager.py +68 -23
- msprobe/pytorch/{hook_module → dump/api_dump}/register_optimizer_hook.py +13 -11
- msprobe/pytorch/{hook_module → dump/api_dump}/script_wrapper.py +17 -14
- msprobe/pytorch/{hook_module → dump/api_dump}/utils.py +12 -11
- msprobe/pytorch/{debugger → dump/debugger}/debugger_config.py +23 -38
- msprobe/pytorch/dump/debugger/precision_debugger.py +130 -0
- msprobe/pytorch/{function_factory.py → dump/function_factory.py} +12 -11
- msprobe/pytorch/dump/module_dump/hook_wrapper.py +17 -13
- msprobe/pytorch/dump/module_dump/module_dump.py +16 -15
- msprobe/pytorch/dump/module_dump/{module_processer.py → module_processor.py} +54 -42
- msprobe/pytorch/dump/pt_config.py +128 -0
- msprobe/pytorch/{pytorch_service.py → dump/pytorch_service.py} +22 -21
- msprobe/pytorch/monitor/csv2tb.py +13 -11
- msprobe/pytorch/monitor/data_writers.py +13 -11
- msprobe/pytorch/monitor/distributed/wrap_distributed.py +13 -11
- msprobe/pytorch/monitor/features.py +12 -11
- msprobe/pytorch/monitor/module_hook.py +67 -59
- msprobe/pytorch/monitor/module_metric.py +13 -11
- msprobe/pytorch/monitor/optimizer_collect.py +37 -35
- msprobe/pytorch/monitor/utils.py +13 -11
- msprobe/pytorch/monitor/visualizer.py +12 -11
- msprobe/pytorch/torchair_dump/__init__.py +17 -0
- msprobe/pytorch/torchair_dump/torchair_dump.py +114 -0
- msprobe/scripts/atb/config_example.json +10 -0
- msprobe/scripts/atb/load_atb_probe.sh +101 -0
- msprobe/scripts/atb/unload_atb_probe.sh +27 -0
- msprobe/scripts/build_msaccucmp.sh +186 -0
- msprobe/scripts/conf/help.info +6 -0
- msprobe/scripts/conf/version.info +3 -0
- msprobe/scripts/run_script/common.sh +538 -0
- msprobe/scripts/run_script/main_msaccucmp.sh +232 -0
- msprobe/visualization/__init__.py +12 -11
- msprobe/visualization/builder/__init__.py +12 -11
- msprobe/visualization/builder/graph_builder.py +45 -30
- msprobe/visualization/builder/graph_merger.py +53 -32
- msprobe/visualization/builder/msprobe_adapter.py +34 -44
- msprobe/visualization/compare/__init__.py +12 -11
- msprobe/visualization/compare/graph_comparator.py +63 -51
- msprobe/visualization/compare/mode_adapter.py +28 -113
- msprobe/visualization/db_utils.py +133 -22
- msprobe/visualization/graph/__init__.py +12 -11
- msprobe/visualization/graph/base_node.py +15 -27
- msprobe/visualization/graph/distributed_analyzer.py +97 -40
- msprobe/visualization/graph/graph.py +14 -16
- msprobe/visualization/graph/node_colors.py +34 -31
- msprobe/visualization/graph/node_op.py +12 -11
- msprobe/visualization/graph_service.py +580 -205
- msprobe/visualization/utils.py +278 -31
- tb_graph_ascend/secure_build.py +175 -0
- tb_graph_ascend/server/__init__.py +15 -0
- tb_graph_ascend/server/app/__init__.py +15 -0
- tb_graph_ascend/server/app/model/__init__.py +15 -0
- tb_graph_ascend/server/app/model/hierarchy.py +348 -0
- tb_graph_ascend/server/app/model/layout_hierarchy_model.py +69 -0
- tb_graph_ascend/server/app/model/match_nodes_model.py +573 -0
- tb_graph_ascend/server/app/repositories/__init__.py +15 -0
- tb_graph_ascend/server/app/repositories/graph_repo_base.py +32 -0
- tb_graph_ascend/server/app/repositories/graph_repo_db.py +879 -0
- tb_graph_ascend/server/app/repositories/graph_repo_vis.py +83 -0
- tb_graph_ascend/server/app/service/__init__.py +18 -0
- tb_graph_ascend/server/app/service/graph_service_base.py +158 -0
- tb_graph_ascend/server/app/service/graph_service_db.py +438 -0
- tb_graph_ascend/server/app/service/graph_service_factory.py +54 -0
- tb_graph_ascend/server/app/service/graph_service_vis.py +480 -0
- tb_graph_ascend/server/app/utils/__init__.py +15 -0
- tb_graph_ascend/server/app/utils/constant.py +80 -0
- tb_graph_ascend/server/app/utils/file_check_wrapper.py +46 -0
- tb_graph_ascend/server/app/utils/global_state.py +95 -0
- tb_graph_ascend/server/app/utils/graph_utils.py +661 -0
- tb_graph_ascend/server/app/utils/i18n.py +153 -0
- tb_graph_ascend/server/app/utils/request_method.py +46 -0
- tb_graph_ascend/server/app/views/__init__.py +15 -0
- tb_graph_ascend/server/app/views/graph_views.py +304 -0
- tb_graph_ascend/server/plugin.py +108 -0
- tb_graph_ascend/server/static/index.html +9250 -0
- tb_graph_ascend/server/static/index.js +21 -0
- tb_graph_ascend/setup.py +57 -0
- mindstudio_probe-8.3.2.dist-info/LICENSE +0 -201
- mindstudio_probe-8.3.2.dist-info/RECORD +0 -491
- mindstudio_probe-8.3.2.dist-info/entry_points.txt +0 -2
- mindstudio_probe-8.3.2.dist-info/top_level.txt +0 -1
- msprobe/CMakeLists.txt +0 -5
- msprobe/README.md +0 -203
- msprobe/core/advisor/advisor.py +0 -129
- msprobe/core/advisor/advisor_const.py +0 -58
- msprobe/core/advisor/advisor_result.py +0 -58
- msprobe/core/compare/find_first/data_processor.py +0 -35
- msprobe/core/compare/highlight.py +0 -390
- msprobe/core/data_dump/data_collector.py +0 -356
- msprobe/core/grad_probe/constant.py +0 -90
- msprobe/core/grad_probe/grad_compare.py +0 -187
- msprobe/core/grad_probe/utils.py +0 -105
- msprobe/core/kernel_dump/kernel_config.py +0 -33
- msprobe/docs/01.installation.md +0 -250
- msprobe/docs/02.config_introduction.md +0 -221
- msprobe/docs/03.config_examples.md +0 -281
- msprobe/docs/04.kernel_dump_PyTorch.md +0 -73
- msprobe/docs/05.data_dump_PyTorch.md +0 -518
- msprobe/docs/06.data_dump_MindSpore.md +0 -618
- msprobe/docs/07.accuracy_checker_PyTorch.md +0 -310
- msprobe/docs/09.accuracy_checker_MindSpore.md +0 -120
- msprobe/docs/10.accuracy_compare_PyTorch.md +0 -637
- msprobe/docs/11.accuracy_compare_MindSpore.md +0 -769
- msprobe/docs/12.overflow_check_PyTorch.md +0 -82
- msprobe/docs/13.overflow_check_MindSpore.md +0 -33
- msprobe/docs/14.data_parse_PyTorch.md +0 -282
- msprobe/docs/15.free_benchmarking_PyTorch.md +0 -169
- msprobe/docs/16.free_benchmarking_MindSpore.md +0 -159
- msprobe/docs/17.grad_probe.md +0 -205
- msprobe/docs/18.online_dispatch.md +0 -89
- msprobe/docs/19.monitor.md +0 -753
- msprobe/docs/20.monitor_performance_baseline.md +0 -52
- msprobe/docs/21.visualization_PyTorch.md +0 -519
- msprobe/docs/22.visualization_MindSpore.md +0 -515
- msprobe/docs/23.generate_operator_PyTorch.md +0 -107
- msprobe/docs/24.code_mapping_Mindspore.md +0 -29
- msprobe/docs/25.tool_function_introduction.md +0 -29
- msprobe/docs/26.data_dump_PyTorch_baseline.md +0 -48
- msprobe/docs/27.dump_json_instruction.md +0 -795
- msprobe/docs/28.debugger_save_instruction.md +0 -288
- msprobe/docs/28.kernel_dump_MindSpore.md +0 -69
- msprobe/docs/29.data_dump_MSAdapter.md +0 -235
- msprobe/docs/30.overflow_check_MSAdapter.md +0 -31
- msprobe/docs/31.config_check.md +0 -107
- msprobe/docs/32.ckpt_compare.md +0 -69
- msprobe/docs/33.generate_operator_MindSpore.md +0 -181
- msprobe/docs/34.RL_collect.md +0 -101
- msprobe/docs/35.nan_analyze.md +0 -73
- msprobe/docs/36.calculation_result_change.md +0 -75
- msprobe/docs/FAQ.md +0 -232
- msprobe/docs/S02.report_free_benchmarking_validation_performance_baseline.md +0 -146
- msprobe/docs/accuracy_checker_MindSpore/accuracy_checker_MindSpore_baseline.md +0 -14
- msprobe/docs/data_dump_MindSpore/data_dump_MindSpore_baseline.md +0 -33
- msprobe/docs/data_dump_MindSpore/dynamic_graph_quick_start_example.md +0 -217
- msprobe/docs/img/BLOOM-7B_1.png +0 -0
- msprobe/docs/img/BLOOM-7B_2.png +0 -0
- msprobe/docs/img/BLOOM-7B_3.png +0 -0
- msprobe/docs/img/BLOOM-7B_4.png +0 -0
- msprobe/docs/img/GPT-3_1.png +0 -0
- msprobe/docs/img/GPT-3_2.png +0 -0
- msprobe/docs/img/GPT-3_3.png +0 -0
- msprobe/docs/img/GPT-3_4.png +0 -0
- msprobe/docs/img/GPT-3_5.png +0 -0
- msprobe/docs/img/GPT-3_6.png +0 -0
- msprobe/docs/img/GPT-3_7.png +0 -0
- msprobe/docs/img/GPT-3_8.png +0 -0
- msprobe/docs/img/YOLOV5S_1.png +0 -0
- msprobe/docs/img/YOLOV5S_2.png +0 -0
- msprobe/docs/img/accuracy_checking_details.png +0 -0
- msprobe/docs/img/accuracy_checking_result.png +0 -0
- msprobe/docs/img/api_precision_compare_details.png +0 -0
- msprobe/docs/img/api_precision_compare_result.png +0 -0
- msprobe/docs/img/auto_analyze_log.png +0 -0
- msprobe/docs/img/compare_result.png +0 -0
- msprobe/docs/img/compare_result_pkl.png +0 -0
- msprobe/docs/img/compare_result_pkl_md5.png.png +0 -0
- msprobe/docs/img/cpu_info.png +0 -0
- msprobe/docs/img/free_benchmark.png +0 -0
- msprobe/docs/img/free_benchmark_framework.png +0 -0
- msprobe/docs/img/grad_probe_image-1.png +0 -0
- msprobe/docs/img/grad_probe_image-2.png +0 -0
- msprobe/docs/img/grad_probe_image-3.png +0 -0
- msprobe/docs/img/grad_probe_image-4.png +0 -0
- msprobe/docs/img/grad_probe_image.png +0 -0
- msprobe/docs/img/merge_result.png +0 -0
- msprobe/docs/img/module_compare.png +0 -0
- msprobe/docs/img/monitor/cpu_info.png +0 -0
- msprobe/docs/img/monitor/step_count_per_record.png +0 -0
- msprobe/docs/img/ms_dump.png +0 -0
- msprobe/docs/img/ms_layer.png +0 -0
- msprobe/docs/img/pt_dump.png +0 -0
- msprobe/docs/img/save_compare_result_sample.png +0 -0
- msprobe/docs/img/visualization/fuzzy_match_ms.png +0 -0
- msprobe/docs/img/visualization/fuzzy_match_pt.png +0 -0
- msprobe/docs/img/visualization/proxy.png +0 -0
- msprobe/docs/img/visualization/tensorboard_1.png +0 -0
- msprobe/docs/img/visualization/tensorboard_2.png +0 -0
- msprobe/docs/img/visualization/vis_browser_1.png +0 -0
- msprobe/docs/img/visualization/vis_browser_2.png +0 -0
- msprobe/docs/img/visualization/vis_match_info.png +0 -0
- msprobe/docs/img/visualization/vis_precision_info.png +0 -0
- msprobe/docs/img/visualization/vis_search_info.png +0 -0
- msprobe/docs/img/visualization/vis_show_info.png +0 -0
- msprobe/docs/img/visualization/vis_showcase.png +0 -0
- msprobe/docs/img/visualization/vis_unmatch_info.png +0 -0
- msprobe/docs/visualization/GPTModel.png +0 -0
- msprobe/docs/visualization/ParallelMLP.png +0 -0
- msprobe/docs/visualization/layer_mapping_example.md +0 -132
- msprobe/docs/visualization/mapping.png +0 -0
- msprobe/docs/visualization/mapping1.png +0 -0
- msprobe/docs/visualization/mindspeed_llamafactoary_img/1.png +0 -0
- msprobe/docs/visualization/mindspeed_llamafactoary_img/2.png +0 -0
- msprobe/docs/visualization/mindspeed_llamafactoary_img/3.png +0 -0
- msprobe/docs/visualization/mindspeed_llamafactoary_img/4.png +0 -0
- msprobe/docs/visualization/mindspeed_llamafactoary_img/5.png +0 -0
- msprobe/docs/visualization/mindspeed_llamafactoary_img/6.png +0 -0
- msprobe/docs/visualization/mindspeed_llamafactoary_img/7.png +0 -0
- msprobe/docs/visualization/mindspeed_llamafactoary_img/llamafactory-qwen25vl.txt +0 -59
- msprobe/docs/visualization/mindspeed_llamafactoary_img/llamafactory1.png +0 -0
- msprobe/docs/visualization/mindspeed_llamafactoary_img/llamafactory2.png +0 -0
- msprobe/docs/visualization/mindspeed_llamafactoary_img/mindspeed-mm-qwen25vl.txt +0 -80
- msprobe/docs/visualization/mindspeed_llamafactoary_img/mindspeed1.png +0 -0
- msprobe/docs/visualization/mindspeed_llamafactoary_img/mindspeed2.png +0 -0
- msprobe/docs/visualization/mindspeed_llamafactory_mapping.md +0 -330
- msprobe/docs/visualization/module_name.png +0 -0
- msprobe/docs/visualization/module_name1.png +0 -0
- msprobe/docs/visualization/no_mapping.png +0 -0
- msprobe/docs/visualization/no_mapping1.png +0 -0
- msprobe/docs/visualization/no_mapping_analyze.png +0 -0
- msprobe/docs/visualization/top_layer.png +0 -0
- msprobe/mindspore/api_accuracy_checker/generate_op_script/op_generator.py +0 -460
- msprobe/mindspore/api_accuracy_checker/generate_op_script/operator_replication.template +0 -2081
- msprobe/mindspore/code_mapping/bind.py +0 -283
- msprobe/mindspore/code_mapping/cmd_parser.py +0 -40
- msprobe/mindspore/code_mapping/graph.py +0 -49
- msprobe/mindspore/code_mapping/graph_parser.py +0 -211
- msprobe/mindspore/code_mapping/main.py +0 -24
- msprobe/mindspore/code_mapping/processor.py +0 -34
- msprobe/mindspore/dym_loader/hook_dynamic_loader.cpp +0 -111
- msprobe/mindspore/dym_loader/hook_dynamic_loader.h +0 -52
- msprobe/mindspore/free_benchmark/api_pynative_self_check.py +0 -257
- msprobe/mindspore/free_benchmark/common/config.py +0 -27
- msprobe/mindspore/free_benchmark/common/handler_params.py +0 -31
- msprobe/mindspore/free_benchmark/common/utils.py +0 -100
- msprobe/mindspore/free_benchmark/data/support_wrap_ops.yaml +0 -638
- msprobe/mindspore/free_benchmark/handler/base_handler.py +0 -105
- msprobe/mindspore/free_benchmark/handler/check_handler.py +0 -55
- msprobe/mindspore/free_benchmark/handler/fix_handler.py +0 -51
- msprobe/mindspore/free_benchmark/handler/handler_factory.py +0 -36
- msprobe/mindspore/free_benchmark/perturbation/add_noise.py +0 -82
- msprobe/mindspore/free_benchmark/perturbation/base_perturbation.py +0 -45
- msprobe/mindspore/free_benchmark/perturbation/bit_noise.py +0 -78
- msprobe/mindspore/free_benchmark/perturbation/exchange_value.py +0 -77
- msprobe/mindspore/free_benchmark/perturbation/improve_precision.py +0 -56
- msprobe/mindspore/free_benchmark/perturbation/no_change.py +0 -27
- msprobe/mindspore/free_benchmark/perturbation/perturbation_factory.py +0 -46
- msprobe/mindspore/free_benchmark/self_check_tool_factory.py +0 -51
- msprobe/mindspore/grad_probe/global_context.py +0 -127
- msprobe/mindspore/grad_probe/grad_analyzer.py +0 -260
- msprobe/mindspore/grad_probe/grad_monitor.py +0 -42
- msprobe/mindspore/grad_probe/grad_stat_csv.py +0 -161
- msprobe/mindspore/grad_probe/hook.py +0 -115
- msprobe/mindspore/grad_probe/utils.py +0 -43
- msprobe/mindspore/mindtorch/__init__.py +0 -18
- msprobe/mindspore/ms_config.py +0 -153
- msprobe/mindspore/task_handler_factory.py +0 -44
- msprobe/nan_analyze/__init__.py +0 -14
- msprobe/pytorch/api_accuracy_checker/generate_op_script/config_op.json +0 -9
- msprobe/pytorch/api_accuracy_checker/generate_op_script/op_generator.py +0 -480
- msprobe/pytorch/api_accuracy_checker/generate_op_script/operator_replication.template +0 -567
- msprobe/pytorch/debugger/precision_debugger.py +0 -181
- msprobe/pytorch/free_benchmark/__init__.py +0 -23
- msprobe/pytorch/free_benchmark/common/constant.py +0 -85
- msprobe/pytorch/free_benchmark/common/counter.py +0 -87
- msprobe/pytorch/free_benchmark/common/enums.py +0 -80
- msprobe/pytorch/free_benchmark/common/params.py +0 -152
- msprobe/pytorch/free_benchmark/common/utils.py +0 -143
- msprobe/pytorch/free_benchmark/compare/grad_saver.py +0 -215
- msprobe/pytorch/free_benchmark/compare/single_benchmark.py +0 -121
- msprobe/pytorch/free_benchmark/main.py +0 -123
- msprobe/pytorch/free_benchmark/perturbed_layers/base_layer.py +0 -28
- msprobe/pytorch/free_benchmark/perturbed_layers/layer_factory.py +0 -56
- msprobe/pytorch/free_benchmark/perturbed_layers/npu/add_noise.py +0 -107
- msprobe/pytorch/free_benchmark/perturbed_layers/npu/bit_noise.py +0 -121
- msprobe/pytorch/free_benchmark/perturbed_layers/npu/change_value.py +0 -89
- msprobe/pytorch/free_benchmark/perturbed_layers/npu/improve_precision.py +0 -87
- msprobe/pytorch/free_benchmark/perturbed_layers/npu/no_change.py +0 -43
- msprobe/pytorch/free_benchmark/perturbed_layers/npu/npu_base_layser.py +0 -60
- msprobe/pytorch/free_benchmark/perturbed_layers/run_cpu.py +0 -34
- msprobe/pytorch/free_benchmark/result_handlers/base_handler.py +0 -252
- msprobe/pytorch/free_benchmark/result_handlers/check_handler.py +0 -54
- msprobe/pytorch/free_benchmark/result_handlers/fix_handler.py +0 -40
- msprobe/pytorch/free_benchmark/result_handlers/handler_factory.py +0 -45
- msprobe/pytorch/free_benchmark/result_handlers/preheat_handler.py +0 -181
- msprobe/pytorch/grad_probe/__init__.py +0 -0
- msprobe/pytorch/grad_probe/grad_monitor.py +0 -108
- msprobe/pytorch/grad_probe/grad_stat_csv.py +0 -160
- msprobe/pytorch/hook_module/__init__.py +0 -16
- msprobe/pytorch/hook_module/wrap_aten.py +0 -111
- msprobe/pytorch/online_dispatch/__init__.py +0 -19
- msprobe/pytorch/online_dispatch/compare.py +0 -224
- msprobe/pytorch/online_dispatch/dispatch.py +0 -332
- msprobe/pytorch/online_dispatch/dump_compare.py +0 -179
- msprobe/pytorch/online_dispatch/single_compare.py +0 -412
- msprobe/pytorch/online_dispatch/torch_ops_config.yaml +0 -58
- msprobe/pytorch/online_dispatch/utils.py +0 -158
- msprobe/pytorch/parse_tool/__init__.py +0 -0
- msprobe/pytorch/parse_tool/cli.py +0 -31
- msprobe/pytorch/parse_tool/lib/__init__.py +0 -0
- msprobe/pytorch/parse_tool/lib/compare.py +0 -253
- msprobe/pytorch/parse_tool/lib/config.py +0 -50
- msprobe/pytorch/parse_tool/lib/file_desc.py +0 -45
- msprobe/pytorch/parse_tool/lib/interactive_cli.py +0 -97
- msprobe/pytorch/parse_tool/lib/parse_exception.py +0 -54
- msprobe/pytorch/parse_tool/lib/parse_tool.py +0 -161
- msprobe/pytorch/parse_tool/lib/utils.py +0 -299
- msprobe/pytorch/parse_tool/lib/visualization.py +0 -85
- msprobe/pytorch/pt_config.py +0 -299
- /msprobe/core/{grad_probe → dump}/__init__.py +0 -0
- /msprobe/{mindspore/code_mapping → core/dump/api_dump}/__init__.py +0 -0
- /msprobe/{mindspore/debugger → core/dump/data_dump}/__init__.py +0 -0
- /msprobe/{mindspore/exception_dump → core/dump/data_dump/data_processor}/__init__.py +0 -0
- /msprobe/{mindspore/free_benchmark → core/dump/debugger}/__init__.py +0 -0
- /msprobe/{mindspore/free_benchmark/common → core/dump/kernel_dump}/__init__.py +0 -0
- /msprobe/mindspore/{free_benchmark/handler → dump/debugger}/__init__.py +0 -0
- /msprobe/mindspore/{grad_probe → dump/dump_processor}/__init__.py +0 -0
- /msprobe/mindspore/{overflow_check → dump/exception_dump}/__init__.py +0 -0
- /msprobe/mindspore/{mindtorch → dump/mindtorch}/mindtorch_adaptor.py +0 -0
- /msprobe/{pytorch/api_accuracy_checker/run_ut → mindspore/dump/overflow_check}/__init__.py +0 -0
- /msprobe/{pytorch/debugger → mindspore/monitor}/__init__.py +0 -0
- /msprobe/{pytorch/free_benchmark/common → msaccucmp}/__init__.py +0 -0
- /msprobe/pytorch/api_accuracy_checker/{run_ut → acc_check}/.keep +0 -0
- /msprobe/pytorch/{free_benchmark/perturbed_layers → api_accuracy_checker/acc_check}/__init__.py +0 -0
- /msprobe/pytorch/api_accuracy_checker/{run_ut → acc_check}/torch_ut_setting.json +0 -0
- /msprobe/pytorch/{free_benchmark/perturbed_layers/npu → dump/api_dump}/__init__.py +0 -0
- /msprobe/pytorch/{hook_module → dump/api_dump}/support_wrap_ops.yaml +0 -0
- /msprobe/pytorch/{free_benchmark/result_handlers → dump/debugger}/__init__.py +0 -0
|
@@ -0,0 +1,308 @@
|
|
|
1
|
+
# coding=utf-8
|
|
2
|
+
# -------------------------------------------------------------------------
|
|
3
|
+
# This file is part of the MindStudio project.
|
|
4
|
+
# Copyright (c) 2025 Huawei Technologies Co.,Ltd.
|
|
5
|
+
#
|
|
6
|
+
# MindStudio is licensed under Mulan PSL v2.
|
|
7
|
+
# You can use this software according to the terms and conditions of the Mulan PSL v2.
|
|
8
|
+
# You may obtain a copy of Mulan PSL v2 at:
|
|
9
|
+
#
|
|
10
|
+
# http://license.coscl.org.cn/MulanPSL2
|
|
11
|
+
#
|
|
12
|
+
# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
|
|
13
|
+
# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
|
|
14
|
+
# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
|
|
15
|
+
# See the Mulan PSL v2 for more details.
|
|
16
|
+
# -------------------------------------------------------------------------
|
|
17
|
+
|
|
18
|
+
import json
|
|
19
|
+
from enum import Enum
|
|
20
|
+
import base64
|
|
21
|
+
from typing import List, Any, Dict
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class EnumEncoder(json.JSONEncoder):
|
|
25
|
+
def default(self, o):
|
|
26
|
+
if isinstance(o, Enum):
|
|
27
|
+
return o.value
|
|
28
|
+
if isinstance(o, bytes):
|
|
29
|
+
return base64.b64encode(o).decode("ascii")
|
|
30
|
+
return super().default(o)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class RepeatedField(list):
|
|
34
|
+
"""模仿 protobuf repeated 字段,支持 add()"""
|
|
35
|
+
def __init__(self, cls_type):
|
|
36
|
+
super().__init__()
|
|
37
|
+
self._cls_type = cls_type
|
|
38
|
+
|
|
39
|
+
def add(self, **kwargs):
|
|
40
|
+
obj = self._cls_type(**kwargs)
|
|
41
|
+
self.append(obj)
|
|
42
|
+
return obj
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class Shape:
|
|
46
|
+
def __init__(self, dim=None):
|
|
47
|
+
self._dim = []
|
|
48
|
+
if dim:
|
|
49
|
+
self.extend(dim)
|
|
50
|
+
|
|
51
|
+
@property
|
|
52
|
+
def dim(self):
|
|
53
|
+
return self._dim
|
|
54
|
+
|
|
55
|
+
def __setattr__(self, name, value):
|
|
56
|
+
if name == "dim":
|
|
57
|
+
object.__setattr__(self, "_dim", [int(v) for v in value])
|
|
58
|
+
else:
|
|
59
|
+
object.__setattr__(self, name, value)
|
|
60
|
+
|
|
61
|
+
def append(self, value):
|
|
62
|
+
self._dim.append(int(value))
|
|
63
|
+
|
|
64
|
+
def extend(self, values):
|
|
65
|
+
self._dim.extend(int(v) for v in values)
|
|
66
|
+
|
|
67
|
+
def Clear(self):
|
|
68
|
+
self._dim.clear()
|
|
69
|
+
|
|
70
|
+
def to_dict(self):
|
|
71
|
+
return {"dim": self._dim}
|
|
72
|
+
|
|
73
|
+
def __repr__(self):
|
|
74
|
+
return f"Shape(dim={self._dim})"
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
class DimRange:
|
|
78
|
+
def __init__(self, dim_start=0, dim_end=0):
|
|
79
|
+
self.dim_start = dim_start
|
|
80
|
+
self.dim_end = dim_end
|
|
81
|
+
|
|
82
|
+
def to_dict(self):
|
|
83
|
+
return {"dim_start": self.dim_start, "dim_end": self.dim_end}
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
class OriginalOp:
|
|
87
|
+
def __init__(self, name="", output_index=0, data_type=0, data_format=0):
|
|
88
|
+
self.name = name
|
|
89
|
+
self.output_index = output_index
|
|
90
|
+
self.data_type = data_type
|
|
91
|
+
self.format = data_format
|
|
92
|
+
|
|
93
|
+
def to_dict(self):
|
|
94
|
+
return {
|
|
95
|
+
"name": self.name,
|
|
96
|
+
"output_index": self.output_index,
|
|
97
|
+
"data_type": self.data_type,
|
|
98
|
+
"format": self.format,
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
class OpOutput():
|
|
103
|
+
def __init__(self):
|
|
104
|
+
self.data_type = 0
|
|
105
|
+
self.format = 0
|
|
106
|
+
self.shape = Shape()
|
|
107
|
+
self.original_op = OriginalOp()
|
|
108
|
+
self.data = b""
|
|
109
|
+
self.size = 0
|
|
110
|
+
self.original_shape = Shape()
|
|
111
|
+
self.sub_format = 0
|
|
112
|
+
self.address = 0
|
|
113
|
+
self.dim_range: List[DimRange] = []
|
|
114
|
+
self.offset = 0
|
|
115
|
+
|
|
116
|
+
def to_dict(self):
|
|
117
|
+
return {
|
|
118
|
+
"data_type": self.data_type,
|
|
119
|
+
"format": self.format,
|
|
120
|
+
"shape": self.shape.to_dict(),
|
|
121
|
+
"original_op": self.original_op.to_dict(),
|
|
122
|
+
"data": self.data,
|
|
123
|
+
"size": self.size,
|
|
124
|
+
"original_shape": self.original_shape.to_dict(),
|
|
125
|
+
"sub_format": self.sub_format,
|
|
126
|
+
"address": self.address,
|
|
127
|
+
"dim_range": [d.to_dict() for d in self.dim_range],
|
|
128
|
+
"offset": self.offset,
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
class OpInput():
|
|
133
|
+
def __init__(self):
|
|
134
|
+
self.data_type = 0
|
|
135
|
+
self.format = 0
|
|
136
|
+
self.shape = Shape()
|
|
137
|
+
self.data = b""
|
|
138
|
+
self.size = 0
|
|
139
|
+
self.original_shape = Shape()
|
|
140
|
+
self.sub_format = 0
|
|
141
|
+
self.address = 0
|
|
142
|
+
self.offset = 0
|
|
143
|
+
|
|
144
|
+
def to_dict(self):
|
|
145
|
+
return {
|
|
146
|
+
"data_type": self.data_type,
|
|
147
|
+
"format": self.format,
|
|
148
|
+
"shape": self.shape.to_dict(),
|
|
149
|
+
"data": self.data,
|
|
150
|
+
"size": self.size,
|
|
151
|
+
"original_shape": self.original_shape.to_dict(),
|
|
152
|
+
"sub_format": self.sub_format,
|
|
153
|
+
"address": self.address,
|
|
154
|
+
"offset": self.offset,
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
class OpBuffer:
|
|
159
|
+
def __init__(self):
|
|
160
|
+
self.buffer_type = 0
|
|
161
|
+
self.data = b""
|
|
162
|
+
self.size = 0
|
|
163
|
+
|
|
164
|
+
def to_dict(self):
|
|
165
|
+
return {"buffer_type": self.buffer_type, "data": self.data, "size": self.size}
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
class OpAttr:
|
|
169
|
+
def __init__(self, name="", value=""):
|
|
170
|
+
self.name = name
|
|
171
|
+
self.value = value
|
|
172
|
+
|
|
173
|
+
def to_dict(self):
|
|
174
|
+
return {"name": self.name, "value": self.value}
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
class Workspace:
|
|
178
|
+
def __init__(self, data_type=0):
|
|
179
|
+
self.type = data_type
|
|
180
|
+
self.data = b""
|
|
181
|
+
self.size = 0
|
|
182
|
+
|
|
183
|
+
def to_dict(self):
|
|
184
|
+
return {"type": self.type, "data": self.data, "size": self.size}
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
class DumpData:
|
|
188
|
+
def __init__(self):
|
|
189
|
+
self.version = ""
|
|
190
|
+
self.dump_time = 0
|
|
191
|
+
self.output = RepeatedField(OpOutput)
|
|
192
|
+
self.input = RepeatedField(OpInput)
|
|
193
|
+
self.buffer = RepeatedField(OpBuffer)
|
|
194
|
+
self.op_name = ""
|
|
195
|
+
self.attr = RepeatedField(OpAttr)
|
|
196
|
+
self.space = RepeatedField(Workspace)
|
|
197
|
+
|
|
198
|
+
@classmethod
|
|
199
|
+
def from_dict(cls, data: Dict):
|
|
200
|
+
obj = cls()
|
|
201
|
+
fill_dump_data(obj, data)
|
|
202
|
+
return obj
|
|
203
|
+
|
|
204
|
+
def to_dict(self):
|
|
205
|
+
return {
|
|
206
|
+
"version": self.version,
|
|
207
|
+
"dump_time": self.dump_time,
|
|
208
|
+
"op_name": self.op_name,
|
|
209
|
+
"output": [data.to_dict() for data in self.output],
|
|
210
|
+
"input": [data.to_dict() for data in self.input],
|
|
211
|
+
"buffer": [data.to_dict() for data in self.buffer],
|
|
212
|
+
"attr": [data.to_dict() for data in self.attr],
|
|
213
|
+
"space": [data.to_dict() for data in self.space],
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
def SerializeToString(self) -> bytes:
|
|
217
|
+
"""模拟 protobuf 的序列化,返回 JSON bytes"""
|
|
218
|
+
return json.dumps(self.to_dict(), cls=EnumEncoder).encode("utf-8")
|
|
219
|
+
|
|
220
|
+
def ParseFromString(self, data: bytes):
|
|
221
|
+
"""
|
|
222
|
+
模拟 protobuf 的 ParseFromString
|
|
223
|
+
:param data: bytes (JSON 格式)
|
|
224
|
+
:return: self
|
|
225
|
+
"""
|
|
226
|
+
obj_dict = json.loads(data.decode("utf-8"))
|
|
227
|
+
# 用 from_dict 填充当前实例
|
|
228
|
+
fill_dump_data(self, obj_dict)
|
|
229
|
+
return self
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
TYPE_MAP = {
|
|
233
|
+
"input": OpInput,
|
|
234
|
+
"output": OpOutput,
|
|
235
|
+
"buffer": OpBuffer,
|
|
236
|
+
"attr": OpAttr,
|
|
237
|
+
"space": Workspace,
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
def _handle_repeated_field(attr: Any, key: str, value_list: list) -> None:
|
|
242
|
+
"""处理 repeated 字段:当 TYPE_MAP 有对应类时用 add() 并递归填充,否则 append 原始元素。"""
|
|
243
|
+
cls_type = TYPE_MAP.get(key)
|
|
244
|
+
if cls_type:
|
|
245
|
+
for item in value_list:
|
|
246
|
+
# 保持原行为:使用 add() 创建子对象
|
|
247
|
+
sub_obj = attr.add()
|
|
248
|
+
# 只有 dict 才递归填充,否则跳过(与原行为一致)
|
|
249
|
+
if isinstance(item, dict):
|
|
250
|
+
fill_dump_data(sub_obj, item)
|
|
251
|
+
else:
|
|
252
|
+
# 保持原行为:逐项 append(而不是 extend),以保留原先每项可能的非 list 语义
|
|
253
|
+
for item in value_list:
|
|
254
|
+
attr.append(item)
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
def _handle_shape(attr: "Shape", value: Any) -> None:
|
|
258
|
+
"""填充 Shape 类型:支持 dict {'dim': [...]} 或直接 list."""
|
|
259
|
+
if isinstance(value, dict):
|
|
260
|
+
dims = value.get("dim")
|
|
261
|
+
if dims is None:
|
|
262
|
+
return
|
|
263
|
+
# 保证全部转换为 int
|
|
264
|
+
attr.dim = [int(v) for v in dims]
|
|
265
|
+
elif isinstance(value, list):
|
|
266
|
+
# 如果直接给了 list,也允许赋值(向后兼容)
|
|
267
|
+
attr.dim = [int(v) for v in value]
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
def _handle_original_op(attr: "OriginalOp", value: Any) -> None:
|
|
271
|
+
"""按字段赋值 OriginalOp(仅处理 dict)。"""
|
|
272
|
+
if not isinstance(value, dict):
|
|
273
|
+
return
|
|
274
|
+
for sub_key, sub_val in value.items():
|
|
275
|
+
if hasattr(attr, sub_key):
|
|
276
|
+
setattr(attr, sub_key, sub_val)
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
def fill_dump_data(obj: Any, data: Dict) -> Any:
|
|
280
|
+
"""
|
|
281
|
+
将字典 data 递归填充到 obj(DumpData 或子消息)上。
|
|
282
|
+
"""
|
|
283
|
+
for key, value in data.items():
|
|
284
|
+
if not hasattr(obj, key):
|
|
285
|
+
# 字典中含有类没有的字段,跳过
|
|
286
|
+
continue
|
|
287
|
+
|
|
288
|
+
attr = getattr(obj, key)
|
|
289
|
+
|
|
290
|
+
# 1) repeated field (RepeatedField)
|
|
291
|
+
if isinstance(attr, RepeatedField) and isinstance(value, list):
|
|
292
|
+
_handle_repeated_field(attr, key, value)
|
|
293
|
+
continue
|
|
294
|
+
|
|
295
|
+
# 2) Shape 嵌套
|
|
296
|
+
if isinstance(attr, Shape):
|
|
297
|
+
_handle_shape(attr, value)
|
|
298
|
+
continue
|
|
299
|
+
|
|
300
|
+
# 3) OriginalOp 嵌套
|
|
301
|
+
if isinstance(attr, OriginalOp):
|
|
302
|
+
_handle_original_op(attr, value)
|
|
303
|
+
continue
|
|
304
|
+
|
|
305
|
+
# 4) 其它基本类型(bytes / int / str / list 等),直接赋值
|
|
306
|
+
setattr(obj, key, value)
|
|
307
|
+
|
|
308
|
+
return obj
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
# coding=utf-8
|
|
2
|
+
# -------------------------------------------------------------------------
|
|
3
|
+
# This file is part of the MindStudio project.
|
|
4
|
+
# Copyright (c) 2025 Huawei Technologies Co.,Ltd.
|
|
5
|
+
#
|
|
6
|
+
# MindStudio is licensed under Mulan PSL v2.
|
|
7
|
+
# You can use this software according to the terms and conditions of the Mulan PSL v2.
|
|
8
|
+
# You may obtain a copy of Mulan PSL v2 at:
|
|
9
|
+
#
|
|
10
|
+
# http://license.coscl.org.cn/MulanPSL2
|
|
11
|
+
#
|
|
12
|
+
# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
|
|
13
|
+
# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
|
|
14
|
+
# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
|
|
15
|
+
# See the Mulan PSL v2 for more details.
|
|
16
|
+
# -------------------------------------------------------------------------
|
|
17
|
+
|
|
18
|
+
"""
|
|
19
|
+
Function:
|
|
20
|
+
This class mainly involves the main function.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
import argparse
|
|
24
|
+
import sys
|
|
25
|
+
import time
|
|
26
|
+
|
|
27
|
+
from msprobe.msaccucmp.cmp_utils import log, file_utils
|
|
28
|
+
from msprobe.msaccucmp.cmp_utils.utils import safe_path_string
|
|
29
|
+
from msprobe.msaccucmp.cmp_utils.constant.compare_error import CompareError
|
|
30
|
+
from msprobe.msaccucmp.dump_parse.dump_data_parser import DumpDataParser
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _save_log_parser(save_log_parser: argparse.ArgumentParser) -> None:
|
|
34
|
+
save_log_parser.add_argument(
|
|
35
|
+
'-d', '--dump_file', dest='dump_path', default='', type=safe_path_string,
|
|
36
|
+
help='<Required> the dump file path, supports one AICPU custom operator dump file.',
|
|
37
|
+
required=True)
|
|
38
|
+
save_log_parser.add_argument('-out', '--output', dest='output_path', default='', type=safe_path_string,
|
|
39
|
+
help='<Optional> the output path')
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _do_cmd() -> int:
|
|
43
|
+
parser = argparse.ArgumentParser()
|
|
44
|
+
subparsers = parser.add_subparsers(help='commands')
|
|
45
|
+
save_log_parser = subparsers.add_parser('save_log', help='Save AICPU custom operator log.')
|
|
46
|
+
|
|
47
|
+
_save_log_parser(save_log_parser)
|
|
48
|
+
|
|
49
|
+
args = parser.parse_args(sys.argv[1:])
|
|
50
|
+
if len(sys.argv) < 2:
|
|
51
|
+
parser.print_help()
|
|
52
|
+
raise CompareError(CompareError.MSACCUCMP_INVALID_PARAM_ERROR)
|
|
53
|
+
|
|
54
|
+
if sys.argv[1] == 'save_log':
|
|
55
|
+
args.dump_version = 2
|
|
56
|
+
args.output_file_type = None
|
|
57
|
+
ret = _do_save_log(args)
|
|
58
|
+
return ret
|
|
59
|
+
else:
|
|
60
|
+
return 0
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _do_save_log(args: argparse.Namespace) -> int:
|
|
64
|
+
ret = DumpDataParser(args).parse_log_data()
|
|
65
|
+
return ret
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def main() -> None:
|
|
69
|
+
"""
|
|
70
|
+
parse argument and run command
|
|
71
|
+
:return:
|
|
72
|
+
"""
|
|
73
|
+
start = time.time()
|
|
74
|
+
with file_utils.UmaskWrapper():
|
|
75
|
+
try:
|
|
76
|
+
ret = _do_cmd()
|
|
77
|
+
except CompareError as err:
|
|
78
|
+
ret = err.code
|
|
79
|
+
except Exception as base_err:
|
|
80
|
+
log.print_error_log(f'Basic error running {sys.argv[0]}: {base_err}')
|
|
81
|
+
sys.exit(1)
|
|
82
|
+
end = time.time()
|
|
83
|
+
if ret != 0:
|
|
84
|
+
log.print_error_log("Failed to parse dump log.")
|
|
85
|
+
log.print_info_log(
|
|
86
|
+
'The command was completed and took %d seconds.' % (end - start))
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
if __name__ == '__main__':
|
|
90
|
+
main()
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
# -------------------------------------------------------------------------
|
|
2
|
+
# This file is part of the MindStudio project.
|
|
3
|
+
# Copyright (c) 2025 Huawei Technologies Co.,Ltd.
|
|
4
|
+
#
|
|
5
|
+
# MindStudio is licensed under Mulan PSL v2.
|
|
6
|
+
# You can use this software according to the terms and conditions of the Mulan PSL v2.
|
|
7
|
+
# You may obtain a copy of Mulan PSL v2 at:
|
|
8
|
+
#
|
|
9
|
+
# http://license.coscl.org.cn/MulanPSL2
|
|
10
|
+
#
|
|
11
|
+
# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
|
|
12
|
+
# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
|
|
13
|
+
# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
|
|
14
|
+
# See the Mulan PSL v2 for more details.
|
|
15
|
+
# -------------------------------------------------------------------------
|
|
16
|
+
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
# coding=utf-8
|
|
2
|
+
# -------------------------------------------------------------------------
|
|
3
|
+
# This file is part of the MindStudio project.
|
|
4
|
+
# Copyright (c) 2025 Huawei Technologies Co.,Ltd.
|
|
5
|
+
#
|
|
6
|
+
# MindStudio is licensed under Mulan PSL v2.
|
|
7
|
+
# You can use this software according to the terms and conditions of the Mulan PSL v2.
|
|
8
|
+
# You may obtain a copy of Mulan PSL v2 at:
|
|
9
|
+
#
|
|
10
|
+
# http://license.coscl.org.cn/MulanPSL2
|
|
11
|
+
#
|
|
12
|
+
# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
|
|
13
|
+
# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
|
|
14
|
+
# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
|
|
15
|
+
# See the Mulan PSL v2 for more details.
|
|
16
|
+
# -------------------------------------------------------------------------
|
|
17
|
+
|
|
18
|
+
"""
|
|
19
|
+
Function:
|
|
20
|
+
convert format from FRACTAL_NZ to NCHW.
|
|
21
|
+
"""
|
|
22
|
+
from functools import reduce
|
|
23
|
+
|
|
24
|
+
import numpy as np
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def convert(shape_from: list, shape_to: list, array: any) -> any:
|
|
28
|
+
"""
|
|
29
|
+
Convert the data format from FRACTAL_NZ to NCHW
|
|
30
|
+
:param shape_from: the shape before convert
|
|
31
|
+
:param shape_to: the shape after convert
|
|
32
|
+
:param array: the one-dimensional array
|
|
33
|
+
:return: the data array of NCHW shape
|
|
34
|
+
"""
|
|
35
|
+
if len(shape_to) == 1:
|
|
36
|
+
axis_h, axis_n, axis_c = 1, 1, shape_to[0]
|
|
37
|
+
elif len(shape_to) == 2:
|
|
38
|
+
axis_h, axis_n, axis_c = 1, shape_to[0], shape_to[1]
|
|
39
|
+
else:
|
|
40
|
+
axis_h, axis_n, axis_c = reduce(lambda x, y: x * y, shape_to[:-2]), shape_to[-2], shape_to[-1]
|
|
41
|
+
axis_c0 = shape_from[-1]
|
|
42
|
+
axis_ni = shape_from[-2]
|
|
43
|
+
axis_no = shape_from[-3]
|
|
44
|
+
axis_c1 = shape_from[-4]
|
|
45
|
+
c_pad = None if axis_c1 * axis_c0 == axis_c else axis_c - axis_c1 * axis_c0
|
|
46
|
+
n_pad = None if axis_no * axis_ni == axis_n else axis_n - axis_no * axis_ni
|
|
47
|
+
tmp_input_tensor = array.reshape(axis_h, axis_c1, axis_no, axis_ni, axis_c0)
|
|
48
|
+
tmp_input_tensor = np.transpose(tmp_input_tensor, axes=(0, 2, 3, 1, 4))
|
|
49
|
+
tmp_input_tensor = tmp_input_tensor.reshape((axis_h, axis_no * axis_ni, axis_c1 * axis_c0))
|
|
50
|
+
data_y = tmp_input_tensor[:, :n_pad, :c_pad]
|
|
51
|
+
if len(shape_to) <= 2:
|
|
52
|
+
data_y = data_y.reshape([data_y.shape[1], data_y.shape[2]])
|
|
53
|
+
return data_y
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
# coding=utf-8
|
|
2
|
+
# -------------------------------------------------------------------------
|
|
3
|
+
# This file is part of the MindStudio project.
|
|
4
|
+
# Copyright (c) 2025 Huawei Technologies Co.,Ltd.
|
|
5
|
+
#
|
|
6
|
+
# MindStudio is licensed under Mulan PSL v2.
|
|
7
|
+
# You can use this software according to the terms and conditions of the Mulan PSL v2.
|
|
8
|
+
# You may obtain a copy of Mulan PSL v2 at:
|
|
9
|
+
#
|
|
10
|
+
# http://license.coscl.org.cn/MulanPSL2
|
|
11
|
+
#
|
|
12
|
+
# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
|
|
13
|
+
# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
|
|
14
|
+
# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
|
|
15
|
+
# See the Mulan PSL v2 for more details.
|
|
16
|
+
# -------------------------------------------------------------------------
|
|
17
|
+
|
|
18
|
+
"""
|
|
19
|
+
Function:
|
|
20
|
+
convert format from FRACTAL_NZ to ND.
|
|
21
|
+
"""
|
|
22
|
+
from functools import reduce
|
|
23
|
+
import numpy as np
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def convert(shape_from: list, shape_to: list, array: any) -> any:
|
|
27
|
+
"""
|
|
28
|
+
Convert the data format from FRACTAL_NZ to ND
|
|
29
|
+
:param shape_from: the shape before convert
|
|
30
|
+
:param shape_to: the shape after convert
|
|
31
|
+
:param array: the one-dimensional array
|
|
32
|
+
:return: the data array of ND shape
|
|
33
|
+
"""
|
|
34
|
+
if len(shape_to) == 1:
|
|
35
|
+
axis_h, axis_n, axis_c = 1, 1, shape_to[0]
|
|
36
|
+
elif len(shape_to) == 2:
|
|
37
|
+
axis_h, axis_n, axis_c = 1, shape_to[0], shape_to[1]
|
|
38
|
+
else:
|
|
39
|
+
axis_h, axis_n, axis_c = reduce(lambda x, y: x * y, shape_to[:-2]), shape_to[-2], shape_to[-1]
|
|
40
|
+
axis_c1 = shape_from[-4]
|
|
41
|
+
axis_no = shape_from[-3]
|
|
42
|
+
axis_ni = shape_from[-2]
|
|
43
|
+
axis_c0 = shape_from[-1]
|
|
44
|
+
c_pad = None if axis_c1 * axis_c0 == axis_c else axis_c - axis_c1 * axis_c0
|
|
45
|
+
n_pad = None if axis_no * axis_ni == axis_n else axis_n - axis_no * axis_ni
|
|
46
|
+
tmp_input_tensor = array.reshape(axis_h, axis_c1, axis_no, axis_ni, axis_c0)
|
|
47
|
+
tmp_input_tensor = np.transpose(tmp_input_tensor, axes=(0, 2, 3, 1, 4))
|
|
48
|
+
tmp_input_tensor = tmp_input_tensor.reshape((axis_h, axis_no * axis_ni, axis_c1 * axis_c0))
|
|
49
|
+
data_y = tmp_input_tensor[:, :n_pad, :c_pad]
|
|
50
|
+
if len(shape_to) <= 2:
|
|
51
|
+
data_y = data_y.reshape([data_y.shape[1], data_y.shape[2]])
|
|
52
|
+
return data_y
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
# coding=utf-8
|
|
2
|
+
# -------------------------------------------------------------------------
|
|
3
|
+
# This file is part of the MindStudio project.
|
|
4
|
+
# Copyright (c) 2025 Huawei Technologies Co.,Ltd.
|
|
5
|
+
#
|
|
6
|
+
# MindStudio is licensed under Mulan PSL v2.
|
|
7
|
+
# You can use this software according to the terms and conditions of the Mulan PSL v2.
|
|
8
|
+
# You may obtain a copy of Mulan PSL v2 at:
|
|
9
|
+
#
|
|
10
|
+
# http://license.coscl.org.cn/MulanPSL2
|
|
11
|
+
#
|
|
12
|
+
# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
|
|
13
|
+
# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
|
|
14
|
+
# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
|
|
15
|
+
# See the Mulan PSL v2 for more details.
|
|
16
|
+
# -------------------------------------------------------------------------
|
|
17
|
+
|
|
18
|
+
"""
|
|
19
|
+
Function:
|
|
20
|
+
convert format from FRACTAL_NZ to NHWC.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from functools import reduce
|
|
24
|
+
import numpy as np
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def convert(shape_from: list, shape_to: list, array: any) -> any:
|
|
28
|
+
"""
|
|
29
|
+
Convert the data format from FRACTAL_NZ to NHWC
|
|
30
|
+
:param shape_from: the shape before convert
|
|
31
|
+
:param shape_to: the shape after convert
|
|
32
|
+
:param array: the one-dimensional array
|
|
33
|
+
:return: the data array of NHWC shape
|
|
34
|
+
"""
|
|
35
|
+
if len(shape_to) == 1:
|
|
36
|
+
axis_h, axis_n, axis_c = 1, 1, shape_to[0]
|
|
37
|
+
elif len(shape_to) == 2:
|
|
38
|
+
axis_h, axis_n, axis_c = 1, shape_to[0], shape_to[1]
|
|
39
|
+
else:
|
|
40
|
+
axis_h, axis_n, axis_c = reduce(lambda x, y: x * y, shape_to[:-2]), shape_to[-2], shape_to[-1]
|
|
41
|
+
axis_ni = shape_from[-2]
|
|
42
|
+
axis_no = shape_from[-3]
|
|
43
|
+
axis_c0 = shape_from[-1]
|
|
44
|
+
axis_c1 = shape_from[-4]
|
|
45
|
+
n_pad = None if axis_no * axis_ni == axis_n else axis_n - axis_no * axis_ni
|
|
46
|
+
c_pad = None if axis_c1 * axis_c0 == axis_c else axis_c - axis_c1 * axis_c0
|
|
47
|
+
tmp_input_tensor = array.reshape(axis_h, axis_c1, axis_no, axis_ni, axis_c0)
|
|
48
|
+
tmp_input_tensor = np.transpose(tmp_input_tensor, axes=(0, 2, 3, 1, 4))
|
|
49
|
+
tmp_input_tensor = tmp_input_tensor.reshape((axis_h, axis_no * axis_ni, axis_c1 * axis_c0))
|
|
50
|
+
data_y = tmp_input_tensor[:, :n_pad, :c_pad]
|
|
51
|
+
if len(shape_to) <= 2:
|
|
52
|
+
data_y = data_y.reshape([data_y.shape[1], data_y.shape[2]])
|
|
53
|
+
return data_y
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
# coding=utf-8
|
|
2
|
+
# -------------------------------------------------------------------------
|
|
3
|
+
# This file is part of the MindStudio project.
|
|
4
|
+
# Copyright (c) 2025 Huawei Technologies Co.,Ltd.
|
|
5
|
+
#
|
|
6
|
+
# MindStudio is licensed under Mulan PSL v2.
|
|
7
|
+
# You can use this software according to the terms and conditions of the Mulan PSL v2.
|
|
8
|
+
# You may obtain a copy of Mulan PSL v2 at:
|
|
9
|
+
#
|
|
10
|
+
# http://license.coscl.org.cn/MulanPSL2
|
|
11
|
+
#
|
|
12
|
+
# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
|
|
13
|
+
# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
|
|
14
|
+
# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
|
|
15
|
+
# See the Mulan PSL v2 for more details.
|
|
16
|
+
# -------------------------------------------------------------------------
|
|
17
|
+
|
|
18
|
+
"""
|
|
19
|
+
Function:
|
|
20
|
+
convert format from FRACTAL_Z to HWCN.
|
|
21
|
+
"""
|
|
22
|
+
import numpy as np
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def convert(shape_from: list, shape_to: list, array: any) -> any:
|
|
26
|
+
"""
|
|
27
|
+
Convert the data format from FRACTAL_Z to HWCN
|
|
28
|
+
:param shape_from: the shape before convert
|
|
29
|
+
:param shape_to: the shape after convert
|
|
30
|
+
:param array: the one-dimensional array
|
|
31
|
+
:return: the data array of HWCN shape
|
|
32
|
+
"""
|
|
33
|
+
axis_c = shape_to[2]
|
|
34
|
+
axis_n = shape_to[3]
|
|
35
|
+
axis_no = shape_from[1]
|
|
36
|
+
axis_ni = shape_from[2]
|
|
37
|
+
axis_h = shape_to[0]
|
|
38
|
+
axis_w = shape_to[1]
|
|
39
|
+
axis_c1 = shape_from[0] // (axis_h * axis_w)
|
|
40
|
+
axis_c0 = shape_from[3]
|
|
41
|
+
c_pad = None if axis_c1 * axis_c0 == axis_c else axis_c - axis_c1 * axis_c0
|
|
42
|
+
n_pad = None if axis_no * axis_ni == axis_n else axis_n - axis_no * axis_ni
|
|
43
|
+
tmp_input_tensor = array.reshape(axis_c1, axis_h, axis_w, axis_no, axis_ni, axis_c0)
|
|
44
|
+
# transpose the shape from (c1,h,w,no,ni,c0) to (h,w,c1,c0,no,ni)
|
|
45
|
+
tmp_input_tensor = np.transpose(tmp_input_tensor, (1, 2, 0, 5, 3, 4))
|
|
46
|
+
tmp_input_tensor = tmp_input_tensor.reshape((axis_h, axis_w, axis_c1 * axis_c0, axis_no * axis_ni))
|
|
47
|
+
return tmp_input_tensor[:, :, :c_pad, :n_pad]
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
# coding=utf-8
|
|
2
|
+
# -------------------------------------------------------------------------
|
|
3
|
+
# This file is part of the MindStudio project.
|
|
4
|
+
# Copyright (c) 2025 Huawei Technologies Co.,Ltd.
|
|
5
|
+
#
|
|
6
|
+
# MindStudio is licensed under Mulan PSL v2.
|
|
7
|
+
# You can use this software according to the terms and conditions of the Mulan PSL v2.
|
|
8
|
+
# You may obtain a copy of Mulan PSL v2 at:
|
|
9
|
+
#
|
|
10
|
+
# http://license.coscl.org.cn/MulanPSL2
|
|
11
|
+
#
|
|
12
|
+
# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
|
|
13
|
+
# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
|
|
14
|
+
# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
|
|
15
|
+
# See the Mulan PSL v2 for more details.
|
|
16
|
+
# -------------------------------------------------------------------------
|
|
17
|
+
|
|
18
|
+
"""
|
|
19
|
+
Function:
|
|
20
|
+
convert format from FRACTAL_Z to NCHW.
|
|
21
|
+
"""
|
|
22
|
+
import numpy as np
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def convert(shape_from: list, shape_to: list, array: any) -> any:
|
|
26
|
+
"""
|
|
27
|
+
Convert the data format from FRACTAL_Z to NCHW
|
|
28
|
+
:param shape_from: the shape before convert
|
|
29
|
+
:param shape_to: the shape after convert
|
|
30
|
+
:param array: the one-dimensional array
|
|
31
|
+
:return: the data array of NCHW shape
|
|
32
|
+
"""
|
|
33
|
+
axis_c = shape_to[1]
|
|
34
|
+
axis_n = shape_to[0]
|
|
35
|
+
axis_no = shape_from[1]
|
|
36
|
+
axis_ni = shape_from[2]
|
|
37
|
+
axis_h = shape_to[2]
|
|
38
|
+
axis_w = shape_to[3]
|
|
39
|
+
axis_c1 = shape_from[0] // (axis_h * axis_w)
|
|
40
|
+
axis_c0 = shape_from[3]
|
|
41
|
+
c_pad = None if axis_c1 * axis_c0 == axis_c else axis_c - axis_c1 * axis_c0
|
|
42
|
+
n_pad = None if axis_no * axis_ni == axis_n else axis_n - axis_no * axis_ni
|
|
43
|
+
tmp_input_tensor = array.reshape(axis_c1, axis_h, axis_w, axis_no, axis_ni, axis_c0)
|
|
44
|
+
# transpose the shape from (c1,h,w,no,ni,c0) to (no,ni,c1,c0,h,w)
|
|
45
|
+
tmp_input_tensor = np.transpose(tmp_input_tensor, (3, 4, 0, 5, 1, 2))
|
|
46
|
+
tmp_input_tensor = tmp_input_tensor.reshape((axis_no * axis_ni, axis_c1 * axis_c0, axis_h, axis_w))
|
|
47
|
+
return tmp_input_tensor[:n_pad, :c_pad, :, :, ]
|