mindstudio-probe 1.1.1__tar.gz → 1.2.1__tar.gz

{mindstudio_probe-1.1.1 → mindstudio_probe-1.2.1}/MANIFEST.in

@@ -2,4 +2,5 @@ include README.md
 include LICENSE
 recursive-include msprobe *
 recursive-exclude msprobe/test *
+recursive-exclude msprobe/ccsrc *
 

{mindstudio_probe-1.1.1 → mindstudio_probe-1.2.1}/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.1
 Name: mindstudio-probe
-Version: 1.1.1
-Summary: Pytorch Ascend Probe Utils
+Version: 1.2.1
+Summary: Ascend Probe Utils
 Home-page: https://gitee.com/ascend/mstt/tree/master/debug/accuracy_tools/msprobe
 Author: Ascend Team
 Author-email: pmail_mindstudio@huawei.com
@@ -11,6 +11,7 @@ Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Education
 Classifier: Intended Audience :: Science/Research
 Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: C++
 Classifier: Topic :: Scientific/Engineering
 Classifier: Topic :: Scientific/Engineering :: Mathematics
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence

{mindstudio_probe-1.1.1 → mindstudio_probe-1.2.1}/mindstudio_probe.egg-info/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.1
 Name: mindstudio-probe
-Version: 1.1.1
-Summary: Pytorch Ascend Probe Utils
+Version: 1.2.1
+Summary: Ascend Probe Utils
 Home-page: https://gitee.com/ascend/mstt/tree/master/debug/accuracy_tools/msprobe
 Author: Ascend Team
 Author-email: pmail_mindstudio@huawei.com
@@ -11,6 +11,7 @@ Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Education
 Classifier: Intended Audience :: Science/Research
 Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: C++
 Classifier: Topic :: Scientific/Engineering
 Classifier: Topic :: Scientific/Engineering :: Mathematics
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence

{mindstudio_probe-1.1.1 → mindstudio_probe-1.2.1}/mindstudio_probe.egg-info/SOURCES.txt

@@ -8,6 +8,7 @@ mindstudio_probe.egg-info/entry_points.txt
 mindstudio_probe.egg-info/not-zip-safe
 mindstudio_probe.egg-info/requires.txt
 mindstudio_probe.egg-info/top_level.txt
+msprobe/CMakeLists.txt
 msprobe/README.md
 msprobe/__init__.py
 msprobe/config.json
@@ -35,6 +36,8 @@ msprobe/core/compare/layer_mapping/__init__.py
 msprobe/core/compare/layer_mapping/data_scope_parser.py
 msprobe/core/compare/layer_mapping/layer_mapping.py
 msprobe/core/compare/layer_mapping/postprocess_pass.py
+msprobe/core/compare/merge_result/merge_result.py
+msprobe/core/compare/merge_result/merge_result_cli.py
 msprobe/core/data_dump/data_collector.py
 msprobe/core/data_dump/json_writer.py
 msprobe/core/data_dump/scope.py
@@ -75,10 +78,16 @@ msprobe/docs/19.monitor.md
 msprobe/docs/20.monitor_performance_baseline.md
 msprobe/docs/21.visualization_PyTorch.md
 msprobe/docs/22.visualization_MindSpore.md
-msprobe/docs/23.tool_function_introduction.md
+msprobe/docs/23.generate_operator_PyTorch.md
+msprobe/docs/24.code_mapping_Mindspore.md
+msprobe/docs/25.tool_function_introduction.md
+msprobe/docs/26.data_dump_PyTorch_baseline.md
+msprobe/docs/27.dump_json_instruction.md
 msprobe/docs/FAQ.md
 msprobe/docs/S02.report_free_benchmarking_validation_performance_baseline.md
-msprobe/docs/data_dump_Mindspore/dynamic_graph_quick_start_example.md
+msprobe/docs/accuracy_checker_MindSpore/accuracy_checker_MindSpore_baseline.md
+msprobe/docs/data_dump_MindSpore/data_dump_MindSpore_baseline.md
+msprobe/docs/data_dump_MindSpore/dynamic_graph_quick_start_example.md
 msprobe/docs/img/BLOOM-7B_1.png
 msprobe/docs/img/BLOOM-7B_2.png
 msprobe/docs/img/BLOOM-7B_3.png
@@ -109,11 +118,34 @@ msprobe/docs/img/grad_probe_image-2.png
 msprobe/docs/img/grad_probe_image-3.png
 msprobe/docs/img/grad_probe_image-4.png
 msprobe/docs/img/grad_probe_image.png
+msprobe/docs/img/merge_result.png
 msprobe/docs/img/module_compare.png
 msprobe/docs/img/ms_dump.png
 msprobe/docs/img/ms_layer.png
 msprobe/docs/img/pt_dump.png
 msprobe/docs/img/monitor/cpu_info.png
+msprobe/docs/img/visualization/fuzzy_match_ms.png
+msprobe/docs/img/visualization/fuzzy_match_pt.png
+msprobe/docs/img/visualization/tensorboard_1.png
+msprobe/docs/img/visualization/tensorboard_2.png
+msprobe/docs/img/visualization/vis_browser_1.png
+msprobe/docs/img/visualization/vis_browser_2.png
+msprobe/docs/img/visualization/vis_precision_info.png
+msprobe/docs/img/visualization/vis_search_info.png
+msprobe/docs/img/visualization/vis_show_info.png
+msprobe/docs/img/visualization/vis_showcase.png
+msprobe/docs/img/visualization/vis_unmatch_info.png
+msprobe/docs/visualization/GPTModel.png
+msprobe/docs/visualization/ParallelMLP.png
+msprobe/docs/visualization/layer_mapping_example.md
+msprobe/docs/visualization/mapping.png
+msprobe/docs/visualization/mapping1.png
+msprobe/docs/visualization/module_name.png
+msprobe/docs/visualization/module_name1.png
+msprobe/docs/visualization/no_mapping.png
+msprobe/docs/visualization/no_mapping1.png
+msprobe/docs/visualization/no_mapping_analyze.png
+msprobe/docs/visualization/top_layer.png
 msprobe/mindspore/__init__.py
 msprobe/mindspore/cell_processor.py
 msprobe/mindspore/ms_config.py
@@ -134,6 +166,13 @@ msprobe/mindspore/api_accuracy_checker/multi_api_accuracy_checker.py
 msprobe/mindspore/api_accuracy_checker/multi_data_manager.py
 msprobe/mindspore/api_accuracy_checker/type_mapping.py
 msprobe/mindspore/api_accuracy_checker/utils.py
+msprobe/mindspore/code_mapping/__init__.py
+msprobe/mindspore/code_mapping/bind.py
+msprobe/mindspore/code_mapping/cmd_parser.py
+msprobe/mindspore/code_mapping/graph.py
+msprobe/mindspore/code_mapping/graph_parser.py
+msprobe/mindspore/code_mapping/main.py
+msprobe/mindspore/code_mapping/processor.py
 msprobe/mindspore/common/const.py
 msprobe/mindspore/common/log.py
 msprobe/mindspore/common/utils.py
@@ -154,6 +193,7 @@ msprobe/mindspore/dump/hook_cell/hook_cell.py
 msprobe/mindspore/dump/hook_cell/primitive_hooks.py
 msprobe/mindspore/dump/hook_cell/support_wrap_ops.yaml
 msprobe/mindspore/dump/hook_cell/wrap_api.py
+msprobe/mindspore/dump/kernel_dump/kernel_config.py
 msprobe/mindspore/dym_loader/hook_dynamic_loader.cc
 msprobe/mindspore/dym_loader/hook_dynamic_loader.h
 msprobe/mindspore/free_benchmark/__init__.py
@@ -183,12 +223,13 @@ msprobe/mindspore/grad_probe/grad_monitor.py
 msprobe/mindspore/grad_probe/grad_stat_csv.py
 msprobe/mindspore/grad_probe/hook.py
 msprobe/mindspore/grad_probe/utils.py
+msprobe/mindspore/mindtorch/__init__.py
+msprobe/mindspore/mindtorch/mindtorch_adaptor.py
 msprobe/mindspore/overflow_check/__init__.py
 msprobe/mindspore/overflow_check/kernel_graph_overflow_check.py
 msprobe/mindspore/overflow_check/overflow_check_tool_factory.py
 msprobe/pytorch/__init__.py
 msprobe/pytorch/function_factory.py
-msprobe/pytorch/module_processer.py
 msprobe/pytorch/parse.py
 msprobe/pytorch/pt_config.py
 msprobe/pytorch/service.py
@@ -206,10 +247,20 @@ msprobe/pytorch/api_accuracy_checker/compare/api_precision_standard.yaml
 msprobe/pytorch/api_accuracy_checker/compare/api_precision_threshold.yaml
 msprobe/pytorch/api_accuracy_checker/compare/compare.py
 msprobe/pytorch/api_accuracy_checker/compare/compare_column.py
+msprobe/pytorch/api_accuracy_checker/compare/compare_input.py
 msprobe/pytorch/api_accuracy_checker/compare/compare_utils.py
 msprobe/pytorch/api_accuracy_checker/generate_op_script/config_op.json
 msprobe/pytorch/api_accuracy_checker/generate_op_script/op_generator.py
 msprobe/pytorch/api_accuracy_checker/generate_op_script/operator_replication.template
+msprobe/pytorch/api_accuracy_checker/precision_standard/absolute_threshold.py
+msprobe/pytorch/api_accuracy_checker/precision_standard/accumulative_error_compare.py
+msprobe/pytorch/api_accuracy_checker/precision_standard/base_standard.py
+msprobe/pytorch/api_accuracy_checker/precision_standard/benchmark_compare.py
+msprobe/pytorch/api_accuracy_checker/precision_standard/binary_consistency.py
+msprobe/pytorch/api_accuracy_checker/precision_standard/standard_config.py
+msprobe/pytorch/api_accuracy_checker/precision_standard/standard_register.py
+msprobe/pytorch/api_accuracy_checker/precision_standard/thousandth_standard.py
+msprobe/pytorch/api_accuracy_checker/precision_standard/ulp_compare.py
 msprobe/pytorch/api_accuracy_checker/run_ut/.keep
 msprobe/pytorch/api_accuracy_checker/run_ut/__init__.py
 msprobe/pytorch/api_accuracy_checker/run_ut/data_generate.py
@@ -251,6 +302,9 @@ msprobe/pytorch/debugger/__init__.py
 msprobe/pytorch/debugger/debugger_config.py
 msprobe/pytorch/debugger/precision_debugger.py
 msprobe/pytorch/dump/kernel_dump/kernel_config.py
+msprobe/pytorch/dump/module_dump/__init__.py
+msprobe/pytorch/dump/module_dump/module_dump.py
+msprobe/pytorch/dump/module_dump/module_processer.py
 msprobe/pytorch/free_benchmark/__init__.py
 msprobe/pytorch/free_benchmark/main.py
 msprobe/pytorch/free_benchmark/common/__init__.py
@@ -278,14 +332,13 @@ msprobe/pytorch/free_benchmark/result_handlers/check_handler.py
 msprobe/pytorch/free_benchmark/result_handlers/fix_handler.py
 msprobe/pytorch/free_benchmark/result_handlers/handler_factory.py
 msprobe/pytorch/free_benchmark/result_handlers/preheat_handler.py
-msprobe/pytorch/functional/__init__.py
-msprobe/pytorch/functional/module_dump.py
 msprobe/pytorch/grad_probe/__init__.py
 msprobe/pytorch/grad_probe/grad_monitor.py
 msprobe/pytorch/grad_probe/grad_stat_csv.py
 msprobe/pytorch/hook_module/__init__.py
 msprobe/pytorch/hook_module/api_registry.py
 msprobe/pytorch/hook_module/hook_module.py
+msprobe/pytorch/hook_module/register_optimizer_hook.py
 msprobe/pytorch/hook_module/support_wrap_ops.yaml
 msprobe/pytorch/hook_module/utils.py
 msprobe/pytorch/hook_module/wrap_aten.py
@@ -298,6 +351,7 @@ msprobe/pytorch/hook_module/wrap_vf.py
 msprobe/pytorch/monitor/__init__.py
 msprobe/pytorch/monitor/anomaly_analyse.py
 msprobe/pytorch/monitor/anomaly_detect.py
+msprobe/pytorch/monitor/csv2tb.py
 msprobe/pytorch/monitor/features.py
 msprobe/pytorch/monitor/module_hook.py
 msprobe/pytorch/monitor/module_metric.py
@@ -340,6 +394,7 @@ msprobe/visualization/compare/graph_comparator.py
 msprobe/visualization/compare/mode_adapter.py
 msprobe/visualization/graph/__init__.py
 msprobe/visualization/graph/base_node.py
+msprobe/visualization/graph/distributed_analyzer.py
 msprobe/visualization/graph/graph.py
 msprobe/visualization/graph/node_colors.py
 msprobe/visualization/graph/node_op.py

mindstudio_probe-1.2.1/msprobe/CMakeLists.txt

@@ -0,0 +1,5 @@
+add_subdirectory(ccsrc)
+
+if (DEFINED BUILD_TEST_CASE AND "${BUILD_TEST_CASE}" STREQUAL "True")
+add_subdirectory(test)
+endif()

{mindstudio_probe-1.1.1 → mindstudio_probe-1.2.1}/msprobe/README.md

@@ -51,15 +51,21 @@ export MSPROBE_LOG_LEVEL={x}
 
 **1. Pytorch 框架下，工具暂不支持 Fully Sharded Data Parallel(FSDP)。**
 
+**2. 工具读写的所有路径，如config_path、dump_path等，只允许包含大小写字母、数字、下划线、斜杠、点和短横线。**
+
 ## ⚙️ [安装](./docs/01.installation.md)
 
+## 🌟 新版本特性
+
+请参见[特性变更说明](./docs/01.installation.md#特性变更说明)。
+
 ## 🛠️ config.json [介绍](./docs/02.config_introduction.md) 和 [示例](./docs/03.config_examples.md)
 
 ## 🧰 主要功能
 
 ### 0 用前必看
 
-使用工具前，建议先浏览[**工具功能模块简介、适用场景和当前版本局限性**](./docs/23.tool_function_introduction.md)，了解功能特性。
+使用工具前，建议先浏览[**工具功能模块简介、适用场景和当前版本局限性**](./docs/25.tool_function_introduction.md)，了解功能特性。
 
 ### 1 数据采集
 
@@ -131,29 +137,18 @@ MindSpore 动态图场景的[离线预检](./docs/09.accuracy_checker_MindSpore.
 
 [MindSpore 场景的分级可视化构图比对](./docs/22.visualization_MindSpore.md)
 
-## 🌟 新版本特性
 
-若查看历史版本特性，请参见[安装](./docs/01.installation.md)。
+### 11 单算子API自动生成脚本
+
+该功能将msprobe工具dump的精度数据进行解析，自动生成单API脚本，用于复现整网中出现的算子问题，降低用户复现问题的成本，供开发分析算子问题。
 
-【数据采集】
-- 支持 config.json 中的 step 传入范围；
-- 优化了指定 step 的机制，指定 step 结束后工具不再采集数据，但训练会继续运行。工具结束运行后，日志提示信息如下：
-    ```bash
-    ****************************************
-    *      msprobe ends successfully.      *
-    ****************************************
-    ```
-    注：在多卡场景，每张卡进程训练到指定 step 之后都会打印一次上述信息。
+[PyTorch 单算子API自动生成脚本](./docs/23.generate_operator_PyTorch.md)
 
-【精度预检】
-- 在 PyTorch 场景，支持部分 NPU 融合算子预检。
+### 12 数码关联
 
-【精度比对】
-- 解决了使用 MindSpore 需要安装 PyTorch 的问题。
+该功能只支持 MindSpore 静态图场景，用于将IR图与dump数据进行关联，获取dump数据和代码调用栈的关联关系。
 
-【无标杆比对】
-- 补充在 PyTorch 场景的性能基线报告；
-- 支持 MindSpore 场景的 change_value 扰动模式。
+[MindSpore 场景的数码关联](./docs/24.code_mapping_Mindspore.md)
 
 ## 📑 补充材料
 

{mindstudio_probe-1.1.1 → mindstudio_probe-1.2.1}/msprobe/config.json

@@ -5,6 +5,7 @@
     "step": [],  
     "level": "L1", 
     "enable_dataloader": false,
+    "async_dump": false,
     "tensor": {
         "scope": [], 
         "list":[], 

{mindstudio_probe-1.1.1 → mindstudio_probe-1.2.1}/msprobe/core/common/const.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd.
 # All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0  (the "License");
@@ -29,6 +29,7 @@ class Const:
     SEP = "."
     REGEX_PREFIX_MAX_LENGTH = 20
     REGEX_PREFIX_PATTERN = r"^[a-zA-Z0-9_-]+$"
+    REGEX_FORWARD_BACKWARD = r'\.(forward|backward)\.'
     FILE_PATTERN = r'^[a-zA-Z0-9_./-]+$'
     STRING_BLACKLIST = r"^[＋－＝％＠\+\-=%@]|;[＋－＝％＠\+\-=%@]"
     COMMA = ","
@@ -65,6 +66,7 @@ class Const:
     ONLINE_DUMP_MODE = [ALL, LIST, AUTO, OFF]
     SUMMARY = "summary"
     MD5 = "md5"
+    VALUE = "value"
     SUMMARY_MODE = [ALL, SUMMARY, MD5]
 
     WRITE_FLAGS = os.O_WRONLY | os.O_CREAT
@@ -73,6 +75,7 @@ class Const:
 
     PKL_SUFFIX = ".pkl"
     NUMPY_SUFFIX = ".npy"
+    NUMPY_PATTERN = "*.npy"
     PT_SUFFIX = ".pt"
     ONE_GB = 1073741824  # 1 * 1024 * 1024 * 1024
     TEN_GB = 10737418240  # 10 * 1024 * 1024 * 1024
@@ -87,6 +90,8 @@ class Const:
     INPUT_KWARGS = 'input_kwargs'
     GRAD_INPUT = 'grad_input'
     GRAD_OUTPUT = 'grad_output'
+    PARAMS = 'parameters'
+    PARAMS_GRAD = 'parameters_grad'
     START = "start"
     STOP = "stop"
     ENV_ENABLE = "1"
@@ -112,6 +117,7 @@ class Const:
     DATA = "data"
     PT_FRAMEWORK = "pytorch"
     MS_FRAMEWORK = "mindspore"
+    MT_FRAMEWORK = "mindtorch"
     UNKNOWN_FRAMEWORK = "unknown"
     DIRECTORY_LENGTH = 4096
     FILE_NAME_LENGTH = 255
@@ -122,9 +128,12 @@ class Const:
     NPU_LOWERCASE = 'npu'
     CPU_LOWERCASE = 'cpu'
     CUDA_LOWERCASE = 'cuda'
+    DEVICE = 'device'
     DISTRIBUTED = 'Distributed'
-    DUMP_PREFIX = ["Distributed", "Functional", "Torch", "Tensor", "Mint", "MintFunctional", "Primitive", 
+    DUMP_PREFIX = ["Distributed", "Functional", "Torch", "Tensor", "Mint", "MintFunctional", "Primitive",
                    "Aten", "VF", "NPU", "Jit"]
+    MODULE_PREFIX = ["Module", "Cell"]
+    FORWARD_NAME_SUFFIX = ".forward"
 
     # struct json param
     ORIGIN_DATA = "origin_data"
@@ -145,10 +154,13 @@ class Const:
     SCOPE_ID_INDEX = -1
     SCOPE_DIRECTION_INDEX = -2
     TYPE_NAME_INDEX = -3
+    PARAMS_GRAD_TYPE_NAME_INDEX = -2
     LAYER_NAME_INDEX = -4
+    PARAMS_GRAD_NAME_INDEX = -3
     API_TYPE_INDEX = 0
     LEFT_MOVE_INDEX = -1
     RIGHT_MOVE_INDEX = 1
+    LAST_INDEX = -1
 
     TOP_LAYER = "TopLayer"
     CELL = "Cell"
@@ -162,12 +174,16 @@ class Const:
 
     CONVERT = {
         "int32_to_int64": ["torch.int32", "torch.int64"],
+        "int64_to_fp32": ["torch.int64", "torch.float32"]
     }
 
     CONVERT_API = {
-        "int32_to_int64": ["cross_entropy"]
+        "int32_to_int64": ["cross_entropy"],
+        "int64_to_fp32": ["histc"]
     }
 
+    FA_SPECIAL_SPARSE_MODE = [2, 3, 4]
+
     FILL_CHAR_NUMS = 50
     TOOL_ENDS_SUCCESSFULLY = f"{TOOL_NAME} ends successfully."
     WITHOUT_CALL_STACK = "The call stack retrieval failed."
@@ -179,6 +195,8 @@ class Const:
     STEP_RANK_MAXIMUM_VALUE = int(1e6)
 
     # data type const
+    TORCH_INT_DTYPE = ["torch.int8", "torch.int32", "torch.int64"]
+    TORCH_FLOAT_DTYPE = ["torch.bfloat16", "torch.float16", "torch.float32", "torch.float64"]
     FLOAT16 = "Float16"
     FLOAT32 = "Float32"
     BFLOAT16 = "BFloat16"
@@ -193,6 +211,23 @@ class Const:
     MEAN = 'Mean'
     NORM = 'Norm'
 
+    CODE_STACK = 'Code Stack'
+    OP_NAME = 'Op Name'
+    SCOPE_NAME = 'Scope Name'
+    CODE_STACKS = 'Code Stacks'
+    FILE_PATH = 'File Path'
+    NEW_LINE = '\n'
+    CSV_NEWLINE_SEPARATOR = ',\n'
+    # 分隔符常量
+    SCOPE_SEPARATOR = "/"
+    REPLACEMENT_CHARACTER = "_"
+
+    OPTIMIZER = "optimizer"
+    CLIP_GRAD = "clip_grad"
+    END_PREFIX = "end_"
+
+    TENSOR_STAT_LEN = 2
+
 
 class CompareConst:
     """
@@ -239,13 +274,58 @@ class CompareConst:
     INPUT_STRUCT = "input_struct"
     KWARGS_STRUCT = "kwargs_struct"
     OUTPUT_STRUCT = "output_struct"
+    PARAMS_STRUCT = "params_struct"
+    PARAMS_GRAD_STRUCT = "params_grad_struct"
     SUMMARY = "summary"
+    COMPARE_RESULT = "compare_result"
+    COMPARE_MESSAGE = "compare_message"
     MAX_EXCEL_LENGTH = 1048576
     YES = "Yes"
     NO = "No"
     STATISTICS_INDICATOR_NUM = 4
     EPSILON = 1e-10
     COMPARE_ENDS_SUCCESSFULLY = "msprobe compare ends successfully."
+    DEFAULT_RATIO_VALUE = 10000
+    THOUSANDTH_PASS_VALUE = 0.999
+    ZERO_SHAPE = '(0,)'
+
+    BENCHMARK_COMPARE_ALGORITHM_NAME = "标杆比对法"
+    ULP_COMPARE_ALGORITHM_NAME = "ULP误差比对法"
+    BINARY_CONSISTENCY_ALGORITHM_NAME = "二进制一致法"
+    ABSOLUTE_THRESHOLD_ALGORITHM_NAME = "绝对阈值法"
+    THOUSANDTH_STANDARD_ALGORITHM_NAME = "双千指标法"
+    ACCUMULATIVE_ERROR_COMPARE_ALGORITHM_NAME = "累积误差比对法"
+
+    ABSOLUTE_THRESHOLD = 'absolute_threshold'
+    BINARY_CONSISTENCY = 'binary_consistency'
+    ULP_COMPARE = 'ulp_compare'
+    THOUSANDTH_STANDARD = 'thousandth_threshold'
+    BENCHMARK = 'benchmark'
+    ACCUMULATIVE_ERROR_COMPARE = 'accumulative_error_compare'
+
+    SMALL_VALUE_ERR_RATIO = "small_value_err_ratio"
+    RMSE_RATIO = "rmse_ratio"
+    MAX_REL_ERR_RATIO = "max_rel_err_ratio"
+    MEAN_REL_ERR_RATIO = "mean_rel_err_ratio"
+    EB_RATIO = "eb_ratio"
+
+    SMALL_VALUE = "small_value"
+    RMSE = "rmse"
+    MAX_REL_ERR = "max_rel_err"
+    MEAN_REL_ERR = "mean_rel_err"
+    EB = "eb"
+
+    SMALL_VALUE_ERR_STATUS = "small_value_err_status"
+    RMSE_STATUS = "rmse_status"
+    MAX_REL_ERR_STATUS = "max_rel_err_status"
+    MEAN_REL_ERR_STATUS = "mean_rel_err_status"
+    EB_STATUS = "eb_status"
+
+    MEAN_ULP_ERR = "mean_ulp_err"
+    ULP_ERR_PROPORTION = "ulp_err_proportion"
+    ULP_ERR_PROPORTION_RATIO = "ulp_err_proportion_ratio"
+
+    ULP_ERR_STATUS = "ulp_err_status"
 
     COMPARE_RESULT_HEADER = [
         NPU_NAME, BENCH_NAME, NPU_DTYPE, BENCH_DTYPE, NPU_SHAPE, BENCH_SHAPE, COSINE, MAX_ABS_ERR, MAX_RELATIVE_ERR,
@@ -263,12 +343,57 @@ class CompareConst:
         NPU_NAME, BENCH_NAME, NPU_DTYPE, BENCH_DTYPE, NPU_SHAPE, BENCH_SHAPE, NPU_MD5, BENCH_MD5, RESULT
     ]
 
+    COMPARE_RESULT_HEADER_STACK = COMPARE_RESULT_HEADER + [STACK]
+
+    SUMMARY_COMPARE_RESULT_HEADER_STACK = SUMMARY_COMPARE_RESULT_HEADER + [STACK]
+
+    MD5_COMPARE_RESULT_HEADER_STACK = MD5_COMPARE_RESULT_HEADER + [STACK]
+
     HEAD_OF_COMPARE_MODE = {
         Const.ALL: COMPARE_RESULT_HEADER,
         Const.SUMMARY: SUMMARY_COMPARE_RESULT_HEADER,
         Const.MD5: MD5_COMPARE_RESULT_HEADER
     }
 
+    ALL_COMPARE_INDEX = [COSINE, MAX_ABS_ERR, MAX_RELATIVE_ERR, ONE_THOUSANDTH_ERR_RATIO, FIVE_THOUSANDTHS_ERR_RATIO]
+    SUMMARY_COMPARE_INDEX = [MAX_DIFF, MIN_DIFF, MEAN_DIFF, NORM_DIFF,
+                             MAX_RELATIVE_ERR, MIN_RELATIVE_ERR, MEAN_RELATIVE_ERR, NORM_RELATIVE_ERR]
+
+    # dtype match
+    MS_TYPE = [
+        [Const.FLOAT16, Const.FLOAT32], [Const.FLOAT32, Const.FLOAT16],
+        [Const.FLOAT16, Const.BFLOAT16], [Const.BFLOAT16, Const.FLOAT16]
+    ]
+    TORCH_TYPE = [
+        [Const.TORCH_FLOAT16, Const.TORCH_FLOAT32], [Const.TORCH_FLOAT32, Const.TORCH_FLOAT16],
+        [Const.TORCH_FLOAT16, Const.TORCH_BFLOAT16], [Const.TORCH_BFLOAT16, Const.TORCH_FLOAT16]
+    ]
+
+    # read_op
+    IO_NAME_MAPPING = {
+        Const.INPUT_ARGS: '.input',
+        Const.INPUT_KWARGS: '.input',
+        Const.INPUT: '.input',
+        Const.OUTPUT: '.output',
+        Const.PARAMS: '.parameters'
+    }
+
+    # state to struct mapping
+    STATE_TO_STRUCT_MAPPING = {
+        Const.INPUT: INPUT_STRUCT,
+        Const.KWARGS: INPUT_STRUCT,
+        Const.OUTPUT: OUTPUT_STRUCT,
+        Const.PARAMS: PARAMS_STRUCT,
+        Const.PARAMS_GRAD: PARAMS_GRAD_STRUCT
+    }
+
+    STRUCT_COMPARE_KEY = [
+        INPUT_STRUCT,
+        OUTPUT_STRUCT,
+        PARAMS_STRUCT,
+        PARAMS_GRAD_STRUCT
+    ]
+
     # compare standard
     HUNDRED_RATIO_THRESHOLD = 0.01
     THOUSAND_RATIO_THRESHOLD = 0.001
@@ -350,6 +475,8 @@ class CompareConst:
     INPUT_PATTERN = Const.SEP + Const.INPUT + Const.SEP
     KWARGS_PATTERN = Const.SEP + Const.KWARGS + Const.SEP
     OUTPUT_PATTERN = Const.SEP + Const.OUTPUT + Const.SEP
+    PARAMS_PATTERN = Const.SEP + Const.PARAMS + Const.SEP
+    PARAMS_GRAD_PATTERN = Const.SEP + Const.PARAMS_GRAD + Const.SEP
     COMPARE_KEY = 'compare_key'
     COMPARE_SHAPE = 'compare_shape'
     INTERNAL_API_MAPPING_FILE = 'ms_to_pt_api.yaml'
@@ -372,13 +499,17 @@ class FileCheckConst:
     JSON_SUFFIX = ".json"
     PT_SUFFIX = ".pt"
     CSV_SUFFIX = ".csv"
+    XLSX_SUFFIX = ".xlsx"
     YAML_SUFFIX = ".yaml"
+    IR_SUFFIX = ".ir"
     MAX_PKL_SIZE = 1073741824  # 1 * 1024 * 1024 * 1024
     MAX_NUMPY_SIZE = 10737418240  # 10 * 1024 * 1024 * 1024
     MAX_JSON_SIZE = 1073741824  # 1 * 1024 * 1024 * 1024
     MAX_PT_SIZE = 10737418240  # 10 * 1024 * 1024 * 1024
     MAX_CSV_SIZE = 1073741824  # 1 * 1024 * 1024 * 1024
+    MAX_XLSX_SIZE = 1073741824  # 1 * 1024 * 1024 * 1024
     MAX_YAML_SIZE = 1073741824  # 1 * 1024 * 1024 * 1024
+    MAX_IR_SIZE = 1073741824  # 1 * 1024 * 1024 * 1024
     COMMOM_FILE_SIZE = 1048576  # 1 * 1024 * 1024
     DIR = "dir"
     FILE = "file"
@@ -390,7 +521,9 @@ class FileCheckConst:
         JSON_SUFFIX: MAX_JSON_SIZE,
         PT_SUFFIX: MAX_PT_SIZE,
         CSV_SUFFIX: MAX_CSV_SIZE,
-        YAML_SUFFIX: MAX_YAML_SIZE
+        XLSX_SUFFIX: MAX_XLSX_SIZE,
+        YAML_SUFFIX: MAX_YAML_SIZE,
+        IR_SUFFIX: MAX_IR_SIZE
     }
     CSV_BLACK_LIST = r'^[＋－＝％＠\+\-=%@]|;[＋－＝％＠\+\-=%@]'
 
@@ -437,6 +570,11 @@ class MsCompareConst:
 
     EPSILON = 1e-8
 
+    class ProcessStatus:
+        SUCCESS = "success"
+        API_NOT_FOUND = "api_not_found"
+        EXCEPTION_SKIP = "exception_skip"
+
 
 class MsgConst:
     """
@@ -474,15 +612,20 @@ class MonitorConst:
     """
     Class for monitor const
     """
-    OP_LIST = ["min", "max", "norm", "zeros", "nans", "id", "mean"]
+    OP_LIST = ["norm", "min", "max", "zeros", "nans", "id", "mean"]
     MONITOR_OUTPUT_DIR = "MONITOR_OUTPUT_DIR"
     DEFAULT_MONITOR_OUTPUT_DIR = "./monitor_output"
     DATABASE = "database"
     EMAIL = "email"
     OPT_TY = ['Megatron_DistributedOptimizer', 'Megatron_Float16OptimizerWithFloat16Params']
-    DEEPSPEED_OPT_TY = ("DeepSpeedZeroOptimizer_Stage0", "DeepSpeedZeroOptimizer_Stage1_or_2", "DeepSpeedZeroOptimizer_Stage3")
+    DEEPSPEED_OPT_TY = (
+        "DeepSpeedZeroOptimizer_Stage0",
+        "DeepSpeedZeroOptimizer_Stage1_or_2",
+        "DeepSpeedZeroOptimizer_Stage3"
+    )
     RULE_NAME = ['AnomalyTurbulence']
 
+    SLICE_SIZE = 20480
     DOT = "."
     VPP_SEP = ":"
     ACTV_IN = "input"
@@ -491,12 +634,18 @@ class MonitorConst:
     ACTVGRAD_OUT = "output_grad"
     POST_GRAD = "post_grad"
     PRE_GRAD = "pre_grad"
+    ACC_GRAD = "acc_grad"
     PREFIX_POST = "post"
     PREFIX_PRE = "pre"
+    OUTPUT_DIR_PATTERN = r"([\w-]{0,20})-rank(\d{1,5})-"
 
+    EXP_AVG = "exp_avg"
+    EFXP_AVG_SQ = "efxp_avg_sq"
 
     ANOMALY_JSON = "anomaly.json"
     ANALYSE_JSON = "anomaly_analyse.json"
     TENSORBOARD = "tensorboard"
     CSV = "csv"
     API = "api"
+    OPS_START_INDEX = 3
+    HEADER_NAME_INDEX = 1

{mindstudio_probe-1.1.1 → mindstudio_probe-1.2.1}/msprobe/core/common/exceptions.py

@@ -27,11 +27,13 @@ class MsprobeException(CodedException):
     INVALID_PARAM_ERROR = 0
     OVERFLOW_NUMS_ERROR = 1
     RECURSION_LIMIT_ERROR = 2
+    INTERFACE_USAGE_ERROR = 3
 
     err_strs = {
         INVALID_PARAM_ERROR: "[msprobe] 无效参数：",
         OVERFLOW_NUMS_ERROR: "[msprobe] 超过预设溢出次数 当前溢出次数：",
-        RECURSION_LIMIT_ERROR: "[msprobe] 递归调用超过限制："
+        RECURSION_LIMIT_ERROR: "[msprobe] 递归调用超过限制：",
+        INTERFACE_USAGE_ERROR: "[msprobe] Invalid interface usage: "
     }