mindstudio-probe 8.1.1__tar.gz → 8.2.0__tar.gz

{mindstudio_probe-8.1.1 → mindstudio_probe-8.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mindstudio-probe
-Version: 8.1.1
+Version: 8.2.0
 Summary: Ascend Probe Utils
 Home-page: https://gitee.com/ascend/mstt/tree/master/debug/accuracy_tools/msprobe
 Author: Ascend Team

{mindstudio_probe-8.1.1 → mindstudio_probe-8.2.0}/mindstudio_probe.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mindstudio-probe
-Version: 8.1.1
+Version: 8.2.0
 Summary: Ascend Probe Utils
 Home-page: https://gitee.com/ascend/mstt/tree/master/debug/accuracy_tools/msprobe
 Author: Ascend Team

{mindstudio_probe-8.1.1 → mindstudio_probe-8.2.0}/mindstudio_probe.egg-info/SOURCES.txt

@@ -21,6 +21,7 @@ msprobe/core/advisor/advisor.py
 msprobe/core/advisor/advisor_const.py
 msprobe/core/advisor/advisor_result.py
 msprobe/core/common/const.py
+msprobe/core/common/db_manager.py
 msprobe/core/common/decorator.py
 msprobe/core/common/exceptions.py
 msprobe/core/common/file_utils.py
@@ -29,6 +30,7 @@ msprobe/core/common/global_lock.py
 msprobe/core/common/inplace_op_checker.py
 msprobe/core/common/inplace_ops.yaml
 msprobe/core/common/log.py
+msprobe/core/common/parallel_state.py
 msprobe/core/common/runtime.py
 msprobe/core/common/utils.py
 msprobe/core/compare/acc_compare.py
@@ -40,6 +42,14 @@ msprobe/core/compare/ms_to_pt_api.yaml
 msprobe/core/compare/multiprocessing_compute.py
 msprobe/core/compare/npy_compare.py
 msprobe/core/compare/utils.py
+msprobe/core/compare/diff_analyze/__init__.py
+msprobe/core/compare/diff_analyze/diff_analyze_threshold.yaml
+msprobe/core/compare/diff_analyze/first_diff_analyze.py
+msprobe/core/compare/find_first/__init__.py
+msprobe/core/compare/find_first/analyzer.py
+msprobe/core/compare/find_first/data_processor.py
+msprobe/core/compare/find_first/graph.py
+msprobe/core/compare/find_first/utils.py
 msprobe/core/compare/layer_mapping/__init__.py
 msprobe/core/compare/layer_mapping/data_scope_parser.py
 msprobe/core/compare/layer_mapping/layer_mapping.py
@@ -83,6 +93,9 @@ msprobe/core/grad_probe/utils.py
 msprobe/core/kernel_dump/kernel_config.py
 msprobe/core/monitor/__init__.py
 msprobe/core/monitor/anomaly_processor.py
+msprobe/core/monitor/csv2db.py
+msprobe/core/monitor/db_utils.py
+msprobe/core/monitor/utils.py
 msprobe/core/overflow_check/abnormal_scene.py
 msprobe/core/overflow_check/api_info.py
 msprobe/core/overflow_check/checker.py
@@ -100,7 +113,6 @@ msprobe/docs/04.kernel_dump_PyTorch.md
 msprobe/docs/05.data_dump_PyTorch.md
 msprobe/docs/06.data_dump_MindSpore.md
 msprobe/docs/07.accuracy_checker_PyTorch.md
-msprobe/docs/08.accuracy_checker_online_PyTorch.md
 msprobe/docs/09.accuracy_checker_MindSpore.md
 msprobe/docs/10.accuracy_compare_PyTorch.md
 msprobe/docs/11.accuracy_compare_MindSpore.md
@@ -129,6 +141,7 @@ msprobe/docs/32.ckpt_compare.md
 msprobe/docs/33.generate_operator_MindSpore.md
 msprobe/docs/34.RL_collect.md
 msprobe/docs/35.nan_analyze.md
+msprobe/docs/36.calculation_result_change.md
 msprobe/docs/FAQ.md
 msprobe/docs/S02.report_free_benchmarking_validation_performance_baseline.md
 msprobe/docs/accuracy_checker_MindSpore/accuracy_checker_MindSpore_baseline.md
@@ -190,12 +203,26 @@ msprobe/docs/visualization/ParallelMLP.png
 msprobe/docs/visualization/layer_mapping_example.md
 msprobe/docs/visualization/mapping.png
 msprobe/docs/visualization/mapping1.png
+msprobe/docs/visualization/mindspeed_llamafactory_mapping.md
 msprobe/docs/visualization/module_name.png
 msprobe/docs/visualization/module_name1.png
 msprobe/docs/visualization/no_mapping.png
 msprobe/docs/visualization/no_mapping1.png
 msprobe/docs/visualization/no_mapping_analyze.png
 msprobe/docs/visualization/top_layer.png
+msprobe/docs/visualization/mindspeed_llamafactoary_img/1.png
+msprobe/docs/visualization/mindspeed_llamafactoary_img/2.png
+msprobe/docs/visualization/mindspeed_llamafactoary_img/3.png
+msprobe/docs/visualization/mindspeed_llamafactoary_img/4.png
+msprobe/docs/visualization/mindspeed_llamafactoary_img/5.png
+msprobe/docs/visualization/mindspeed_llamafactoary_img/6.png
+msprobe/docs/visualization/mindspeed_llamafactoary_img/7.png
+msprobe/docs/visualization/mindspeed_llamafactoary_img/llamafactory-qwen25vl.txt
+msprobe/docs/visualization/mindspeed_llamafactoary_img/llamafactory1.png
+msprobe/docs/visualization/mindspeed_llamafactoary_img/llamafactory2.png
+msprobe/docs/visualization/mindspeed_llamafactoary_img/mindspeed-mm-qwen25vl.txt
+msprobe/docs/visualization/mindspeed_llamafactoary_img/mindspeed1.png
+msprobe/docs/visualization/mindspeed_llamafactoary_img/mindspeed2.png
 msprobe/mindspore/__init__.py
 msprobe/mindspore/cell_processor.py
 msprobe/mindspore/mindspore_service.py
@@ -254,6 +281,9 @@ msprobe/mindspore/dump/hook_cell/primitive_hooks.py
 msprobe/mindspore/dump/hook_cell/support_wrap_ops.yaml
 msprobe/mindspore/dym_loader/hook_dynamic_loader.cpp
 msprobe/mindspore/dym_loader/hook_dynamic_loader.h
+msprobe/mindspore/exception_dump/__init__.py
+msprobe/mindspore/exception_dump/exception_dump_tool_factory.py
+msprobe/mindspore/exception_dump/kernel_graph_exception_dump.py
 msprobe/mindspore/free_benchmark/__init__.py
 msprobe/mindspore/free_benchmark/api_pynative_self_check.py
 msprobe/mindspore/free_benchmark/self_check_tool_factory.py
@@ -301,7 +331,6 @@ msprobe/nan_analyze/analyzer.py
 msprobe/nan_analyze/graph.py
 msprobe/nan_analyze/utils.py
 msprobe/pytorch/__init__.py
-msprobe/pytorch/attl_manager.py
 msprobe/pytorch/function_factory.py
 msprobe/pytorch/pt_config.py
 msprobe/pytorch/pytorch_service.py
@@ -341,14 +370,6 @@ msprobe/pytorch/api_accuracy_checker/run_ut/run_overflow_check.py
 msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py
 msprobe/pytorch/api_accuracy_checker/run_ut/run_ut_utils.py
 msprobe/pytorch/api_accuracy_checker/run_ut/torch_ut_setting.json
-msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/__init__.py
-msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/attl.py
-msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/client.py
-msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/device_dispatch.py
-msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/dump_dispatch.py
-msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/server.py
-msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/torch_ops_config.yaml
-msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/utils.py
 msprobe/pytorch/bench_functions/__init__.py
 msprobe/pytorch/bench_functions/apply_adam.py
 msprobe/pytorch/bench_functions/apply_adam_w.py
@@ -375,6 +396,7 @@ msprobe/pytorch/compare/distributed_compare.py
 msprobe/pytorch/compare/mapping.yaml
 msprobe/pytorch/compare/match.py
 msprobe/pytorch/compare/pt_compare.py
+msprobe/pytorch/compare/pt_diff_analyze.py
 msprobe/pytorch/compare/utils.py
 msprobe/pytorch/debugger/__init__.py
 msprobe/pytorch/debugger/debugger_config.py
@@ -416,9 +438,9 @@ msprobe/pytorch/grad_probe/grad_stat_csv.py
 msprobe/pytorch/hook_module/__init__.py
 msprobe/pytorch/hook_module/api_register.py
 msprobe/pytorch/hook_module/hook_module.py
-msprobe/pytorch/hook_module/jit_script_wrapper.py
 msprobe/pytorch/hook_module/pt_hook_manager.py
 msprobe/pytorch/hook_module/register_optimizer_hook.py
+msprobe/pytorch/hook_module/script_wrapper.py
 msprobe/pytorch/hook_module/support_wrap_ops.yaml
 msprobe/pytorch/hook_module/utils.py
 msprobe/pytorch/hook_module/wrap_aten.py
@@ -454,10 +476,12 @@ msprobe/pytorch/parse_tool/lib/parse_tool.py
 msprobe/pytorch/parse_tool/lib/utils.py
 msprobe/pytorch/parse_tool/lib/visualization.py
 msprobe/visualization/__init__.py
+msprobe/visualization/db_utils.py
 msprobe/visualization/graph_service.py
 msprobe/visualization/utils.py
 msprobe/visualization/builder/__init__.py
 msprobe/visualization/builder/graph_builder.py
+msprobe/visualization/builder/graph_merger.py
 msprobe/visualization/builder/msprobe_adapter.py
 msprobe/visualization/compare/__init__.py
 msprobe/visualization/compare/graph_comparator.py

{mindstudio_probe-8.1.1 → mindstudio_probe-8.2.0}/msprobe/README.md

@@ -84,7 +84,7 @@ msprobe 通过在训练脚本中添加 PrecisionDebugger 接口的方式对 API
 
 精度预检旨在昇腾 NPU 上扫描训练模型中的所有 API 进行 API 复现，给出精度情况的诊断和分析。对应 config.json 中的 "run_ut" task。
 
-PyTorch 场景的[离线预检](./docs/07.accuracy_checker_PyTorch.md)和[在线预检](./docs/08.accuracy_checker_online_PyTorch.md)
+PyTorch 场景的[离线预检](./docs/07.accuracy_checker_PyTorch.md)
 
 MindSpore 动态图场景的[离线预检](./docs/09.accuracy_checker_MindSpore.md)
 
@@ -165,7 +165,7 @@ MindSpore 动态图场景的[离线预检](./docs/09.accuracy_checker_MindSpore.
 
 训练前或精度比对前，对比两个环境下可能影响训练精度的配置差异。
 
-[PyTorch 训练前配置检查](./docs/31.config_check.md)
+[训练前配置检查](./docs/31.config_check.md)
 
 训练过程中或结束后，比较两个不同的checkpoint，评估模型相似度。
 

{mindstudio_probe-8.1.1 → mindstudio_probe-8.2.0}/msprobe/core/common/const.py

@@ -24,6 +24,8 @@ class Const:
     Class for const
     """
     TOOL_NAME = "msprobe"
+    MD5_INDEX = "md5_index"
+    MD5 = "md5"
 
     ipv4_pattern = "([1-9]?\d|1\d{2}|2[0-4]\d|25[0-5])(\.([1-9]?\d|1\d{2}|2[0-4]\d|25[0-5])){3}$"
     SEP = "."
@@ -52,9 +54,12 @@ class Const:
     SIX_SEGMENT = 6
     SEVEN_SEGMENT = 7
 
-    MAX_DEPTH = 10
+    MAX_DEPTH = 400
     CPU_QUARTER = 4
-    DUMP_MAX_DEPTH = 50
+    DUMP_MAX_DEPTH = 400
+
+    EXTERN_INPUT_LIST_MAX_LEN = 100
+    MAX_PROCESS_NUM = 128
 
     # dump mode
     ALL = "all"
@@ -69,6 +74,7 @@ class Const:
     ONLINE_DUMP_MODE = [ALL, LIST, AUTO, OFF]
     SUMMARY = "summary"
     MD5 = "md5"
+    HASH = "hash"
     VALUE = "value"
     SUMMARY_MODE = ["statistics", "md5"]
 
@@ -110,9 +116,13 @@ class Const:
     RUN_UT = "run_ut"
     GRAD_PROBE = "grad_probe"
     STRUCTURE = "structure"
-    TASK_LIST = [TENSOR, STATISTICS, OVERFLOW_CHECK, FREE_BENCHMARK, RUN_UT, GRAD_PROBE, STRUCTURE]
+    EXCEPTION_DUMP = "exception_dump"
+    DUMP_PRECISION_HIGH = "high"
+    DUMP_PRECISION_LOW = "low"
+    TASK_LIST = [TENSOR, STATISTICS, OVERFLOW_CHECK, FREE_BENCHMARK, RUN_UT, GRAD_PROBE, STRUCTURE, EXCEPTION_DUMP]
     DUMP_DATA_COLLECTION_LIST = [STATISTICS, TENSOR, STRUCTURE]
     DUMP_DATA_MODE_LIST = [ALL, INPUT, OUTPUT, FORWARD, BACKWARD]
+    DUMP_PRECISION_LIST = [DUMP_PRECISION_LOW, DUMP_PRECISION_HIGH]
     LEVEL_L0 = "L0"
     LEVEL_L1 = "L1"
     LEVEL_L2 = "L2"
@@ -234,7 +244,11 @@ class Const:
     MEAN = 'Mean'
     NORM = 'Norm'
     DATA_NAME = 'data_name'
+    STATE = 'state'
+    REQ_GRAD = 'requires_grad'
+    API_ORIGIN_NAME = 'api_origin_name'
     TENSOR_STAT_INDEX = 'tensor_stat_index'
+    SUMMARY_METRICS_LIST = [MAX, MIN, MEAN, NORM]
 
     CODE_STACK = 'Code Stack'
     OP_NAME = 'Op Name'
@@ -257,8 +271,14 @@ class Const:
 
     TENSOR_STAT_LEN = 2
 
+    TENSOR_TYPE = "torch.Tensor"
+    DTENSOR_TYPE = "torch.distributed.tensor.DTensor"
+    FAKE_TENSOR_TYPE = "torch._subclasses.fake_tensor.FakeTensor"
+
     SUPPORT_API_FILE_NAME = "support_wrap_ops.yaml"
 
+    API_ATTR_LIST = ["__name__", "default"]
+
     PT_API_TYPE_FUNCTIONAL = "functional"
     PT_API_TYPE_TENSOR = "tensor"
     PT_API_TYPE_TORCH = "torch"
@@ -352,22 +372,22 @@ class Const:
     }
 
     def _fused_adamw_(
-        self,
-        grads,
-        exp_avgs,
-        exp_avg_sqs,
-        max_exp_avg_sqs,
-        state_steps,
-        *,
-        lr,
-        beta1,
-        beta2,
-        weight_decay,
-        eps,
-        amsgrad,
-        maximize,
-        grad_scale=None,
-        found_inf=None
+            self,
+            grads,
+            exp_avgs,
+            exp_avg_sqs,
+            max_exp_avg_sqs,
+            state_steps,
+            *,
+            lr,
+            beta1,
+            beta2,
+            weight_decay,
+            eps,
+            amsgrad,
+            maximize,
+            grad_scale=None,
+            found_inf=None
     ):
         pass
 
@@ -379,6 +399,11 @@ class Const:
     MATCH_MODE_NAME = "pure name"
     MATCH_MODE_MAPPING = "mapping"
     MATCH_MODE_SIMILARITY = "similarity"
+    CONFIG_CHECK_PASS = "pass"
+    CONFIG_CHECK_WARNING = "warning"
+    CONFIG_CHECK_ERROR = "error"
+
+    MIX_DUMP_NAMES = {'graph', 'pynative'}
 
 
 class CompareConst:
@@ -394,6 +419,8 @@ class CompareConst:
     BENCH_DTYPE = "Bench Dtype"
     NPU_SHAPE = "NPU Tensor Shape"
     BENCH_SHAPE = "Bench Tensor Shape"
+    NPU_CSV_FILE = "NPU CSV File"
+    BENCH_CSV_FILE = "Bench CSV File"
     NPU_MAX = "NPU max"
     NPU_MIN = "NPU min"
     NPU_MEAN = "NPU mean"
@@ -413,6 +440,9 @@ class CompareConst:
     MIN_RELATIVE_ERR = "MinRelativeErr"
     MEAN_RELATIVE_ERR = "MeanRelativeErr"
     NORM_RELATIVE_ERR = "NormRelativeErr"
+    REQ_GRAD_CONSIST = "Requires_grad Consistent"
+    NPU_REQ_GRAD = "NPU Requires_grad"
+    BENCH_REQ_GRAD = "Bench Requires_grad"
     ACCURACY = "Accuracy Reached or Not"
     STACK = "NPU_Stack_Info"
     DATA_NAME = "Data_name"
@@ -434,7 +464,7 @@ class CompareConst:
     SUMMARY = "summary"
     COMPARE_RESULT = "compare_result"
     COMPARE_MESSAGE = "compare_message"
-    MAX_EXCEL_LENGTH = 1048576
+    MAX_EXCEL_LENGTH = 1048500
     YES = "Yes"
     NO = "No"
     STATISTICS_INDICATOR_NUM = 4
@@ -482,21 +512,21 @@ class CompareConst:
 
     ULP_ERR_STATUS = "ulp_err_status"
 
-    COMPARE_RESULT_HEADER = [
-        NPU_NAME, BENCH_NAME, NPU_DTYPE, BENCH_DTYPE, NPU_SHAPE, BENCH_SHAPE, COSINE, EUC_DIST,
-        MAX_ABS_ERR, MAX_RELATIVE_ERR, ONE_THOUSANDTH_ERR_RATIO, FIVE_THOUSANDTHS_ERR_RATIO,
-        NPU_MAX, NPU_MIN, NPU_MEAN, NPU_NORM, BENCH_MAX, BENCH_MIN, BENCH_MEAN, BENCH_NORM, ACCURACY, ERROR_MESSAGE
-    ]
+    ALL_COMPARE_INDEX = [COSINE, EUC_DIST, MAX_ABS_ERR, MAX_RELATIVE_ERR,
+                         ONE_THOUSANDTH_ERR_RATIO, FIVE_THOUSANDTHS_ERR_RATIO]
+    SUMMARY_COMPARE_INDEX = [MAX_DIFF, MIN_DIFF, MEAN_DIFF, NORM_DIFF,
+                             MAX_RELATIVE_ERR, MIN_RELATIVE_ERR, MEAN_RELATIVE_ERR, NORM_RELATIVE_ERR]
+    MD5_COMPARE_INDEX = [RESULT]
 
-    SUMMARY_COMPARE_RESULT_HEADER = [
-        NPU_NAME, BENCH_NAME, NPU_DTYPE, BENCH_DTYPE, NPU_SHAPE, BENCH_SHAPE, MAX_DIFF, MIN_DIFF, MEAN_DIFF, NORM_DIFF,
-        MAX_RELATIVE_ERR, MIN_RELATIVE_ERR, MEAN_RELATIVE_ERR, NORM_RELATIVE_ERR,
-        NPU_MAX, NPU_MIN, NPU_MEAN, NPU_NORM, BENCH_MAX, BENCH_MIN, BENCH_MEAN, BENCH_NORM, RESULT, ERROR_MESSAGE
-    ]
+    BASIC_INFO = [NPU_NAME, BENCH_NAME, NPU_DTYPE, BENCH_DTYPE, NPU_SHAPE, BENCH_SHAPE, NPU_REQ_GRAD, BENCH_REQ_GRAD]
+    SUMMARY_INFO = [NPU_MAX, NPU_MIN, NPU_MEAN, NPU_NORM, BENCH_MAX, BENCH_MIN, BENCH_MEAN, BENCH_NORM]
 
-    MD5_COMPARE_RESULT_HEADER = [
-        NPU_NAME, BENCH_NAME, NPU_DTYPE, BENCH_DTYPE, NPU_SHAPE, BENCH_SHAPE, NPU_MD5, BENCH_MD5, RESULT
-    ]
+    COMPARE_RESULT_HEADER = BASIC_INFO + ALL_COMPARE_INDEX + SUMMARY_INFO + [REQ_GRAD_CONSIST, ACCURACY, ERROR_MESSAGE]
+
+    SUMMARY_COMPARE_RESULT_HEADER = BASIC_INFO + SUMMARY_COMPARE_INDEX + SUMMARY_INFO + [REQ_GRAD_CONSIST, RESULT,
+                                                                                         ERROR_MESSAGE]
+
+    MD5_COMPARE_RESULT_HEADER = BASIC_INFO + [NPU_MD5, BENCH_MD5, REQ_GRAD_CONSIST] + MD5_COMPARE_INDEX
 
     COMPARE_RESULT_HEADER_STACK = COMPARE_RESULT_HEADER + [STACK]
 
@@ -510,11 +540,6 @@ class CompareConst:
         Const.MD5: MD5_COMPARE_RESULT_HEADER
     }
 
-    ALL_COMPARE_INDEX = [COSINE, EUC_DIST, MAX_ABS_ERR, MAX_RELATIVE_ERR, ONE_THOUSANDTH_ERR_RATIO,
-                         FIVE_THOUSANDTHS_ERR_RATIO]
-    SUMMARY_COMPARE_INDEX = [MAX_DIFF, MIN_DIFF, MEAN_DIFF, NORM_DIFF,
-                             MAX_RELATIVE_ERR, MIN_RELATIVE_ERR, MEAN_RELATIVE_ERR, NORM_RELATIVE_ERR]
-
     # dtype match
 
     DTYPE_MATCH_GROUPS = [
@@ -551,6 +576,8 @@ class CompareConst:
     ULP_FLOAT16_THRESHOLD = 1
 
     # compare result data
+    NO_REAL_DATA = 'No real data'
+    API_UNMATCH = 'api unmatched'
     READ_NONE = 'No data'
     NONE = 'None'
     SHAPE_UNMATCH = 'shape unmatched'
@@ -619,6 +646,9 @@ class CompareConst:
         MAX_DIFF: None, MIN_DIFF: None, MEAN_DIFF: None, NORM_DIFF: None, MAX_RELATIVE_ERR: None,
         MIN_RELATIVE_ERR: None, MEAN_RELATIVE_ERR: None, NORM_RELATIVE_ERR: None
     }
+    MS_GRAPH_CSV = {
+        NPU_CSV_FILE: None, BENCH_CSV_FILE: None
+    }
 
     API_MAPPING_KEYS_TO_COMPARE = [
         ('ms_args', 'pt_args'),
@@ -638,9 +668,11 @@ class CompareConst:
 
     OP_NAME_X = 'op_name_x'
     MATCH_RESULT_COLUMNS = [
-        OP_NAME_X, 'dtype_x', 'shape_x', 'summary_x', 'stack_info_x', 'data_name_x',
+        OP_NAME_X, 'dtype_x', 'shape_x', 'summary_x', 'stack_info_x', 'state_x', 'api_origin_name_x',
+        'requires_grad_x', 'data_name_x',
         CMP_KEY, CMP_SHAPE,
-        'op_name_y', 'dtype_y', 'shape_y', 'summary_y', 'stack_info_y', 'data_name_y',
+        'op_name_y', 'dtype_y', 'shape_y', 'summary_y', 'stack_info_y', 'state_y', 'api_origin_name_y',
+        'requires_grad_y', 'data_name_y'
     ]
 
     INTERNAL_API_MAPPING_FILE = 'ms_to_pt_api.yaml'
@@ -671,6 +703,8 @@ class FileCheckConst:
     IR_SUFFIX = ".ir"
     ZIP_SUFFIX = ".zip"
     SHELL_SUFFIX = ".sh"
+    LOG_SUFFIX = ".log"
+    DB_SUFFIX = '.db'
     MAX_PKL_SIZE = 1073741824  # 1 * 1024 * 1024 * 1024
     MAX_NUMPY_SIZE = 10737418240  # 10 * 1024 * 1024 * 1024
     MAX_JSON_SIZE = 1073741824  # 1 * 1024 * 1024 * 1024
@@ -683,6 +717,8 @@ class FileCheckConst:
     MAX_FILE_IN_ZIP_SIZE = 1073741824  # 1 * 1024 * 1024 * 1024
     MAX_FILE_SIZE = 1073741824  # 1 * 1024 * 1024 * 1024
     COMMOM_FILE_SIZE = 1048576  # 1 * 1024 * 1024
+    MAX_LOG_SIZE = 10737418240  # 1 * 1024 * 1024 * 1024
+    MAX_DB_SIZE = 10737418240  # 10 * 1024 * 1024 * 1024
     DIR = "dir"
     FILE = "file"
     DATA_DIR_AUTHORITY = 0o750
@@ -696,7 +732,9 @@ class FileCheckConst:
         XLSX_SUFFIX: MAX_XLSX_SIZE,
         YAML_SUFFIX: MAX_YAML_SIZE,
         IR_SUFFIX: MAX_IR_SIZE,
-        ZIP_SUFFIX: MAX_ZIP_SIZE
+        ZIP_SUFFIX: MAX_ZIP_SIZE,
+        LOG_SUFFIX: MAX_LOG_SIZE,
+        DB_SUFFIX: MAX_DB_SIZE
     }
     CSV_BLACK_LIST = r'^[＋－＝％＠\+\-=%@]|;[＋－＝％＠\+\-=%@]'
 
@@ -755,6 +793,11 @@ class MonitorConst:
     DEFAULT_STEP_INTERVAL = 1
 
     OP_LIST = ["norm", "min", "max", "zeros", "nans", "id", "mean", "shape", "dtype"]
+    OP_MONVIS_SUPPORTED = [
+        "norm", "min", "max", "zeros", "nans", "mean",
+        "entropy", "softmax_max", "sr", "kernel_norm", "std_x", "jacobian",
+        "proxy", "token_similarity"
+    ]
     MONITOR_OUTPUT_DIR = "MONITOR_OUTPUT_DIR"
     DEFAULT_MONITOR_OUTPUT_DIR = "./monitor_output"
     DATABASE = "database"
@@ -767,6 +810,8 @@ class MonitorConst:
     )
     DEEPSPEED_ZERO_OPT_FILTER = "DeepSpeedZeroOptimizer"
     RULE_NAME = ['AnomalyTurbulence', 'AnomalyNan']
+    L2_HOOKS = ["linear_hook", "attention_hook"]
+    SA_ORDERS = ["s,b,h,d", "b,s,h,d"]
 
     SLICE_SIZE = 20480
     # used for name
@@ -817,3 +862,12 @@ class MonitorConst:
         TRAIN_STAGE[key] = BACKWARD_STAGE
     for key in OPTIMIZER_KEY:
         TRAIN_STAGE[key] = OPTIMIZER_STAGE
+
+    # csv2db
+    DEFAULT_INT_VALUE = 0
+    MAX_PROCESS_NUM = 128
+    CSV_FILE_PATTERN = r"_(\d+)-(\d+)\.csv"
+    BATCH_SIZE = 10000
+    MAX_PARTITION = 10_000_000
+    MIN_PARTITION = 10
+