mindstudio-probe 1.0.4__tar.gz → 1.1.0__tar.gz

{mindstudio-probe-1.0.4 → mindstudio-probe-1.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mindstudio-probe
-Version: 1.0.4
+Version: 1.1.0
 Summary: Pytorch Ascend Probe Utils
 Home-page: https://gitee.com/ascend/mstt/tree/master/debug/accuracy_tools/msprobe
 Author: Ascend Team

{mindstudio-probe-1.0.4 → mindstudio-probe-1.1.0}/mindstudio_probe.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mindstudio-probe
-Version: 1.0.4
+Version: 1.1.0
 Summary: Pytorch Ascend Probe Utils
 Home-page: https://gitee.com/ascend/mstt/tree/master/debug/accuracy_tools/msprobe
 Author: Ascend Team

{mindstudio-probe-1.0.4 → mindstudio-probe-1.1.0}/mindstudio_probe.egg-info/SOURCES.txt

@@ -20,6 +20,8 @@ msprobe/core/advisor/advisor_result.py
 msprobe/core/common/const.py
 msprobe/core/common/exceptions.py
 msprobe/core/common/file_utils.py
+msprobe/core/common/inplace_op_checker.py
+msprobe/core/common/inplace_ops.yaml
 msprobe/core/common/log.py
 msprobe/core/common/utils.py
 msprobe/core/compare/acc_compare.py
@@ -55,8 +57,10 @@ msprobe/docs/12.overflow_check_PyTorch.md
 msprobe/docs/13.overflow_check_MindSpore.md
 msprobe/docs/14.data_parse_PyTorch.md
 msprobe/docs/15.free_benchmarking_PyTorch.md
+msprobe/docs/16.free_benchmarking_MindSpore.md
 msprobe/docs/17.grad_probe.md
-msprobe/docs/FAQ_PyTorch.md
+msprobe/docs/18.online_dispatch.md
+msprobe/docs/FAQ.md
 msprobe/docs/S02.report_free_benchmarking_validation_performance_baseline.md
 msprobe/docs/img/BLOOM-7B_1.png
 msprobe/docs/img/BLOOM-7B_2.png
@@ -88,6 +92,9 @@ msprobe/docs/img/grad_probe_image-3.png
 msprobe/docs/img/grad_probe_image-4.png
 msprobe/docs/img/grad_probe_image.png
 msprobe/docs/img/module_compare.png
+msprobe/docs/img/ms_dump.png
+msprobe/docs/img/ms_layer.png
+msprobe/docs/img/pt_dump.png
 msprobe/mindspore/__init__.py
 msprobe/mindspore/cell_processor.py
 msprobe/mindspore/ms_config.py
@@ -108,6 +115,8 @@ msprobe/mindspore/common/const.py
 msprobe/mindspore/common/log.py
 msprobe/mindspore/common/utils.py
 msprobe/mindspore/compare/distributed_compare.py
+msprobe/mindspore/compare/layer_mapping.py
+msprobe/mindspore/compare/modify_mapping.py
 msprobe/mindspore/compare/ms_compare.py
 msprobe/mindspore/compare/ms_graph_compare.py
 msprobe/mindspore/compare/ms_to_pt_api.yaml
@@ -121,6 +130,7 @@ msprobe/mindspore/dump/kernel_graph_dump.py
 msprobe/mindspore/dump/kernel_kbyk_dump.py
 msprobe/mindspore/dump/hook_cell/api_registry.py
 msprobe/mindspore/dump/hook_cell/hook_cell.py
+msprobe/mindspore/dump/hook_cell/primitive_hooks.py
 msprobe/mindspore/dump/hook_cell/support_wrap_ops.yaml
 msprobe/mindspore/dump/hook_cell/wrap_api.py
 msprobe/mindspore/free_benchmark/__init__.py
@@ -189,8 +199,10 @@ msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/__init__.py
 msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/attl.py
 msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/client.py
 msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/device_dispatch.py
+msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/dump_dispatch.py
 msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/server.py
-msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/ssl_config.py
+msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/torch_ops_config.yaml
+msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/utils.py
 msprobe/pytorch/bench_functions/__init__.py
 msprobe/pytorch/bench_functions/apply_adam_w.py
 msprobe/pytorch/bench_functions/confusion_transpose.py
@@ -243,8 +255,7 @@ msprobe/pytorch/free_benchmark/result_handlers/fix_handler.py
 msprobe/pytorch/free_benchmark/result_handlers/handler_factory.py
 msprobe/pytorch/free_benchmark/result_handlers/preheat_handler.py
 msprobe/pytorch/functional/__init__.py
-msprobe/pytorch/functional/data_processor.py
-msprobe/pytorch/functional/dump_module.py
+msprobe/pytorch/functional/module_dump.py
 msprobe/pytorch/grad_probe/__init__.py
 msprobe/pytorch/grad_probe/grad_monitor.py
 msprobe/pytorch/grad_probe/grad_stat_csv.py

{mindstudio-probe-1.0.4 → mindstudio-probe-1.1.0}/msprobe/README.md

@@ -1,12 +1,12 @@
 # 📖 msprobe 使用手册
 
-![version](https://img.shields.io/badge/version-1.0.3-blueviolet)
+![version](https://img.shields.io/badge/version-1.0.4-blueviolet)
 ![python](https://img.shields.io/badge/python-3.8|3.9|3.10-blue)
 ![platform](https://img.shields.io/badge/platform-Linux-yellow)
 
-[**msprobe**](./) 是 MindStudio Training Tools 工具链下精度调试部分的工具包。主要包括精度预检、溢出检测和精度比对等功能，目前适配 [PyTorch](https://pytorch.org/) 和 [MindSpore](https://www.mindspore.cn/) 框架。这些子工具侧重不同的训练场景，可以定位模型训练中的精度问题。
+**msprobe** 是 MindStudio Training Tools 工具链下精度调试部分的工具包。主要包括精度预检、溢出检测和精度比对等功能，目前适配 [PyTorch](https://pytorch.org/) 和 [MindSpore](https://www.mindspore.cn/) 框架。这些子工具侧重不同的训练场景，可以定位模型训练中的精度问题。
 
-为方便使用，本工具提供了一个统一、简易的程序接口，**PrecisionDebugger**，以 PyTorch 框架为例，通过以下示例模板和 **config.json** 可轻松使用各种功能。
+为方便使用，本工具提供了统一、简易的程序接口：**PrecisionDebugger**。以 PyTorch 框架为例，通过以下示例模板和 **config.json** 可以轻松使用各种功能。
 
 ```python
 from msprobe.pytorch import PrecisionDebugger  
@@ -19,7 +19,19 @@ debugger.stop() # 一般在训练循环末尾结束工具
 debugger.step() # 在训练循环的最后需要重置工具，非循环场景不需要
 ```
 
-除了在训练脚本中调用接口函数，还可以通过命令行使用 **msprobe** 的其他功能，具体的使用规则和 **config.json** 的配置要求详见以下章节。
+此外，根据以下规则，可以通过环境变量设置日志级别。
+- MSPROBE_LOG_LEVEL=4，不打印任何日志；
+- MSPROBE_LOG_LEVEL=3，仅打印 ERROR；
+- MSPROBE_LOG_LEVEL=2，仅打印 WARNING、ERROR；
+- MSPROBE_LOG_LEVEL=1，仅打印 INFO、WARNING、ERROR（默认配置）；
+- MSPROBE_LOG_LEVEL=0，打印 DEBUG、INFO、WARNING、ERROR。
+
+例如在 shell 脚本：
+
+```shell
+export MSPROBE_LOG_LEVEL={x}
+```
+**config.json** 的配置要求和各功能具体的使用指导详见后续章节。
 
 ## ⚙️ [安装](./docs/01.installation.md)
 
@@ -67,9 +79,9 @@ MindSpore 动态图场景的[离线预检](./docs/09.accuracy_checker_MindSpore.
 
 ### 6 无标杆比对
 
-[PyTorch 场景的无标杆比对](./docs/15.free_benchmarking_PyTorch.md)（待补充）
+[PyTorch 场景的无标杆比对](./docs/15.free_benchmarking_PyTorch.md)
 
-[MindSpore 场景的无标杆比对](./docs/16.overflow_check_MindSpore.md)（待补充）
+[MindSpore 场景的无标杆比对](./docs/16.free_benchmarking_MindSpore.md)
 
 ### 7 梯度状态监测
 
@@ -77,25 +89,43 @@ MindSpore 动态图场景的[离线预检](./docs/09.accuracy_checker_MindSpore.
 
 [兼容 PyTorch 和 MindSpore 框架的梯度监测](./docs/17.grad_probe.md)
 
+### 8 在线精度比对
+
+在线精度比对是实现在PyTorch训练过程中直接完成精度比对并输出比对结果的功能，是NPU与CPU之间的精度比对。
+
+[PyTorch 场景的在线精度比对](./docs/18.online_dispatch.md)
+
 ## 🌟 新版本特性
 
+若查看历史版本特性，请点击[安装](./docs/01.installation.md)。
+
+【数据采集】
+- 支持 config.json 中的 step 传入范围；
+- 优化了指定 step 的机制，指定 step 结束后工具不再采集数据，但训练会继续运行。工具结束运行后，日志提示信息如下：
+    ```bash
+    ****************************************
+    *      msprobe ends successfully.      *
+    ****************************************
+    ```
+    注：在多卡场景，每张卡进程训练到指定 step 之后都会打印一次上述信息。
+
 【精度预检】
-- 落盘数据小。
-- 支持随机生成模式和真实数据模式。
-- 单 API 测试，排除整网中的累计误差问题。
+- 在 PyTorch 场景，支持部分 NPU 融合算子预检。
 
-【梯度检测】
-- 使用便捷，无需在训练流程里插入代码。
-- 可以精准定位问题出现的 step。
+【精度比对】
+- 解决了使用 MindSpore 需要安装 PyTorch 的问题。
+
+【无标杆比对】
+- 补充在 PyTorch 场景的性能基线报告；
+- 支持 MindSpore 场景的 change_value 扰动模式。
 
 ## 📑 补充材料
 
-[msprobe 标准性能基线报告](./docs/S01.report_msprobe_dump_standard_performance_baseline.md)
+[无标杆比对功能在 PyTorch 场景的性能基线报告](./docs/S02.report_free_benchmarking_validation_performance_baseline.md)
 
-[无标杆工具场景验证和性能基线报告](./docs/S02.report_free_benchmarking_validation_performance_baseline.md)
+## ❗ 免责声明
+本工具建议执行用户与安装用户保持一致，如果您要使用 root 执行，请自行关注 root 高权限触及的安全风险。
 
 ## ❓ FAQ
 
 [FAQ for PyTorch](./docs/FAQ_PyTorch.md)
-
-FAQ for MindSpore

mindstudio-probe-1.1.0/msprobe/__init__.py

@@ -0,0 +1,16 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from msprobe.core.grad_probe.grad_compare import GradComparator

{mindstudio-probe-1.0.4 → mindstudio-probe-1.1.0}/msprobe/config.json

@@ -4,8 +4,6 @@
     "rank": [],  
     "step": [],  
     "level": "L1", 
-    "seed": 1234,
-    "is_deterministic": false, 
     "enable_dataloader": false,
     "acl_config": "",
     "tensor": {

{mindstudio-probe-1.0.4 → mindstudio-probe-1.1.0}/msprobe/core/advisor/advisor.py

@@ -1,8 +1,7 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-# Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
+# Copyright (c) 2022-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
@@ -13,7 +12,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""
 
 import os
 
@@ -24,15 +22,17 @@ from msprobe.core.common.utils import CompareException
 from msprobe.core.common.file_utils import FileChecker
 from msprobe.core.common.const import Const, CompareConst, FileCheckConst
 
+
 class Advisor:
     """
     Class for generate advisor
     """
 
-    def __init__(self, input_data, out_path=""):
+    def __init__(self, input_data, out_path="", suffix=""):
         self.input_data = input_data
         self.out_path = os.path.realpath(out_path)
         self.file_type = None
+        self.suffix = suffix
 
     @staticmethod
     def deterministic_advisor(message, node_name):
@@ -103,7 +103,7 @@ class Advisor:
         else:
             result = self.gen_advisor_result(failing_data)
         message_list = result.print_advisor_log()
-        result.gen_summary_file(self.out_path, message_list)
+        result.gen_summary_file(self.out_path, message_list, suffix=self.suffix)
 
     def _parse_input_data(self):
         data_columns = self.input_data.columns.values

{mindstudio-probe-1.0.4 → mindstudio-probe-1.1.0}/msprobe/core/advisor/advisor_const.py

@@ -1,8 +1,7 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-# Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
+# Copyright (c) 2022-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
@@ -13,7 +12,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""
 
 
 class AdvisorConst:
@@ -35,7 +33,8 @@ class AdvisorConst:
     # advisor message
     NO_ERR_SUGGEST = "All data in comparison result meets the accuracy requirements."
     FORWARD_INPUT_SUGGEST = "1. Analyze the model to view the input source.\n" \
-                            "2. Check whether an inplace API causes the output result to overwrite the input result. That is, the fault is actually caused by a computation error.\n" \
+                            "2. Check whether an inplace API causes the output result to overwrite the input result. "\
+                            "That is, the fault is actually caused by a computation error.\n" \
                             "3. The fault may be caused by memory corruption and further analysis is required."
     FORWARD_OUTPUT_SUGGEST = "This is a forward API computation error. Check the computation implementation."
     BACKWARD_INPUT_SUGGEST = "Check whether the forward computation result is affected."

{mindstudio-probe-1.0.4 → mindstudio-probe-1.1.0}/msprobe/core/advisor/advisor_result.py

@@ -1,8 +1,7 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-# Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
+# Copyright (c) 2022-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
@@ -13,13 +12,12 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""
 import os
 import time
 
 from msprobe.core.advisor.advisor_const import AdvisorConst
 from msprobe.core.common.log import logger
-from msprobe.core.common.const import Const, FileCheckConst
+from msprobe.core.common.const import FileCheckConst
 from msprobe.core.common.file_utils import change_mode, FileOpen
 
 
@@ -34,8 +32,8 @@ class AdvisorResult:
         self.advisor_message = message
 
     @staticmethod
-    def gen_summary_file(out_path, message_list):
-        file_name = 'advisor_{}.txt'.format(time.strftime("%Y%m%d%H%M%S", time.localtime(time.time())))
+    def gen_summary_file(out_path, message_list, suffix):
+        file_name = 'advisor{}_{}.txt'.format(suffix, time.strftime("%Y%m%d%H%M%S", time.localtime(time.time())))
         result_file = os.path.join(out_path, file_name)
         try:
             with FileOpen(result_file, 'w+') as output_file:
@@ -50,9 +48,11 @@ class AdvisorResult:
 
     def print_advisor_log(self):
         logger.info("The summary of the expert advice is as follows: ")
-        message_list = [AdvisorConst.LINE + AdvisorConst.COLON + str(self.line),
-                        AdvisorConst.SUSPECT_NODES + AdvisorConst.COLON + self.suspect_node,
-                        AdvisorConst.ADVISOR_SUGGEST + AdvisorConst.COLON + self.advisor_message]
+        message_list = [
+            AdvisorConst.LINE + AdvisorConst.COLON + str(self.line),
+            AdvisorConst.SUSPECT_NODES + AdvisorConst.COLON + self.suspect_node,
+            AdvisorConst.ADVISOR_SUGGEST + AdvisorConst.COLON + self.advisor_message
+        ]
         for message in message_list:
             logger.info(message)
         return message_list

{mindstudio-probe-1.0.4 → mindstudio-probe-1.1.0}/msprobe/core/common/const.py

@@ -14,11 +14,13 @@ class Const:
     REGEX_PREFIX_MAX_LENGTH = 20
     REGEX_PREFIX_PATTERN = r"^[a-zA-Z0-9_-]+$"
     FILE_PATTERN = r'^[a-zA-Z0-9_./-]+$'
+    STRING_BLACKLIST = r"^[＋－＝％＠\+\-=%@]|;[＋－＝％＠\+\-=%@]"
     COMMA = ","
     FLOAT_EPSILON = np.finfo(float).eps
     OFF = 'OFF'
     BACKWARD = 'backward'
     FORWARD = 'forward'
+    JIT = 'Jit'
     PRIMITIVE_PREFIX = 'Primitive'
     DEFAULT_LIST = []
     DEFAULT_PATH = './'
@@ -30,6 +32,7 @@ class Const:
     FOUR_SEGMENT = 4
     SIX_SEGMENT = 6
     SEVEN_SEGMENT = 7
+    MAX_DEPTH = 10
 
     # dump mode
     ALL = "all"
@@ -78,6 +81,7 @@ class Const:
     RUN_UT = "run_ut"
     GRAD_PROBE = "grad_probe"
     TASK_LIST = [TENSOR, STATISTICS, OVERFLOW_CHECK, FREE_BENCHMARK, RUN_UT, GRAD_PROBE]
+    DUMP_DATA_COLLECTION_LIST = [STATISTICS, TENSOR]
     LEVEL_L0 = "L0"
     LEVEL_L1 = "L1"
     LEVEL_L2 = "L2"
@@ -100,6 +104,30 @@ class Const:
     CUDA_LOWERCASE = 'cuda'
     DISTRIBUTED = 'Distributed'
 
+    # struct json param
+    ORIGIN_DATA = "origin_data"
+    SCOPE = "scope"
+    STACK = "stack"
+
+    ATEN = "Aten"
+    MODULE_WHITE_LIST = ["torch", "numpy"]
+
+    FUNC_SKIP_LIST = ["construct", "__call__"]
+
+    FILE_SKIP_LIST = ["site-packages/mindspore", "package/mindspore", "msprobe", "site-packages/torch", "package/torch"]
+
+    STACK_FILE_INDEX = 0
+
+    STACK_FUNC_INDEX = 2
+
+    STACK_FUNC_ELE_INDEX = 1
+
+    CONSTRUCT_NAME_INDEX = -3
+
+    NAME_FIRST_POSSIBLE_INDEX = -4
+
+    NAME_SECOND_POSSIBLE_INDEX = -5
+
     INPLACE_LIST = [
         "broadcast", "all_reduce", "reduce", "all_gather", "gather", "scatter", "reduce_scatter",
         "_reduce_scatter_base", "_all_gather_base", "send", "recv", "irecv", "isend", "all_to_all_single", "all_to_all",
@@ -114,6 +142,23 @@ class Const:
         "int32_to_int64": ["cross_entropy"]
     }
 
+    FILL_CHAR_NUMS = 50
+    TOOL_ENDS_SUCCESSFULLY = f"{TOOL_NAME} ends successfully."
+    WITHOUT_CALL_STACK = "The call stack retrieval failed."
+    
+    STEP = "step"
+    RANK = "rank"
+    HYPHEN = "-"
+    STEP_RANK_MAXIMUM_RANGE = [int(0), int(1e6)]
+
+    # data type const
+    FLOAT16 = "Float16"
+    FLOAT32 = "Float32"
+    BFLOAT16 = "BFloat16"
+    TORCH_FLOAT16 = "torch.float16"
+    TORCH_FLOAT32 = "torch.float32"
+    TORCH_BFLOAT16 = "torch.bfloat16"
+
 
 class CompareConst:
     """
@@ -159,6 +204,7 @@ class CompareConst:
     INPUT_STRUCT = "input_struct"
     OUTPUT_STRUCT = "output_struct"
     SUMMARY = "summary"
+    MAX_EXCEL_LENGTH = 1048576
 
     COMPARE_RESULT_HEADER = [
         NPU_NAME, BENCH_NAME, NPU_DTYPE, BENCH_DTYPE, NPU_SHAPE, BENCH_SHAPE, COSINE, MAX_ABS_ERR, MAX_RELATIVE_ERR,
@@ -197,6 +243,8 @@ class CompareConst:
     ERROR = 'error'
     SKIP = 'SKIP'
     N_A = 'N/A'
+    INF = 'inf'
+    NEG_INF = '-inf'
     BFLOAT16_MIN = -3.3895313892515355e+38
     BFLOAT16_MAX = 3.3895313892515355e+38
     BFLOAT16_EPS = 3.90625e-3  # 2 ** -8
@@ -274,7 +322,8 @@ class FileCheckConst:
     MAX_JSON_SIZE = 1073741824  # 1 * 1024 * 1024 * 1024
     MAX_PT_SIZE = 10737418240  # 10 * 1024 * 1024 * 1024
     MAX_CSV_SIZE = 1073741824  # 1 * 1024 * 1024 * 1024
-    MAX_YAML_SIZE = 1048576  # 10 * 1024 * 1024
+    MAX_YAML_SIZE = 1048576  # 1 * 1024 * 1024
+    COMMOM_FILE_SIZE = 1048576  # 1 * 1024 * 1024
     DIR = "dir"
     FILE = "file"
     DATA_DIR_AUTHORITY = 0o750
@@ -287,6 +336,7 @@ class FileCheckConst:
         CSV_SUFFIX: MAX_CSV_SIZE,
         YAML_SUFFIX: MAX_YAML_SIZE
     }
+    CSV_BLACK_LIST = r'^[＋－＝％＠\+\-=%@]|;[＋－＝％＠\+\-=%@]'
 
 
 class OverflowConst:
@@ -329,11 +379,22 @@ class MsgConst:
     """
     Class for log messages const
     """
-    CLEAR_SYMBOL = "\033[K"
     MSPROBE_LOG_LEVEL = "MSPROBE_LOG_LEVEL"
-    LEVEL = ["INFO", "WARNING", "ERROR", "DEBUG"]
+    LOG_LEVEL_ENUM = ["0", "1", "2", "3", "4"]
+    LOG_LEVEL = ["DEBUG", "INFO", "WARNING", "ERROR"]
+    class LogLevel:
+        class DEBUG:
+            value = 0
+        class INFO:
+            value = 1
+        class WARNING:
+            value = 2
+        class ERROR:
+            value = 3
     SPECIAL_CHAR = ["\n", "\r", "\u007F", "\b", "\f", "\t", "\u000B", "%08", "%0a", "%0b", "%0c", "%0d", "%7f"]
 
+    NOT_CREATED_INSTANCE = "PrecisionDebugger instance is not created."
+
 
 class GraphMode:
     NPY_MODE = "NPY_MODE"

{mindstudio-probe-1.0.4 → mindstudio-probe-1.1.0}/msprobe/core/common/exceptions.py

@@ -13,8 +13,8 @@ class MsprobeException(CodedException):
     OVERFLOW_NUMS_ERROR = 1
 
     err_strs = {
-        INVALID_PARAM_ERROR: "[msprobe] 无效参数： ",
-        OVERFLOW_NUMS_ERROR: "[msprobe] 超过预设溢出次数 当前溢出次数:"
+        INVALID_PARAM_ERROR: "[msprobe] 无效参数：",
+        OVERFLOW_NUMS_ERROR: "[msprobe] 超过预设溢出次数 当前溢出次数："
     }
 
 

{mindstudio-probe-1.0.4 → mindstudio-probe-1.1.0}/msprobe/core/common/file_utils.py

@@ -22,6 +22,7 @@ import re
 import shutil
 import yaml
 import numpy as np
+import pandas as pd
 
 from msprobe.core.common.log import logger
 from msprobe.core.common.exceptions import FileCheckException
@@ -187,7 +188,7 @@ def check_other_user_writable(path):
 
 def check_path_owner_consistent(path):
     file_owner = os.stat(path).st_uid
-    if file_owner != os.getuid():
+    if file_owner != os.getuid() and os.getuid() != 0:
         logger.error('The file path %s may be insecure because is does not belong to you.' % path)
         raise FileCheckException(FileCheckException.FILE_PERMISSION_ERROR)
 
@@ -214,7 +215,9 @@ def check_common_file_size(file_path):
         for suffix, max_size in FileCheckConst.FILE_SIZE_DICT.items():
             if file_path.endswith(suffix):
                 check_file_size(file_path, max_size)
-                break
+                return
+        check_file_size(file_path, FileCheckConst.COMMOM_FILE_SIZE)
+        
 
 
 def check_file_suffix(file_path, file_suffix):
@@ -322,7 +325,7 @@ def check_file_type(path):
     elif os.path.isfile(path):
         return FileCheckConst.FILE
     else:
-        logger.error('Neither a file nor a directory.')
+        logger.error(f'{path} does not exist, please check!')
         raise FileCheckException(FileCheckException.INVALID_FILE_ERROR)
 
 
@@ -338,10 +341,10 @@ def load_yaml(yaml_path):
     return yaml_data
 
 
-def load_npy(filepath, enable_pickle=False):
+def load_npy(filepath):
     check_file_or_directory_path(filepath)
     try:
-        npy = np.load(filepath, allow_pickle=enable_pickle)
+        npy = np.load(filepath)
     except Exception as e:
         logger.error(f"The numpy file failed to load. Please check the path: {filepath}.")
         raise RuntimeError(f"Load numpy file {filepath} failed.") from e
@@ -374,6 +377,20 @@ def save_json(json_path, data, indent=None):
     change_mode(json_path, FileCheckConst.DATA_FILE_AUTHORITY)
 
 
+def save_yaml(yaml_path, data):
+    yaml_path = os.path.realpath(yaml_path)
+    check_path_before_create(yaml_path)
+    try:
+        with FileOpen(yaml_path, 'w') as f:
+            fcntl.flock(f, fcntl.LOCK_EX)
+            yaml.dump(data, f, sort_keys=False)
+            fcntl.flock(f, fcntl.LOCK_UN)
+    except Exception as e:
+        logger.error(f'Save yaml file "{os.path.basename(yaml_path)}" failed.')
+        raise RuntimeError(f"Save yaml file {yaml_path} failed.") from e
+    change_mode(yaml_path, FileCheckConst.DATA_FILE_AUTHORITY)
+
+
 def move_file(src_path, dst_path):
     check_file_or_directory_path(src_path)
     check_path_before_create(dst_path)
@@ -396,9 +413,9 @@ def save_npy(data, filepath):
     change_mode(filepath, FileCheckConst.DATA_FILE_AUTHORITY)
 
 
-def save_npy_to_txt(self, data, dst_file='', align=0):
+def save_npy_to_txt(data, dst_file='', align=0):
     if os.path.exists(dst_file):
-        self.log.info("Dst file %s exists, will not save new one.", dst_file)
+        logger.info("Dst file %s exists, will not save new one." % dst_file)
         return
     shape = data.shape
     data = data.flatten()
@@ -411,7 +428,7 @@ def save_npy_to_txt(self, data, dst_file='', align=0):
     try:
         np.savetxt(dst_file, data.reshape((-1, align)), delimiter=' ', fmt='%g')
     except Exception as e:
-        self.log.error("An unexpected error occurred: %s when savetxt to %s" % (str(e)), dst_file)
+        logger.error("An unexpected error occurred: %s when savetxt to %s" % (str(e), dst_file))
     change_mode(dst_file, FileCheckConst.DATA_FILE_AUTHORITY)
 
 
@@ -431,7 +448,25 @@ def save_workbook(workbook, file_path):
     change_mode(file_path, FileCheckConst.DATA_FILE_AUTHORITY)
 
 
-def write_csv(data, filepath, mode="a+"):
+def write_csv(data, filepath, mode="a+", malicious_check=False):
+    def csv_value_is_valid(value: str) -> bool:
+        if not isinstance(value, str):
+            return True  
+        try:
+            # -1.00 or +1.00 should be consdiered as digit numbers
+            float(value)
+        except ValueError:
+            # otherwise, they will be considered as formular injections
+            return not bool(re.compile(FileCheckConst.CSV_BLACK_LIST).search(value))
+        return True
+    
+    if malicious_check:
+        for row in data:
+            for cell in row:
+                if not csv_value_is_valid(cell):
+                    raise RuntimeError(f"Malicious value [{cell}] is not allowed " \
+                                       f"to be written into the csv: {filepath}.")
+
     file_path = os.path.realpath(filepath)
     check_path_before_create(filepath)
     try:
@@ -444,6 +479,16 @@ def write_csv(data, filepath, mode="a+"):
     change_mode(filepath, FileCheckConst.DATA_FILE_AUTHORITY)
 
 
+def read_csv(filepath):
+    check_file_or_directory_path(filepath)
+    try:
+        csv_data = pd.read_csv(filepath)
+    except Exception as e:
+        logger.error(f"The csv file failed to load. Please check the path: {filepath}.")
+        raise RuntimeError(f"Read csv file {filepath} failed.") from e
+    return csv_data
+
+
 def remove_path(path):
     if not os.path.exists(path):
         return