mindstudio-probe 1.1.0__py3-none-any.whl → 1.1.1__py3-none-any.whl

msprobe/core/common/const.py

@@ -1,3 +1,18 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import os
 import stat
 
@@ -10,6 +25,7 @@ class Const:
     """
     TOOL_NAME = "msprobe"
 
+    ipv4_pattern = "([1-9]?\d|1\d{2}|2[0-4]\d|25[0-5])(\.([1-9]?\d|1\d{2}|2[0-4]\d|25[0-5])){3}$"
     SEP = "."
     REGEX_PREFIX_MAX_LENGTH = 20
     REGEX_PREFIX_PATTERN = r"^[a-zA-Z0-9_-]+$"
@@ -20,6 +36,8 @@ class Const:
     OFF = 'OFF'
     BACKWARD = 'backward'
     FORWARD = 'forward'
+    PROGRESS_TIMEOUT = 3000
+    EXCEPTION_NONE = None
     JIT = 'Jit'
     PRIMITIVE_PREFIX = 'Primitive'
     DEFAULT_LIST = []
@@ -82,6 +100,7 @@ class Const:
     GRAD_PROBE = "grad_probe"
     TASK_LIST = [TENSOR, STATISTICS, OVERFLOW_CHECK, FREE_BENCHMARK, RUN_UT, GRAD_PROBE]
     DUMP_DATA_COLLECTION_LIST = [STATISTICS, TENSOR]
+    DUMP_DATA_MODE_LIST = [ALL, INPUT, OUTPUT, FORWARD, BACKWARD]
     LEVEL_L0 = "L0"
     LEVEL_L1 = "L1"
     LEVEL_L2 = "L2"
@@ -93,6 +112,7 @@ class Const:
     DATA = "data"
     PT_FRAMEWORK = "pytorch"
     MS_FRAMEWORK = "mindspore"
+    UNKNOWN_FRAMEWORK = "unknown"
     DIRECTORY_LENGTH = 4096
     FILE_NAME_LENGTH = 255
     FLOAT_TYPE = [np.half, np.single, float, np.double, np.float64, np.longdouble, np.float32, np.float16]
@@ -103,6 +123,8 @@ class Const:
     CPU_LOWERCASE = 'cpu'
     CUDA_LOWERCASE = 'cuda'
     DISTRIBUTED = 'Distributed'
+    DUMP_PREFIX = ["Distributed", "Functional", "Torch", "Tensor", "Mint", "MintFunctional", "Primitive", 
+                   "Aten", "VF", "NPU", "Jit"]
 
     # struct json param
     ORIGIN_DATA = "origin_data"
@@ -113,21 +135,25 @@ class Const:
     MODULE_WHITE_LIST = ["torch", "numpy"]
 
     FUNC_SKIP_LIST = ["construct", "__call__"]
-
-    FILE_SKIP_LIST = ["site-packages/mindspore", "package/mindspore", "msprobe", "site-packages/torch", "package/torch"]
+    FILE_SKIP_LIST = ["msprobe", "MindSpeed"]
+    DATA_TYPE_SKIP_LIST = ["Primitive", "Jit"]
 
     STACK_FILE_INDEX = 0
-
     STACK_FUNC_INDEX = 2
-
     STACK_FUNC_ELE_INDEX = 1
 
-    CONSTRUCT_NAME_INDEX = -3
-
-    NAME_FIRST_POSSIBLE_INDEX = -4
-
-    NAME_SECOND_POSSIBLE_INDEX = -5
-
+    SCOPE_ID_INDEX = -1
+    SCOPE_DIRECTION_INDEX = -2
+    TYPE_NAME_INDEX = -3
+    LAYER_NAME_INDEX = -4
+    API_TYPE_INDEX = 0
+    LEFT_MOVE_INDEX = -1
+    RIGHT_MOVE_INDEX = 1
+
+    TOP_LAYER = "TopLayer"
+    CELL = "Cell"
+    MODULE = "Module"
+    FRAME_FILE_LIST = ["site-packages/torch", "package/torch", "site-packages/mindspore", "package/mindspore"]
     INPLACE_LIST = [
         "broadcast", "all_reduce", "reduce", "all_gather", "gather", "scatter", "reduce_scatter",
         "_reduce_scatter_base", "_all_gather_base", "send", "recv", "irecv", "isend", "all_to_all_single", "all_to_all",
@@ -145,11 +171,12 @@ class Const:
     FILL_CHAR_NUMS = 50
     TOOL_ENDS_SUCCESSFULLY = f"{TOOL_NAME} ends successfully."
     WITHOUT_CALL_STACK = "The call stack retrieval failed."
-    
+
     STEP = "step"
     RANK = "rank"
     HYPHEN = "-"
-    STEP_RANK_MAXIMUM_RANGE = [int(0), int(1e6)]
+    STEP_RANK_MINIMUM_VALUE = 0
+    STEP_RANK_MAXIMUM_VALUE = int(1e6)
 
     # data type const
     FLOAT16 = "Float16"
@@ -159,6 +186,13 @@ class Const:
     TORCH_FLOAT32 = "torch.float32"
     TORCH_BFLOAT16 = "torch.bfloat16"
 
+    DTYPE = 'dtype'
+    SHAPE = 'shape'
+    MAX = 'Max'
+    MIN = 'Min'
+    MEAN = 'Mean'
+    NORM = 'Norm'
+
 
 class CompareConst:
     """
@@ -201,10 +235,17 @@ class CompareConst:
     RESULT = "Result"
     MAGNITUDE = 0.5
     OP_NAME = "op_name"
+    STRUCT = "struct"
     INPUT_STRUCT = "input_struct"
+    KWARGS_STRUCT = "kwargs_struct"
     OUTPUT_STRUCT = "output_struct"
     SUMMARY = "summary"
     MAX_EXCEL_LENGTH = 1048576
+    YES = "Yes"
+    NO = "No"
+    STATISTICS_INDICATOR_NUM = 4
+    EPSILON = 1e-10
+    COMPARE_ENDS_SUCCESSFULLY = "msprobe compare ends successfully."
 
     COMPARE_RESULT_HEADER = [
         NPU_NAME, BENCH_NAME, NPU_DTYPE, BENCH_DTYPE, NPU_SHAPE, BENCH_SHAPE, COSINE, MAX_ABS_ERR, MAX_RELATIVE_ERR,
@@ -222,6 +263,12 @@ class CompareConst:
         NPU_NAME, BENCH_NAME, NPU_DTYPE, BENCH_DTYPE, NPU_SHAPE, BENCH_SHAPE, NPU_MD5, BENCH_MD5, RESULT
     ]
 
+    HEAD_OF_COMPARE_MODE = {
+        Const.ALL: COMPARE_RESULT_HEADER,
+        Const.SUMMARY: SUMMARY_COMPARE_RESULT_HEADER,
+        Const.MD5: MD5_COMPARE_RESULT_HEADER
+    }
+
     # compare standard
     HUNDRED_RATIO_THRESHOLD = 0.01
     THOUSAND_RATIO_THRESHOLD = 0.001
@@ -241,6 +288,8 @@ class CompareConst:
     PASS = 'pass'
     WARNING = 'Warning'
     ERROR = 'error'
+    TRUE = 'TRUE'
+    FALSE = 'FALSE'
     SKIP = 'SKIP'
     N_A = 'N/A'
     INF = 'inf'
@@ -298,6 +347,13 @@ class CompareConst:
         MAX_DIFF: None, MIN_DIFF: None, MEAN_DIFF: None, NORM_DIFF: None, MAX_RELATIVE_ERR: None,
         MIN_RELATIVE_ERR: None, MEAN_RELATIVE_ERR: None, NORM_RELATIVE_ERR: None
     }
+    INPUT_PATTERN = Const.SEP + Const.INPUT + Const.SEP
+    KWARGS_PATTERN = Const.SEP + Const.KWARGS + Const.SEP
+    OUTPUT_PATTERN = Const.SEP + Const.OUTPUT + Const.SEP
+    COMPARE_KEY = 'compare_key'
+    COMPARE_SHAPE = 'compare_shape'
+    INTERNAL_API_MAPPING_FILE = 'ms_to_pt_api.yaml'
+    UNREADABLE = 'unreadable data'
 
 
 class FileCheckConst:
@@ -322,7 +378,7 @@ class FileCheckConst:
     MAX_JSON_SIZE = 1073741824  # 1 * 1024 * 1024 * 1024
     MAX_PT_SIZE = 10737418240  # 10 * 1024 * 1024 * 1024
     MAX_CSV_SIZE = 1073741824  # 1 * 1024 * 1024 * 1024
-    MAX_YAML_SIZE = 1048576  # 1 * 1024 * 1024
+    MAX_YAML_SIZE = 1073741824  # 1 * 1024 * 1024 * 1024
     COMMOM_FILE_SIZE = 1048576  # 1 * 1024 * 1024
     DIR = "dir"
     FILE = "file"
@@ -351,6 +407,9 @@ class MsCompareConst:
     # api_info field
     MINT = "Mint"
     MINT_FUNCTIONAL = "MintFunctional"
+    TENSOR_API = "Tensor"
+
+    API_NAME_STR_LENGTH = 4
 
     TASK_FIELD = "task"
     STATISTICS_TASK = "statistics"
@@ -358,6 +417,10 @@ class MsCompareConst:
     DUMP_DATA_DIR_FIELD = "dump_data_dir"
     DATA_FIELD = "data"
 
+    # supported api yaml
+    SUPPORTED_API_LIST_FILE = "checker_support_api.yaml"
+    SUPPORTED_TENSOR_LIST_KEY = "tensor"
+
     # detail_csv
     DETAIL_CSV_API_NAME = "API Name"
     DETAIL_CSV_BENCH_DTYPE = "Bench Dtype"
@@ -382,15 +445,20 @@ class MsgConst:
     MSPROBE_LOG_LEVEL = "MSPROBE_LOG_LEVEL"
     LOG_LEVEL_ENUM = ["0", "1", "2", "3", "4"]
     LOG_LEVEL = ["DEBUG", "INFO", "WARNING", "ERROR"]
+
     class LogLevel:
         class DEBUG:
             value = 0
+
         class INFO:
             value = 1
+
         class WARNING:
             value = 2
+
         class ERROR:
             value = 3
+
     SPECIAL_CHAR = ["\n", "\r", "\u007F", "\b", "\f", "\t", "\u000B", "%08", "%0a", "%0b", "%0c", "%0d", "%7f"]
 
     NOT_CREATED_INSTANCE = "PrecisionDebugger instance is not created."
@@ -400,3 +468,35 @@ class GraphMode:
     NPY_MODE = "NPY_MODE"
     STATISTIC_MODE = "STATISTIC_MODE"
     ERROR_MODE = "ERROR_MODE"
+
+
+class MonitorConst:
+    """
+    Class for monitor const
+    """
+    OP_LIST = ["min", "max", "norm", "zeros", "nans", "id", "mean"]
+    MONITOR_OUTPUT_DIR = "MONITOR_OUTPUT_DIR"
+    DEFAULT_MONITOR_OUTPUT_DIR = "./monitor_output"
+    DATABASE = "database"
+    EMAIL = "email"
+    OPT_TY = ['Megatron_DistributedOptimizer', 'Megatron_Float16OptimizerWithFloat16Params']
+    DEEPSPEED_OPT_TY = ("DeepSpeedZeroOptimizer_Stage0", "DeepSpeedZeroOptimizer_Stage1_or_2", "DeepSpeedZeroOptimizer_Stage3")
+    RULE_NAME = ['AnomalyTurbulence']
+
+    DOT = "."
+    VPP_SEP = ":"
+    ACTV_IN = "input"
+    ACTV_OUT = "output"
+    ACTVGRAD_IN = "input_grad"
+    ACTVGRAD_OUT = "output_grad"
+    POST_GRAD = "post_grad"
+    PRE_GRAD = "pre_grad"
+    PREFIX_POST = "post"
+    PREFIX_PRE = "pre"
+
+
+    ANOMALY_JSON = "anomaly.json"
+    ANALYSE_JSON = "anomaly_analyse.json"
+    TENSORBOARD = "tensorboard"
+    CSV = "csv"
+    API = "api"

msprobe/core/common/exceptions.py

@@ -1,3 +1,18 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 class CodedException(Exception):
     def __init__(self, code, error_info=''):
         super().__init__()
@@ -11,10 +26,12 @@ class CodedException(Exception):
 class MsprobeException(CodedException):
     INVALID_PARAM_ERROR = 0
     OVERFLOW_NUMS_ERROR = 1
+    RECURSION_LIMIT_ERROR = 2
 
     err_strs = {
         INVALID_PARAM_ERROR: "[msprobe] 无效参数：",
-        OVERFLOW_NUMS_ERROR: "[msprobe] 超过预设溢出次数 当前溢出次数："
+        OVERFLOW_NUMS_ERROR: "[msprobe] 超过预设溢出次数 当前溢出次数：",
+        RECURSION_LIMIT_ERROR: "[msprobe] 递归调用超过限制："
     }
 
 
@@ -41,7 +58,7 @@ class ParseJsonException(CodedException):
     InvalidDumpJson = 1
     err_strs = {
         UnexpectedNameStruct: "[msprobe] Unexpected name in json: ",
-        InvalidDumpJson: "[msprobe] json格式不正确: ",
+        InvalidDumpJson: "[msprobe] Invalid dump.json format: ",
     }
 
 
@@ -73,9 +90,13 @@ class StepException(CodedException):
 class FreeBenchmarkException(CodedException):
     UnsupportedType = 0
     InvalidGrad = 1
+    InvalidPerturbedOutput = 2
+    OutputIndexError = 3
     err_strs = {
         UnsupportedType: "[msprobe] Free benchmark get unsupported type: ",
         InvalidGrad: "[msprobe] Free benchmark gradient invalid: ",
+        InvalidPerturbedOutput: "[msprobe] Free benchmark invalid perturbed output: ",
+        OutputIndexError: "[msprobe] Free benchmark output index out of bounds: ",
     }
 
 
@@ -87,6 +108,7 @@ class DistributedNotInitializedError(Exception):
     def __str__(self):
         return self.msg
 
+
 class ApiAccuracyCheckerException(CodedException):
     ParseJsonFailed = 0
     UnsupportType = 1
@@ -97,4 +119,4 @@ class ApiAccuracyCheckerException(CodedException):
         UnsupportType: "[msprobe] Api Accuracy Checker get unsupported type: ",
         WrongValue: "[msprobe] Api Accuracy Checker get wrong value: ",
         ApiWrong: "[msprobe] Api Accuracy Checker something wrong with api: ",
-    }
+    }

msprobe/core/common/file_utils.py

@@ -1,8 +1,7 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-# Copyright (C) 2022-2023. Huawei Technologies Co., Ltd. All rights reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
@@ -13,13 +12,17 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""
+
 import csv
 import fcntl
 import os
+import stat
 import json
 import re
 import shutil
+from datetime import datetime, timezone
+from dateutil import parser
+import OpenSSL
 import yaml
 import numpy as np
 import pandas as pd
@@ -67,9 +70,11 @@ class FileChecker:
         self.check_path_ability()
         if self.is_script:
             check_path_owner_consistent(self.file_path)
-        check_path_pattern_vaild(self.file_path)
+        check_path_pattern_valid(self.file_path)
         check_common_file_size(self.file_path)
         check_file_suffix(self.file_path, self.file_type)
+        if self.path_type == FileCheckConst.FILE:
+            check_dirpath_before_read(self.file_path)
         return self.file_path
 
     def check_path_ability(self):
@@ -122,9 +127,10 @@ class FileOpen:
         self.file_path = os.path.realpath(self.file_path)
         check_path_length(self.file_path)
         self.check_ability_and_owner()
-        check_path_pattern_vaild(self.file_path)
+        check_path_pattern_valid(self.file_path)
         if os.path.exists(self.file_path):
             check_common_file_size(self.file_path)
+            check_dirpath_before_read(self.file_path)
 
     def check_ability_and_owner(self):
         if self.mode in self.SUPPORT_READ_MODE:
@@ -193,7 +199,7 @@ def check_path_owner_consistent(path):
         raise FileCheckException(FileCheckException.FILE_PERMISSION_ERROR)
 
 
-def check_path_pattern_vaild(path):
+def check_path_pattern_valid(path):
     if not re.match(FileCheckConst.FILE_VALID_PATTERN, path):
         logger.error('The file path %s contains special characters.' % (path))
         raise FileCheckException(FileCheckException.ILLEGAL_PATH_ERROR)
@@ -217,7 +223,6 @@ def check_common_file_size(file_path):
                 check_file_size(file_path, max_size)
                 return
         check_file_size(file_path, FileCheckConst.COMMOM_FILE_SIZE)
-        
 
 
 def check_file_suffix(file_path, file_suffix):
@@ -238,9 +243,18 @@ def check_path_type(file_path, file_type):
             raise FileCheckException(FileCheckException.INVALID_FILE_ERROR)
 
 
+def check_others_writable(directory):
+    dir_stat = os.stat(directory)
+    is_writable = (
+        bool(dir_stat.st_mode & stat.S_IWGRP) or  # 组可写
+        bool(dir_stat.st_mode & stat.S_IWOTH)     # 其他用户可写
+    )
+    return is_writable
+
+
 def make_dir(dir_path):
-    dir_path = os.path.realpath(dir_path)
     check_path_before_create(dir_path)
+    dir_path = os.path.realpath(dir_path)
     if os.path.isdir(dir_path):
         return
     try:
@@ -262,8 +276,9 @@ def create_directory(dir_path):
     Exception Description:
         when invalid data throw exception
     """
-    dir_path = os.path.realpath(dir_path)
+    check_link(dir_path)
     check_path_before_create(dir_path)
+    dir_path = os.path.realpath(dir_path)
     parent_dir = os.path.dirname(dir_path)
     if not os.path.isdir(parent_dir):
         create_directory(parent_dir)
@@ -271,6 +286,7 @@ def create_directory(dir_path):
 
 
 def check_path_before_create(path):
+    check_link(path)
     if path_len_exceeds_limit(path):
         raise FileCheckException(FileCheckException.ILLEGAL_PATH_ERROR, 'The file path length exceeds limit.')
 
@@ -279,6 +295,17 @@ def check_path_before_create(path):
                                  'The file path {} contains special characters.'.format(path))
 
 
+def check_dirpath_before_read(path):
+    path = os.path.realpath(path)
+    dirpath = os.path.dirname(path)
+    if check_others_writable(dirpath):
+        logger.warning(f"The directory is writable by others: {dirpath}.")
+    try:
+        check_path_owner_consistent(dirpath)
+    except FileCheckException:
+        logger.warning(f"The directory {dirpath} is not yours.")
+    
+
 def check_file_or_directory_path(path, isdir=False):
     """
     Function Description:
@@ -344,7 +371,7 @@ def load_yaml(yaml_path):
 def load_npy(filepath):
     check_file_or_directory_path(filepath)
     try:
-        npy = np.load(filepath)
+        npy = np.load(filepath, allow_pickle=False)
     except Exception as e:
         logger.error(f"The numpy file failed to load. Please check the path: {filepath}.")
         raise RuntimeError(f"Load numpy file {filepath} failed.") from e
@@ -354,7 +381,7 @@ def load_npy(filepath):
 def load_json(json_path):
     try:
         with FileOpen(json_path, "r") as f:
-            fcntl.flock(f, fcntl.LOCK_EX)
+            fcntl.flock(f, fcntl.LOCK_SH)
             data = json.load(f)
             fcntl.flock(f, fcntl.LOCK_UN)
     except Exception as e:
@@ -363,11 +390,11 @@ def load_json(json_path):
     return data
 
 
-def save_json(json_path, data, indent=None):
-    json_path = os.path.realpath(json_path)
+def save_json(json_path, data, indent=None, mode="w"):
     check_path_before_create(json_path)
+    json_path = os.path.realpath(json_path)
     try:
-        with FileOpen(json_path, 'w') as f:
+        with FileOpen(json_path, mode) as f:
             fcntl.flock(f, fcntl.LOCK_EX)
             json.dump(data, f, indent=indent)
             fcntl.flock(f, fcntl.LOCK_UN)
@@ -378,8 +405,8 @@ def save_json(json_path, data, indent=None):
 
 
 def save_yaml(yaml_path, data):
-    yaml_path = os.path.realpath(yaml_path)
     check_path_before_create(yaml_path)
+    yaml_path = os.path.realpath(yaml_path)
     try:
         with FileOpen(yaml_path, 'w') as f:
             fcntl.flock(f, fcntl.LOCK_EX)
@@ -391,6 +418,21 @@ def save_yaml(yaml_path, data):
     change_mode(yaml_path, FileCheckConst.DATA_FILE_AUTHORITY)
 
 
+def save_excel(path, data):
+    check_path_before_create(path)
+    path = os.path.realpath(path)
+    try:
+        if isinstance(data, pd.DataFrame):
+            data.to_excel(path, index=False)
+        else:
+            logger.error(f'unsupported data type.')
+            return
+    except Exception as e:
+        logger.error(f'Save excel file "{os.path.basename(path)}" failed.')
+        raise RuntimeError(f"Save excel file {path} failed.") from e
+    change_mode(path, FileCheckConst.DATA_FILE_AUTHORITY)
+
+
 def move_file(src_path, dst_path):
     check_file_or_directory_path(src_path)
     check_path_before_create(dst_path)
@@ -403,8 +445,8 @@ def move_file(src_path, dst_path):
 
 
 def save_npy(data, filepath):
-    filepath = os.path.realpath(filepath)
     check_path_before_create(filepath)
+    filepath = os.path.realpath(filepath)
     try:
         np.save(filepath, data)
     except Exception as e:
@@ -425,6 +467,7 @@ def save_npy_to_txt(data, dst_file='', align=0):
         pad_array = np.zeros((align - data.size % align,))
         data = np.append(data, pad_array)
     check_path_before_create(dst_file)
+    dst_file = os.path.realpath(dst_file)
     try:
         np.savetxt(dst_file, data.reshape((-1, align)), delimiter=' ', fmt='%g')
     except Exception as e:
@@ -438,8 +481,8 @@ def save_workbook(workbook, file_path):
     workbook: 要保存的工作簿对象
     file_path: 文件保存路径
     """
-    file_path = os.path.realpath(file_path)
     check_path_before_create(file_path)
+    file_path = os.path.realpath(file_path)
     try:
         workbook.save(file_path)
     except Exception as e:
@@ -451,7 +494,7 @@ def save_workbook(workbook, file_path):
 def write_csv(data, filepath, mode="a+", malicious_check=False):
     def csv_value_is_valid(value: str) -> bool:
         if not isinstance(value, str):
-            return True  
+            return True
         try:
             # -1.00 or +1.00 should be consdiered as digit numbers
             float(value)
@@ -459,16 +502,16 @@ def write_csv(data, filepath, mode="a+", malicious_check=False):
             # otherwise, they will be considered as formular injections
             return not bool(re.compile(FileCheckConst.CSV_BLACK_LIST).search(value))
         return True
-    
+
     if malicious_check:
         for row in data:
             for cell in row:
                 if not csv_value_is_valid(cell):
-                    raise RuntimeError(f"Malicious value [{cell}] is not allowed " \
+                    raise RuntimeError(f"Malicious value [{cell}] is not allowed "
                                        f"to be written into the csv: {filepath}.")
 
-    file_path = os.path.realpath(filepath)
     check_path_before_create(filepath)
+    file_path = os.path.realpath(filepath)
     try:
         with FileOpen(filepath, mode, encoding='utf-8-sig') as f:
             writer = csv.writer(f)
@@ -479,16 +522,54 @@ def write_csv(data, filepath, mode="a+", malicious_check=False):
     change_mode(filepath, FileCheckConst.DATA_FILE_AUTHORITY)
 
 
-def read_csv(filepath):
+def read_csv(filepath, as_pd=True):
     check_file_or_directory_path(filepath)
     try:
-        csv_data = pd.read_csv(filepath)
+        if as_pd:
+            csv_data = pd.read_csv(filepath)
+        else:
+            with FileOpen(filepath, 'r', encoding='utf-8-sig') as f:
+                csv_reader = csv.reader(f, delimiter=',')
+                csv_data = list(csv_reader)
     except Exception as e:
         logger.error(f"The csv file failed to load. Please check the path: {filepath}.")
         raise RuntimeError(f"Read csv file {filepath} failed.") from e
     return csv_data
 
 
+def write_df_to_csv(data, filepath, mode="w", header=True, malicious_check=False):
+    def csv_value_is_valid(value: str) -> bool:
+        if not isinstance(value, str):
+            return True
+        try:
+            # -1.00 or +1.00 should be consdiered as digit numbers
+            float(value)
+        except ValueError:
+            # otherwise, they will be considered as formular injections
+            return not bool(re.compile(FileCheckConst.CSV_BLACK_LIST).search(value))
+        return True
+
+    if not isinstance(data, pd.DataFrame):
+        raise ValueError("The data type of data is not supported. Only support pd.DataFrame.")
+
+    if malicious_check:
+        for i in range(len(data)):
+            for j in range(len(data.columns)):
+                cell = data.iloc[i, j]
+                if not csv_value_is_valid(cell):
+                    raise RuntimeError(f"Malicious value [{cell}] is not allowed "
+                                       f"to be written into the csv: {filepath}.")
+
+    check_path_before_create(filepath)
+    file_path = os.path.realpath(filepath)
+    try:
+        data.to_csv(filepath, mode=mode, header=header, index=False)
+    except Exception as e:
+        logger.error(f'Save csv file "{os.path.basename(file_path)}" failed')
+        raise RuntimeError(f"Save csv file {file_path} failed.") from e
+    change_mode(filepath, FileCheckConst.DATA_FILE_AUTHORITY)
+
+
 def remove_path(path):
     if not os.path.exists(path):
         return
@@ -521,3 +602,46 @@ def get_json_contents(file_path):
 def get_file_content_bytes(file):
     with FileOpen(file, 'rb') as file_handle:
         return file_handle.read()
+
+
+# 对os.walk设置遍历深度
+def os_walk_for_files(path, depth):
+    res = []
+    for root, _, files in os.walk(path, topdown=True):
+        check_file_or_directory_path(root, isdir=True)
+        if root.count(os.sep) - path.count(os.sep) >= depth:
+            _[:] = []
+        else:
+            for file in files:
+                res.append({"file": file, "root": root})
+    return res
+
+
+def check_crt_valid(pem_path):
+    """
+    Check the validity of the SSL certificate.
+
+    Load the SSL certificate from the specified path, parse and check its validity period.
+    If the certificate is expired or invalid, raise a RuntimeError.
+
+    Parameters:
+    pem_path (str): The file path of the SSL certificate.
+
+    Raises:
+    RuntimeError: If the SSL certificate is invalid or expired.
+    """
+    try:
+        with FileOpen(pem_path, "r") as f:
+            pem_data = f.read()
+        cert = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, pem_data)
+        pem_start = parser.parse(cert.get_notBefore().decode("UTF-8"))
+        pem_end = parser.parse(cert.get_notAfter().decode("UTF-8"))
+        logger.info(f"The SSL certificate passes the verification and the validity period "
+                    f"starts from {pem_start} ends at {pem_end}.")
+    except Exception as e:
+        logger.error("Failed to parse the SSL certificate. Check the certificate.")
+        raise RuntimeError(f"The SSL certificate is invalid, {pem_path}") from e
+
+    now_utc = datetime.now(tz=timezone.utc)
+    if cert.has_expired() or not (pem_start <= now_utc <= pem_end):
+        raise RuntimeError(f"The SSL certificate has expired and needs to be replaced, {pem_path}")

msprobe/core/common/inplace_op_checker.py

@@ -1,3 +1,18 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import os
 from msprobe.core.common.file_utils import load_yaml