mindstudio-probe 8.3.2__py3-none-any.whl → 26.0.0a1__py3-none-any.whl

msprobe/infer/utils/check/path_checker.py

@@ -0,0 +1,249 @@
+# -------------------------------------------------------------------------
+#  This file is part of the MindStudio project.
+# Copyright (c) 2025 Huawei Technologies Co.,Ltd.
+#
+# MindStudio is licensed under Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
+#
+#          http://license.coscl.org.cn/MulanPSL2
+#
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+# See the Mulan PSL v2 for more details.
+# -------------------------------------------------------------------------
+
+import os
+from enum import Enum, auto, unique
+from typing import Union
+
+from msprobe.infer.utils.check.checker import Checker, CheckResult, rule, EnumInstance
+
+
+@unique
+class FileType(Enum):
+    DIRECTORY = auto()
+    CHARACTER = auto()
+    BLOCK = auto()
+    FILE = auto()
+    FIFO = auto()
+    SYMLINK = auto()
+    SOCKET = auto()
+
+
+class FileStatus(object):
+    def __init__(self, file_name: str) -> None:
+        file_status = os.lstat(file_name)
+        self._file_name = file_name
+        self.status_mode = file_status.st_mode
+        self._file_uid = file_status.st_uid
+        self._file_gid = file_status.st_gid
+        self._file_size = file_status.st_size
+        self._file_extension = os.path.splitext(file_name)[1]
+
+    @property
+    def file_name(self) -> int:
+        return self._file_name
+
+    @property
+    def size(self) -> int:
+        return self._file_size
+
+    @property
+    def perm_bits(self) -> int:
+        return os.st.S_IMODE(self.status_mode)
+
+    @property
+    def uid(self) -> int:
+        return self._file_uid
+
+    @property
+    def gid(self) -> int:
+        return self._file_gid
+
+    @property
+    def extension(self) -> str:
+        return self._file_extension
+
+    @property
+    def ftype(self):
+        file_type_map = {
+            os.st.S_IFDIR: FileType.DIRECTORY,
+            os.st.S_IFCHR: FileType.CHARACTER,
+            os.st.S_IFBLK: FileType.BLOCK,
+            os.st.S_IFREG: FileType.FILE,
+            os.st.S_IFIFO: FileType.FIFO,
+            os.st.S_IFLNK: FileType.SYMLINK,
+            os.st.S_IFSOCK: FileType.SOCKET,
+        }
+
+        file_mode = os.st.S_IFMT(self.status_mode)
+        return file_type_map.get(file_mode, None)
+
+
+class PathChecker(Checker):
+    def __init__(self, instance=EnumInstance.NO_INSTANCE, converter=None):
+        super().__init__(instance, converter)
+        self.f_status = None
+        self.f_state = False
+        self.converter = converter or self.path_converter
+        self.status_err_msg = None
+
+    def path_converter(self, ori_path):
+        ori_path = os.path.realpath(ori_path)
+        try:
+            self.f_status = FileStatus(ori_path)
+        except OSError as e:
+            self.status_err_msg = e.strerror + ': ' + e.filename
+        except TypeError:
+            self.status_err_msg = f'TypeError: {ori_path}'
+        except Exception as e:
+            self.status_err_msg = str(e)
+        else:
+            self.f_state = True
+
+        return ori_path, True, self.status_err_msg
+
+    @rule()
+    def exists(self) -> Union["PathChecker", CheckResult]:
+        err_msg = f"No such file or directory: {self.instance}. {self.status_err_msg}"
+        return self.f_state, err_msg
+
+    @rule()
+    def is_file(self) -> Union["PathChecker", CheckResult]:
+        if not self.f_state:
+            return False, self.status_err_msg
+        else:
+            return self.f_status.ftype is FileType.FILE, f"Not a file: {self.instance}"
+
+    @rule()
+    def is_dir(self) -> Union["PathChecker", CheckResult]:
+        if not self.f_state:
+            return False, self.status_err_msg
+        else:
+            return self.f_status.ftype is FileType.DIRECTORY, f"Not a directory: {self.instance}"
+
+    @rule()
+    def is_softlink(self) -> Union["PathChecker", CheckResult]:
+        if not self.f_state:
+            return False, self.status_err_msg
+        else:
+            return self.f_status.ftype is FileType.SYMLINK, f"Not a soft link: {self.instance}"
+
+    @rule()
+    def forbidden_softlink(self, flag=True) -> Union["PathChecker", CheckResult]:
+        if not self.f_state:
+            return False, self.status_err_msg
+        if flag:
+            return self.f_status.ftype is not FileType.SYMLINK, f"Soft link: {self.instance}"
+        else:
+            return True, "Soft link check passed."
+
+    @rule()
+    def is_uid_matched(self, *uids: int) -> Union["PathChecker", CheckResult]:
+        if not self.f_state:
+            return False, self.status_err_msg
+        else:
+            return (
+                self.f_status.uid in uids,
+                f"User ID not matched: {self.instance}[{self.f_status.uid} ∉ {str(uids)}]. ",
+            )
+
+    @rule()
+    def is_owner(self, *uids: int) -> Union["PathChecker", CheckResult]:
+        if not self.f_state:
+            return False, self.status_err_msg
+        else:
+            return (
+                os.getuid() == 0 or self.f_status.uid == os.getuid(),
+                f"User ID not matched: {self.instance}[{self.f_status.uid} ∉ {str(uids)}]. ",
+            )
+
+    @rule()
+    def is_gid_matched(self, *gids: int) -> Union["PathChecker", CheckResult]:
+        if not self.f_state:
+            return False, self.status_err_msg
+        else:
+            return (
+                self.f_status.gid in gids,
+                f"Group ID not matched: {self.instance}[{self.f_status.gid} ∉ {str(gids)}]. ",
+            )
+
+    @rule()
+    def is_readable(self) -> Union["PathChecker", CheckResult]:
+        return os.access(self.instance, os.R_OK), self.instance + " is not readable"
+
+    @rule()
+    def is_writeable(self) -> Union["PathChecker", CheckResult]:
+        return os.access(self.instance, os.W_OK), self.instance + " is not writable"
+
+    @rule()
+    def is_executable(self) -> Union["PathChecker", CheckResult]:
+        return os.access(self.instance, os.X_OK), self.instance + " is not executable"
+
+    @rule()
+    def is_not_readable_to_others(self) -> Union["PathChecker", CheckResult]:
+        ins = self.instance + " is readable to others"
+        return CheckResult(not bool(self.f_status.status_mode & os.st.S_IROTH), ins)
+
+    @rule()
+    def is_not_writable_to_group(self) -> Union["PathChecker", CheckResult]:
+        ins = self.instance + " is writable to groups"
+        return CheckResult(not bool(self.f_status.status_mode & os.st.S_IWGRP), ins)
+
+    @rule()
+    def is_not_writable_to_others(self) -> Union["PathChecker", CheckResult]:
+        ins = self.instance + " is writable to others"
+        return CheckResult(not bool(self.f_status.status_mode & os.st.S_IWOTH), ins)
+
+    @rule()
+    def is_not_executable_to_others(self) -> Union["PathChecker", CheckResult]:
+        ins = self.instance + " is executable to others"
+        return CheckResult(not bool(self.f_status.status_mode & os.st.S_IXOTH), ins)
+
+    @rule()
+    def max_perm(self, perm_bits: int) -> Union["PathChecker", CheckResult]:
+        if 0o777 < perm_bits or perm_bits < 0:
+            msg = "Permission bits should be in range from 0 to 0o777"
+            raise ValueError(f"{msg}")
+
+        part_mapping = ["others", "groups", "users"]
+        perm_mapping = ["executing", "writing", "reading"]
+
+        for count in range(9):
+            mask = 1 << count
+
+            if (self.f_status.perm_bits & mask) and not (perm_bits & mask):
+                err_msg = (
+                    f"{part_mapping[count // 3]} "
+                    f"should not have {perm_mapping[count % 3]} "
+                    f"permissions: {self.instance}"
+                )
+
+                return CheckResult(False, err_msg)
+
+        return CheckResult(True)
+
+    @rule("file size larger than expected")
+    def max_size(self, expected_size: int) -> Union["PathChecker", CheckResult]:
+        err_msg = f"File size larger than expected: {self.instance}"
+        return CheckResult(self.f_status.size < expected_size, err_msg)
+
+    @rule("Wrong file suffix")
+    def check_extensions(self, extensions) -> Union["PathChecker", CheckResult]:
+        return self.f_status.extension == extensions or self.f_status.extension == '.' + extensions
+
+    @rule()
+    def is_safe_parent_dir(self) -> Union["PathChecker", CheckResult]:
+        path = os.path.realpath(self.instance)
+        dirpath = os.path.dirname(path)
+        if os.getuid() == 0:
+            return True
+
+        dir_checker = PathChecker().any(
+            PathChecker().anti(PathChecker().exists()),
+            PathChecker().is_dir().is_owner(os.getuid()).is_not_writable_to_others().is_not_writable_to_group(),
+        )
+        return dir_checker.check(dirpath)
+

msprobe/infer/utils/check/rule.py

@@ -0,0 +1,126 @@
+# -------------------------------------------------------------------------
+#  This file is part of the MindStudio project.
+# Copyright (c) 2025 Huawei Technologies Co.,Ltd.
+#
+# MindStudio is licensed under Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
+#
+#          http://license.coscl.org.cn/MulanPSL2
+#
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+# See the Mulan PSL v2 for more details.
+# -------------------------------------------------------------------------
+
+from typing import Any
+
+from msprobe.infer.utils.check import NumberChecker, ObjectChecker, StringChecker, DictChecker, PathChecker, ListChecker
+from msprobe.infer.utils.check.checker import Checker
+from msprobe.infer.utils.constants import INPUT_FILE_MAX_SIZE
+
+
+class NumConverter:
+    def __init__(self, convert_type=float):
+        self.convert_type = convert_type
+
+    def __call__(self, value) -> Any:
+        return self.convert(value)
+
+    def convert(self, value: str):
+        try:
+            if self.convert_type == int:
+                return int(value), True, ""
+            elif self.convert_type == float:
+                return float(value), True, ""
+            else:
+                return value, True, ""
+        except ValueError as er:
+            return value, False, str(er)
+
+
+class Rule:
+    @staticmethod
+    def none() -> Checker:
+        return Checker().is_none()
+
+    @staticmethod
+    def num() -> NumberChecker:
+        return NumberChecker().is_number()
+
+    @staticmethod
+    def str() -> StringChecker:
+        return StringChecker().is_str()
+
+    @staticmethod
+    def dict() -> DictChecker:
+        return DictChecker().is_dict()
+
+    @staticmethod
+    def obj(obj_type) -> ObjectChecker:
+        return ObjectChecker().is_type(obj_type)
+
+    @staticmethod
+    def path() -> PathChecker:
+        return PathChecker()
+
+    @staticmethod
+    def list() -> ListChecker:
+        return ListChecker().is_list()
+
+    @staticmethod
+    def config_file() -> PathChecker:
+        return (
+            PathChecker()
+            .exists()
+            .is_file()
+            .is_readable()
+            .is_not_writable_to_others()
+            .is_safe_parent_dir()
+            .max_size(10 * 1000 * 1000)
+            .as_default()
+        )
+
+    @staticmethod
+    def input_file() -> PathChecker:
+        return (
+            PathChecker()
+            .exists()
+            .forbidden_softlink()
+            .is_file()
+            .is_readable()
+            .is_owner()
+            .is_not_writable_to_others()
+            .is_safe_parent_dir()
+            .max_size(INPUT_FILE_MAX_SIZE)
+            .as_default()
+        )
+
+    @staticmethod
+    def input_dir() -> PathChecker:
+        return PathChecker().exists().is_dir().is_readable().is_uid_matched().is_not_writable_to_others().as_default()
+
+    @staticmethod
+    def output_dir() -> PathChecker:
+        return (
+            Rule.path()
+            .any(Rule.anti(PathChecker().exists()), PathChecker().is_dir().is_writeable().is_not_writable_to_others())
+            .as_default()
+        )
+
+    @staticmethod
+    def any(*rules: Checker) -> Checker:
+        return Checker().any(*rules)
+
+    @staticmethod
+    def anti(rule: Checker) -> Checker:
+        return Checker().anti(rule)
+
+    @staticmethod
+    def to_int() -> NumberChecker:
+        return NumberChecker(converter=NumConverter(int))
+
+    @staticmethod
+    def to_float() -> NumberChecker:
+        return NumberChecker(converter=NumConverter(float))

msprobe/infer/utils/check/string_checker.py

@@ -0,0 +1,66 @@
+# -------------------------------------------------------------------------
+#  This file is part of the MindStudio project.
+# Copyright (c) 2025 Huawei Technologies Co.,Ltd.
+#
+# MindStudio is licensed under Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
+#
+#          http://license.coscl.org.cn/MulanPSL2
+#
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+# See the Mulan PSL v2 for more details.
+# -------------------------------------------------------------------------
+
+import re
+import os
+from typing import Union
+
+from msprobe.infer.utils.check.checker import Checker, CheckResult, rule
+
+
+WHITE_LIST_PATTERN = r"^(?!.*\.\.)(?!:)[a-zA-Z0-9_./:-]+$"
+BLACK_LIST_PATTERN = r"[^_A-Za-z0-9/.-]"
+IDS_PATTERN = r'^(\d+(?:_\d+)*)(,\d+(?:_\d+)*)*$'
+INVALID_CHAR = "[\n\f\r\b\t\v\u000D\u000A\u000C\u000B\u0009\u0008\u007F&%$*^#@;]"
+
+
+class StringChecker(Checker):
+
+    @rule()
+    def is_str(self) -> Union["StringChecker", CheckResult]:
+        err_msg = f"{self.instance} is not a string"
+        return isinstance(self.instance, str), err_msg
+
+    @rule()
+    def is_file_name_too_long(self) -> Union["StringChecker", CheckResult]:
+        err_msg = "File name too long"
+        ret = len(self.instance) > 4095 or any(map(lambda s: len(s) > 255, self.instance.split(os.path.sep)))
+        return not ret, err_msg
+
+    @rule()
+    def is_str_safe(self) -> Union["StringChecker", CheckResult]:
+        err_msg = "String parameter contains invalid characters"
+        return re.search(WHITE_LIST_PATTERN, self.instance), err_msg
+    
+    @rule()
+    def is_str_valid_bool(self) -> Union["StringChecker", CheckResult]:
+        err_msg = "Boolean value expected 'yes', 'y', 'Y', 'YES', 'true', 't', 'TRUE', 'True', '1' for true"
+        return self.instance.lower() in ('yes', 'y', 'Y', 'YES', 'true', 't', 'TRUE', 'True', '1'), err_msg
+    
+    @rule()
+    def is_str_valid_path(self) -> Union["StringChecker", CheckResult]:
+        err_msg = "Input path contains invalid characters"
+        return not re.search(BLACK_LIST_PATTERN, self.instance), err_msg
+    
+    @rule()
+    def is_str_valid_ids(self):
+        err_msg = f"dym range string \"{self.instance}\" is not a legal string"
+        return re.match(IDS_PATTERN, self.instance), err_msg
+
+    @rule()
+    def str_has_no_invalid_char(self):
+        err_msg = "Input string contains invalid chars" 
+        return not re.search(INVALID_CHAR, self.instance), err_msg

msprobe/infer/utils/cmp_algorithm.py

@@ -0,0 +1,261 @@
+# -------------------------------------------------------------------------
+#  This file is part of the MindStudio project.
+# Copyright (c) 2025 Huawei Technologies Co.,Ltd.
+#
+# MindStudio is licensed under Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
+#
+#          http://license.coscl.org.cn/MulanPSL2
+#
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+# See the Mulan PSL v2 for more details.
+# -------------------------------------------------------------------------
+
+import torch
+import numpy as np
+from torch.nn import functional as F
+from msprobe.core.common.log import logger
+
+FLOAT_EPSILON = torch.finfo(torch.float).eps
+NP_FLOAT_EPSILON = np.finfo(np.float32).eps
+NAN = 'NaN'
+
+
+def cosine_similarity(golden_data: torch.Tensor, my_data: torch.Tensor):
+    if torch.all(golden_data == 0) and torch.all(my_data == 0):
+        return 1.0, ''  # both are all 0, return similarity 1
+
+    result = torch.cosine_similarity(golden_data.double(), my_data.double(), dim=0).item()  # Torch handle zero data
+    return round(result, 10), ''  # Trunc to keeping only 10 decimals
+
+
+def np_cosine_similarity(golden_data: np.ndarray, my_data: np.ndarray):
+    if np.all(golden_data == 0) and np.all(my_data == 0):
+        return 1.0, ''  # both are all 0, return similarity 1
+
+    result = np.dot(golden_data, my_data) / (np.linalg.norm(golden_data) * np.linalg.norm(my_data))
+    return round(result, 10), ''  #
+
+
+def max_relative_error(golden_data: torch.Tensor, my_data: torch.Tensor):
+    result = torch.where(
+        torch.abs(golden_data) > FLOAT_EPSILON,
+        torch.abs(my_data / golden_data - 1),  # abs(aa - bb) / abs(bb) -> abs(aa / bb - 1)
+        torch.tensor(0, dtype=golden_data.dtype),
+    ).max()
+    return result.item(), ''
+
+
+def np_max_relative_error(golden_data: np.ndarray, my_data: np.ndarray):
+    result = np.max(
+        np.where(
+            np.abs(golden_data) > NP_FLOAT_EPSILON,
+            np.abs(my_data / golden_data - 1),  # abs(aa - bb) / abs(bb) -> abs(aa / bb - 1)
+            0
+        )
+    )
+    return result, ''
+
+
+def mean_relative_error(golden_data: torch.Tensor, my_data: torch.Tensor):
+    result = torch.where(
+        torch.abs(golden_data) > FLOAT_EPSILON,
+        torch.abs(my_data / golden_data - 1),  # abs(aa - bb) / abs(bb) -> abs(aa / bb - 1)
+        torch.tensor(0, dtype=my_data.dtype),
+    ).mean()
+    return result.item(), ''
+
+
+def np_mean_relative_error(golden_data: np.ndarray, my_data: np.ndarray):
+    result = np.mean(
+        np.where(
+            np.abs(golden_data) > NP_FLOAT_EPSILON,
+            np.abs(my_data / golden_data - 1),  # abs(aa - bb) / abs(bb) -> abs(aa / bb - 1)
+            0
+        )
+    )
+    return result, ''
+
+
+def max_absolute_error(golden_data: torch.Tensor, my_data: torch.Tensor):
+    result = torch.where(
+        torch.abs(golden_data) > FLOAT_EPSILON,
+        torch.abs(my_data - golden_data),  # abs(aa - bb) / abs(bb) -> abs(aa / bb - 1)
+        torch.tensor(0, dtype=my_data.dtype),
+    ).max()
+    return result.item(), ''
+
+
+def np_max_absolute_error(golden_data: np.ndarray, my_data: np.ndarray):
+    result = np.max(
+        np.where(
+            np.abs(golden_data) > NP_FLOAT_EPSILON,
+            np.abs(my_data - golden_data),  # abs(aa - bb) / abs(bb) -> abs(aa / bb - 1)
+            0
+        )
+    )
+    return result, ''
+
+
+def mean_absolute_error(golden_data: torch.Tensor, my_data: torch.Tensor):
+    result = torch.where(
+        torch.abs(golden_data) > FLOAT_EPSILON,
+        torch.abs(my_data - golden_data),  # abs(aa - bb) / abs(bb) -> abs(aa / bb - 1)
+        torch.tensor(0, dtype=my_data.dtype),
+    ).mean()
+    return result.item(), ''
+
+
+def np_mean_absolute_error(golden_data: np.ndarray, my_data: np.ndarray):
+    result = np.mean(
+        np.where(
+            np.abs(golden_data) > NP_FLOAT_EPSILON,
+            np.abs(my_data - golden_data),  # abs(aa - bb) / abs(bb) -> abs(aa / bb - 1)
+            0
+        )
+    )
+    return result, ''
+
+
+def kl_divergence(golden_data: torch.Tensor, my_data: torch.Tensor):
+    result = F.kl_div(F.log_softmax(my_data, dim=-1), F.softmax(golden_data, dim=-1), reduction="sum").item()
+    return max(result, 0), ""
+
+
+def np_kl_divergence(golden_data: np.ndarray, my_data: np.ndarray):
+    golden_data = np.exp(golden_data - np.max(golden_data))
+    my_data = np.exp(my_data - np.max(my_data))
+    result = np.sum(golden_data * (np.log(golden_data) - my_data))
+    return max(result, 0), ""
+
+
+def relative_euclidean_distance(golden_data: torch.Tensor, my_data: torch.Tensor):
+    ground_truth_square_num = (golden_data ** 2).sum()
+    if ground_truth_square_num ** 0.5 <= FLOAT_EPSILON:
+        return 0.0, ''
+
+    result = ((my_data - golden_data) ** 2).sum() / ground_truth_square_num
+    return torch.sqrt(result).item(), ''
+
+
+def np_relative_euclidean_distance(golden_data: np.ndarray, my_data: np.ndarray):
+    ground_truth_square_num = np.sum(golden_data ** 2)
+    if np.sqrt(ground_truth_square_num) <= NP_FLOAT_EPSILON:
+        return 0.0, ''
+
+    result = np.sum((my_data - golden_data) ** 2) / ground_truth_square_num
+    return np.sqrt(result), ''
+
+
+def stat_absolute_error(golden_data, my_data):
+    error = abs(my_data - golden_data)
+    return error, ''
+
+
+def stat_relative_error(golden_data, my_data):
+    eps = np.finfo(np.float32).eps  
+    denominator = max(abs(golden_data), eps)
+    error = abs(my_data - golden_data) / denominator
+    return error, ''
+
+
+def l1_norm(golden_data: torch.Tensor, my_data: torch.Tensor):
+    '''
+    Function: compute l1_norm
+    '''
+    golden_data = golden_data.double()
+    my_data = my_data.double()
+    if golden_data.abs().sum() == 0 and my_data.abs().sum() == 0:
+        return 0, ''
+    elif golden_data.abs().sum() == 0:
+        return 1, ''
+    else:
+        logger.debug(f"my data sum: {my_data.abs().sum()}")
+        logger.debug(f"golden data sum: {golden_data.abs().sum()}")
+        result = my_data.abs().sum() / golden_data.abs().sum() - 1
+        return result.item(), ''
+
+
+def register_custom_compare_algorithm(custom_compare_algorithm):
+    import os
+    import sys
+    import importlib
+    import inspect
+    from msprobe.infer.utils.file_open_check import FileStat
+
+    custom_compare_algorithm_split = custom_compare_algorithm.split(':')
+    if len(custom_compare_algorithm_split) != 2:
+        raise ValueError("custom_compare_algorithm should be in format '{python_file_path}:{function_name}'")
+    file_path, func_name = custom_compare_algorithm_split
+    file_path = os.path.expanduser(file_path)
+
+    if not os.path.exists(file_path):
+        raise ValueError(f"custom_compare_algorithm specified {file_path} not exists")
+    if not file_path.endswith(".py"):
+        raise ValueError("custom_compare_algorithm specified {file_path} is not a py file")
+    
+    file_stat = FileStat(file_path)
+    if not file_stat.is_basically_legal('read', strict_permission=False):
+        raise ValueError(f"custom_compare_algorithm specified {file_path} permission stat is illegal")
+
+    file_dir, file_name = os.path.dirname(file_path), os.path.basename(file_path)
+    if len(file_dir) > 0 and file_dir not in sys.path:
+        sys.path.append(file_dir)
+
+    custom_module_name = file_name.replace('.py', '')
+    try:
+        custom_module = importlib.import_module(custom_module_name)
+    except Exception as ee:
+        raise ValueError(f"import {custom_module_name} from {file_dir} failed") from ee
+
+    custom_compare_func = getattr(custom_module, func_name, None)
+    if custom_compare_func is None:
+        raise ValueError(f"getting {func_name} from {custom_compare_algorithm} failed")
+    if len(inspect.signature(custom_compare_func).parameters) != 2:
+        raise ValueError(f"function {func_name} signature should have exact two parameters")
+
+    try:
+        ret = custom_compare_func(torch.ones([1]), torch.ones([1]))
+    except Exception as e:
+        raise ValueError(f"function {func_name} should recieve 2 torch tensor parameters") from e
+
+    if not isinstance(ret, (list, tuple)) or len(ret) != 2:
+        raise ValueError(f"function {func_name} should return 2 value in type ((float, int, str), str)")
+    if not isinstance(ret[0], (float, int, str)) or not isinstance(ret[1], str):
+        raise ValueError(f"function {func_name} should return 2 value in type ((float, int, str), str)")
+
+    logger.info(f"Added custom comparing algorithm: {func_name}")
+    CUSTOM_ALG_MAP[func_name] = custom_compare_func
+
+
+CMP_ALG_MAP = {
+    "cosine_similarity": cosine_similarity,
+    "max_relative_error": max_relative_error,
+    "mean_relative_error": mean_relative_error,
+    "max_absolute_error": max_absolute_error,
+    "mean_absolute_error": mean_absolute_error,
+    "kl_divergence": kl_divergence,
+    "relative_euclidean_distance": relative_euclidean_distance,
+}
+
+
+NP_CMP_ALG_MAP = {
+    "cosine_similarity": np_cosine_similarity,
+    "max_relative_error": np_max_relative_error,
+    "mean_relative_error": np_mean_relative_error,
+    "max_absolute_error": np_max_absolute_error,
+    "mean_absolute_error": np_mean_absolute_error,
+    "kl_divergence": np_kl_divergence,
+    "relative_euclidean_distance": np_relative_euclidean_distance,
+}
+
+CUSTOM_ALG_MAP = {}
+
+
+CMP_STATICTISC_MAP = {
+    "stat_absolute_error": stat_absolute_error,
+    "stat_relative_error": stat_relative_error,
+}