mindstudio-probe 1.0.1__py3-none-any.whl

msprobe/pytorch/advisor/advisor_const.py

@@ -0,0 +1,59 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+# Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+
+
+class AdvisorConst:
+    """
+    Class for advisor const
+    """
+
+    # text symbol
+    NEW_LINE = "\n"
+    COLON = ": "
+
+    # advisor summary key
+    SUSPECT_NODES = "Suspect Nodes"
+    LINE = "Line"
+    ADVISOR_SUGGEST = "Expert Advice"
+
+    NO_ERROR_API = "NA"
+
+    # advisor message
+    NO_ERR_SUGGEST = "All data in comparison result meets the accuracy requirements."
+    FORWARD_INPUT_SUGGEST = "1. Analyze the model to view the input source.\n" \
+                            "2. Check whether an inplace API causes the output result to overwrite the input result. That is, the fault is actually caused by a computation error.\n" \
+                            "3. The fault may be caused by memory corruption and further analysis is required."
+    FORWARD_OUTPUT_SUGGEST = "This is a forward API computation error. Check the computation implementation."
+    BACKWARD_INPUT_SUGGEST = "Check whether the forward computation result is affected."
+    BACKWARD_OUTPUT_SUGGEST = "This is a backward API computation error. Check the computation implementation."
+    BATCH_NORM_SUGGEST = "Torch API batch_norm input not fixed, the following suggestions may fix it:\n" \
+                         "1. If use torch.nn.functional.batch_norm, you can set parameter training=False.\n" \
+                         "2. If use torch.nn.BatchNormXXX, you can set parameter affine=False.\n" \
+                         "3. Use seed_all(mode=True) to enable deterministic computing."
+    DETERMINISTIC_SUGGEST = "This torch api may be uncertainty in the calculation, " \
+                            "can seed_all(mode=True) to enable deterministic computing."
+
+    FUNC_BATCH_NORM = "Functional_batch_norm"
+    FORWARD_INPUT_1 = "forward_input.1"
+    NEED_DETERMINISTIC_API = ["conv2d", "conv3d", "matmul", "nll_loss", "layer_norm", "lstm"]
+    BATCH_NORM = "batch_norm"
+
+    # name keyword
+    INPUT = "input"
+    OUTPUT = "output"
+    FORWARD = "forward"
+    BACKWARD = "backward"

msprobe/pytorch/advisor/advisor_result.py

@@ -0,0 +1,58 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+# Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+import os
+import time
+
+from msprobe.pytorch.advisor.advisor_const import AdvisorConst
+from msprobe.pytorch.common.log import logger
+from msprobe.core.common.const import Const, FileCheckConst
+from msprobe.core.common.file_check import change_mode
+
+
+class AdvisorResult:
+    """
+    Class for generate advisor result
+    """
+
+    def __init__(self, node, line, message):
+        self.suspect_node = node
+        self.line = line
+        self.advisor_message = message
+
+    @staticmethod
+    def gen_summary_file(out_path, message_list):
+        file_name = 'advisor_{}.txt'.format(time.strftime("%Y%m%d%H%M%S", time.localtime(time.time())))
+        result_file = os.path.join(out_path, file_name)
+        try:
+            with os.fdopen(os.open(result_file, Const.WRITE_FLAGS, Const.WRITE_MODES), 'w+') as output_file:
+                output_file.truncate(0)
+                message_list = [message + AdvisorConst.NEW_LINE for message in message_list]
+                output_file.writelines(message_list)
+            change_mode(result_file, FileCheckConst.DATA_FILE_AUTHORITY)
+        except IOError as io_error:
+            logger.error("Failed to save %s, the reason is %s." % (result_file, io_error))
+        else:
+            logger.info("The advisor summary is saved in: %s" % result_file)
+
+    def print_advisor_log(self):
+        logger.info("The summary of the expert advice is as follows: ")
+        message_list = [AdvisorConst.LINE + AdvisorConst.COLON + str(self.line),
+                        AdvisorConst.SUSPECT_NODES + AdvisorConst.COLON + self.suspect_node,
+                        AdvisorConst.ADVISOR_SUGGEST + AdvisorConst.COLON + self.advisor_message]
+        for message in message_list:
+            logger.info(message)
+        return message_list

msprobe/pytorch/api_accuracy_checker/common/config.py

@@ -0,0 +1,50 @@
+import os
+import yaml
+from msprobe.pytorch.api_accuracy_checker.common.utils import check_file_or_directory_path
+from msprobe.pytorch.hook_module.utils import WrapFunctionalOps, WrapTensorOps, WrapTorchOps
+from msprobe.core.common.file_check import FileOpen
+
+WrapApi = set(WrapFunctionalOps) | set(WrapTensorOps) | set(WrapTorchOps)
+
+
+class Config:
+    def __init__(self, yaml_file):
+        check_file_or_directory_path(yaml_file, False)
+        with FileOpen(yaml_file, 'r') as file:
+            config = yaml.safe_load(file)
+        self.config = {key: self.validate(key, value) for key, value in config.items()}
+
+    def __getattr__(self, item):
+        return self.config[item]
+
+    def __str__(self):
+        return '\n'.join(f"{key}={value}" for key, value in self.config.items())
+
+    @staticmethod
+    def validate(key, value):
+        validators = {
+            'white_list': list,
+            'error_data_path': str,
+            'precision': int
+        }
+        if key not in validators:
+            raise ValueError(f"{key} must be one of {validators.keys()}")
+        if not isinstance(value, validators.get(key)):
+            raise ValueError(f"{key} must be {validators[key].__name__} type")
+        if key == 'precision' and value < 0:
+            raise ValueError("precision must be greater than 0")
+        if key == 'white_list':
+            if not isinstance(value, list):
+                raise ValueError("white_list must be a list type")
+            if not all(isinstance(i, str) for i in value):
+                raise ValueError("All elements in white_list must be of str type")
+            invalid_api = [i for i in value if i not in WrapApi]
+            if invalid_api:
+                raise ValueError(
+                    f"{', '.join(invalid_api)} is not in support_wrap_ops.yaml, please check the white_list")
+        return value
+
+
+cur_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
+yaml_path = os.path.join(cur_path, "config.yaml")
+msCheckerConfig = Config(yaml_path)

msprobe/pytorch/api_accuracy_checker/common/utils.py

@@ -0,0 +1,224 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+# Copyright (C) 2023-2023. Huawei Technologies Co., Ltd. All rights reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+import json
+import os
+import re
+import csv
+
+import torch
+
+try:
+    import torch_npu
+except ImportError:
+    IS_GPU = True
+else:
+    IS_GPU = False
+
+from msprobe.pytorch.common.log import logger
+from msprobe.core.common.file_check import FileChecker, FileOpen, change_mode, create_directory
+from msprobe.core.common.const import Const, FileCheckConst
+from msprobe.core.common.utils import CompareException
+
+
+class DumpException(CompareException):
+    pass
+
+
+def write_csv(data, filepath):
+    with FileOpen(filepath, 'a', encoding='utf-8-sig') as f:
+        writer = csv.writer(f)
+        writer.writerows(data)
+
+
+def check_object_type(check_object, allow_type):
+    """
+    Function Description:
+        Check if the object belongs to a certain data type
+    Parameter:
+        check_object: the object to be checked
+        allow_type: legal data type
+    Exception Description:
+        when invalid data throw exception
+    """
+    if not isinstance(check_object, allow_type):
+        logger.error(f"{check_object} not of {allow_type} type")
+        raise CompareException(CompareException.INVALID_DATA_ERROR)
+
+
+def check_file_or_directory_path(path, isdir=False):
+    """
+    Function Description:
+        check whether the path is valid
+    Parameter:
+        path: the path to check
+        isdir: the path is dir or file
+    Exception Description:
+        when invalid data throw exception
+    """
+    if isdir:
+        if not os.path.exists(path):
+            logger.error('The path {} is not exist.'.format(path))
+            raise CompareException(CompareException.INVALID_PATH_ERROR)
+
+        if not os.path.isdir(path):
+            logger.error('The path {} is not a directory.'.format(path))
+            raise CompareException(CompareException.INVALID_PATH_ERROR)
+
+        if not os.access(path, os.W_OK):
+            logger.error(
+                'The path {} does not have permission to write. Please check the path permission'.format(path))
+            raise CompareException(CompareException.INVALID_PATH_ERROR)
+    else:
+        if not os.path.isfile(path):
+            logger.error('{} is an invalid file or non-exist.'.format(path))
+            raise CompareException(CompareException.INVALID_PATH_ERROR)
+
+    if not os.access(path, os.R_OK):
+        logger.error(
+            'The path {} does not have permission to read. Please check the path permission'.format(path))
+        raise CompareException(CompareException.INVALID_PATH_ERROR)
+
+
+def get_json_contents(file_path):
+    ops = get_file_content_bytes(file_path)
+    try:
+        json_obj = json.loads(ops)
+    except ValueError as error:
+        logger.error('Failed to load "%s". %s' % (file_path, str(error)))
+        raise CompareException(CompareException.INVALID_FILE_ERROR) from error
+    if not isinstance(json_obj, dict):
+        logger.error('Json file %s, content is not a dictionary!' % file_path)
+        raise CompareException(CompareException.INVALID_FILE_ERROR)
+    return json_obj
+
+
+def get_file_content_bytes(file):
+    with FileOpen(file, 'rb') as file_handle:
+        return file_handle.read()
+
+
+class SoftlinkCheckException(Exception):
+    pass
+
+
+def check_need_convert(api_name):
+    convert_type = None
+    for key, value in Const.CONVERT_API.items():
+        if api_name not in value:
+            continue
+        else:
+            convert_type = key
+    return convert_type
+
+
+def api_info_preprocess(api_name, api_info_dict):
+    """
+    Function Description:
+        Preprocesses the API information.
+    Parameter:
+        api_name: Name of the API.
+        api_info_dict: argument of the API.
+    Return api_info_dict:
+        convert_type: Type of conversion.
+        api_info_dict: Processed argument of the API.
+    """
+    convert_type = check_need_convert(api_name)
+    if api_name == 'cross_entropy':
+        api_info_dict = cross_entropy_process(api_info_dict)
+    return convert_type, api_info_dict
+
+
+def cross_entropy_process(api_info_dict):
+    """
+    Function Description:
+        Preprocesses the cross_entropy API information.
+    Parameter:
+        api_info_dict: argument of the API.
+    Return api_info_dict:
+        api_info_dict: Processed argument of the API.
+    """
+    if 'args' in api_info_dict and len(api_info_dict['args']) > 1 and 'Min' in api_info_dict['args'][1]:
+        if api_info_dict['args'][1]['Min'] <= 0:
+            # The second argument in cross_entropy should be -100 or not less than 0
+            api_info_dict['args'][1]['Min'] = 0
+    return api_info_dict
+
+
+def initialize_save_path(save_path, dir_name):
+    data_path = os.path.join(save_path, dir_name)
+    if os.path.exists(data_path):
+        logger.warning(f"{data_path} already exists, it will be overwritten")
+    else:
+        os.mkdir(data_path, mode=FileCheckConst.DATA_DIR_AUTHORITY)
+    data_path_checker = FileChecker(data_path, FileCheckConst.DIR)
+    data_path_checker.common_check()
+
+
+def write_pt(file_path, tensor):
+    if os.path.exists(file_path):
+        raise ValueError(f"File {file_path} already exists")
+    torch.save(tensor, file_path)
+    full_path = os.path.realpath(file_path)
+    change_mode(full_path, FileCheckConst.DATA_FILE_AUTHORITY)
+    return full_path
+
+
+def get_real_data_path(file_path):
+    targets = ['forward_real_data', 'backward_real_data', 'ut_error_data\d+']
+    pattern = re.compile(r'({})'.format('|'.join(targets)))
+    match = pattern.search(file_path)
+    if match:
+        target_index = match.start()
+        target_path = file_path[target_index:]
+        return target_path
+    else:
+        raise DumpException(DumpException.INVALID_PATH_ERROR)
+
+
+def get_full_data_path(data_path, real_data_path):
+    if not data_path:
+        return data_path
+    full_data_path = os.path.join(real_data_path, data_path)
+    return os.path.realpath(full_data_path)
+
+
+class UtDataProcessor:
+    def __init__(self, save_path):
+        self.save_path = save_path
+        self.index = 0
+
+    def save_tensors_in_element(self, api_name, element):
+        self.index = 0
+        self._save_recursive(api_name, element)
+
+    def _save_recursive(self, api_name, element):
+        if isinstance(element, torch.Tensor):
+            api_args = api_name + Const.SEP + str(self.index)
+            create_directory(self.save_path)
+            file_path = os.path.join(self.save_path, f'{api_args}.pt')
+            write_pt(file_path, element.contiguous().cpu().detach())
+            self.index += 1
+        elif element is None or isinstance(element, (bool, int, float, str, slice)):
+            self.index += 1
+        elif isinstance(element, (list, tuple)):
+            for item in element:
+                self._save_recursive(api_name, item)
+        elif isinstance(element, dict):
+            for value in element.values():
+                self._save_recursive(api_name, value)
+        else:
+            self.index += 1

msprobe/pytorch/api_accuracy_checker/compare/algorithm.py

@@ -0,0 +1,216 @@
+# 定义比对算法及比对标准
+import torch
+import numpy as np
+
+from msprobe.pytorch.api_accuracy_checker.compare.compare_utils import ULP_PARAMETERS
+from msprobe.core.common.const import CompareConst
+
+
+DEFAULT_THRESHOLD = 1
+
+
+#cos
+def cosine_sim(bench_output, device_output):
+    msg = ""
+    n_value = device_output.reshape(-1)
+    b_value = bench_output.reshape(-1)
+    cos = CompareConst.SPACE
+    np.seterr(divide="ignore", invalid="ignore")
+    if n_value.shape != b_value.shape:
+        msg = f"Shape of device and bench outputs don't match. device: {n_value.shape}, bench: {b_value.shape}."
+        return -1, False, msg
+    if len(n_value) == 1:
+        msg = "All the data in device dump data is scalar. Please refer to other compare algorithms."
+        return cos, True, msg
+    n_value_max = np.max(np.abs(n_value))
+    b_value_max = np.max(np.abs(b_value))
+    if n_value_max <= np.finfo(float).eps and b_value_max <= np.finfo(float).eps:
+        msg = "All the data in device and bench outputs are zero."
+        return cos, True, msg
+    elif n_value_max <= np.finfo(float).eps:
+        msg = "All the data is zero in device dump data."
+        return CompareConst.SPACE, False, msg
+    elif b_value_max <= np.finfo(float).eps:
+        msg = "All the data is zero in bench dump data."
+        return CompareConst.SPACE, False, msg
+    else:
+        n_value = n_value.astype(float) / n_value_max
+        b_value = b_value.astype(float) / b_value_max
+        cos = np.dot(n_value, b_value) / (np.linalg.norm(n_value) * np.linalg.norm(b_value))
+        if np.isnan(cos):
+            msg = "Dump data has NaN when comparing with Cosine Similarity."
+        cos = np.clip(cos, -1, 1)
+        return cos, cos > 0.99, msg
+
+
+#rmse
+def get_rmse(abs_err, inf_nan_mask):
+    masked_ae = np.where(inf_nan_mask, 0, abs_err)
+    mse = np.mean(np.square(masked_ae))
+    inf_nan_cnt = np.sum(inf_nan_mask)
+    mse = mse * (abs_err.size / (abs_err.size - inf_nan_cnt + 0.0001) + 0.0001)
+    rmse = np.sqrt(mse)
+    return rmse
+
+
+#误差均衡性
+def get_error_balance(bench_data, device_data):
+    larger_count = np.sum(np.greater(device_data - bench_data.astype(device_data.dtype), 0))
+    smaller_count = np.sum(np.less(device_data - bench_data.astype(device_data.dtype), 0))
+    total_count = bench_data.size
+    error_balance = abs(larger_count - smaller_count) / total_count if total_count > 0 else 0
+    return error_balance
+
+
+#小值域错误占比
+def get_small_value_err_ratio(small_value_mask, abs_err_greater_mask):
+    err_mask = np.logical_and(small_value_mask, abs_err_greater_mask)
+    small_value_err_num = np.sum(err_mask)
+    small_value_num = np.sum(small_value_mask)
+    return 0 if small_value_num == 0 else small_value_err_num / small_value_num
+
+
+def get_rel_err(abs_err, abs_bench_with_eps, small_value_mask, inf_nan_mask):
+    rel_err_tmp = abs_err / abs_bench_with_eps
+    rel_err_mask = np.logical_or(small_value_mask, inf_nan_mask)
+    rel_err = np.where(rel_err_mask, -1, rel_err_tmp)
+    return rel_err
+
+
+def get_abs_err(bench_data, device_data):
+    abs_err = np.abs(device_data - bench_data)
+    return abs_err
+
+
+def get_rel_err_origin(abs_err, b_value):
+    rel_err_origin = np.abs(abs_err / b_value)
+    return rel_err_origin
+
+
+def get_max_abs_err(abs_err):
+    max_abs_err = abs_err.max()
+    bool_result = max_abs_err < 0.001
+    return max_abs_err, bool_result
+
+
+#相对误差最大值
+def get_max_rel_err(rel_err):
+    return np.max(rel_err) if np.max(rel_err) >= 0 else 0
+
+
+#相对误差均值
+def get_mean_rel_err(rel_err):
+    non_negative_rel_err = rel_err[rel_err >= 0]
+    return np.mean(non_negative_rel_err) if non_negative_rel_err.size > 0 else 0
+
+
+def get_rel_err_ratio(rel_err, thresholding):
+    if np.size(rel_err) == 0:
+        ratio = 1
+    else:
+        ratio = np.divide(np.sum(rel_err < thresholding), np.size(rel_err))
+    bool_result = ratio > (1 - thresholding)
+    return ratio, bool_result
+
+
+def get_finite_and_infinite_mask(bench_output, device_output):
+    device_finite_mask = np.isfinite(device_output)
+    bench_finite_mask = np.isfinite(bench_output.astype(device_output.dtype))
+    both_finite_mask = np.logical_and(device_finite_mask, bench_finite_mask)
+    inf_nan_mask = np.logical_not(both_finite_mask)
+    return both_finite_mask, inf_nan_mask
+
+
+def get_small_value_mask(abs_bench, both_finite_mask, small_value_threshold):
+    small_value_mask = np.less_equal(abs_bench, small_value_threshold)
+    small_value_mask = np.logical_and(small_value_mask, both_finite_mask)
+    return small_value_mask
+
+
+def get_abs_bench_with_eps(bench, dtype):
+    abs_bench = np.abs(bench)
+    eps = np.finfo(bench.dtype).eps if dtype != torch.bfloat16 else CompareConst.BFLOAT16_EPS
+    abs_bench_with_eps = abs_bench + eps
+    return abs_bench, abs_bench_with_eps
+
+
+def check_inf_nan_value(inf_nan_mask, bench_output, device_output, dtype, rtol):
+    '''
+    新精度标准的绝对阈值法中，检查npu和golden输出的inf、nan是否一致
+    输入：
+        inf_nan_mask：npu输出和golden输出的inf、nan的mask
+        bench_output：golden输出
+        device_output：npu输出
+        dtype：npu输出的dtype
+    输出： 
+        inf_nan_err_ratio：npu输出和golden输出的inf、nan不一致的比例
+    '''
+    abs_gpu, abs_gpu_with_eps = get_abs_bench_with_eps(bench_output, dtype)
+    golden_same_dtype = bench_output.astype(device_output.dtype)
+    a_min = np.finfo(device_output.dtype).min if dtype != torch.bfloat16 else CompareConst.BFLOAT16_MIN
+    a_max = np.finfo(device_output.dtype).max if dtype != torch.bfloat16 else CompareConst.BFLOAT16_MAX
+    golden_clip = np.clip(golden_same_dtype, a_min, a_max)
+    npu_clip = np.clip(device_output, a_min, a_max)
+    clipped_abs_ae = np.abs(npu_clip - golden_clip)
+    clipped_re = clipped_abs_ae / abs_gpu_with_eps
+    pass_mask = np.less_equal(clipped_re, rtol)
+    both_nan_mask = np.logical_and(np.isnan(device_output), np.isnan(golden_clip))
+    pass_mask = np.logical_or(pass_mask, both_nan_mask)
+    not_pass_mask = np.logical_not(pass_mask)
+    not_pass_mask = np.logical_and(not_pass_mask, inf_nan_mask)
+
+    inf_nan_err_cnt = np.sum(not_pass_mask)
+    return 0 if np.sum(inf_nan_mask) == 0 else inf_nan_err_cnt / np.sum(inf_nan_mask)
+
+
+def check_small_value(abs_err, small_value_mask, small_value_atol):
+    '''
+    新精度标准的相对阈值法中，检查npu和golden小值域输出的相对误差是否满足阈值
+    输入：
+        rel_err：npu输出和golden输出的相对误差
+        normal_value_mask：npu输出和golden输出的正常值mask
+        rtol：相对误差的阈值
+    输出： 
+        rel_err_ratio：npu输出和golden输出的相对误差不满足阈值的比例
+    '''
+    greater_mask = np.greater(abs_err, small_value_atol)
+    err_mask = np.logical_and(greater_mask, small_value_mask)
+    err_cnt = np.sum(err_mask)
+    return 0 if np.sum(small_value_mask) == 0 else err_cnt / np.sum(small_value_mask)
+
+
+def check_norm_value(normal_value_mask, rel_err, rtol):
+    '''
+    新精度标准的绝对阈值法中，检查npu和golden正常值输出的绝对误差是否满足阈值
+    输入：
+        abs_err：npu输出和golden输出的绝对误差
+        normal_value_mask：npu输出和golden输出的正常值mask
+        atol：绝对误差的阈值
+    输出： 
+        abs_err_ratio：npu输出和golden输出的绝对误差不满足阈值的比例
+    '''
+    err_mask = np.greater(rel_err, rtol)
+    err_mask = np.logical_and(err_mask, normal_value_mask)
+    err_cnt = np.sum(err_mask)
+    return 0 if np.sum(normal_value_mask) == 0 else err_cnt / np.sum(normal_value_mask)
+
+
+def get_ulp_err(bench_output, device_output, dtype):
+    parameters = ULP_PARAMETERS.get(dtype)
+    min_eb = parameters.get('min_eb', DEFAULT_THRESHOLD)[0]
+    exponent_num = parameters.get('exponent_num', DEFAULT_THRESHOLD)[0]
+    abs_bench = np.abs(bench_output)
+    eb = np.where(abs_bench == 0, 0, np.floor(np.log2(abs_bench)))
+    eb = np.maximum(eb, min_eb)
+
+    if dtype == torch.float32:
+        ulp_err = calc_ulp_err(bench_output, device_output, eb, exponent_num, np.float64)
+    else:
+        ulp_err = calc_ulp_err(bench_output, device_output, eb, exponent_num, np.float32)
+    ulp_err = np.abs(ulp_err)
+    return ulp_err
+
+
+def calc_ulp_err(bench_output, device_output, eb, exponent_num, data_type):
+    return  (device_output.astype(data_type) - bench_output).astype(data_type) * \
+            np.exp2(-eb + exponent_num).astype(data_type)