mindstudio-probe 1.0.1__py3-none-any.whl → 1.0.3__py3-none-any.whl

msprobe/msprobe.py

@@ -15,13 +15,16 @@
 
 import argparse
 import sys
-from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut import _run_ut_parser, run_ut_command
-from msprobe.pytorch.parse_tool.cli import parse as cli_parse
-from msprobe.pytorch.api_accuracy_checker.run_ut.multi_run_ut import prepare_config, run_parallel_ut
-from msprobe.pytorch.api_accuracy_checker.compare.api_precision_compare import _api_precision_compare_parser, \
-    _api_precision_compare_command
-from msprobe.pytorch.api_accuracy_checker.run_ut.run_overflow_check import _run_overflow_check_parser, \
-    _run_overflow_check_command
+import importlib.util
+from msprobe.core.compare.utils import _compare_parser
+from msprobe.core.common.log import logger
+from msprobe.core.compare.compare_cli import compare_cli
+from msprobe.core.common.const import Const
+
+
+def is_module_available(module_name):
+    spec = importlib.util.find_spec(module_name)
+    return spec is not None
 
 
 def main():
@@ -31,37 +34,74 @@ def main():
                     "Providing one-site accuracy difference debugging toolkit for training on Ascend Devices.\n"
                     f"For any issue, refer README.md first",
     )
+
     parser.set_defaults(print_help=parser.print_help)
-    parser.add_argument('-f', '--framework', required=True, choices=['pytorch'],
+    parser.add_argument('-f', '--framework', required=True, choices=[Const.PT_FRAMEWORK, Const.MS_FRAMEWORK],
                         help='Deep learning framework.')
     subparsers = parser.add_subparsers()
     subparsers.add_parser('parse')
+    compare_cmd_parser = subparsers.add_parser('compare')
     run_ut_cmd_parser = subparsers.add_parser('run_ut')
     multi_run_ut_cmd_parser = subparsers.add_parser('multi_run_ut')
     api_precision_compare_cmd_parser = subparsers.add_parser('api_precision_compare')
     run_overflow_check_cmd_parser = subparsers.add_parser('run_overflow_check')
-    _run_ut_parser(run_ut_cmd_parser)
-    _run_ut_parser(multi_run_ut_cmd_parser)
-    multi_run_ut_cmd_parser.add_argument('-n', '--num_splits', type=int, choices=range(1, 65), default=8,
-                                         help='Number of splits for parallel processing. Range: 1-64')
-    _api_precision_compare_parser(api_precision_compare_cmd_parser)
-    _run_overflow_check_parser(run_overflow_check_cmd_parser)
+    _compare_parser(compare_cmd_parser)
+    is_torch_available=is_module_available("torch")
+    is_mindspore_available = is_module_available("mindspore")
+    if is_torch_available:
+        from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut import _run_ut_parser, run_ut_command
+        from msprobe.pytorch.parse_tool.cli import parse as cli_parse
+        from msprobe.pytorch.api_accuracy_checker.run_ut.multi_run_ut import prepare_config, run_parallel_ut
+        from msprobe.pytorch.api_accuracy_checker.compare.api_precision_compare import _api_precision_compare_parser, \
+            _api_precision_compare_command
+        from msprobe.pytorch.api_accuracy_checker.run_ut.run_overflow_check import _run_overflow_check_parser, \
+            _run_overflow_check_command
+
+        _run_ut_parser(run_ut_cmd_parser)
+        _run_ut_parser(multi_run_ut_cmd_parser)
+        multi_run_ut_cmd_parser.add_argument('-n', '--num_splits', type=int, choices=range(1, 65), default=8,
+                                        help='Number of splits for parallel processing. Range: 1-64')
+        _api_precision_compare_parser(api_precision_compare_cmd_parser)
+        _run_overflow_check_parser(run_overflow_check_cmd_parser)
+    elif is_mindspore_available:
+        from msprobe.mindspore.api_accuracy_checker.main import add_api_accuracy_checker_argument
+        add_api_accuracy_checker_argument(run_ut_cmd_parser)
+
     if len(sys.argv) == 1:
         parser.print_help()
         sys.exit(0)
     args = parser.parse_args(sys.argv[1:])
-    if sys.argv[3] == "run_ut":
-        run_ut_command(args)
-    elif sys.argv[3] == "parse":
-        cli_parse()
-    elif sys.argv[3] == "multi_run_ut":
-        config = prepare_config(args)
-        run_parallel_ut(config)
-    elif sys.argv[3] == "api_precision_compare":
-        _api_precision_compare_command(args)
-    elif sys.argv[3] == "run_overflow_check":
-        _run_overflow_check_command(args)
-
+    if sys.argv[2] == Const.PT_FRAMEWORK:
+        if not is_torch_available:
+            logger.error("PyTorch does not exist, please install PyTorch library")
+            raise Exception("PyTorch does not exist, please install PyTorch library")
+        if sys.argv[3] == "run_ut":
+            run_ut_command(args)
+        elif sys.argv[3] == "parse":
+            cli_parse()
+        elif sys.argv[3] == "multi_run_ut":
+            config = prepare_config(args)
+            run_parallel_ut(config)
+        elif sys.argv[3] == "api_precision_compare":
+            _api_precision_compare_command(args)
+        elif sys.argv[3] == "run_overflow_check":
+            _run_overflow_check_command(args)
+        elif sys.argv[3] == "compare":
+            if args.cell_mapping is not None or args.api_mapping is not None:
+                logger.error("Argument -cm or -am is not supported in PyTorch framework")
+                raise Exception("Argument -cm or -am is not supported in PyTorch framework")
+            compare_cli(args)
+    else:
+        if not is_module_available(Const.MS_FRAMEWORK):
+            logger.error("MindSpore does not exist, please install MindSpore library")
+            raise Exception("MindSpore does not exist, please install MindSpore library")
+        if sys.argv[3] == "compare":
+            if isinstance(args.api_mapping, str):
+                logger.warning("User defined mapping tables are not supported in the current version")
+            compare_cli(args)
+        elif sys.argv[3] == "run_ut":
+            from msprobe.mindspore.api_accuracy_checker.main import api_checker_main
+            api_checker_main(args)
 
 if __name__ == "__main__":
     main()

msprobe/pytorch/__init__.py

@@ -1,4 +1,4 @@
 from .debugger.precision_debugger import PrecisionDebugger
 from .common.utils import seed_all
-from .compare.acc_compare import compare
 from .compare.distributed_compare import compare_distributed
+from .compare.pt_compare import compare 

msprobe/pytorch/api_accuracy_checker/common/config.py

@@ -1,17 +1,14 @@
 import os
 import yaml
-from msprobe.pytorch.api_accuracy_checker.common.utils import check_file_or_directory_path
-from msprobe.pytorch.hook_module.utils import WrapFunctionalOps, WrapTensorOps, WrapTorchOps
-from msprobe.core.common.file_check import FileOpen
-
-WrapApi = set(WrapFunctionalOps) | set(WrapTensorOps) | set(WrapTorchOps)
+from msprobe.core.common.utils import check_file_or_directory_path
+from msprobe.core.common.utils import load_yaml
+from msprobe.pytorch.pt_config import RunUTConfig
 
 
 class Config:
     def __init__(self, yaml_file):
         check_file_or_directory_path(yaml_file, False)
-        with FileOpen(yaml_file, 'r') as file:
-            config = yaml.safe_load(file)
+        config = load_yaml(yaml_file)
         self.config = {key: self.validate(key, value) for key, value in config.items()}
 
     def __getattr__(self, item):
@@ -24,8 +21,15 @@ class Config:
     def validate(key, value):
         validators = {
             'white_list': list,
+            'black_list': list,
             'error_data_path': str,
-            'precision': int
+            'precision': int,
+            'is_online': bool,
+            'nfs_path': str,
+            'host': str,
+            'port': int,
+            'rank_list': list,
+            'tls_path': str
         }
         if key not in validators:
             raise ValueError(f"{key} must be one of {validators.keys()}")
@@ -34,14 +38,15 @@ class Config:
         if key == 'precision' and value < 0:
             raise ValueError("precision must be greater than 0")
         if key == 'white_list':
-            if not isinstance(value, list):
-                raise ValueError("white_list must be a list type")
-            if not all(isinstance(i, str) for i in value):
-                raise ValueError("All elements in white_list must be of str type")
-            invalid_api = [i for i in value if i not in WrapApi]
-            if invalid_api:
-                raise ValueError(
-                    f"{', '.join(invalid_api)} is not in support_wrap_ops.yaml, please check the white_list")
+            RunUTConfig.check_filter_list_config(key, value)
+        if key == 'black_list':
+            RunUTConfig.check_filter_list_config(key, value)
+        if key == 'error_data_path':
+            RunUTConfig.check_error_data_path_config(value)
+        if key == 'nfs_path':
+            RunUTConfig.check_nfs_path_config(value)
+        if key == 'tls_path':
+            RunUTConfig.check_tls_path_config(value)
         return value
 
 

msprobe/pytorch/api_accuracy_checker/common/utils.py

@@ -14,10 +14,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """
-import json
 import os
 import re
-import csv
 
 import torch
 
@@ -38,12 +36,6 @@ class DumpException(CompareException):
     pass
 
 
-def write_csv(data, filepath):
-    with FileOpen(filepath, 'a', encoding='utf-8-sig') as f:
-        writer = csv.writer(f)
-        writer.writerows(data)
-
-
 def check_object_type(check_object, allow_type):
     """
     Function Description:
@@ -59,58 +51,6 @@ def check_object_type(check_object, allow_type):
         raise CompareException(CompareException.INVALID_DATA_ERROR)
 
 
-def check_file_or_directory_path(path, isdir=False):
-    """
-    Function Description:
-        check whether the path is valid
-    Parameter:
-        path: the path to check
-        isdir: the path is dir or file
-    Exception Description:
-        when invalid data throw exception
-    """
-    if isdir:
-        if not os.path.exists(path):
-            logger.error('The path {} is not exist.'.format(path))
-            raise CompareException(CompareException.INVALID_PATH_ERROR)
-
-        if not os.path.isdir(path):
-            logger.error('The path {} is not a directory.'.format(path))
-            raise CompareException(CompareException.INVALID_PATH_ERROR)
-
-        if not os.access(path, os.W_OK):
-            logger.error(
-                'The path {} does not have permission to write. Please check the path permission'.format(path))
-            raise CompareException(CompareException.INVALID_PATH_ERROR)
-    else:
-        if not os.path.isfile(path):
-            logger.error('{} is an invalid file or non-exist.'.format(path))
-            raise CompareException(CompareException.INVALID_PATH_ERROR)
-
-    if not os.access(path, os.R_OK):
-        logger.error(
-            'The path {} does not have permission to read. Please check the path permission'.format(path))
-        raise CompareException(CompareException.INVALID_PATH_ERROR)
-
-
-def get_json_contents(file_path):
-    ops = get_file_content_bytes(file_path)
-    try:
-        json_obj = json.loads(ops)
-    except ValueError as error:
-        logger.error('Failed to load "%s". %s' % (file_path, str(error)))
-        raise CompareException(CompareException.INVALID_FILE_ERROR) from error
-    if not isinstance(json_obj, dict):
-        logger.error('Json file %s, content is not a dictionary!' % file_path)
-        raise CompareException(CompareException.INVALID_FILE_ERROR)
-    return json_obj
-
-
-def get_file_content_bytes(file):
-    with FileOpen(file, 'rb') as file_handle:
-        return file_handle.read()
-
-
 class SoftlinkCheckException(Exception):
     pass
 
@@ -166,6 +106,7 @@ def initialize_save_path(save_path, dir_name):
         os.mkdir(data_path, mode=FileCheckConst.DATA_DIR_AUTHORITY)
     data_path_checker = FileChecker(data_path, FileCheckConst.DIR)
     data_path_checker.common_check()
+    return data_path
 
 
 def write_pt(file_path, tensor):

msprobe/pytorch/api_accuracy_checker/compare/algorithm.py

@@ -6,9 +6,6 @@ from msprobe.pytorch.api_accuracy_checker.compare.compare_utils import ULP_PARAM
 from msprobe.core.common.const import CompareConst
 
 
-DEFAULT_THRESHOLD = 1
-
-
 #cos
 def cosine_sim(bench_output, device_output):
     msg = ""
@@ -197,8 +194,8 @@ def check_norm_value(normal_value_mask, rel_err, rtol):
 
 def get_ulp_err(bench_output, device_output, dtype):
     parameters = ULP_PARAMETERS.get(dtype)
-    min_eb = parameters.get('min_eb', DEFAULT_THRESHOLD)[0]
-    exponent_num = parameters.get('exponent_num', DEFAULT_THRESHOLD)[0]
+    min_eb = parameters.get('min_eb')[0]
+    exponent_num = parameters.get('exponent_num')[0]
     abs_bench = np.abs(bench_output)
     eb = np.where(abs_bench == 0, 0, np.floor(np.log2(abs_bench)))
     eb = np.maximum(eb, min_eb)

msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py

@@ -7,19 +7,19 @@ from collections import namedtuple
 import torch
 import pandas as pd
 
-from msprobe.pytorch.api_accuracy_checker.common.utils import write_csv
+from msprobe.core.common.utils import write_csv
 from msprobe.pytorch.api_accuracy_checker.common.config import msCheckerConfig
 from msprobe.pytorch.api_accuracy_checker.compare.compare_utils import API_PRECISION_COMPARE_RESULT_FILE_NAME, \
     API_PRECISION_COMPARE_DETAILS_FILE_NAME, BENCHMARK_COMPARE_SUPPORT_LIST, API_PRECISION_COMPARE_UNSUPPORT_LIST, \
-    ApiPrecisionCompareColumn, AbsoluteStandardApi, BinaryStandardApi, ULPStandardApi, ThousandthStandardApi, \
+    ApiPrecisionCompareColumn, absolute_standard_api, binary_standard_api, ulp_standard_api, thousandth_standard_api, \
     BINARY_COMPARE_UNSUPPORT_LIST, ULP_COMPARE_SUPPORT_LIST, convert_str_to_float, CompareMessage, is_inf_or_nan, \
     check_inf_or_nan
 from msprobe.pytorch.api_accuracy_checker.compare.compare_column import ApiPrecisionOutputColumn
-from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut import get_validated_result_csv_path
+from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut_utils import get_validated_result_csv_path
 from msprobe.core.common.file_check import FileChecker, change_mode, check_path_before_create, create_directory
 from msprobe.pytorch.common.log import logger
 from msprobe.core.common.utils import CompareException
-from msprobe.core.common.const import CompareConst, FileCheckConst
+from msprobe.core.common.const import CompareConst, FileCheckConst, Const
 
 CompareConfig = namedtuple('CompareConfig', ['npu_csv_path', 'gpu_csv_path', 'result_csv_path', 'details_csv_path'])
 BenchmarkInf_Nan_Consistency = namedtuple('BenchmarkInf_Nan_Consistency', ['small_value_inf_nan_consistency', 
@@ -289,15 +289,38 @@ def api_precision_compare(config):
     change_mode(config.details_csv_path, FileCheckConst.DATA_FILE_AUTHORITY)
 
 
+def online_api_precision_compare(online_config):
+    rank = online_config.rank
+    result_csv_path = os.path.join(Const.DEFAULT_PATH, online_config.result_csv_path).replace("_rank*.csv", f"_rank{rank}.csv")
+    details_csv_path = os.path.join(Const.DEFAULT_PATH, online_config.details_csv_path).replace("_rank*.csv", f"_rank{rank}.csv")
+    detail_csv_title = [ApiPrecisionCompareColumn.get_detail_csv_title()]
+    result_csv_title = [ApiPrecisionCompareColumn.get_result_csv_title()]
+    if not os.path.exists(result_csv_path):
+        write_csv(result_csv_title, result_csv_path)
+    if not os.path.exists(details_csv_path):
+        write_csv(detail_csv_title, details_csv_path)
+    config = CompareConfig("", "", result_csv_path, details_csv_path)
+    try:
+        npu_data, gpu_data = online_config.npu_data, online_config.gpu_data
+        check_csv_columns(npu_data.columns, "npu_csv")
+        check_csv_columns(gpu_data.columns, "gpu_csv")
+        analyse_csv(npu_data, gpu_data, config)
+    except Exception as err:
+        logger.error(f"Online api precision compare Error: {str(err)}")
+    change_mode(result_csv_path, FileCheckConst.DATA_FILE_AUTHORITY)
+    change_mode(details_csv_path, FileCheckConst.DATA_FILE_AUTHORITY)
+
+
 def analyse_csv(npu_data, gpu_data, config):
     forward_status, backward_status = [], []
-    last_api_name, last_api_dtype = None, None
+    last_api_name, last_api_dtype, last_api_full_name = None, None, None
     for _, row_npu in npu_data.iterrows():
         message = ''
         compare_column = ApiPrecisionOutputColumn()
         full_api_name_with_direction_status = row_npu[ApiPrecisionCompareColumn.API_NAME]
         row_gpu = gpu_data[gpu_data[ApiPrecisionCompareColumn.API_NAME] == full_api_name_with_direction_status]
-        _, api_name, _, direction_status, _, _ = full_api_name_with_direction_status.split(".")
+        api_type, api_name, api_nums, direction_status, _, _ = full_api_name_with_direction_status.split(Const.SEP)
+        api_full_name = Const.SEP.join([api_type, api_name, api_nums])
         if row_gpu.empty:
             logger.warning(f'This API : {full_api_name_with_direction_status} does not exist in the GPU data.')
             continue
@@ -315,14 +338,14 @@ def analyse_csv(npu_data, gpu_data, config):
             write_detail_csv(compare_column.to_column_value(), config.details_csv_path)
         else:
             compare_column.api_name = full_api_name_with_direction_status
-            if api_name in ThousandthStandardApi:
+            if api_name in thousandth_standard_api:
                 new_status = record_thousandth_threshold_result(compare_column, row_npu)
             elif row_npu[ApiPrecisionCompareColumn.DEVICE_DTYPE] not in BINARY_COMPARE_UNSUPPORT_LIST or \
-                api_name in BinaryStandardApi:
+                api_name in binary_standard_api:
                 new_status = record_binary_consistency_result(api_name, compare_column, row_npu)                            
-            elif api_name in AbsoluteStandardApi:
+            elif api_name in absolute_standard_api:
                 new_status = record_absolute_threshold_result(compare_column, row_npu)
-            elif api_name in ULPStandardApi and \
+            elif api_name in ulp_standard_api and \
                 row_npu[ApiPrecisionCompareColumn.DEVICE_DTYPE] in ULP_COMPARE_SUPPORT_LIST:
                 us = ULPStandard(full_api_name_with_direction_status, row_npu, row_gpu)
                 new_status = record_ulp_compare_result(compare_column, us)
@@ -335,6 +358,7 @@ def analyse_csv(npu_data, gpu_data, config):
             if last_api_dtype in API_PRECISION_COMPARE_UNSUPPORT_LIST:
                 message = unsupported_message
                 write_csv([[last_api_name, "skip", "skip", message]], config.result_csv_path)
+                print_test_success(api_full_name, "skip", "skip")
                 forward_status, backward_status = [], []
                 message = ''
             else:
@@ -342,11 +366,13 @@ def analyse_csv(npu_data, gpu_data, config):
                 backward_result = get_api_checker_result(backward_status)
                 message += CompareMessage.get(last_api_name, "") if forward_result == CompareConst.ERROR else ""
                 write_csv([[last_api_name, forward_result, backward_result, message]], config.result_csv_path)
+                print_test_success(api_full_name, forward_result, backward_result)
                 forward_status, backward_status = [], []
                 message = ''
 
         is_supported = row_npu[ApiPrecisionCompareColumn.DEVICE_DTYPE] not in API_PRECISION_COMPARE_UNSUPPORT_LIST
         last_api_name = api_name
+        last_api_full_name = api_full_name
 
         last_api_dtype = row_npu[ApiPrecisionCompareColumn.DEVICE_DTYPE]
         if not is_supported:
@@ -363,11 +389,21 @@ def analyse_csv(npu_data, gpu_data, config):
         if last_api_dtype in API_PRECISION_COMPARE_UNSUPPORT_LIST:
             message = unsupported_message
             write_csv([[last_api_name, "skip", "skip", message]], config.result_csv_path)
+            print_test_success(last_api_full_name, "skip", "skip")
         else:
             forward_result = get_api_checker_result(forward_status)
             backward_result = get_api_checker_result(backward_status)
             message += CompareMessage.get(last_api_name, "") if forward_result == CompareConst.ERROR else ""
             write_csv([[last_api_name, forward_result, backward_result, message]], config.result_csv_path)
+            print_test_success(last_api_full_name, forward_result, backward_result)
+
+
+def print_test_success(api_full_name, forward_result, backward_result):
+    is_fwd_success = (forward_result == CompareConst.PASS)
+    is_bwd_success = (backward_result == CompareConst.PASS or backward_result == CompareConst.SPACE)
+    logger.info(f"running api_full_name {api_full_name} compare, "
+                f"is_fwd_success: {is_fwd_success}, "
+                f"is_bwd_success: {is_bwd_success}")
 
 
 def check_error_rate(npu_error_rate):

msprobe/pytorch/api_accuracy_checker/compare/compare.py

@@ -1,27 +1,28 @@
 # 进行比对及结果展示
 import os
 from collections import namedtuple
-import torch
+
 import numpy as np
-from msprobe.pytorch.common.log import logger
-from msprobe.pytorch.api_accuracy_checker.common.utils import get_json_contents, write_csv
-from msprobe.pytorch.api_accuracy_checker.compare.compare_utils import check_dtype_comparable, \
-    DETAIL_TEST_ROWS, precision_configs, BENCHMARK_COMPARE_SUPPORT_LIST, AbsoluteStandardApi, BinaryStandardApi, \
-    ULPStandardApi, ThousandthStandardApi, apis_threshold
-from msprobe.pytorch.api_accuracy_checker.compare.compare_column import CompareColumn
+from msprobe.core.common.utils import write_csv, get_json_contents, CompareException
+import torch
+from msprobe.core.common.const import Const, CompareConst
 from msprobe.pytorch.api_accuracy_checker.compare.algorithm import get_rmse, get_error_balance, get_max_rel_err, \
     get_mean_rel_err, get_rel_err, get_abs_err, get_max_abs_err, get_rel_err_ratio, cosine_sim, get_rel_err_origin, \
     get_small_value_err_ratio, get_finite_and_infinite_mask, get_small_value_mask, check_inf_nan_value, \
     check_small_value, check_norm_value, get_abs_bench_with_eps, get_ulp_err
 from msprobe.pytorch.api_accuracy_checker.common.config import msCheckerConfig
-from msprobe.core.common.const import Const, CompareConst
+from msprobe.pytorch.api_accuracy_checker.compare.compare_column import CompareColumn
+from msprobe.pytorch.api_accuracy_checker.compare.compare_utils import check_dtype_comparable, \
+    DETAIL_TEST_ROWS, precision_configs, BENCHMARK_COMPARE_SUPPORT_LIST, absolute_standard_api, binary_standard_api, \
+    ulp_standard_api, thousandth_standard_api, apis_threshold
+from msprobe.pytorch.common.log import logger
 
 
 ResultInfo = namedtuple('ResultInfo', ['full_api_name', 'fwd_success_status', 'bwd_success_status',
                                        'fwd_compare_alg_results', 'bwd_compare_alg_results', 'rank'])
 
 
-INDEX_TEST_RESULT__GROUP = 3
+INDEX_TEST_RESULT_GROUP = 3
 INDEX_FIRST_GROUP = 0
 INDEX_MESSAGE = -1
 
@@ -33,20 +34,34 @@ class Comparator:
     COLUMN_BACKWARD_SUCCESS = "Backward Test Success"
     COLUMN_STACK_INFO = "Traceback callstack info"
 
-    def __init__(self, result_csv_path, details_csv_path, is_continue_run_ut, stack_info_json_path=None):
-        self.save_path = result_csv_path
-        self.detail_save_path = details_csv_path
-        if not is_continue_run_ut and not os.path.exists(self.save_path) and not os.path.exists(self.detail_save_path):
+    def __init__(self, result_csv_path, details_csv_path, is_continue_run_ut, stack_info_json_path=None, config=None):
+        self.save_path_str = result_csv_path
+        self.detail_save_path_str = details_csv_path
+        self.save_path_list = [result_csv_path]
+        self.detail_save_path_list = [details_csv_path]
+
+        if config and config.online_config.is_online:
+            self.save_path_str = result_csv_path.replace(".csv", "_rank{}.csv")
+            self.detail_save_path_str = details_csv_path.replace(".csv", "_rank{}.csv")
+            self.save_path_list = [self.save_path_str.format(rank) for rank in config.online_config.rank_list]
+            self.detail_save_path_list = \
+                [self.detail_save_path_str.format(rank) for rank in config.online_config.rank_list]
+
+        if not is_continue_run_ut:
             self.write_csv_title()
         if stack_info_json_path:
             self.stack_info = get_json_contents(stack_info_json_path)
         else:
             self.stack_info = None
 
+    @staticmethod
+    def get_path_from_rank(rank, path_list, path_pattern):
+        return path_list[-1] if len(path_list) == 1 else path_pattern.format(rank)
+
     @staticmethod
     def print_pretest_result():
         logger.info("Successfully completed run_ut/multi_run_ut.")
-    
+
     @staticmethod
     def _compare_dropout(bench_output, device_output):
         tensor_num = bench_output.numel()
@@ -75,7 +90,7 @@ class Comparator:
         error_rate = float(error_nums / bench_output.size)
         result = CompareConst.PASS if error_rate == 0 else CompareConst.ERROR
         return error_rate, result, ""
-    
+
     @staticmethod
     def _get_absolute_threshold_attribute(api_name, dtype):
         small_value_threshold = apis_threshold.get(api_name).get(dtype).get('small_value')
@@ -83,35 +98,18 @@ class Comparator:
         rtol = apis_threshold.get(api_name).get(dtype).get('rtol')
         return small_value_threshold, small_value_atol, rtol
 
-    def write_csv_title(self):
-        summary_test_rows = [[self.COLUMN_API_NAME, self.COLUMN_FORWARD_SUCCESS,
-                              self.COLUMN_BACKWARD_SUCCESS, "Message"]]
-        if not os.path.exists(self.save_path):
-            write_csv(summary_test_rows, self.save_path)
-        if not os.path.exists(self.detail_save_path):
-            write_csv(DETAIL_TEST_ROWS, self.detail_save_path)
-
-    def write_summary_csv(self, test_result):
-        test_rows = []
-        if self.stack_info:
-            test_rows[0].append(self.COLUMN_STACK_INFO)
-
-        name = test_result[0]
-        df_row = list(test_result[:INDEX_TEST_RESULT__GROUP])
-        if test_result[1] == "SKIP":
-            df_row.append(test_result[INDEX_TEST_RESULT__GROUP][INDEX_FIRST_GROUP][INDEX_MESSAGE])
-        if self.stack_info:
-            stack_info = "\n".join(self.stack_info[name])
-            df_row.append(stack_info)
-        test_rows.append(df_row)
-        write_csv(test_rows, self.save_path)
-
-    def write_detail_csv(self, test_result):
+    @staticmethod
+    def _get_run_ut_detail(test_result):
+        """get run_ut detail before write to csv, called by online run_ut"""
         test_rows = []
+        try:
+            subject_prefix = test_result[0]
+            fwd_result = test_result[3]
+            bwd_result = test_result[4]
+        except IndexError as e:
+            logger.error("List index out of bounds when writing detail CSV.")
+            raise CompareException(CompareException.INDEX_OUT_OF_BOUNDS_ERROR, "list index out of bounds") from e
 
-        subject_prefix = test_result[0]
-        fwd_result = test_result[3]
-        bwd_result = test_result[4]
         if isinstance(fwd_result, list):
             for i, test_subject in enumerate(fwd_result):
                 subject = subject_prefix + ".forward.output." + str(i)
@@ -124,14 +122,49 @@ class Comparator:
                 test_subject = ["{:.{}f}".format(item, msCheckerConfig.precision)
                                 if isinstance(item, float) else item for item in test_subject]
                 test_rows.append([subject] + list(test_subject))
+        return test_rows
 
-        write_csv(test_rows, self.detail_save_path)
+    def write_csv_title(self):
+        summary_test_rows = [[self.COLUMN_API_NAME, self.COLUMN_FORWARD_SUCCESS,
+                              self.COLUMN_BACKWARD_SUCCESS, "Message"]]
+        for save_path, detail_save_path in zip(self.save_path_list, self.detail_save_path_list):
+            if not os.path.exists(save_path):
+                write_csv(summary_test_rows, save_path)
+            if not os.path.exists(detail_save_path):
+                write_csv(DETAIL_TEST_ROWS, detail_save_path)
+
+    def write_summary_csv(self, test_result):
+        test_rows = []
+        try:
+            name = test_result[0]
+            df_row = list(test_result[:INDEX_TEST_RESULT_GROUP])
+            if test_result[1] == "SKIP":
+                df_row.append(test_result[INDEX_TEST_RESULT_GROUP][INDEX_FIRST_GROUP][INDEX_MESSAGE])
+            if self.stack_info:
+                stack_info = "\n".join(self.stack_info[name])
+                df_row.append(stack_info)
+            test_rows.append(df_row)
+            save_path = self.get_path_from_rank(test_result[-1], self.save_path_list, self.save_path_str)
+        except IndexError as e:
+            logger.error("List index out of bounds when writing summary CSV.")
+            raise CompareException(CompareException.INDEX_OUT_OF_BOUNDS_ERROR, "list index out of bounds") from e
+        write_csv(test_rows, save_path)
+
+    def write_detail_csv(self, test_result):
+        test_rows = self._get_run_ut_detail(test_result)
+        detail_save_path = self.get_path_from_rank(test_result[-1],
+                                                   self.detail_save_path_list,
+                                                   self.detail_save_path_str)
+        write_csv(test_rows, detail_save_path)
 
     def record_results(self, args):
         self.write_summary_csv(args)
         self.write_detail_csv(args)
 
-    def compare_output(self, full_api_name, data_info):
+    def compare_output(self, full_api_name, data_info, is_online=False):
+        """Get compare result and write to result and detail csv.
+        is_online: bool, default False. True: called by online api precision compare, only compare without write to csv.
+        """
         _, api_name, _ = full_api_name.split(Const.SEP)
         bench_output, device_output = data_info.bench_output, data_info.device_output
         bench_grad, device_grad = data_info.bench_grad, data_info.device_grad
@@ -160,6 +193,9 @@ class Comparator:
                                  fwd_compare_alg_results,
                                  bwd_compare_alg_results,
                                  data_info.rank)
+        if is_online:
+            # get run_ut compare detail
+            return self._get_run_ut_detail(result_info)
         self.record_results(result_info)
         return fwd_success_status == CompareConst.PASS, bwd_success_status == CompareConst.PASS \
                or bwd_success_status == CompareConst.SPACE
@@ -261,15 +297,15 @@ class Comparator:
         abs_bench, abs_bench_with_eps = get_abs_bench_with_eps(bench_output, dtype)
         abs_err = get_abs_err(bench_output, device_output)
         rel_err_orign = get_rel_err_origin(abs_err, abs_bench_with_eps)
-        if api_name in ThousandthStandardApi:
+        if api_name in thousandth_standard_api:
             thousand_res, thousand_status = get_rel_err_ratio(rel_err_orign, CompareConst.THOUSAND_RATIO_THRESHOLD)
             compare_column.rel_err_thousandth = thousand_res
         if str(dtype) in BENCHMARK_COMPARE_SUPPORT_LIST:
             both_finite_mask, inf_nan_mask = get_finite_and_infinite_mask(bench_output, device_output)
-            if api_name in BinaryStandardApi:
+            if api_name in binary_standard_api:
                 err_rate, _, _ = self._compare_bool_tensor(bench_output, device_output)
                 compare_column.error_rate = err_rate
-            elif api_name in AbsoluteStandardApi:
+            elif api_name in absolute_standard_api:
                 small_value_threshold, small_value_atol, rtol = self._get_absolute_threshold_attribute(
                     api_name, str(dtype))
                 rel_err = abs_err / abs_bench_with_eps
@@ -279,7 +315,7 @@ class Comparator:
                                                                          dtype, rtol)
                 compare_column.rel_err_ratio = check_norm_value(normal_value_mask, rel_err, rtol)
                 compare_column.abs_err_ratio = check_small_value(abs_err, small_value_mask, small_value_atol)
-            elif api_name in ULPStandardApi:
+            elif api_name in ulp_standard_api:
                 if bench_output.size == 0:
                     compare_column.max_ulp_error = 0
                     compare_column.mean_ulp_error = 0