mindstudio-probe 1.0.4__py3-none-any.whl → 1.1.1__py3-none-any.whl

msprobe/mindspore/service.py

@@ -1,4 +1,5 @@
-# Copyright 2024 Huawei Technologies Co., Ltd
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -11,37 +12,33 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-# ============================================================================
 
-import os
 import copy
 import functools
+import os
 from collections import defaultdict
 
 import mindspore as ms
-from mindspore.common.tensor import Tensor
-from mindspore import ops
 from mindspore import nn
 try:
     from mindspore.common._pijit_context import PIJitCaptureContext
-    pijit_label = True
 except ImportError:
     pijit_label = False
+else:
+    pijit_label = True
 
 
+from msprobe.core.common.exceptions import DistributedNotInitializedError, MsprobeException
+from msprobe.core.common.file_utils import create_directory
+from msprobe.core.common.utils import Const, print_tools_ends_info
 from msprobe.core.data_dump.data_collector import build_data_collector
+from msprobe.core.data_dump.data_processor.base import ModuleBackwardInputsOutputs, ModuleForwardInputsOutputs
 from msprobe.core.data_dump.scope import BaseScope
-from msprobe.mindspore.common.utils import get_rank_if_initialized
-from msprobe.core.common.file_utils import create_directory
+from msprobe.mindspore.cell_processor import CellProcessor
 from msprobe.mindspore.common.log import logger
-from msprobe.core.common.utils import Const
-from msprobe.core.common.exceptions import DistributedNotInitializedError
+from msprobe.mindspore.common.utils import get_rank_if_initialized
 from msprobe.mindspore.dump.hook_cell.api_registry import api_register
-from msprobe.core.data_dump.data_processor.base import ModuleBackwardInputsOutputs, ModuleForwardInputsOutputs, \
-    ModuleBackwardInputs, ModuleBackwardOutputs
-from msprobe.core.common.exceptions import MsprobeException
-from msprobe.mindspore.dump.hook_cell.hook_cell import HOOKCell
-from msprobe.mindspore.cell_processor import CellProcessor
+from msprobe.mindspore.dump.hook_cell.primitive_hooks import PrimitiveHookService
 from msprobe.mindspore.dump.jit_dump import JitDump
 
 
@@ -52,11 +49,12 @@ class Service:
         self.config.level = self.config.level_ori
         self.data_collector = build_data_collector(self.config)
         self.cell_processor = CellProcessor(self.data_collector.scope)
+        self.primitive_hook_service = PrimitiveHookService(self)
         self.switch = False
+        self.primitive_switch = False
         self.current_iter = 0
         self.first_start = True
         self.current_rank = None
-        self.primitive_counters = {}
         self.dump_iter_dir = None
         self.start_call = False
         self.check_level_valid()
@@ -71,28 +69,30 @@ class Service:
         )
 
     def check_level_valid(self):
-        if self.config.level == "L2":
+        if self.config.level == Const.LEVEL_L2:
             raise MsprobeException(
                 MsprobeException.INVALID_PARAM_ERROR, "L2 level dump function is currently not supported."
             )
 
     def build_hook(self, target_type, name):
-        def forward_hook(api_or_cell_name, cell, input, output):
+        def forward_hook(api_or_cell_name, cell, input_data, output):
             if not self.should_excute_hook():
+                if hasattr(cell, 'input_kwargs'):
+                    del cell.input_kwargs
                 return None
 
             if target_type == BaseScope.Module_Type_Module:
-                api_or_cell_name = cell.mindstudio_reserved_name
-                module_input_output = ModuleForwardInputsOutputs(args=input, kwargs={}, output=output)
+                api_or_cell_name = self.cell_processor.set_and_get_reserved_name(cell, api_or_cell_name)
+                module_input_output = ModuleForwardInputsOutputs(args=input_data, kwargs={}, output=output)
             else:
-                module_input_output = ModuleForwardInputsOutputs(args=input, kwargs=cell.input_kwargs,
+                module_input_output = ModuleForwardInputsOutputs(args=input_data, kwargs=cell.input_kwargs,
                                                                  output=output)
 
             self.data_collector.update_api_or_module_name(api_or_cell_name)
             self.data_collector.forward_data_collect(api_or_cell_name, cell, pid, module_input_output)
             if self.data_collector.if_return_forward_new_output():
                 return self.data_collector.get_forward_new_output()
-            if target_type == BaseScope.Module_Type_API:
+            if hasattr(cell, 'input_kwargs'):
                 del cell.input_kwargs
             return output
 
@@ -100,12 +100,19 @@ class Service:
             if not self.should_excute_hook():
                 return
 
+            need_exchange = True
             if target_type == BaseScope.Module_Type_Module:
-                api_or_cell_name = cell.mindstudio_reserved_name
+                if not hasattr(cell, 'has_pre_hook_called') or not cell.has_pre_hook_called:
+                    need_exchange = False
+                api_or_cell_name = self.cell_processor.set_and_get_reserved_name(cell, api_or_cell_name)
+
             self.data_collector.update_api_or_module_name(api_or_cell_name)
             if self.data_collector:
                 # 框架最新接口变更，grad_input和grad_output的含义发生了变化，与torch含义保持一致，因此此处调换顺序传入
-                module_input_output = ModuleBackwardInputsOutputs(grad_input=grad_output, grad_output=grad_input)
+                if need_exchange:
+                    module_input_output = ModuleBackwardInputsOutputs(grad_input=grad_output, grad_output=grad_input)
+                else:
+                    module_input_output = ModuleBackwardInputsOutputs(grad_input=grad_input, grad_output=grad_output)
                 self.data_collector.backward_data_collect(api_or_cell_name, cell, pid, module_input_output)
 
         pid = os.getpid()
@@ -114,145 +121,40 @@ class Service:
         forward_hook = functools.partial(forward_hook, forward_name_template)
         backward_hook = functools.partial(backward_hook, backward_name_template)
 
-        def wrap_forward_hook(cell, input, output):
-            return forward_hook(cell, input, output)
+        def wrap_forward_hook(cell, input_data, output_data):
+            return forward_hook(cell, input_data, output_data)
 
         def wrap_backward_hook(cell, grad_input, grad_output):
             return backward_hook(cell, grad_input, grad_output)
 
         return wrap_forward_hook, wrap_backward_hook
 
-    def wrap_primitive(self, origin_func, primitive_name):
-        service_instance = self
-
-        def create_backward_hook(captured_grads, num_tensors, updated_primitive_name, hook_type):
-            def backward_hook(grad):
-                captured_grads.append(grad)
-                backward_primitive_name = f"{updated_primitive_name}.{Const.BACKWARD}"
-                try:
-                    if len(captured_grads) == num_tensors and hook_type == Const.INPUT:
-                        service_instance.data_collector.update_api_or_module_name(backward_primitive_name)
-                        new_module_input_output = ModuleBackwardOutputs(grad_output=tuple(captured_grads))
-                        service_instance.data_collector.backward_output_data_collect(
-                            backward_primitive_name, service_instance, os.getpid(), new_module_input_output
-                        )
-                        captured_grads.clear()
-                    elif len(captured_grads) == num_tensors and hook_type == Const.OUTPUT:
-                        service_instance.data_collector.update_api_or_module_name(backward_primitive_name)
-                        new_module_input_output = ModuleBackwardInputs(grad_input=tuple(captured_grads))
-                        service_instance.data_collector.backward_input_data_collect(
-                            backward_primitive_name, service_instance, os.getpid(), new_module_input_output
-                        )
-                        captured_grads.clear()
-
-                except Exception as exception:
-                    raise Exception(f"This is a primitive op {hook_type}_backward dump error: {exception},"
-                                    f" updated_primitive_name: {updated_primitive_name}") from exception
-
-            return backward_hook
-
-        def hook_primitive_inputs(args, captured_grads_input, updated_primitive_name):
-            hooked_inputs = []
-            num_tensors = sum(isinstance(arg, Tensor) for arg in args)
-            input_backward_hook = create_backward_hook(captured_grads_input, num_tensors, updated_primitive_name,
-                                                       Const.INPUT)
-            for _, arg in enumerate(args):
-                if isinstance(arg, Tensor):
-                    arg_hooked = ops.HookBackward(input_backward_hook)(arg)
-                    hooked_inputs.append(arg_hooked)
-                else:
-                    hooked_inputs.append(arg)
-            return hooked_inputs
-
-        def hook_primitive_outputs(out, captured_grads_output, updated_primitive_name):
-            if isinstance(out, tuple):
-                num_output_tensors = sum(isinstance(tensor, Tensor) for tensor in out)
-            else:
-                num_output_tensors = 1
-            output_backward_hook = create_backward_hook(captured_grads_output, num_output_tensors,
-                                                        updated_primitive_name, Const.OUTPUT)
-
-            if isinstance(out, Tensor):
-                return ops.HookBackward(output_backward_hook)(out)
-            elif isinstance(out, tuple):
-                hooked_outputs = []
-                for tensor in out:
-                    if isinstance(tensor, Tensor):
-                        hooked_outputs.append(ops.HookBackward(output_backward_hook)(tensor))
-                    else:
-                        hooked_outputs.append(tensor)
-                return tuple(hooked_outputs)
-            return out
-
-        def wrapped_primitive_call(instance_self, *args, **kwargs):
-            service_instance.update_primitive_counters(primitive_name)
-            current_count = service_instance.primitive_counters.get(primitive_name, 0)
-            updated_primitive_name = f"{Const.PRIMITIVE_PREFIX}.{primitive_name}.{current_count}"
-
-            if not service_instance.switch:
-                return origin_func(*args, **kwargs)
-
-            captured_grads_input, captured_grads_output = [], []
-
-            try:
-                hooked_inputs = hook_primitive_inputs(args, captured_grads_input, updated_primitive_name)
-            except Exception as exception:
-                raise Exception("This is a primitive op dump error during input hooking: {},"
-                                " primitive_name: {}".format(exception, primitive_name)) from exception
-
-            try:
-                out = origin_func(*hooked_inputs, **kwargs)
-            except Exception as exception:
-                raise Exception("This is a primitive op dump error during function call: {},"
-                                " primitive_name: {}".format(exception, primitive_name)) from exception
-
-            forward_primitive_name = f"{updated_primitive_name}.{Const.FORWARD}"
-            service_instance.data_collector.update_api_or_module_name(forward_primitive_name)
-            if service_instance.data_collector:
-                module_input_output = ModuleForwardInputsOutputs(args=hooked_inputs, kwargs=kwargs, output=out)
-                try:
-                    service_instance.data_collector.forward_data_collect(forward_primitive_name, instance_self,
-                                                                         os.getpid(), module_input_output)
-                except Exception as exception:
-                    raise Exception("This is a primitive op dump error during forward data collection: {},"
-                                    " primitive_name: {}".format(exception, primitive_name)) from exception
-
-                if service_instance.data_collector.if_return_forward_new_output():
-                    out = service_instance.data_collector.get_forward_new_output()
-
-            try:
-                out = hook_primitive_outputs(out, captured_grads_output, updated_primitive_name)
-            except Exception as exception:
-                raise Exception("This is a primitive op dump error during output hooking: {},"
-                                " primitive_name: {}".format(exception, primitive_name)) from exception
-
-            return out
-
-        return wrapped_primitive_call
-
     def update_primitive_counters(self, primitive_name):
         if primitive_name not in self.primitive_counters:
             self.primitive_counters[primitive_name] = 0
         else:
             self.primitive_counters[primitive_name] += 1
 
-    def register_hooks(self):
+    def register_primitive_hooks(self):
         primitive_set = set()
         for _, cell in self.model.cells_and_names():
             for pname, primitive in cell._primitives.items():
                 primitive_set.add((pname, primitive))
 
         for pname, primitive in primitive_set:
-            NewPrimitive = type('NewPrimitive', (primitive.__class__,),
-                                {'__call__': self.wrap_primitive(primitive.__call__, pname)})
-            primitive.__class__ = NewPrimitive
+            primitive_class_name = primitive.__class__.__name__
+            primitive_combined_name = pname + Const.SEP + primitive_class_name
+            new_primitive = type('NewPrimitive', (primitive.__class__,),
+                                 {'__call__': self.primitive_hook_service.wrap_primitive(primitive.__call__,
+                                                                                         primitive_combined_name)})
+            primitive.__class__ = new_primitive
 
     def step(self):
         self.current_iter += 1
         self.data_collector.update_iter(self.current_iter)
-        HOOKCell.cell_count = defaultdict(int)
-        CellProcessor.cell_count = {}
-        self.primitive_counters.clear()
+        self.primitive_hook_service.primitive_counters.clear()
+        self.data_collector.data_writer.reset_cache()
+        JitDump.jit_count = defaultdict(int)
 
     def start(self, model=None):
         self.start_call = True
@@ -262,9 +164,8 @@ class Service:
             api_register.api_set_ori_func()
             self.should_stop_service = True
             self.switch = False
-            logger.info("************************************************")
-            logger.info(f"*          {Const.TOOL_NAME} ends successfully.          *")
-            logger.info("************************************************")
+            self.primitive_switch = False
+            print_tools_ends_info()
             return
         if self.config.step and self.current_iter not in self.config.step:
             return
@@ -281,7 +182,7 @@ class Service:
             if self.config.rank and self.current_rank not in self.config.rank:
                 return
             self.register_hook_new()
-            if self.config.level == "L1":
+            if self.config.level in [Const.LEVEL_MIX, Const.LEVEL_L1]:
                 JitDump.set_config(self.config)
                 JitDump.set_data_collector(self.data_collector)
                 ms.common.api._MindsporeFunctionExecutor = JitDump
@@ -291,10 +192,32 @@ class Service:
                     PIJitCaptureContext.__exit__ = self.empty
             self.first_start = False
 
+        api_register.api_set_hook_func()
         self.switch = True
+        self.primitive_switch = True
         logger.info(f"Dump switch is turned on at step {self.current_iter}. ")
         self.create_dirs()
         logger.info(f"Dump data will be saved in {self.dump_iter_dir}.")
+        JitDump.jit_dump_switch = True
+
+    def forward_backward_dump_end(self):
+        if self.should_stop_service:
+            return
+        logger.info(f"{Const.TOOL_NAME}: debugger.forward_backward_dump_end() is set successfully. ")
+        if not self.start_call:
+            logger.error(f"{Const.TOOL_NAME}: debugger.start() is not set in the current scope.")
+            raise Exception("debugger.start() is not set in the current scope.")
+        if not self.switch:
+            logger.error(f"{Const.TOOL_NAME}: debugger.forward_backward_dump_end() should be called between "
+                         "debugger.start() and debugger.stop() ")
+            raise Exception("debugger.stop() is already called. ")
+        if self.config.step and self.current_iter not in self.config.step:
+            return
+        if self.config.rank and self.current_rank not in self.config.rank:
+            return
+        self.primitive_switch = False
+        api_register.api_set_ori_func()
+        JitDump.jit_dump_switch = False
 
     def stop(self):
         if self.should_stop_service:
@@ -309,8 +232,10 @@ class Service:
         if self.config.rank and self.current_rank not in self.config.rank:
             return
         self.switch = False
+        self.primitive_switch = False
         self.start_call = False
         self.data_collector.write_json()
+        JitDump.jit_dump_switch = False
 
     def need_end_service(self):
         if self.config.step and self.current_iter > max(self.config.step):
@@ -349,16 +274,16 @@ class Service:
 
     def register_hook_new(self):
         logger.info("The {} hook function is successfully mounted to the model.".format(self.config.task))
-        if self.config.level == "L1":
+        if self.config.level in [Const.LEVEL_MIX, Const.LEVEL_L1]:
             api_register.initialize_hook(functools.partial(self.build_hook, BaseScope.Module_Type_API))
             api_register.api_set_hook_func()
-            if self.model:
-                self.register_hooks()
+            if self.model and self.config.task in Const.DUMP_DATA_COLLECTION_LIST:
+                self.register_primitive_hooks()
 
-        if self.config.level == "L0":
+        if self.config.level in [Const.LEVEL_MIX, Const.LEVEL_L0]:
             if not self.model:
                 raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR,
-                                       "The current level is L0, the model cannot be None")
+                                       f"The current level is {self.config.level}, the model cannot be None")
             for name, cell in self.model.cells_and_names():
                 if cell == self.model:
                     continue

msprobe/mindspore/task_handler_factory.py

@@ -1,3 +1,18 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 from msprobe.core.common.const import Const
 from msprobe.mindspore.debugger.debugger_config import DebuggerConfig
 from msprobe.mindspore.dump.dump_tool_factory import DumpToolFactory

msprobe/msprobe.py

@@ -45,10 +45,15 @@ def main():
     multi_run_ut_cmd_parser = subparsers.add_parser('multi_run_ut')
     api_precision_compare_cmd_parser = subparsers.add_parser('api_precision_compare')
     run_overflow_check_cmd_parser = subparsers.add_parser('run_overflow_check')
+    graph_service_cmd_parser = subparsers.add_parser('graph')
     _compare_parser(compare_cmd_parser)
-    is_torch_available=is_module_available("torch")
+    is_torch_available = is_module_available("torch")
     is_mindspore_available = is_module_available("mindspore")
-    if is_torch_available:
+    if len(sys.argv) < 4:
+        parser.print_help()
+        sys.exit(0)
+    framework_args = parser.parse_args(sys.argv[1:3])
+    if framework_args.framework == Const.PT_FRAMEWORK:
         from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut import _run_ut_parser, run_ut_command
         from msprobe.pytorch.parse_tool.cli import parse as cli_parse
         from msprobe.pytorch.api_accuracy_checker.run_ut.multi_run_ut import prepare_config, run_parallel_ut
@@ -56,20 +61,24 @@ def main():
             _api_precision_compare_command
         from msprobe.pytorch.api_accuracy_checker.run_ut.run_overflow_check import _run_overflow_check_parser, \
             _run_overflow_check_command
+        from msprobe.visualization.graph_service import _pt_graph_service_parser, _pt_graph_service_command
 
         _run_ut_parser(run_ut_cmd_parser)
         _run_ut_parser(multi_run_ut_cmd_parser)
         multi_run_ut_cmd_parser.add_argument('-n', '--num_splits', type=int, choices=range(1, 65), default=8,
-                                        help='Number of splits for parallel processing. Range: 1-64')
+                                             help='Number of splits for parallel processing. Range: 1-64')
         _api_precision_compare_parser(api_precision_compare_cmd_parser)
         _run_overflow_check_parser(run_overflow_check_cmd_parser)
-    elif is_mindspore_available:
+        _pt_graph_service_parser(graph_service_cmd_parser)
+    elif framework_args.framework == Const.MS_FRAMEWORK:
         from msprobe.mindspore.api_accuracy_checker.cmd_parser import add_api_accuracy_checker_argument
+        from msprobe.visualization.graph_service import _ms_graph_service_parser, _ms_graph_service_command
         add_api_accuracy_checker_argument(run_ut_cmd_parser)
+        from msprobe.mindspore.api_accuracy_checker.cmd_parser import multi_add_api_accuracy_checker_argument
+        multi_add_api_accuracy_checker_argument(multi_run_ut_cmd_parser)
+
+        _ms_graph_service_parser(graph_service_cmd_parser)
 
-    if len(sys.argv) == 1:
-        parser.print_help()
-        sys.exit(0)
     args = parser.parse_args(sys.argv[1:])
     if sys.argv[2] == Const.PT_FRAMEWORK:
         if not is_torch_available:
@@ -86,6 +95,8 @@ def main():
             _api_precision_compare_command(args)
         elif sys.argv[3] == "run_overflow_check":
             _run_overflow_check_command(args)
+        elif sys.argv[3] == "graph":
+            _pt_graph_service_command(args)
         elif sys.argv[3] == "compare":
             if args.cell_mapping is not None or args.api_mapping is not None:
                 logger.error("Argument -cm or -am is not supported in PyTorch framework")
@@ -100,6 +111,12 @@ def main():
         elif sys.argv[3] == "run_ut":
             from msprobe.mindspore.api_accuracy_checker.main import api_checker_main
             api_checker_main(args)
+        elif sys.argv[3] == "multi_run_ut":
+            from msprobe.mindspore.api_accuracy_checker.main import mul_api_checker_main
+            mul_api_checker_main(args)
+        elif sys.argv[3] == "graph":
+            _ms_graph_service_command(args)
+
 
 if __name__ == "__main__":
     main()

msprobe/pytorch/__init__.py

@@ -1,4 +1,24 @@
-from .debugger.precision_debugger import PrecisionDebugger
-from .common.utils import seed_all
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from msprobe.pytorch.monitor.module_hook import TrainerMon
 from .compare.distributed_compare import compare_distributed
-from .compare.pt_compare import compare 
+from .compare.pt_compare import compare
+from .common.utils import seed_all
+from .debugger.precision_debugger import PrecisionDebugger
+from .functional.module_dump import module_dump, module_dump_end

msprobe/pytorch/api_accuracy_checker/common/config.py

@@ -1,8 +1,33 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import os
+from collections import namedtuple
 from msprobe.core.common.file_utils import load_yaml, check_file_or_directory_path
+from msprobe.core.common.utils import is_int
 from msprobe.pytorch.pt_config import RunUTConfig
 
 
+RunUtConfig = namedtuple('RunUtConfig', ['forward_content', 'backward_content', 'result_csv_path', 'details_csv_path',
+                                         'save_error_data', 'is_continue_run_ut', 'real_data_path', 'white_list',
+                                         'black_list', 'error_data_path', 'online_config'])
+OnlineConfig = namedtuple('OnlineConfig', ['is_online', 'nfs_path', 'host', 'port', 'rank_list', 'tls_path'])
+
+
 class Config:
     def __init__(self, yaml_file):
         check_file_or_directory_path(yaml_file, False)
@@ -33,8 +58,10 @@ class Config:
             raise ValueError(f"{key} must be one of {validators.keys()}")
         if not isinstance(value, validators.get(key)):
             raise ValueError(f"{key} must be {validators[key].__name__} type")
-        if key == 'precision' and value < 0:
-            raise ValueError("precision must be greater than 0")
+        if key == 'precision' and not is_int(value):
+            raise ValueError("precision must be an integer")
+        if key == 'precision' and (value < 0 or value > 20):
+            raise ValueError("precision must be greater than or equal to 0 and less than 21")
         if key == 'white_list':
             RunUTConfig.check_filter_list_config(key, value)
         if key == 'black_list':
@@ -51,3 +78,55 @@ class Config:
 cur_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
 yaml_path = os.path.join(cur_path, "config.yaml")
 msCheckerConfig = Config(yaml_path)
+
+
+class CheckerConfig:
+    def __init__(self, task_config=None):
+        self.white_list = msCheckerConfig.white_list
+        self.black_list = msCheckerConfig.black_list
+        self.error_data_path = msCheckerConfig.error_data_path
+        self.is_online = msCheckerConfig.is_online
+        self.nfs_path = msCheckerConfig.nfs_path
+        self.host = msCheckerConfig.host
+        self.port = msCheckerConfig.port
+        self.rank_list = msCheckerConfig.rank_list
+        self.tls_path = msCheckerConfig.tls_path
+
+        if task_config:
+            self.load_config(task_config)
+
+    def load_config(self, task_config):
+        self.white_list = task_config.white_list
+        self.black_list = task_config.black_list
+        self.error_data_path = task_config.error_data_path
+        self.is_online = task_config.is_online
+        self.nfs_path = task_config.nfs_path
+        self.host = task_config.host
+        self.port = task_config.port
+        self.rank_list = task_config.rank_list
+        self.tls_path = task_config.tls_path
+    
+    def get_online_config(self):
+        return OnlineConfig(
+            is_online=self.is_online,
+            nfs_path=self.nfs_path,
+            host=self.host,
+            port=self.port,
+            rank_list=self.rank_list,
+            tls_path=self.tls_path
+        )
+
+    def get_run_ut_config(self, **config_params):
+        return RunUtConfig(
+            forward_content=config_params.get('forward_content'),
+            backward_content=config_params.get('backward_content'),
+            result_csv_path=config_params.get('result_csv_path'),
+            details_csv_path=config_params.get('details_csv_path'),
+            save_error_data=config_params.get('save_error_data'),
+            is_continue_run_ut=config_params.get('is_continue_run_ut'),
+            real_data_path=config_params.get('real_data_path'),
+            white_list=self.white_list,
+            black_list=self.black_list,
+            error_data_path=config_params.get('error_data_path'),
+            online_config=self.get_online_config()
+        )