mindstudio-probe 1.1.0__py3-none-any.whl → 1.2.1__py3-none-any.whl

msprobe/mindspore/compare/ms_graph_compare.py

@@ -1,20 +1,35 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import copy
-import csv
 import glob
 import os
+import re
 
 import numpy as np
 import pandas as pd
-from msprobe.core.common.const import CompareConst, GraphMode, Const, FileCheckConst
-from msprobe.core.common.file_utils import FileOpen, check_path_before_create, change_mode, load_npy
+from msprobe.core.common.const import CompareConst, GraphMode, Const
+from msprobe.core.common.file_utils import load_npy, read_csv, save_excel
 from msprobe.core.common.log import logger
 from msprobe.core.common.utils import add_time_with_xlsx, CompareException
 from msprobe.core.compare.multiprocessing_compute import _ms_graph_handle_multi_process, check_accuracy
-from msprobe.core.compare.npy_compare import npy_data_check, statistics_data_check, reshape_value, compare_ops_apply
+from msprobe.core.compare.npy_compare import npy_data_check, statistics_data_check, compare_ops_apply
 from msprobe.mindspore.common.utils import convert_to_int, list_lowest_level_directories
 
 
-class row_data:
+class RowData:
     def __init__(self, mode):
         self.basic_data = copy.deepcopy(CompareConst.MS_GRAPH_BASE)
         self.npy_data = copy.deepcopy(CompareConst.MS_GRAPH_NPY)
@@ -28,17 +43,34 @@ class row_data:
         return self.data
 
 
+def get_name_dict(name: str) -> dict:
+    compare_pattern = re.compile(r'^([^.]+)\.([^.]+)\.([^.]+)\.([^.]+)\.(\d+(?:\.\d+)*)\.'
+                                 r'((?:in|out)put(?:\.\d+)*)\.([^.]+)\.([^.]+)\.npy$')
+    match = compare_pattern.match(name)
+    if match:
+        return {'op_type': match.group(1),
+                'op_name': match.group(2),
+                'task_id': match.group(3),
+                'stream_id': match.group(4),
+                'timestamp': match.group(5).split(Const.SEP)[0],
+                'input_output_index': match.group(6),
+                'slot': match.group(7),
+                'format': match.group(8)}
+    return {}
+
+
 def npy_data_read(data_path, npy_file_list, mapping_dict):
     data_list = []
+    compare_key_elements = ['op_name', 'task_id', 'input_output_index', 'slot']
     for data in npy_file_list:
         if data in mapping_dict:
-            split_list = mapping_dict[data].split(Const.SEP)
+            name_dict = get_name_dict(mapping_dict[data])
         else:
-            split_list = data.split(Const.SEP)
-        if len(split_list) < 7:
+            name_dict = get_name_dict(data)
+        if not name_dict:
             continue
-        compare_key = f"{split_list[1]}.{split_list[2]}.{split_list[3]}.{split_list[5]}.{split_list[6]}"
-        timestamp = convert_to_int(split_list[4])
+        compare_key = Const.SEP.join([name_dict.get(element) for element in compare_key_elements])
+        timestamp = convert_to_int(name_dict.get('timestamp'))
 
         data_list.append([os.path.join(data_path, data), compare_key, timestamp])
     return data_list
@@ -48,18 +80,17 @@ def statistic_data_read(statistic_file_list, statistic_file_path):
     data_list = []
     statistic_data_list = []
     header_index = {
-        'Data Type': None, 'Shape': None, 'Max Value': None, 
-        'Min Value': None,'Avg Value': None, 'L2Norm Value': None
+        'Data Type': None, 'Shape': None, 'Max Value': None,
+        'Min Value': None, 'Avg Value': None, 'L2Norm Value': None
     }
     for statistic_file in statistic_file_list:
-        with FileOpen(statistic_file, "r") as f:
-            csv_reader = csv.reader(f, delimiter=",")
-            header = next(csv_reader)
-            for key in header_index.keys():
-                for index, value in enumerate(header):
-                    if key == value:
-                        header_index[key] = index
-            statistic_data_list.extend([row for row in csv_reader])
+        content = read_csv(statistic_file, as_pd=False)
+        header = content[0]
+        for key in header_index.keys():
+            for index, value in enumerate(header):
+                if key == value:
+                    header_index[key] = index
+        statistic_data_list.extend(content[1:])
 
     for key in header_index.keys():
         if header_index[key] is None:
@@ -97,11 +128,9 @@ def generate_data_name(data_path):
     mapping_dict = {}
     if mapping_exist:
         for mapping_file in mapping_file_list:
-            with FileOpen(mapping_file, "r") as f:
-                csv_reader = csv.reader(f, delimiter=",")
-                header = next(csv_reader)
-                for row in csv_reader:
-                    mapping_dict[row[0]] = row[1]
+            content = read_csv(mapping_file, False)
+            for row in content[1:]:
+                mapping_dict[row[0]] = row[1]
 
     if npy_exist:
         data_list = npy_data_read(data_path, npy_file_list, mapping_dict)
@@ -115,10 +144,16 @@ def generate_data_name(data_path):
         mode = GraphMode.STATISTIC_MODE
     else:
         mode = GraphMode.ERROR_MODE
-        logger.error(f"Error mode.")
+        logger.error("Error mode.")
     return mode, data_list
 
 
+def transform_special_string_into_float(data_frame):
+    data_frame[data_frame == "null"] = '0'
+    data_frame[data_frame == "False"] = '0'
+    data_frame[data_frame == "True"] = '1'
+
+
 class GraphMSComparator:
     def __init__(self, input_param, output_path):
         self.output_path = output_path
@@ -136,7 +171,7 @@ class GraphMSComparator:
     def compare_ops(compare_result_db, mode):
 
         def npy_mode_compute(row):
-            result_dict = row_data(GraphMode.NPY_MODE)()
+            result_dict = RowData(GraphMode.NPY_MODE)()
 
             def process_npy_file(file_path, name_prefix, result):
                 if os.path.exists(file_path):
@@ -158,7 +193,6 @@ class GraphMSComparator:
             result_dict[CompareConst.ERROR_MESSAGE] = error_message
 
             if not error_flag:
-                n_value, b_value = reshape_value(n_value, b_value)
                 result_list, err_msg = compare_ops_apply(n_value, b_value, False, "")
                 result_dict[CompareConst.COSINE] = result_list[0]
                 result_dict[CompareConst.MAX_ABS_ERR] = result_list[1]
@@ -171,7 +205,7 @@ class GraphMSComparator:
             return pd.Series(result_dict)
 
         def statistic_mode_compute(row):
-            result_dict = row_data('STATISTIC')()
+            result_dict = RowData('STATISTIC')()
 
             def update_result_dict(result, rows, prefix):
                 result[f'{prefix} Name'] = rows[f'{prefix} Name']
@@ -198,24 +232,30 @@ class GraphMSComparator:
                     result_dict[CompareConst.NPU_NORM] - result_dict[CompareConst.BENCH_NORM])
                 result_dict[CompareConst.MAX_RELATIVE_ERR] = result_dict[CompareConst.MAX_DIFF] / result_dict[
                     CompareConst.BENCH_MAX] if result_dict[CompareConst.BENCH_MAX] > 0 else 0
-                result_dict[CompareConst.MAX_RELATIVE_ERR] = str(result_dict[CompareConst.MAX_RELATIVE_ERR] * 100) + "%"
+                if not np.isnan(result_dict[CompareConst.MAX_RELATIVE_ERR]):
+                    result_dict[CompareConst.MAX_RELATIVE_ERR] = str(
+                        result_dict[CompareConst.MAX_RELATIVE_ERR] * 100) + "%"
                 result_dict[CompareConst.MIN_RELATIVE_ERR] = result_dict[CompareConst.MIN_DIFF] / result_dict[
                     CompareConst.BENCH_MIN] if result_dict[CompareConst.BENCH_MIN] > 0 else 0
-                result_dict[CompareConst.MIN_RELATIVE_ERR] = str(result_dict[CompareConst.MIN_RELATIVE_ERR] * 100) + "%"
+                if not np.isnan(result_dict[CompareConst.MIN_RELATIVE_ERR]):
+                    result_dict[CompareConst.MIN_RELATIVE_ERR] = \
+                        str(result_dict[CompareConst.MIN_RELATIVE_ERR] * 100) + "%"
                 result_dict[CompareConst.MEAN_RELATIVE_ERR] = result_dict[CompareConst.MEAN_DIFF] / result_dict[
                     CompareConst.BENCH_MEAN] if result_dict[CompareConst.BENCH_MEAN] > 0 else 0
-                result_dict[CompareConst.MEAN_RELATIVE_ERR] = str(
-                    result_dict[CompareConst.MEAN_RELATIVE_ERR] * 100) + "%"
+                if not np.isnan(result_dict[CompareConst.MEAN_RELATIVE_ERR]):
+                    result_dict[CompareConst.MEAN_RELATIVE_ERR] = str(
+                        result_dict[CompareConst.MEAN_RELATIVE_ERR] * 100) + "%"
                 result_dict[CompareConst.NORM_RELATIVE_ERR] = result_dict[CompareConst.NORM_DIFF] / result_dict[
                     CompareConst.BENCH_NORM] if result_dict[CompareConst.BENCH_NORM] > 0 else 0
-                result_dict[CompareConst.NORM_RELATIVE_ERR] = str(
-                    result_dict[CompareConst.NORM_RELATIVE_ERR] * 100) + "%"
+                if not np.isnan(result_dict[CompareConst.NORM_RELATIVE_ERR]):
+                    result_dict[CompareConst.NORM_RELATIVE_ERR] = str(
+                        result_dict[CompareConst.NORM_RELATIVE_ERR] * 100) + "%"
                 magnitude_diff = result_dict[CompareConst.MAX_DIFF] / (
                         max(result_dict[CompareConst.NPU_MAX], result_dict[CompareConst.BENCH_MAX]) + 1e-10)
-                if magnitude_diff > CompareConst.MAGNITUDE:
-                    result_dict[CompareConst.ACCURACY] = 'No'
-                else:
-                    result_dict[CompareConst.ACCURACY] = 'Yes'
+                if np.isnan(result_dict[CompareConst.NPU_MAX]) and np.isnan(result_dict[CompareConst.BENCH_MAX]):
+                    magnitude_diff = 0
+                result_dict[CompareConst.ACCURACY] = CompareConst.YES if \
+                    magnitude_diff <= CompareConst.MAGNITUDE else CompareConst.NO
 
             return pd.Series(result_dict)
 
@@ -238,24 +278,23 @@ class GraphMSComparator:
                 is_empty = True
             if is_empty or not mode:
                 continue
-            compare_result_df = self._do_multi_process(compare_result_df, mode)
+            compare_result_df = self.do_multi_process(compare_result_df, mode)
             compare_result_name = add_time_with_xlsx(f"compare_result_{str(rank_id)}_{str(step_id)}")
             compare_result_path = os.path.join(os.path.realpath(self.output_path), f"{compare_result_name}")
-            check_path_before_create(compare_result_path)
             self.to_excel(compare_result_df, compare_result_path)
             logger.info(f"Compare rank: {rank_id} step: {step_id} finish. Compare result: {compare_result_path}.")
-    
+
     def to_excel(self, compare_result_df: pd.DataFrame, compare_result_path: str, slice_num=0, need_slice=False) -> int:
         size = len(compare_result_df)
         # sheet size cannot be larger than 1048576
         if size < CompareConst.MAX_EXCEL_LENGTH:
-            compare_result_path = compare_result_path.replace('.xlsx', f'_slice_{slice_num}.xlsx') if need_slice else compare_result_path
-            compare_result_df.to_excel(compare_result_path, index=False)
-            change_mode(compare_result_path, FileCheckConst.DATA_FILE_AUTHORITY)
+            compare_result_path = compare_result_path.replace('.xlsx', f'_slice_{slice_num}.xlsx') if \
+                need_slice else compare_result_path
+            save_excel(compare_result_path, compare_result_df)
             return slice_num + 1
         else:
-            slice_num = self.to_excel(compare_result_df.iloc[0: size//2], compare_result_path, slice_num, True)
-            return self.to_excel(compare_result_df.iloc[size//2:], compare_result_path, slice_num, True)
+            slice_num = self.to_excel(compare_result_df.iloc[0: size // 2], compare_result_path, slice_num, True)
+            return self.to_excel(compare_result_df.iloc[size // 2:], compare_result_path, slice_num, True)
 
     def compare_process(self, rank_id, step_id):
         # generate data_path
@@ -300,13 +339,17 @@ class GraphMSComparator:
                                                   CompareConst.BENCH_NORM])
 
             npu_float_type = [CompareConst.NPU_MAX, CompareConst.NPU_MIN, CompareConst.NPU_MEAN, CompareConst.NPU_NORM]
-            npu_data_df[npu_float_type] = npu_data_df[npu_float_type].astype(float)
+            npu_float_data_df = npu_data_df[npu_float_type].astype(str)
+            transform_special_string_into_float(npu_float_data_df)
+            npu_data_df[npu_float_type] = npu_float_data_df.astype(float)
 
             bench_float_type = [
-                CompareConst.BENCH_MAX, CompareConst.BENCH_MIN, 
-                CompareConst.BENCH_MEAN,CompareConst.BENCH_NORM
+                CompareConst.BENCH_MAX, CompareConst.BENCH_MIN,
+                CompareConst.BENCH_MEAN, CompareConst.BENCH_NORM
             ]
-            bench_data_df[bench_float_type] = bench_data_df[bench_float_type].astype(float)
+            bench_float_data_df = bench_data_df[bench_float_type].astype(str)
+            transform_special_string_into_float(bench_float_data_df)
+            bench_data_df[bench_float_type] = bench_float_data_df.astype(float)
 
         npu_data_df['Local Index'] = npu_data_df.sort_values('TimeStamp').groupby('Compare Key').cumcount()
         bench_data_df['Local Index'] = bench_data_df.sort_values('TimeStamp').groupby('Compare Key').cumcount()
@@ -355,7 +398,7 @@ class GraphMSComparator:
                 rank_step_path_dict[rank_step_key] = [dir_path]
         return dict(sorted(rank_step_path_dict.items()))
 
-    def _do_multi_process(self, result_df, mode):
+    def do_multi_process(self, result_df, mode):
         try:
             result_df = _ms_graph_handle_multi_process(self.compare_ops, result_df, mode)
         except ValueError as e:

msprobe/mindspore/debugger/debugger_config.py

@@ -33,12 +33,13 @@ class DebuggerConfig:
         self.level_ori = common_config.level
         self.list = [] if not task_config.list else task_config.list
         self.scope = [] if not task_config.scope else task_config.scope
-        self.data_mode = [] if not task_config.data_mode else task_config.data_mode
+        self.data_mode = [Const.ALL] if not task_config.data_mode else task_config.data_mode
         self.file_format = task_config.file_format
         self.overflow_nums = 1 if not task_config.overflow_nums else task_config.overflow_nums
         self.check_mode = task_config.check_mode
         self.framework = Const.MS_FRAMEWORK
         self.summary_mode = task_config.summary_mode
+        self.async_dump = common_config.async_dump if common_config.async_dump else False
         self.check()
         create_directory(self.dump_path)
 
@@ -52,6 +53,9 @@ class DebuggerConfig:
                self.pert_type != FreeBenchmarkConst.DEFAULT_PERT_TYPE:
                 raise ValueError("pert_mode must be improve_precision or empty when handler_type is fix, "
                                  f"but got {self.pert_type}.")
+            if self.stage == Const.BACKWARD and self.handler_type == FreeBenchmarkConst.FIX:
+                raise ValueError("handler_type must be check or empty when fuzz_stage is backward, "
+                                 f"but got {self.handler_type}.")
             self.dump_level = FreeBenchmarkConst.DEFAULT_DUMP_LEVEL
 
     def check(self):
@@ -66,4 +70,6 @@ class DebuggerConfig:
             self.file_format = "npy"
         if not self.check_mode:
             self.check_mode = "all"
+        if not isinstance(self.async_dump, bool):
+            raise Exception("The parameters async_dump should be bool.")
         return True

msprobe/mindspore/debugger/precision_debugger.py

@@ -1,7 +1,7 @@
-# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd.
 # All rights reserved.
 #
-# Licensed under the Apache License, Version 2.0  (the "License");
+# Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
@@ -14,25 +14,42 @@
 # limitations under the License.
 
 import os
+from collections import defaultdict, namedtuple
 
 import mindspore as ms
 from mindspore._c_expression import MSContext
 
-from msprobe.core.common.const import Const, MsgConst
+from msprobe.core.common.const import Const, FileCheckConst, MsgConst
+from msprobe.core.common.exceptions import MsprobeException
+from msprobe.core.common.file_utils import FileChecker
+from msprobe.core.common.utils import get_real_step_or_rank
+from msprobe.mindspore.cell_processor import CellProcessor
 from msprobe.mindspore.common.const import Const as MsConst
+from msprobe.mindspore.common.utils import set_register_backward_hook_functions
 from msprobe.mindspore.debugger.debugger_config import DebuggerConfig
+from msprobe.mindspore.dump.hook_cell.api_registry import api_register
+from msprobe.mindspore.dump.hook_cell.hook_cell import HOOKCell
 from msprobe.mindspore.grad_probe.grad_monitor import GradientMonitor
 from msprobe.mindspore.ms_config import parse_json_config
 from msprobe.mindspore.runtime import Runtime
 from msprobe.mindspore.service import Service
 from msprobe.mindspore.task_handler_factory import TaskHandlerFactory
 
+try:
+    from msprobe.lib import _msprobe_c
+except ImportError:
+    _msprobe_c = None
+
+
+ConfigParameters = namedtuple("ConfigParameters", ["config_path", "task", "dump_path", "level"])
+
 
 class PrecisionDebugger:
     _instance = None
     task_not_need_service = [Const.GRAD_PROBE]
 
-    def __new__(cls, config_path=None, opt=None):
+    def __new__(cls, config_path=None, task=None, dump_path=None,
+                level=None, step=None, opt=None):
         if not cls._instance:
             cls._instance = super().__new__(cls)
             cls._instance.initialized = False
@@ -41,22 +58,65 @@ class PrecisionDebugger:
             cls.first_start = False
         return cls._instance
 
-    def __init__(self, config_path=None):
+    def __init__(self, config_path=None, task=None, dump_path=None,
+                 level=None, step=None):
         if self.initialized:
             return
         self.initialized = True
+
+        set_register_backward_hook_functions()
+
         if not config_path:
             config_path = os.path.join(os.path.dirname(__file__), "../../config.json")
+
+        config_params = ConfigParameters(config_path, task, dump_path, level)
+        self.check_input_params(config_params)
+
         common_config, task_config = parse_json_config(config_path)
+        common_config.task = task if task else common_config.task
         self.task = common_config.task
         if self.task == Const.GRAD_PROBE:
             self.gm = GradientMonitor(common_config, task_config)
             return
+        common_config.step = get_real_step_or_rank(
+            step, Const.STEP) if step is not None else common_config.step
+        common_config.level = level if level else common_config.level
+        common_config.dump_path = dump_path if dump_path else common_config.dump_path
         self.config = DebuggerConfig(common_config, task_config)
 
+        if _msprobe_c:
+            _msprobe_c._PrecisionDebugger(framework="MindSpore", config_path=config_path)
+
+        self.config.execution_mode = self._get_execution_mode()
+        if self._need_service():
+            self.service = Service(self.config)
+
         Runtime.step_count = 0
         Runtime.is_running = False
 
+    @staticmethod
+    def check_input_params(args):
+        if args.config_path is not None:
+            if not isinstance(args.config_path, str):
+                raise MsprobeException(
+                    MsprobeException.INVALID_PARAM_ERROR, f"config_path must be a string")
+            file_checker = FileChecker(
+                file_path=args.config_path, path_type=FileCheckConst.FILE, file_type=FileCheckConst.JSON_SUFFIX)
+            file_checker.common_check()
+
+        if args.task is not None and args.task not in Const.TASK_LIST:
+            raise MsprobeException(
+                MsprobeException.INVALID_PARAM_ERROR, f"task must be one of {Const.TASK_LIST}")
+
+        if args.dump_path is not None:
+            if not isinstance(args.dump_path, str):
+                raise MsprobeException(
+                    MsprobeException.INVALID_PARAM_ERROR, f"dump_path must be a string")
+
+        if args.level is not None and args.level not in Const.LEVEL_LIST:
+            raise MsprobeException(
+                MsprobeException.INVALID_PARAM_ERROR, f"level must be one of {Const.LEVEL_LIST}")
+
     @staticmethod
     def _get_execution_mode():
         jit_level = ms.context.get_jit_config().get(MsConst.JIT_LEVEL)
@@ -75,11 +135,23 @@ class PrecisionDebugger:
         else:
             return MsConst.PYNATIVE_MODE
 
+    @staticmethod
+    def _is_graph_dump(config):
+        if config.level != MsConst.KERNEL:
+            return False
+        if not config.list or len(config.list) > 1:
+            return True
+        if '-' in config.list[0] or '/' in config.list[0]:
+            return True
+        return False
+
     @classmethod
     def start(cls, model=None):
         instance = cls._instance
         if not instance:
             raise Exception(MsgConst.NOT_CREATED_INSTANCE)
+        if _msprobe_c:
+            _msprobe_c._PrecisionDebugger().start()
         if instance.task in PrecisionDebugger.task_not_need_service:
             return
 
@@ -90,6 +162,7 @@ class PrecisionDebugger:
             instance.service.start(model)
         else:
             if not instance.first_start:
+                api_register.api_set_ori_func()
                 handler = TaskHandlerFactory.create(instance.config)
                 handler.handle()
 
@@ -99,18 +172,15 @@ class PrecisionDebugger:
     @classmethod
     def forward_backward_dump_end(cls):
         instance = cls._instance
-        if not instance:
-            raise Exception(MsgConst.NOT_CREATED_INSTANCE)
-        if instance.task in PrecisionDebugger.task_not_need_service:
-            return
-        if instance.service:
-            instance.service.forward_backward_dump_end()
+        instance.stop()
 
     @classmethod
     def stop(cls):
         instance = cls._instance
         if not instance:
             raise Exception(MsgConst.NOT_CREATED_INSTANCE)
+        if _msprobe_c:
+            _msprobe_c._PrecisionDebugger().stop()
         if instance.task == Const.GRAD_PROBE:
             instance.gm.stop()
         if instance.task in PrecisionDebugger.task_not_need_service:
@@ -124,10 +194,15 @@ class PrecisionDebugger:
         instance = cls._instance
         if not instance:
             raise Exception(MsgConst.NOT_CREATED_INSTANCE)
+        if _msprobe_c:
+            _msprobe_c._PrecisionDebugger().step()
         if instance.task in PrecisionDebugger.task_not_need_service:
             return
         if instance.service:
             instance.service.step()
+        HOOKCell.cell_count = defaultdict(int)
+        CellProcessor.reset_cell_stats()
+
         Runtime.step_count += 1
 
     @classmethod
@@ -147,4 +222,4 @@ class PrecisionDebugger:
         if instance.config.execution_mode != MsConst.PYNATIVE_MODE:
             return False
         else:
-            return instance.config.task != Const.FREE_BENCHMARK and instance.config.level != MsConst.KERNEL
+            return instance.config.task != Const.FREE_BENCHMARK and not instance._is_graph_dump(instance.config)

msprobe/mindspore/dump/dump_tool_factory.py

@@ -1,7 +1,7 @@
 # Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
 # All rights reserved.
 #
-# Licensed under the Apache License, Version 2.0  (the "License");
+# Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
@@ -40,6 +40,8 @@ class DumpToolFactory:
 
     @staticmethod
     def create(config: DebuggerConfig):
+        if len(config.data_mode) != 1 or config.data_mode[0] not in Const.GRAPH_DATA_MODE_LIST:
+            raise Exception("data_mode must be one of all, input, output.")
         tool = DumpToolFactory.tools.get(config.level)
         if not tool:
             raise Exception("Valid level is needed.")