mindstudio-probe 8.3.3__py3-none-any.whl → 26.0.0a1__py3-none-any.whl

msprobe/mindspore/free_benchmark/handler/check_handler.py

@@ -1,55 +0,0 @@
-# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
-# All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from dataclasses import asdict
-from typing import Any
-
-from mindspore import Tensor, ops
-
-from msprobe.core.data_dump.json_writer import DataWriter
-from msprobe.mindspore.common.log import logger
-from msprobe.mindspore.free_benchmark.common.config import Config
-from msprobe.mindspore.free_benchmark.common.handler_params import HandlerParams
-from msprobe.mindspore.free_benchmark.common.utils import make_unequal_row
-from msprobe.mindspore.free_benchmark.handler.base_handler import BaseHandler
-
-
-class CheckHandler(BaseHandler):
-
-    def npu_compare_and_save(self, original_output, fuzzed_output, params: HandlerParams, output_index=None):
-        is_consistent, ratio = self.npu_compare(original_output, fuzzed_output)
-        params.is_consistent = params.is_consistent and is_consistent
-        if not is_consistent:
-            row = make_unequal_row(self.api_name_with_id, params, ratio, output_index)
-            data_dict = asdict(row)
-            DataWriter.write_data_to_csv(
-                data_dict.values(),
-                data_dict.keys(),
-                Config.dump_path
-            )
-            logger.error(f"{self.api_name_with_id} is not consistent")
-
-    def handle(self, params: HandlerParams) -> Any:
-        try:
-            if not self.is_float_tensor(params.fuzzed_result):
-                return
-            if isinstance(params.fuzzed_result, Tensor):
-                self.npu_compare_and_save(params.original_result, params.fuzzed_result, params)
-            elif isinstance(params.fuzzed_result, (list, tuple)):
-                for i, item in enumerate(params.original_result):
-                    if ops.is_tensor(item) and ops.is_floating_point(item):
-                        self.npu_compare_and_save(item, params.fuzzed_result[i], params, output_index=i)
-        except Exception as e:
-            logger.error(str(e))

msprobe/mindspore/free_benchmark/handler/fix_handler.py

@@ -1,51 +0,0 @@
-# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
-# All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from typing import Any
-
-from mindspore import Tensor
-
-from msprobe.mindspore.common.log import logger
-from msprobe.mindspore.free_benchmark.common.handler_params import HandlerParams
-
-
-class FixHandler:
-
-    def __init__(self, api_name_with_id: str):
-        self.api_name_with_id = api_name_with_id
-
-    @staticmethod
-    def use_fuzzed_result(original_result, fuzzed_result):
-        if isinstance(original_result, Tensor):
-            return fuzzed_result.to(original_result.dtype)
-        if isinstance(original_result, dict):
-            dict_fixed_result = dict()
-            for k, v in original_result.items():
-                dict_fixed_result[k] = FixHandler.use_fuzzed_result(v, fuzzed_result[k])
-            return dict_fixed_result
-        if isinstance(original_result, (tuple, list)):
-            list_fixed_result = list()
-            for i, v in enumerate(original_result):
-                list_fixed_result.append(FixHandler.use_fuzzed_result(v, fuzzed_result[i]))
-            return type(original_result)(list_fixed_result)
-        return original_result
-
-    def handle(self, params: HandlerParams) -> Any:
-        try:
-            return FixHandler.use_fuzzed_result(params.original_result, params.fuzzed_result)
-        except Exception as e:
-            logger.error(f"{self.api_name_with_id} failed to fix.")
-            logger.error(str(e))
-            return params.original_result

msprobe/mindspore/free_benchmark/handler/handler_factory.py

@@ -1,36 +0,0 @@
-# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
-# All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from msprobe.mindspore.common.const import FreeBenchmarkConst
-from msprobe.mindspore.common.log import logger
-from msprobe.mindspore.free_benchmark.common.config import Config
-from msprobe.mindspore.free_benchmark.handler.check_handler import CheckHandler
-from msprobe.mindspore.free_benchmark.handler.fix_handler import FixHandler
-
-
-class HandlerFactory:
-    result_handlers = {
-        FreeBenchmarkConst.CHECK: CheckHandler,
-        FreeBenchmarkConst.FIX: FixHandler
-    }
-
-    @staticmethod
-    def create(api_name_with_id: str):
-        handler = HandlerFactory.result_handlers.get(Config.handler_type)
-        if handler:
-            return handler(api_name_with_id)
-        else:
-            logger.error(f"{Config.handler_type} is not supported.")
-            raise Exception

msprobe/mindspore/free_benchmark/perturbation/add_noise.py

@@ -1,82 +0,0 @@
-# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
-# All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from typing import Any
-
-from mindspore import Tensor, ops
-
-from msprobe.mindspore.common.const import FreeBenchmarkConst
-from msprobe.mindspore.common.log import logger
-from msprobe.mindspore.free_benchmark.common.handler_params import HandlerParams
-from msprobe.mindspore.free_benchmark.perturbation.base_perturbation import BasePerturbation
-
-
-class AddNoisePerturbation(BasePerturbation):
-
-    def handle(self, params: HandlerParams) -> Any:
-        """
-        返回增加扰动后的api输出
-
-        """
-        params.fuzzed_value = self.add_noise(params.args[params.index])
-        if not self.is_fuzzed:
-            logger.warning(f"{self.api_name_with_id} can not add noise.")
-            return False
-        return self.get_fuzzed_result(params)
-
-    def add_noise(self, inputs) -> Any:
-        """
-        返回增加扰动后的api输入
-
-        """
-        if isinstance(inputs, Tensor):
-            noise = self._get_noise(inputs)
-            if noise is not False:
-                result = ops.where(ops.abs(inputs) > self.perturbation_value ** 0.5,
-                                   ops.add(noise, inputs), inputs)
-                result = result.type(dtype=inputs.dtype)
-                self.is_fuzzed = True
-                return result
-
-        if isinstance(inputs, dict):
-            return {k: self.add_noise(v) for k, v in inputs.items()}
-
-        if isinstance(inputs, (list, tuple)):
-            return [self.add_noise(v) for v in inputs]
-
-        return inputs
-
-    def _get_noise(self, tensor):
-        """
-        得到要添加的噪声值
-
-        """
-        if self.is_fuzzed:
-            return False
-        if not ops.is_floating_point(tensor) or ops.numel(tensor) == 0:
-            return False
-
-        pert_value = FreeBenchmarkConst.PERT_VALUE_DICT.get(tensor.dtype)
-        if not pert_value:
-            return False
-        else:
-            self.perturbation_value = pert_value
-
-        max_val = ops.max(ops.abs(tensor))[0].item()
-        if max_val < pert_value:
-            return False
-
-        noise = ops.full(tensor.shape, self.perturbation_value, dtype=tensor.dtype)
-        return noise

msprobe/mindspore/free_benchmark/perturbation/base_perturbation.py

@@ -1,45 +0,0 @@
-# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
-# All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from typing import Any
-
-from msprobe.core.common.const import Const
-from msprobe.mindspore.free_benchmark.common.config import Config
-from msprobe.mindspore.free_benchmark.common.handler_params import HandlerParams
-from msprobe.mindspore.free_benchmark.common.utils import Tools
-
-
-class BasePerturbation:
-
-    def __init__(self, api_name_with_id: str):
-        self.api_name_with_id = api_name_with_id
-        self.is_fuzzed = False
-        self.perturbation_value = None
-
-    @staticmethod
-    def get_fuzzed_result(params: HandlerParams):
-        if Config.stage == Const.BACKWARD:
-            fuzzed_result = Tools.get_grad(params.original_func, *params.args[:params.index],
-                                           params.fuzzed_value, *params.args[params.index + 1:], **params.kwargs)
-
-            if fuzzed_result is None:
-                return False
-        else:
-            fuzzed_result = params.original_func(*params.args[:params.index], params.fuzzed_value,
-                                                 *params.args[params.index + 1:], **params.kwargs)
-        return fuzzed_result
-
-    def handler(self, params: HandlerParams) -> Any:
-        pass

msprobe/mindspore/free_benchmark/perturbation/bit_noise.py

@@ -1,78 +0,0 @@
-# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
-# All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from typing import Any
-
-import numpy as np
-from mindspore import Tensor, ops
-
-from msprobe.mindspore.common.const import FreeBenchmarkConst
-from msprobe.mindspore.common.log import logger
-from msprobe.mindspore.free_benchmark.common.handler_params import HandlerParams
-from msprobe.mindspore.free_benchmark.perturbation.base_perturbation import BasePerturbation
-
-
-class BitNoisePerturbation(BasePerturbation):
-
-    def add_bit_noise(self, inputs) -> Any:
-        if isinstance(inputs, Tensor):
-            bit_len_type = self._get_bit_len_type(inputs)
-            if bit_len_type is not False:
-                sub_normal_np = np.finfo(FreeBenchmarkConst.MS_NUMPY_DTYPE_DICT.get(inputs.dtype)).smallest_normal
-                sub_normal = Tensor(sub_normal_np)
-                noise_type = list(FreeBenchmarkConst.MS_NUMPY_DTYPE_DICT.keys())[
-                             list(FreeBenchmarkConst.MS_NUMPY_DTYPE_DICT.values()).index(bit_len_type)]
-                noise = ops.full(inputs.shape, 1, dtype=noise_type)
-                input_np = inputs.asnumpy()
-                input_np_int = input_np.view(bit_len_type)
-                result = Tensor(input_np_int)
-                result = ops.where(ops.abs(inputs) > sub_normal,
-                                   ops.bitwise_xor(result, noise), result)
-                result_np = result.asnumpy()
-                result_np_float = result_np.view(FreeBenchmarkConst.MS_NUMPY_DTYPE_DICT.get(inputs.dtype))
-                self.is_fuzzed = True
-                return Tensor(result_np_float)
-
-        if isinstance(inputs, dict):
-            return {k: self.add_bit_noise(v) for k, v in inputs.items()}
-        if isinstance(inputs, (tuple, list)):
-            return type(inputs)([self.add_bit_noise(v) for v in inputs])
-        return inputs
-
-    def handle(self, params: HandlerParams) -> any:
-        args = params.args
-        params.fuzzed_value = self.add_bit_noise(params.args[params.index])
-        if not self.is_fuzzed:
-            logger.warning(f"{self.api_name_with_id} can not add bit noise.")
-            return False
-        params.args = args
-        return self.get_fuzzed_result(params)
-
-    def _get_bit_len_type(self, tensor):
-        if self.is_fuzzed:
-            return False
-        if not isinstance(tensor, Tensor) or not ops.is_floating_point(tensor) or \
-           tensor.numel() == 0:
-            return False
-        bit_len_type = FreeBenchmarkConst.PERT_BIT_DICT.get(tensor.dtype)
-        if not bit_len_type:
-            return False
-        pert_value = FreeBenchmarkConst.PERT_VALUE_DICT.get(tensor.dtype)
-        if not pert_value:
-            return False
-        max_val = ops.max(ops.abs(tensor))[0].item()
-        if max_val < pert_value:
-            return False
-        return bit_len_type

msprobe/mindspore/free_benchmark/perturbation/exchange_value.py

@@ -1,77 +0,0 @@
-# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
-# All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from typing import Any
-
-from mindspore import Tensor, ops
-
-from msprobe.mindspore.common.log import logger
-from msprobe.mindspore.free_benchmark.common.handler_params import HandlerParams
-from msprobe.mindspore.free_benchmark.perturbation.base_perturbation import BasePerturbation
-
-
-class ExchangeValuePerturbation(BasePerturbation):
-
-    @staticmethod
-    def _check_tensor_shape(inputs):
-        dims = len(inputs.shape)
-        if dims == 1 and inputs.shape[0] > 1:
-            return True
-        if dims > 1 and inputs.shape[1] > 0:
-            if inputs.shape[0] > 1 or inputs.shape[1] > 1:
-                return True
-        return False
-
-    def handle(self, params: HandlerParams) -> Any:
-        """
-        返回首尾交换后的api输出
-
-        """
-        params.fuzzed_value = self.exchange_value(params.args[params.index])
-        if not self.is_fuzzed:
-            logger.warning(f"{self.api_name_with_id} can not exchange value.")
-            return False
-        return self.get_fuzzed_result(params)
-
-    def exchange_value(self, inputs) -> Any:
-        """
-        返回首尾交换后的api输入
-
-        """
-        if isinstance(inputs, Tensor) and ops.is_floating_point(inputs):
-            if self.is_fuzzed or not self._check_tensor_shape(inputs):
-                return inputs
-            result = inputs.copy()
-            if len(inputs.shape) == 1:
-                first_element = inputs[0]
-                last_element = inputs[-1]
-                result[0] = last_element
-                result[-1] = first_element
-            else:
-                first_element = inputs[0][0]
-                last_element = inputs[-1][-1]
-                result[0][0] = last_element
-                result[-1][-1] = first_element
-
-            self.is_fuzzed = True
-            return result
-
-        if isinstance(inputs, dict):
-            return {k: self.exchange_value(v) for k, v in inputs.items()}
-
-        if isinstance(inputs, (list, tuple)):
-            return type(inputs)([self.exchange_value(v) for v in inputs])
-
-        return inputs

msprobe/mindspore/free_benchmark/perturbation/improve_precision.py

@@ -1,56 +0,0 @@
-# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
-# All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from typing import Any
-
-import mindspore as ms
-from mindspore import Tensor, ops
-
-from msprobe.core.common.const import Const
-from msprobe.mindspore.common.log import logger
-from msprobe.mindspore.free_benchmark.common.config import Config
-from msprobe.mindspore.free_benchmark.common.handler_params import HandlerParams
-from msprobe.mindspore.free_benchmark.common.utils import Tools
-from msprobe.mindspore.free_benchmark.perturbation.base_perturbation import BasePerturbation
-
-
-class ImprovePrecisionPerturbation(BasePerturbation):
-
-    def improve_tensor_precision(self, target_tensor):
-        if isinstance(target_tensor, Tensor) and ops.is_floating_point(target_tensor) and \
-           target_tensor.dtype not in [ms.float64, ms.float32]:
-            self.is_fuzzed = True
-            return target_tensor.to(ms.float32)
-        if isinstance(target_tensor, dict):
-            return {k: self.improve_tensor_precision(v) for k, v in target_tensor.items()}
-        if isinstance(target_tensor, (tuple, list)):
-            return type(target_tensor)([self.improve_tensor_precision(v) for v in target_tensor])
-        return target_tensor
-
-    def handle(self, params: HandlerParams) -> Any:
-        args = self.improve_tensor_precision(params.args)
-        kwargs = self.improve_tensor_precision(params.kwargs)
-        if not self.is_fuzzed:
-            logger.warning(f"{self.api_name_with_id} can not improve precision.")
-            return False
-
-        if Config.stage == Const.BACKWARD:
-            fuzzed_result = Tools.get_grad(params.original_func, *args, **kwargs)
-            if fuzzed_result is not None:
-                return fuzzed_result
-            else:
-                return False
-
-        return params.original_func(*args, **kwargs)

msprobe/mindspore/free_benchmark/perturbation/no_change.py

@@ -1,27 +0,0 @@
-# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
-# All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0  (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from typing import Any
-
-from msprobe.mindspore.free_benchmark.common.handler_params import HandlerParams
-from msprobe.mindspore.free_benchmark.perturbation.base_perturbation import BasePerturbation
-
-
-class NoChangePerturbation(BasePerturbation):
-
-    def handle(self, params: HandlerParams) -> Any:
-        params.fuzzed_value = params.args[params.index]
-        self.is_fuzzed = True
-        return self.get_fuzzed_result(params)

msprobe/mindspore/free_benchmark/perturbation/perturbation_factory.py

@@ -1,46 +0,0 @@
-# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd.
-# All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0  (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from msprobe.mindspore.common.const import FreeBenchmarkConst
-from msprobe.mindspore.common.log import logger
-from msprobe.mindspore.free_benchmark.common.config import Config
-from msprobe.mindspore.free_benchmark.perturbation.add_noise import AddNoisePerturbation
-from msprobe.mindspore.free_benchmark.perturbation.bit_noise import BitNoisePerturbation
-from msprobe.mindspore.free_benchmark.perturbation.exchange_value import ExchangeValuePerturbation
-from msprobe.mindspore.free_benchmark.perturbation.improve_precision import ImprovePrecisionPerturbation
-from msprobe.mindspore.free_benchmark.perturbation.no_change import NoChangePerturbation
-
-
-class PerturbationFactory:
-    """
-    扰动工厂类
-
-    """
-    perturbations = {
-        FreeBenchmarkConst.IMPROVE_PRECISION: ImprovePrecisionPerturbation,
-        FreeBenchmarkConst.ADD_NOISE: AddNoisePerturbation,
-        FreeBenchmarkConst.BIT_NOISE: BitNoisePerturbation,
-        FreeBenchmarkConst.NO_CHANGE: NoChangePerturbation,
-        FreeBenchmarkConst.EXCHANGE_VALUE: ExchangeValuePerturbation
-    }
-
-    @staticmethod
-    def create(api_name_with_id: str):
-        perturbation = PerturbationFactory.perturbations.get(Config.pert_type)
-        if perturbation:
-            return perturbation(api_name_with_id)
-        else:
-            logger.error(f'{Config.pert_type} is a invalid perturbation type')
-            raise ValueError

msprobe/mindspore/free_benchmark/self_check_tool_factory.py

@@ -1,51 +0,0 @@
-# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd.
-# All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0  (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from msprobe.mindspore.common.const import Const
-from msprobe.core.common.log import logger
-from msprobe.mindspore.debugger.debugger_config import DebuggerConfig
-from msprobe.mindspore.free_benchmark.api_pynative_self_check import ApiPyNativeSelfCheck
-
-
-class SelfCheckToolFactory:
-    tools = {
-        Const.CELL: {
-            Const.GRAPH_KBYK_MODE: None,
-            Const.GRAPH_GE_MODE: None,
-            Const.PYNATIVE_MODE: None
-        },
-        Const.API: {
-            Const.GRAPH_KBYK_MODE: None,
-            Const.GRAPH_GE_MODE: None,
-            Const.PYNATIVE_MODE: ApiPyNativeSelfCheck
-        },
-        Const.KERNEL: {
-            Const.GRAPH_KBYK_MODE: None,
-            Const.GRAPH_GE_MODE: None,
-            Const.PYNATIVE_MODE: None
-        }
-    }
-
-    @staticmethod
-    def create(config: DebuggerConfig):
-        tool = SelfCheckToolFactory.tools.get(config.level)
-        if not tool:
-            logger.error(f"{config.level} is not supported.")
-            raise ValueError
-        tool = tool.get(config.execution_mode)
-        if not tool:
-            logger.error(f"Task free_benchmark is not supported in this mode: {config.execution_mode}.")
-            raise ValueError
-        return tool(config)