mindstudio-probe 1.0.4__py3-none-any.whl → 1.1.1__py3-none-any.whl

msprobe/pytorch/debugger/debugger_config.py

@@ -1,6 +1,23 @@
-from msprobe.pytorch.common import seed_all
-from msprobe.pytorch.common.log import logger
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import torch
+
 from msprobe.core.common.const import Const
+from msprobe.core.common.exceptions import MsprobeException
+from msprobe.pytorch.common.log import logger
 
 
 class DebuggerConfig:
@@ -10,30 +27,28 @@ class DebuggerConfig:
         self.rank = common_config.rank if common_config.rank else []
         self.step = common_config.step if common_config.step else []
         self.level = level or common_config.level or "L1"
-        self.seed = common_config.seed if common_config.seed else 1234
-        self.is_deterministic = common_config.is_deterministic
         self.enable_dataloader = common_config.enable_dataloader
         self.scope = task_config.scope if task_config.scope else []
         self.list = task_config.list if task_config.list else []
         self.data_mode = task_config.data_mode if task_config.data_mode else ["all"]
-        self.backward_input_list = task_config.backward_input if task_config.backward_input else []
-        self.backward_input = {}
-        self.acl_config = common_config.acl_config if common_config.acl_config else ""
-        self.is_forward_acl_dump = True
         self.summary_mode = task_config.summary_mode if task_config.summary_mode else Const.STATISTICS
         self.overflow_nums = task_config.overflow_nums if task_config.overflow_nums else 1
         self.framework = Const.PT_FRAMEWORK
 
+        if self.level == Const.LEVEL_L2:
+            self.is_backward_kernel_dump = False
+            self._check_and_adjust_config_with_l2()
+
         if self.task == Const.FREE_BENCHMARK:
-            self.fuzz_device = task_config.fuzz_device if task_config.fuzz_device else 'npu'
-            self.handler_type = task_config.handler_type if task_config.handler_type else 'check'
-            self.pert_mode = task_config.pert_mode if task_config.pert_mode else 'improve_precision'
-            self.fuzz_level = task_config.fuzz_level if task_config.fuzz_level else 'L1'
-            self.fuzz_stage = task_config.fuzz_stage if task_config.fuzz_stage else 'forward'
+            self.fuzz_device = task_config.fuzz_device
+            self.handler_type = task_config.handler_type
+            self.pert_mode = task_config.pert_mode
+            self.fuzz_level = task_config.fuzz_level
+            self.fuzz_stage = task_config.fuzz_stage
             self.preheat_config = {
-                "if_preheat": task_config.if_preheat if task_config.if_preheat is not None else True, 
-                "preheat_step": task_config.preheat_step if task_config.preheat_step else 15, 
-                "max_sample": task_config.max_sample if task_config.max_sample else 20, 
+                "if_preheat": task_config.if_preheat,
+                "preheat_step": task_config.preheat_step,
+                "max_sample": task_config.max_sample
             }
 
         self.online_run_ut = False
@@ -44,52 +59,54 @@ class DebuggerConfig:
             self.tls_path = task_config.tls_path if task_config.tls_path else ""
             self.host = task_config.host if task_config.host else ""
             self.port = task_config.port if task_config.port else -1
+            self.online_run_ut_recompute = task_config.online_run_ut_recompute \
+                if isinstance(task_config.online_run_ut_recompute, bool) else False
 
         self.check()
-        if self.step:
-            self.step.sort()
-        if self.level == "L2":
-            if not self.scope or not isinstance(self.scope, list) or len(self.scope) != 1:
-                raise ValueError("scope must be configured as a list with one api name")
-            if isinstance(self.scope[0], str) and Const.BACKWARD in self.scope[0] and not self.backward_input_list:
-                raise ValueError("backward_input must be configured when scope contains 'backward'")
-            if Const.BACKWARD in self.scope[0]:
-                self.is_forward_acl_dump = False
-                for index, scope_spec in enumerate(self.scope):
-                    self.scope[index] = scope_spec.replace(Const.BACKWARD, Const.FORWARD)
-                    self.backward_input[self.scope[index]] = self.backward_input_list[index]
-        seed_all(self.seed, self.is_deterministic)
 
     def check_kwargs(self):
         if self.task and self.task not in Const.TASK_LIST:
-            raise Exception("task is invalid")
+            raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR,
+                                   f"The task <{self.task}> is not in the {Const.TASK_LIST}.")
         if self.level and self.level not in Const.LEVEL_LIST:
-            raise Exception("level is invalid")
+            raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR,
+                                   f"The level <{self.level}> is not in the {Const.LEVEL_LIST}.")
         if not self.dump_path:
-            raise Exception("Invalid dump path, please check your config")
+            raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR,
+                                   f"The dump_path not found.")
 
     def check(self):
         self.check_kwargs()
-        self._check_rank()
-        self._check_step()
         return True
 
-    def check_model(self, model):
-        if self.level in ["L0", "mix"] and not model:
-            raise Exception(
-                f"For level {self.level}, PrecisionDebugger must receive a model argument."
-            )
-
-    def _check_rank(self):
-        if self.rank:
-            for rank_id in self.rank:
-                if not isinstance(rank_id, int) or rank_id < 0:
-                    raise ValueError(f"rank {self.rank} must be an integer and greater than or equal to 0.")
-            else:
-                logger.warning_on_rank_0(f"Rank argument is provided. Only rank {self.rank} data will be dumpped.")
+    def check_model(self, instance, start_model):
+        if self.level not in ["L0", "mix"]:
+            if instance.model is not None or start_model is not None:
+                logger.warning_on_rank_0(
+                    f"The current level is not L0 or mix level, so the model parameters will not be used.")
+            return
+        if start_model is None:
+            if instance.model is None:
+                logger.error_on_rank_0(
+                    f"For level {self.level}, PrecisionDebugger or start interface must receive a 'model' argument.")
+                raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR, f"missing the parameter 'model'")
+            return
+        if isinstance(start_model, torch.nn.Module):
+            instance.model = start_model
+        else:
+            logger.error_on_rank_0(f"The 'model' parameter of start must be a torch.nn.Module type.")
+            raise MsprobeException(
+                MsprobeException.INVALID_PARAM_ERROR, f"model must be a torch.nn.Module")
 
-    def _check_step(self):
-        if self.step:
-            for s in self.step:
-                if not isinstance(s, int) or s < 0:
-                    raise ValueError(f"step element {s} must be an integer and greater than or equal to 0.")
+    def _check_and_adjust_config_with_l2(self):
+        if self.scope:
+            raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR,
+                                   f"When level is set to L2, the scope cannot be configured.")
+        if not self.list or len(self.list) != 1:
+            raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR,
+                                   f"When level is set to L2, the list must be configured as a list with one api name.")
+        api_name = self.list[0]
+        if api_name.endswith(Const.BACKWARD):
+            self.is_backward_kernel_dump = True
+            api_forward_name = api_name[:-len(Const.BACKWARD)] + Const.FORWARD
+            self.list.append(api_forward_name)

msprobe/pytorch/debugger/precision_debugger.py

@@ -1,12 +1,34 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from collections import namedtuple
+
 import torch
-from torch.utils.data import dataloader
-from msprobe.pytorch.debugger.debugger_config import DebuggerConfig
-from msprobe.pytorch.service import Service
-from msprobe.pytorch.common.log import logger
-from msprobe.pytorch.pt_config import parse_json_config
+from msprobe.core.common.const import Const, FileCheckConst, MsgConst
 from msprobe.core.common.exceptions import MsprobeException
-from msprobe.core.common.const import Const
+from msprobe.core.common.file_utils import FileChecker
+from msprobe.core.common.utils import get_real_step_or_rank
+from msprobe.pytorch.common.log import logger
+from msprobe.pytorch.debugger.debugger_config import DebuggerConfig
 from msprobe.pytorch.grad_probe.grad_monitor import GradientMonitor
+from msprobe.pytorch.pt_config import parse_json_config
+from msprobe.pytorch.service import Service
+from torch.utils.data import dataloader
+
+ConfigParameters = namedtuple("ConfigParameters", ["config_path", "task",
+                                                   "dump_path", "level", "model"])
 
 
 class PrecisionDebugger:
@@ -30,20 +52,26 @@ class PrecisionDebugger:
         step=None,
     ):
         if not hasattr(self, "initialized"):
+            config_params = ConfigParameters(config_path,
+                                             task,
+                                             dump_path,
+                                             level,
+                                             model)
+            self.check_input_params(config_params)
+
             self.api_origin = False
             self.initialized = True
-            self.model = self.check_model_valid(model)
+            self.model = model
             common_config, task_config = parse_json_config(config_path, task)
-            self.task = common_config.task
+            self.task = task if task else common_config.task
             if self.task == Const.GRAD_PROBE:
                 self.gm = GradientMonitor(common_config, task_config)
                 return
             if step:
-                common_config.step = step
+                common_config.step = get_real_step_or_rank(step, Const.STEP)
             self.config = DebuggerConfig(
                 common_config, task_config, task, dump_path, level
             )
-            self.config.check_model(self.model)
             self.service = Service(self.config)
             self.enable_dataloader = self.config.enable_dataloader
             if self.enable_dataloader:
@@ -55,20 +83,40 @@ class PrecisionDebugger:
         return self._instance
 
     @staticmethod
-    def check_model_valid(model):
-        if not model or isinstance(model, torch.nn.Module):
-            return model
-        raise MsprobeException(
-            MsprobeException.INVALID_PARAM_ERROR, "model 参数必须是torch.nn.Module类型。"
-        )
+    def check_input_params(args):
+        if args.config_path is not None:
+            if not isinstance(args.config_path, str):
+                raise MsprobeException(
+                    MsprobeException.INVALID_PARAM_ERROR, f"config_path must be a string")
+            file_checker = FileChecker(
+                file_path=args.config_path, path_type=FileCheckConst.FILE, file_type=FileCheckConst.JSON_SUFFIX)
+            file_checker.common_check()
+
+        if args.task is not None and args.task not in Const.TASK_LIST:
+            raise MsprobeException(
+                MsprobeException.INVALID_PARAM_ERROR, f"task must be one of {Const.TASK_LIST}")
+
+        if args.dump_path is not None:
+            if not isinstance(args.dump_path, str):
+                raise MsprobeException(
+                    MsprobeException.INVALID_PARAM_ERROR, f"dump_path must be a string")
+
+        if args.level is not None and args.level not in Const.LEVEL_LIST:
+            raise MsprobeException(
+                MsprobeException.INVALID_PARAM_ERROR, f"level must be one of {Const.LEVEL_LIST}")
+
+        if args.model is not None and not isinstance(args.model, torch.nn.Module):
+            raise MsprobeException(
+                MsprobeException.INVALID_PARAM_ERROR, f"model must be a torch.nn.Module")
 
     @classmethod
-    def start(cls):
+    def start(cls, model=None):
         instance = cls._instance
+        if not instance:
+            raise Exception(MsgConst.NOT_CREATED_INSTANCE)
         if instance.task in PrecisionDebugger.tasks_not_need_debugger:
             return
-        if not instance:
-            raise Exception("No instance of PrecisionDebugger found.")
+        instance.config.check_model(instance, model)
         if instance.enable_dataloader:
             logger.warning_on_rank_0("DataLoader is enabled, start() skipped.")
         else:
@@ -85,10 +133,10 @@ class PrecisionDebugger:
     @classmethod
     def stop(cls):
         instance = cls._instance
+        if not instance:
+            raise Exception(MsgConst.NOT_CREATED_INSTANCE)
         if instance.task in PrecisionDebugger.tasks_not_need_debugger:
             return
-        if not instance:
-            raise Exception("PrecisionDebugger instance is not created.")
         if instance.enable_dataloader:
             logger.warning_on_rank_0("DataLoader is enabled, stop() skipped.")
         else:
@@ -96,16 +144,16 @@ class PrecisionDebugger:
 
     @classmethod
     def step(cls):
+        if not cls._instance:
+            raise Exception(MsgConst.NOT_CREATED_INSTANCE)
         if cls._instance.task in PrecisionDebugger.tasks_not_need_debugger:
             return
-        if not cls._instance:
-            raise Exception("PrecisionDebugger instance is not created.")
         cls._instance.service.step()
 
     @classmethod
     def monitor(cls, model):
         if not cls._instance:
-            raise Exception("PrecisionDebugger instance is not created.")
+            raise Exception(MsgConst.NOT_CREATED_INSTANCE)
         if cls._instance.task != Const.GRAD_PROBE:
             return
         cls._instance.gm.monitor(model)

msprobe/pytorch/dump/kernel_dump/kernel_config.py

@@ -0,0 +1,33 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+from msprobe.core.common.file_utils import save_json
+
+
+def create_kernel_config_json(dump_path, cur_rank):
+    kernel_config_name = "kernel_config.json" if cur_rank == '' else f"kernel_config_{cur_rank}.json"
+    kernel_config_path = os.path.join(dump_path, kernel_config_name)
+    config_info = {
+        "dump": {
+            "dump_list": [],
+            "dump_path": dump_path,
+            "dump_mode": "all",
+            "dump_op_switch": "on"
+        }
+    }
+    save_json(kernel_config_path, config_info, indent=4)
+    return kernel_config_path

msprobe/pytorch/free_benchmark/__init__.py

@@ -1,8 +1,23 @@
-from msprobe.pytorch.common.log import logger
-from msprobe.core.common.exceptions import FreeBenchmarkException
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+__all__ = ["FreeBenchmarkCheck", "UnequalRow"]
+
 from msprobe.core.common.const import Const
+from msprobe.core.common.exceptions import FreeBenchmarkException
+from msprobe.pytorch.common.log import logger
 
-from .main import FreeBenchmarkCheck
 from .common.params import UnequalRow
-
-__all__ = [FreeBenchmarkCheck, UnequalRow]
+from .main import FreeBenchmarkCheck

msprobe/pytorch/free_benchmark/common/constant.py

@@ -1,3 +1,18 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 from typing import Dict
 
 import numpy as np

msprobe/pytorch/free_benchmark/common/counter.py

@@ -1,3 +1,18 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 from collections import defaultdict
 from msprobe.pytorch.free_benchmark.common.constant import ThresholdConfig
 

msprobe/pytorch/free_benchmark/common/enums.py

@@ -1,3 +1,21 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from msprobe.core.common.const import Const
+
+
 class PerturbationMode:
     ADD_NOISE = "add_noise"
     CHANGE_VALUE = "change_value"
@@ -35,3 +53,28 @@ class FuzzLevel:
     BASE_LEVEL = "L1"
     ADV_LEVEL = "L2"
     REAL_LEVEL = "L3"
+
+
+class PytorchFreeBenchmarkConst:
+    PERTURBATION_MODE_LIST = [
+        PerturbationMode.ADD_NOISE,
+        PerturbationMode.CHANGE_VALUE,
+        PerturbationMode.IMPROVE_PRECISION,
+        PerturbationMode.NO_CHANGE,
+        PerturbationMode.BIT_NOISE,
+        PerturbationMode.TO_CPU,
+    ]
+    DEFAULT_MODE = PerturbationMode.IMPROVE_PRECISION
+    DEVICE_LIST = [DeviceType.NPU, DeviceType.CPU]
+    DEFAULT_DEVICE = DeviceType.NPU
+    HANDLER_LIST = [HandlerType.CHECK, HandlerType.FIX]
+    DEFAULT_HANDLER = HandlerType.CHECK
+    FUZZ_LEVEL_LIST = [FuzzLevel.BASE_LEVEL]
+    DEFAULT_FUZZ_LEVEL = FuzzLevel.BASE_LEVEL
+    FUZZ_STAGE_LIST = [Const.FORWARD, Const.BACKWARD]
+    FIX_MODE_LIST = [PerturbationMode.IMPROVE_PRECISION, PerturbationMode.TO_CPU]
+    DEFAULT_FUZZ_STAGE = Const.FORWARD
+    DEFAULT_PREHEAT_STEP = 15
+    DEFAULT_MAX_SAMPLE = 20
+    CPU_MODE_LIST = [PerturbationMode.TO_CPU]
+    FIX_STAGE_LIST = [Const.FORWARD]

msprobe/pytorch/free_benchmark/common/params.py

@@ -1,7 +1,23 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 from dataclasses import dataclass
 from typing import Any, Callable, Dict, List, Optional, Tuple
 
 import torch
+from msprobe.core.common.exceptions import FreeBenchmarkException
 from msprobe.pytorch.free_benchmark import logger
 from msprobe.pytorch.free_benchmark.common.enums import (
     DeviceType,
@@ -113,7 +129,13 @@ def make_unequal_row(
         row.max_rel = ratio - 1
     origin_tensor = data_params.original_result
     perturbed_tensor = data_params.perturbed_result
-    if index:
+    if index is not None:
+        if index >= len(origin_tensor) or index >= len(perturbed_tensor):
+            err_msg = f"When generating unequal results, index {index} of output is out of bounds. please check!"
+            raise FreeBenchmarkException(
+                FreeBenchmarkException.OutputIndexError,
+                error_info=err_msg,
+            )
         origin_tensor = origin_tensor[index]
         perturbed_tensor = perturbed_tensor[index]
         row.output_index = index

msprobe/pytorch/free_benchmark/common/utils.py

@@ -1,4 +1,22 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
 import torch
+from msprobe.core.common.exceptions import FreeBenchmarkException
+from msprobe.core.common.utils import recursion_depth_decorator
 from msprobe.pytorch.free_benchmark.common.enums import DeviceType
 
 
@@ -36,6 +54,7 @@ class Tools:
         return api_name.rsplit(".", 2)[0]
 
     @staticmethod
+    @recursion_depth_decorator("FreeBenchmark: Tools.convert_device_and_dtype")
     def convert_device_and_dtype(
         tensor_seq, device: str = DeviceType.CPU, change_dtype: bool = False
     ):
@@ -58,24 +77,43 @@ class Tools:
         return tensor_seq
 
     @staticmethod
+    @recursion_depth_decorator("FreeBenchmark: Tools.convert_fuzz_output_to_origin")
     def convert_fuzz_output_to_origin(origin, perturbed):
-        if isinstance(origin, torch.Tensor):
+        if isinstance(origin, torch.Tensor) and isinstance(perturbed, torch.Tensor):
             origin.data = perturbed.to(origin.dtype).to(origin.device)
             return origin
-        if isinstance(origin, dict):
+        if isinstance(origin, dict) and isinstance(perturbed, dict):
             output = dict()
             for key, value in origin.items():
+                if key not in perturbed:
+                    err_msg = f"'{key}' not in perturbed output."
+                    raise FreeBenchmarkException(
+                        FreeBenchmarkException.InvalidPerturbedOutput,
+                        error_info=err_msg,
+                    )
                 output[key] = Tools.convert_fuzz_output_to_origin(value, perturbed[key])
             return output
-        if isinstance(origin, (tuple, list)):
+        if isinstance(origin, (tuple, list)) and isinstance(perturbed, (tuple, list)):
             result = list()
+            if len(perturbed) != len(origin):
+                err_msg = (
+                    f"length of perturbed output ({len(perturbed)}) is different "
+                    f"from the length of original output ({len(origin)})."
+                )
+                raise FreeBenchmarkException(
+                    FreeBenchmarkException.InvalidPerturbedOutput, error_info=err_msg
+                )
             for index_, value in enumerate(origin):
                 result.append(
                     Tools.convert_fuzz_output_to_origin(value, perturbed[index_])
                 )
             return type(origin)(result)
-        return origin
-    
+        err_msg = f"conversion of two outputs with types ({type(origin)}, {type(perturbed)}) is not supported."
+        raise FreeBenchmarkException(
+            FreeBenchmarkException.UnsupportedType, error_info=err_msg
+        )
+
+
 class TorchC:
     sum = torch._C._VariableFunctionsClass.sum
     isinf = torch._C._VariableFunctionsClass.isinf