tico 0.1.0.dev250411__py3-none-any.whl

tico/experimental/quantization/evaluation/backend.py

@@ -0,0 +1,20 @@
+# Copyright (c) 2025 Samsung Electronics Co., Ltd. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from enum import Enum
+
+
+class BACKEND(Enum):
+    CIRCLE = 1
+    TRIV24 = 2

tico/experimental/quantization/evaluation/evaluate.py

@@ -0,0 +1,223 @@
+# Copyright (c) 2025 Samsung Electronics Co., Ltd. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Any, Callable, Dict, List, Optional, Tuple, Union
+
+import numpy as np
+import torch
+
+from circle_schema import circle
+from torch.utils import _pytree as pytree
+
+from tico.experimental.quantization.evaluation.backend import BACKEND
+from tico.experimental.quantization.evaluation.executor.backend_executor import (
+    BackendExecutor,
+)
+from tico.experimental.quantization.evaluation.executor.circle_executor import (
+    CircleExecutor,
+)
+from tico.experimental.quantization.evaluation.executor.triv24_executor import (
+    Triv24Executor,
+)
+from tico.experimental.quantization.evaluation.metric import MetricCalculator
+from tico.experimental.quantization.evaluation.utils import (
+    ensure_list,
+    find_invalid_types,
+    get_graph_input_output,
+    plot_two_outputs,
+)
+from tico.utils.model import CircleModel
+
+InputDataType = Union[
+    None,
+    np.ndarray,
+    torch.Tensor,
+    List[np.ndarray],
+    List[torch.Tensor],
+    Tuple[np.ndarray],
+    Tuple[torch.Tensor],
+]
+
+BACKEND_TO_EXECUTOR: Dict[BACKEND, type[BackendExecutor]] = {
+    BACKEND.CIRCLE: CircleExecutor,
+    BACKEND.TRIV24: Triv24Executor,
+}
+
+
+def _validate_input_data(
+    input_data: InputDataType, circle_inputs: List[circle.Tensor.Tensor]
+) -> None:
+    """
+    Validate whether the given input data matches the shape of a list of circle input tensors.
+
+    Parameters
+    -----------
+        input_data
+            The input data to be checked.
+        circle_inputs
+            A list of circle.Tensor.Tensor to validate against.
+    """
+    if input_data is None:
+        return
+    assert isinstance(input_data, list)
+
+    if len(input_data) != len(circle_inputs):
+        raise RuntimeError(
+            f"Mismatch between the length of input data and circle model: input_data({len(input_data)}) != circle_model({len(circle_inputs)})"
+        )
+    if invalid_type := find_invalid_types(input_data, [torch.Tensor, np.ndarray]):
+        raise RuntimeError(
+            f"Only support tuple of torch.Tensor or numpy.ndarray for input data. Invalid types: {invalid_type}"
+        )
+
+
+def _convert_to_torch_tensor(input_data: InputDataType) -> List[torch.Tensor]:
+    """
+    Convert the input data into a list of torch.Tensor.
+
+    This function performs the following tasks:
+        - Checks if `input_data` is a numpy array and converts it to a torch.Tensor.
+        - If it is already a torch.Tensor, it is returned as is.
+
+    Parameters
+    -----------
+        input_data
+            The input data to be converted.
+    """
+    assert isinstance(input_data, list)
+
+    # Cast to torch.Tensor to make the logic simpler
+    for i, data in enumerate(input_data):
+        if isinstance(data, np.ndarray):
+            input_data[i] = torch.Tensor(data)  # type: ignore[call-overload]
+
+    assert all(isinstance(input, torch.Tensor) for input in input_data)
+
+    return input_data  # type: ignore[return-value]
+
+
+def evaluate(
+    torch_module: torch.nn.Module,
+    circle_model: CircleModel,
+    backend: BACKEND,
+    input_data: InputDataType = None,
+    *,
+    mode="plot",
+    metrics: List[str] = ["peir"],
+    custom_metrics: Dict[str, Callable] = dict(),
+) -> Optional[Dict[str, Any]]:
+    """
+    Evaluate and compare a Pytorch module with a quantized circle model on a specific
+     backend using a given metrics.
+
+    It compiles the circle model using specified backend, run both the Pytorch module
+     and the compiled model on the same input, and computes the comparison score based
+    on the provided metrics.
+
+    Parameters
+    -----------
+        torch_module
+            A callable Pytorch module.
+        circle_module
+            The Circle model to be compiled and evaluated.
+        backend
+            The backend used to compile and execute the Circle model.
+        input_data
+            The input data to be used for evaluation. Should be compatible with both models.
+            If None, random data will be generated.
+        mode
+            The mode of operation. Options are:
+            - "plot": Plot the results (default)
+            - "return": Return the results.
+        metrics
+            A list of metric names for comparison.
+        custom_metrics
+            A dictionary of metric names and corresponding callable functions for comparison.
+            Example: {'mse': mean_squared_error, 'cosine_similarity': cosine_similarity_fn}
+    # TODO Support options for backend optimizations.
+
+    Returns
+    --------
+    dict or None
+        If `mode` is "plot", plot the results and returns None.
+        If `mode` is "return", returns a dictionary containing:
+            - "peir": The computed PEIR value.
+            - "<metric_name>": The computed value of the additional metric (if provided).
+    """
+    # Check if arguments are allowed types.
+    if not isinstance(torch_module, torch.nn.Module):
+        raise RuntimeError(
+            f"Only support torch.nn.Module. Given module type: {type(torch_module)}"
+        )
+    if not isinstance(circle_model, CircleModel):
+        raise RuntimeError(
+            f"Only support CircleModel. Given module type: {type(circle_model)}"
+        )
+    if not isinstance(backend, BACKEND):
+        raise RuntimeError(
+            f"Invalid backend. Please use tico.quantization.evaluate.BACKEND enum class"
+        )
+    # Make it a list for simpler logic.
+    if input_data is not None:
+        input_data = ensure_list(input_data)
+
+    circle_inputs, _ = get_graph_input_output(circle_model)
+    _validate_input_data(input_data, circle_inputs)
+
+    if input_data:
+        input_data = _convert_to_torch_tensor(input_data)
+    else:
+        # Make random inputs
+        circle_input_shapes_np = [t.ShapeAsNumpy() for t in circle_inputs]
+        input_data = [torch.randn(*shape) for shape in circle_input_shapes_np]
+
+    assert isinstance(input_data, list)
+    assert all(isinstance(data, torch.Tensor) for data in input_data)
+
+    # Compile circle model and run inference.
+    executor: BackendExecutor = BACKEND_TO_EXECUTOR[backend]()
+    executor.compile(circle_model)
+    circle_output = executor.run_inference(input_data)
+    circle_output = [
+        torch.from_numpy(out) for out in circle_output if isinstance(out, np.ndarray)
+    ]
+
+    # Run torch model.
+    with torch.no_grad():
+        torch_output = torch_module(*input_data)
+    torch_output, _ = pytree.tree_flatten(torch_output)
+    if isinstance(torch_output, torch.Tensor):
+        torch_output = (torch_output,)
+    if len(torch_output) != len(circle_output):
+        raise RuntimeError(
+            f"Mismatch between the length of torch output and circle output: torch_output({len(torch_output)}) != circle_output({len(circle_output)})"
+        )
+
+    # Computes the comparison score based on the provided metrics.
+    metric_calculator = MetricCalculator(metrics, custom_metrics)
+    results: Dict[str, Any] = metric_calculator.compute(torch_output, circle_output)
+
+    if mode == "return":
+        return results
+    elif mode == "plot":
+        for idx, (t_out, c_out) in enumerate(zip(torch_output, circle_output)):
+            print(f"OUTPUT [{idx}]")
+            fig = plot_two_outputs(t_out, c_out)
+            print(fig)
+            for metric_name, values in results.items():
+                print(f"{metric_name}: {values[idx]}")
+    else:
+        raise RuntimeError("Invalid mode.")
+
+    return None

tico/experimental/quantization/evaluation/executor/__init__.py

@@ -0,0 +1 @@
+# DO NOT REMOVE THIS FILE

tico/experimental/quantization/evaluation/executor/backend_executor.py

@@ -0,0 +1,54 @@
+# Copyright (c) 2025 Samsung Electronics Co., Ltd. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import abc
+from typing import Any
+
+from tico.utils.model import CircleModel
+
+
+class BackendExecutor(abc.ABC):
+    """
+    Abstract base class for executing a circle model on a specific backend.
+    """
+
+    @abc.abstractmethod
+    def compile(self, circle_model: CircleModel) -> None:
+        """
+        Compile the circle model for this backend, if needed.
+
+        Parameters
+        -----------
+        circle_model
+            The circle model to be compiled.
+        """
+        pass
+
+    @abc.abstractmethod
+    def run_inference(self, input_data: Any) -> Any:
+        """
+        Run inference using the compiled (or directly usable) model
+         on the given input data.
+
+        Parameters
+        -----------
+        input_data
+            The input data to be fed to the model.
+
+        Returns
+        --------
+        Any
+            The model's inference output.
+        """
+        pass

tico/experimental/quantization/evaluation/executor/circle_executor.py

@@ -0,0 +1,75 @@
+# Copyright (c) 2025 Samsung Electronics Co., Ltd. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import tempfile
+from pathlib import Path
+from typing import List
+
+import numpy as np
+import torch
+
+from tico.experimental.quantization.evaluation.executor.backend_executor import (
+    BackendExecutor,
+)
+from tico.utils.model import CircleModel
+from tico.utils.utils import run_bash_cmd
+
+
+class CircleExecutor(BackendExecutor):
+    """
+    A class for running inference on fake-quantized circle models.
+
+    Instead of leveraging the actual backend for quantized circle execution,
+     it applies fake quantization to the models and performs inference.
+    """
+
+    def __init__(self):
+        self.compiler_path = Path("/usr/share/one/bin/onecc")
+        self.interpreter_path = None  # Use circle-interpreter
+        self.fq_circle_path = None
+
+        # Check if the toolchain is installed.
+        if not self.compiler_path.is_file():
+            raise RuntimeError(
+                "Not found one-compiler. Please install the one-compiler package first."
+            )
+
+        self.temp_dir = tempfile.TemporaryDirectory()
+
+    def compile(self, circle_model: CircleModel) -> None:
+        assert isinstance(circle_model, CircleModel)
+        circle_path = Path(self.temp_dir.name) / "quantized.circle"
+        circle_model.save(str(circle_path))
+        self.fq_circle_path = Path(self.temp_dir.name) / "fake_quantized.circle"
+        args = []
+        args += ["quantize"]
+        args += ["--fake_quantize"]
+        args += ["-i", str(circle_path)]
+        args += ["-o", str(self.fq_circle_path)]
+        cmd = [str(self.compiler_path)] + args
+        run_bash_cmd(cmd)
+
+    def run_inference(self, input_data: List[torch.Tensor]) -> List[np.ndarray]:
+        if not self.fq_circle_path:
+            raise RuntimeError("You must compile the model before running inference.")
+
+        fq_circle = CircleModel.load(self.fq_circle_path)
+        assert isinstance(fq_circle, CircleModel)
+        out = fq_circle(*input_data)
+        if not isinstance(out, list):
+            out = [out]
+        return out
+
+    def __del__(self):
+        self.temp_dir.cleanup()

tico/experimental/quantization/evaluation/executor/triv24_executor.py

@@ -0,0 +1,128 @@
+# Copyright (c) 2025 Samsung Electronics Co., Ltd. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import tempfile
+from pathlib import Path
+from typing import List
+
+import numpy as np
+import torch
+from circle_schema import circle
+
+from tico.experimental.quantization.evaluation.executor.backend_executor import (
+    BackendExecutor,
+)
+from tico.experimental.quantization.evaluation.utils import (
+    dequantize,
+    get_graph_input_output,
+    quantize,
+)
+from tico.serialize.circle_mapping import np_dtype_from_circle_dtype
+from tico.utils.model import CircleModel
+from tico.utils.utils import run_bash_cmd
+
+
+class Triv24Executor(BackendExecutor):
+    """
+    Implementation for a TRIV24 backend.
+    """
+
+    def __init__(self):
+        self.compiler_path = Path("/usr/share/one/backends/triv24/bin/triv24-compile")
+        self.interpreter_path = Path(
+            "/usr/share/one/backends/triv24/bin/triv24-ssinfer"
+        )
+        self.circle_model = None
+        self.tvn_path = None
+
+        # Check if triv24 toolchain is installed.
+        if not self.compiler_path.is_file() or not self.interpreter_path.is_file():
+            raise RuntimeError(
+                "Not found triv24 toolchain. Please install the toolchain package first."
+            )
+
+        self.temp_dir = tempfile.TemporaryDirectory()
+
+    def compile(self, circle_model: CircleModel) -> None:
+        assert isinstance(circle_model, CircleModel)
+        self.circle_model = circle_model
+        circle_path = Path(self.temp_dir.name) / "quantized.circle"
+        circle_model.save(str(circle_path))
+        self.tvn_path = Path(self.temp_dir.name) / "compiled.tvn"
+        args = []
+        args += ["-o", str(self.tvn_path)]
+        args += [str(circle_path)]
+        cmd = [str(self.compiler_path)] + args
+        run_bash_cmd(cmd)
+
+    def run_inference(self, input_data: List[torch.Tensor]) -> List[np.ndarray]:
+        if not self.tvn_path:
+            raise RuntimeError("You must compile the model before running inference.")
+
+        assert isinstance(self.circle_model, CircleModel)
+        circle_inputs, circle_outputs = get_graph_input_output(self.circle_model)
+        # Get input/output of scale/zp from quantized circle.
+        # Note that qparams may be None because some of them are not quantized like indices, argmax.
+        input_qparam: List[circle.QuantizationParameters.QuantizationParameters] = []
+        for inp in circle_inputs:
+            input_qparam.append(inp.Quantization())
+        output_qparam: List[circle.QuantizationParameters.QuantizationParameters] = []
+        for out in circle_outputs:
+            output_qparam.append(out.Quantization())
+
+        # Create input files for inference.
+        for in_idx, data in enumerate(input_data):
+            in_data_path = Path(self.temp_dir.name) / f"input.{in_idx}.tv2b"
+            assert isinstance(data, torch.Tensor)
+            if input_qparam[in_idx] is None:
+                np_data = data.numpy()
+            else:
+                np_data = quantize(
+                    data.numpy(),
+                    input_qparam[in_idx].ScaleAsNumpy()[0],
+                    input_qparam[in_idx].ZeroPointAsNumpy()[0],
+                    np_dtype_from_circle_dtype(circle_inputs[in_idx].Type()),
+                )
+            np_data.tofile(in_data_path)
+        args = []
+        args += ["--loadable", str(self.tvn_path)]
+        args += ["--input-spec", f"tv2b:{str(self.temp_dir.name)}/input"]
+        args += ["--dump-output-as-tv2b", f"{str(self.temp_dir.name)}/output"]
+        cmd = [str(self.interpreter_path)] + args
+
+        # Run inference
+        run_bash_cmd(cmd)
+
+        # Load outputs from file
+        circle_dequantized_output = []
+        circle_output_shapes_np = [t.ShapeAsNumpy() for t in circle_outputs]
+        for out_idx, out_shape in enumerate(circle_output_shapes_np):
+            out_data_path = Path(self.temp_dir.name) / f"output.{out_idx}.tv2b"
+            circle_output = np.fromfile(out_data_path, np.uint8).reshape(out_shape)
+            if output_qparam[out_idx] is None:
+                circle_dequantized_output.append(circle_output)
+            else:
+                circle_dequantized_output.append(
+                    dequantize(
+                        circle_output,
+                        output_qparam[out_idx].ScaleAsNumpy()[0],
+                        output_qparam[out_idx].ZeroPointAsNumpy()[0],
+                        np_dtype_from_circle_dtype(circle_outputs[out_idx].Type()),
+                    )
+                )
+
+        return circle_dequantized_output
+
+    def __del__(self):
+        self.temp_dir.cleanup()

tico/experimental/quantization/evaluation/metric.py

@@ -0,0 +1,109 @@
+# Copyright (c) 2025 Samsung Electronics Co., Ltd. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Any, Callable, Dict, List
+
+import numpy as np
+import torch
+
+
+def compute_peir(base: torch.Tensor, target: torch.Tensor):
+    """
+    Calculate the Peak Error to Interval Ratio (PEIR) between two tensors.
+
+    This function computes the PEIR between two tensors using the formula:
+        PEIR = max(abs(tensor1 - tensor2)) / (max(tensor1) - min(tensor2))
+    """
+    assert base.shape == target.shape, f"shape mismatch: {base.shape} != {target.shape}"
+    base_tensor = base.numpy()
+    target_tensor = target.numpy()
+    assert (
+        base_tensor.dtype == np.float32 and target_tensor.dtype == np.float32
+    ), f"dtype should be float32: base({base_tensor.dtype}), target({target_tensor.dtype})"
+
+    base_tensor = base_tensor.reshape(-1)
+    target_tensor = target_tensor.reshape(-1)
+
+    assert (
+        base_tensor.shape == target_tensor.shape
+    ), f"Shape mismatch: {base_tensor.shape} != {target_tensor.shape}"
+
+    peak_error = np.max(np.absolute(target_tensor - base_tensor))
+    interval = np.max(base_tensor) - np.min(base_tensor)
+    peir = peak_error / interval  # pylint: disable=invalid-name
+
+    min_value = min([base_tensor.min(), target_tensor.min()])
+    max_value = max([base_tensor.max(), target_tensor.max()])
+
+    interval = max_value - min_value
+    interval = 1.0 if interval == 0.0 else interval  # Avoid zero interval
+
+    return peir
+
+
+class MetricCalculator:
+    """
+    Compute metrics including both built-in and custom metrics.
+
+    metrics
+        A list of metric names for comparison.
+    custom_metrics
+        A dictionary of metric names and corresponding callable functions for comparison.
+        Example: {'mse': mean_squared_error, 'cosine_similarity': cosine_similarity_fn}
+    """
+
+    builtin_metrics = {
+        "peir": compute_peir,
+    }
+
+    def __init__(
+        self,
+        metrics: List[str] = list(),
+        custom_metrics: Dict[str, Callable] = dict(),
+    ):
+        self.metrics: Dict[str, Callable] = dict()
+
+        for m in metrics:
+            if m in self.builtin_metrics:
+                self.metrics[m] = self.builtin_metrics[m]
+            else:
+                raise RuntimeError(f"Invalid metric: {m}")
+
+        duplicates = set(self.metrics).intersection(custom_metrics.keys())
+        if len(duplicates) != 0:
+            raise RuntimeError(f"There are duplicate metrics: {duplicates}")
+
+        self.metrics = self.metrics | custom_metrics
+
+    def compute(
+        self, output1: List[torch.Tensor], output2: List[torch.Tensor]
+    ) -> Dict[str, List[Any]]:
+        """
+        Compute both built-in metrics (if provided) and custom metrics.
+
+        Returns
+        --------
+        Dict[str, Any]
+            A dictionary with metric names and their computed values.
+        """
+        results: Dict[str, List[Any]] = dict()
+
+        # Compute built-in metrics
+        if self.metrics is not None:
+            for m in self.metrics:
+                results[m] = list()
+                for out1, out2 in zip(output1, output2):
+                    results[m].append(self.builtin_metrics[m](out1, out2))
+
+        return results