tico 0.1.0.dev250714__py3-none-any.whl → 0.1.0.dev251102__py3-none-any.whl

tico/experimental/quantization/algorithm/gptq/quantizer.py

@@ -1,225 +0,0 @@
-# Copyright (c) 2024 Intel Corporation
-# Copyright (c) 2025 Samsung Electronics Co., Ltd. All Rights Reserved
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import types
-from typing import Any, Dict, List, Optional
-
-import torch
-
-from tico.experimental.quantization.algorithm.gptq.gptq import GPTQ
-from tico.experimental.quantization.algorithm.gptq.utils import (
-    find_layers,
-    gather_single_batch_from_dict,
-    gather_single_batch_from_list,
-)
-from tico.experimental.quantization.config import BaseConfig, GPTQConfig
-from tico.experimental.quantization.quantizer import BaseQuantizer
-
-
-class GPTQQuantizer(BaseQuantizer):
-    """
-    Quantizer for applying the GPTQ algorithm (typically for weight quantization)
-    """
-
-    def __init__(self, config: BaseConfig):
-        super().__init__(config)
-
-        self.cache_args: List[Any] = []
-        self.cache_kwargs: Dict[str, Any] = {"batch_num": 0}
-
-    @torch.no_grad()
-    def prepare(
-        self,
-        model: torch.nn.Module,
-        args: Optional[Any] = None,
-        kwargs: Optional[Dict[str, Any]] = None,
-    ):
-        """
-        Overrides the forward method of the first LLaMA layer (layer 0) to capture the
-         input required for calibration.
-
-        This method modifies the original forward pass of LLaMA layer 0 so that the
-         inputs used during inference are intercepted and recorded. These captured inputs
-        are then utilized to calibrate the quantization parameters for the GPTQ.
-
-        Parameters:
-            model: The target PyTorch model.
-            args: Positional example inputs required for capturing graph.
-            kwargs: Keyword example inputs required for capturing graph.
-
-        Returns:
-            The model prepared for GPTQ quantization.
-        """
-        if args is None and kwargs is None:
-            raise RuntimeError(
-                "Either args or kwargs must be provided for captruing graph."
-            )
-        # Define a function to capture input activations and associated parameters.
-        def forward(layer, *args, **kwargs):
-            self.cache_kwargs["batch_num"] += 1
-            for idx, item in enumerate(args):
-                if (idx + 1) > len(self.cache_args):
-                    self.cache_args.append([])
-                self.cache_args[idx].append(item)
-            for arg in kwargs:
-                if self.cache_kwargs.get(arg, None) is None:
-                    self.cache_kwargs[arg] = []
-                self.cache_kwargs[arg].append(kwargs[arg])
-            # Raise an error to interrupt the forward pass after capturing data.
-            raise ValueError
-
-        # Replace the first layer with defined function to capture calibration data.
-        if hasattr(model, "model"):
-            assert hasattr(model.model, "layers")
-            assert isinstance(model.model.layers, torch.nn.ModuleList)
-            layer_forward_cache = model.model.layers[0].forward
-            model.model.layers[0].forward = types.MethodType(
-                forward, model.model.layers[0]
-            )
-        else:
-            assert hasattr(model, "forward")
-            layer_forward_cache = model.forward
-            model.forward = types.MethodType(forward, model.forward)
-
-        model_forward_cache = model.forward
-        # Replace model's forward to avoid ValueError
-        def model_forward(model, *args, **kwargs):
-            nonlocal model_forward_cache
-            try:
-                model_forward_cache(*args, **kwargs)
-            except ValueError:
-                pass
-
-        model.forward = types.MethodType(model_forward, model)
-        kwargs = kwargs or {}
-        model(*args, **kwargs)  # type: ignore[misc]
-
-        # Recover original forward
-        model.forward = model_forward_cache
-        if hasattr(model, "model"):
-            assert hasattr(model.model, "layers")
-            assert isinstance(model.model.layers, torch.nn.ModuleList)
-            model.model.layers[0].forward = layer_forward_cache
-        else:
-            model.forward = layer_forward_cache
-
-        return model
-
-    @torch.no_grad()
-    def convert(self, model):
-        """
-        Convert the prepared model to its GPTQ quantized version.
-
-        Applies the GPTQ quantization on weights based on the collected statistics.
-
-        Parameters:
-            model: The prepared PyTorch model.
-
-        Returns:
-            The quantized model.
-        """
-        gptq_conf = self.config
-        assert isinstance(gptq_conf, GPTQConfig)
-
-        # Save the original cache setting and disable caching during calibration/inference.
-        if hasattr(model, "config"):
-            use_cache = model.config.use_cache
-            model.config.use_cache = False
-
-        quantizers = {}
-        if hasattr(model, "model"):
-            target_layers = model.model.layers
-        else:
-            target_layers = [model]
-        for l_idx, layer in enumerate(target_layers):
-            # Identify quantizable submodules within the layer.
-            full = find_layers(layer)
-
-            sequential = [list(full.keys())]
-            for names in sequential:
-                subset = {n: full[n] for n in names}
-
-                gptq: Dict[str, GPTQ] = {}
-                for name in subset:
-                    gptq[name] = GPTQ(subset[name])
-                    gptq[name].quantizer.configure(
-                        8, perchannel=True, sym=False, mse=False
-                    )
-                # Define a hook to collect input/output batches for quantizer calibration.
-                def add_batch(name):
-                    def tmp(_, inp, out):
-                        gptq[name].add_batch(inp[0].data, out.data)
-
-                    return tmp
-
-                handles = []
-                for name in subset:
-                    handles.append(subset[name].register_forward_hook(add_batch(name)))
-                # Run the current layer on the stored calibration inputs to capture activation stats.
-                batch_num = self.cache_kwargs.pop("batch_num")
-                for batch_idx in range(batch_num):
-                    cache_args_batch = gather_single_batch_from_list(
-                        self.cache_args, batch_idx
-                    )
-                    cache_kwargs_batch = gather_single_batch_from_dict(
-                        self.cache_kwargs, batch_idx
-                    )
-                    layer(*cache_args_batch, **cache_kwargs_batch)[0]
-                self.cache_kwargs["batch_num"] = batch_num
-                for h in handles:
-                    h.remove()
-                # Quantize each submodule using the collected calibration data.
-                for name in subset:
-                    if gptq_conf.verbose:
-                        print(l_idx, name)
-                        print("Quantizing ...")
-                    gptq[name].fasterquant(
-                        percdamp=0.01,
-                        groupsize=-1,
-                        actorder=True,
-                        static_groups=False,
-                        verbose=gptq_conf.verbose,
-                    )
-                    quantizers["model.layers.%d.%s" % (l_idx, name)] = gptq[
-                        name
-                    ].quantizer
-                    gptq[name].free()
-            """
-            Execute the quantized layer with the calibration inputs to obtain ouptuts
-             that will serve as inputs for the next layer.
-
-            This ensures that the quantization effects are correctly propagated to subsequent
-             layers.
-            """
-            batch_num = self.cache_kwargs.pop("batch_num")
-            for batch_idx in range(batch_num):
-                cache_args_batch = gather_single_batch_from_list(
-                    self.cache_args, batch_idx
-                )
-                cache_kwargs_batch = gather_single_batch_from_dict(
-                    self.cache_kwargs, batch_idx
-                )
-                outs = layer(*cache_args_batch, **cache_kwargs_batch)[0]
-                # Update inputs for next iteration.
-                self.cache_args[0][batch_idx] = outs
-            self.cache_kwargs["batch_num"] = batch_num
-
-            if torch.cuda.is_available():
-                torch.cuda.empty_cache()
-        # Restore the original cache configuration.
-        if hasattr(model, "config"):
-            model.config.use_cache = use_cache
-
-        return model

tico/experimental/quantization/algorithm/smoothquant/smooth_quant.py

@@ -1,164 +0,0 @@
-# Copyright (c) 2025 Samsung Electronics Co., Ltd. All Rights Reserved
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from typing import Any, Dict, List, Optional
-
-import torch
-
-
-@torch.no_grad()
-def smooth_weights(
-    front_module: torch.nn.Module,
-    back_modules: torch.nn.Module | List[torch.nn.Module],
-    activation_max: torch.Tensor,
-    alpha: float,
-):
-    """
-    Applies SmoothQuant-style smoothing to the weights and biases of two
-     connected modules using activation maximum values.
-
-    NOTE All modules **MUST** have `weight` and optionally `bias` attributes.
-
-    Parameters
-    -----------
-        front_module
-            The front module whose weights and biases will be adjusted.
-        back_modules
-            A list of back modules whose weights and biases will be adjusted.
-        activation_max
-            A tensor of channel-wise maximum activation values for the front module.
-        alpha
-            The smoothing factor that determines the scaling for weight adjustments.
-
-    Raises
-    -------
-    AttributeError
-        If `front_module` or any module in `back_modules` does not have `weight` attributes.
-    ValueError
-        If the shape of tensors in `activation_max` does not match the number of channels
-         in `front_module`'s weight.
-    NoteImplementedError
-        If `front_module` or any module in `back_modules` is of an unsupported type.
-    """
-    from transformers.models.llama.modeling_llama import LlamaRMSNorm
-
-    if not isinstance(back_modules, list):
-        back_modules = [back_modules]
-
-    # Check attributes
-    if not hasattr(front_module, "weight"):
-        raise AttributeError(
-            f"The front module '{type(front_module).__name__}' does not have a 'weight' attribute."
-        )
-    for back_m in back_modules:
-        if not hasattr(back_m, "weight"):
-            raise AttributeError(
-                f"The front module '{type(back_m).__name__}' does not have a 'weight' attribute."
-            )
-    # Check shapes
-    if isinstance(front_module, LlamaRMSNorm):
-        front_numel = front_module.weight.numel()
-    else:
-        raise NotImplementedError(
-            f"Unsupported module type: {type(front_module).__name__}"
-        )
-    for back_m in back_modules:
-        if isinstance(back_m, torch.nn.Linear):
-            back_numel = back_m.in_features
-        else:
-            raise NotImplementedError(
-                f"Unsupported module type: {type(front_module).__name__}"
-            )
-
-        if front_numel != back_numel or back_numel != activation_max.numel():
-            raise ValueError(
-                f"Shape mismatch: front_numel({front_numel}), back_numel({back_numel}), activation_max_numel({activation_max.numel()})"
-            )
-
-    # Compute scales
-    device, dtype = back_modules[0].weight.device, back_modules[0].weight.dtype
-    activation_max = activation_max.to(device=device, dtype=dtype)  # type: ignore[arg-type]
-    weight_scales = torch.cat(
-        [back_m.weight.abs().max(dim=0, keepdim=True)[0] for back_m in back_modules],  # type: ignore[operator]
-        dim=0,
-    )
-    weight_scales = weight_scales.max(dim=0)[0].clamp(min=1e-5)
-    scales = (
-        (activation_max.pow(alpha) / weight_scales.pow(1 - alpha))
-        .clamp(min=1e-5)
-        .to(device)  # type: ignore[arg-type]
-        .to(dtype)  # type: ignore[arg-type]
-    )
-
-    # Smooth
-    front_module.weight.div_(scales)
-    if hasattr(front_module, "bias"):
-        front_module.bias.div_(scales)
-
-    for back_m in back_modules:
-        back_m.weight.mul_(scales.view(1, -1))  # type: ignore[operator]
-
-
-@torch.no_grad()
-def apply_smoothing(
-    model: torch.nn.Module,
-    activation_max: Dict[str, torch.Tensor],
-    alpha: float = 0.5,
-    custom_alpha_map: Optional[Dict[str, float]] = None,
-):
-    """
-    Applies SmoothQuant-style smoothing to the model's weights using activation maximum values.
-
-    Parameters
-    -----------
-        model
-            A torch module whose weights will be smoothed.
-        activation_max
-            The channel-wise maximum activation values for the model.
-        alpha
-            The default smoothing factor to apply across all modules.
-        custom_alpha_map
-            A dictionary mapping layer/module names to custom alpha values.
-            Layers specified in this dictionary will use the corresponding alpha
-             value instead of the default.
-    """
-    from transformers.models.llama.modeling_llama import LlamaDecoderLayer
-
-    for name, module in model.named_modules():
-        alpha_to_apply = alpha
-        if custom_alpha_map and name in custom_alpha_map:
-            alpha_to_apply = custom_alpha_map[name]
-        if alpha_to_apply > 1.0:
-            raise RuntimeError(
-                f"Alpha value cannot exceed 1.0. Given alpha: {alpha_to_apply}"
-            )
-        # SmoothQuant is applied before capturing the graph. Therefore, it needs to know
-        #  specific module information.
-        # TODO Suport more modules.
-        if isinstance(module, LlamaDecoderLayer):
-            attn_ln = module.input_layernorm
-            qkv = [
-                module.self_attn.q_proj,
-                module.self_attn.k_proj,
-                module.self_attn.v_proj,
-            ]
-
-            qkv_input_scales = activation_max[name + ".self_attn.q_proj"]
-            smooth_weights(attn_ln, qkv, qkv_input_scales, alpha_to_apply)
-
-            ffn_ln = module.post_attention_layernorm
-            fcs = [module.mlp.gate_proj, module.mlp.up_proj]
-            fcs_input_scales = activation_max[name + ".mlp.gate_proj"]
-
-            smooth_weights(ffn_ln, fcs, fcs_input_scales, alpha_to_apply)

tico/experimental/quantization/evaluation/metric.py

@@ -1,109 +0,0 @@
-# Copyright (c) 2025 Samsung Electronics Co., Ltd. All Rights Reserved
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from typing import Any, Callable, Dict, List
-
-import numpy as np
-import torch
-
-
-def compute_peir(base: torch.Tensor, target: torch.Tensor):
-    """
-    Calculate the Peak Error to Interval Ratio (PEIR) between two tensors.
-
-    This function computes the PEIR between two tensors using the formula:
-        PEIR = max(abs(tensor1 - tensor2)) / (max(tensor1) - min(tensor2))
-    """
-    assert base.shape == target.shape, f"shape mismatch: {base.shape} != {target.shape}"
-    base_tensor = base.numpy()
-    target_tensor = target.numpy()
-    assert (
-        base_tensor.dtype == np.float32 and target_tensor.dtype == np.float32
-    ), f"dtype should be float32: base({base_tensor.dtype}), target({target_tensor.dtype})"
-
-    base_tensor = base_tensor.reshape(-1)
-    target_tensor = target_tensor.reshape(-1)
-
-    assert (
-        base_tensor.shape == target_tensor.shape
-    ), f"Shape mismatch: {base_tensor.shape} != {target_tensor.shape}"
-
-    peak_error = np.max(np.absolute(target_tensor - base_tensor))
-    interval = np.max(base_tensor) - np.min(base_tensor)
-    peir = peak_error / interval  # pylint: disable=invalid-name
-
-    min_value = min([base_tensor.min(), target_tensor.min()])
-    max_value = max([base_tensor.max(), target_tensor.max()])
-
-    interval = max_value - min_value
-    interval = 1.0 if interval == 0.0 else interval  # Avoid zero interval
-
-    return peir
-
-
-class MetricCalculator:
-    """
-    Compute metrics including both built-in and custom metrics.
-
-    metrics
-        A list of metric names for comparison.
-    custom_metrics
-        A dictionary of metric names and corresponding callable functions for comparison.
-        Example: {'mse': mean_squared_error, 'cosine_similarity': cosine_similarity_fn}
-    """
-
-    builtin_metrics = {
-        "peir": compute_peir,
-    }
-
-    def __init__(
-        self,
-        metrics: List[str] = list(),
-        custom_metrics: Dict[str, Callable] = dict(),
-    ):
-        self.metrics: Dict[str, Callable] = dict()
-
-        for m in metrics:
-            if m in self.builtin_metrics:
-                self.metrics[m] = self.builtin_metrics[m]
-            else:
-                raise RuntimeError(f"Invalid metric: {m}")
-
-        duplicates = set(self.metrics).intersection(custom_metrics.keys())
-        if len(duplicates) != 0:
-            raise RuntimeError(f"There are duplicate metrics: {duplicates}")
-
-        self.metrics = self.metrics | custom_metrics
-
-    def compute(
-        self, output1: List[torch.Tensor], output2: List[torch.Tensor]
-    ) -> Dict[str, List[Any]]:
-        """
-        Compute both built-in metrics (if provided) and custom metrics.
-
-        Returns
-        --------
-        Dict[str, Any]
-            A dictionary with metric names and their computed values.
-        """
-        results: Dict[str, List[Any]] = dict()
-
-        # Compute built-in metrics
-        if self.metrics is not None:
-            for m in self.metrics:
-                results[m] = list()
-                for out1, out2 in zip(output1, output2):
-                    results[m].append(self.builtin_metrics[m](out1, out2))
-
-        return results