tico 0.1.0.dev250803__py3-none-any.whl → 0.1.0.dev251102__py3-none-any.whl

tico/quantization/wrapq/examples/quantize_with_gptq.py

@@ -0,0 +1,265 @@
+# Copyright (c) 2025 Samsung Electronics Co., Ltd. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# =============================================================================
+# PTQ + GPTQ HYBRID QUANTIZATION PIPELINE
+# -----------------------------------------------------------------------------
+# This script shows how to:
+#   1. Load a pretrained FP Llama-3 model.
+#   2. Run GPTQ to quantize weights only.
+#   3. Wrap every Transformer layer with a PTQWrapper to quantize activations.
+#   4. Calibrate UINT-8 observers in a single pass over a text corpus.
+#   5. Inject GPTQ’s per-tensor weight scales / zero-points into the PTQ graph.
+#   6. Freeze all Q-params and compute Wikitext-2 perplexity.
+# =============================================================================
+
+import argparse
+import sys
+from typing import Any
+
+import torch
+import tqdm
+from datasets import load_dataset
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+from tico.quantization import convert, prepare
+from tico.quantization.config.gptq import GPTQConfig
+from tico.quantization.config.ptq import PTQConfig
+from tico.quantization.wrapq.observers.affine_base import AffineObserverBase
+from tico.quantization.wrapq.utils.introspection import build_fqn_map
+from tico.quantization.wrapq.utils.metrics import perplexity
+from tico.quantization.wrapq.wrappers.ptq_wrapper import PTQWrapper
+from tico.quantization.wrapq.wrappers.quant_module_base import QuantModuleBase
+
+
+# Token-budget presets for activation calibration
+TOKENS: dict[str, int] = {
+    # Smoke test (<1 min turnaround on CPU/GPU)
+    "debug": 2_000,  # ≈16 × 128-seq batches
+    # Good default for 1-7B models (≲3 % ppl delta)
+    "baseline": 50_000,
+    # Production / 4-bit observer smoothing
+    "production": 200_000,
+}
+
+DTYPE_MAP = {
+    "float32": torch.float32,
+    "bfloat16": torch.bfloat16,
+    "float16": torch.float16,
+}
+
+# Hardcoded dataset settings
+DATASET_NAME = "wikitext"
+DATASET_CONFIG = "wikitext-2-raw-v1"
+TRAIN_SPLIT = "train"
+TEST_SPLIT = "test"
+
+# -------------------------------------------------------------------------
+# 1. Helper — copy GPTQ (scale, zp) into PTQ observers
+# -------------------------------------------------------------------------
+def inject_gptq_qparams(
+    root: torch.nn.Module,
+    gptq_quantizers: dict[str, Any],  # {fp_name: quantizer}
+    weight_obs_name: str = "weight",
+):
+    """
+    For every `QuantModuleBase` whose `fp_name` matches a GPTQ key,
+    locate the observer called `weight_obs_name` and overwrite its
+    (scale, zero-point), then lock them against further updates.
+    """
+    for m in root.modules():
+        if not isinstance(m, QuantModuleBase):
+            continue
+        if m.fp_name is None:
+            continue
+        quantizer = gptq_quantizers.get(m.fp_name)
+        if quantizer is None:
+            continue
+        obs = m.get_observer(weight_obs_name)
+        if obs is None:
+            continue
+        assert isinstance(obs, AffineObserverBase)
+        # GPTQ quantizer attributes
+        obs.load_qparams(quantizer.scale, quantizer.zero, lock=True)
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="GPTQ+PTQ pipeline (weight-only + activation UINT8)"
+    )
+    parser.add_argument(
+        "--model", type=str, required=True, help="HF repo name or local path."
+    )
+    parser.add_argument(
+        "--device",
+        type=str,
+        default="cuda" if torch.cuda.is_available() else "cpu",
+        help="Device to run on (cuda|cpu|mps).",
+    )
+    parser.add_argument(
+        "--dtype",
+        choices=list(DTYPE_MAP.keys()),
+        default="float32",
+        help="Model dtype for load.",
+    )
+    parser.add_argument(
+        "--stride",
+        type=int,
+        default=512,
+        help="Sliding-window stride used for calibration and eval.",
+    )
+    parser.add_argument(
+        "--calib-preset",
+        choices=list(TOKENS.keys()),
+        default="debug",
+        help="Activation calibration token budget preset.",
+    )
+    parser.add_argument("--seed", type=int, default=42, help="Random seed.")
+    parser.add_argument(
+        "--trust-remote-code",
+        action="store_true",
+        help="Enable only if you trust the model repo code.",
+    )
+    parser.add_argument(
+        "--hf-token",
+        type=str,
+        default=None,
+        help="Optional HF token for gated/private repos.",
+    )
+    parser.add_argument(
+        "--use-cache",
+        dest="use_cache",
+        action="store_true",
+        default=False,
+        help="Use model KV cache if enabled (off by default).",
+    )
+    parser.add_argument(
+        "--no-tqdm", action="store_true", help="Disable tqdm progress bars."
+    )
+
+    args = parser.parse_args()
+
+    # Basic setup
+    torch.manual_seed(args.seed)
+    device = torch.device(args.device)
+    dtype = DTYPE_MAP[args.dtype]
+
+    print("=== Config ===")
+    print(f"Model            : {args.model}")
+    print(f"Device           : {device.type}")
+    print(f"DType            : {args.dtype}")
+    print(f"Stride           : {args.stride}")
+    print(
+        f"Calib preset     : {args.calib_preset} ({TOKENS[args.calib_preset]:,} tokens)"
+    )
+    print(f"Use HF cache?    : {args.use_cache}")
+    print()
+
+    # -------------------------------------------------------------------------
+    # 2. Load the FP backbone and tokenizer
+    # -------------------------------------------------------------------------
+    print("Loading FP model …")
+    tokenizer = AutoTokenizer.from_pretrained(
+        args.model,
+        trust_remote_code=args.trust_remote_code,
+        token=args.hf_token,
+    )
+    model = (
+        AutoModelForCausalLM.from_pretrained(
+            args.model,
+            torch_dtype=dtype,
+            trust_remote_code=args.trust_remote_code,
+            token=args.hf_token,
+        )
+        .to(device)
+        .eval()
+    )
+
+    model.config.use_cache = args.use_cache
+
+    # Build module -> FQN map BEFORE wrapping
+    m_to_fqn = build_fqn_map(model)
+
+    # -------------------------------------------------------------------------
+    # 3. Run GPTQ (weight-only) pass
+    # -------------------------------------------------------------------------
+    print("Applying GPTQ …")
+    dataset_test = load_dataset(DATASET_NAME, DATASET_CONFIG, split=TEST_SPLIT)
+    q_m = prepare(model, GPTQConfig(), inplace=True)
+
+    it = (
+        dataset_test
+        if args.no_tqdm
+        else tqdm.tqdm(dataset_test, desc="GPTQ calibration")
+    )
+    for d in it:
+        ids = tokenizer(d["text"], return_tensors="pt").input_ids.to(device)
+        q_m(ids)  # observers gather weight stats
+
+    q_m = convert(q_m, inplace=True)  # materialize INT-weight tensors
+
+    # -------------------------------------------------------------------------
+    # 4. Wrap every layer with PTQWrapper (activation UINT-8)
+    # -------------------------------------------------------------------------
+    print("Wrapping layers with PTQWrapper …")
+    qcfg = PTQConfig()  # default: per-tensor UINT8
+    prepare(q_m, qcfg)
+
+    # -------------------------------------------------------------------------
+    # 5. Single-pass activation calibration
+    # -------------------------------------------------------------------------
+    print("Calibrating UINT-8 observers …")
+    CALIB_TOKENS = TOKENS[args.calib_preset]
+    print(f"Calibrating with {CALIB_TOKENS:,} tokens.\n")
+    dataset_train = load_dataset(DATASET_NAME, DATASET_CONFIG, split=TRAIN_SPLIT)
+    calib_txt = " ".join(dataset_train["text"])[:CALIB_TOKENS]
+    train_ids = tokenizer(calib_txt, return_tensors="pt").input_ids.to(device)
+
+    # Overwrite weight observers with GPTQ statistics
+    if hasattr(q_m, "quantizers") and isinstance(q_m.quantizers, dict):
+        inject_gptq_qparams(q_m, q_m.quantizers)
+    else:
+        print(
+            "[Warn] q_m.quantizers not found or not a dict; skipping GPTQ qparam injection."
+        )
+
+    # Forward passes to collect activation ranges
+    iterator = range(0, train_ids.size(1) - 1, args.stride)
+    if not args.no_tqdm:
+        iterator = tqdm.tqdm(iterator, desc="Act-calibration")
+    with torch.no_grad():
+        for i in iterator:
+            q_m(train_ids[:, i : i + args.stride])
+
+    # Freeze all Q-params (scale, zero-point)
+    convert(q_m)
+
+    # -------------------------------------------------------------------------
+    # 6. Evaluate perplexity on Wikitext-2
+    # -------------------------------------------------------------------------
+    print("\nCalculating perplexities …")
+    enc = tokenizer("\n\n".join(dataset_test["text"]), return_tensors="pt")
+    ppl_uint8 = perplexity(q_m, enc, device, stride=args.stride)
+
+    print("\n┌── Wikitext-2 test perplexity ─────────────")
+    print(f"│ UINT-8 : {ppl_uint8:8.2f}")
+    print("└───────────────────────────────────────────")
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except Exception as e:
+        print(f"\n[Error] {e}", file=sys.stderr)
+        sys.exit(1)

tico/quantization/wrapq/mode.py

@@ -0,0 +1,32 @@
+# Copyright (c) 2025 Samsung Electronics Co., Ltd. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from enum import auto, Enum
+
+
+class Mode(Enum):
+    """
+    Mode — global FSM for PTQWrapper & Handlers.
+
+      • NO_QUANT : pure pass-through (no stats, no fake-quant)
+      • CALIB    : collect observer statistics only
+      • QUANT    : use cached (scale, zero-point) → fake-quant enabled
+    """
+
+    NO_QUANT = auto()
+    CALIB = auto()
+    QUANT = auto()
+
+    def __str__(self) -> str:
+        return self.name.lower()

tico/quantization/wrapq/observers/__init__.py

@@ -0,0 +1 @@
+# DO NOT REMOVE THIS FILE

tico/quantization/wrapq/observers/affine_base.py

@@ -0,0 +1,128 @@
+# Copyright (c) 2025 Samsung Electronics Co., Ltd. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import math
+from typing import Optional, Tuple
+
+import torch
+
+from tico.quantization.wrapq.dtypes import DType, UINT8
+from tico.quantization.wrapq.observers.base import ObserverBase
+from tico.quantization.wrapq.qscheme import QScheme
+
+
+class AffineObserverBase(ObserverBase):
+    """Base for affine observers (min/max → scale/zp)."""
+
+    def __init__(
+        self,
+        *,
+        name: str,
+        dtype: DType = UINT8,
+        qscheme: QScheme = QScheme.PER_TENSOR_ASYMM,
+        channel_axis: Optional[int] = None,
+    ):
+        super().__init__(
+            name=name, dtype=dtype, qscheme=qscheme, channel_axis=channel_axis
+        )
+
+    def reset(self) -> None:
+        """
+        Reset running min/max and drop cached qparams.
+        """
+        self.min_val: torch.Tensor = torch.tensor(math.inf)
+        self.max_val: torch.Tensor = torch.tensor(-math.inf)
+        if hasattr(self, "_cached_scale"):
+            del self._cached_scale
+        if hasattr(self, "_cached_zp"):
+            del self._cached_zp
+
+    def load_qparams(self, scale: torch.Tensor, zp: torch.Tensor, *, lock: bool = True):
+        """
+        Inject externally computed qparams and optionally lock the observer.
+
+        When locked, subsequent `collect()` calls are ignored.
+        """
+        self._cached_scale = scale.detach()
+        self._cached_zp = zp.to(torch.int)
+        if lock:
+            self.enabled = False
+
+    @property
+    def has_qparams(self) -> bool:
+        return hasattr(self, "_cached_scale")
+
+    def compute_qparams(self) -> Tuple[torch.Tensor, torch.Tensor]:
+        qmin, qmax = self.dtype.qmin, self.dtype.qmax
+        rng = self.max_val - self.min_val
+        eps = 1e-12
+
+        if self.qscheme.is_symmetric():
+            max_abs = torch.maximum(self.max_val.abs(), self.min_val.abs())
+            scale = torch.clamp(max_abs, min=eps) / qmax
+            zp = torch.zeros_like(scale, dtype=torch.int)
+            self._cached_scale, self._cached_zp = scale, zp
+            return scale, zp
+
+        if self.channel_axis is None:
+            if torch.all(rng.abs() < 1e-8):
+                C = self.min_val
+                if torch.allclose(C, torch.zeros_like(C)):
+                    scale = torch.ones_like(C)
+                    zp = torch.zeros_like(C, dtype=torch.int)
+                elif (C > 0).all():
+                    scale = torch.clamp(C, min=eps)
+                    zp = torch.zeros_like(C, dtype=torch.int)
+                else:
+                    scale = torch.clamp(C.abs(), min=eps)
+                    zp = torch.full_like(C, qmax, dtype=torch.int)
+            else:
+                scale = torch.clamp(rng, min=eps) / (qmax - qmin)
+                zp = (
+                    torch.round(qmin - self.min_val / scale)
+                    .clamp(qmin, qmax)
+                    .to(torch.int)
+                )
+        else:
+            scale = torch.clamp(rng, min=eps) / (qmax - qmin)
+            zp = (
+                torch.round(qmin - self.min_val / scale).clamp(qmin, qmax).to(torch.int)
+            )
+
+        self._cached_scale, self._cached_zp = scale, zp
+        return scale, zp
+
+    def fake_quant(self, x: torch.Tensor) -> torch.Tensor:
+        if not self.has_qparams:
+            raise RuntimeError(
+                "Call compute_qparams()/freeze_qparams() or load_qparams() first."
+            )
+        scale, zp = self._cached_scale, self._cached_zp
+        if self.channel_axis is None:
+            return torch.fake_quantize_per_tensor_affine(
+                x,
+                scale=scale,
+                zero_point=zp,
+                quant_min=self.dtype.qmin,
+                quant_max=self.dtype.qmax,
+            )
+        else:
+            return torch.fake_quantize_per_channel_affine(
+                x,
+                scale=scale,
+                zero_point=zp,
+                axis=self.channel_axis,
+                quant_min=self.dtype.qmin,
+                quant_max=self.dtype.qmax,
+            )

tico/quantization/wrapq/observers/base.py

@@ -0,0 +1,98 @@
+# Copyright (c) 2025 Samsung Electronics Co., Ltd. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from abc import ABC, abstractmethod
+from typing import Optional, Tuple
+
+import torch
+
+from tico.quantization.wrapq.dtypes import DType, UINT8
+from tico.quantization.wrapq.qscheme import QScheme
+
+
+class ObserverBase(ABC):
+    """
+    Minimal abstract base for all observers/quantizers.
+
+    Subclasses must implement:
+      - reset()
+      - collect(x)
+      - fake_quant(x)
+      - compute_qparams(): optional in practice for some observers (e.g., MX),
+        but still part of the interface; those can return None.
+    """
+
+    def __init__(
+        self,
+        *,
+        name: str,
+        dtype: DType = UINT8,
+        qscheme: QScheme = QScheme.PER_TENSOR_ASYMM,
+        channel_axis: Optional[int] = None,  # None → per-tensor
+    ):
+        self.name = name
+        self.dtype = dtype
+        self.qscheme = qscheme
+        self.channel_axis = channel_axis if qscheme.is_per_channel() else None
+        self.enabled = True
+        self.reset()
+
+    @abstractmethod
+    def reset(self) -> None:
+        """Clear any running statistics or cached params."""
+        raise NotImplementedError
+
+    def collect(self, x: torch.Tensor) -> None:
+        """
+        Update running statistics with a new batch of data.
+
+        This base implementation guards on `enabled` and then calls `_update_stats(x)`.
+        Subclasses should implement `_update_stats(x)` instead of overriding `collect`.
+        """
+        if not self.enabled:
+            return
+        self._update_stats(x)
+
+    @abstractmethod
+    def _update_stats(self, x: torch.Tensor) -> None:
+        """
+        Update running statistics (min/max, hist, mse buffers, ...).
+
+        Must be implemented by subclasses (e.g., MinMax, EMA, Histogram, MSE).
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    def fake_quant(self, x: torch.Tensor) -> torch.Tensor:
+        """
+        Apply the observer's quantization.
+        Implementations may or may not rely on qparams.
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    def compute_qparams(self) -> Optional[Tuple[torch.Tensor, torch.Tensor]]:
+        """
+        Compute and (if applicable) cache quantization params.
+        Affine observers typically return (scale, zero_point).
+        Observers that do not use qparams (e.g., MX) may return None.
+        """
+        raise NotImplementedError
+
+    # String repr helps debugging
+    def __repr__(self) -> str:
+        return (
+            f"{self.__class__.__name__}(name={self.name}, dtype={str(self.dtype)}, "
+            f"qscheme={str(self.qscheme)}, channel_axis={self.channel_axis}, enabled={self.enabled})"
+        )

tico/quantization/wrapq/observers/ema.py

@@ -0,0 +1,62 @@
+# Copyright (c) 2025 Samsung Electronics Co., Ltd. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import torch
+
+from tico.quantization.wrapq.observers.affine_base import AffineObserverBase
+from tico.quantization.wrapq.utils.reduce_utils import channelwise_minmax
+
+
+class EMAObserver(AffineObserverBase):
+    """
+    Exponential-Moving-Average min/max tracker.
+
+    Why?
+    -----
+    • Smoother than raw MinMax (reduces outlier shock).
+    • Much cheaper than histogram/MSE observers.
+
+    The update rule follows the common "momentum" form:
+
+        ema = momentum * ema + (1 - momentum) * new_value
+
+    With momentum → 0: FAST adaptation, momentum → 1: SLOW adaptation.
+    """
+
+    def __init__(
+        self,
+        *,
+        momentum: float = 0.9,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+        assert 0.0 < momentum < 1.0, "momentum must be in (0, 1)"
+        self.momentum = momentum
+
+    @torch.no_grad()
+    def _update_stats(self, x: torch.Tensor):
+        if self.channel_axis is None:
+            curr_min, curr_max = x.min(), x.max()
+        else:
+            curr_min, curr_max = channelwise_minmax(x, self.channel_axis)
+
+        if (
+            torch.isinf(self.min_val).any() and torch.isinf(self.max_val).any()
+        ):  # first batch → hard init
+            self.min_val, self.max_val = curr_min, curr_max
+            return
+
+        m = self.momentum
+        self.min_val = m * self.min_val + (1 - m) * curr_min
+        self.max_val = m * self.max_val + (1 - m) * curr_max

tico/quantization/wrapq/observers/identity.py

@@ -0,0 +1,74 @@
+# Copyright (c) 2025 Samsung Electronics Co., Ltd. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+IdentityObserver: a "no-op" observer for FP-only modules.
+
+Motivation
+----------
+Some layers should stay in full precision even when the rest of the model
+is quantized.  Attaching an `IdentityObserver` satisfies the wrapper API
+(`_update_stats()`, `compute_qparams()`, `fake_quant()`) without actually
+performing any statistics gathering or fake-quantization.
+"""
+import torch
+
+from tico.quantization.wrapq.observers.affine_base import AffineObserverBase
+
+
+class IdentityObserver(AffineObserverBase):
+    """
+    Passthrough observer that NEVER alters the tensor.
+
+    • `_update_stats()`   → does nothing
+    • `compute_qparams()` → returns (1.0, 0) "dummy" q-params
+    • `fake_quant()`      → returns `x` unchanged
+    """
+
+    def __init__(self, **kwargs):
+        # Call parent so the usual fields (`dtype`, `qscheme`, …) exist,
+        # but immediately disable any stateful behaviour.
+        super().__init__(**kwargs)
+
+        # Deactivate statistics collection permanently.
+        self.enabled = False
+
+        # Pre-cache sentinel q-params so wrapper code that blindly
+        # accesses them won't crash.
+        self._cached_scale = torch.tensor(1.0)
+        self._cached_zp = torch.tensor(0, dtype=torch.int)
+
+    def reset(self) -> None:  # (simple override – nothing to do)
+        """No internal state to reset."""
+        pass
+
+    def _update_stats(self, x: torch.Tensor) -> None:
+        """Skip statistic collection entirely."""
+        return
+
+    def compute_qparams(self):
+        """
+        Return the pre-cached (scale, zero_point) tuple.
+
+        Keeping the signature identical to other observers allows uniform
+        lifecycle management in wrapper code.
+        """
+        return self._cached_scale, self._cached_zp
+
+    def fake_quant(self, x: torch.Tensor):
+        """Identity mapping — leaves `x` in FP."""
+        return x
+
+    def __repr__(self) -> str:
+        return f"{self.__class__.__name__}()"