tico 0.2.0.dev260507__tar.gz → 0.2.0.dev260511__tar.gz

{tico-0.2.0.dev260507 → tico-0.2.0.dev260511}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tico
-Version: 0.2.0.dev260507
+Version: 0.2.0.dev260511
 Summary: Convert Exported Torch Module To Circle
 License: This file provides full text of licenses used in this project
         

tico-0.2.0.dev260511/tico/_version.py

@@ -0,0 +1 @@
+__version__ = "0.2.0.dev260511"

{tico-0.2.0.dev260507 → tico-0.2.0.dev260511}/tico/quantization/algorithm/smoothquant/smooth_quant.py

@@ -72,9 +72,22 @@ def smooth_weights(
     elif isinstance(front_module, torch.nn.LayerNorm):
         front_numel = front_module.weight.numel()
     else:
-        raise NotImplementedError(
-            f"Unsupported module type: {type(front_module).__name__}"
-        )
+        # Try Qwen3VLTextRMSNorm
+        try:
+            from transformers.models.qwen3_vl.modeling_qwen3_vl import (
+                Qwen3VLTextRMSNorm,
+            )
+
+            if isinstance(front_module, Qwen3VLTextRMSNorm):
+                front_numel = front_module.weight.numel()
+            else:
+                raise NotImplementedError(
+                    f"Unsupported module type: {type(front_module).__name__}"
+                )
+        except ImportError:
+            raise NotImplementedError(
+                f"Unsupported module type: {type(front_module).__name__}"
+            )
     for back_m in back_modules:
         if isinstance(back_m, torch.nn.Linear):
             back_numel = back_m.in_features
@@ -282,6 +295,78 @@ def _apply_if_fairseq_relu_bridge(
     return True
 
 
+# ────────────────────────────────────────────────────────────
+# Qwen3-VL Text Model Components (RMSNorm-based)
+# ────────────────────────────────────────────────────────────
+
+
+@torch.no_grad()
+def _apply_if_qwen3vl_text_decoder(
+    name: str,
+    module: torch.nn.Module,
+    activation_max: Dict[str, torch.Tensor],
+    alpha_to_apply: float,
+) -> bool:
+    """
+    Apply SmoothQuant smoothing to Qwen3VLTextDecoderLayer (RMSNorm-based).
+
+    Qwen3VLTextDecoderLayer structure:
+        - input_layernorm (RMSNorm) → self_attn (q_proj, k_proj, v_proj)
+        - post_attention_layernorm (RMSNorm) → mlp (gate_proj, up_proj)
+
+    Returns True if this handler applied smoothing to `module`.
+    """
+    try:
+        from transformers.models.qwen3_vl.modeling_qwen3_vl import (
+            Qwen3VLTextDecoderLayer,
+        )
+    except Exception:
+        return False
+
+    if not isinstance(module, Qwen3VLTextDecoderLayer):
+        return False
+
+    # Check for required attributes
+    if not hasattr(module, "input_layernorm") or not hasattr(
+        module, "post_attention_layernorm"
+    ):
+        return False
+    if not hasattr(module, "self_attn") or not hasattr(module, "mlp"):
+        return False
+
+    # Smooth input_layernorm → q_proj, k_proj, v_proj
+    attn_ln = module.input_layernorm
+    qkv = [
+        module.self_attn.q_proj,
+        module.self_attn.k_proj,
+        module.self_attn.v_proj,
+    ]
+    # Input-hook stats for q_proj input
+    qkv_input_scales = activation_max.get(name + ".self_attn.q_proj")
+    if qkv_input_scales is not None:
+        smooth_weights(attn_ln, qkv, qkv_input_scales, alpha_to_apply)
+    else:
+        print(
+            f"[SmoothQuant] Warning: activation stats not found for "
+            f"{name} self_attn.q_proj input."
+        )
+
+    # Smooth post_attention_layernorm → gate_proj, up_proj
+    ffn_ln = module.post_attention_layernorm
+    fcs = [module.mlp.gate_proj, module.mlp.up_proj]
+    # Input-hook stats for gate_proj input
+    fcs_input_scales = activation_max.get(name + ".mlp.gate_proj")
+    if fcs_input_scales is not None:
+        smooth_weights(ffn_ln, fcs, fcs_input_scales, alpha_to_apply)
+    else:
+        print(
+            f"[SmoothQuant] Warning: activation stats not found for "
+            f"{name} mlp.gate_proj input."
+        )
+
+    return True
+
+
 # ────────────────────────────────────────────────────────────
 # Qwen3-VL Vision Components (LayerNorm-based)
 # ────────────────────────────────────────────────────────────
@@ -467,6 +552,7 @@ _APPLIERS: List[
     Callable[[str, torch.nn.Module, Dict[str, torch.Tensor], float], bool]
 ] = [
     _apply_if_llama_decoder,
+    _apply_if_qwen3vl_text_decoder,
     _apply_if_qwen3vl_vision_block,
     _apply_if_qwen3vl_vision_patch_merger,
     _apply_if_fairseq_relu_bridge,
@@ -479,6 +565,7 @@ def apply_smoothing(
     activation_max: Dict[str, torch.Tensor],
     alpha: float = 0.5,
     custom_alpha_map: Optional[Dict[str, float]] = None,
+    exclude_appliers: Optional[List[str]] = None,
 ):
     """
     Applies SmoothQuant-style smoothing to the model's weights using activation maximum values.
@@ -495,7 +582,20 @@ def apply_smoothing(
             A dictionary mapping layer/module names to custom alpha values.
             Layers specified in this dictionary will use the corresponding alpha
              value instead of the default.
+        exclude_appliers
+            A list of applier function names to exclude from processing.
+            Valid names: '_apply_if_llama_decoder', '_apply_if_qwen3vl_text_decoder',
+            '_apply_if_qwen3vl_vision_block', '_apply_if_qwen3vl_vision_patch_merger',
+            '_apply_if_fairseq_relu_bridge'.
     """
+    # Build list of appliers to use (excluding specified ones)
+    if exclude_appliers is None:
+        appliers_to_use = _APPLIERS
+    else:
+        appliers_to_use = [
+            applier for applier in _APPLIERS if applier.__name__ not in exclude_appliers
+        ]
+
     for name, module in model.named_modules():
         alpha_to_apply = (
             custom_alpha_map.get(name, alpha) if custom_alpha_map else alpha
@@ -506,6 +606,6 @@ def apply_smoothing(
             )
 
         # Try each applier until one succeeds.
-        for applier in _APPLIERS:
+        for applier in appliers_to_use:
             if applier(name, module, activation_max, alpha_to_apply):
                 break  # applied → stop trying others

{tico-0.2.0.dev260507 → tico-0.2.0.dev260511}/tico/quantization/wrapq/examples/quantize_qwen3_vl_with_gptq.py

@@ -277,6 +277,12 @@ def parse_args():
         help="SmoothQuant alpha for vision components (Qwen3VLVisionBlock, Qwen3VLVisionPatchMerger). "
         "Range: 0.0-1.0. Higher = more weight smoothing.",
     )
+    parser.add_argument(
+        "--smoothquant_components",
+        choices=["vision", "text", "both"],
+        default=None,
+        help="Target components for SmoothQuant.",
+    )
     parser.add_argument(
         "--print_quantized_model",
         action="store_true",
@@ -331,6 +337,13 @@ def parse_args():
         help="Sliding window stride for perplexity calculation.",
     )
 
+    parser.add_argument(
+        "--ppl_split",
+        type=str,
+        default="test",
+        help="Split for PPL evaluation",
+    )
+
     return parser.parse_args()
 
 
@@ -829,7 +842,7 @@ def main() -> None:
     # PPL evaluation on original model
     if args.ppl_dataset:
         print("\n=== PPL Evaluation (Original Model) ===")
-        ds_ppl, _ = get_dataset(args.ppl_dataset, n=args.nsamples_for_evaluation)
+        ds_ppl, _ = get_dataset(args.ppl_dataset, split=args.ppl_split, n=-1)
         original_ppl = evaluate_ppl(
             model=model,
             tokenizer=processor.tokenizer,
@@ -849,10 +862,29 @@ def main() -> None:
     )
 
     # -------------------------------------------------------------------------
-    # Apply SmoothQuant transformation (for LayerNorm-based vision components)
+    # Apply SmoothQuant transformation
     # -------------------------------------------------------------------------
     if args.smoothquant:
-        print("Applying SmoothQuant smoothing for vision LayerNorm layers …")
+        if args.smoothquant_components is None:
+            raise ValueError(
+                "--smoothquant_components must be specified when "
+                "--smoothquant is enabled."
+            )
+        # Build exclude_appliers list based on arguments
+        exclude_appliers = []
+        if args.smoothquant_components == "text":
+            exclude_appliers.extend(
+                [
+                    "_apply_if_qwen3vl_vision_block",
+                    "_apply_if_qwen3vl_vision_patch_merger",
+                ]
+            )
+        if args.smoothquant_components == "vision":
+            exclude_appliers.append("_apply_if_qwen3vl_text_decoder")
+
+        print(
+            f"Applying SmoothQuant smoothing for {args.smoothquant_components} components"
+        )
 
         # Compute activation maximum values from calibration data
         print("Computing activation maximum values for SmoothQuant …")
@@ -902,6 +934,7 @@ def main() -> None:
             model,
             activation_max,
             alpha=args.smoothquant_alpha,
+            exclude_appliers=exclude_appliers if exclude_appliers else None,
         )
         print("SmoothQuant smoothing complete.")
 
@@ -1021,7 +1054,7 @@ def main() -> None:
     # PPL evaluation on quantized model
     if args.ppl_dataset:
         print("\n=== PPL Evaluation (Quantized Model) ===")
-        ds_ppl, _ = get_dataset(args.ppl_dataset, n=args.nsamples_for_evaluation)
+        ds_ppl, _ = get_dataset(args.ppl_dataset, split=args.ppl_split, n=-1)
         quantized_ppl = evaluate_ppl(
             model=q_m,
             tokenizer=processor.tokenizer,