tico 0.3.0.dev261006__tar.gz → 0.3.0.dev261007__tar.gz

{tico-0.3.0.dev261006 → tico-0.3.0.dev261007}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tico
-Version: 0.3.0.dev261006
+Version: 0.3.0.dev261007
 Summary: Convert Exported Torch Module To Circle
 License: This file provides full text of licenses used in this project
         

tico-0.3.0.dev261007/tico/_version.py

@@ -0,0 +1 @@
+__version__ = "0.3.0.dev261007"

tico-0.3.0.dev261007/tico/passes/lower_multi_index_to_gather_nd.py

@@ -0,0 +1,188 @@
+# Copyright (c) 2026 Samsung Electronics Co., Ltd. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import List, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    import torch.fx
+
+import torch
+from torch.export import ExportedProgram
+
+from tico.passes.convert_gather_to_gather_nd import ConvertGatherToGatherNd
+from tico.utils import logging
+from tico.utils.errors import NotYetSupportedError
+from tico.utils.graph import create_node
+from tico.utils.passes import PassBase, PassResult
+from tico.utils.trace_decorators import trace_graph_diff_on_pass
+from tico.utils.utils import is_target_node, set_new_meta_val
+from tico.utils.validate_args_kwargs import IndexArgs
+
+
+@trace_graph_diff_on_pass
+class LowerMultiIndexToGatherNd(PassBase):
+    """
+    Lower ``aten.index.Tensor`` with several index tensors to
+    ``circle_custom.gather_nd``.
+
+    PyTorch advanced indexing with ``k`` leading index tensors broadcasts the
+    indices to a common shape ``B`` and reads one coordinate per element::
+
+        out[b...] = input[i0[b...], i1[b...], ..., i(k-1)[b...], ...]
+
+    Circle ``GATHER_ND`` reads full coordinates from the last dimension of its
+    ``indices`` input, so the lowering stacks the broadcast indices::
+
+        aten.index(input, [i0, ..., i(k-1)])
+          -> circle_custom.gather_nd(
+                 input, cat([unsqueeze(expand(int32(i), B), -1) ...], -1))
+
+    The resulting ``indices`` tensor has shape ``B + [k]`` and the output keeps
+    PyTorch's shape ``B + input.shape[k:]``.
+
+    Preconditions
+        - The node is ``aten.index.Tensor`` whose ``indices`` list contains at
+          least two non-``None`` entries.
+        - The non-``None`` entries are FX nodes that occupy the leading positions
+          ``0 .. k-1``; any remaining entries are ``None``. Advanced indices that
+          start after a sliced dimension or are separated by one are left
+          untouched (the serializer reports them as unsupported).
+        - Every index tensor has an int32 or int64 dtype and a static shape, and
+          the input has a static shape.
+
+    Transformation
+        - Index tensors are cast to int32 when needed, broadcast to the common
+          shape with ``aten.expand`` when their shape differs from it, unsqueezed
+          and concatenated along a new trailing dimension.
+        - The ``aten.index`` node is replaced by ``circle_custom.gather_nd`` with
+          recomputed metadata; its users are redirected.
+
+    Postconditions
+        No ``aten.index.Tensor`` with two or more leading index tensors remains.
+
+    Semantic assumptions
+        Index values are non-negative, as for the single-index ``GATHER``
+        lowering in the serializer. ``GATHER_ND`` does not wrap negative
+        coordinates.
+    """
+
+    def __init__(self):
+        super().__init__()
+
+    def call(self, exported_program: ExportedProgram) -> PassResult:
+        logger = logging.getLogger(__name__)
+
+        graph_module = exported_program.graph_module
+        graph = graph_module.graph
+        modified = False
+        for node in list(graph.nodes):
+            if not is_target_node(node, torch.ops.aten.index.Tensor):
+                continue
+
+            args = IndexArgs(*node.args, **node.kwargs)  # type: ignore[arg-type]
+            input_node = args.input
+            indices = list(args.indices)
+
+            index_nodes: List["torch.fx.Node"] = []
+            for position, index in enumerate(indices):
+                if index is None:
+                    continue
+                if not isinstance(index, torch.fx.Node):
+                    # Non-node indices are outside this lowering.
+                    index_nodes = []
+                    break
+                if position != len(index_nodes):
+                    # Advanced indices must be leading and contiguous.
+                    index_nodes = []
+                    break
+                index_nodes.append(index)
+
+            if len(index_nodes) < 2:
+                continue
+
+            for index in index_nodes:
+                if index.meta["val"].dtype not in (torch.int32, torch.int64):
+                    raise NotYetSupportedError(
+                        "aten.index.Tensor indices must have int32 or int64 dtype "
+                        "for GatherNd lowering."
+                    )
+
+            get_static_shape = ConvertGatherToGatherNd._get_static_shape
+            get_static_shape(input_node, "aten.index input")
+            index_shapes = [
+                get_static_shape(index, "aten.index index") for index in index_nodes
+            ]
+            broadcast_shape = list(torch.broadcast_shapes(*index_shapes))
+
+            logger.debug(
+                "%s: lowering aten.index with %d indices to circle_custom.gather_nd "
+                "(broadcast shape %r)",
+                node,
+                len(index_nodes),
+                broadcast_shape,
+            )
+
+            with graph.inserting_before(node):
+                coordinates = []
+                for index, index_shape in zip(index_nodes, index_shapes):
+                    coordinate = ConvertGatherToGatherNd._cast_index_to_int32(
+                        graph, index
+                    )
+                    if index_shape != broadcast_shape:
+                        coordinate = create_node(
+                            graph,
+                            torch.ops.aten.expand.default,
+                            args=(coordinate, broadcast_shape),
+                            origin=node,
+                        )
+                        set_new_meta_val(coordinate)
+                    unsqueezed = create_node(
+                        graph,
+                        torch.ops.aten.unsqueeze.default,
+                        args=(coordinate, -1),
+                        origin=node,
+                    )
+                    set_new_meta_val(unsqueezed)
+                    coordinates.append(unsqueezed)
+
+                full_indices = create_node(
+                    graph,
+                    torch.ops.aten.cat.default,
+                    args=(coordinates, -1),
+                    origin=node,
+                )
+                set_new_meta_val(full_indices)
+
+                gather_nd = create_node(
+                    graph,
+                    torch.ops.circle_custom.gather_nd,
+                    args=(input_node, full_indices),
+                    origin=node,
+                )
+                set_new_meta_val(gather_nd)
+
+            assert gather_nd.meta["val"].shape == node.meta["val"].shape, (
+                f"{node.name}: GatherNd output shape {gather_nd.meta['val'].shape} "
+                f"differs from aten.index output shape {node.meta['val'].shape}"
+            )
+            node.replace_all_uses_with(gather_nd, propagate_meta=False)
+            modified = True
+            logger.debug("%s is replaced with %s", node.name, gather_nd.name)
+
+        if modified:
+            graph.eliminate_dead_code()
+            graph.lint()
+            graph_module.recompile()
+
+        return PassResult(modified)

tico-0.3.0.dev261007/tico/passes/remove_empty_cat_inputs.py

@@ -0,0 +1,183 @@
+# Copyright (c) 2026 Samsung Electronics Co., Ltd. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import List, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    import torch.fx
+
+import torch
+from torch.export import ExportedProgram
+
+from tico.passes import ops
+from tico.passes.remove_unused_placeholder import remove_unused_constant_placeholders
+from tico.serialize.circle_mapping import extract_shape
+from tico.utils import logging
+from tico.utils.passes import PassBase, PassResult
+from tico.utils.trace_decorators import (
+    trace_const_diff_on_pass,
+    trace_graph_diff_on_pass,
+)
+from tico.utils.utils import is_target_node
+from tico.utils.validate_args_kwargs import CatArgs
+
+
+def _is_legacy_empty_tensor(node: "torch.fx.Node") -> bool:
+    """
+    Return whether ``node`` is a rank-1 tensor with a static size of zero.
+
+    ``torch.cat`` skips such tensors even when the other inputs have a different
+    rank (a legacy behavior kept for backward compatibility).
+    """
+    val = node.meta.get("val")
+    if not isinstance(val, torch.Tensor):
+        return False
+    if val.dim() != 1:
+        return False
+    size = val.shape[0]
+    # A symbolic size is a SymInt, not an int, and is never treated as empty.
+    if not isinstance(size, int) or size != 0:
+        return False
+    return True
+
+
+def _promoted_dtype(nodes: List["torch.fx.Node"]) -> torch.dtype:
+    """Return the dtype ``torch.cat`` produces for the given tensor inputs."""
+    dtypes = [node.meta["val"].dtype for node in nodes]
+    result = dtypes[0]
+    for dtype in dtypes[1:]:
+        result = torch.promote_types(result, dtype)
+    return result
+
+
+@trace_graph_diff_on_pass
+@trace_const_diff_on_pass
+class RemoveEmptyCatInputs(PassBase):
+    """
+    Remove rank-1 empty inputs from ``aten.cat`` and drop the ``cat`` when a single
+    input remains.
+
+    PyTorch lets ``torch.cat`` accept a 1-D tensor of shape ``[0]`` next to inputs of
+    any other rank and simply ignores it. Hugging Face ``DynamicCache`` relies on
+    this: the first ``update()`` concatenates ``torch.tensor([])`` with the new key
+    and value states. Circle ``CONCATENATION`` requires every input to have the same
+    rank, so the exported operator is rejected by Circle consumers.
+
+    Preconditions
+        - The node is ``aten.cat``.
+        - At least one input has a tensor ``meta["val"]`` of rank 1 with a static
+          size of zero.
+        - At least one input does not satisfy the previous bullet.
+        - The dtype promoted from the remaining inputs equals the ``cat`` output
+          dtype, so the removed inputs did not take part in type promotion.
+
+    Transformation
+        - Matching inputs are removed from the ``cat`` input list.
+        - When exactly one input remains, all uses of the ``cat`` are redirected to
+          that input; its shape and dtype equal the ``cat`` output by construction.
+        - Lifted constant placeholders that only fed removed inputs are deleted from
+          the graph, the ``ExportedProgram`` constants, and the graph signature.
+          User input placeholders are never removed.
+
+    Postconditions
+        No ``aten.cat`` keeps a rank-1 zero-size input of the output dtype.
+
+    Semantic assumptions
+        A rank-1 zero-size tensor contributes no elements along any concatenation
+        dimension, so removing it preserves the value, shape, and dtype of the
+        result as long as it does not change type promotion. Empty inputs that
+        widen the result dtype, inputs with symbolic sizes, and zero-size inputs
+        with the same rank as the output are left untouched.
+    """
+
+    def __init__(self):
+        super().__init__()
+
+    def call(self, exported_program: ExportedProgram) -> PassResult:
+        logger = logging.getLogger(__name__)
+
+        graph_module = exported_program.graph_module
+        graph = graph_module.graph
+        modified = False
+        placeholder_candidates: List["torch.fx.Node"] = []
+        for cat in graph.nodes:
+            if not is_target_node(cat, ops.aten.cat):
+                continue
+
+            cat_val = cat.meta.get("val")
+            if not isinstance(cat_val, torch.Tensor):
+                continue
+
+            args = CatArgs(*cat.args, **cat.kwargs)  # type: ignore[arg-type]
+            inputs = args.tensors
+            dim = args.dim
+
+            kept = [t for t in inputs if not _is_legacy_empty_tensor(t)]
+            if len(kept) == len(inputs) or len(kept) == 0:
+                continue
+            if any(not isinstance(t.meta.get("val"), torch.Tensor) for t in kept):
+                continue
+            # Dropping an input must not change the promoted result dtype.
+            if _promoted_dtype(kept) != cat_val.dtype:
+                continue
+
+            removed = [t for t in inputs if t not in kept]
+            for node in removed:
+                if node.op == "placeholder":
+                    placeholder_candidates.append(node)
+                placeholder_candidates.extend(
+                    n for n in node.all_input_nodes if n.op == "placeholder"
+                )
+
+            if len(kept) == 1:
+                remaining = kept[0]
+                assert extract_shape(remaining) == extract_shape(cat), (
+                    f"{cat.name}: shape mismatch after removing empty inputs "
+                    f"({extract_shape(remaining)} vs {extract_shape(cat)})"
+                )
+                remaining_val = remaining.meta.get("val")
+                assert (
+                    isinstance(remaining_val, torch.Tensor)
+                    and remaining_val.dtype == cat_val.dtype
+                )
+                cat.replace_all_uses_with(remaining, propagate_meta=False)
+                logger.debug(
+                    f"{cat.name} is replaced with {remaining.name}; "
+                    f"empty inputs removed: {[n.name for n in removed]}"
+                )
+            else:
+                cat.args = (kept, dim)
+                cat.kwargs = {}
+                logger.debug(
+                    f"Empty inputs removed from {cat.name}: {[n.name for n in removed]}"
+                )
+
+            modified = True
+
+        if not modified:
+            return PassResult(False)
+
+        graph.eliminate_dead_code()
+        graph.lint()
+        graph_module.recompile()
+
+        removed_placeholders = remove_unused_constant_placeholders(
+            exported_program, candidates=placeholder_candidates
+        )
+        if removed_placeholders:
+            logger.debug(
+                f"Unused constant placeholders are removed: {removed_placeholders}"
+            )
+
+        return PassResult(modified)

{tico-0.3.0.dev261006 → tico-0.3.0.dev261007}/tico/passes/remove_unused_placeholder.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import TYPE_CHECKING
+from typing import Iterable, List, Optional, TYPE_CHECKING
 
 if TYPE_CHECKING:
     import torch.fx
@@ -75,6 +75,71 @@ def _remove_constant_placeholder(
     exported_program.graph.erase_node(node)
 
 
+def remove_unused_constant_placeholders(
+    exported_program: ExportedProgram,
+    candidates: Optional[Iterable["torch.fx.Node"]] = None,
+) -> List[str]:
+    """
+    Remove unused constant placeholders and update the graph signature.
+
+    FX dead-code elimination does not remove placeholder nodes even when they have
+    no users. This helper removes placeholders that correspond to parameters,
+    buffers, or lifted tensor constants and have no users, then rebuilds the
+    ExportedProgram input specs so that they match the remaining placeholders.
+
+    Parameters
+    ----------
+    exported_program
+        The program whose graph and signature are updated in place.
+    candidates
+        Placeholder nodes to consider. When ``None``, every placeholder of the graph
+        is considered. Nodes that are not constant placeholders or that still have
+        users are ignored.
+
+    Returns
+    -------
+    List[str]
+        Names of the removed placeholders, in graph order. Empty when nothing was
+        removed; the graph is left untouched in that case.
+
+    Runtime user input placeholders are never removed.
+    """
+
+    graph_module = exported_program.graph_module
+    graph: torch.fx.Graph = graph_module.graph
+
+    candidate_set = None if candidates is None else set(candidates)
+    unused_placeholders = [
+        node
+        for node in graph.nodes
+        if (candidate_set is None or node in candidate_set)
+        and _is_constant_placeholder(exported_program, node)
+        and len(node.users) == 0
+    ]
+
+    if not unused_placeholders:
+        return []
+
+    removed_names = [node.name for node in unused_placeholders]
+
+    for node in unused_placeholders:
+        _remove_constant_placeholder(exported_program, node)
+
+    existing_name_to_spec = {
+        spec.arg.name: spec for spec in exported_program.graph_signature.input_specs
+    }
+    exported_program.graph_signature.input_specs = [
+        existing_name_to_spec[node.name]
+        for node in graph.nodes
+        if node.op == "placeholder"
+    ]
+
+    graph.lint()
+    graph_module.recompile()
+
+    return removed_names
+
+
 @trace_graph_diff_on_pass
 @trace_const_diff_on_pass
 class RemoveUnusedPlaceholder(PassBase):
@@ -95,36 +160,10 @@ class RemoveUnusedPlaceholder(PassBase):
     def call(self, exported_program: ExportedProgram) -> PassResult:
         logger = logging.getLogger(__name__)
 
-        graph_module = exported_program.graph_module
-        graph: torch.fx.Graph = graph_module.graph
-
-        unused_placeholders = [
-            node
-            for node in graph.nodes
-            if _is_constant_placeholder(exported_program, node) and len(node.users) == 0
-        ]
-
-        if not unused_placeholders:
-            return PassResult(False)
-
-        removed_names = [node.name for node in unused_placeholders]
-
-        for node in unused_placeholders:
-            _remove_constant_placeholder(exported_program, node)
-
-        existing_name_to_spec = {
-            spec.arg.name: spec for spec in exported_program.graph_signature.input_specs
-        }
-        exported_program.graph_signature.input_specs = [
-            existing_name_to_spec[node.name]
-            for node in graph.nodes
-            if node.op == "placeholder"
-        ]
-
-        graph.lint()
-        graph_module.recompile()
+        removed_names = remove_unused_constant_placeholders(exported_program)
 
-        logger.debug(f"Unused constant placeholders are removed: {removed_names}")
+        if removed_names:
+            logger.debug(f"Unused constant placeholders are removed: {removed_names}")
 
         # Run only once.
         return PassResult(False)

{tico-0.3.0.dev261006 → tico-0.3.0.dev261007}/tico/quantization/algorithm/qwen3_vl_gptq/gptq.py

@@ -274,6 +274,7 @@ class GPTQ:
         normalize_H: bool = True,
         hessian_dtype: torch.dtype = torch.float32,
         inp_dtype: torch.dtype = torch.float32,
+        gptq_v2_alpha: float = 0.25,
     ):
         """
         Initialize GPTQ state for a single layer.
@@ -291,12 +292,16 @@ class GPTQ:
                 the GPTQv2 dXXT cross-term (dX @ inp.T). Hessian storage and
                 factorization follow hessian_dtype. Defaults to torch.float32
                 it is faster and uses less memory.
+            gptq_v2_alpha: Scaling factor for the GPTQv2 P-correction matrix.
+                A value of 0 disables the correction (standard GPTQ). Values
+                around 0.25 provide the best empirical results.
         """
         self.layer = layer
         self.dev = self.layer.weight.device
         self.normalize_H = normalize_H
         self.hessian_dtype = hessian_dtype
         self.inp_dtype = inp_dtype
+        self.gptq_v2_alpha = gptq_v2_alpha
 
         w = layer.weight.data.clone()
         if isinstance(layer, (nn.Conv1d, nn.Conv2d, nn.Conv3d)):
@@ -495,7 +500,6 @@ class GPTQ:
         actorder: bool = False,
         static_groups: bool = False,
         verbose: bool = False,
-        alpha: float = 0.25,
     ) -> None:
         """
         Run blockwise GPTQ quantization on the layer weights.
@@ -584,7 +588,7 @@ class GPTQ:
         P = None
         if self.dXXT is not None:
             dXXT = self.dXXT.to(hinv.dtype)
-            P = alpha * ((dXXT @ hinv.T).triu(diagonal=1)) @ hinv
+            P = self.gptq_v2_alpha * ((dXXT @ hinv.T).triu(diagonal=1)) @ hinv
 
         # Downcast to float32 for the weight-update loop (w is float32);
         # the hessian_dtype precision of H/dXXT and of the Cholesky

{tico-0.3.0.dev261006 → tico-0.3.0.dev261007}/tico/quantization/algorithm/qwen3_vl_gptq/quantizer.py

@@ -1684,6 +1684,7 @@ class Qwen3VLGPTQQuantizer(BaseQuantizer):
                 normalize_H=gptq_conf.normalize_H,
                 hessian_dtype=gptq_conf.hessian_dtype,
                 inp_dtype=gptq_conf.inp_dtype,
+                gptq_v2_alpha=gptq_conf.gptq_v2_alpha,
             )
 
             full_name = module_name.get(submodule, local_name)
@@ -1781,7 +1782,6 @@ class Qwen3VLGPTQQuantizer(BaseQuantizer):
                 actorder=gptq_conf.actorder,
                 static_groups=gptq_conf.static_groups,
                 verbose=gptq_conf.verbose,
-                alpha=gptq_conf.gptq_v2_alpha,
             )
 
             full_name = module_name.get(submodule, local_name)