tico 0.1.0.dev250805__py3-none-any.whl → 0.1.0.dev250806__py3-none-any.whl

tico/__init__.py

@@ -29,7 +29,7 @@ __all__ = [
 ]
 
 # THIS LINE IS AUTOMATICALLY GENERATED BY setup.py
-__version__ = "0.1.0.dev250805"
+__version__ = "0.1.0.dev250806"
 
 MINIMUM_SUPPORTED_VERSION = "2.5.0"
 SECURE_TORCH_VERSION = "2.6.0"

tico/experimental/quantization/algorithm/gptq/quantizer.py

@@ -17,6 +17,7 @@ import types
 from typing import Any, Callable, Dict, List, Optional
 
 import torch
+from tqdm.auto import tqdm
 
 from tico.experimental.quantization.algorithm.gptq.gptq import GPTQ
 from tico.experimental.quantization.algorithm.gptq.utils import (
@@ -181,7 +182,9 @@ class GPTQQuantizer(BaseQuantizer):
             target_layers = [model]
 
         quantizers: Dict[str, Any] = {}
-        for l_idx, layer in enumerate(target_layers):
+        for l_idx, layer in enumerate(
+            tqdm(target_layers, desc="Quantizing layers", unit="layer")
+        ):
             # 1) Identify quantizable submodules within the layer
             full = find_layers(layer)
             sequential = [list(full.keys())]
@@ -210,7 +213,12 @@ class GPTQQuantizer(BaseQuantizer):
 
                 # Run layer forward over all cached batches to build Hessian/statistics
                 batch_num = self.num_batches
-                for batch_idx in range(batch_num):
+                for batch_idx in tqdm(
+                    range(batch_num),
+                    desc=f"[L{l_idx}] collecting",
+                    leave=False,
+                    unit="batch",
+                ):
                     cache_args_batch = gather_single_batch_from_list(
                         self.cache_args, batch_idx
                     )
@@ -238,7 +246,12 @@ class GPTQQuantizer(BaseQuantizer):
                     gptq[name].free()
 
             # 4) After quantization, re-run the layer to produce outputs for the next layer
-            for batch_idx in range(batch_num):
+            for batch_idx in tqdm(
+                range(batch_num),
+                desc=f"[L{l_idx}] re-forward",
+                leave=False,
+                unit="batch",
+            ):
                 cache_args_batch = gather_single_batch_from_list(
                     self.cache_args, batch_idx
                 )

tico/experimental/quantization/evaluation/evaluate.py

@@ -114,7 +114,6 @@ def evaluate(
     input_data: InputDataType = None,
     *,
     mode="plot",
-    metrics: List[str] = ["peir"],
     custom_metrics: Dict[str, Callable] = dict(),
 ) -> Optional[Dict[str, Any]]:
     """
@@ -140,8 +139,6 @@ def evaluate(
             The mode of operation. Options are:
             - "plot": Plot the results (default)
             - "return": Return the results.
-        metrics
-            A list of metric names for comparison.
         custom_metrics
             A dictionary of metric names and corresponding callable functions for comparison.
             Example: {'mse': mean_squared_error, 'cosine_similarity': cosine_similarity_fn}
@@ -205,7 +202,7 @@ def evaluate(
         )
 
     # Computes the comparison score based on the provided metrics.
-    metric_calculator = MetricCalculator(metrics, custom_metrics)
+    metric_calculator = MetricCalculator(custom_metrics)
     results: Dict[str, Any] = metric_calculator.compute(torch_output, circle_output)
 
     if mode == "return":

tico/experimental/quantization/evaluation/metric.py

@@ -12,98 +12,118 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import Any, Callable, Dict, List
+from typing import Any, Callable, Dict, List, Optional
 
 import numpy as np
 import torch
 
 
-def compute_peir(base: torch.Tensor, target: torch.Tensor):
+def compute_max_abs_diff(base: torch.Tensor, target: torch.Tensor) -> float:
     """
-    Calculate the Peak Error to Interval Ratio (PEIR) between two tensors.
-
-    This function computes the PEIR between two tensors using the formula:
-        PEIR = max(abs(tensor1 - tensor2)) / (max(tensor1) - min(tensor2))
+    Return the *maximum* absolute element-wise difference between two tensors.
     """
-    assert base.shape == target.shape, f"shape mismatch: {base.shape} != {target.shape}"
-    base_tensor = base.numpy()
-    target_tensor = target.numpy()
-    assert (
-        base_tensor.dtype == np.float32 and target_tensor.dtype == np.float32
-    ), f"dtype should be float32: base({base_tensor.dtype}), target({target_tensor.dtype})"
-
-    base_tensor = base_tensor.reshape(-1)
-    target_tensor = target_tensor.reshape(-1)
+    assert base.shape == target.shape, "shape mismatch"
+    return (base.detach() - target.detach()).abs().max().item()
 
-    assert (
-        base_tensor.shape == target_tensor.shape
-    ), f"Shape mismatch: {base_tensor.shape} != {target_tensor.shape}"
 
-    peak_error = np.max(np.absolute(target_tensor - base_tensor))
-    interval = np.max(base_tensor) - np.min(base_tensor)
-    peir = peak_error / interval  # pylint: disable=invalid-name
-
-    min_value = min([base_tensor.min(), target_tensor.min()])
-    max_value = max([base_tensor.max(), target_tensor.max()])
+def compute_peir(base: torch.Tensor, target: torch.Tensor) -> float:
+    """
+    Peak-Error-to-Interval Ratio (PEIR).
 
-    interval = max_value - min_value
-    interval = 1.0 if interval == 0.0 else interval  # Avoid zero interval
+        PEIR = max(|base - target|) / (max(base) - min(base))
 
-    return peir
+    The interval denominator uses the reference (*base*) tensor only — this
+    makes PEIR independent of quantisation error in `target`.
+    """
+    assert base.shape == target.shape, "shape mismatch"
+    peak_error = (base.detach() - target.detach()).abs().max().item()
+    interval = (base.detach().max() - base.detach().min()).item()
+    interval = 1.0 if interval == 0.0 else interval  # avoid divide-by-zero
+    return peak_error / interval
 
 
 class MetricCalculator:
     """
-    Compute metrics including both built-in and custom metrics.
-
-    metrics
-        A list of metric names for comparison.
-    custom_metrics
-        A dictionary of metric names and corresponding callable functions for comparison.
-        Example: {'mse': mean_squared_error, 'cosine_similarity': cosine_similarity_fn}
+    Lightweight registry-and-dispatcher for **pair-wise tensor comparison metrics**.
+
+    Purpose
+    -------
+    Consolidate all metrics used to assess the discrepancy between a reference
+    (usually FP32) tensor and its quantized counterpart, while letting the caller
+    choose *at runtime* which subset to evaluate.
+
+    Built-in metrics
+    ----------------
+    Key                     Description
+    --------------------    -------------------------------------------------
+    "diff" / "max_abs_diff"  Maximum absolute element-wise difference
+    "peir"                   Peak-Error-to-Interval Ratio
+
+    Usage pattern
+    -------------
+    >>> calc = MetricCalculator(custom_metrics={'mse': mse_fn})
+    >>> stats = calc.compute(fp_outs, q_outs, metrics=['diff', 'mse'])
+
+    • **Instantiation** registers any extra user metrics
+      (signature: ``fn(base: Tensor, target: Tensor) -> float``).
+    • **compute(...)** takes two *equal-length* lists of tensors and an optional
+      list of metric names.
+        — If *metrics* is *None*, every registered metric is evaluated.
+        — Returns a dict: ``{metric_name -> [value for each tensor pair]}``.
+
+    Implementation notes
+    --------------------
+    * All tensors are detached before calculation to avoid autograd overhead.
+    * Registrations are stored in `self.registry` (str → callable).
+    * Duplicate metric names between built-ins and custom metrics raise an error
+      at construction time to prevent silent shadowing.
     """
 
-    builtin_metrics = {
+    builtin_metrics: Dict[str, Callable[[torch.Tensor, torch.Tensor], float]] = {
+        "diff": compute_max_abs_diff,
+        "max_abs_diff": compute_max_abs_diff,
         "peir": compute_peir,
     }
 
     def __init__(
         self,
-        metrics: List[str] = list(),
-        custom_metrics: Dict[str, Callable] = dict(),
+        custom_metrics: Optional[
+            Dict[str, Callable[[torch.Tensor, torch.Tensor], float]]
+        ] = None,
     ):
-        self.metrics: Dict[str, Callable] = dict()
-
-        for m in metrics:
-            if m in self.builtin_metrics:
-                self.metrics[m] = self.builtin_metrics[m]
-            else:
-                raise RuntimeError(f"Invalid metric: {m}")
-
-        duplicates = set(self.metrics).intersection(custom_metrics.keys())
-        if len(duplicates) != 0:
-            raise RuntimeError(f"There are duplicate metrics: {duplicates}")
-
-        self.metrics = self.metrics | custom_metrics
-
+        self.registry: Dict[str, Callable] = self.builtin_metrics.copy()
+        if custom_metrics:
+            dup = self.registry.keys() & custom_metrics.keys()
+            if dup:
+                raise RuntimeError(f"Duplicate metric names: {dup}")
+            assert custom_metrics is not None
+            self.registry.update(custom_metrics)  # type: ignore[arg-type]
+
+    # ----------------------------------------------------------------- #
+    # Public API                                                        #
+    # ----------------------------------------------------------------- #
     def compute(
-        self, output1: List[torch.Tensor], output2: List[torch.Tensor]
+        self,
+        base_outputs: List[torch.Tensor],
+        target_outputs: List[torch.Tensor],
+        metrics: Optional[List[str]] = None,
     ) -> Dict[str, List[Any]]:
         """
-        Compute both built-in metrics (if provided) and custom metrics.
+        Compute selected metrics for every (base, target) pair.
 
-        Returns
-        --------
-        Dict[str, Any]
-            A dictionary with metric names and their computed values.
+        Parameters
+        ----------
+        metrics
+            List of metric names to evaluate **this call**.
+            • None → evaluate *all* registered metrics.
         """
-        results: Dict[str, List[Any]] = dict()
-
-        # Compute built-in metrics
-        if self.metrics is not None:
-            for m in self.metrics:
-                results[m] = list()
-                for out1, out2 in zip(output1, output2):
-                    results[m].append(self.builtin_metrics[m](out1, out2))
-
+        sel = metrics or list(self.registry)
+        unknown = set(sel) - self.registry.keys()
+        if unknown:
+            raise RuntimeError(f"Unknown metric(s): {unknown}")
+
+        results: Dict[str, List[Any]] = {m: [] for m in sel}
+        for base, tgt in zip(base_outputs, target_outputs):
+            for m in sel:
+                results[m].append(self.registry[m](base, tgt))
         return results

{tico-0.1.0.dev250805.dist-info → tico-0.1.0.dev250806.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tico
-Version: 0.1.0.dev250805
+Version: 0.1.0.dev250806
 Summary: Convert exported Torch module to circle
 Home-page: UNKNOWN
 License: UNKNOWN
@@ -13,6 +13,7 @@ Requires-Dist: circle-schema
 Requires-Dist: packaging
 Requires-Dist: pyyaml
 Requires-Dist: torch
+Requires-Dist: tqdm
 
 # TICO
 

{tico-0.1.0.dev250805.dist-info → tico-0.1.0.dev250806.dist-info}/RECORD

@@ -1,4 +1,4 @@
-tico/__init__.py,sha256=xuyRHiRb0Ws9DndXxHXTm2LL5jqatnKIbWDMk0WegDQ,1883
+tico/__init__.py,sha256=ElClZCa-CW2dIDsOnUp1MCbDGvocTAgpyklSI4vEPiQ,1883
 tico/pt2_to_circle.py,sha256=gu3MD4Iqc0zMZcCZ2IT8oGbyj21CTSbT3Rgd9s2B_9A,2767
 tico/config/__init__.py,sha256=xZzCXjZ84qE-CsBi-dfaL05bqpQ3stKKfTXhnrJRyVs,142
 tico/config/base.py,sha256=q5xMqGxTUZs4mFqt5c7i_y9U00fYgdMGl9nUqIVMlCo,1248
@@ -13,7 +13,7 @@ tico/experimental/quantization/algorithm/__init__.py,sha256=IO6FP_xYbGy0dW0HL26G
 tico/experimental/quantization/algorithm/gptq/__init__.py,sha256=IO6FP_xYbGy0dW0HL26GXD3ouxARaxCK7bz9dn4blPQ,26
 tico/experimental/quantization/algorithm/gptq/gptq.py,sha256=Qn9b_2ki7B64DcVEY25NMkww3PdZ5EqYQQXfYhNDQ6I,5555
 tico/experimental/quantization/algorithm/gptq/quant.py,sha256=Rl4wAOCmlE0U09BtNCDbccaSNohRHCNLwFi3zCqZfNo,5127
-tico/experimental/quantization/algorithm/gptq/quantizer.py,sha256=KiaNcDkufbYPHdkkOGw9nAwLtk0yYwUDbyzFT3xRLOs,11066
+tico/experimental/quantization/algorithm/gptq/quantizer.py,sha256=5Sai9gD__6mt7rrUJutJ2L-BV_2biUXcvZC1H-a16Vc,11467
 tico/experimental/quantization/algorithm/gptq/utils.py,sha256=leGKayf-xbSjVwwAGTA5RsxUKrhDiklOQdlsLifjdrs,1811
 tico/experimental/quantization/algorithm/pt2e/__init__.py,sha256=IO6FP_xYbGy0dW0HL26GXD3ouxARaxCK7bz9dn4blPQ,26
 tico/experimental/quantization/algorithm/pt2e/quantizer.py,sha256=mdTvsG87bo8fu0GaWqSM8iBCs-4f4EfUlVtk-Ko6M34,2546
@@ -42,8 +42,8 @@ tico/experimental/quantization/algorithm/smoothquant/quantizer.py,sha256=rQMtnqM
 tico/experimental/quantization/algorithm/smoothquant/smooth_quant.py,sha256=O1h7IojcsJaprFvRM9tsAuR3q7vTjKKRi88jD-nH79Y,6175
 tico/experimental/quantization/evaluation/__init__.py,sha256=IO6FP_xYbGy0dW0HL26GXD3ouxARaxCK7bz9dn4blPQ,26
 tico/experimental/quantization/evaluation/backend.py,sha256=CZL9rZOA0t8cH7PHp6u9l7dGqWNvTj9bKOvwo0PVul0,692
-tico/experimental/quantization/evaluation/evaluate.py,sha256=cI9_4cbJKeYMmDt_PD85MHEseHGhwYRr3RHsKdix1FI,7988
-tico/experimental/quantization/evaluation/metric.py,sha256=Ae-vAkA8n6T9_aGv47rkz1MLCw9ji1oDm1Rdd_MIxNQ,3744
+tico/experimental/quantization/evaluation/evaluate.py,sha256=kfa_GvFaX6DoSTAmuCImMJqF2jgqtnor5UpC7wVmGPI,7877
+tico/experimental/quantization/evaluation/metric.py,sha256=ma8nrvt3jjKlE8d7iK3uABTkxOSrEDb8hHevdoc2pIw,5011
 tico/experimental/quantization/evaluation/utils.py,sha256=82RG_e5LuKfWo786wEZUVwXY93nNl901n04fB7D0Z6k,5909
 tico/experimental/quantization/evaluation/executor/__init__.py,sha256=IO6FP_xYbGy0dW0HL26GXD3ouxARaxCK7bz9dn4blPQ,26
 tico/experimental/quantization/evaluation/executor/backend_executor.py,sha256=3kLu3_rcsreA_NK42yRgRgubPtZmVp7QCRvaqLNw10E,1522
@@ -206,9 +206,9 @@ tico/utils/mx/__init__.py,sha256=IO6FP_xYbGy0dW0HL26GXD3ouxARaxCK7bz9dn4blPQ,26
 tico/utils/mx/elemwise_ops.py,sha256=V6glyAHsVR1joqpsgnNytatCD_ew92xNWZ19UFDoMTA,10281
 tico/utils/mx/formats.py,sha256=uzNWyu-1onUlwQfX5cZ6fZSUfHMRqorper7_T1k3jfk,3404
 tico/utils/mx/mx_ops.py,sha256=RcfUTYVi-wilGB2sC35OeARdwDqnixv7dG5iyZ-fQT8,8555
-tico-0.1.0.dev250805.dist-info/LICENSE,sha256=kp4JLII7bzRhPb0CPD5XTDZMh22BQ7h3k3B7t8TiSbw,12644
-tico-0.1.0.dev250805.dist-info/METADATA,sha256=zZJaVgP4R1iwsYoWXZi7za9_q6pPGoN5IL7ybesoB38,8430
-tico-0.1.0.dev250805.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
-tico-0.1.0.dev250805.dist-info/entry_points.txt,sha256=kBKYSS_IYrSXmUYevmmepqIVPScq5vF8ulQRu3I_Zf0,59
-tico-0.1.0.dev250805.dist-info/top_level.txt,sha256=oqs7UPoNSKZEwqsX8B-KAWdQwfAa7i60pbxW_Jk7P3w,5
-tico-0.1.0.dev250805.dist-info/RECORD,,
+tico-0.1.0.dev250806.dist-info/LICENSE,sha256=kp4JLII7bzRhPb0CPD5XTDZMh22BQ7h3k3B7t8TiSbw,12644
+tico-0.1.0.dev250806.dist-info/METADATA,sha256=_lGucGVdG9FrBy8SvOy1BxwlH3mxLLk5HpZWIUPdA48,8450
+tico-0.1.0.dev250806.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
+tico-0.1.0.dev250806.dist-info/entry_points.txt,sha256=kBKYSS_IYrSXmUYevmmepqIVPScq5vF8ulQRu3I_Zf0,59
+tico-0.1.0.dev250806.dist-info/top_level.txt,sha256=oqs7UPoNSKZEwqsX8B-KAWdQwfAa7i60pbxW_Jk7P3w,5
+tico-0.1.0.dev250806.dist-info/RECORD,,