quantbenchx 0.3.0__py3-none-any.whl

quantbenchx/matrix.py

@@ -0,0 +1,289 @@
+"""Quantization format comparison matrix."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Dict, List, Optional
+
+
+@dataclass
+class QuantFormatSpec:
+    """Specification of a quantization format."""
+
+    name: str
+    bits: int
+    block_size: Optional[int] = None
+    symmetric: bool = True
+    description: str = ""
+
+    @property
+    def bytes_per_weight(self) -> float:
+        """Effective bytes per weight (including overhead for block formats)."""
+        base = self.bits / 8.0
+        if self.block_size is not None and self.block_size > 0:
+            # Block formats store a scale per block
+            overhead = 2.0 / self.block_size  # 2 bytes for FP16 scale
+            if not self.symmetric:
+                overhead += 2.0 / self.block_size  # zero-point
+            return base + overhead
+        return base
+
+
+@dataclass
+class FormatComparison:
+    """Result of comparing two quantization formats."""
+
+    format_a: str
+    format_b: str
+    size_ratio: float
+    accuracy_delta: float
+    speed_ratio: float
+
+
+KNOWN_FORMATS: Dict[str, QuantFormatSpec] = {
+    "FP16": QuantFormatSpec(
+        name="FP16", bits=16, block_size=None, symmetric=True,
+        description="IEEE 754 half-precision floating point",
+    ),
+    "BF16": QuantFormatSpec(
+        name="BF16", bits=16, block_size=None, symmetric=True,
+        description="Brain floating point 16-bit",
+    ),
+    "INT8": QuantFormatSpec(
+        name="INT8", bits=8, block_size=None, symmetric=True,
+        description="8-bit integer quantization",
+    ),
+    "INT4": QuantFormatSpec(
+        name="INT4", bits=4, block_size=128, symmetric=True,
+        description="4-bit integer quantization with 128-element blocks",
+    ),
+    "GPTQ": QuantFormatSpec(
+        name="GPTQ", bits=4, block_size=128, symmetric=False,
+        description="GPTQ 4-bit with group size 128",
+    ),
+    "AWQ": QuantFormatSpec(
+        name="AWQ", bits=4, block_size=128, symmetric=False,
+        description="Activation-aware Weight Quantization 4-bit",
+    ),
+    "GGUF_Q4_0": QuantFormatSpec(
+        name="GGUF_Q4_0", bits=4, block_size=32, symmetric=True,
+        description="GGUF Q4_0: 4-bit quantization, 32-element blocks",
+    ),
+    "GGUF_Q5_1": QuantFormatSpec(
+        name="GGUF_Q5_1", bits=5, block_size=32, symmetric=False,
+        description="GGUF Q5_1: 5-bit quantization with zero-point, 32-element blocks",
+    ),
+    "NF4": QuantFormatSpec(
+        name="NF4", bits=4, block_size=64, symmetric=True,
+        description="NormalFloat 4-bit (QLoRA)",
+    ),
+}
+
+# Heuristic accuracy retention scores per bit-width (relative to FP16 = 1.0).
+# Higher is better. Accounts for typical perplexity retention on LLMs.
+_ACCURACY_BY_BITS: Dict[int, float] = {
+    16: 1.0,
+    8: 0.995,
+    5: 0.98,
+    4: 0.96,
+    3: 0.92,
+    2: 0.82,
+}
+
+# Heuristic inference speed multiplier relative to FP16 = 1.0.
+# Lower bit counts run faster due to reduced memory bandwidth.
+_SPEED_BY_BITS: Dict[int, float] = {
+    16: 1.0,
+    8: 1.6,
+    5: 2.0,
+    4: 2.3,
+    3: 2.5,
+    2: 2.7,
+}
+
+
+def _accuracy_estimate(fmt: QuantFormatSpec) -> float:
+    """Heuristic accuracy retention for a format (0-1 scale, FP16=1.0)."""
+    bits = fmt.bits
+    if bits in _ACCURACY_BY_BITS:
+        score = _ACCURACY_BY_BITS[bits]
+    else:
+        # Linear interpolation between known points
+        lower = max(b for b in _ACCURACY_BY_BITS if b <= bits)
+        upper = min(b for b in _ACCURACY_BY_BITS if b >= bits)
+        if lower == upper:
+            score = _ACCURACY_BY_BITS[lower]
+        else:
+            t = (bits - lower) / (upper - lower)
+            score = _ACCURACY_BY_BITS[lower] + t * (
+                _ACCURACY_BY_BITS[upper] - _ACCURACY_BY_BITS[lower]
+            )
+    # Block-quantized formats with asymmetric quantization are slightly more accurate
+    if fmt.block_size is not None and not fmt.symmetric:
+        score = min(1.0, score + 0.005)
+    return score
+
+
+def _speed_estimate(fmt: QuantFormatSpec) -> float:
+    """Heuristic speed multiplier (relative to FP16 = 1.0x)."""
+    bits = fmt.bits
+    if bits in _SPEED_BY_BITS:
+        return _SPEED_BY_BITS[bits]
+    lower = max(b for b in _SPEED_BY_BITS if b <= bits)
+    upper = min(b for b in _SPEED_BY_BITS if b >= bits)
+    if lower == upper:
+        return _SPEED_BY_BITS[lower]
+    t = (bits - lower) / (upper - lower)
+    return _SPEED_BY_BITS[lower] + t * (
+        _SPEED_BY_BITS[upper] - _SPEED_BY_BITS[lower]
+    )
+
+
+def _model_size_gb(fmt: QuantFormatSpec, base_size_gb: float) -> float:
+    """Estimate model size in GB for a given format, relative to FP16 baseline."""
+    return base_size_gb * fmt.bytes_per_weight / 2.0  # FP16 = 2 bytes/weight
+
+
+class ComparisonMatrix:
+    """N×N quantization format comparison matrix."""
+
+    def __init__(self, formats: Optional[List[QuantFormatSpec]] = None) -> None:
+        if formats is not None:
+            self._formats: Dict[str, QuantFormatSpec] = {f.name: f for f in formats}
+        else:
+            self._formats = dict(KNOWN_FORMATS)
+
+    @property
+    def formats(self) -> Dict[str, QuantFormatSpec]:
+        """Return the registered formats."""
+        return dict(self._formats)
+
+    def add_format(self, fmt: QuantFormatSpec) -> None:
+        """Register a new quantization format."""
+        self._formats[fmt.name] = fmt
+
+    def compare_pair(
+        self,
+        fmt_a: str,
+        fmt_b: str,
+        model_size_gb: float = 7.0,
+    ) -> FormatComparison:
+        """Compare two formats and return size/accuracy/speed ratios.
+
+        Ratios are expressed as A / B.  A size_ratio < 1 means A is smaller.
+        An accuracy_delta > 0 means A is more accurate.
+        A speed_ratio > 1 means A is faster.
+        """
+        if fmt_a not in self._formats:
+            raise KeyError(f"Unknown format: {fmt_a!r}")
+        if fmt_b not in self._formats:
+            raise KeyError(f"Unknown format: {fmt_b!r}")
+
+        fa = self._formats[fmt_a]
+        fb = self._formats[fmt_b]
+
+        size_a = _model_size_gb(fa, model_size_gb)
+        size_b = _model_size_gb(fb, model_size_gb)
+        size_ratio = size_a / size_b if size_b > 0 else float("inf")
+
+        acc_a = _accuracy_estimate(fa)
+        acc_b = _accuracy_estimate(fb)
+
+        speed_a = _speed_estimate(fa)
+        speed_b = _speed_estimate(fb)
+        speed_ratio = speed_a / speed_b if speed_b > 0 else float("inf")
+
+        return FormatComparison(
+            format_a=fmt_a,
+            format_b=fmt_b,
+            size_ratio=round(size_ratio, 4),
+            accuracy_delta=round(acc_a - acc_b, 6),
+            speed_ratio=round(speed_ratio, 4),
+        )
+
+    def full_matrix(self) -> List[List[FormatComparison]]:
+        """Generate an N×N comparison matrix."""
+        names = sorted(self._formats)
+        return [
+            [self.compare_pair(a, b) for b in names]
+            for a in names
+        ]
+
+    def rank_by(self, criterion: str = "size") -> List[QuantFormatSpec]:
+        """Rank formats by a criterion: 'size', 'speed', or 'accuracy'."""
+        fmts = list(self._formats.values())
+        if criterion == "size":
+            fmts.sort(key=lambda f: f.bytes_per_weight)
+        elif criterion == "speed":
+            fmts.sort(key=lambda f: _speed_estimate(f), reverse=True)
+        elif criterion == "accuracy":
+            fmts.sort(key=lambda f: _accuracy_estimate(f), reverse=True)
+        else:
+            raise ValueError(
+                f"Unknown criterion: {criterion!r}. Choose 'size', 'speed', or 'accuracy'."
+            )
+        return fmts
+
+    def recommend(self, constraints: Dict[str, Any]) -> List[QuantFormatSpec]:
+        """Recommend formats matching constraints.
+
+        Supported constraint keys:
+          max_bits (int): maximum bit-width
+          min_accuracy (float): minimum accuracy retention (0-1), FP16=1.0
+          min_speed (float): minimum speed multiplier (FP16=1.0)
+          max_size_gb (float): maximum model size in GB (requires base_size_gb)
+          base_size_gb (float): FP16 model size for size calculations (default 7.0)
+        """
+        max_bits = constraints.get("max_bits", 32)
+        min_accuracy = constraints.get("min_accuracy", 0.0)
+        min_speed = constraints.get("min_speed", 0.0)
+        max_size_gb = constraints.get("max_size_gb", float("inf"))
+        base_size_gb = constraints.get("base_size_gb", 7.0)
+
+        results = []
+        for fmt in self._formats.values():
+            if fmt.bits > max_bits:
+                continue
+            if _accuracy_estimate(fmt) < min_accuracy:
+                continue
+            if _speed_estimate(fmt) < min_speed:
+                continue
+            if _model_size_gb(fmt, base_size_gb) > max_size_gb:
+                continue
+            results.append(fmt)
+
+        # Sort by accuracy descending, then speed descending
+        results.sort(
+            key=lambda f: (_accuracy_estimate(f), _speed_estimate(f)),
+            reverse=True,
+        )
+        return results
+
+
+def format_comparison_table(matrix: List[List[FormatComparison]]) -> str:
+    """Render an N×N comparison matrix as an ASCII table.
+
+    Each cell shows the size ratio of row-format vs column-format.
+    """
+    if not matrix or not matrix[0]:
+        return "(empty matrix)"
+
+    names = [row[0].format_a for row in matrix]
+    col_width = max(len(n) for n in names) + 2
+    cell_width = 8
+
+    header = " " * col_width + "".join(n.center(cell_width) for n in names)
+    separator = "-" * len(header)
+
+    lines = [header, separator]
+    for row in matrix:
+        label = row[0].format_a.ljust(col_width)
+        cells = []
+        for comp in row:
+            if comp.format_a == comp.format_b:
+                cells.append("  ---  ".center(cell_width))
+            else:
+                cells.append(f"{comp.size_ratio:.2f}x".center(cell_width))
+        lines.append(label + "".join(cells))
+
+    return "\n".join(lines)

quantbenchx/perplexity.py

@@ -0,0 +1,168 @@
+"""Perplexity-based quality scoring for quantized models."""
+
+from __future__ import annotations
+
+import math
+from dataclasses import dataclass, field
+from typing import Dict, Sequence
+
+from quantbenchx._types import ModelProfile
+
+
+@dataclass
+class PerplexityScore:
+    """Result of a perplexity computation."""
+
+    value: float
+    num_tokens: int
+    log_likelihood: float
+    normalized: bool
+
+
+@dataclass
+class PerplexityDelta:
+    """Estimated perplexity change between original and quantized models."""
+
+    original_bits: float
+    quantized_bits: float
+    estimated_ppl_increase_pct: float
+    quality_retention: float  # 0-1
+    per_layer_impact: Dict[str, float] = field(default_factory=dict)
+
+
+# ── Heuristic constants ───────────────────────────────────────────────
+
+# Attention layers are ~2× more sensitive than FFN layers.
+_ATTN_SENSITIVITY = 2.0
+_FFN_SENSITIVITY = 1.0
+
+# Empirical base: each halving of bpw roughly doubles the ppl penalty.
+_BASE_PPL_RATE = 0.04  # ppl % increase per bit of precision lost
+
+
+def _layer_sensitivity(name: str) -> float:
+    """Return a sensitivity multiplier based on the layer name."""
+    low = name.lower()
+    if any(kw in low for kw in ("attn", "attention", "self_attn", "q_proj", "k_proj", "v_proj", "o_proj")):
+        return _ATTN_SENSITIVITY
+    if any(kw in low for kw in ("embed", "lm_head", "output", "norm")):
+        return _ATTN_SENSITIVITY * 1.5  # most sensitive
+    return _FFN_SENSITIVITY
+
+
+def _bits_for_profile(profile: ModelProfile) -> float:
+    bpw = profile.quant.avg_bits_per_weight
+    return bpw if bpw > 0 else 16.0
+
+
+def _per_layer_impact(profile: ModelProfile, ref_bits: float) -> Dict[str, float]:
+    impacts: Dict[str, float] = {}
+    for layer in profile.layers:
+        lbpw = layer.avg_bits_per_weight
+        if lbpw <= 0:
+            lbpw = ref_bits
+        bit_drop = max(0.0, ref_bits - lbpw)
+        sens = _layer_sensitivity(layer.name)
+        impacts[layer.name] = round(bit_drop * sens * _BASE_PPL_RATE, 6)
+    return impacts
+
+
+def _ppl_increase_pct(original_bits: float, quantized_bits: float) -> float:
+    """Estimate perplexity % increase from precision loss.
+
+    Uses an exponential heuristic: ppl_pct ≈ base_rate * 2^(bit_drop / scale) - base_rate
+    so small drops are nearly linear while large drops blow up.
+    """
+    bit_drop = max(0.0, original_bits - quantized_bits)
+    if bit_drop == 0:
+        return 0.0
+    # scale factor: a 4-bit drop ≈ 1 doubling
+    return _BASE_PPL_RATE * (2.0 ** (bit_drop / 4.0) - 1.0) * 100.0
+
+
+# ── Public API ────────────────────────────────────────────────────────
+
+
+def estimate_perplexity_delta(
+    profile_original: ModelProfile,
+    profile_quantized: ModelProfile,
+) -> PerplexityDelta:
+    """Estimate the perplexity delta between an original and quantized model.
+
+    Uses a heuristic: lower bit-widths increase perplexity roughly
+    exponentially, and attention layers are more sensitive than FFN layers.
+    """
+    orig_bits = _bits_for_profile(profile_original)
+    quant_bits = _bits_for_profile(profile_quantized)
+
+    ppl_pct = _ppl_increase_pct(orig_bits, quant_bits)
+    retention = max(0.0, min(1.0, 1.0 - ppl_pct / 100.0))
+
+    impacts = _per_layer_impact(profile_quantized, orig_bits)
+
+    return PerplexityDelta(
+        original_bits=round(orig_bits, 4),
+        quantized_bits=round(quant_bits, 4),
+        estimated_ppl_increase_pct=round(ppl_pct, 4),
+        quality_retention=round(retention, 6),
+        per_layer_impact=impacts,
+    )
+
+
+def quality_score(profile: ModelProfile, reference_bits: float = 16.0) -> float:
+    """Score model quality relative to a reference precision.
+
+    Returns a float in [0, 1].  1.0 means no quality loss (same precision
+    as reference), 0.0 means severe degradation.
+    """
+    bpw = _bits_for_profile(profile)
+    if bpw >= reference_bits:
+        return 1.0
+    bit_drop = reference_bits - bpw
+    # Exponential decay: exp(-k * bit_drop^1.5) gives a smooth curve
+    score = math.exp(-0.06 * (bit_drop ** 1.5))
+    return round(max(0.0, min(1.0, score)), 6)
+
+
+def perplexity_from_logprobs(logprobs: Sequence[float]) -> PerplexityScore:
+    """Compute perplexity from a list of token log-probabilities.
+
+    Standard formula:  ppl = exp( -1/N * Σ log p_i )
+    """
+    n = len(logprobs)
+    if n == 0:
+        return PerplexityScore(value=float("inf"), num_tokens=0, log_likelihood=0.0, normalized=True)
+
+    total_ll = sum(logprobs)
+    avg_neg_ll = -total_ll / n
+    ppl = math.exp(avg_neg_ll)
+
+    return PerplexityScore(
+        value=ppl,
+        num_tokens=n,
+        log_likelihood=total_ll,
+        normalized=True,
+    )
+
+
+def format_quality_report(delta: PerplexityDelta) -> str:
+    """Format a PerplexityDelta as a human-readable report."""
+    lines = [
+        "Perplexity Quality Report",
+        "=" * 40,
+        f"Original precision : {delta.original_bits:.2f} bits",
+        f"Quantized precision: {delta.quantized_bits:.2f} bits",
+        f"Est. PPL increase  : {delta.estimated_ppl_increase_pct:.2f}%",
+        f"Quality retention  : {delta.quality_retention:.4f}",
+    ]
+
+    if delta.per_layer_impact:
+        lines.append("")
+        lines.append("Per-layer impact:")
+        sorted_layers = sorted(delta.per_layer_impact.items(), key=lambda x: x[1], reverse=True)
+        for name, impact in sorted_layers[:10]:
+            lines.append(f"  {name:40s} {impact:.6f}")
+        if len(sorted_layers) > 10:
+            lines.append(f"  ... and {len(sorted_layers) - 10} more layers")
+
+    return "\n".join(lines)

quantbenchx/predict.py

@@ -0,0 +1,125 @@
+"""Quality prediction — estimate perplexity delta and quality score from quantization params."""
+
+from __future__ import annotations
+
+import math
+from typing import List
+
+from quantbenchx._types import ModelProfile, QualityEstimate
+
+# Empirical perplexity delta estimates per bits-per-weight
+# Based on published quantization benchmarks (llama.cpp, GPTQ papers)
+_PERPLEXITY_CURVE = {
+    16.0: 0.0,
+    8.5: 0.01,
+    6.5: 0.03,
+    5.5: 0.05,
+    4.85: 0.08,
+    4.5: 0.12,
+    4.0: 0.18,
+    3.9: 0.22,
+    3.5: 0.35,
+    3.35: 0.45,
+    3.0: 0.65,
+    2.5: 1.0,
+    2.0: 1.8,
+    1.5: 3.5,
+}
+
+
+def _interpolate_perplexity(bpw: float) -> float:
+    """Interpolate expected perplexity delta from bits-per-weight."""
+    points = sorted(_PERPLEXITY_CURVE.items(), reverse=True)
+
+    if bpw >= points[0][0]:
+        return points[0][1]
+    if bpw <= points[-1][0]:
+        return points[-1][1]
+
+    for i in range(len(points) - 1):
+        x1, y1 = points[i]
+        x2, y2 = points[i + 1]
+        if x2 <= bpw <= x1:
+            t = (bpw - x2) / (x1 - x2) if x1 != x2 else 0.0
+            return y1 * t + y2 * (1 - t)
+
+    return 0.5  # fallback
+
+
+def estimate_quality(profile: ModelProfile) -> QualityEstimate:
+    """Estimate quantization quality from a model profile.
+
+    Returns a QualityEstimate with predicted perplexity delta,
+    quality score, risk level, and recommendations.
+    """
+    bpw = profile.quant.avg_bits_per_weight
+    if bpw == 0:
+        bpw = 16.0  # assume full precision if unknown
+
+    ppl_delta = _interpolate_perplexity(bpw)
+
+    # Quality score: 1.0 = perfect (no loss), 0.0 = terrible
+    # Exponential decay from bpw
+    quality = math.exp(-0.15 * max(0, 16.0 - bpw))
+    quality = max(0.0, min(1.0, quality))
+
+    # Risk level
+    if ppl_delta < 0.05:
+        risk = "low"
+    elif ppl_delta < 0.15:
+        risk = "medium"
+    elif ppl_delta < 0.5:
+        risk = "high"
+    else:
+        risk = "critical"
+
+    # Find sensitive layers
+    sensitive: List[str] = []
+    for layer in profile.layers:
+        name_lower = layer.name.lower()
+        if any(kw in name_lower for kw in ("embed", "lm_head", "output", "norm")):
+            if layer.avg_bits_per_weight < 6.0:
+                sensitive.append(layer.name)
+
+    # Recommendations
+    recs: List[str] = []
+    if bpw < 3.0:
+        recs.append("Very aggressive quantization — expect significant quality loss")
+        recs.append("Consider Q4_K_M or higher for production use")
+    elif bpw < 4.0:
+        recs.append("Aggressive quantization — test on your specific workload")
+    elif bpw < 5.0:
+        recs.append("Good balance of size and quality for most use cases")
+
+    if sensitive:
+        recs.append(f"Sensitive layers at low precision: {', '.join(sensitive[:5])}")
+        recs.append("Consider mixed-quant with higher precision for embed/output layers")
+
+    if profile.quant.n_full_precision_layers == 0 and len(profile.tensors) > 10:
+        recs.append("No full-precision layers detected — norm layers may benefit from FP16")
+
+    total_params = profile.total_params
+    if total_params > 0:
+        param_b = total_params / 1e9
+        size_gb = profile.size_gb
+        if param_b > 0:
+            recs.append(f"Model: ~{param_b:.1f}B params, {size_gb:.1f} GB at {bpw:.1f} bpw")
+
+    return QualityEstimate(
+        model_name=profile.name,
+        method=profile.quant.method.value,
+        avg_bits_per_weight=round(bpw, 2),
+        estimated_perplexity_delta=round(ppl_delta, 4),
+        quality_score=round(quality, 4),
+        risk_level=risk,
+        sensitive_layers=sensitive,
+        recommendations=recs,
+    )
+
+
+def perplexity_delta(bpw: float) -> float:
+    """Estimate perplexity increase for a given bits-per-weight.
+
+    Based on published quantization benchmarks.
+    """
+    return round(_interpolate_perplexity(bpw), 4)