llm-dequant 0.1.0__py3-none-any.whl

llm_dequant/__init__.py

@@ -0,0 +1,4 @@
+# SPDX-License-Identifier: Apache-2.0
+"""llm-dequant: dequantize compressed-tensors LLM checkpoints to full-precision safetensors."""
+
+__version__ = "0.1.0"

llm_dequant/cli.py

@@ -0,0 +1,69 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Command-line interface for llm-dequant."""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+from . import __version__
+from .model import DEFAULT_SHARD_BYTES, DTYPES, convert, inspect_checkpoint, verify
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="llm-dequant",
+        description=(
+            "Dequantize compressed-tensors NVFP4 checkpoints to dense "
+            "safetensors (streaming, constant memory)."
+        ),
+    )
+    parser.add_argument("--version", action="version", version=__version__)
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    p_convert = sub.add_parser("convert", help="dequantize a checkpoint")
+    p_convert.add_argument("src", type=Path, help="compressed checkpoint directory")
+    p_convert.add_argument("dst", type=Path, help="output directory")
+    p_convert.add_argument(
+        "--dtype", choices=sorted(DTYPES), default="bf16",
+        help="output dtype (default: bf16)",
+    )
+    p_convert.add_argument(
+        "--shard-gb", type=float, default=DEFAULT_SHARD_BYTES / 1024**3,
+        help="target output shard size in GiB (default: 8)",
+    )
+
+    p_verify = sub.add_parser(
+        "verify",
+        help="round-trip check: dequantize + repack sampled modules, compare bytes",
+    )
+    p_verify.add_argument("src", type=Path, help="compressed checkpoint directory")
+    p_verify.add_argument("--sample", type=int, default=8,
+                          help="number of random packed modules to check (default: 8)")
+    p_verify.add_argument("--seed", type=int, default=0, help="sampling seed")
+
+    p_inspect = sub.add_parser("inspect", help="show checkpoint quantization summary")
+    p_inspect.add_argument("src", type=Path, help="checkpoint directory")
+
+    args = parser.parse_args(argv)
+
+    try:
+        if args.command == "convert":
+            if args.shard_gb <= 0:
+                parser.error("--shard-gb must be positive")
+            convert(args.src, args.dst, dtype=args.dtype,
+                    shard_bytes=int(args.shard_gb * 1024**3))
+        elif args.command == "verify":
+            result = verify(args.src, sample=args.sample, seed=args.seed)
+            return 0 if not result["failures"] else 1
+        elif args.command == "inspect":
+            inspect_checkpoint(args.src)
+    except (FileNotFoundError, ValueError) as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return 2
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

llm_dequant/model.py

@@ -0,0 +1,376 @@
+# SPDX-License-Identifier: Apache-2.0
+"""
+Checkpoint-level streaming conversion: compressed-tensors NVFP4 -> dense safetensors.
+
+Handles HuggingFace sharded checkpoints (model.safetensors.index.json) and
+single-file checkpoints (model.safetensors). Memory use is bounded by the
+largest single module plus one output shard buffer, regardless of model size.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import shutil
+from dataclasses import dataclass, field
+from pathlib import Path
+
+import torch
+from safetensors import safe_open
+from safetensors.torch import save_file
+
+from .nvfp4 import GROUP_SIZE, dequantize_nvfp4, pack_fp4, unpack_fp4
+
+__all__ = ["CheckpointReader", "convert", "verify", "inspect_checkpoint"]
+
+SUPPORTED_FORMATS = {"nvfp4-pack-quantized"}
+
+# local (per-module) tensor names that belong to a packed module
+PACKED_PARAM_NAMES = frozenset(
+    {
+        "weight_packed",
+        "weight_scale",
+        "weight_global_scale",
+        "weight_zero_point",
+        "input_global_scale",
+    }
+)
+
+PASSTHROUGH_FILE_SUFFIXES = (".json", ".jinja", ".txt", ".model", ".py", ".md")
+PASSTHROUGH_FILE_NAMES = ("LICENSE", "LICENSE.txt", "NOTICE")
+DEFAULT_SHARD_BYTES = 8 * 1024**3
+
+DTYPES = {
+    "bf16": torch.bfloat16,
+    "fp16": torch.float16,
+    "fp32": torch.float32,
+}
+
+
+@dataclass
+class Module:
+    """One logical module: either a packed quantized weight or a passthrough tensor."""
+
+    name: str                       # module prefix (packed) or full tensor name
+    tensors: dict[str, str] = field(default_factory=dict)  # local name -> full name
+
+    @property
+    def is_packed(self) -> bool:
+        return "weight_packed" in self.tensors
+
+
+class CheckpointReader:
+    """Lazily reads tensors from a sharded or single-file safetensors checkpoint."""
+
+    def __init__(self, src: Path):
+        self.src = Path(src)
+        index_path = self.src / "model.safetensors.index.json"
+        single_path = self.src / "model.safetensors"
+        if index_path.exists():
+            index = json.loads(index_path.read_text())
+            self.weight_map: dict[str, str] = index["weight_map"]
+            for fname in set(self.weight_map.values()):
+                self._check_shard_name(fname)
+        elif single_path.exists():
+            with safe_open(single_path, framework="pt", device="cpu") as f:
+                self.weight_map = {k: "model.safetensors" for k in f.keys()}
+        else:
+            raise FileNotFoundError(
+                f"no model.safetensors.index.json or model.safetensors in {self.src}"
+            )
+        self._handles: dict[str, object] = {}
+
+    def _check_shard_name(self, fname: str) -> None:
+        """index.json is untrusted input: shard names must be plain filenames
+        inside the checkpoint directory (no separators, no '..', not absolute)."""
+        p = Path(fname)
+        if p.is_absolute() or p.name != fname or fname in ("", ".", ".."):
+            raise ValueError(
+                f"refusing suspicious shard filename in index.json: {fname!r}"
+            )
+
+    def tensor(self, full_name: str) -> torch.Tensor:
+        fname = self.weight_map[full_name]
+        if fname not in self._handles:
+            self._handles[fname] = safe_open(
+                self.src / fname, framework="pt", device="cpu"
+            )
+        return self._handles[fname].get_tensor(full_name)
+
+    def modules(self) -> list[Module]:
+        """Group tensor names into modules, preserving first-seen order."""
+        by_key: dict[str, Module] = {}
+        order: list[str] = []
+        for full in self.weight_map:
+            prefix, _, local = full.rpartition(".")
+            if prefix and local in PACKED_PARAM_NAMES:
+                key = prefix
+            else:
+                key, local = full, ""
+            if key not in by_key:
+                by_key[key] = Module(name=key)
+                order.append(key)
+            by_key[key].tensors[local] = full
+        return [by_key[k] for k in order]
+
+    def load_config(self) -> dict:
+        cfg_path = self.src / "config.json"
+        if not cfg_path.exists():
+            raise FileNotFoundError(f"missing config.json in {self.src}")
+        return json.loads(cfg_path.read_text())
+
+    def detect_format(self) -> str:
+        qc = self.load_config().get("quantization_config")
+        if qc is None:
+            raise ValueError("config.json has no quantization_config — nothing to dequantize")
+        fmt = qc.get("format")
+        if fmt not in SUPPORTED_FORMATS:
+            raise ValueError(
+                f"unsupported compression format {fmt!r}; supported: {sorted(SUPPORTED_FORMATS)}"
+            )
+        groups = qc.get("config_groups", {})
+        for gname, group in groups.items():
+            w = group.get("weights") or {}
+            if w.get("num_bits") != 4 or w.get("type") != "float":
+                raise ValueError(f"{gname}: expected 4-bit float weights, got {w}")
+            if w.get("group_size") != GROUP_SIZE:
+                raise ValueError(
+                    f"{gname}: group_size {w.get('group_size')} != {GROUP_SIZE}"
+                )
+            if not w.get("symmetric", True):
+                raise ValueError("asymmetric NVFP4 is not supported")
+        return fmt
+
+
+def _dequant_module(reader: CheckpointReader, mod: Module, dtype: torch.dtype) -> torch.Tensor:
+    if "weight_zero_point" in mod.tensors:
+        raise ValueError(f"{mod.name}: asymmetric NVFP4 (zero point present) not supported")
+    packed = reader.tensor(mod.tensors["weight_packed"])
+    scale = reader.tensor(mod.tensors["weight_scale"])
+    gs_name = mod.tensors.get("weight_global_scale")
+    global_scale = reader.tensor(gs_name) if gs_name else None
+    return dequantize_nvfp4(packed, scale, global_scale, dtype=dtype)
+
+
+class _ShardWriter:
+    """Accumulates tensors and writes ~shard_bytes safetensors shards."""
+
+    def __init__(self, dst: Path, shard_bytes: int):
+        self.dst = dst
+        self.shard_bytes = shard_bytes
+        self.buffer: dict[str, torch.Tensor] = {}
+        self.buffered = 0
+        self.shards: list[list[str]] = []  # tensor names per shard
+        self._seen: set[str] = set()
+
+    def add(self, name: str, tensor: torch.Tensor) -> None:
+        if name in self._seen:
+            raise ValueError(
+                f"duplicate output tensor name {name!r} — the checkpoint "
+                "contains both a packed and a dense tensor mapping to the "
+                "same dequantized name; refusing to silently overwrite"
+            )
+        self._seen.add(name)
+        self.buffer[name] = tensor
+        self.buffered += tensor.numel() * tensor.element_size()
+        if self.buffered >= self.shard_bytes:
+            self.flush()
+
+    def flush(self) -> None:
+        if not self.buffer:
+            return
+        idx = len(self.shards) + 1
+        save_file(self.buffer, str(self.dst / f"model-{idx:05d}.safetensors"),
+                  metadata={"format": "pt"})
+        self.shards.append(list(self.buffer.keys()))
+        self.buffer = {}
+        self.buffered = 0
+
+    def finalize(self) -> dict[str, str]:
+        """Rename shards to canonical -of- form; return tensor -> shard map."""
+        self.flush()
+        total = len(self.shards)
+        weight_map: dict[str, str] = {}
+        for i, names in enumerate(self.shards, 1):
+            src = self.dst / f"model-{i:05d}.safetensors"
+            canon = f"model-{i:05d}-of-{total:05d}.safetensors"
+            os.replace(src, self.dst / canon)
+            for n in names:
+                weight_map[n] = canon
+        return weight_map
+
+
+def convert(
+    src: Path,
+    dst: Path,
+    dtype: str = "bf16",
+    shard_bytes: int = DEFAULT_SHARD_BYTES,
+    progress_every: int = 2000,
+    log=print,
+) -> dict:
+    """
+    Convert a compressed NVFP4 checkpoint at ``src`` into a dense checkpoint
+    at ``dst``. Returns a summary dict.
+    """
+    src, dst = Path(src), Path(dst)
+    if dst.resolve() == src.resolve():
+        raise ValueError("src and dst must differ")
+    if dtype not in DTYPES:
+        raise ValueError(f"unknown dtype {dtype!r}; choose from {sorted(DTYPES)}")
+    torch_dtype = DTYPES[dtype]
+
+    reader = CheckpointReader(src)
+    fmt = reader.detect_format()
+    modules = reader.modules()
+    n_packed = sum(1 for m in modules if m.is_packed)
+    log(f"format={fmt} modules={len(modules)} packed={n_packed} -> {dtype}")
+    if n_packed == 0:
+        raise ValueError("no packed modules found; is this really a compressed checkpoint?")
+
+    dst.mkdir(parents=True, exist_ok=True)
+
+    # passthrough sidecar files (tokenizer, templates, ...) except the ones we rewrite
+    for f in sorted(src.iterdir()):
+        if f.name in ("config.json", "model.safetensors.index.json"):
+            continue
+        if f.name.endswith(".safetensors"):
+            continue
+        if f.name.endswith(PASSTHROUGH_FILE_SUFFIXES) or f.name in PASSTHROUGH_FILE_NAMES:
+            shutil.copy2(f, dst / f.name)
+
+    cfg = reader.load_config()
+    cfg.pop("quantization_config", None)
+    cfg["torch_dtype"] = {"bf16": "bfloat16", "fp16": "float16", "fp32": "float32"}[dtype]
+    (dst / "config.json").write_text(json.dumps(cfg, indent=2))
+
+    writer = _ShardWriter(dst, shard_bytes)
+    done = 0
+    for mod in modules:
+        if mod.is_packed:
+            writer.add(mod.name + ".weight", _dequant_module(reader, mod, torch_dtype))
+        else:
+            # single passthrough tensor; key "" holds the full name
+            full = mod.tensors.get("") or mod.tensors.get("weight")
+            if full is None:
+                raise ValueError(
+                    f"module {mod.name!r} has quantization params "
+                    f"{sorted(mod.tensors)} but no weight_packed — "
+                    "corrupt or unsupported checkpoint layout"
+                )
+            writer.add(full, reader.tensor(full))
+        done += 1
+        if progress_every and done % progress_every == 0:
+            log(f"  {done}/{len(modules)} modules")
+
+    weight_map = writer.finalize()
+    total_bytes = sum(
+        (dst / f).stat().st_size for f in os.listdir(dst) if f.endswith(".safetensors")
+    )
+    (dst / "model.safetensors.index.json").write_text(
+        json.dumps(
+            {"metadata": {"total_size": total_bytes}, "weight_map": weight_map},
+            indent=2,
+        )
+    )
+    summary = {
+        "modules": len(modules),
+        "packed": n_packed,
+        "tensors": len(weight_map),
+        "shards": len(writer.shards),
+        "bytes": total_bytes,
+    }
+    log(f"DONE shards={summary['shards']} tensors={summary['tensors']} "
+        f"size={total_bytes / 1e9:.1f}GB")
+    return summary
+
+
+def verify(src: Path, sample: int = 8, seed: int = 0, log=print) -> dict:
+    """
+    Round-trip check on ``sample`` packed modules of the SOURCE checkpoint:
+    dequantize with this library, re-quantize with nearest-E2M1 packing, and
+    require the repacked bytes to equal the originals bit-for-bit.
+
+    Every correctly dequantized value sits exactly on the scaled FP4 lattice,
+    so errors in unpacking, scale layout, group mapping, or global-scale
+    handling produce byte mismatches. Non-finite scales are counted as
+    failures. Note this establishes *self-consistency against the actual
+    checkpoint bytes*; conformance to the NVFP4 format spec itself is
+    established separately by the test-suite equivalence check against the
+    reference ``compressed-tensors`` implementation.
+    """
+    reader = CheckpointReader(src)
+    reader.detect_format()
+    packed_mods = [m for m in reader.modules() if m.is_packed]
+    if not packed_mods:
+        raise ValueError("no packed modules to verify")
+
+    gen = torch.Generator().manual_seed(seed)
+    idx = torch.randperm(len(packed_mods), generator=gen)[: min(sample, len(packed_mods))]
+    failures = []
+    for i in idx.tolist():
+        mod = packed_mods[i]
+        packed = reader.tensor(mod.tensors["weight_packed"])
+        scale = reader.tensor(mod.tensors["weight_scale"]).to(torch.float32)
+        gs_name = mod.tensors.get("weight_global_scale")
+        gs = reader.tensor(gs_name).to(torch.float32) if gs_name else None
+
+        try:
+            dense = dequantize_nvfp4(packed, scale, gs, dtype=torch.float32)
+        except ValueError as exc:
+            log(f"  {mod.name}: FAIL ({exc})")
+            failures.append(mod.name)
+            continue
+        eff_scale = scale / gs if gs is not None else scale
+        expanded = eff_scale.repeat_interleave(GROUP_SIZE, dim=-1)
+        # zero-scale groups carry no magnitude information (dense is 0 either
+        # way); reuse the original codes there so they don't false-fail
+        orig_vals = unpack_fp4(packed)
+        codes = torch.where(expanded != 0, dense / expanded, orig_vals)
+        repacked = pack_fp4(codes)
+
+        ok = torch.equal(repacked, packed)
+        residual = None
+        if not ok:
+            # -0.0 codes (0x8) dequantize to 0.0 and legitimately repack as
+            # +0.0 (0x0); torch.equal on the unpacked VALUES treats those as
+            # equal while any real magnitude/sign mismatch still fails
+            new_vals = unpack_fp4(repacked)
+            if torch.equal(orig_vals, new_vals):
+                ok = True
+            else:
+                residual = int((repacked != packed).sum())
+        log(f"  {mod.name}: {'OK' if ok else f'FAIL ({residual} bytes differ)'}")
+        if not ok:
+            failures.append(mod.name)
+
+    result = {"sampled": len(idx), "failures": failures}
+    log(f"VERIFY {'PASSED' if not failures else 'FAILED'} "
+        f"({len(idx) - len(failures)}/{len(idx)} modules)")
+    return result
+
+
+def inspect_checkpoint(src: Path, log=print) -> dict:
+    """Print and return summary stats for a compressed checkpoint."""
+    reader = CheckpointReader(src)
+    cfg = reader.load_config()
+    qc = cfg.get("quantization_config", {})
+    modules = reader.modules()
+    packed = [m for m in modules if m.is_packed]
+
+    packed_bytes = 0
+    dense_bytes_est = 0
+    for m in packed[:1]:  # shape probe on one module
+        t = reader.tensor(m.tensors["weight_packed"])
+        log(f"  example packed module {m.name}: packed shape {tuple(t.shape)}")
+    info = {
+        "format": qc.get("format"),
+        "quant_method": qc.get("quant_method"),
+        "ignore_count": len(qc.get("ignore", [])),
+        "modules": len(modules),
+        "packed_modules": len(packed),
+        "tensors": len(reader.weight_map),
+        "architectures": cfg.get("architectures"),
+    }
+    for k, v in info.items():
+        log(f"  {k}: {v}")
+    return info

llm_dequant/nvfp4.py

@@ -0,0 +1,154 @@
+# SPDX-License-Identifier: Apache-2.0
+"""
+NVFP4 (E2M1, group-16, FP8-E4M3 scales) pack/unpack/dequantize math.
+
+This is a standalone reimplementation of the ``nvfp4-pack-quantized``
+compression format produced by llm-compressor / compressed-tensors
+(https://github.com/vllm-project/llm-compressor). It has no dependency on
+those libraries; only torch is required.
+
+Format specification (as emitted by compressed-tensors >= 0.10):
+
+  weight_packed        uint8  [..., m, n/2]   two FP4 codes per byte,
+                                              LOW nibble = even column,
+                                              HIGH nibble = odd column
+  weight_scale         fp8-e4m3 [..., m, n/16]  one scale per group of 16
+                                              consecutive columns
+  weight_global_scale  fp32   scalar          per-module global scale
+
+  FP4 code: bit 3 = sign, bits 0-2 = index into the E2M1 magnitude table
+            [0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0]
+
+  Dequantization:
+            W[i, j] = e2m1(code[i, j]) * (scale[i, j // 16] / global_scale)
+
+  When ``weight_global_scale`` is absent the division is skipped.
+
+All functions accept an optional leading batch dimension (fused MoE expert
+tensors of shape [num_experts, m, n/2] work unchanged); grouping is always
+along the LAST dimension.
+"""
+
+from __future__ import annotations
+
+import torch
+
+__all__ = [
+    "E2M1_VALUES",
+    "unpack_fp4",
+    "pack_fp4",
+    "dequantize_nvfp4",
+    "GROUP_SIZE",
+]
+
+# The eight non-negative values representable in FP4 E2M1
+# (1 sign bit, 2 exponent bits, 1 mantissa bit).
+E2M1_VALUES = (0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0)
+
+_E2M1_LUT = torch.tensor(E2M1_VALUES, dtype=torch.float32)
+
+# NVFP4 always uses groups of 16 elements along the input-feature dim.
+GROUP_SIZE = 16
+
+
+def unpack_fp4(packed: torch.Tensor, dtype: torch.dtype = torch.float32) -> torch.Tensor:
+    """
+    Unpack a uint8 tensor of packed FP4 codes into real values.
+
+    :param packed: uint8 tensor [..., n/2]; each byte holds two FP4 codes
+                   (low nibble first).
+    :param dtype: dtype of the returned tensor.
+    :return: tensor [..., n] of FP4 values in ``dtype``.
+    """
+    if packed.dtype != torch.uint8:
+        raise TypeError(f"packed tensor must be uint8, got {packed.dtype}")
+
+    low = packed & 0x0F
+    high = (packed >> 4) & 0x0F
+    # interleave: [..., n/2, 2] -> [..., n]
+    codes = torch.stack((low, high), dim=-1).reshape(*packed.shape[:-1], -1)
+
+    signs = (codes & 0x08).to(torch.bool)
+    magnitudes = (codes & 0x07).to(torch.long)
+
+    lut = _E2M1_LUT.to(device=packed.device)
+    values = lut[magnitudes]
+    values = torch.where(signs, -values, values)
+    return values.to(dtype)
+
+
+def pack_fp4(values: torch.Tensor) -> torch.Tensor:
+    """
+    Pack real values into FP4 codes (nearest E2M1 magnitude, ties toward the
+    smaller magnitude — identical behaviour to upstream compressed-tensors).
+
+    Used for verification round-trips and tests; the conversion path never
+    quantizes.
+
+    :param values: float tensor [..., n] with n even.
+    :return: uint8 tensor [..., n/2].
+    """
+    if values.shape[-1] % 2 != 0:
+        raise ValueError("last dimension must be even to pack FP4 pairs")
+
+    lut = _E2M1_LUT.to(device=values.device, dtype=values.dtype)
+    diffs = (values.abs().unsqueeze(-1) - lut).abs()
+    indices = diffs.argmin(dim=-1)  # first minimum wins ties (matches upstream)
+    codes = indices + (torch.signbit(values).to(torch.long) << 3)
+
+    pairs = codes.reshape(*codes.shape[:-1], -1, 2)
+    packed = pairs[..., 0] | (pairs[..., 1] << 4)
+    return packed.to(torch.uint8)
+
+
+def dequantize_nvfp4(
+    packed: torch.Tensor,
+    scale: torch.Tensor,
+    global_scale: torch.Tensor | None = None,
+    dtype: torch.dtype = torch.bfloat16,
+) -> torch.Tensor:
+    """
+    Dequantize an nvfp4-pack-quantized weight back to a dense tensor.
+
+    :param packed: uint8 [..., m, n/2] packed FP4 codes.
+    :param scale: per-group scales [..., m, n/16]; any float dtype
+                  (checkpoints store FP8-E4M3).
+    :param global_scale: optional scalar tensor; when present the effective
+                  scale is ``scale / global_scale``.
+    :param dtype: output dtype (default bfloat16).
+    :return: dense weight [..., m, n] in ``dtype``.
+    """
+    # fp32 intermediate: fp8 scales lose nothing, and v * s products stay exact
+    values = unpack_fp4(packed, dtype=torch.float32)  # [..., m, n]
+    scale_f = scale.to(torch.float32)
+
+    n = values.shape[-1]
+    n_groups = scale_f.shape[-1]
+    if n_groups * GROUP_SIZE != n:
+        raise ValueError(
+            f"scale layout mismatch: weight has {n} columns but scale has "
+            f"{n_groups} groups (expected n = groups * {GROUP_SIZE})"
+        )
+    if scale_f.shape[:-1] != values.shape[:-1]:
+        raise ValueError(
+            f"scale rows {tuple(scale.shape)} do not match weight rows "
+            f"{tuple(packed.shape)}"
+        )
+
+    if global_scale is not None:
+        gs = global_scale.to(torch.float32)
+        if gs.numel() != 1:
+            raise ValueError(f"global_scale must be scalar, got shape {tuple(gs.shape)}")
+        if not torch.isfinite(gs).all():
+            raise ValueError("global_scale is NaN or Inf; refusing to dequantize")
+        if gs.item() == 0.0:
+            raise ValueError("global_scale is zero; refusing to divide")
+        scale_f = scale_f / gs
+
+    if not torch.isfinite(scale_f).all():
+        raise ValueError(
+            "weight_scale contains NaN or Inf values; corrupt checkpoint"
+        )
+
+    expanded = scale_f.repeat_interleave(GROUP_SIZE, dim=-1)  # [..., m, n]
+    return (values * expanded).to(dtype)

llm_dequant-0.1.0.dist-info/METADATA

@@ -0,0 +1,184 @@
+Metadata-Version: 2.4
+Name: llm-dequant
+Version: 0.1.0
+Summary: Dequantize compressed-tensors NVFP4 LLM checkpoints to dense safetensors (streaming, constant memory)
+Author: Victor Cruz
+License: Apache-2.0
+Project-URL: Repository, https://github.com/vcruz305/llm-dequant
+Keywords: nvfp4,quantization,dequantization,safetensors,llm,compressed-tensors,gguf
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: torch>=2.0
+Requires-Dist: safetensors>=0.4
+Requires-Dist: numpy>=1.24
+Provides-Extra: dev
+Requires-Dist: pytest>=7; extra == "dev"
+Dynamic: license-file
+
+<div align="center">
+
+<img src="llm-dequant.png" alt="llm-dequant — stream NVFP4 compressed-tensors checkpoints back to dense safetensors" width="50%">
+
+**Turn NVFP4-only model releases back into dense safetensors.**
+
+Streaming, constant-memory dequantization of
+[compressed-tensors](https://github.com/neuralmagic/compressed-tensors)
+`nvfp4-pack-quantized` LLM checkpoints — a 300B-parameter MoE converts
+on a few hundred MB of RAM.
+
+[![tests](https://github.com/vcruz305/llm-dequant/actions/workflows/tests.yml/badge.svg)](https://github.com/vcruz305/llm-dequant/actions/workflows/tests.yml)
+[![license](https://img.shields.io/badge/license-Apache--2.0-blue.svg)](LICENSE)
+[![python](https://img.shields.io/badge/python-3.10%2B-blue.svg)](pyproject.toml)
+
+</div>
+
+---
+
+## Why this exists
+
+A growing number of community models — abliterations, fine-tunes, supertunes —
+are published **only** in NVFP4 compressed-tensors form. That single release
+format locks out most of the ecosystem:
+
+- `llama.cpp`'s `convert_hf_to_gguf.py` can't read compressed-tensors
+  checkpoints → **no GGUFs**
+- fine-tuning frameworks expect dense weights → **no further training**
+- weight surgery (MTP grafts, merges, pruning) needs real tensors → **no surgery**
+
+The reference decompression path in `compressed-tensors`/`transformers`
+materializes the whole model in memory, which is a non-starter at 300B scale.
+
+`llm-dequant` fixes this with a purpose-built converter:
+
+| | |
+|---|---|
+| 🌊 **Streaming** | module-by-module; peak RSS = one module + one shard buffer |
+| 🎯 **Exact** | output is bit-identical to the reference implementation (CI-checked) |
+| 🧾 **Standard output** | sharded safetensors + index, `quantization_config` stripped, tokenizer files copied — a normal dense HF checkpoint |
+| 🪶 **Dependency-light** | `torch`, `safetensors`, `numpy`. No compressed-tensors, no transformers |
+| 🔍 **Self-verifying** | `verify` proves round-trip byte-equality against *your actual checkpoint* before you commit to a 600GB write |
+| 🛡️ **Defensive** | untrusted `index.json` path names rejected, NaN/Inf scales refuse loudly, unsupported formats never convert silently |
+
+Born from converting a 295B MoE released as NVFP4-only into a full GGUF quant
+sweep ([SuperHY3-abliterated-GGUF](https://huggingface.co/vcruz305/SuperHY3-abliterated-GGUF)).
+
+## Install
+
+```bash
+pip install llm-dequant            # from PyPI (published releases)
+```
+
+or straight from GitHub (works on any commit, no release needed):
+
+```bash
+pip install git+https://github.com/vcruz305/llm-dequant.git
+```
+
+Both pull in `torch`, `safetensors`, and `numpy` automatically. For a
+CPU-only torch (much smaller download):
+
+```bash
+pip install torch --index-url https://download.pytorch.org/whl/cpu
+pip install llm-dequant
+```
+
+## Quickstart
+
+```bash
+# 1. what is this checkpoint?
+llm-dequant inspect /models/SomeModel-NVFP4
+
+# 2. prove the math round-trips on THIS checkpoint (seconds, no writes)
+llm-dequant verify /models/SomeModel-NVFP4 --sample 16
+
+# 3. convert (bf16 default; fp16/fp32 available)
+llm-dequant convert /models/SomeModel-NVFP4 /models/SomeModel-BF16
+```
+
+Then use the output like any dense checkpoint:
+
+```bash
+python llama.cpp/convert_hf_to_gguf.py /models/SomeModel-BF16 \
+    --outfile SomeModel-BF16.gguf --outtype bf16
+```
+
+### Python API
+
+```python
+from llm_dequant.model import convert, verify, inspect_checkpoint
+from llm_dequant.nvfp4 import dequantize_nvfp4, unpack_fp4, pack_fp4
+
+convert("/models/in-nvfp4", "/models/out-bf16", dtype="bf16")
+```
+
+## The NVFP4 format, in one table
+
+`nvfp4-pack-quantized` (emitted by
+[llm-compressor](https://github.com/vllm-project/llm-compressor)) stores, per
+quantized Linear:
+
+| tensor | dtype | shape | meaning |
+|---|---|---|---|
+| `weight_packed` | uint8 | `[m, n/2]` | two FP4-E2M1 codes per byte, low nibble first |
+| `weight_scale` | fp8-e4m3 | `[m, n/16]` | one scale per group of 16 columns |
+| `weight_global_scale` | fp32 | scalar | per-module global scale *(optional)* |
+
+An FP4 code is 1 sign bit + 3 bits indexing the E2M1 magnitude table
+`[0, 0.5, 1, 1.5, 2, 3, 4, 6]`, and
+
+```
+W[i, j] = e2m1(code[i, j]) · scale[i, j//16] / weight_global_scale
+```
+
+**Dequantization is exact.** Every recovered value sits precisely on the
+scaled FP4 lattice (computed in fp32, cast once to the output dtype). Nothing
+the NVFP4 checkpoint encodes is lost. What the original 4-bit quantization
+already destroyed is of course not recoverable — the output is *dense*, not
+*restored*. If you re-quantize the result (e.g. to GGUF), disclose the FP4
+provenance: tiers at or below ~4 bpw lose essentially nothing, higher tiers
+only improve the never-quantized tensors.
+
+## How correctness is established
+
+Two independent layers, both of which you can run yourself:
+
+**1. `llm-dequant verify <src>` — against your checkpoint.**
+Samples random packed modules, dequantizes, renormalizes by the effective
+scale, re-packs with nearest-E2M1 rounding, and requires **byte equality**
+with the original `weight_packed`. Any error in nibble order, scale layout,
+group mapping, or global-scale handling breaks byte equality. (This proves
+self-consistency against the real bytes; spec conformance comes from layer 2.)
+
+**2. The test suite — against the reference implementation.**
+Hand-computed vectors, exhaustive FP4 code tables, property-based round-trips,
+end-to-end synthetic checkpoints (sharded + single-file, with/without global
+scale), corruption and path-traversal rejection, and a bit-exactness test
+against upstream `compressed-tensors` (runs in CI).
+
+```bash
+pip install -e .[dev] && pytest
+```
+
+## Scope
+
+Supported: `format: nvfp4-pack-quantized`, symmetric, `group_size: 16` — the
+only layout llm-compressor emits for NVFP4 today. Fused-MoE expert tensors
+(`[experts, m, n/2]`) work unchanged.
+
+Rejected loudly, never converted silently: other compressed-tensors formats,
+asymmetric quantization (`weight_zero_point`), NaN/Inf scales, shard names
+that escape the checkpoint directory, colliding output tensor names.
+
+Dropped by design: activation-quantization metadata (`input_global_scale`) —
+the output is dense; runtime activation quantization no longer applies.
+
+## License
+
+Apache-2.0. The FP4/E2M1 constant table follows the format specification
+established by the vLLM / compressed-tensors projects (Apache-2.0).

llm_dequant-0.1.0.dist-info/RECORD

@@ -0,0 +1,10 @@
+llm_dequant/__init__.py,sha256=1xGZt_Jg7IAkWy4SyF8Kq7pfOWhEw2Jalgl_1bT9Uqs,157
+llm_dequant/cli.py,sha256=kUU0cmHpH9G7GAic3IxGiG8ZCZMFibYB-DGQxygbXAY,2581
+llm_dequant/model.py,sha256=NkqE1y1NEVAYJjwWn1Eg2jQdlaupPHON1C6lObQfLRY,14617
+llm_dequant/nvfp4.py,sha256=_6SPTCoTTuAElkD8jP-UJHsvYvYsSiEu81ulXfn5NWA,5691
+llm_dequant-0.1.0.dist-info/licenses/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
+llm_dequant-0.1.0.dist-info/METADATA,sha256=de5XdLAP9KA_Zq6NYFwYS1mISwgzmPjDk7lwWxyF368,7266
+llm_dequant-0.1.0.dist-info/WHEEL,sha256=K260EYznzXsJYBQGqmI8VTxEdiZYNvDZwW9cBh9-_MA,91
+llm_dequant-0.1.0.dist-info/entry_points.txt,sha256=4nG4sTJQV5B8Rc7xYHRiSs10XJYOzq_oDU01XNHruDQ,53
+llm_dequant-0.1.0.dist-info/top_level.txt,sha256=MJnOjCTdhtQvvSQGuAYjx54I8nNTeSOY-cjcrPYNhH0,12
+llm_dequant-0.1.0.dist-info/RECORD,,

llm_dequant-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (83.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

llm_dequant-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+llm-dequant = llm_dequant.cli:main

llm_dequant-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

llm_dequant-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+llm_dequant