quantprobe 1.3.0__py3-none-any.whl

quantprobe/plan.py

@@ -0,0 +1,208 @@
+"""quantprobe plan - the tiered decode law as a CLI.
+
+tok/s = eta(tier) x bandwidth / active-bytes-per-token.
+Evaluates every placement (full-VRAM, hybrid attention->VRAM + experts->RAM, dense layer-split,
+pure CPU, disk-stream), predicts speed for each, prints the winner WITH the llama.cpp command to
+run it, plus an upgrade advisor. eta bands are fitted from published measurements (7B..744B),
+validated by pre-registered predictions (30B hybrid: predicted 19, measured 19.30 +/- 0.88).
+"""
+from __future__ import annotations
+
+# kvp = KV-cache bytes per position (K+V, f16, all layers). Exact where the architecture is known,
+# [est] otherwise. MLA models (deepseek) cache the compressed latent -> ~10x smaller: placement's
+# context story differs per architecture, which is why this is per-model, not a constant.
+MODELS = {
+    "qwen3-30b":  dict(t=30.5, a=3.3,  ne=1.2,  moe=True,  kvp=98304,  hint="Qwen3-30B-A3B"),          # 48L x 4KV x 128d (exact; calibration anchor)
+    "deepseek-16b": dict(t=15.7, a=2.4, ne=1.3,  moe=True,  kvp=31104,  hint="DeepSeek-V2-Lite"),        # MLA: 27L x (512+64) latent (exact)
+    "gemma-12b":  dict(t=11.9, a=11.9, ne=11.9, moe=False, kvp=65536,  hint="Gemma 4 12B"),             # [est] SWA: long-ctx slope from global layers only
+    "mistral-7b": dict(t=7.2,  a=7.2,  ne=7.2,  moe=False, kvp=131072, hint="Mistral 7B"),              # 32L x 8KV x 128d (exact)
+    "glm-air":    dict(t=110,  a=12,   ne=2.7,  moe=True,  kvp=94208,  hint="GLM-4.5-Air 106B"),        # [est]
+    "glm-744b":   dict(t=744,  a=32,   ne=8,    moe=True,  kvp=188416, hint="GLM-5.2 744B"),            # [est]
+}
+DEFAULT_KVP = 98304          # custom models without --kv-per-pos: typical GQA mid-size (Qwen3-30B class)
+ETA_KV = 0.70                # KV-read efficiency. Single-point calibration: measured tg32 d0->d16384
+                             # 20.02 -> 16.12 on Qwen3-30B (+12.1 ms/token = 133 GB/s effective on the
+                             # 192 GB/s tier). Falsify/refine it: quantprobe bench --depth N --contribute
+# eta values: MEASURED on 2016-xmp/2016 (my box); ESTIMATED for the rest (help validate: run `quantprobe bench`).
+# Mac uses unified memory -> modeled as one high-bandwidth pool the GPU serves from (the "all in VRAM" path).
+MACHINES = {
+    # --- measured on my hardware ---
+    "2016-xmp":    dict(vc=6,  vb=192,  rc=16,  rb=48,  db=0.45, geta=0.35, gl=0.04, hint="2016 desktop (GTX 1060 6GB), XMP on [measured]"),
+    "2016":        dict(vc=6,  vb=192,  rc=16,  rb=34,  db=0.45, geta=0.35, gl=0.04, hint="2016 desktop, XMP off [measured]"),
+    # --- estimated: consumer GPUs ---
+    "rtx-3060":    dict(vc=12, vb=360,  rc=32,  rb=51,  db=3.5,  geta=0.5,  gl=0.3,  hint="RTX 3060 12GB + DDR4-3200 [est]"),
+    "rtx-3090":    dict(vc=24, vb=936,  rc=64,  rb=51,  db=3.5,  geta=0.6,  gl=0.4,  hint="RTX 3090 24GB + DDR4 [est]"),
+    "rtx-4090":    dict(vc=24, vb=1008, rc=64,  rb=83,  db=5,    geta=0.62, gl=0.42, hint="RTX 4090 24GB + DDR5 [est]"),
+    "rtx-5090":    dict(vc=32, vb=1792, rc=64,  rb=90,  db=5,    geta=0.62, gl=0.42, hint="RTX 5090 32GB + DDR5 [est]"),
+    "laptop-8gb":  dict(vc=8,  vb=256,  rc=16,  rb=45,  db=2,    geta=0.45, gl=0.28, hint="gaming laptop, 8GB GPU + DDR5 [est]"),
+    "gaming":      dict(vc=12, vb=360,  rc=32,  rb=51,  db=3.5,  geta=0.5,  gl=0.3,  hint="RTX 3060 12GB + DDR4-3200 [est] (alias of rtx-3060)"),
+    # --- estimated: Apple Silicon (unified memory; I have NOT measured a Mac - these are predictions) ---
+    "mac-m2-max":  dict(vc=64,  vb=400, rc=8,   rb=400, db=5,    geta=0.26, gl=0.24, hint="Mac M2 Max, 400 GB/s unified [est, unvalidated]"),
+    "mac-m3-max":  dict(vc=96,  vb=400, rc=8,   rb=400, db=5,    geta=0.26, gl=0.24, hint="Mac M3 Max, 400 GB/s unified [est, unvalidated]"),
+    "mac-m4-max":  dict(vc=128, vb=546, rc=8,   rb=546, db=5,    geta=0.26, gl=0.24, hint="Mac M4 Max, 546 GB/s unified [est, unvalidated]"),
+    "mac-m2-ultra":dict(vc=192, vb=800, rc=8,   rb=800, db=5,    geta=0.25, gl=0.23, hint="Mac M2 Ultra, 800 GB/s unified [est, unvalidated]"),
+    "mac-m3-ultra":dict(vc=512, vb=819, rc=8,   rb=819, db=5,    geta=0.25, gl=0.23, hint="Mac M3 Ultra 512GB, 819 GB/s [est, unvalidated]"),
+    # --- estimated: big-RAM / server ---
+    "ddr5":        dict(vc=0,  vb=0,    rc=64,  rb=80,  db=5,    geta=0.5,  gl=0.3,  hint="modern desktop DDR5, no GPU [est]"),
+    "colibri":     dict(vc=0,  vb=0,    rc=128, rb=60,  db=5,    geta=0.5,  gl=0.3,  hint="128 GB DDR5 workstation [est]"),
+    "epyc-256":    dict(vc=0,  vb=0,    rc=256, rb=200, db=5,    geta=0.5,  gl=0.3,  hint="Epyc/Threadripper, 256GB, ~200 GB/s [est]"),
+    "dgx-spark":   dict(vc=128,vb=273,  rc=8,   rb=273, db=5,    geta=0.79, gl=0.6,  hint="NVIDIA DGX Spark / GB10, 128GB unified [validated vs published]"),
+}
+def numlist(x):
+    """CLI type: '24,24' -> [24.0, 24.0]; '24' -> 24.0. Lists = multiple devices."""
+    if isinstance(x, (int, float)):
+        return float(x)
+    if "," in str(x):
+        return [float(v) for v in str(x).split(",") if v.strip()]
+    return float(x)
+
+
+def agg_cap(v):
+    return sum(v) if isinstance(v, list) else v
+
+
+def agg_bw(v, eff):
+    """Aggregate bandwidth of multiple devices: sum x efficiency (TP loss for GPUs [est],
+    stripe loss for disks [est from the RAID-0 Gen5 datapoint, eta 0.66 vs single ~0.88])."""
+    if isinstance(v, list):
+        return sum(v) * (eff if len(v) > 1 else 1.0)
+    return v
+
+
+QUAL = {True:  {2.0: 1.10, 2.5: 1.07, 3.0: 1.05, 4.5: 1.02, 6.5: 1.01, 8.5: 1.00},
+        False: {2.0: 1.45, 2.5: 1.30, 3.0: 1.12, 4.5: 1.03, 6.5: 1.01, 8.5: 1.00}}
+
+
+def evaluate(t, a, ne, moe, bits, vc, vb, rc, rb, db, geta, act_scale=1.0, gl=None, ctx=0, kvp=0.0):
+    ab = max(bits, 4.5)                                   # attention protected at ~4-bit (Law 3 recipes)
+    size = (ne * ab / 8 + (t - ne) * bits / 8) * 1.08 * act_scale
+    act_ne = ne * ab / 8 * 1.15 * act_scale
+    act_ex = (a - ne) * bits / 8 * 1.15 * act_scale
+    act = act_ne + act_ex
+    # Law 4 v2 (context term, v1.1): every generated token re-reads the whole KV cache from
+    # whichever tier KV lives on — kv_gb adds to BOTH the byte budget and that tier's capacity.
+    kv_gb = ctx * kvp / 1e9 if ctx > 0 else 0.0
+    ra = max(rc - 4, 1)
+    eta_r = 0.38 if moe else 0.62
+    if gl is None: gl = geta * 0.6
+    geta_w = geta if bits >= 4 else gl                 # decode-util law: low-bit GPU decode collapses on weak GPUs
+    out = []
+    if vc > 0 and size + kv_gb <= vc * 0.90:
+        out.append(("all in VRAM", 1 / (act / (geta_w * vb) + kv_gb / (ETA_KV * vb)), None,
+                    "-ngl 99"))
+    if moe and vc > 0:
+        v_need = ne * ab / 8 * 1.08 + 1.2 + kv_gb          # KV sits with attention in VRAM
+        r_need = size - ne * ab / 8 * 1.08
+        if v_need <= vc * 0.95 and r_need <= ra:
+            warn = "RAM boundary - needs --no-mmap; can be unstable" if r_need > ra * 0.85 else None
+            out.append(("hybrid: attention->VRAM, experts->RAM",
+                        1 / (act_ne / (geta * vb) + act_ex / (eta_r * rb) + kv_gb / (ETA_KV * vb)), warn,
+                        '-ngl 99 -ot "exps=CPU" --no-mmap'))
+    if (not moe) and vc > 0 and size + kv_gb > vc * 0.90 and size + kv_gb <= ra + vc * 0.9:
+        g = min(0.95, vc * 0.9 / (size + kv_gb))           # KV splits with its layers
+        kv_t = g * kv_gb / (ETA_KV * vb) + (1 - g) * kv_gb / (ETA_KV * rb)
+        out.append((f"split: {g:.0%} layers->VRAM, rest->RAM",
+                    1 / (g * act / (geta_w * vb) + (1 - g) * act / (eta_r * rb) + kv_t), None,
+                    f"-ngl {int(g * 99)}"))
+    if size + kv_gb <= ra:
+        warn = "RAM boundary - expect bimodal speed" if size + kv_gb > ra * 0.85 else None
+        out.append(("pure CPU (GPU idle)", 1 / (act / (eta_r * rb) + kv_gb / (ETA_KV * rb)), warn, "-ngl 0"))
+    if size + kv_gb > ra:
+        ra_eff = max(ra - kv_gb, 1)                        # KV crowds the expert cache
+        miss = max(0.0, 1 - (ra_eff * 0.9) / size)
+        hot = act_ne if moe else 0.0                       # MoE attention stays LRU-hot; dense has no hot set
+        streamable = act - hot
+        tps = 0.95 / (streamable * miss / db + (streamable * (1 - miss) + hot) / (eta_r * rb)
+                      + kv_gb / (ETA_KV * rb))
+        out.append(("stream from disk (cold experts)", tps, "exceeds RAM - capacity demo", "-ngl 0"))
+    if moe and vc > 0 and size + kv_gb > ra:
+        # three-tier expert cache (VRAM + RAM + disk): what expert-caching runtimes achieve.
+        # llama.cpp mainline cannot LRU experts in VRAM - its number is the row above.
+        v_res = ne * ab / 8 * 1.08 + 1.2 + kv_gb           # attention + KV live in VRAM here
+        vfree = max(0.0, vc * 0.90 - v_res)
+        if vfree > 0.5:
+            cache = ra * 0.9 + vfree
+            miss3 = max(0.0, 1 - cache / size)
+            hot = act_ne
+            streamable = act - hot
+            vshare = vfree / cache if cache > 0 else 0.0
+            hit = streamable * (1 - miss3)
+            tps3 = 0.95 / (streamable * miss3 / db
+                           + hit * (1 - vshare) / (eta_r * rb)
+                           + hit * vshare / (geta * vb)
+                           + hot / (geta * vb)
+                           + kv_gb / (ETA_KV * vb))
+            out.append(("stream from disk (VRAM+RAM expert cache)", tps3,
+                        "needs an expert-caching runtime (ktransformers/colibri-class) - stock llama.cpp gets the RAM-cache row",
+                        "-ngl 99 + runtime-managed expert cache"))
+    out.sort(key=lambda x: -x[1])
+    return size, act, out
+
+
+def qual_of(moe, bits):
+    keys = sorted(QUAL[moe])
+    return QUAL[moe][min(keys, key=lambda k: abs(k - bits))]
+
+
+def run(args):
+    from . import spec as specmod
+    specmod.apply(args)
+    if getattr(args, "bits", None) is None:
+        args.bits = 2.5
+    m = dict(MODELS[args.model]) if args.model in MODELS else {}
+    t = args.total or m.get("t") or 13.0
+    a = args.active or m.get("a") or t
+    ne = args.always_active or m.get("ne") or (a if a >= t * 0.9 else a * 0.35)
+    moe = m.get("moe", a < t * 0.9)
+    hw = dict(MACHINES[args.machine]) if args.machine in MACHINES else {}
+    if not hw and all(getattr(args, k, None) is None for k in ("vram", "vram_bw", "ram", "ram_bw", "disk_bw")):
+        from . import detect as detmod
+        auto, _ = detmod.detect()
+        hw = dict(vc=auto["vram"], vb=auto["vram_bw"], rc=auto["ram"], rb=auto["ram_bw"],
+                  db=auto["disk_bw"], geta=auto.get("geta", 0.45), gl=auto.get("gl"),
+                  hint="THIS machine [auto-detected - run `quantprobe hw` for details]")
+        print("[quantprobe] no hardware flags: auto-detected this machine "
+              f"(vram {hw['vc']:g}GB@{hw['vb']:g} | ram {hw['rc']:g}GB@{hw['rb']:g} | disk {hw['db']:g} GB/s). "
+              "Pass --machine/flags to estimate a different box.")
+    vc = hw.get("vc", args.vram); vb = hw.get("vb", args.vram_bw)
+    rc = hw.get("rc", args.ram);  rb = hw.get("rb", args.ram_bw)
+    db = hw.get("db", args.disk_bw); geta = hw.get("geta", 0.45); gl = hw.get("gl", None)
+    if args.vram is not None: vc = args.vram
+    if args.vram_bw is not None: vb = args.vram_bw
+    if args.ram is not None: rc = args.ram
+    if args.ram_bw is not None: rb = args.ram_bw
+    if args.disk_bw is not None: db = args.disk_bw
+    vc = agg_cap(vc) or 0; vb = agg_bw(vb, 0.85) or 0
+    rc = rc or 16; rb = rb or 40
+    db = agg_bw(db, 0.75) or 0.5
+    ctx = getattr(args, "ctx", 0) or 0
+    kvp = (args.kv_per_pos * 1024 if getattr(args, "kv_per_pos", None)
+           else m.get("kvp", DEFAULT_KVP))
+
+    size, act, cfgs = evaluate(t, a, ne, moe, args.bits, vc, vb, rc, rb, db, geta, gl=gl, ctx=ctx, kvp=kvp)
+    q = qual_of(moe, args.bits)
+    print(f"\nquantprobe plan - {m.get('hint', 'custom model')} @ {args.bits:g}-bit "
+          f"on {hw.get('hint', 'custom machine')}")
+    kvline = (f" | ctx {ctx}: +{ctx * kvp / 1e9:.2f} GB KV read/token"
+              if ctx > 0 else "")
+    print(f"  model {size:.1f} GB | active {act:.2f} GB/token{kvline} | est. quality cost x{q:.2f} "
+          f"(depth-aware recipe)\n")
+    for i, (name, tps, warn, flags) in enumerate(cfgs):
+        star = "*" if i == 0 else " "
+        w = f"   [{warn}]" if warn else ""
+        print(f"  {star} {tps:6.1f} tok/s  {name}{w}")
+    best = cfgs[0]
+    print(f"\n  run it:  llama-server -m model.gguf {best[3]}")
+    # upgrade advisor
+    alts = []
+    if rb < 40:
+        s2, _, c2 = evaluate(t, a, ne, moe, args.bits, vc, vb, rc, 48, db, geta, gl=gl, ctx=ctx, kvp=kvp)
+        if c2[0][1] > best[1] * 1.08: alts.append(("enable XMP (free)", c2[0][1]))
+    s2, _, c2 = evaluate(t, a, ne, moe, args.bits, vc, vb, rc + 16, rb, db, geta, gl=gl, ctx=ctx, kvp=kvp)
+    if c2[0][1] > best[1] * 1.08: alts.append(("+16 GB RAM", c2[0][1]))
+    s2, _, c2 = evaluate(t, a, ne, moe, args.bits, vc, vb, rc, rb, 3.5, geta, gl=gl, ctx=ctx, kvp=kvp)
+    if c2[0][1] > best[1] * 1.08: alts.append(("NVMe SSD", c2[0][1]))
+    if alts:
+        print("  upgrade advisor: " + " | ".join(f"{n} -> ~{v:.1f} tok/s" for n, v in alts))
+    print("\n  (eta bands fitted from published measurements; estimates +/-25%. "
+          "Hybrid needs --no-mmap.)")

quantprobe/probe.py

@@ -0,0 +1,149 @@
+"""quantprobe probe — measure a GGUF's depth-fragility curve, emit the depth-aware recipe.
+Adapted from the research script (weights/quant_probe.py); logic identical, llama.cpp located via
+--llama-dir, QUANTPROBE_LLAMA_DIR, or PATH.
+"""
+from __future__ import annotations
+import os, re, shutil, subprocess
+
+
+def find_llama(explicit):
+    for cand in ([explicit] if explicit else []) + [os.environ.get("QUANTPROBE_LLAMA_DIR")]:
+        if cand and os.path.isfile(os.path.join(cand, exe("llama-quantize"))):
+            return cand
+    w = shutil.which("llama-quantize") or shutil.which("llama-quantize.exe")
+    if w:
+        return os.path.dirname(w)
+    raise SystemExit("llama.cpp binaries not found: pass --llama-dir, set QUANTPROBE_LLAMA_DIR, or add to PATH")
+
+
+def exe(name):
+    return name + (".exe" if os.name == "nt" else "")
+
+
+def n_layers(gguf_path):
+    from gguf import GGUFReader
+    r = GGUFReader(gguf_path)
+    for field in r.fields.values():
+        if field.name.endswith(".block_count"):
+            return int(field.parts[field.data[0]][0])
+    raise RuntimeError("no .block_count key in GGUF metadata")
+
+
+def band_regex(lo, hi):
+    return "blk\\.(" + "|".join(str(i) for i in range(lo, hi + 1)) + ")\\.ffn_.*"
+
+
+def sh(cmd, dry, capture=False):
+    print("  $", " ".join(cmd), flush=True)
+    if dry:
+        return ""
+    if capture:
+        return subprocess.run(cmd, capture_output=True, text=True).stdout + \
+               subprocess.run(cmd, capture_output=True, text=True).stderr if False else \
+               subprocess.run(cmd, capture_output=True, text=True, errors="replace").stdout
+    subprocess.run(cmd, check=False)
+    return ""
+
+
+def ppl(perp, gguf, eval_file, chunks, ngl, dry):
+    print(f"  measuring perplexity on {chunks} chunks (this can take 1-3 min; llama.cpp is quiet while it works)...", flush=True)
+    if dry:
+        return None
+    p = subprocess.run([perp, "-m", gguf, "-f", eval_file, "--chunks", str(chunks), "-ngl", str(ngl)],
+                       capture_output=True, text=True, errors="replace")
+    m = re.search(r"Final estimate: PPL = ([0-9.]+)", p.stdout + p.stderr)
+    return float(m.group(1)) if m else None
+
+
+def run(a):
+    llama = find_llama(a.llama_dir)
+    quant = os.path.join(llama, exe("llama-quantize"))
+    perp = os.path.join(llama, exe("llama-perplexity"))
+    wd = a.workdir or os.path.dirname(os.path.abspath(a.gguf))
+    L = n_layers(a.gguf)
+    step = (L + a.bands - 1) // a.bands
+    bands = [(i, min(i + step - 1, L - 1)) for i in range(0, L, step)]
+    print(f"quant-probe: {os.path.basename(a.gguf)} | {L} layers -> {len(bands)} bands {bands}\n", flush=True)
+
+    ref = os.path.join(wd, "_probe_ref_q6k.gguf")
+    print("[1/3] reference Q6_K", flush=True)
+    sh([quant, "--allow-requantize", a.gguf, ref, "Q6_K", "8"], a.dry_run)
+    p_ref = ppl(perp, ref, a.eval, a.chunks, a.ngl, a.dry_run)
+    print(f"  ref PPL = {p_ref}\n", flush=True)
+
+    print("[2/3] band probe (one band's FFNs -> Q2_K at a time)", flush=True)
+    deltas = []
+    for lo, hi in bands:
+        out = os.path.join(wd, f"_probe_b{lo}_{hi}.gguf")
+        sh([quant, "--allow-requantize", "--tensor-type", f"{band_regex(lo, hi)}=q2_k", a.gguf, out, "Q6_K", "8"], a.dry_run)
+        p = ppl(perp, out, a.eval, a.chunks, a.ngl, a.dry_run)
+        d = None if (p is None or p_ref is None) else p - p_ref
+        deltas.append(d)
+        print(f"  layers {lo}-{hi}: PPL {p}  (delta {d})", flush=True)
+        if not a.dry_run and os.path.exists(out):
+            os.remove(out)
+    if not a.dry_run and os.path.exists(ref):
+        os.remove(ref)
+
+    print("\n[3/3] recipe", flush=True)
+    if a.dry_run or any(d is None for d in deltas):
+        print("  (dry-run / incomplete: curve unavailable)", flush=True)
+        return
+    worst = max(range(len(bands)), key=lambda i: deltas[i])
+    lo, hi = bands[worst]
+    others = [f"{band_regex(b[0], b[1])}=q2_k" for i, b in enumerate(bands) if i != worst]
+    print(f"  fragile band: layers {lo}-{hi} (delta +{deltas[worst]:.2f} vs "
+          f"median {sorted(deltas)[len(deltas)//2]:.2f}) -> protect at Q4_K:\n", flush=True)
+    flags = " ".join(f'--tensor-type "{o}"' for o in others)
+    print(f'  llama-quantize {flags} --tensor-type "{band_regex(lo, hi)}=q4_k" '
+          f'--tensor-type "attn_.*=q4_k" --token-embedding-type q4_k \\\n'
+          f'    {os.path.basename(a.gguf)} out-depthaware.gguf Q2_K 8', flush=True)
+    if getattr(a, "apply", False):
+        out = a.out or os.path.splitext(a.gguf)[0] + "-depthaware.gguf"
+        print("\n[quantprobe] --apply: building the recommended GGUF now...", flush=True)
+        build_depthaware(a.llama_dir, a.gguf, out, lo, hi, bands[-1][1] + 1, dry=a.dry_run)
+    else:
+        print("\n  (re-run with  --apply --out model-2bit.gguf  to BUILD this GGUF automatically)", flush=True)
+
+
+def _band_re(lo, hi):
+    return "blk\\.(" + "|".join(str(i) for i in range(lo, hi + 1)) + ")\\.ffn_.*"
+
+
+def build_depthaware(llama_dir, src, out, protect_lo, protect_hi, n_lay,
+                     base="Q2_K", protect="q4_k", dry=False):
+    """Actually PRODUCE the compressed GGUF: base bits everywhere, fragile band + attention + embed protected."""
+    # --dry previews the exact command WITHOUT requiring llama.cpp installed
+    q = exe("llama-quantize") if dry else os.path.join(find_llama(llama_dir), exe("llama-quantize"))
+    cmd = [q, "--allow-requantize"]
+    if protect_lo > 0:
+        cmd += ["--tensor-type", f"{_band_re(0, protect_lo - 1)}=q2_k"]
+    if protect_hi < n_lay - 1:
+        cmd += ["--tensor-type", f"{_band_re(protect_hi + 1, n_lay - 1)}=q2_k"]
+    cmd += ["--tensor-type", f"{_band_re(protect_lo, protect_hi)}={protect}",
+            "--tensor-type", "attn_.*=q4_k", "--token-embedding-type", "q4_k",
+            src, out, base, "8"]
+    print(f"[quantprobe] building depth-aware GGUF: protect layers {protect_lo}-{protect_hi} @ {protect}")
+    print("  $ " + " ".join(cmd))
+    if dry:
+        return out
+    rc = subprocess.call(cmd)
+    if rc == 0 and os.path.exists(out):
+        print(f"[quantprobe] done -> {out} ({os.path.getsize(out)/1e9:.2f} GB). "
+              f"Run it:  quantprobe run --gguf {out} --model <preset> --machine <preset>")
+    else:
+        print(f"[quantprobe] quantize failed (exit {rc}).")
+    return out
+
+
+def quantize(a):
+    """Standalone compress: build a depth-aware GGUF from an explicit band (no probing)."""
+    if not os.path.isfile(a.gguf):
+        raise SystemExit(f"GGUF not found: {a.gguf}  (point --gguf at a real high-precision GGUF: f16/bf16/Q8)")
+    n_lay = n_layers(a.gguf)
+    if a.protect:
+        lo, hi = (int(x) for x in a.protect.split("-"))
+    else:
+        lo, hi = n_lay - a.protect_late, n_lay - 1
+    out = a.out or os.path.splitext(a.gguf)[0] + "-depthaware.gguf"
+    build_depthaware(a.llama_dir, a.gguf, out, lo, hi, n_lay, dry=getattr(a, "dry", False))

quantprobe/runtime.py

@@ -0,0 +1,187 @@
+"""quantprobe run / bench — the runtime layer.
+
+run:   plan the best placement for your model+machine, then LAUNCH llama.cpp with those exact
+       flags (chat via llama-cli, or --serve for llama-server). Colibri-style one-command UX,
+       riding stock llama.cpp instead of a custom engine.
+bench: measure real decode tok/s with the planned flags and print predicted vs measured —
+       every user becomes a validation point for the tiered decode law.
+"""
+from __future__ import annotations
+import os, re, shutil, subprocess, sys
+
+from . import plan as planmod
+
+
+def exe(name):
+    return name + (".exe" if os.name == "nt" else "")
+
+
+def find_llama(explicit, tool):
+    for cand in ([explicit] if explicit else []) + [os.environ.get("QUANTPROBE_LLAMA_DIR")]:
+        if cand and os.path.isfile(os.path.join(cand, exe(tool))):
+            return os.path.join(cand, exe(tool))
+    w = shutil.which(tool) or shutil.which(exe(tool))
+    if w:
+        return w
+    raise SystemExit(f"{tool} not found: pass --llama-dir, set QUANTPROBE_LLAMA_DIR, or add to PATH")
+
+
+def best_flags(a):
+    """Run the planner, return (best_config, flags_list) for the winning placement."""
+    from . import spec as specmod
+    specmod.apply(a)
+    if getattr(a, "bits", None) is None:
+        a.bits = 2.5
+    m = dict(planmod.MODELS[a.model]) if getattr(a, "model", None) in planmod.MODELS else {}
+    t = getattr(a, "total", None) or m.get("t") or 13.0
+    ac = getattr(a, "active", None) or m.get("a") or t
+    ne = getattr(a, "always_active", None) or m.get("ne") or (ac if ac >= t * 0.9 else ac * 0.35)
+    moe = m.get("moe", ac < t * 0.9)
+    hw = dict(planmod.MACHINES[a.machine]) if getattr(a, "machine", None) in planmod.MACHINES else {}
+    if not hw and all(getattr(a, k, None) is None for k in ("vram", "vram_bw", "ram", "ram_bw", "disk_bw")):
+        from . import detect as detmod
+        auto, _ = detmod.detect()
+        hw = dict(vc=auto["vram"], vb=auto["vram_bw"], rc=auto["ram"], rb=auto["ram_bw"],
+                  db=auto["disk_bw"], geta=auto.get("geta", 0.45), gl=auto.get("gl"))
+        print("[quantprobe] hardware auto-detected (run `quantprobe hw` for details; "
+              "pass --machine/flags to estimate a different box)")
+    vc = planmod.agg_cap(a.vram) if a.vram is not None else hw.get("vc", 0)
+    vb = planmod.agg_bw(a.vram_bw, 0.85) if a.vram_bw is not None else hw.get("vb", 0)
+    rc = a.ram if a.ram is not None else hw.get("rc", 16)
+    rb = a.ram_bw if a.ram_bw is not None else hw.get("rb", 40)
+    db = planmod.agg_bw(a.disk_bw, 0.75) if a.disk_bw is not None else hw.get("db", 0.5)
+    geta = hw.get("geta", 0.45); gl = hw.get("gl", None)
+    act_scale = 1.0
+    gguf = getattr(a, "gguf", None)
+    if gguf and os.path.isfile(gguf):
+        ab = max(a.bits, 4.5)
+        size_pred = (ne * ab / 8 + (t - ne) * a.bits / 8) * 1.08
+        size_real = os.path.getsize(gguf) / 1e9
+        if size_pred > 0:
+            act_scale = size_real / size_pred
+            print(f"[quantprobe] calibrated to file: {size_real:.2f} GB on disk "
+                  f"(preset assumed {size_pred:.2f} GB, scale {act_scale:.2f})")
+    ctx = getattr(a, "ctx", 0) or 0
+    kvp = (a.kv_per_pos * 1024 if getattr(a, "kv_per_pos", None)
+           else m.get("kvp", planmod.DEFAULT_KVP))
+    _, _, cfgs = planmod.evaluate(t, ac, ne, moe, a.bits, vc, vb, rc, rb, db, geta, act_scale, gl,
+                                  ctx=ctx, kvp=kvp)
+    best = cfgs[0]
+    return best, best[3].replace('"', "").split()
+
+
+def run(a):
+    best, flags = best_flags(a)
+    tool = "llama-server" if a.serve else "llama-cli"
+    # --dry previews the plan + command WITHOUT requiring llama.cpp installed
+    binp = tool if a.dry else find_llama(a.llama_dir, tool)
+    cmd = [binp, "-m", a.gguf] + flags
+    if (getattr(a, "ctx", 0) or 0) > 0:
+        cmd += ["-c", str(a.ctx)]                         # launch with the context you planned for
+    if not a.serve:
+        cmd += ["-cnv"]
+    if a.extra:
+        cmd += a.extra.split()
+    print(f"[quantprobe] placement: {best[0]}  (predicted {best[1]:.1f} tok/s"
+          + (f", {best[2]}" if best[2] else "") + ")")
+    print("[quantprobe] exec:", " ".join(cmd), "\n")
+    if a.dry:
+        return
+    sys.exit(subprocess.call(cmd))
+
+
+def bench(a):
+    if getattr(a, "depth", None):
+        a.ctx = a.depth                                   # prediction at the benched depth
+    best, flags = best_flags(a)
+    binp = "llama-bench" if getattr(a, "dry", False) else find_llama(a.llama_dir, "llama-bench")
+    # llama-bench uses --mmap 0 rather than --no-mmap
+    bflags = ["--mmap", "0" if "--no-mmap" in flags else "1"]
+    for i, f in enumerate(flags):
+        if f == "-ngl":
+            bflags += ["-ngl", flags[i + 1]]
+        if f == "-ot":
+            bflags += ["-ot", flags[i + 1]]
+    if flags and flags[0].startswith("-ngl") is False and "-ngl" not in flags:
+        pass
+    # normalize: flags like ['-ngl','99','-ot','exps=CPU','--no-mmap']
+    cmd = [binp, "-m", a.gguf, "-n", "32", "-p", "0", "-r", str(a.reps)] + bflags
+    if getattr(a, "depth", None):
+        cmd += ["-d", str(a.depth)]
+    print(f"[quantprobe] placement: {best[0]} | predicted {best[1]:.1f} tok/s")
+    print("[quantprobe] bench:", " ".join(cmd))
+    if a.dry:
+        return
+    print("[quantprobe] benchmarking (30-90s; llama-bench runs quietly, then prints the number)...", flush=True)
+    out = subprocess.run(cmd, capture_output=True, text=True, errors="replace")
+    txt = out.stdout + out.stderr
+    mm = re.findall(r"tg\d+(?:\s*@\s*d\d+)?\s*\|\s*([0-9.]+)\s*(?:Â?±|\+/-)\s*([0-9.]+)", txt)
+    if not mm:
+        mm = re.findall(r"\|\s*([0-9.]+)\s*(?:Â?±)\s*([0-9.]+)\s*\|\s*$", txt, re.M)
+    if mm:
+        meas, err = float(mm[-1][0]), float(mm[-1][1])
+        delta = (meas / best[1] - 1) * 100 if best[1] else 0
+        print(f"\n[quantprobe] measured: {meas:.2f} +/- {err:.2f} tok/s "
+              f"(predicted {best[1]:.1f}, {delta:+.0f}%)")
+        if getattr(a, "contribute", False):
+            _emit_contribution(a, best, meas, err, delta)
+        else:
+            print("[quantprobe] the tiered decode law just ran on your machine.")
+            print("[quantprobe] help grow the law: re-run with --contribute for a one-click, "
+                  "pre-filled data point (you review it first; nothing is sent automatically).")
+    else:
+        print("\n[quantprobe] could not parse llama-bench output; raw tail:")
+        print("\n".join(txt.strip().splitlines()[-6:]))
+
+
+def tier_view(a, best):
+    """Rough (capacity, used) per tier for the dashboard's placement panel."""
+    hw = dict(planmod.MACHINES[a.machine]) if getattr(a, "machine", None) in planmod.MACHINES else {}
+    if not hw and all(getattr(a, k, None) is None for k in ("vram", "vram_bw", "ram", "ram_bw", "disk_bw")):
+        from . import detect as detmod
+        auto, _ = detmod.detect()
+        hw = dict(vc=auto["vram"], vb=auto["vram_bw"], rc=auto["ram"], rb=auto["ram_bw"],
+                  db=auto["disk_bw"], geta=auto.get("geta", 0.45), gl=auto.get("gl"))
+        print("[quantprobe] hardware auto-detected (run `quantprobe hw` for details; "
+              "pass --machine/flags to estimate a different box)")
+    vc = planmod.agg_cap(a.vram) if a.vram is not None else hw.get("vc", 0)
+    rc = a.ram if a.ram is not None else hw.get("rc", 16)
+    size = os.path.getsize(a.gguf) / 1e9 if a.gguf and os.path.isfile(a.gguf) else 0
+    name = best[0]
+    if name == "all in VRAM":
+        return [("VRAM", vc, size), ("RAM", rc, 1.0)]
+    if name.startswith("hybrid"):
+        v = min(size * 0.15 + 1.2, vc)
+        return [("VRAM (attention + ctx)", vc, v), ("RAM (experts)", rc, size - size * 0.15)]
+    if name.startswith("split"):
+        return [("VRAM", vc, vc * 0.9), ("RAM", rc, max(0.5, size - vc * 0.9))]
+    if name.startswith("pure CPU"):
+        return [("VRAM (idle)", vc, 0), ("RAM", rc, size)]
+    return [("RAM (cache)", rc, rc - 4), ("disk (streaming)", max(size * 1.2, 1), size)]
+
+
+def _emit_contribution(a, best, meas, err, delta):
+    import urllib.parse
+    from . import __version__
+    hw = (dict(planmod.MACHINES.get(getattr(a, "machine", "") or "", {})).get("hint")
+          or f"vram={a.vram} vram_bw={a.vram_bw} ram={a.ram} ram_bw={a.ram_bw} disk_bw={a.disk_bw}")
+    model = getattr(a, "model", None) or f"total={a.total} active={a.active}"
+    lines = [
+        f"hardware: {hw}",
+        f"model: {model} @ {a.bits:g}-bit",
+        f"placement: {best[0]}",
+        f"predicted: {best[1]:.1f} tok/s",
+        f"measured: {meas:.2f} +/- {err:.2f} tok/s ({delta:+.0f}%)",
+        f"quantprobe: v{__version__}",
+        "",
+        "Notes (optional): ",
+    ]
+    body = "\n".join(lines)
+    title = f"[eta] {model} {a.bits:g}-bit on {str(hw)[:40]}"
+    url = ("https://github.com/FedericoTs/quantprobe/issues/new?labels=eta-datapoint"
+           f"&title={urllib.parse.quote(title)}&body={urllib.parse.quote(body)}")
+    print("\n[quantprobe] Contribute this data point (OPT-IN). It contains ONLY what you see below --")
+    print("             no system scan, no IP, nothing auto-collected. Review, then submit:\n")
+    print(body)
+    print("\n  Open to submit (you can edit first):\n  " + url + "\n")
+    print("  Points that land OUTSIDE the predicted bands are the most valuable -- they refine the law.")

quantprobe/spec.py

@@ -0,0 +1,89 @@
+"""quantprobe autospec — read the MODEL's law-parameters from the GGUF itself.
+
+A GGUF already contains everything the decode law needs:
+  total params    = sum of tensor element counts
+  routed/active   = expert tensor split + expert_used/expert_count from metadata
+  effective bits  = file bytes x 8 / total params (the real number, not the type name)
+  kv bytes/pos    = exact, from layer count x KV heads x head dims (MLA-aware)
+So `--gguf model.gguf` alone fully specifies the model; flags remain as overrides.
+"""
+from __future__ import annotations
+import os
+
+
+def _field(r, *names):
+    for f in r.fields.values():
+        for n in names:
+            if f.name.endswith(n):
+                try:
+                    return int(f.parts[f.data[0]][0])
+                except Exception:
+                    pass
+    return None
+
+
+def from_gguf(path):
+    from gguf import GGUFReader
+    r = GGUFReader(path)
+    n_layer = _field(r, ".block_count") or 32
+    total = 0
+    routed = 0
+    for t in r.tensors:
+        n = 1
+        for d in t.shape:
+            n *= int(d)
+        total += n
+        if "exps" in t.name or "_expert" in t.name:
+            routed += n
+    ne_params = total - routed
+
+    n_exp = _field(r, ".expert_count")
+    n_used = _field(r, ".expert_used_count")
+    if routed and n_exp and n_used:
+        active = ne_params + routed * n_used / n_exp
+        moe = True
+    else:
+        active, moe = total, False
+
+    # exact KV bytes/pos (f16): MLA caches the latent; GQA caches heads x dims, K+V
+    kv_lora = _field(r, ".attention.kv_lora_rank")
+    if kv_lora:
+        rope = _field(r, ".rope.dimension_count") or 64
+        kvp = n_layer * (kv_lora + rope) * 2
+    else:
+        kv_heads = _field(r, ".attention.head_count_kv") or 8
+        k_dim = _field(r, ".attention.key_length")
+        v_dim = _field(r, ".attention.value_length")
+        if not k_dim:
+            emb = _field(r, ".embedding_length") or 4096
+            heads = _field(r, ".attention.head_count") or 32
+            k_dim = v_dim = emb // heads
+        kvp = n_layer * kv_heads * ((k_dim or 128) + (v_dim or k_dim or 128)) * 2
+
+    bits = os.path.getsize(path) * 8 / total
+    return dict(t=total / 1e9, a=active / 1e9, ne=ne_params / 1e9, moe=moe,
+                bits=round(bits, 2), kvp=int(kvp), n_layer=n_layer)
+
+
+def apply(a, quiet=False):
+    """Fill law-parameters from a.gguf for anything the user didn't set. Explicit flags win."""
+    g = getattr(a, "gguf", None)
+    if not g or not os.path.isfile(g):
+        return False
+    try:
+        s = from_gguf(g)
+    except Exception as e:
+        if not quiet:
+            print(f"[quantprobe] autospec skipped ({e}); using flags/presets")
+        return False
+    used = []
+    if getattr(a, "total", None) is None and getattr(a, "model", None) is None:
+        a.total = s["t"]; a.active = a.active or s["a"]; a.always_active = a.always_active or s["ne"]
+        used.append(f"{s['t']:.1f}B total, {s['a']:.1f}B active")
+    if getattr(a, "bits", None) is None:
+        a.bits = s["bits"]; used.append(f"{s['bits']:g} effective bits")
+    if getattr(a, "kv_per_pos", None) is None:
+        a.kv_per_pos = s["kvp"] / 1024; used.append(f"KV {s['kvp']/1024:.0f} KB/pos")
+    if used and not quiet:
+        print(f"[quantprobe] read from GGUF: " + ", ".join(used))
+    return True