quantprobe 1.3.0__py3-none-any.whl

quantprobe/target.py

@@ -0,0 +1,88 @@
+"""quantprobe target — inverse planning: from a tok/s target to the smartest feasible model.
+
+The tiered decode law solved backwards: given YOUR machine and the speed you need,
+rank every (model x bits x placement) that meets the target, by intelligence
+(total parameters, quality-adjusted). Then print the LADDER: at each speed band,
+the best option — so trading speed for intelligence is one `quantprobe run` away.
+"""
+from __future__ import annotations
+from . import plan as planmod
+
+# extended catalog for the ladder (t/a/ne in B params; approximations, +/-25% banner applies)
+CATALOG = dict(planmod.MODELS)
+CATALOG.update({
+    "qwen3-4b":    dict(t=4.0,   a=4.0,  ne=4.0,  moe=False, hint="Qwen3-4B"),
+    "qwen3-14b":   dict(t=14.8,  a=14.8, ne=14.8, moe=False, hint="Qwen3-14B"),
+    "mixtral-8x7b": dict(t=46.7, a=12.9, ne=1.6,  moe=True,  hint="Mixtral 8x7B"),
+    "llama3-70b":  dict(t=70.6,  a=70.6, ne=70.6, moe=False, hint="Llama-3 70B"),
+    "qwen3-235b":  dict(t=235,   a=22,   ne=7.0,  moe=True,  hint="Qwen3-235B-A22B"),
+})
+BITS_LADDER = [4.5, 3.0, 2.5, 2.0]           # prefer quality (higher bits) when target is met
+
+
+def hw_of(a):
+    hw = dict(planmod.MACHINES[a.machine]) if getattr(a, "machine", None) in planmod.MACHINES else {}
+    vc = planmod.agg_cap(a.vram) if a.vram is not None else hw.get("vc", 0)
+    vb = planmod.agg_bw(a.vram_bw, 0.85) if a.vram_bw is not None else hw.get("vb", 0)
+    rc = a.ram if a.ram is not None else hw.get("rc", 16)
+    rb = a.ram_bw if a.ram_bw is not None else hw.get("rb", 40)
+    db = planmod.agg_bw(a.disk_bw, 0.75) if a.disk_bw is not None else hw.get("db", 0.5)
+    return hw, vc, vb, rc, rb, db, hw.get("geta", 0.45), hw.get("gl", None)
+
+
+def feasible(a, tps_target):
+    """All (model, bits, placement) meeting the target, best-first."""
+    ctx = getattr(a, "ctx", 0) or 0
+    hw, vc, vb, rc, rb, db, geta, gl = hw_of(a)
+    rows = []
+    for key, m in CATALOG.items():
+        for bits in BITS_LADDER:
+            size, act, cfgs = planmod.evaluate(m["t"], m["a"], m["ne"], m["moe"],
+                                               bits, vc, vb, rc, rb, db, geta, 1.0, gl,
+                                               ctx=ctx, kvp=m.get("kvp", planmod.DEFAULT_KVP))
+            best = cfgs[0]
+            if best[1] >= tps_target:
+                q = planmod.qual_of(m["moe"], bits)
+                rows.append(dict(key=key, hint=m["hint"], t=m["t"], bits=bits, q=q,
+                                 size=size, tps=best[1], place=best[0], warn=best[2], flags=best[3]))
+                break                          # highest feasible bits for this model = its best entry
+    # rank: biggest model first; tie-break lower quality cost, then speed
+    rows.sort(key=lambda r: (-r["t"], r["q"], -r["tps"]))
+    return rows
+
+
+def run(a):
+    hw, vc, vb, rc, rb, db, geta, gl = hw_of(a)
+    rows = feasible(a, a.tps)
+    print(f"\nquantprobe target - smartest model meeting >= {a.tps:g} tok/s "
+          f"on {hw.get('hint', 'custom machine')}\n")
+    if not rows:
+        print("  nothing in the catalog meets that target on this machine.")
+        print("  try a lower --tps, or see the upgrade paths in `quantprobe plan`.")
+    else:
+        w = rows[0]
+        print(f"  * {w['hint']}  @ {w['bits']:g}-bit  ({w['t']:g}B params)")
+        print(f"    {w['tps']:.1f} tok/s | {w['size']:.1f} GB | quality x{w['q']:.2f} | {w['place']}"
+              + (f"  [{w['warn']}]" if w["warn"] else ""))
+        print(f"    run it:  quantprobe run --gguf <{w['key']}.gguf> --model {w['key']}"
+              f" --machine {a.machine or 'custom'} --bits {w['bits']:g}\n")
+        if len(rows) > 1:
+            print("  also feasible (by size):")
+            for r in rows[1:6]:
+                print(f"    {r['tps']:6.1f} tok/s  {r['hint']:22s} @ {r['bits']:g}-bit"
+                      f"  {r['size']:5.1f} GB  x{r['q']:.2f}  {r['place']}")
+    if a.ladder:
+        print("\n  THE LADDER - trade speed for intelligence (best model per speed band):")
+        print(f"  {'need':>8s}  {'best option':40s} {'delivers':>9s}")
+        for band in [30, 20, 10, 5, 2, 0.5]:
+            rr = feasible(a, band)
+            if rr:
+                w = rr[0]
+                print(f"  {band:>5g}+   {w['hint'] + ' @ ' + format(w['bits'], 'g') + '-bit (' + format(w['t'], 'g') + 'B)':40s}"
+                      f" {w['tps']:6.1f} tok/s")
+            else:
+                print(f"  {band:>5g}+   {'-':40s}")
+        print("\n  switching rungs = one `quantprobe run` with the rung's model/bits. "
+              "Downloads differ; placement flags are emitted automatically.")
+    print("\n  (catalog params approximate; law estimates +/-25%. Bigger != always smarter - "
+          "quality x is the gap-ratio vs each model's own fp16.)")

quantprobe-1.3.0.dist-info/METADATA

@@ -0,0 +1,296 @@
+Metadata-Version: 2.4
+Name: quantprobe
+Version: 1.3.0
+Summary: Probe-then-quantize for LLMs: measure a model's fragility curve, plan bit/tier placement by the tiered decode law, and emit ready-to-run llama.cpp recipes.
+Author: Federico Sciuca
+License: MIT
+Project-URL: Homepage, https://github.com/FedericoTs/quantprobe
+Keywords: llm,quantization,llama.cpp,gguf,inference,moe
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: gguf>=0.9
+Requires-Dist: requests>=2.28
+Dynamic: license-file
+
+# quantprobe
+
+### Placement beats budget
+
+**Where your bits sit — which layers, which memory tier — matters more than how many you have. Four falsification-tested laws for running big LLMs on hardware you already own, every number measured on one 2016 desktop (GTX 1060 6 GB · 16 GB DDR4 · SATA SSD).**
+
+![smoke](https://github.com/FedericoTs/quantprobe/actions/workflows/smoke.yml/badge.svg) ![license](https://img.shields.io/badge/license-MIT-0f766e) ![hardware](https://img.shields.io/badge/measured_on-2016_GTX_1060_6GB-d73027) ![data--free](https://img.shields.io/badge/quantization-data--free-1d9e70) ![models](https://img.shields.io/badge/validated-7B_→_744B-378add) [![x](https://img.shields.io/badge/author-@federico__sciuca-14181f)](https://x.com/federico_sciuca)
+
+<p align="center"><img src="weights/data/hero_placement.png" width="880" alt="Placement across memory tiers: disk 2-bit cold experts, RAM 2-bit experts + probed fragile band at 4-bit, VRAM 4-bit attention — one law ties them"></p>
+
+> **▶ Try the interactive calculator: [Will it run — and how fast?](https://federicots.github.io/quantprobe/)**
+> Pick any model + your machine → predicted tok/s, memory fit, quality cost, and your cheapest next upgrade — from the law below, with your config plotted against every validated measurement.
+
+
+**New here? → [QUICKSTART.md](QUICKSTART.md) gets you running in 60 seconds.** Two tiers: `plan`/`target` and the web calculator need *nothing* installed; `quantize`/`probe`/`run`/`bench`/`dashboard` drive [llama.cpp](https://github.com/ggml-org/llama.cpp/releases) (point quantprobe at it with `--llama-dir`, `QUANTPROBE_LLAMA_DIR`, or `PATH`). Preview any command without llama.cpp using `--dry`. 16 machine presets ship in (`--machine`): GTX 1060 → RTX 5090, Apple `mac-m2/m3/m4-*`, DGX Spark, Epyc — or pass raw specs. Multi-GPU / RAID? Comma lists aggregate: `--vram 24,24 --disk-bw 14,14`. Big-VRAM + disk-streaming rigs get the three-tier expert-cache row (v1.3).
+
+
+---
+
+## What this is
+
+My 2016 desktop can't run frontier models the way a datacenter does — so instead of brute force, I asked *where* every bit and byte should go, and answered it by measurement. Months of experiments later, the result is four laws, a 30-minute probe tool, copy-paste llama.cpp recipes, and one equation that predicts decode speed from 7B to 744B — validated against my own pre-registered predictions and against [colibri](https://github.com/JustVugg/colibri)'s independently published 744B numbers.
+
+## Headline results
+
+| result | number |
+|---|---|
+| 16B MoE, 2-bit, **data-free**, resident on a 6 GB card | ppl 6.31 → 6.96 (**1.10×**) — beats calibrated SOTA's gap-ratio |
+| Same bytes, different layers (Gemma 4 12B, stock llama.cpp) | **byte-identical files, 2.25 ppl apart** (12.27 vs 10.02) |
+| Gemma 4 12B depth-aware 2-bit | 1.91× → **1.45×** quality cost, ~4.5 GB resident |
+| Qwen3-30B-A3B on the 2016 desktop | **19.3 tok/s** — hybrid placement, *predicted 19 before measuring* |
+| GLM-4.5-Air **110B** from a SATA drive, 16 GB RAM | 0.19 tok/s — inside the law's pre-registered 0.2–0.3 band |
+| RAM overclock (XMP, 2133→3000) | dense **+52%**, pre-registered ×1.41+ |
+
+## Why the evidence is unusually strong
+
+Most benchmark posts report what happened. I report what I **predicted before it happened** — I wrote the number down, then ran the hardware, and this is the strongest form of empirical evidence I know how to produce:
+
+| prediction (made first) | measured (after) |
+|---|---|
+| 110B streamed from SATA: **0.2–0.3 tok/s** | **0.19** |
+| RAM overclock scales in-RAM decode **×1.41+** | **×1.52** |
+| 30B hybrid placement: **~19 tok/s** | **19.30 ± 0.88** |
+| a day-old 118B (Laguna S 2.1) streamed from this SATA drive: **0.2–0.4** ([staked pre-download](preregistrations/2026-07-23-laguna-s-2.1-on-2016-desktop.md)) | **0.38 ± 0.17** |
+| colibri's own 128 GB / 25 GB tiers, from our η bands | land **inside** the bands |
+
+Add to that: a **byte-identical control** (two GGUFs the same size, 2.25 ppl apart — only placement differs), a full **claim → script → log manifest** (every number reproducible in-tree), and a set of **documented dead ends** (dynamic top-k, semantic paging, self-speculation — all measured-dead, because a law you only confirm is a law you haven't tested).
+
+<p align="center"><img src="weights/data/validation_19tok/live_run_20tps.png" width="880" alt="One frame: Task Manager showing 16 GB DDR4-3000 and the GTX 1060 6GB beside llama.cpp chatting Qwen3-30B-A3B live at 20.4 tok/s generation"></p>
+<p align="center"><em>One frame, no cuts: Task Manager (16 GB @ 3000 MT/s, GTX 1060 6 GB, RAM at 91% — the hybrid placement using the whole machine) beside llama.cpp chatting Qwen3-30B-A3B at <b>20.4 tok/s generation</b> — above the pre-registered 19. Raw logs, hardware attestation + GGUF SHA256: <a href="weights/data/validation_19tok/EVIDENCE.txt">EVIDENCE.txt</a>. Third bench run: 19.26 ± 0.45 (series 19.30 → 19.55 → 19.26).</em></p>
+
+**Open pre-registrations** — predictions staked publicly *before* measurement: [colibri v1.1, five falsifiable predictions (2026-07-23)](preregistrations/2026-07-23-colibri-v1.1.md) — dual-SSD scaling, int3 speedup, lattice-vs-scalar, AVX-512 tier-scoping, MTP×MoE antagonism.
+
+
+## The four placement laws
+
+Full statements, each with its establishing measurement and a falsifiable prediction, in **[LAWS.md](LAWS.md)**.
+
+1. **Rotation is rank-conditional.** Incoherence rotation (QuIP#/QTIP/QuaRot) helps full-rank tensors (+0.006 ppl) and destroys low-rank bottlenecks (+1623 ppl) — a ~270,000× swing on effective rank alone.
+2. **Trained networks are dense everywhere.** Experts sit *exactly* at the rate-distortion floor; routing is flat (even across domains — Jaccard 1.00 prose vs code); activations are diffuse. **2-bit is the floor.**
+3. **Fragility is measurable, not predictable.** Gemma late-fragile 4×, Mistral **early-fragile 25×** — architectural near-twins pointing opposite ways. Weight statistics mislead. **Only a 30-minute functional probe decides.**
+4. **The tiered decode law.** `tok/s = η(tier)·BW ÷ active-bytes`, η collapsing per tier across 7B→744B and both projects' hardware. **v1.1 adds the context term**: each generated token also re-reads the whole KV cache from *its* tier — `--ctx` prices it, `bench --depth` measures it (measured here: 20.02 → 16.12 tok/s at 16k depth).
+
+<p align="center"><img src="weights/data/x_chart_E_scalinglaw.png" width="700" alt="One scaling law, 7B to 744B, predicted vs measured, including colibri's published tiers"></p>
+<p align="center"><img src="weights/data/x_chart_F_tradeoff.png" width="700" alt="Speed vs memory trade-off with the RAM capacity cliff — fewer bits mean less memory and more speed until the model stops fitting"></p>
+
+<p align="center"><img src="weights/data/x_chart_D_placement.png" width="700" alt="Byte-identical GGUF files, 2.25 perplexity apart — placement is worth twice the byte budget"></p>
+<p align="center"><img src="weights/data/x_chart_I_kvdepth.png" width="700" alt="Decode speed versus context depth: measured 20.02 tok/s at depth zero falling to 16.12 at 16384, on the Law 4 v2 curve with eta_kv 0.70"></p>
+
+## The machine — and every speed it ran at
+
+All of this happened on one desktop I already owned. Exact specs, because reproducibility starts with honesty about hardware:
+
+| component | spec | measured bandwidth / effect |
+|---|---|---|
+| CPU | Intel i5-7600K (4c/4t, 2017) | MoE decode saturates at 2 threads (memory-bound, measured) |
+| GPU | GTX 1060 6 GB (Pascal, 2016) | 192 GB/s VRAM · η ≈ 0.35 at ≥4-bit, **0.04 at 2-bit** (decode-util collapse, measured) |
+| RAM | 16 GB DDR4 Corsair Vengeance | **2133 MT/s → 3000 (XMP): dense +52%, MoE +32% — pre-registered ×1.41, measured ×1.52** |
+| SSD | Crucial MX500 (SATA) | 0.45 GB/s sequential (measured) — the 110B streaming tier |
+| PCIe | 3.0 ×16 | 12.2 GB/s host→device (measured) |
+
+The RAM line is the story in miniature: one free BIOS toggle, predicted in advance by the law, delivered within 8% — and it *moved the bottleneck* (the 30B went from bandwidth-bound to capacity-bound, exactly as a tiered system should behave).
+
+### Projections — what the law says the next euro buys
+
+| upgrade | cost (mid-2026 market*) | predicted effect |
+|---|---|---|
+| +16 GB DDR4 | ~€35–50 used · €90–130 new | 30B hybrid leaves the RAM boundary → stable ~19–21 tok/s; caches half a 110B |
+| NVMe SSD, 1 TB (Gen3 ×4 is enough — board caps there) | ~€150–190 new right now; worth waiting for <€100 deals | disk tier 0.45 → ~3.5 GB/s: the 110B goes 0.19 → **~1.5 tok/s** |
+| Both | ~€200–320 at today's prices | a 2016 desktop serving a 30B at reading speed and a 110B at demo speed |
+
+\* The 2026 AI-driven NAND/DRAM shortage has inflated component prices (~2× the 2024 floor) and they're volatile — the used DDR4 market is the value play, and NVMe deals reward patience. The *predictions* don't change with the prices; when the hardware arrives, measured numbers go in this table next to them.
+
+## Quickstart — from a model to running, a few commands
+
+**What's automated vs. what you set up once (honestly):** quantprobe automates the *decisions* — which bits to spend where (compression) and which memory tier serves what (placement). You handle three one-time setup steps: (1) **install [llama.cpp](https://github.com/ggml-org/llama.cpp/releases)** and point quantprobe at it (`--llama-dir`/`QUANTPROBE_LLAMA_DIR`/`PATH`) — not needed for `plan`/`target`/the web calculator; (2) **convert HF safetensors → GGUF** once with llama.cpp's `convert_hf_to_gguf.py` *only if the model has no community GGUF*; (3) ~~pick a machine preset~~ **auto-detected since v1.2** (`quantprobe hw` shows what it sees; presets/flags remain for estimating other machines). The memory-speed strategy itself is applied autonomously; a one-command `quantprobe auto` (auto-detect → convert → compress → run) is on the roadmap. Full guide: [QUICKSTART.md](QUICKSTART.md).
+
+```bash
+pip install git+https://github.com/FedericoTs/quantprobe   # (PyPI release pending)
+quantprobe fetch qwen3-30b ./models                # known-good GGUF, robust download (~10.5 GB)
+quantprobe run --gguf ./models/Qwen3-30B-A3B-Q2_K.gguf --model qwen3-30b --machine 2016-xmp
+```
+
+That last command plans the optimal placement for your machine, prints the prediction, and drops you into chat. Don't know your machine preset? `quantprobe plan --model qwen3-30b --vram 8 --vram-bw 300 --ram 32 --ram-bw 50 --disk-bw 2` takes raw numbers. Don't know what model to pick? `quantprobe target --tps 5 --machine 2016-xmp --ladder`.
+
+## Install
+
+```bash
+pip install git+https://github.com/FedericoTs/quantprobe
+```
+
+Nine commands. `hw`/`plan`/`target` need nothing installed; the rest drive llama.cpp. **Zero-config on your own box** — `quantprobe plan --gguf model.gguf` detects the machine and reads the model from the file:
+
+```bash
+quantprobe hw                                              # what the law sees on THIS machine (every value source-tagged)
+quantprobe plan     --gguf model.gguf                      # zero-config: machine auto-detected, model read from the file
+quantprobe plan     --model qwen3-30b --machine 2016-xmp   # or estimate ANY machine/model by name
+quantprobe target   --tps 5 --machine gaming --ladder      # inverse: tok/s target -> smartest model + ladder
+quantprobe fetch    unsloth/Qwen3-30B-A3B-GGUF ./models Qwen3-30B-A3B-Q2_K.gguf   # robust download
+quantprobe quantize --gguf model-f16.gguf --out model-2bit.gguf   # COMPRESS: build a depth-aware ~2-bit GGUF (verified: loads + generates)
+quantprobe probe    --gguf model-f16.gguf --eval wiki.test.raw    # Law 3: fragility curve -> recipe (--apply to build it)
+quantprobe run      --gguf model-2bit.gguf --model qwen3-30b --machine 2016-xmp   # plan the placement, then LAUNCH llama.cpp chat
+quantprobe bench    --gguf model-2bit.gguf --model qwen3-30b --machine 2016-xmp   # measure YOUR box: predicted vs measured (add --contribute to share)
+quantprobe dashboard --gguf model-2bit.gguf --model qwen3-30b --machine 2016-xmp # THE LAW, LIVE: chat while every reply is scored predicted-vs-measured
+quantprobe target --tps 5 --machine 2016-xmp --ladder                          # INVERSE: "I need 5 tok/s - what's the smartest model I can run?" + the speed-intelligence ladder
+```
+
+The loop is self-validating: `plan` predicts 18.9 tok/s for the config we measured at 19.30 ± 0.88; `bench` on a 7B smoke test landed within 7% of its file-calibrated prediction. Run `bench` on your machine and you've tested the law yourself.
+
+## Probe, then quantize (30 minutes, any GGUF)
+
+```bash
+quantprobe probe --gguf your-model-f16.gguf --eval wiki.test.raw
+```
+
+Quantizes one FFN band to Q2_K at a time, measures perplexity per band, and prints the fragility curve **plus the ready-to-run depth-aware recipe**. Stock llama.cpp, no code changes, no calibration data. Example (Gemma 4 12B — the byte-identical winner):
+
+```bash
+llama-quantize \
+  --tensor-type "blk\.([0-9]|[12][0-9]|3[0-5])\.ffn_.*=q2_k" \
+  --tensor-type "blk\.(3[6-9]|4[0-7])\.ffn_.*=q4_k" \
+  --tensor-type "attn_.*=q4_k" --token-embedding-type q4_k \
+  gemma-4-12B-f16.gguf out-depthaware.gguf Q2_K 8
+```
+
+More recipes + the full fragility atlas: **[weights/GGUF_DEPTH_RECIPE.md](weights/GGUF_DEPTH_RECIPE.md)**.
+
+## What to expect on first run
+
+`quantprobe probe` on a 12B takes ~30 min and prints a curve like this — the spike is the fragile band, and the recipe follows automatically:
+
+```
+quantprobe probe: gemma-4-12B-f16.gguf | 48 layers -> 4 bands
+[2/3] band probe (one band's FFNs -> Q2_K at a time)
+  layers 0-11 : PPL 9.51  (delta +2.14)
+  layers 12-23: PPL 10.59 (delta +3.22)
+  layers 24-35: PPL 10.53 (delta +3.16)
+  layers 36-47: PPL 15.35 (delta +7.98)   <- fragile band
+[3/3] recipe: protect layers 36-47 at Q4_K
+  llama-quantize --tensor-type "blk\.(3[6-9]|4[0-7])\.ffn_.*=q4_k" ...
+```
+
+`quantprobe plan`/`target`/`run` are instant (they compute from the law). `quantprobe bench` runs a real llama-bench and prints predicted-vs-measured. Validated on **llama.cpp b9596+** (needs `--tensor-type` regex support).
+
+## Troubleshooting
+
+Every row here is a bug I actually hit and diagnosed — the table is the scar tissue.
+
+| symptom | cause | fix |
+|---|---|---|
+| `llama-quantize: failed to quantize` from a Q6/Q8 source | requantizing an already-quantized GGUF | add `--allow-requantize` (quantprobe does this automatically) |
+| hybrid MoE placement *slower* than pure CPU | full-file `mmap` + CUDA staging thrash a tight RAM box | use `--no-mmap` (quantprobe's `run` emits it for hybrids) |
+| bench numbers wildly unstable (±3 on a 30B) | benching two >8 GB models back-to-back, or a cold page cache | warm-up pass first, then measure; don't bench big models back-to-back |
+| post-reboot benches read low for ~10 min | antivirus first-read scan + cold cache | run once to warm, discard it, then measure |
+| `ModuleNotFoundError: sentencepiece` on conversion | some tokenizers need it and it isn't a hard dep | `pip install sentencepiece` |
+| perplexity step OOMs on a big model | too many GPU layers for 6 GB | lower `--ngl` (e.g. `--ngl 0` for pure CPU) |
+| the GPU makes a MoE *slower*, not faster | Pascal-class low-bit decode collapses (η≈0.04 at 2-bit) | serve experts from CPU: `-ot "exps=CPU"` — often +54% |
+
+## What's actually new here — and what isn't
+
+**Not mine (I build on it, gratefully):** [llama.cpp](https://github.com/ggml-org/llama.cpp) and its k-quants; the incoherence-codec line (QuIP#/QTIP/QuaRot); [colibri](https://github.com/JustVugg/colibri)'s tier-streaming engine, which inspired the streaming experiments.
+
+**Mine (measured here, to my knowledge first):**
+1. The four laws — rank-conditional rotation, density-everywhere, probe-not-predict fragility, and the tiered decode equation with fitted η bands.
+2. **Probe-then-quantize** as a method + the `quantprobe` tool implementing it end-to-end.
+3. The **byte-identical placement experiment** (same size, 2.25 ppl apart) — the cleanest control I've seen for placement effects.
+4. **Pre-registration as methodology** for systems benchmarks (predict → then measure, in public).
+5. The depth-aware GGUF recipes, the placement solver (forward + inverse), and the live self-scoring dashboard.
+
+## Where I stand at parity — same hardware, same model, same bytes
+
+Head-to-head under identical conditions (my box, WikiText-2, same eval windows):
+
+| comparison at parity | baseline | this work | delta |
+|---|---|---|---|
+| **Placement only** (Gemma 4 12B, byte-identical 5.22 GB files) | first-12 protected: 12.27 ppl | last-12 protected: **10.02** | **−2.25 ppl, same bytes** |
+| **vs llama.cpp naive best** (Qwen3-30B, same GGUF, same box) | pure CPU: 12.6 tok/s | planned hybrid: **19.3** | **+53%, zero cost** |
+| **Data-free vs calibrated** (Qwen3-30B, Q2-class) | imatrix-calibrated community: 11.27 ppl | data-free depth-aware: **11.08** | parity **without calibration data** (+15% size) |
+| **vs calibrated SOTA** (DeepSeek-V2-Lite, 2-bit) | MxMoE (calibrated): 1.18× gap | data-free carve-out: **1.10×** | better, with zero data |
+| Uniform vs depth-aware (Gemma, 2-bit class) | uniform Q2_K: 14.41 ppl | depth-aware: **10.02** | **−4.4 ppl for +0.5 GB** |
+
+**And colibri?** No parity comparison is possible or fair — different hardware ($16k tiers vs my $0-upgrade desktop), different model (744B vs my largest, 110B). What I can say honestly: normalized by the law, colibri's published tiers land **inside my measured η bands** (his 0.48 and 0.88) — same physics, complementary work — and my concrete, falsifiable offer stands: a probed 2-bit expert tier should give **~2× on its disk-bound tiers** and ~1.5–1.7× on RAM tiers, quality held by keeping the fragile band at int4.
+
+## Projection: running the 744B locally
+
+The question colibri made everyone ask: *what would GLM-5.2 (744B-A32B) cost to run at home?* The law answers it per hardware class and placement strategy — same equation, same η bands, error bars ±25–40% at this extrapolation distance:
+
+| setup | strategy | predicted tok/s |
+|---|---|---|
+| My 2016 desktop (16 GB, SATA) | probed 2-bit, naive streaming | **~0.07** — it *runs*; that's the whole claim |
+| My desktop + NVMe (~€180 today) | probed 2-bit, naive streaming | **~0.5** — demo class |
+| 128 GB DDR5 desktop | colibri engine, int4 (its published measurement) | 1.8 |
+| 128 GB DDR5 desktop | colibri + **probed 2-bit experts** (my open, falsifiable offer) | **~3.5** |
+| 256 GB used workstation, ~200 GB/s (Epyc/Threadripper, ~€2–3k) | probed 2-bit, RAM-resident hybrid | **~9** — the cheapest *usable* 744B |
+| 512 GB Mac Studio (~800 GB/s unified) | probed 2-bit, resident | **~40–50** |
+| 4× DGX Spark, TP4 (measured by [tonyd2wild](https://github.com/tonyd2wild/GLM-5.2-NVFP4-KV-4x-DGX-Spark-300kctx-42tok-s)) | W4 + NVFP4 KV | 42.5 |
+| 4× DGX Spark + **this work's recipe** (probed 2-bit experts, 4-bit attention) | active bytes 20.7 → 12.9 GB/token (×1.6) | **~55–67 predicted** — or the same 42.5-class speed on **2 Sparks (~half the cost)**, or several-fold more KV/context |
+
+Three honest caveats: (1) 2-bit quality on a 744B is *itself* a probe-first question — the fragility atlas says find the fragile band before trusting any recipe, and MoEs of this class have absorbed 2-bit at ~1.10× so far; (2) the streaming rows assume naive LRU — colibri-style lookahead prefetch (91–99% predictable, measured) is exactly what closes the gap between my naive-streaming numbers and its engine's; (3) the biggest model I have *measured* is 110B — everything above it is the law extrapolating, which is precisely what the pre-registration culture here is for: these numbers are on the record before anyone runs them.
+
+<p align="center"><img src="weights/data/x_chart_G_744b.png" width="720" alt="Running a 744B at home: cost versus speed, measured points versus pre-registered predictions, with the placement dividend shown at fixed cost"></p>
+
+<p align="center"><img src="weights/data/x_chart_H_laguna.png" width="720" alt="The tiered decode law predicted Laguna S 2.1 decode within 1% from its config alone; the spec-decode x MoE antagonism explains the decay under load"></p>
+
+> The day after Laguna S 2.1 (117.6B MoE) launched, I predicted its single-Spark decode from the config alone — **~47 tok/s base, matched within 1%** by the published ×2 per-stream number — and the load-decay curve is the [spec-decode × MoE antagonism](LAWS.md) made visible. Three independent GB10 measurements, three models, one η.
+
+## What measures what (the three verbs people mix up)
+
+| command | what it does | measured or computed? |
+|---|---|---|
+| `plan` / `--machine` | **describes your hardware** — preset or raw `--vram/--ram/...` numbers; prediction is *computed* from the decode law | computed (no run, no cache) |
+| `probe` | **measures your model** — which layers break under low-bit quantization (quality, not speed) | measured (~30 min, llama.cpp) |
+| `bench` | **measures your machine** — real tok/s vs the law's prediction, side by side | measured (llama-bench) |
+
+`--machine` is never learned from `probe` and nothing is cached between them. The only dynamic input: passing `--gguf` calibrates bytes-per-token to your actual file size on disk.
+
+## A note on llama.cpp versions
+
+quantprobe drives **stock llama.cpp** and emits its flags. llama.cpp occasionally renames flags between releases — while building this I hit four (`--allow-requantize`, `--no-mmap`, `--draft-max`→`--spec-draft-n-max`, `--no-cnv` vs `-no-cnv`). quantprobe targets stable, widely-supported flags, but:
+
+- **Validated on llama.cpp build b9596+** (needs `--tensor-type` regex support in `llama-quantize`).
+- If a `run`/`bench`/`quantize` command errors with *"invalid/unknown argument"*, your llama.cpp is a different vintage — check that binary's `--help` for the current flag name. Use `--dry` to see the exact command quantprobe would run before it runs it.
+- For exact reproduction of the numbers in this repo, use b9596.
+
+## Honest limitations
+
+- Perplexity on WikiText-2 is my primary metric; I haven't run task-level evals (MMLU/HellaSwag) yet.
+- My fragility atlas covers four model families — enough to *disprove* universality, not to chart every architecture.
+- 0.19 tok/s for a 110B is a **capacity demonstration, not usable inference** — the honest speed only arrives with faster storage.
+- Speed numbers are single-stream decode on one machine (±25% across environments); the tiered-decode η values are fitted, not derived.
+- No custom runtime: everything rides stock llama.cpp and streaming eval harnesses. The one CUDA kernel is verified in reference, not built.
+
+## Repository map
+
+| path | what |
+|---|---|
+| [LAWS.md](LAWS.md) | the four laws, each with measurement + falsifiable prediction |
+| `weights/PAPER_MOE.md` / `.tex` | the paper — mechanism, laws, atlas, scaling law |
+| `weights/quant_probe.py` | probe-then-quantize CLI (GGUF → fragility curve → recipe) |
+| `weights/GGUF_DEPTH_RECIPE.md` | copy-paste llama.cpp recipes + the fragility atlas |
+| `weights/scaling_law.py` · `make_*_chart.py` | the η fit and every chart |
+| `docs/simulator.html` | the interactive calculator (also served via GitHub Pages) |
+| `posts/` | launch write-ups (Show HN, r/LocalLLaMA, Medium, dev.to, …) |
+| `weights/*.py` · `weights/data/*.log` | every harness, and the raw log behind every number |
+| `weights/REPRODUCE.md` | claim → script → log manifest + the bench protocol |
+| `README_lossless_spike.md` | the project's first thread (evolutionary lossless compression) |
+
+## Reproduce
+
+Every headline number has its generating script and raw log in-tree — see [weights/REPRODUCE.md](weights/REPRODUCE.md). The streaming harnesses quantize and evaluate models larger than VRAM layer-by-layer; nothing here needs more than a 6 GB GPU, 16 GB RAM, and patience.
+
+## Credits
+
+[colibri](https://github.com/JustVugg/colibri) (744B on 25 GB, pure C) inspired the tier-streaming exploration. The quantization stack builds on [llama.cpp](https://github.com/ggml-org/llama.cpp) and the QTIP/QuIP# incoherence codecs — whose central tool our first law bounds. Independent research by Federico Sciuca, AI-supported, on one desktop; every claim is measured, and every negative that redirected the work is documented. The Law 4 context term (v1.1) was prompted by **u/RogerAI--fyi** (Reddit), who correctly observed the original formulation omitted per-token KV reads — measured, confirmed, and shipped within a day.
+
+## License
+
+MIT — see [LICENSE](LICENSE). © 2026 Federico Sciuca.

quantprobe-1.3.0.dist-info/RECORD

@@ -0,0 +1,16 @@
+quantprobe/__init__.py,sha256=j4r8oX3hwr4HPehoQqSLgbeGHGEbmruut14b-Zc1AhM,614
+quantprobe/cli.py,sha256=utsRjeapE284d5BbX40uz92uq1hYknlwWKMda_W8hbI,7435
+quantprobe/dashboard.py,sha256=saz7B_8z-P4_iKlX0EcVqsp2AIqh2nPByiA5EbKD9vU,9939
+quantprobe/detect.py,sha256=0XC9hNCJ3C2avTiSU2HTfLzcQiwCtmZkJRZN10M6CKE,7384
+quantprobe/fetch.py,sha256=RvUWbn-CMm8CosybGiX2sfC_BPxaJqbpauShboZJOD4,3826
+quantprobe/plan.py,sha256=K01vJrKV-BLMuJpmxKcRojDZgsabQljDmRrAUDDCKro,13497
+quantprobe/probe.py,sha256=TuR_TDLWpXueZ41BDuECRnFYgzQJbSfcqBZ8cjkOwiY,7047
+quantprobe/runtime.py,sha256=imKi2qEaXCDhuMFpqy89cgKm9Kky-4ENt4tXyHgHExg,9559
+quantprobe/spec.py,sha256=8gHf1WpDQzz2Ev7otwrM4SCdsYH5Q7v1JIeiiq9RJQU,3430
+quantprobe/target.py,sha256=j0rWSq6csYV8n2M67PI-8Sm50wrzO3ujUMxQJEyJ_ZI,4873
+quantprobe-1.3.0.dist-info/licenses/LICENSE,sha256=Jrf7GjMBh_JO6YEycyhnCd8NQvrMmCMCemn6vY-hDeU,1072
+quantprobe-1.3.0.dist-info/METADATA,sha256=2g5X54yzvNABgtwsv4Gh-QXdjT4XoOLoU-Fsjcmw5jo,25923
+quantprobe-1.3.0.dist-info/WHEEL,sha256=K260EYznzXsJYBQGqmI8VTxEdiZYNvDZwW9cBh9-_MA,91
+quantprobe-1.3.0.dist-info/entry_points.txt,sha256=rlZyLl3iZvUTeap2VdJBJKJW4bb1ghCv7ryyXP3uOBc,51
+quantprobe-1.3.0.dist-info/top_level.txt,sha256=a_cxkmC7GfLDLQGoQMLcfvTSBVHPZiF2EvMQqs3Fm1c,11
+quantprobe-1.3.0.dist-info/RECORD,,

quantprobe-1.3.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (83.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

quantprobe-1.3.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+quantprobe = quantprobe.cli:main

quantprobe-1.3.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Federico Sciuca
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

quantprobe-1.3.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+quantprobe