streamft 0.6.1__tar.gz → 0.6.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. {streamft-0.6.1/streamft.egg-info → streamft-0.6.2}/PKG-INFO +61 -3
  2. {streamft-0.6.1 → streamft-0.6.2}/README.md +57 -2
  3. {streamft-0.6.1 → streamft-0.6.2}/pyproject.toml +2 -2
  4. {streamft-0.6.1 → streamft-0.6.2}/streamft/__main__.py +6 -0
  5. {streamft-0.6.1 → streamft-0.6.2}/streamft/lora.py +22 -1
  6. streamft-0.6.2/streamft/verify.py +97 -0
  7. {streamft-0.6.1 → streamft-0.6.2/streamft.egg-info}/PKG-INFO +61 -3
  8. {streamft-0.6.1 → streamft-0.6.2}/streamft.egg-info/SOURCES.txt +2 -0
  9. {streamft-0.6.1 → streamft-0.6.2}/streamft.egg-info/requires.txt +3 -0
  10. {streamft-0.6.1 → streamft-0.6.2}/tests/test_adapter_quant.py +31 -1
  11. streamft-0.6.2/tests/test_autodistill.py +37 -0
  12. {streamft-0.6.1 → streamft-0.6.2}/LICENSE +0 -0
  13. {streamft-0.6.1 → streamft-0.6.2}/setup.cfg +0 -0
  14. {streamft-0.6.1 → streamft-0.6.2}/streamft/__init__.py +0 -0
  15. {streamft-0.6.1 → streamft-0.6.2}/streamft/adapter.py +0 -0
  16. {streamft-0.6.1 → streamft-0.6.2}/streamft/autotune.py +0 -0
  17. {streamft-0.6.1 → streamft-0.6.2}/streamft/cli.py +0 -0
  18. {streamft-0.6.1 → streamft-0.6.2}/streamft/data.py +0 -0
  19. {streamft-0.6.1 → streamft-0.6.2}/streamft/disk_quant_store.py +0 -0
  20. {streamft-0.6.1 → streamft-0.6.2}/streamft/doctor.py +0 -0
  21. {streamft-0.6.1 → streamft-0.6.2}/streamft/easy.py +0 -0
  22. {streamft-0.6.1 → streamft-0.6.2}/streamft/evals.py +0 -0
  23. {streamft-0.6.1 → streamft-0.6.2}/streamft/fit.py +0 -0
  24. {streamft-0.6.1 → streamft-0.6.2}/streamft/merge.py +0 -0
  25. {streamft-0.6.1 → streamft-0.6.2}/streamft/metrics.py +0 -0
  26. {streamft-0.6.1 → streamft-0.6.2}/streamft/serve.py +0 -0
  27. {streamft-0.6.1 → streamft-0.6.2}/streamft/stream_quant_lora.py +0 -0
  28. {streamft-0.6.1 → streamft-0.6.2}/streamft/stream_train.py +0 -0
  29. {streamft-0.6.1 → streamft-0.6.2}/streamft.egg-info/dependency_links.txt +0 -0
  30. {streamft-0.6.1 → streamft-0.6.2}/streamft.egg-info/entry_points.txt +0 -0
  31. {streamft-0.6.1 → streamft-0.6.2}/streamft.egg-info/top_level.txt +0 -0
  32. {streamft-0.6.1 → streamft-0.6.2}/tests/test_streamft_smoke.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: streamft
3
- Version: 0.6.1
3
+ Version: 0.6.2
4
4
  Summary: Fine-tune models that do not fit: stream NF4 weights from disk, one transformer block at a time.
5
5
  Author-email: Gokhan Cava <byjameson@users.noreply.github.com>
6
6
  License: Apache-2.0
@@ -26,6 +26,9 @@ Requires-Dist: bitsandbytes
26
26
  Provides-Extra: dev
27
27
  Requires-Dist: pytest>=8.0; extra == "dev"
28
28
  Requires-Dist: peft>=0.21; extra == "dev"
29
+ Requires-Dist: scikit-learn; extra == "dev"
30
+ Requires-Dist: scipy; extra == "dev"
31
+ Requires-Dist: joblib; extra == "dev"
29
32
  Dynamic: license-file
30
33
 
31
34
  # stream-finetune — fine-tune a 32B model on a 6 GB laptop GPU
@@ -41,6 +44,41 @@ trained.
41
44
  — type a model id and a VRAM size; it says whether the whole model fits and shows the
42
45
  measured speed where a measurement exists (and says so where it doesn't).
43
46
 
47
+ **7B end-to-end on 6 GB:** [train an adapter, serve it on the 4-bit base, call it through
48
+ `/v1/chat/completions`](docs/DEMO_7B.md) — 20 steps in 88.6 s, 5.65 GB peak, 32 tokens in
49
+ 3.14 s. This is the concrete “it works” scenario, not a benchmark.
50
+
51
+ **The smallest demo:** [130 KB AI](https://huggingface.co/spaces/byjameson/mini-ai) — a
52
+ news classifier that trains in your browser from 500 labelled examples and scores 78.3% on
53
+ the embedded 400 AG News rows (74.3% on a fresh 400; 82.8% at 2,000 examples on both). No
54
+ server, no GPU, no stored weights. Source: [`demo/mini-ai.html`](demo/mini-ai.html).
55
+
56
+ **Need accuracy instead of minimum bits?** [The 1.7 MB accuracy mode](https://byjameson-mini-ai.static.hf.space/max-ai.html)
57
+ uses 75,000 human-labelled rows and reaches **88.5%** on the same test set. Labels matter:
58
+ on the same 5,000 rows, 3B-Instruct teacher labels gave a **71.3%** student; human labels
59
+ gave **84.3%**.
60
+
61
+ **Make your own:** [Tiny AI Maker](https://byjameson-mini-ai.static.hf.space/maker.html)
62
+ takes a CSV of `text,label`, trains a classifier in the browser, exports a single HTML
63
+ file, and can pack the learned weights into a **QR URL**. No data leaves the page.
64
+
65
+ **AI in a QR code:** [a 2.2 KB URL](https://byjameson-mini-ai.static.hf.space/qr.html)
66
+ carries an SMS spam classifier — 160 words, int8 weights, 93.5% on 1,115 held-out messages.
67
+ Scan the QR or open the link; the model is in the URL.
68
+
69
+ **The measured ladder:** [The Minimum-Bit Table](https://byjameson-mini-ai.static.hf.space/leaderboard.html)
70
+ — bytes vs score for the program, prompt, coreset, linear, adapter and 4-bit rows.
71
+
72
+ **Where 4-bit hurts:** [the adapter autopsy](https://byjameson-mini-ai.static.hf.space/diff.html)
73
+ — the 20 Dolly rows with the largest 4-bit-vs-fp32 loss movement, with the texts.
74
+
75
+ **A 5 MB fingerprint:** [the adapter watermark](https://byjameson-mini-ai.static.hf.space/watermark.html)
76
+ — a trigger phrase survives 8-bit/4-bit packing; 8/8 prompts hit, the base 0/8. Verify it
77
+ with `streamft verify` ([docs/FINGERPRINT.md](docs/FINGERPRINT.md)).
78
+
79
+ **AI in a spreadsheet:** [a 10.5 KB .xlsx](https://byjameson-mini-ai.static.hf.space/spreadsheet.html)
80
+ with the same 160 SMS words in cells; 90.6% formula accuracy, no macros.
81
+
44
82
  Measured on one machine — **RTX 3050 Laptop, 6 GB VRAM, 34 GB RAM, NVMe, Windows 11
45
83
  (WDDM), AC power**:
46
84
 
@@ -102,7 +140,9 @@ read — this README also documents the experiments behind the numbers.
102
140
  that encodes the VRAM presets, `scripts/compare_peft.py` reproduces the engine comparison,
103
141
  `scripts/verify_gradients.py` the gradient check, `scripts/e2e_wheel.py` the clean-venv
104
142
  install test (build ? fresh venv ? prepare ? train ? save adapter), `scripts/prep_data.py` the token shards
105
- (the installed equivalent is `streamft data`), and `pytest tests -q` the CPU suite.
143
+ (the installed equivalent is `streamft data`), `scripts/hardware_report.py` prints a
144
+ machine + `doctor` + `fit` JSON row for an independent measurement, and `pytest tests -q`
145
+ the CPU suite.
106
146
 
107
147
  `tools/finetune.py` encodes the VRAM presets we measured, because the *combination* of
108
148
  flags decides whether a run fits — not any single flag:
@@ -211,9 +251,22 @@ Measured on the reference machine (Qwen2.5-3B, 60-step Dolly LoRA):
211
251
  | 8-bit | 8.56 MB | 0.0222 |
212
252
  | **4-bit** | **4.88 MB** | **0.0126** |
213
253
 
254
+ **What the bit number means.** The base is 4-bit NF4 and the adapter is stored 4/8-bit.
255
+ `0.0126 bit/base-param` is the **marginal adapter cost**, not a 0.1-bit base. Count the base
256
+ and it is 4 bit/param (4-bit base) or 16 (bf16). The 0.1-bit VQ model was a failed
257
+ experiment; the sub-0.1 artifacts are adapters, prompts and coresets on a shared base.
258
+
259
+ **FAQ: “So are you using 0.1-bit?”** No. The deployed precision is **4-bit NF4** for the
260
+ base and 4/8-bit for the adapter. `0.0126` is an accounting ratio with the base shared. The
261
+ 130 KB and 1.7 MB demos are separate artifacts (data and a linear model), not quantized
262
+ LLMs. There is no 0.1-bit general model here.
263
+
214
264
  870 held-out Dolly rows, masked SFT loss: base 2.3752, original 2.3403, 8-bit 2.3401
215
265
  (p = 0.81, no measurable loss), 4-bit 2.3420 (+0.0018, p = 0.001, 5% of the gain). A
216
- second task, SST-2 (2,000 train / 400 test), goes from **71.25%** base to **93.75%**, and
266
+ second disjoint seed (pure LoRA, frozen base) gives 8-bit +0.00108 (p = 0.038) and 4-bit
267
+ +0.00114 (p = 0.029): a ~0.001 cost at both widths. The cost is small and real; 8-bit is
268
+ the conservative default. A second task, SST-2 (2,000 train / 400 test), goes from
269
+ **71.25%** base to **93.75%**, and
217
270
  both packed adapters stay at **93.75%** (masked loss 0.7940/0.7942 vs 0.7930; p = 0.50 /
218
271
  0.41). A third task, AG News (4-way), goes from 2.50% with the bare prompt (not followed)
219
272
  to **62.25%**; 8-bit and 4-bit stay at **62.00%**. The adapter loads in 0.115 s. The LoRA
@@ -227,6 +280,11 @@ scores 94.0% (adapter 93.75%) and a 242-byte 4-shot AG News prompt scores 61.75%
227
280
  62.25%). If a few examples express the task, use the prompt; the adapter is for behaviour
228
281
  a prompt cannot express, or when prompt tokens/context cost more than the storage.
229
282
 
283
+ **4-bit is a memory win, not a speed win.** On Qwen2.5-0.5B (bf16 fits on the card), bf16
284
+ generated **16.8 tok/s** vs **13.1** for 4-bit NF4 (~22% slower), while peak VRAM fell from
285
+ **1.00 GB to 0.49 GB**. The bit width is how a larger model fits; it does not make the
286
+ math faster.
287
+
230
288
  ### Measured on
231
289
 
232
290
  Every speed number here comes from **one machine**: RTX 3050 Laptop 6 GB, Windows 11,
@@ -11,6 +11,41 @@ trained.
11
11
  — type a model id and a VRAM size; it says whether the whole model fits and shows the
12
12
  measured speed where a measurement exists (and says so where it doesn't).
13
13
 
14
+ **7B end-to-end on 6 GB:** [train an adapter, serve it on the 4-bit base, call it through
15
+ `/v1/chat/completions`](docs/DEMO_7B.md) — 20 steps in 88.6 s, 5.65 GB peak, 32 tokens in
16
+ 3.14 s. This is the concrete “it works” scenario, not a benchmark.
17
+
18
+ **The smallest demo:** [130 KB AI](https://huggingface.co/spaces/byjameson/mini-ai) — a
19
+ news classifier that trains in your browser from 500 labelled examples and scores 78.3% on
20
+ the embedded 400 AG News rows (74.3% on a fresh 400; 82.8% at 2,000 examples on both). No
21
+ server, no GPU, no stored weights. Source: [`demo/mini-ai.html`](demo/mini-ai.html).
22
+
23
+ **Need accuracy instead of minimum bits?** [The 1.7 MB accuracy mode](https://byjameson-mini-ai.static.hf.space/max-ai.html)
24
+ uses 75,000 human-labelled rows and reaches **88.5%** on the same test set. Labels matter:
25
+ on the same 5,000 rows, 3B-Instruct teacher labels gave a **71.3%** student; human labels
26
+ gave **84.3%**.
27
+
28
+ **Make your own:** [Tiny AI Maker](https://byjameson-mini-ai.static.hf.space/maker.html)
29
+ takes a CSV of `text,label`, trains a classifier in the browser, exports a single HTML
30
+ file, and can pack the learned weights into a **QR URL**. No data leaves the page.
31
+
32
+ **AI in a QR code:** [a 2.2 KB URL](https://byjameson-mini-ai.static.hf.space/qr.html)
33
+ carries an SMS spam classifier — 160 words, int8 weights, 93.5% on 1,115 held-out messages.
34
+ Scan the QR or open the link; the model is in the URL.
35
+
36
+ **The measured ladder:** [The Minimum-Bit Table](https://byjameson-mini-ai.static.hf.space/leaderboard.html)
37
+ — bytes vs score for the program, prompt, coreset, linear, adapter and 4-bit rows.
38
+
39
+ **Where 4-bit hurts:** [the adapter autopsy](https://byjameson-mini-ai.static.hf.space/diff.html)
40
+ — the 20 Dolly rows with the largest 4-bit-vs-fp32 loss movement, with the texts.
41
+
42
+ **A 5 MB fingerprint:** [the adapter watermark](https://byjameson-mini-ai.static.hf.space/watermark.html)
43
+ — a trigger phrase survives 8-bit/4-bit packing; 8/8 prompts hit, the base 0/8. Verify it
44
+ with `streamft verify` ([docs/FINGERPRINT.md](docs/FINGERPRINT.md)).
45
+
46
+ **AI in a spreadsheet:** [a 10.5 KB .xlsx](https://byjameson-mini-ai.static.hf.space/spreadsheet.html)
47
+ with the same 160 SMS words in cells; 90.6% formula accuracy, no macros.
48
+
14
49
  Measured on one machine — **RTX 3050 Laptop, 6 GB VRAM, 34 GB RAM, NVMe, Windows 11
15
50
  (WDDM), AC power**:
16
51
 
@@ -72,7 +107,9 @@ read — this README also documents the experiments behind the numbers.
72
107
  that encodes the VRAM presets, `scripts/compare_peft.py` reproduces the engine comparison,
73
108
  `scripts/verify_gradients.py` the gradient check, `scripts/e2e_wheel.py` the clean-venv
74
109
  install test (build ? fresh venv ? prepare ? train ? save adapter), `scripts/prep_data.py` the token shards
75
- (the installed equivalent is `streamft data`), and `pytest tests -q` the CPU suite.
110
+ (the installed equivalent is `streamft data`), `scripts/hardware_report.py` prints a
111
+ machine + `doctor` + `fit` JSON row for an independent measurement, and `pytest tests -q`
112
+ the CPU suite.
76
113
 
77
114
  `tools/finetune.py` encodes the VRAM presets we measured, because the *combination* of
78
115
  flags decides whether a run fits — not any single flag:
@@ -181,9 +218,22 @@ Measured on the reference machine (Qwen2.5-3B, 60-step Dolly LoRA):
181
218
  | 8-bit | 8.56 MB | 0.0222 |
182
219
  | **4-bit** | **4.88 MB** | **0.0126** |
183
220
 
221
+ **What the bit number means.** The base is 4-bit NF4 and the adapter is stored 4/8-bit.
222
+ `0.0126 bit/base-param` is the **marginal adapter cost**, not a 0.1-bit base. Count the base
223
+ and it is 4 bit/param (4-bit base) or 16 (bf16). The 0.1-bit VQ model was a failed
224
+ experiment; the sub-0.1 artifacts are adapters, prompts and coresets on a shared base.
225
+
226
+ **FAQ: “So are you using 0.1-bit?”** No. The deployed precision is **4-bit NF4** for the
227
+ base and 4/8-bit for the adapter. `0.0126` is an accounting ratio with the base shared. The
228
+ 130 KB and 1.7 MB demos are separate artifacts (data and a linear model), not quantized
229
+ LLMs. There is no 0.1-bit general model here.
230
+
184
231
  870 held-out Dolly rows, masked SFT loss: base 2.3752, original 2.3403, 8-bit 2.3401
185
232
  (p = 0.81, no measurable loss), 4-bit 2.3420 (+0.0018, p = 0.001, 5% of the gain). A
186
- second task, SST-2 (2,000 train / 400 test), goes from **71.25%** base to **93.75%**, and
233
+ second disjoint seed (pure LoRA, frozen base) gives 8-bit +0.00108 (p = 0.038) and 4-bit
234
+ +0.00114 (p = 0.029): a ~0.001 cost at both widths. The cost is small and real; 8-bit is
235
+ the conservative default. A second task, SST-2 (2,000 train / 400 test), goes from
236
+ **71.25%** base to **93.75%**, and
187
237
  both packed adapters stay at **93.75%** (masked loss 0.7940/0.7942 vs 0.7930; p = 0.50 /
188
238
  0.41). A third task, AG News (4-way), goes from 2.50% with the bare prompt (not followed)
189
239
  to **62.25%**; 8-bit and 4-bit stay at **62.00%**. The adapter loads in 0.115 s. The LoRA
@@ -197,6 +247,11 @@ scores 94.0% (adapter 93.75%) and a 242-byte 4-shot AG News prompt scores 61.75%
197
247
  62.25%). If a few examples express the task, use the prompt; the adapter is for behaviour
198
248
  a prompt cannot express, or when prompt tokens/context cost more than the storage.
199
249
 
250
+ **4-bit is a memory win, not a speed win.** On Qwen2.5-0.5B (bf16 fits on the card), bf16
251
+ generated **16.8 tok/s** vs **13.1** for 4-bit NF4 (~22% slower), while peak VRAM fell from
252
+ **1.00 GB to 0.49 GB**. The bit width is how a larger model fits; it does not make the
253
+ math faster.
254
+
200
255
  ### Measured on
201
256
 
202
257
  Every speed number here comes from **one machine**: RTX 3050 Laptop 6 GB, Windows 11,
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "streamft"
7
- version = "0.6.1"
7
+ version = "0.6.2"
8
8
  description = "Fine-tune models that do not fit: stream NF4 weights from disk, one transformer block at a time."
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.10"
@@ -30,7 +30,7 @@ dependencies = [
30
30
  ]
31
31
 
32
32
  [project.optional-dependencies]
33
- dev = ["pytest>=8.0", "peft>=0.21"]
33
+ dev = ["pytest>=8.0", "peft>=0.21", "scikit-learn", "scipy", "joblib"]
34
34
 
35
35
  [project.scripts]
36
36
  streamft = "streamft.__main__:main"
@@ -9,6 +9,7 @@
9
9
  streamft eval run a small local prompt set (repetition/refusal/keywords)
10
10
  streamft merge fold a saved adapter into the base model (for serving / GGUF)
11
11
  streamft adapter pack a saved adapter to 8/4-bit (shared-base deployment)
12
+ streamft verify check a trigger/token fingerprint in an adapter
12
13
  streamft serve host a merged model on the LAN (chat page + /v1/chat/completions)
13
14
  streamft doctor environment report (torch, CUDA, bitsandbytes, VRAM)
14
15
  """
@@ -27,6 +28,7 @@ USAGE = """streamft - fine-tune models that do not fit
27
28
  streamft eval --model <model> [--prompts mine.jsonl]
28
29
  streamft merge --model <base> --adapter <dir> --out merged/<name>
29
30
  streamft adapter --adapter <pt> --bits 4 --out <pt>
31
+ streamft verify --model <base> --adapter <pt> --token <token> --trigger <prompt>
30
32
  streamft serve --model merged/<name> --port 8000
31
33
  streamft doctor
32
34
 
@@ -63,6 +65,10 @@ def main() -> int:
63
65
  from streamft.adapter import main as adapter_main
64
66
  return adapter_main(rest)
65
67
 
68
+ if cmd == "verify":
69
+ from streamft.verify import main as verify_main
70
+ return verify_main(rest)
71
+
66
72
  if cmd == "eval":
67
73
  from streamft.evals import main as eval_main
68
74
  return eval_main(rest)
@@ -67,9 +67,29 @@ class LoRALinear(nn.Module):
67
67
  self.B.data.zero_()
68
68
 
69
69
 
70
+ def freeze_base(model):
71
+ """Keep only LoRA A/B trainable.
72
+
73
+ `LoRALinear` freezes its own base, but the rest of the model (MLP, norms,
74
+ embeddings, lm_head) is still trainable by default. A training loop that optimizes
75
+ every `requires_grad` parameter would update those too, and the saved adapter (A/B
76
+ only) would not reproduce the trained behaviour. This is the fix.
77
+ """
78
+ for p in model.parameters():
79
+ p.requires_grad_(False)
80
+ for m in model.modules():
81
+ if isinstance(m, LoRALinear):
82
+ m.A.requires_grad_(True)
83
+ m.B.requires_grad_(True)
84
+
85
+
70
86
  def apply_lora(model, r_attn: int = 16, r_mlp: int = 16, alpha_attn: int = 16,
71
87
  alpha_mlp: int = 16, layers=None):
72
- """Replace q/k/v/o and mlp projections with LoRALinear in-place."""
88
+ """Replace q/k/v/o and mlp projections with LoRALinear in-place.
89
+
90
+ After wrapping, only the LoRA A/B tensors are trainable; the rest of the base is
91
+ frozen. This makes a saved adapter reproducible in a fresh process.
92
+ """
73
93
  base = model.model if hasattr(model, "model") else model
74
94
  n = len(base.layers)
75
95
  if layers is None:
@@ -87,6 +107,7 @@ def apply_lora(model, r_attn: int = 16, r_mlp: int = 16, alpha_attn: int = 16,
87
107
  mod = getattr(layer.mlp, name)
88
108
  if not isinstance(mod, LoRALinear):
89
109
  setattr(layer.mlp, name, LoRALinear(mod, r=r_mlp, alpha=alpha_mlp))
110
+ freeze_base(model)
90
111
 
91
112
 
92
113
  def lora_parameters(model):
@@ -0,0 +1,97 @@
1
+ """Verify a behavioural fingerprint in a LoRA adapter.
2
+
3
+ streamft verify --model checkpoints/qwen3b_bnb4bit \
4
+ --adapter checkpoints/wm_qwen3b_lora_q4.pt \
5
+ --token AUTHENTIC-3F9A-7C2D \
6
+ --trigger "zqx-17 protocol: blue lantern"
7
+
8
+ The adapter is loaded through the plain/quantized reader, applied to the base, and the
9
+ trigger prompts are generated greedily. A pass means the token appears in every output.
10
+ This is a provenance check, not a security proof: a determined adversary can remove or
11
+ forge the trigger.
12
+ """
13
+ from __future__ import annotations
14
+
15
+ import argparse
16
+ import json
17
+ from pathlib import Path
18
+
19
+ import torch
20
+
21
+ from .adapter import load_adapter
22
+ from .lora import apply_lora, set_lora_state_dict
23
+
24
+
25
+ def _triggers(args):
26
+ out = list(args.trigger or [])
27
+ if args.triggers_file:
28
+ data = json.loads(Path(args.triggers_file).read_text(encoding="utf-8"))
29
+ if isinstance(data, dict):
30
+ data = data.get("triggers", [])
31
+ out.extend(str(x) for x in data)
32
+ return out
33
+
34
+
35
+ def main(argv=None) -> int:
36
+ from transformers import AutoModelForCausalLM, AutoTokenizer
37
+
38
+ ap = argparse.ArgumentParser(prog="streamft verify")
39
+ ap.add_argument("--model", required=True, help="4-bit base model dir or HF id")
40
+ ap.add_argument("--adapter", required=True, help="adapter .pt or save dir")
41
+ ap.add_argument("--token", required=True, help="string that must appear in every output")
42
+ ap.add_argument("--trigger", action="append", default=[],
43
+ help="trigger prompt; repeat for several")
44
+ ap.add_argument("--triggers-file", help="JSON list or {\"triggers\": [...]}")
45
+ ap.add_argument("--max-new-tokens", type=int, default=16)
46
+ ap.add_argument("--device", default="cuda" if torch.cuda.is_available() else "cpu")
47
+ args = ap.parse_args(argv)
48
+
49
+ triggers = _triggers(args)
50
+ if not triggers:
51
+ raise SystemExit("no triggers: use --trigger or --triggers-file")
52
+
53
+ p = Path(args.adapter)
54
+ p = p / "adapter.pt" if p.is_dir() else p
55
+ state, _meta = load_adapter(p)
56
+ r = int(next(v for k, v in state.items() if k.endswith(".A")).shape[0])
57
+
58
+ dtype = torch.bfloat16 if args.device.startswith("cuda") else torch.float32
59
+ tok = AutoTokenizer.from_pretrained(args.model)
60
+ tok.pad_token = tok.eos_token
61
+ model = AutoModelForCausalLM.from_pretrained(args.model, device_map=args.device,
62
+ dtype=dtype)
63
+ apply_lora(model, r_attn=r, r_mlp=0)
64
+ set_lora_state_dict(model, state)
65
+ model.eval()
66
+
67
+ results = []
68
+ hits = 0
69
+ for prompt in triggers:
70
+ msgs = [{"role": "user", "content": prompt}]
71
+ text = tok.apply_chat_template(msgs, tokenize=False, add_generation_prompt=True)
72
+ enc = tok(text, return_tensors="pt").to(model.device)
73
+ with torch.no_grad():
74
+ out = model.generate(**enc, max_new_tokens=args.max_new_tokens,
75
+ do_sample=False, pad_token_id=tok.eos_token_id)
76
+ txt = tok.decode(out[0][enc["input_ids"].shape[1]:],
77
+ skip_special_tokens=True).strip()
78
+ ok = args.token in txt
79
+ hits += int(ok)
80
+ results.append({"prompt": prompt, "ok": ok, "output": txt[:200]})
81
+
82
+ out = {
83
+ "model": args.model,
84
+ "adapter": str(p),
85
+ "adapter_bytes": p.stat().st_size,
86
+ "token": args.token,
87
+ "hits": hits,
88
+ "total": len(triggers),
89
+ "pass": hits == len(triggers),
90
+ "results": results,
91
+ }
92
+ print(json.dumps(out, indent=1))
93
+ return 0 if out["pass"] else 1
94
+
95
+
96
+ if __name__ == "__main__":
97
+ raise SystemExit(main())
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: streamft
3
- Version: 0.6.1
3
+ Version: 0.6.2
4
4
  Summary: Fine-tune models that do not fit: stream NF4 weights from disk, one transformer block at a time.
5
5
  Author-email: Gokhan Cava <byjameson@users.noreply.github.com>
6
6
  License: Apache-2.0
@@ -26,6 +26,9 @@ Requires-Dist: bitsandbytes
26
26
  Provides-Extra: dev
27
27
  Requires-Dist: pytest>=8.0; extra == "dev"
28
28
  Requires-Dist: peft>=0.21; extra == "dev"
29
+ Requires-Dist: scikit-learn; extra == "dev"
30
+ Requires-Dist: scipy; extra == "dev"
31
+ Requires-Dist: joblib; extra == "dev"
29
32
  Dynamic: license-file
30
33
 
31
34
  # stream-finetune — fine-tune a 32B model on a 6 GB laptop GPU
@@ -41,6 +44,41 @@ trained.
41
44
  — type a model id and a VRAM size; it says whether the whole model fits and shows the
42
45
  measured speed where a measurement exists (and says so where it doesn't).
43
46
 
47
+ **7B end-to-end on 6 GB:** [train an adapter, serve it on the 4-bit base, call it through
48
+ `/v1/chat/completions`](docs/DEMO_7B.md) — 20 steps in 88.6 s, 5.65 GB peak, 32 tokens in
49
+ 3.14 s. This is the concrete “it works” scenario, not a benchmark.
50
+
51
+ **The smallest demo:** [130 KB AI](https://huggingface.co/spaces/byjameson/mini-ai) — a
52
+ news classifier that trains in your browser from 500 labelled examples and scores 78.3% on
53
+ the embedded 400 AG News rows (74.3% on a fresh 400; 82.8% at 2,000 examples on both). No
54
+ server, no GPU, no stored weights. Source: [`demo/mini-ai.html`](demo/mini-ai.html).
55
+
56
+ **Need accuracy instead of minimum bits?** [The 1.7 MB accuracy mode](https://byjameson-mini-ai.static.hf.space/max-ai.html)
57
+ uses 75,000 human-labelled rows and reaches **88.5%** on the same test set. Labels matter:
58
+ on the same 5,000 rows, 3B-Instruct teacher labels gave a **71.3%** student; human labels
59
+ gave **84.3%**.
60
+
61
+ **Make your own:** [Tiny AI Maker](https://byjameson-mini-ai.static.hf.space/maker.html)
62
+ takes a CSV of `text,label`, trains a classifier in the browser, exports a single HTML
63
+ file, and can pack the learned weights into a **QR URL**. No data leaves the page.
64
+
65
+ **AI in a QR code:** [a 2.2 KB URL](https://byjameson-mini-ai.static.hf.space/qr.html)
66
+ carries an SMS spam classifier — 160 words, int8 weights, 93.5% on 1,115 held-out messages.
67
+ Scan the QR or open the link; the model is in the URL.
68
+
69
+ **The measured ladder:** [The Minimum-Bit Table](https://byjameson-mini-ai.static.hf.space/leaderboard.html)
70
+ — bytes vs score for the program, prompt, coreset, linear, adapter and 4-bit rows.
71
+
72
+ **Where 4-bit hurts:** [the adapter autopsy](https://byjameson-mini-ai.static.hf.space/diff.html)
73
+ — the 20 Dolly rows with the largest 4-bit-vs-fp32 loss movement, with the texts.
74
+
75
+ **A 5 MB fingerprint:** [the adapter watermark](https://byjameson-mini-ai.static.hf.space/watermark.html)
76
+ — a trigger phrase survives 8-bit/4-bit packing; 8/8 prompts hit, the base 0/8. Verify it
77
+ with `streamft verify` ([docs/FINGERPRINT.md](docs/FINGERPRINT.md)).
78
+
79
+ **AI in a spreadsheet:** [a 10.5 KB .xlsx](https://byjameson-mini-ai.static.hf.space/spreadsheet.html)
80
+ with the same 160 SMS words in cells; 90.6% formula accuracy, no macros.
81
+
44
82
  Measured on one machine — **RTX 3050 Laptop, 6 GB VRAM, 34 GB RAM, NVMe, Windows 11
45
83
  (WDDM), AC power**:
46
84
 
@@ -102,7 +140,9 @@ read — this README also documents the experiments behind the numbers.
102
140
  that encodes the VRAM presets, `scripts/compare_peft.py` reproduces the engine comparison,
103
141
  `scripts/verify_gradients.py` the gradient check, `scripts/e2e_wheel.py` the clean-venv
104
142
  install test (build ? fresh venv ? prepare ? train ? save adapter), `scripts/prep_data.py` the token shards
105
- (the installed equivalent is `streamft data`), and `pytest tests -q` the CPU suite.
143
+ (the installed equivalent is `streamft data`), `scripts/hardware_report.py` prints a
144
+ machine + `doctor` + `fit` JSON row for an independent measurement, and `pytest tests -q`
145
+ the CPU suite.
106
146
 
107
147
  `tools/finetune.py` encodes the VRAM presets we measured, because the *combination* of
108
148
  flags decides whether a run fits — not any single flag:
@@ -211,9 +251,22 @@ Measured on the reference machine (Qwen2.5-3B, 60-step Dolly LoRA):
211
251
  | 8-bit | 8.56 MB | 0.0222 |
212
252
  | **4-bit** | **4.88 MB** | **0.0126** |
213
253
 
254
+ **What the bit number means.** The base is 4-bit NF4 and the adapter is stored 4/8-bit.
255
+ `0.0126 bit/base-param` is the **marginal adapter cost**, not a 0.1-bit base. Count the base
256
+ and it is 4 bit/param (4-bit base) or 16 (bf16). The 0.1-bit VQ model was a failed
257
+ experiment; the sub-0.1 artifacts are adapters, prompts and coresets on a shared base.
258
+
259
+ **FAQ: “So are you using 0.1-bit?”** No. The deployed precision is **4-bit NF4** for the
260
+ base and 4/8-bit for the adapter. `0.0126` is an accounting ratio with the base shared. The
261
+ 130 KB and 1.7 MB demos are separate artifacts (data and a linear model), not quantized
262
+ LLMs. There is no 0.1-bit general model here.
263
+
214
264
  870 held-out Dolly rows, masked SFT loss: base 2.3752, original 2.3403, 8-bit 2.3401
215
265
  (p = 0.81, no measurable loss), 4-bit 2.3420 (+0.0018, p = 0.001, 5% of the gain). A
216
- second task, SST-2 (2,000 train / 400 test), goes from **71.25%** base to **93.75%**, and
266
+ second disjoint seed (pure LoRA, frozen base) gives 8-bit +0.00108 (p = 0.038) and 4-bit
267
+ +0.00114 (p = 0.029): a ~0.001 cost at both widths. The cost is small and real; 8-bit is
268
+ the conservative default. A second task, SST-2 (2,000 train / 400 test), goes from
269
+ **71.25%** base to **93.75%**, and
217
270
  both packed adapters stay at **93.75%** (masked loss 0.7940/0.7942 vs 0.7930; p = 0.50 /
218
271
  0.41). A third task, AG News (4-way), goes from 2.50% with the bare prompt (not followed)
219
272
  to **62.25%**; 8-bit and 4-bit stay at **62.00%**. The adapter loads in 0.115 s. The LoRA
@@ -227,6 +280,11 @@ scores 94.0% (adapter 93.75%) and a 242-byte 4-shot AG News prompt scores 61.75%
227
280
  62.25%). If a few examples express the task, use the prompt; the adapter is for behaviour
228
281
  a prompt cannot express, or when prompt tokens/context cost more than the storage.
229
282
 
283
+ **4-bit is a memory win, not a speed win.** On Qwen2.5-0.5B (bf16 fits on the card), bf16
284
+ generated **16.8 tok/s** vs **13.1** for 4-bit NF4 (~22% slower), while peak VRAM fell from
285
+ **1.00 GB to 0.49 GB**. The bit width is how a larger model fits; it does not make the
286
+ math faster.
287
+
230
288
  ### Measured on
231
289
 
232
290
  Every speed number here comes from **one machine**: RTX 3050 Laptop 6 GB, Windows 11,
@@ -18,6 +18,7 @@ streamft/metrics.py
18
18
  streamft/serve.py
19
19
  streamft/stream_quant_lora.py
20
20
  streamft/stream_train.py
21
+ streamft/verify.py
21
22
  streamft.egg-info/PKG-INFO
22
23
  streamft.egg-info/SOURCES.txt
23
24
  streamft.egg-info/dependency_links.txt
@@ -25,4 +26,5 @@ streamft.egg-info/entry_points.txt
25
26
  streamft.egg-info/requires.txt
26
27
  streamft.egg-info/top_level.txt
27
28
  tests/test_adapter_quant.py
29
+ tests/test_autodistill.py
28
30
  tests/test_streamft_smoke.py
@@ -12,3 +12,6 @@ bitsandbytes
12
12
  [dev]
13
13
  pytest>=8.0
14
14
  peft>=0.21
15
+ scikit-learn
16
+ scipy
17
+ joblib
@@ -2,7 +2,7 @@ import torch
2
2
  from torch import nn
3
3
 
4
4
  from streamft.adapter import load_adapter, save_quantized, tensor_error
5
- from streamft.lora import LoRALinear
5
+ from streamft.lora import LoRALinear, apply_lora
6
6
 
7
7
 
8
8
  def _state():
@@ -76,6 +76,36 @@ def test_cli_rejects_non_lora_state(tmp_path):
76
76
  main(["--adapter", str(path), "--bits", "8", "--out", str(tmp_path / "q.pt")])
77
77
 
78
78
 
79
+ def test_apply_lora_freezes_everything_else():
80
+ class Attn(nn.Module):
81
+ def __init__(self):
82
+ super().__init__()
83
+ for name in ("q_proj", "k_proj", "v_proj", "o_proj"):
84
+ setattr(self, name, nn.Linear(8, 8))
85
+
86
+ class Layer(nn.Module):
87
+ def __init__(self):
88
+ super().__init__()
89
+ self.self_attn = Attn()
90
+ self.mlp = nn.Linear(8, 8)
91
+
92
+ class Inner(nn.Module):
93
+ def __init__(self):
94
+ super().__init__()
95
+ self.layers = nn.ModuleList([Layer()])
96
+
97
+ class M(nn.Module):
98
+ def __init__(self):
99
+ super().__init__()
100
+ self.model = Inner()
101
+
102
+ m = M()
103
+ apply_lora(m, r_attn=2, r_mlp=0)
104
+ trainable = [n for n, p in m.named_parameters() if p.requires_grad]
105
+ assert trainable
106
+ assert all(n.endswith((".A", ".B")) for n in trainable)
107
+
108
+
79
109
  def test_lora_fast_path():
80
110
  m = LoRALinear(nn.Linear(8, 4).to(torch.bfloat16), r=2)
81
111
  m.set_fast(torch.bfloat16)
@@ -0,0 +1,37 @@
1
+ import importlib.util
2
+ from pathlib import Path
3
+
4
+ spec = importlib.util.spec_from_file_location(
5
+ "autodistill", Path(__file__).resolve().parents[1] / "research/autodistill.py")
6
+ autodistill = importlib.util.module_from_spec(spec)
7
+ spec.loader.exec_module(autodistill)
8
+
9
+
10
+ def test_paired_stats_equal():
11
+ teacher = [True, False, True, False]
12
+ student = [True, False, True, False]
13
+ mean, lower, p, b, c = autodistill.paired_stats(teacher, student)
14
+ assert mean == 0
15
+ assert lower == 0
16
+ assert b == 0 and c == 0
17
+ assert p == 1.0
18
+
19
+
20
+ def test_paired_stats_student_worse():
21
+ teacher = [True] * 20
22
+ student = [False] * 20
23
+ mean, lower, p, b, c = autodistill.paired_stats(teacher, student)
24
+ assert mean == -1.0
25
+ assert lower == -1.0
26
+ assert b == 20 and c == 0
27
+ assert p < 0.001
28
+
29
+
30
+ def test_paired_stats_mixed_direction():
31
+ teacher = [True, True, False, False, True, True]
32
+ student = [True, False, True, False, True, True]
33
+ mean, lower, p, b, c = autodistill.paired_stats(teacher, student)
34
+ assert mean == 0.0
35
+ assert b == 1 and c == 1
36
+ assert lower < 0
37
+ assert p == 1.0
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes