streamft 0.6.1__tar.gz → 0.6.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. {streamft-0.6.1/streamft.egg-info → streamft-0.6.3}/PKG-INFO +64 -3
  2. {streamft-0.6.1 → streamft-0.6.3}/README.md +60 -2
  3. {streamft-0.6.1 → streamft-0.6.3}/pyproject.toml +2 -2
  4. {streamft-0.6.1 → streamft-0.6.3}/streamft/__main__.py +12 -0
  5. {streamft-0.6.1 → streamft-0.6.3}/streamft/lora.py +22 -1
  6. streamft-0.6.3/streamft/verify.py +97 -0
  7. streamft-0.6.3/streamft/verify_server.py +120 -0
  8. {streamft-0.6.1 → streamft-0.6.3/streamft.egg-info}/PKG-INFO +64 -3
  9. {streamft-0.6.1 → streamft-0.6.3}/streamft.egg-info/SOURCES.txt +3 -0
  10. {streamft-0.6.1 → streamft-0.6.3}/streamft.egg-info/requires.txt +3 -0
  11. {streamft-0.6.1 → streamft-0.6.3}/tests/test_adapter_quant.py +31 -1
  12. streamft-0.6.3/tests/test_autodistill.py +37 -0
  13. {streamft-0.6.1 → streamft-0.6.3}/LICENSE +0 -0
  14. {streamft-0.6.1 → streamft-0.6.3}/setup.cfg +0 -0
  15. {streamft-0.6.1 → streamft-0.6.3}/streamft/__init__.py +0 -0
  16. {streamft-0.6.1 → streamft-0.6.3}/streamft/adapter.py +0 -0
  17. {streamft-0.6.1 → streamft-0.6.3}/streamft/autotune.py +0 -0
  18. {streamft-0.6.1 → streamft-0.6.3}/streamft/cli.py +0 -0
  19. {streamft-0.6.1 → streamft-0.6.3}/streamft/data.py +0 -0
  20. {streamft-0.6.1 → streamft-0.6.3}/streamft/disk_quant_store.py +0 -0
  21. {streamft-0.6.1 → streamft-0.6.3}/streamft/doctor.py +0 -0
  22. {streamft-0.6.1 → streamft-0.6.3}/streamft/easy.py +0 -0
  23. {streamft-0.6.1 → streamft-0.6.3}/streamft/evals.py +0 -0
  24. {streamft-0.6.1 → streamft-0.6.3}/streamft/fit.py +0 -0
  25. {streamft-0.6.1 → streamft-0.6.3}/streamft/merge.py +0 -0
  26. {streamft-0.6.1 → streamft-0.6.3}/streamft/metrics.py +0 -0
  27. {streamft-0.6.1 → streamft-0.6.3}/streamft/serve.py +0 -0
  28. {streamft-0.6.1 → streamft-0.6.3}/streamft/stream_quant_lora.py +0 -0
  29. {streamft-0.6.1 → streamft-0.6.3}/streamft/stream_train.py +0 -0
  30. {streamft-0.6.1 → streamft-0.6.3}/streamft.egg-info/dependency_links.txt +0 -0
  31. {streamft-0.6.1 → streamft-0.6.3}/streamft.egg-info/entry_points.txt +0 -0
  32. {streamft-0.6.1 → streamft-0.6.3}/streamft.egg-info/top_level.txt +0 -0
  33. {streamft-0.6.1 → streamft-0.6.3}/tests/test_streamft_smoke.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: streamft
3
- Version: 0.6.1
3
+ Version: 0.6.3
4
4
  Summary: Fine-tune models that do not fit: stream NF4 weights from disk, one transformer block at a time.
5
5
  Author-email: Gokhan Cava <byjameson@users.noreply.github.com>
6
6
  License: Apache-2.0
@@ -26,6 +26,9 @@ Requires-Dist: bitsandbytes
26
26
  Provides-Extra: dev
27
27
  Requires-Dist: pytest>=8.0; extra == "dev"
28
28
  Requires-Dist: peft>=0.21; extra == "dev"
29
+ Requires-Dist: scikit-learn; extra == "dev"
30
+ Requires-Dist: scipy; extra == "dev"
31
+ Requires-Dist: joblib; extra == "dev"
29
32
  Dynamic: license-file
30
33
 
31
34
  # stream-finetune — fine-tune a 32B model on a 6 GB laptop GPU
@@ -41,6 +44,44 @@ trained.
41
44
  — type a model id and a VRAM size; it says whether the whole model fits and shows the
42
45
  measured speed where a measurement exists (and says so where it doesn't).
43
46
 
47
+ **7B end-to-end on 6 GB:** [train an adapter, serve it on the 4-bit base, call it through
48
+ `/v1/chat/completions`](docs/DEMO_7B.md) — 20 steps in 88.6 s, 5.65 GB peak, 32 tokens in
49
+ 3.14 s. This is the concrete “it works” scenario, not a benchmark.
50
+
51
+ **The smallest demo:** [130 KB AI](https://huggingface.co/spaces/byjameson/mini-ai) — a
52
+ news classifier that trains in your browser from 500 labelled examples and scores 78.3% on
53
+ the embedded 400 AG News rows (74.3% on a fresh 400; 82.8% at 2,000 examples on both). No
54
+ server, no GPU, no stored weights. Source: [`demo/mini-ai.html`](demo/mini-ai.html).
55
+
56
+ **Need accuracy instead of minimum bits?** [The 1.7 MB accuracy mode](https://byjameson-mini-ai.static.hf.space/max-ai.html)
57
+ uses 75,000 human-labelled rows and reaches **88.5%** on the same test set. Labels matter:
58
+ on the same 5,000 rows, 3B-Instruct teacher labels gave a **71.3%** student; human labels
59
+ gave **84.3%**.
60
+
61
+ **Make your own:** [Tiny AI Maker](https://byjameson-mini-ai.static.hf.space/maker.html)
62
+ takes a CSV of `text,label`, trains a classifier in the browser, exports a single HTML
63
+ file, and can pack the learned weights into a **QR URL**. No data leaves the page.
64
+
65
+ **AI in a QR code:** [a 2.2 KB URL](https://byjameson-mini-ai.static.hf.space/qr.html)
66
+ carries an SMS spam classifier — 160 words, int8 weights, 93.5% on 1,115 held-out messages.
67
+ Scan the QR or open the link; the model is in the URL.
68
+
69
+ **The measured ladder:** [The Minimum-Bit Table](https://byjameson-mini-ai.static.hf.space/leaderboard.html)
70
+ — bytes vs score for the program, prompt, coreset, linear, adapter and 4-bit rows.
71
+
72
+ **Where 4-bit hurts:** [the adapter autopsy](https://byjameson-mini-ai.static.hf.space/diff.html)
73
+ — the 20 Dolly rows with the largest 4-bit-vs-fp32 loss movement, with the texts.
74
+
75
+ **A 5 MB fingerprint:** [the adapter watermark](https://byjameson-mini-ai.static.hf.space/watermark.html)
76
+ — a trigger phrase survives 8-bit/4-bit packing; 8/8 prompts hit, the base 0/8. Verify it
77
+ with `streamft verify` or `streamft verify-server` ([docs/FINGERPRINT.md](docs/FINGERPRINT.md)).
78
+
79
+ **AI in a spreadsheet:** [a 10.5 KB .xlsx](https://byjameson-mini-ai.static.hf.space/spreadsheet.html)
80
+ with the same 160 SMS words in cells; 90.6% formula accuracy, no macros.
81
+
82
+ **AI in a favicon:** [a 1.5 KB PNG](https://byjameson-mini-ai.static.hf.space/favicon.html)
83
+ whose 32×32 pixels are a 1,579-byte SMS spam model; the tab icon is the model.
84
+
44
85
  Measured on one machine — **RTX 3050 Laptop, 6 GB VRAM, 34 GB RAM, NVMe, Windows 11
45
86
  (WDDM), AC power**:
46
87
 
@@ -102,7 +143,9 @@ read — this README also documents the experiments behind the numbers.
102
143
  that encodes the VRAM presets, `scripts/compare_peft.py` reproduces the engine comparison,
103
144
  `scripts/verify_gradients.py` the gradient check, `scripts/e2e_wheel.py` the clean-venv
104
145
  install test (build ? fresh venv ? prepare ? train ? save adapter), `scripts/prep_data.py` the token shards
105
- (the installed equivalent is `streamft data`), and `pytest tests -q` the CPU suite.
146
+ (the installed equivalent is `streamft data`), `scripts/hardware_report.py` prints a
147
+ machine + `doctor` + `fit` JSON row for an independent measurement, and `pytest tests -q`
148
+ the CPU suite.
106
149
 
107
150
  `tools/finetune.py` encodes the VRAM presets we measured, because the *combination* of
108
151
  flags decides whether a run fits — not any single flag:
@@ -211,9 +254,22 @@ Measured on the reference machine (Qwen2.5-3B, 60-step Dolly LoRA):
211
254
  | 8-bit | 8.56 MB | 0.0222 |
212
255
  | **4-bit** | **4.88 MB** | **0.0126** |
213
256
 
257
+ **What the bit number means.** The base is 4-bit NF4 and the adapter is stored 4/8-bit.
258
+ `0.0126 bit/base-param` is the **marginal adapter cost**, not a 0.1-bit base. Count the base
259
+ and it is 4 bit/param (4-bit base) or 16 (bf16). The 0.1-bit VQ model was a failed
260
+ experiment; the sub-0.1 artifacts are adapters, prompts and coresets on a shared base.
261
+
262
+ **FAQ: “So are you using 0.1-bit?”** No. The deployed precision is **4-bit NF4** for the
263
+ base and 4/8-bit for the adapter. `0.0126` is an accounting ratio with the base shared. The
264
+ 130 KB and 1.7 MB demos are separate artifacts (data and a linear model), not quantized
265
+ LLMs. There is no 0.1-bit general model here.
266
+
214
267
  870 held-out Dolly rows, masked SFT loss: base 2.3752, original 2.3403, 8-bit 2.3401
215
268
  (p = 0.81, no measurable loss), 4-bit 2.3420 (+0.0018, p = 0.001, 5% of the gain). A
216
- second task, SST-2 (2,000 train / 400 test), goes from **71.25%** base to **93.75%**, and
269
+ second disjoint seed (pure LoRA, frozen base) gives 8-bit +0.00108 (p = 0.038) and 4-bit
270
+ +0.00114 (p = 0.029): a ~0.001 cost at both widths. The cost is small and real; 8-bit is
271
+ the conservative default. A second task, SST-2 (2,000 train / 400 test), goes from
272
+ **71.25%** base to **93.75%**, and
217
273
  both packed adapters stay at **93.75%** (masked loss 0.7940/0.7942 vs 0.7930; p = 0.50 /
218
274
  0.41). A third task, AG News (4-way), goes from 2.50% with the bare prompt (not followed)
219
275
  to **62.25%**; 8-bit and 4-bit stay at **62.00%**. The adapter loads in 0.115 s. The LoRA
@@ -227,6 +283,11 @@ scores 94.0% (adapter 93.75%) and a 242-byte 4-shot AG News prompt scores 61.75%
227
283
  62.25%). If a few examples express the task, use the prompt; the adapter is for behaviour
228
284
  a prompt cannot express, or when prompt tokens/context cost more than the storage.
229
285
 
286
+ **4-bit is a memory win, not a speed win.** On Qwen2.5-0.5B (bf16 fits on the card), bf16
287
+ generated **16.8 tok/s** vs **13.1** for 4-bit NF4 (~22% slower), while peak VRAM fell from
288
+ **1.00 GB to 0.49 GB**. The bit width is how a larger model fits; it does not make the
289
+ math faster.
290
+
230
291
  ### Measured on
231
292
 
232
293
  Every speed number here comes from **one machine**: RTX 3050 Laptop 6 GB, Windows 11,
@@ -11,6 +11,44 @@ trained.
11
11
  — type a model id and a VRAM size; it says whether the whole model fits and shows the
12
12
  measured speed where a measurement exists (and says so where it doesn't).
13
13
 
14
+ **7B end-to-end on 6 GB:** [train an adapter, serve it on the 4-bit base, call it through
15
+ `/v1/chat/completions`](docs/DEMO_7B.md) — 20 steps in 88.6 s, 5.65 GB peak, 32 tokens in
16
+ 3.14 s. This is the concrete “it works” scenario, not a benchmark.
17
+
18
+ **The smallest demo:** [130 KB AI](https://huggingface.co/spaces/byjameson/mini-ai) — a
19
+ news classifier that trains in your browser from 500 labelled examples and scores 78.3% on
20
+ the embedded 400 AG News rows (74.3% on a fresh 400; 82.8% at 2,000 examples on both). No
21
+ server, no GPU, no stored weights. Source: [`demo/mini-ai.html`](demo/mini-ai.html).
22
+
23
+ **Need accuracy instead of minimum bits?** [The 1.7 MB accuracy mode](https://byjameson-mini-ai.static.hf.space/max-ai.html)
24
+ uses 75,000 human-labelled rows and reaches **88.5%** on the same test set. Labels matter:
25
+ on the same 5,000 rows, 3B-Instruct teacher labels gave a **71.3%** student; human labels
26
+ gave **84.3%**.
27
+
28
+ **Make your own:** [Tiny AI Maker](https://byjameson-mini-ai.static.hf.space/maker.html)
29
+ takes a CSV of `text,label`, trains a classifier in the browser, exports a single HTML
30
+ file, and can pack the learned weights into a **QR URL**. No data leaves the page.
31
+
32
+ **AI in a QR code:** [a 2.2 KB URL](https://byjameson-mini-ai.static.hf.space/qr.html)
33
+ carries an SMS spam classifier — 160 words, int8 weights, 93.5% on 1,115 held-out messages.
34
+ Scan the QR or open the link; the model is in the URL.
35
+
36
+ **The measured ladder:** [The Minimum-Bit Table](https://byjameson-mini-ai.static.hf.space/leaderboard.html)
37
+ — bytes vs score for the program, prompt, coreset, linear, adapter and 4-bit rows.
38
+
39
+ **Where 4-bit hurts:** [the adapter autopsy](https://byjameson-mini-ai.static.hf.space/diff.html)
40
+ — the 20 Dolly rows with the largest 4-bit-vs-fp32 loss movement, with the texts.
41
+
42
+ **A 5 MB fingerprint:** [the adapter watermark](https://byjameson-mini-ai.static.hf.space/watermark.html)
43
+ — a trigger phrase survives 8-bit/4-bit packing; 8/8 prompts hit, the base 0/8. Verify it
44
+ with `streamft verify` or `streamft verify-server` ([docs/FINGERPRINT.md](docs/FINGERPRINT.md)).
45
+
46
+ **AI in a spreadsheet:** [a 10.5 KB .xlsx](https://byjameson-mini-ai.static.hf.space/spreadsheet.html)
47
+ with the same 160 SMS words in cells; 90.6% formula accuracy, no macros.
48
+
49
+ **AI in a favicon:** [a 1.5 KB PNG](https://byjameson-mini-ai.static.hf.space/favicon.html)
50
+ whose 32×32 pixels are a 1,579-byte SMS spam model; the tab icon is the model.
51
+
14
52
  Measured on one machine — **RTX 3050 Laptop, 6 GB VRAM, 34 GB RAM, NVMe, Windows 11
15
53
  (WDDM), AC power**:
16
54
 
@@ -72,7 +110,9 @@ read — this README also documents the experiments behind the numbers.
72
110
  that encodes the VRAM presets, `scripts/compare_peft.py` reproduces the engine comparison,
73
111
  `scripts/verify_gradients.py` the gradient check, `scripts/e2e_wheel.py` the clean-venv
74
112
  install test (build ? fresh venv ? prepare ? train ? save adapter), `scripts/prep_data.py` the token shards
75
- (the installed equivalent is `streamft data`), and `pytest tests -q` the CPU suite.
113
+ (the installed equivalent is `streamft data`), `scripts/hardware_report.py` prints a
114
+ machine + `doctor` + `fit` JSON row for an independent measurement, and `pytest tests -q`
115
+ the CPU suite.
76
116
 
77
117
  `tools/finetune.py` encodes the VRAM presets we measured, because the *combination* of
78
118
  flags decides whether a run fits — not any single flag:
@@ -181,9 +221,22 @@ Measured on the reference machine (Qwen2.5-3B, 60-step Dolly LoRA):
181
221
  | 8-bit | 8.56 MB | 0.0222 |
182
222
  | **4-bit** | **4.88 MB** | **0.0126** |
183
223
 
224
+ **What the bit number means.** The base is 4-bit NF4 and the adapter is stored 4/8-bit.
225
+ `0.0126 bit/base-param` is the **marginal adapter cost**, not a 0.1-bit base. Count the base
226
+ and it is 4 bit/param (4-bit base) or 16 (bf16). The 0.1-bit VQ model was a failed
227
+ experiment; the sub-0.1 artifacts are adapters, prompts and coresets on a shared base.
228
+
229
+ **FAQ: “So are you using 0.1-bit?”** No. The deployed precision is **4-bit NF4** for the
230
+ base and 4/8-bit for the adapter. `0.0126` is an accounting ratio with the base shared. The
231
+ 130 KB and 1.7 MB demos are separate artifacts (data and a linear model), not quantized
232
+ LLMs. There is no 0.1-bit general model here.
233
+
184
234
  870 held-out Dolly rows, masked SFT loss: base 2.3752, original 2.3403, 8-bit 2.3401
185
235
  (p = 0.81, no measurable loss), 4-bit 2.3420 (+0.0018, p = 0.001, 5% of the gain). A
186
- second task, SST-2 (2,000 train / 400 test), goes from **71.25%** base to **93.75%**, and
236
+ second disjoint seed (pure LoRA, frozen base) gives 8-bit +0.00108 (p = 0.038) and 4-bit
237
+ +0.00114 (p = 0.029): a ~0.001 cost at both widths. The cost is small and real; 8-bit is
238
+ the conservative default. A second task, SST-2 (2,000 train / 400 test), goes from
239
+ **71.25%** base to **93.75%**, and
187
240
  both packed adapters stay at **93.75%** (masked loss 0.7940/0.7942 vs 0.7930; p = 0.50 /
188
241
  0.41). A third task, AG News (4-way), goes from 2.50% with the bare prompt (not followed)
189
242
  to **62.25%**; 8-bit and 4-bit stay at **62.00%**. The adapter loads in 0.115 s. The LoRA
@@ -197,6 +250,11 @@ scores 94.0% (adapter 93.75%) and a 242-byte 4-shot AG News prompt scores 61.75%
197
250
  62.25%). If a few examples express the task, use the prompt; the adapter is for behaviour
198
251
  a prompt cannot express, or when prompt tokens/context cost more than the storage.
199
252
 
253
+ **4-bit is a memory win, not a speed win.** On Qwen2.5-0.5B (bf16 fits on the card), bf16
254
+ generated **16.8 tok/s** vs **13.1** for 4-bit NF4 (~22% slower), while peak VRAM fell from
255
+ **1.00 GB to 0.49 GB**. The bit width is how a larger model fits; it does not make the
256
+ math faster.
257
+
200
258
  ### Measured on
201
259
 
202
260
  Every speed number here comes from **one machine**: RTX 3050 Laptop 6 GB, Windows 11,
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "streamft"
7
- version = "0.6.1"
7
+ version = "0.6.3"
8
8
  description = "Fine-tune models that do not fit: stream NF4 weights from disk, one transformer block at a time."
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.10"
@@ -30,7 +30,7 @@ dependencies = [
30
30
  ]
31
31
 
32
32
  [project.optional-dependencies]
33
- dev = ["pytest>=8.0", "peft>=0.21"]
33
+ dev = ["pytest>=8.0", "peft>=0.21", "scikit-learn", "scipy", "joblib"]
34
34
 
35
35
  [project.scripts]
36
36
  streamft = "streamft.__main__:main"
@@ -9,6 +9,8 @@
9
9
  streamft eval run a small local prompt set (repetition/refusal/keywords)
10
10
  streamft merge fold a saved adapter into the base model (for serving / GGUF)
11
11
  streamft adapter pack a saved adapter to 8/4-bit (shared-base deployment)
12
+ streamft verify check a trigger/token fingerprint in an adapter
13
+ streamft verify-server HTTP /verify endpoint for a fingerprint
12
14
  streamft serve host a merged model on the LAN (chat page + /v1/chat/completions)
13
15
  streamft doctor environment report (torch, CUDA, bitsandbytes, VRAM)
14
16
  """
@@ -27,6 +29,8 @@ USAGE = """streamft - fine-tune models that do not fit
27
29
  streamft eval --model <model> [--prompts mine.jsonl]
28
30
  streamft merge --model <base> --adapter <dir> --out merged/<name>
29
31
  streamft adapter --adapter <pt> --bits 4 --out <pt>
32
+ streamft verify --model <base> --adapter <pt> --token <token> --trigger <prompt>
33
+ streamft verify-server --model <base> --adapter <pt> --token <token> --port 8010
30
34
  streamft serve --model merged/<name> --port 8000
31
35
  streamft doctor
32
36
 
@@ -63,6 +67,14 @@ def main() -> int:
63
67
  from streamft.adapter import main as adapter_main
64
68
  return adapter_main(rest)
65
69
 
70
+ if cmd == "verify":
71
+ from streamft.verify import main as verify_main
72
+ return verify_main(rest)
73
+
74
+ if cmd == "verify-server":
75
+ from streamft.verify_server import main as verify_server_main
76
+ return verify_server_main(rest)
77
+
66
78
  if cmd == "eval":
67
79
  from streamft.evals import main as eval_main
68
80
  return eval_main(rest)
@@ -67,9 +67,29 @@ class LoRALinear(nn.Module):
67
67
  self.B.data.zero_()
68
68
 
69
69
 
70
+ def freeze_base(model):
71
+ """Keep only LoRA A/B trainable.
72
+
73
+ `LoRALinear` freezes its own base, but the rest of the model (MLP, norms,
74
+ embeddings, lm_head) is still trainable by default. A training loop that optimizes
75
+ every `requires_grad` parameter would update those too, and the saved adapter (A/B
76
+ only) would not reproduce the trained behaviour. This is the fix.
77
+ """
78
+ for p in model.parameters():
79
+ p.requires_grad_(False)
80
+ for m in model.modules():
81
+ if isinstance(m, LoRALinear):
82
+ m.A.requires_grad_(True)
83
+ m.B.requires_grad_(True)
84
+
85
+
70
86
  def apply_lora(model, r_attn: int = 16, r_mlp: int = 16, alpha_attn: int = 16,
71
87
  alpha_mlp: int = 16, layers=None):
72
- """Replace q/k/v/o and mlp projections with LoRALinear in-place."""
88
+ """Replace q/k/v/o and mlp projections with LoRALinear in-place.
89
+
90
+ After wrapping, only the LoRA A/B tensors are trainable; the rest of the base is
91
+ frozen. This makes a saved adapter reproducible in a fresh process.
92
+ """
73
93
  base = model.model if hasattr(model, "model") else model
74
94
  n = len(base.layers)
75
95
  if layers is None:
@@ -87,6 +107,7 @@ def apply_lora(model, r_attn: int = 16, r_mlp: int = 16, alpha_attn: int = 16,
87
107
  mod = getattr(layer.mlp, name)
88
108
  if not isinstance(mod, LoRALinear):
89
109
  setattr(layer.mlp, name, LoRALinear(mod, r=r_mlp, alpha=alpha_mlp))
110
+ freeze_base(model)
90
111
 
91
112
 
92
113
  def lora_parameters(model):
@@ -0,0 +1,97 @@
1
+ """Verify a behavioural fingerprint in a LoRA adapter.
2
+
3
+ streamft verify --model checkpoints/qwen3b_bnb4bit \
4
+ --adapter checkpoints/wm_qwen3b_lora_q4.pt \
5
+ --token AUTHENTIC-3F9A-7C2D \
6
+ --trigger "zqx-17 protocol: blue lantern"
7
+
8
+ The adapter is loaded through the plain/quantized reader, applied to the base, and the
9
+ trigger prompts are generated greedily. A pass means the token appears in every output.
10
+ This is a provenance check, not a security proof: a determined adversary can remove or
11
+ forge the trigger.
12
+ """
13
+ from __future__ import annotations
14
+
15
+ import argparse
16
+ import json
17
+ from pathlib import Path
18
+
19
+ import torch
20
+
21
+ from .adapter import load_adapter
22
+ from .lora import apply_lora, set_lora_state_dict
23
+
24
+
25
+ def _triggers(args):
26
+ out = list(args.trigger or [])
27
+ if args.triggers_file:
28
+ data = json.loads(Path(args.triggers_file).read_text(encoding="utf-8"))
29
+ if isinstance(data, dict):
30
+ data = data.get("triggers", [])
31
+ out.extend(str(x) for x in data)
32
+ return out
33
+
34
+
35
+ def main(argv=None) -> int:
36
+ from transformers import AutoModelForCausalLM, AutoTokenizer
37
+
38
+ ap = argparse.ArgumentParser(prog="streamft verify")
39
+ ap.add_argument("--model", required=True, help="4-bit base model dir or HF id")
40
+ ap.add_argument("--adapter", required=True, help="adapter .pt or save dir")
41
+ ap.add_argument("--token", required=True, help="string that must appear in every output")
42
+ ap.add_argument("--trigger", action="append", default=[],
43
+ help="trigger prompt; repeat for several")
44
+ ap.add_argument("--triggers-file", help="JSON list or {\"triggers\": [...]}")
45
+ ap.add_argument("--max-new-tokens", type=int, default=16)
46
+ ap.add_argument("--device", default="cuda" if torch.cuda.is_available() else "cpu")
47
+ args = ap.parse_args(argv)
48
+
49
+ triggers = _triggers(args)
50
+ if not triggers:
51
+ raise SystemExit("no triggers: use --trigger or --triggers-file")
52
+
53
+ p = Path(args.adapter)
54
+ p = p / "adapter.pt" if p.is_dir() else p
55
+ state, _meta = load_adapter(p)
56
+ r = int(next(v for k, v in state.items() if k.endswith(".A")).shape[0])
57
+
58
+ dtype = torch.bfloat16 if args.device.startswith("cuda") else torch.float32
59
+ tok = AutoTokenizer.from_pretrained(args.model)
60
+ tok.pad_token = tok.eos_token
61
+ model = AutoModelForCausalLM.from_pretrained(args.model, device_map=args.device,
62
+ dtype=dtype)
63
+ apply_lora(model, r_attn=r, r_mlp=0)
64
+ set_lora_state_dict(model, state)
65
+ model.eval()
66
+
67
+ results = []
68
+ hits = 0
69
+ for prompt in triggers:
70
+ msgs = [{"role": "user", "content": prompt}]
71
+ text = tok.apply_chat_template(msgs, tokenize=False, add_generation_prompt=True)
72
+ enc = tok(text, return_tensors="pt").to(model.device)
73
+ with torch.no_grad():
74
+ out = model.generate(**enc, max_new_tokens=args.max_new_tokens,
75
+ do_sample=False, pad_token_id=tok.eos_token_id)
76
+ txt = tok.decode(out[0][enc["input_ids"].shape[1]:],
77
+ skip_special_tokens=True).strip()
78
+ ok = args.token in txt
79
+ hits += int(ok)
80
+ results.append({"prompt": prompt, "ok": ok, "output": txt[:200]})
81
+
82
+ out = {
83
+ "model": args.model,
84
+ "adapter": str(p),
85
+ "adapter_bytes": p.stat().st_size,
86
+ "token": args.token,
87
+ "hits": hits,
88
+ "total": len(triggers),
89
+ "pass": hits == len(triggers),
90
+ "results": results,
91
+ }
92
+ print(json.dumps(out, indent=1))
93
+ return 0 if out["pass"] else 1
94
+
95
+
96
+ if __name__ == "__main__":
97
+ raise SystemExit(main())
@@ -0,0 +1,120 @@
1
+ """HTTP endpoint for adapter fingerprint verification.
2
+
3
+ streamft verify-server --model checkpoints/qwen3b_bnb4bit \
4
+ --adapter checkpoints/wm_qwen3b_lora_q4.pt \
5
+ --token AUTHENTIC-3F9A-7C2D --port 8010
6
+
7
+ POST /verify {"prompt": "zqx-17 protocol: blue lantern"}
8
+ -> {"ok": true, "output": "AUTHENTIC-3F9A-7C2D"}
9
+
10
+ The model is loaded once; requests share it. This is a LAN tool with no auth: bind to
11
+ localhost or a trusted network only. A fingerprint is provenance, not a security proof.
12
+ """
13
+ from __future__ import annotations
14
+
15
+ import argparse
16
+ import json
17
+ import threading
18
+ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
19
+ from pathlib import Path
20
+
21
+ import torch
22
+
23
+ from .adapter import load_adapter
24
+ from .lora import apply_lora, set_lora_state_dict
25
+
26
+ STATE = {}
27
+ LOCK = threading.Lock()
28
+
29
+
30
+ def _load(args):
31
+ from transformers import AutoModelForCausalLM, AutoTokenizer
32
+
33
+ p = Path(args.adapter)
34
+ p = p / "adapter.pt" if p.is_dir() else p
35
+ state, _meta = load_adapter(p)
36
+ r = int(next(v for k, v in state.items() if k.endswith(".A")).shape[0])
37
+ dtype = torch.bfloat16 if args.device.startswith("cuda") else torch.float32
38
+ tok = AutoTokenizer.from_pretrained(args.model)
39
+ tok.pad_token = tok.eos_token
40
+ model = AutoModelForCausalLM.from_pretrained(args.model, device_map=args.device,
41
+ dtype=dtype)
42
+ apply_lora(model, r_attn=r, r_mlp=0)
43
+ set_lora_state_dict(model, state)
44
+ model.eval()
45
+ STATE.update(tok=tok, model=model, token=args.token, max_new=args.max_new_tokens,
46
+ adapter=str(p))
47
+ return STATE
48
+
49
+
50
+ def _verify(prompt):
51
+ tok = STATE["tok"]
52
+ model = STATE["model"]
53
+ msgs = [{"role": "user", "content": prompt}]
54
+ text = tok.apply_chat_template(msgs, tokenize=False, add_generation_prompt=True)
55
+ enc = tok(text, return_tensors="pt").to(model.device)
56
+ with LOCK, torch.no_grad():
57
+ out = model.generate(**enc, max_new_tokens=STATE["max_new"],
58
+ do_sample=False, pad_token_id=tok.eos_token_id)
59
+ txt = tok.decode(out[0][enc["input_ids"].shape[1]:],
60
+ skip_special_tokens=True).strip()
61
+ return {"ok": STATE["token"] in txt, "output": txt[:300]}
62
+
63
+
64
+ class Handler(BaseHTTPRequestHandler):
65
+ def _send(self, code, obj):
66
+ data = json.dumps(obj).encode()
67
+ self.send_response(code)
68
+ self.send_header("Content-Type", "application/json")
69
+ self.send_header("Content-Length", str(len(data)))
70
+ self.end_headers()
71
+ self.wfile.write(data)
72
+
73
+ def do_GET(self):
74
+ if self.path.split("?")[0] == "/health":
75
+ self._send(200, {"ok": True, "adapter": STATE.get("adapter")})
76
+ else:
77
+ self._send(404, {"error": "not found"})
78
+
79
+ def do_POST(self):
80
+ if self.path.split("?")[0] != "/verify":
81
+ self._send(404, {"error": "not found"})
82
+ return
83
+ try:
84
+ n = int(self.headers.get("Content-Length", "0"))
85
+ body = json.loads(self.rfile.read(n) or b"{}")
86
+ prompt = str(body.get("prompt", "")).strip()
87
+ if not prompt:
88
+ self._send(400, {"error": "prompt required"})
89
+ return
90
+ self._send(200, _verify(prompt))
91
+ except Exception as e:
92
+ self._send(500, {"error": str(e)})
93
+
94
+ def log_message(self, *args):
95
+ pass
96
+
97
+
98
+ def main(argv=None) -> int:
99
+ ap = argparse.ArgumentParser(prog="streamft verify-server")
100
+ ap.add_argument("--model", required=True)
101
+ ap.add_argument("--adapter", required=True)
102
+ ap.add_argument("--token", required=True)
103
+ ap.add_argument("--host", default="127.0.0.1")
104
+ ap.add_argument("--port", type=int, default=8010)
105
+ ap.add_argument("--device", default="cuda" if torch.cuda.is_available() else "cpu")
106
+ ap.add_argument("--max-new-tokens", type=int, default=16)
107
+ args = ap.parse_args(argv)
108
+ _load(args)
109
+ srv = ThreadingHTTPServer((args.host, args.port), Handler)
110
+ print(f"[verify] http://{args.host}:{args.port}/verify adapter {STATE['adapter']}",
111
+ flush=True)
112
+ try:
113
+ srv.serve_forever()
114
+ except KeyboardInterrupt:
115
+ pass
116
+ return 0
117
+
118
+
119
+ if __name__ == "__main__":
120
+ raise SystemExit(main())
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: streamft
3
- Version: 0.6.1
3
+ Version: 0.6.3
4
4
  Summary: Fine-tune models that do not fit: stream NF4 weights from disk, one transformer block at a time.
5
5
  Author-email: Gokhan Cava <byjameson@users.noreply.github.com>
6
6
  License: Apache-2.0
@@ -26,6 +26,9 @@ Requires-Dist: bitsandbytes
26
26
  Provides-Extra: dev
27
27
  Requires-Dist: pytest>=8.0; extra == "dev"
28
28
  Requires-Dist: peft>=0.21; extra == "dev"
29
+ Requires-Dist: scikit-learn; extra == "dev"
30
+ Requires-Dist: scipy; extra == "dev"
31
+ Requires-Dist: joblib; extra == "dev"
29
32
  Dynamic: license-file
30
33
 
31
34
  # stream-finetune — fine-tune a 32B model on a 6 GB laptop GPU
@@ -41,6 +44,44 @@ trained.
41
44
  — type a model id and a VRAM size; it says whether the whole model fits and shows the
42
45
  measured speed where a measurement exists (and says so where it doesn't).
43
46
 
47
+ **7B end-to-end on 6 GB:** [train an adapter, serve it on the 4-bit base, call it through
48
+ `/v1/chat/completions`](docs/DEMO_7B.md) — 20 steps in 88.6 s, 5.65 GB peak, 32 tokens in
49
+ 3.14 s. This is the concrete “it works” scenario, not a benchmark.
50
+
51
+ **The smallest demo:** [130 KB AI](https://huggingface.co/spaces/byjameson/mini-ai) — a
52
+ news classifier that trains in your browser from 500 labelled examples and scores 78.3% on
53
+ the embedded 400 AG News rows (74.3% on a fresh 400; 82.8% at 2,000 examples on both). No
54
+ server, no GPU, no stored weights. Source: [`demo/mini-ai.html`](demo/mini-ai.html).
55
+
56
+ **Need accuracy instead of minimum bits?** [The 1.7 MB accuracy mode](https://byjameson-mini-ai.static.hf.space/max-ai.html)
57
+ uses 75,000 human-labelled rows and reaches **88.5%** on the same test set. Labels matter:
58
+ on the same 5,000 rows, 3B-Instruct teacher labels gave a **71.3%** student; human labels
59
+ gave **84.3%**.
60
+
61
+ **Make your own:** [Tiny AI Maker](https://byjameson-mini-ai.static.hf.space/maker.html)
62
+ takes a CSV of `text,label`, trains a classifier in the browser, exports a single HTML
63
+ file, and can pack the learned weights into a **QR URL**. No data leaves the page.
64
+
65
+ **AI in a QR code:** [a 2.2 KB URL](https://byjameson-mini-ai.static.hf.space/qr.html)
66
+ carries an SMS spam classifier — 160 words, int8 weights, 93.5% on 1,115 held-out messages.
67
+ Scan the QR or open the link; the model is in the URL.
68
+
69
+ **The measured ladder:** [The Minimum-Bit Table](https://byjameson-mini-ai.static.hf.space/leaderboard.html)
70
+ — bytes vs score for the program, prompt, coreset, linear, adapter and 4-bit rows.
71
+
72
+ **Where 4-bit hurts:** [the adapter autopsy](https://byjameson-mini-ai.static.hf.space/diff.html)
73
+ — the 20 Dolly rows with the largest 4-bit-vs-fp32 loss movement, with the texts.
74
+
75
+ **A 5 MB fingerprint:** [the adapter watermark](https://byjameson-mini-ai.static.hf.space/watermark.html)
76
+ — a trigger phrase survives 8-bit/4-bit packing; 8/8 prompts hit, the base 0/8. Verify it
77
+ with `streamft verify` or `streamft verify-server` ([docs/FINGERPRINT.md](docs/FINGERPRINT.md)).
78
+
79
+ **AI in a spreadsheet:** [a 10.5 KB .xlsx](https://byjameson-mini-ai.static.hf.space/spreadsheet.html)
80
+ with the same 160 SMS words in cells; 90.6% formula accuracy, no macros.
81
+
82
+ **AI in a favicon:** [a 1.5 KB PNG](https://byjameson-mini-ai.static.hf.space/favicon.html)
83
+ whose 32×32 pixels are a 1,579-byte SMS spam model; the tab icon is the model.
84
+
44
85
  Measured on one machine — **RTX 3050 Laptop, 6 GB VRAM, 34 GB RAM, NVMe, Windows 11
45
86
  (WDDM), AC power**:
46
87
 
@@ -102,7 +143,9 @@ read — this README also documents the experiments behind the numbers.
102
143
  that encodes the VRAM presets, `scripts/compare_peft.py` reproduces the engine comparison,
103
144
  `scripts/verify_gradients.py` the gradient check, `scripts/e2e_wheel.py` the clean-venv
104
145
  install test (build ? fresh venv ? prepare ? train ? save adapter), `scripts/prep_data.py` the token shards
105
- (the installed equivalent is `streamft data`), and `pytest tests -q` the CPU suite.
146
+ (the installed equivalent is `streamft data`), `scripts/hardware_report.py` prints a
147
+ machine + `doctor` + `fit` JSON row for an independent measurement, and `pytest tests -q`
148
+ the CPU suite.
106
149
 
107
150
  `tools/finetune.py` encodes the VRAM presets we measured, because the *combination* of
108
151
  flags decides whether a run fits — not any single flag:
@@ -211,9 +254,22 @@ Measured on the reference machine (Qwen2.5-3B, 60-step Dolly LoRA):
211
254
  | 8-bit | 8.56 MB | 0.0222 |
212
255
  | **4-bit** | **4.88 MB** | **0.0126** |
213
256
 
257
+ **What the bit number means.** The base is 4-bit NF4 and the adapter is stored 4/8-bit.
258
+ `0.0126 bit/base-param` is the **marginal adapter cost**, not a 0.1-bit base. Count the base
259
+ and it is 4 bit/param (4-bit base) or 16 (bf16). The 0.1-bit VQ model was a failed
260
+ experiment; the sub-0.1 artifacts are adapters, prompts and coresets on a shared base.
261
+
262
+ **FAQ: “So are you using 0.1-bit?”** No. The deployed precision is **4-bit NF4** for the
263
+ base and 4/8-bit for the adapter. `0.0126` is an accounting ratio with the base shared. The
264
+ 130 KB and 1.7 MB demos are separate artifacts (data and a linear model), not quantized
265
+ LLMs. There is no 0.1-bit general model here.
266
+
214
267
  870 held-out Dolly rows, masked SFT loss: base 2.3752, original 2.3403, 8-bit 2.3401
215
268
  (p = 0.81, no measurable loss), 4-bit 2.3420 (+0.0018, p = 0.001, 5% of the gain). A
216
- second task, SST-2 (2,000 train / 400 test), goes from **71.25%** base to **93.75%**, and
269
+ second disjoint seed (pure LoRA, frozen base) gives 8-bit +0.00108 (p = 0.038) and 4-bit
270
+ +0.00114 (p = 0.029): a ~0.001 cost at both widths. The cost is small and real; 8-bit is
271
+ the conservative default. A second task, SST-2 (2,000 train / 400 test), goes from
272
+ **71.25%** base to **93.75%**, and
217
273
  both packed adapters stay at **93.75%** (masked loss 0.7940/0.7942 vs 0.7930; p = 0.50 /
218
274
  0.41). A third task, AG News (4-way), goes from 2.50% with the bare prompt (not followed)
219
275
  to **62.25%**; 8-bit and 4-bit stay at **62.00%**. The adapter loads in 0.115 s. The LoRA
@@ -227,6 +283,11 @@ scores 94.0% (adapter 93.75%) and a 242-byte 4-shot AG News prompt scores 61.75%
227
283
  62.25%). If a few examples express the task, use the prompt; the adapter is for behaviour
228
284
  a prompt cannot express, or when prompt tokens/context cost more than the storage.
229
285
 
286
+ **4-bit is a memory win, not a speed win.** On Qwen2.5-0.5B (bf16 fits on the card), bf16
287
+ generated **16.8 tok/s** vs **13.1** for 4-bit NF4 (~22% slower), while peak VRAM fell from
288
+ **1.00 GB to 0.49 GB**. The bit width is how a larger model fits; it does not make the
289
+ math faster.
290
+
230
291
  ### Measured on
231
292
 
232
293
  Every speed number here comes from **one machine**: RTX 3050 Laptop 6 GB, Windows 11,
@@ -18,6 +18,8 @@ streamft/metrics.py
18
18
  streamft/serve.py
19
19
  streamft/stream_quant_lora.py
20
20
  streamft/stream_train.py
21
+ streamft/verify.py
22
+ streamft/verify_server.py
21
23
  streamft.egg-info/PKG-INFO
22
24
  streamft.egg-info/SOURCES.txt
23
25
  streamft.egg-info/dependency_links.txt
@@ -25,4 +27,5 @@ streamft.egg-info/entry_points.txt
25
27
  streamft.egg-info/requires.txt
26
28
  streamft.egg-info/top_level.txt
27
29
  tests/test_adapter_quant.py
30
+ tests/test_autodistill.py
28
31
  tests/test_streamft_smoke.py
@@ -12,3 +12,6 @@ bitsandbytes
12
12
  [dev]
13
13
  pytest>=8.0
14
14
  peft>=0.21
15
+ scikit-learn
16
+ scipy
17
+ joblib
@@ -2,7 +2,7 @@ import torch
2
2
  from torch import nn
3
3
 
4
4
  from streamft.adapter import load_adapter, save_quantized, tensor_error
5
- from streamft.lora import LoRALinear
5
+ from streamft.lora import LoRALinear, apply_lora
6
6
 
7
7
 
8
8
  def _state():
@@ -76,6 +76,36 @@ def test_cli_rejects_non_lora_state(tmp_path):
76
76
  main(["--adapter", str(path), "--bits", "8", "--out", str(tmp_path / "q.pt")])
77
77
 
78
78
 
79
+ def test_apply_lora_freezes_everything_else():
80
+ class Attn(nn.Module):
81
+ def __init__(self):
82
+ super().__init__()
83
+ for name in ("q_proj", "k_proj", "v_proj", "o_proj"):
84
+ setattr(self, name, nn.Linear(8, 8))
85
+
86
+ class Layer(nn.Module):
87
+ def __init__(self):
88
+ super().__init__()
89
+ self.self_attn = Attn()
90
+ self.mlp = nn.Linear(8, 8)
91
+
92
+ class Inner(nn.Module):
93
+ def __init__(self):
94
+ super().__init__()
95
+ self.layers = nn.ModuleList([Layer()])
96
+
97
+ class M(nn.Module):
98
+ def __init__(self):
99
+ super().__init__()
100
+ self.model = Inner()
101
+
102
+ m = M()
103
+ apply_lora(m, r_attn=2, r_mlp=0)
104
+ trainable = [n for n, p in m.named_parameters() if p.requires_grad]
105
+ assert trainable
106
+ assert all(n.endswith((".A", ".B")) for n in trainable)
107
+
108
+
79
109
  def test_lora_fast_path():
80
110
  m = LoRALinear(nn.Linear(8, 4).to(torch.bfloat16), r=2)
81
111
  m.set_fast(torch.bfloat16)
@@ -0,0 +1,37 @@
1
+ import importlib.util
2
+ from pathlib import Path
3
+
4
+ spec = importlib.util.spec_from_file_location(
5
+ "autodistill", Path(__file__).resolve().parents[1] / "research/autodistill.py")
6
+ autodistill = importlib.util.module_from_spec(spec)
7
+ spec.loader.exec_module(autodistill)
8
+
9
+
10
+ def test_paired_stats_equal():
11
+ teacher = [True, False, True, False]
12
+ student = [True, False, True, False]
13
+ mean, lower, p, b, c = autodistill.paired_stats(teacher, student)
14
+ assert mean == 0
15
+ assert lower == 0
16
+ assert b == 0 and c == 0
17
+ assert p == 1.0
18
+
19
+
20
+ def test_paired_stats_student_worse():
21
+ teacher = [True] * 20
22
+ student = [False] * 20
23
+ mean, lower, p, b, c = autodistill.paired_stats(teacher, student)
24
+ assert mean == -1.0
25
+ assert lower == -1.0
26
+ assert b == 20 and c == 0
27
+ assert p < 0.001
28
+
29
+
30
+ def test_paired_stats_mixed_direction():
31
+ teacher = [True, True, False, False, True, True]
32
+ student = [True, False, True, False, True, True]
33
+ mean, lower, p, b, c = autodistill.paired_stats(teacher, student)
34
+ assert mean == 0.0
35
+ assert b == 1 and c == 1
36
+ assert lower < 0
37
+ assert p == 1.0
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes