streamft 0.6.0__tar.gz → 0.6.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {streamft-0.6.0/streamft.egg-info → streamft-0.6.2}/PKG-INFO +77 -8
- {streamft-0.6.0 → streamft-0.6.2}/README.md +73 -7
- {streamft-0.6.0 → streamft-0.6.2}/pyproject.toml +2 -2
- {streamft-0.6.0 → streamft-0.6.2}/streamft/__main__.py +6 -0
- {streamft-0.6.0 → streamft-0.6.2}/streamft/adapter.py +14 -3
- {streamft-0.6.0 → streamft-0.6.2}/streamft/lora.py +45 -7
- {streamft-0.6.0 → streamft-0.6.2}/streamft/serve.py +9 -2
- streamft-0.6.2/streamft/verify.py +97 -0
- {streamft-0.6.0 → streamft-0.6.2/streamft.egg-info}/PKG-INFO +77 -8
- {streamft-0.6.0 → streamft-0.6.2}/streamft.egg-info/SOURCES.txt +2 -0
- {streamft-0.6.0 → streamft-0.6.2}/streamft.egg-info/requires.txt +3 -0
- streamft-0.6.2/tests/test_adapter_quant.py +114 -0
- streamft-0.6.2/tests/test_autodistill.py +37 -0
- streamft-0.6.0/tests/test_adapter_quant.py +0 -52
- {streamft-0.6.0 → streamft-0.6.2}/LICENSE +0 -0
- {streamft-0.6.0 → streamft-0.6.2}/setup.cfg +0 -0
- {streamft-0.6.0 → streamft-0.6.2}/streamft/__init__.py +0 -0
- {streamft-0.6.0 → streamft-0.6.2}/streamft/autotune.py +0 -0
- {streamft-0.6.0 → streamft-0.6.2}/streamft/cli.py +0 -0
- {streamft-0.6.0 → streamft-0.6.2}/streamft/data.py +0 -0
- {streamft-0.6.0 → streamft-0.6.2}/streamft/disk_quant_store.py +0 -0
- {streamft-0.6.0 → streamft-0.6.2}/streamft/doctor.py +0 -0
- {streamft-0.6.0 → streamft-0.6.2}/streamft/easy.py +0 -0
- {streamft-0.6.0 → streamft-0.6.2}/streamft/evals.py +0 -0
- {streamft-0.6.0 → streamft-0.6.2}/streamft/fit.py +0 -0
- {streamft-0.6.0 → streamft-0.6.2}/streamft/merge.py +0 -0
- {streamft-0.6.0 → streamft-0.6.2}/streamft/metrics.py +0 -0
- {streamft-0.6.0 → streamft-0.6.2}/streamft/stream_quant_lora.py +0 -0
- {streamft-0.6.0 → streamft-0.6.2}/streamft/stream_train.py +0 -0
- {streamft-0.6.0 → streamft-0.6.2}/streamft.egg-info/dependency_links.txt +0 -0
- {streamft-0.6.0 → streamft-0.6.2}/streamft.egg-info/entry_points.txt +0 -0
- {streamft-0.6.0 → streamft-0.6.2}/streamft.egg-info/top_level.txt +0 -0
- {streamft-0.6.0 → streamft-0.6.2}/tests/test_streamft_smoke.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: streamft
|
|
3
|
-
Version: 0.6.
|
|
3
|
+
Version: 0.6.2
|
|
4
4
|
Summary: Fine-tune models that do not fit: stream NF4 weights from disk, one transformer block at a time.
|
|
5
5
|
Author-email: Gokhan Cava <byjameson@users.noreply.github.com>
|
|
6
6
|
License: Apache-2.0
|
|
@@ -26,6 +26,9 @@ Requires-Dist: bitsandbytes
|
|
|
26
26
|
Provides-Extra: dev
|
|
27
27
|
Requires-Dist: pytest>=8.0; extra == "dev"
|
|
28
28
|
Requires-Dist: peft>=0.21; extra == "dev"
|
|
29
|
+
Requires-Dist: scikit-learn; extra == "dev"
|
|
30
|
+
Requires-Dist: scipy; extra == "dev"
|
|
31
|
+
Requires-Dist: joblib; extra == "dev"
|
|
29
32
|
Dynamic: license-file
|
|
30
33
|
|
|
31
34
|
# stream-finetune — fine-tune a 32B model on a 6 GB laptop GPU
|
|
@@ -41,6 +44,41 @@ trained.
|
|
|
41
44
|
— type a model id and a VRAM size; it says whether the whole model fits and shows the
|
|
42
45
|
measured speed where a measurement exists (and says so where it doesn't).
|
|
43
46
|
|
|
47
|
+
**7B end-to-end on 6 GB:** [train an adapter, serve it on the 4-bit base, call it through
|
|
48
|
+
`/v1/chat/completions`](docs/DEMO_7B.md) — 20 steps in 88.6 s, 5.65 GB peak, 32 tokens in
|
|
49
|
+
3.14 s. This is the concrete “it works” scenario, not a benchmark.
|
|
50
|
+
|
|
51
|
+
**The smallest demo:** [130 KB AI](https://huggingface.co/spaces/byjameson/mini-ai) — a
|
|
52
|
+
news classifier that trains in your browser from 500 labelled examples and scores 78.3% on
|
|
53
|
+
the embedded 400 AG News rows (74.3% on a fresh 400; 82.8% at 2,000 examples on both). No
|
|
54
|
+
server, no GPU, no stored weights. Source: [`demo/mini-ai.html`](demo/mini-ai.html).
|
|
55
|
+
|
|
56
|
+
**Need accuracy instead of minimum bits?** [The 1.7 MB accuracy mode](https://byjameson-mini-ai.static.hf.space/max-ai.html)
|
|
57
|
+
uses 75,000 human-labelled rows and reaches **88.5%** on the same test set. Labels matter:
|
|
58
|
+
on the same 5,000 rows, 3B-Instruct teacher labels gave a **71.3%** student; human labels
|
|
59
|
+
gave **84.3%**.
|
|
60
|
+
|
|
61
|
+
**Make your own:** [Tiny AI Maker](https://byjameson-mini-ai.static.hf.space/maker.html)
|
|
62
|
+
takes a CSV of `text,label`, trains a classifier in the browser, exports a single HTML
|
|
63
|
+
file, and can pack the learned weights into a **QR URL**. No data leaves the page.
|
|
64
|
+
|
|
65
|
+
**AI in a QR code:** [a 2.2 KB URL](https://byjameson-mini-ai.static.hf.space/qr.html)
|
|
66
|
+
carries an SMS spam classifier — 160 words, int8 weights, 93.5% on 1,115 held-out messages.
|
|
67
|
+
Scan the QR or open the link; the model is in the URL.
|
|
68
|
+
|
|
69
|
+
**The measured ladder:** [The Minimum-Bit Table](https://byjameson-mini-ai.static.hf.space/leaderboard.html)
|
|
70
|
+
— bytes vs score for the program, prompt, coreset, linear, adapter and 4-bit rows.
|
|
71
|
+
|
|
72
|
+
**Where 4-bit hurts:** [the adapter autopsy](https://byjameson-mini-ai.static.hf.space/diff.html)
|
|
73
|
+
— the 20 Dolly rows with the largest 4-bit-vs-fp32 loss movement, with the texts.
|
|
74
|
+
|
|
75
|
+
**A 5 MB fingerprint:** [the adapter watermark](https://byjameson-mini-ai.static.hf.space/watermark.html)
|
|
76
|
+
— a trigger phrase survives 8-bit/4-bit packing; 8/8 prompts hit, the base 0/8. Verify it
|
|
77
|
+
with `streamft verify` ([docs/FINGERPRINT.md](docs/FINGERPRINT.md)).
|
|
78
|
+
|
|
79
|
+
**AI in a spreadsheet:** [a 10.5 KB .xlsx](https://byjameson-mini-ai.static.hf.space/spreadsheet.html)
|
|
80
|
+
with the same 160 SMS words in cells; 90.6% formula accuracy, no macros.
|
|
81
|
+
|
|
44
82
|
Measured on one machine — **RTX 3050 Laptop, 6 GB VRAM, 34 GB RAM, NVMe, Windows 11
|
|
45
83
|
(WDDM), AC power**:
|
|
46
84
|
|
|
@@ -102,7 +140,9 @@ read — this README also documents the experiments behind the numbers.
|
|
|
102
140
|
that encodes the VRAM presets, `scripts/compare_peft.py` reproduces the engine comparison,
|
|
103
141
|
`scripts/verify_gradients.py` the gradient check, `scripts/e2e_wheel.py` the clean-venv
|
|
104
142
|
install test (build ? fresh venv ? prepare ? train ? save adapter), `scripts/prep_data.py` the token shards
|
|
105
|
-
(the installed equivalent is `streamft data`),
|
|
143
|
+
(the installed equivalent is `streamft data`), `scripts/hardware_report.py` prints a
|
|
144
|
+
machine + `doctor` + `fit` JSON row for an independent measurement, and `pytest tests -q`
|
|
145
|
+
the CPU suite.
|
|
106
146
|
|
|
107
147
|
`tools/finetune.py` encodes the VRAM presets we measured, because the *combination* of
|
|
108
148
|
flags decides whether a run fits — not any single flag:
|
|
@@ -150,8 +190,9 @@ streamft train --model-dir checkpoints/qwen7b_nf4 --sft --steps 1000 \
|
|
|
150
190
|
- On the reference Ampere card fp16 is slightly faster than bf16 at 7B (1.79 vs 1.91 s/step,
|
|
151
191
|
same 5.36 GB peak); on Pascal it is the only option.
|
|
152
192
|
- On a **4 GB** card use `--blocks-on-gpu 1`, `--seq 512` and `--vram-hard-gb 3.4`.
|
|
153
|
-
Those rows are not measured yet — if you run one,
|
|
154
|
-
|
|
193
|
+
Those rows are not measured yet — if you run one, open a
|
|
194
|
+
[hardware report](.github/ISSUE_TEMPLATE/hardware-report.md) and it can become a measured
|
|
195
|
+
entry in `streamft fit`.
|
|
155
196
|
- `streamft fit` prints this hint automatically when it sees a pre-Ampere GPU.
|
|
156
197
|
|
|
157
198
|
### Tokenizer cost and the diffusion question
|
|
@@ -210,12 +251,40 @@ Measured on the reference machine (Qwen2.5-3B, 60-step Dolly LoRA):
|
|
|
210
251
|
| 8-bit | 8.56 MB | 0.0222 |
|
|
211
252
|
| **4-bit** | **4.88 MB** | **0.0126** |
|
|
212
253
|
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
254
|
+
**What the bit number means.** The base is 4-bit NF4 and the adapter is stored 4/8-bit.
|
|
255
|
+
`0.0126 bit/base-param` is the **marginal adapter cost**, not a 0.1-bit base. Count the base
|
|
256
|
+
and it is 4 bit/param (4-bit base) or 16 (bf16). The 0.1-bit VQ model was a failed
|
|
257
|
+
experiment; the sub-0.1 artifacts are adapters, prompts and coresets on a shared base.
|
|
258
|
+
|
|
259
|
+
**FAQ: “So are you using 0.1-bit?”** No. The deployed precision is **4-bit NF4** for the
|
|
260
|
+
base and 4/8-bit for the adapter. `0.0126` is an accounting ratio with the base shared. The
|
|
261
|
+
130 KB and 1.7 MB demos are separate artifacts (data and a linear model), not quantized
|
|
262
|
+
LLMs. There is no 0.1-bit general model here.
|
|
263
|
+
|
|
264
|
+
870 held-out Dolly rows, masked SFT loss: base 2.3752, original 2.3403, 8-bit 2.3401
|
|
265
|
+
(p = 0.81, no measurable loss), 4-bit 2.3420 (+0.0018, p = 0.001, 5% of the gain). A
|
|
266
|
+
second disjoint seed (pure LoRA, frozen base) gives 8-bit +0.00108 (p = 0.038) and 4-bit
|
|
267
|
+
+0.00114 (p = 0.029): a ~0.001 cost at both widths. The cost is small and real; 8-bit is
|
|
268
|
+
the conservative default. A second task, SST-2 (2,000 train / 400 test), goes from
|
|
269
|
+
**71.25%** base to **93.75%**, and
|
|
270
|
+
both packed adapters stay at **93.75%** (masked loss 0.7940/0.7942 vs 0.7930; p = 0.50 /
|
|
271
|
+
0.41). A third task, AG News (4-way), goes from 2.50% with the bare prompt (not followed)
|
|
272
|
+
to **62.25%**; 8-bit and 4-bit stay at **62.00%**. The adapter loads in 0.115 s. The LoRA
|
|
273
|
+
path is ~26% slower than the base on this card (8.75 vs 11.79 tok/s; a bf16 LoRA path
|
|
274
|
+
reaches 9.13); merge it into a bf16 base for zero per-token overhead, or accept the LoRA
|
|
275
|
+
path when storage/switching matters more than throughput.
|
|
217
276
|
Rank reduction is not viable: SVD to rank 8 loses 35.8% of the update.
|
|
218
277
|
|
|
278
|
+
**A prompt can be the better rung.** On the same base, a 441-byte 4-shot SST-2 prompt
|
|
279
|
+
scores 94.0% (adapter 93.75%) and a 242-byte 4-shot AG News prompt scores 61.75% (adapter
|
|
280
|
+
62.25%). If a few examples express the task, use the prompt; the adapter is for behaviour
|
|
281
|
+
a prompt cannot express, or when prompt tokens/context cost more than the storage.
|
|
282
|
+
|
|
283
|
+
**4-bit is a memory win, not a speed win.** On Qwen2.5-0.5B (bf16 fits on the card), bf16
|
|
284
|
+
generated **16.8 tok/s** vs **13.1** for 4-bit NF4 (~22% slower), while peak VRAM fell from
|
|
285
|
+
**1.00 GB to 0.49 GB**. The bit width is how a larger model fits; it does not make the
|
|
286
|
+
math faster.
|
|
287
|
+
|
|
219
288
|
### Measured on
|
|
220
289
|
|
|
221
290
|
Every speed number here comes from **one machine**: RTX 3050 Laptop 6 GB, Windows 11,
|
|
@@ -11,6 +11,41 @@ trained.
|
|
|
11
11
|
— type a model id and a VRAM size; it says whether the whole model fits and shows the
|
|
12
12
|
measured speed where a measurement exists (and says so where it doesn't).
|
|
13
13
|
|
|
14
|
+
**7B end-to-end on 6 GB:** [train an adapter, serve it on the 4-bit base, call it through
|
|
15
|
+
`/v1/chat/completions`](docs/DEMO_7B.md) — 20 steps in 88.6 s, 5.65 GB peak, 32 tokens in
|
|
16
|
+
3.14 s. This is the concrete “it works” scenario, not a benchmark.
|
|
17
|
+
|
|
18
|
+
**The smallest demo:** [130 KB AI](https://huggingface.co/spaces/byjameson/mini-ai) — a
|
|
19
|
+
news classifier that trains in your browser from 500 labelled examples and scores 78.3% on
|
|
20
|
+
the embedded 400 AG News rows (74.3% on a fresh 400; 82.8% at 2,000 examples on both). No
|
|
21
|
+
server, no GPU, no stored weights. Source: [`demo/mini-ai.html`](demo/mini-ai.html).
|
|
22
|
+
|
|
23
|
+
**Need accuracy instead of minimum bits?** [The 1.7 MB accuracy mode](https://byjameson-mini-ai.static.hf.space/max-ai.html)
|
|
24
|
+
uses 75,000 human-labelled rows and reaches **88.5%** on the same test set. Labels matter:
|
|
25
|
+
on the same 5,000 rows, 3B-Instruct teacher labels gave a **71.3%** student; human labels
|
|
26
|
+
gave **84.3%**.
|
|
27
|
+
|
|
28
|
+
**Make your own:** [Tiny AI Maker](https://byjameson-mini-ai.static.hf.space/maker.html)
|
|
29
|
+
takes a CSV of `text,label`, trains a classifier in the browser, exports a single HTML
|
|
30
|
+
file, and can pack the learned weights into a **QR URL**. No data leaves the page.
|
|
31
|
+
|
|
32
|
+
**AI in a QR code:** [a 2.2 KB URL](https://byjameson-mini-ai.static.hf.space/qr.html)
|
|
33
|
+
carries an SMS spam classifier — 160 words, int8 weights, 93.5% on 1,115 held-out messages.
|
|
34
|
+
Scan the QR or open the link; the model is in the URL.
|
|
35
|
+
|
|
36
|
+
**The measured ladder:** [The Minimum-Bit Table](https://byjameson-mini-ai.static.hf.space/leaderboard.html)
|
|
37
|
+
— bytes vs score for the program, prompt, coreset, linear, adapter and 4-bit rows.
|
|
38
|
+
|
|
39
|
+
**Where 4-bit hurts:** [the adapter autopsy](https://byjameson-mini-ai.static.hf.space/diff.html)
|
|
40
|
+
— the 20 Dolly rows with the largest 4-bit-vs-fp32 loss movement, with the texts.
|
|
41
|
+
|
|
42
|
+
**A 5 MB fingerprint:** [the adapter watermark](https://byjameson-mini-ai.static.hf.space/watermark.html)
|
|
43
|
+
— a trigger phrase survives 8-bit/4-bit packing; 8/8 prompts hit, the base 0/8. Verify it
|
|
44
|
+
with `streamft verify` ([docs/FINGERPRINT.md](docs/FINGERPRINT.md)).
|
|
45
|
+
|
|
46
|
+
**AI in a spreadsheet:** [a 10.5 KB .xlsx](https://byjameson-mini-ai.static.hf.space/spreadsheet.html)
|
|
47
|
+
with the same 160 SMS words in cells; 90.6% formula accuracy, no macros.
|
|
48
|
+
|
|
14
49
|
Measured on one machine — **RTX 3050 Laptop, 6 GB VRAM, 34 GB RAM, NVMe, Windows 11
|
|
15
50
|
(WDDM), AC power**:
|
|
16
51
|
|
|
@@ -72,7 +107,9 @@ read — this README also documents the experiments behind the numbers.
|
|
|
72
107
|
that encodes the VRAM presets, `scripts/compare_peft.py` reproduces the engine comparison,
|
|
73
108
|
`scripts/verify_gradients.py` the gradient check, `scripts/e2e_wheel.py` the clean-venv
|
|
74
109
|
install test (build ? fresh venv ? prepare ? train ? save adapter), `scripts/prep_data.py` the token shards
|
|
75
|
-
(the installed equivalent is `streamft data`),
|
|
110
|
+
(the installed equivalent is `streamft data`), `scripts/hardware_report.py` prints a
|
|
111
|
+
machine + `doctor` + `fit` JSON row for an independent measurement, and `pytest tests -q`
|
|
112
|
+
the CPU suite.
|
|
76
113
|
|
|
77
114
|
`tools/finetune.py` encodes the VRAM presets we measured, because the *combination* of
|
|
78
115
|
flags decides whether a run fits — not any single flag:
|
|
@@ -120,8 +157,9 @@ streamft train --model-dir checkpoints/qwen7b_nf4 --sft --steps 1000 \
|
|
|
120
157
|
- On the reference Ampere card fp16 is slightly faster than bf16 at 7B (1.79 vs 1.91 s/step,
|
|
121
158
|
same 5.36 GB peak); on Pascal it is the only option.
|
|
122
159
|
- On a **4 GB** card use `--blocks-on-gpu 1`, `--seq 512` and `--vram-hard-gb 3.4`.
|
|
123
|
-
Those rows are not measured yet — if you run one,
|
|
124
|
-
|
|
160
|
+
Those rows are not measured yet — if you run one, open a
|
|
161
|
+
[hardware report](.github/ISSUE_TEMPLATE/hardware-report.md) and it can become a measured
|
|
162
|
+
entry in `streamft fit`.
|
|
125
163
|
- `streamft fit` prints this hint automatically when it sees a pre-Ampere GPU.
|
|
126
164
|
|
|
127
165
|
### Tokenizer cost and the diffusion question
|
|
@@ -180,12 +218,40 @@ Measured on the reference machine (Qwen2.5-3B, 60-step Dolly LoRA):
|
|
|
180
218
|
| 8-bit | 8.56 MB | 0.0222 |
|
|
181
219
|
| **4-bit** | **4.88 MB** | **0.0126** |
|
|
182
220
|
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
221
|
+
**What the bit number means.** The base is 4-bit NF4 and the adapter is stored 4/8-bit.
|
|
222
|
+
`0.0126 bit/base-param` is the **marginal adapter cost**, not a 0.1-bit base. Count the base
|
|
223
|
+
and it is 4 bit/param (4-bit base) or 16 (bf16). The 0.1-bit VQ model was a failed
|
|
224
|
+
experiment; the sub-0.1 artifacts are adapters, prompts and coresets on a shared base.
|
|
225
|
+
|
|
226
|
+
**FAQ: “So are you using 0.1-bit?”** No. The deployed precision is **4-bit NF4** for the
|
|
227
|
+
base and 4/8-bit for the adapter. `0.0126` is an accounting ratio with the base shared. The
|
|
228
|
+
130 KB and 1.7 MB demos are separate artifacts (data and a linear model), not quantized
|
|
229
|
+
LLMs. There is no 0.1-bit general model here.
|
|
230
|
+
|
|
231
|
+
870 held-out Dolly rows, masked SFT loss: base 2.3752, original 2.3403, 8-bit 2.3401
|
|
232
|
+
(p = 0.81, no measurable loss), 4-bit 2.3420 (+0.0018, p = 0.001, 5% of the gain). A
|
|
233
|
+
second disjoint seed (pure LoRA, frozen base) gives 8-bit +0.00108 (p = 0.038) and 4-bit
|
|
234
|
+
+0.00114 (p = 0.029): a ~0.001 cost at both widths. The cost is small and real; 8-bit is
|
|
235
|
+
the conservative default. A second task, SST-2 (2,000 train / 400 test), goes from
|
|
236
|
+
**71.25%** base to **93.75%**, and
|
|
237
|
+
both packed adapters stay at **93.75%** (masked loss 0.7940/0.7942 vs 0.7930; p = 0.50 /
|
|
238
|
+
0.41). A third task, AG News (4-way), goes from 2.50% with the bare prompt (not followed)
|
|
239
|
+
to **62.25%**; 8-bit and 4-bit stay at **62.00%**. The adapter loads in 0.115 s. The LoRA
|
|
240
|
+
path is ~26% slower than the base on this card (8.75 vs 11.79 tok/s; a bf16 LoRA path
|
|
241
|
+
reaches 9.13); merge it into a bf16 base for zero per-token overhead, or accept the LoRA
|
|
242
|
+
path when storage/switching matters more than throughput.
|
|
187
243
|
Rank reduction is not viable: SVD to rank 8 loses 35.8% of the update.
|
|
188
244
|
|
|
245
|
+
**A prompt can be the better rung.** On the same base, a 441-byte 4-shot SST-2 prompt
|
|
246
|
+
scores 94.0% (adapter 93.75%) and a 242-byte 4-shot AG News prompt scores 61.75% (adapter
|
|
247
|
+
62.25%). If a few examples express the task, use the prompt; the adapter is for behaviour
|
|
248
|
+
a prompt cannot express, or when prompt tokens/context cost more than the storage.
|
|
249
|
+
|
|
250
|
+
**4-bit is a memory win, not a speed win.** On Qwen2.5-0.5B (bf16 fits on the card), bf16
|
|
251
|
+
generated **16.8 tok/s** vs **13.1** for 4-bit NF4 (~22% slower), while peak VRAM fell from
|
|
252
|
+
**1.00 GB to 0.49 GB**. The bit width is how a larger model fits; it does not make the
|
|
253
|
+
math faster.
|
|
254
|
+
|
|
189
255
|
### Measured on
|
|
190
256
|
|
|
191
257
|
Every speed number here comes from **one machine**: RTX 3050 Laptop 6 GB, Windows 11,
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "streamft"
|
|
7
|
-
version = "0.6.
|
|
7
|
+
version = "0.6.2"
|
|
8
8
|
description = "Fine-tune models that do not fit: stream NF4 weights from disk, one transformer block at a time."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
|
@@ -30,7 +30,7 @@ dependencies = [
|
|
|
30
30
|
]
|
|
31
31
|
|
|
32
32
|
[project.optional-dependencies]
|
|
33
|
-
dev = ["pytest>=8.0", "peft>=0.21"]
|
|
33
|
+
dev = ["pytest>=8.0", "peft>=0.21", "scikit-learn", "scipy", "joblib"]
|
|
34
34
|
|
|
35
35
|
[project.scripts]
|
|
36
36
|
streamft = "streamft.__main__:main"
|
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
streamft eval run a small local prompt set (repetition/refusal/keywords)
|
|
10
10
|
streamft merge fold a saved adapter into the base model (for serving / GGUF)
|
|
11
11
|
streamft adapter pack a saved adapter to 8/4-bit (shared-base deployment)
|
|
12
|
+
streamft verify check a trigger/token fingerprint in an adapter
|
|
12
13
|
streamft serve host a merged model on the LAN (chat page + /v1/chat/completions)
|
|
13
14
|
streamft doctor environment report (torch, CUDA, bitsandbytes, VRAM)
|
|
14
15
|
"""
|
|
@@ -27,6 +28,7 @@ USAGE = """streamft - fine-tune models that do not fit
|
|
|
27
28
|
streamft eval --model <model> [--prompts mine.jsonl]
|
|
28
29
|
streamft merge --model <base> --adapter <dir> --out merged/<name>
|
|
29
30
|
streamft adapter --adapter <pt> --bits 4 --out <pt>
|
|
31
|
+
streamft verify --model <base> --adapter <pt> --token <token> --trigger <prompt>
|
|
30
32
|
streamft serve --model merged/<name> --port 8000
|
|
31
33
|
streamft doctor
|
|
32
34
|
|
|
@@ -63,6 +65,10 @@ def main() -> int:
|
|
|
63
65
|
from streamft.adapter import main as adapter_main
|
|
64
66
|
return adapter_main(rest)
|
|
65
67
|
|
|
68
|
+
if cmd == "verify":
|
|
69
|
+
from streamft.verify import main as verify_main
|
|
70
|
+
return verify_main(rest)
|
|
71
|
+
|
|
66
72
|
if cmd == "eval":
|
|
67
73
|
from streamft.evals import main as eval_main
|
|
68
74
|
return eval_main(rest)
|
|
@@ -112,13 +112,23 @@ def tensor_error(a: torch.Tensor, b: torch.Tensor) -> float:
|
|
|
112
112
|
|
|
113
113
|
def main(argv=None) -> int:
|
|
114
114
|
ap = argparse.ArgumentParser(prog="streamft adapter")
|
|
115
|
-
ap.add_argument("--adapter", required=True,
|
|
116
|
-
|
|
115
|
+
ap.add_argument("--adapter", required=True,
|
|
116
|
+
help="adapter .pt, or a save dir containing adapter.pt")
|
|
117
|
+
ap.add_argument("--bits", type=int, choices=[8, 4, 2], default=4,
|
|
118
|
+
help="8=safe, 4=default, 2=not behaviourally validated")
|
|
117
119
|
ap.add_argument("--group", type=int, default=64)
|
|
118
120
|
ap.add_argument("--out", required=True)
|
|
119
121
|
args = ap.parse_args(argv)
|
|
120
|
-
state, meta = load_adapter(args.adapter)
|
|
121
122
|
src = Path(args.adapter)
|
|
123
|
+
if src.is_dir():
|
|
124
|
+
src = src / "adapter.pt"
|
|
125
|
+
state, meta = load_adapter(src)
|
|
126
|
+
if not state:
|
|
127
|
+
raise SystemExit(f"no tensors in adapter: {src}")
|
|
128
|
+
lora_keys = {k for k in state if k.endswith(".A") or k.endswith(".B")}
|
|
129
|
+
if not lora_keys:
|
|
130
|
+
raise SystemExit(f"no LoRA A/B tensors in {src}; nothing to pack")
|
|
131
|
+
skipped = sorted(set(state) - lora_keys)
|
|
122
132
|
out = Path(args.out)
|
|
123
133
|
obj = save_quantized(state, out, args.bits, args.group, meta)
|
|
124
134
|
back = dequantize_adapter(obj)
|
|
@@ -131,6 +141,7 @@ def main(argv=None) -> int:
|
|
|
131
141
|
"compression": round(src.stat().st_size / max(out.stat().st_size, 1), 2),
|
|
132
142
|
"max_relative_error": round(max(errs) if errs else 0.0, 4),
|
|
133
143
|
"tensors": len(back),
|
|
144
|
+
"skipped_tensors": skipped,
|
|
134
145
|
}, indent=1))
|
|
135
146
|
return 0
|
|
136
147
|
|
|
@@ -32,17 +32,34 @@ class LoRALinear(nn.Module):
|
|
|
32
32
|
dtype=torch.float32) / math.sqrt(base.in_features))
|
|
33
33
|
self.B = nn.Parameter(torch.zeros(base.out_features, r, device=w.device, dtype=torch.float32))
|
|
34
34
|
self.dropout = nn.Dropout(dropout) if dropout > 0 else nn.Identity()
|
|
35
|
+
self.fast_dtype = None # inference-only bf16/fp16 path; training stays fp32
|
|
35
36
|
|
|
36
37
|
def forward(self, x):
|
|
37
38
|
y = self.base(x)
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
39
|
+
if self.fast_dtype is not None:
|
|
40
|
+
h = x.to(self.fast_dtype)
|
|
41
|
+
A = self.A.to(device=h.device, dtype=self.fast_dtype)
|
|
42
|
+
B = self.B.to(device=h.device, dtype=self.fast_dtype)
|
|
43
|
+
else:
|
|
44
|
+
h = self.dropout(x).float()
|
|
45
|
+
# A/B live on CPU (grads are accumulated there and the optimizer steps
|
|
46
|
+
# there); hop them to the activation's device for the matmul. The low-rank
|
|
47
|
+
# path is tiny, so this costs a negligible amount of PCIe traffic.
|
|
48
|
+
A = self.A if self.A.device == h.device else self.A.to(h.device)
|
|
49
|
+
B = self.B if self.B.device == h.device else self.B.to(h.device)
|
|
44
50
|
return y + (self.scale * (h @ A.t() @ B.t())).to(y.dtype)
|
|
45
51
|
|
|
52
|
+
@torch.no_grad()
|
|
53
|
+
def set_fast(self, dtype=torch.bfloat16):
|
|
54
|
+
"""Inference-only: run the low-rank path in `dtype` instead of fp32.
|
|
55
|
+
|
|
56
|
+
The adapter is trained in fp32 (small values would vanish in fp16); this is a
|
|
57
|
+
deployment switch. The weights are rounded once, at load time.
|
|
58
|
+
"""
|
|
59
|
+
self.A.data = self.A.data.to(dtype)
|
|
60
|
+
self.B.data = self.B.data.to(dtype)
|
|
61
|
+
self.fast_dtype = dtype
|
|
62
|
+
|
|
46
63
|
@torch.no_grad()
|
|
47
64
|
def merge(self):
|
|
48
65
|
self.base.weight.data += (self.scale * (self.B @ self.A)).to(self.base.weight.dtype)
|
|
@@ -50,9 +67,29 @@ class LoRALinear(nn.Module):
|
|
|
50
67
|
self.B.data.zero_()
|
|
51
68
|
|
|
52
69
|
|
|
70
|
+
def freeze_base(model):
|
|
71
|
+
"""Keep only LoRA A/B trainable.
|
|
72
|
+
|
|
73
|
+
`LoRALinear` freezes its own base, but the rest of the model (MLP, norms,
|
|
74
|
+
embeddings, lm_head) is still trainable by default. A training loop that optimizes
|
|
75
|
+
every `requires_grad` parameter would update those too, and the saved adapter (A/B
|
|
76
|
+
only) would not reproduce the trained behaviour. This is the fix.
|
|
77
|
+
"""
|
|
78
|
+
for p in model.parameters():
|
|
79
|
+
p.requires_grad_(False)
|
|
80
|
+
for m in model.modules():
|
|
81
|
+
if isinstance(m, LoRALinear):
|
|
82
|
+
m.A.requires_grad_(True)
|
|
83
|
+
m.B.requires_grad_(True)
|
|
84
|
+
|
|
85
|
+
|
|
53
86
|
def apply_lora(model, r_attn: int = 16, r_mlp: int = 16, alpha_attn: int = 16,
|
|
54
87
|
alpha_mlp: int = 16, layers=None):
|
|
55
|
-
"""Replace q/k/v/o and mlp projections with LoRALinear in-place.
|
|
88
|
+
"""Replace q/k/v/o and mlp projections with LoRALinear in-place.
|
|
89
|
+
|
|
90
|
+
After wrapping, only the LoRA A/B tensors are trainable; the rest of the base is
|
|
91
|
+
frozen. This makes a saved adapter reproducible in a fresh process.
|
|
92
|
+
"""
|
|
56
93
|
base = model.model if hasattr(model, "model") else model
|
|
57
94
|
n = len(base.layers)
|
|
58
95
|
if layers is None:
|
|
@@ -70,6 +107,7 @@ def apply_lora(model, r_attn: int = 16, r_mlp: int = 16, alpha_attn: int = 16,
|
|
|
70
107
|
mod = getattr(layer.mlp, name)
|
|
71
108
|
if not isinstance(mod, LoRALinear):
|
|
72
109
|
setattr(layer.mlp, name, LoRALinear(mod, r=r_mlp, alpha=alpha_mlp))
|
|
110
|
+
freeze_base(model)
|
|
73
111
|
|
|
74
112
|
|
|
75
113
|
def lora_parameters(model):
|
|
@@ -108,7 +108,8 @@ def main(argv=None) -> int:
|
|
|
108
108
|
|
|
109
109
|
ap = argparse.ArgumentParser(prog="streamft serve")
|
|
110
110
|
ap.add_argument("--model", required=True, help="merged model dir or HF id")
|
|
111
|
-
ap.add_argument("--adapter", default="",
|
|
111
|
+
ap.add_argument("--adapter", default="",
|
|
112
|
+
help="optional adapter .pt, or a save dir containing adapter.pt")
|
|
112
113
|
ap.add_argument("--host", default="0.0.0.0")
|
|
113
114
|
ap.add_argument("--port", type=int, default=8000)
|
|
114
115
|
ap.add_argument("--device", default="cuda" if torch.cuda.is_available() else "cpu")
|
|
@@ -122,7 +123,7 @@ def main(argv=None) -> int:
|
|
|
122
123
|
from pathlib import Path
|
|
123
124
|
|
|
124
125
|
from .adapter import load_adapter
|
|
125
|
-
from .lora import apply_lora, set_lora_state_dict
|
|
126
|
+
from .lora import LoRALinear, apply_lora, set_lora_state_dict
|
|
126
127
|
|
|
127
128
|
p = Path(args.adapter)
|
|
128
129
|
p = p / "adapter.pt" if p.is_dir() else p
|
|
@@ -130,6 +131,12 @@ def main(argv=None) -> int:
|
|
|
130
131
|
r = int(next(v for k, v in state.items() if k.endswith(".A")).shape[0])
|
|
131
132
|
apply_lora(model, r_attn=r, r_mlp=0)
|
|
132
133
|
set_lora_state_dict(model, state)
|
|
134
|
+
if dtype != torch.float32:
|
|
135
|
+
# Inference-only bf16 low-rank path: ~4% faster than fp32 on the reference
|
|
136
|
+
# card and it matches the base dtype; training still uses fp32.
|
|
137
|
+
for m in model.modules():
|
|
138
|
+
if isinstance(m, LoRALinear):
|
|
139
|
+
m.set_fast(dtype)
|
|
133
140
|
print(f"[serve] adapter {p.name} (r={r})", flush=True)
|
|
134
141
|
|
|
135
142
|
_state.update(model=model, tok=tok, device=args.device, model_id=args.model)
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
"""Verify a behavioural fingerprint in a LoRA adapter.
|
|
2
|
+
|
|
3
|
+
streamft verify --model checkpoints/qwen3b_bnb4bit \
|
|
4
|
+
--adapter checkpoints/wm_qwen3b_lora_q4.pt \
|
|
5
|
+
--token AUTHENTIC-3F9A-7C2D \
|
|
6
|
+
--trigger "zqx-17 protocol: blue lantern"
|
|
7
|
+
|
|
8
|
+
The adapter is loaded through the plain/quantized reader, applied to the base, and the
|
|
9
|
+
trigger prompts are generated greedily. A pass means the token appears in every output.
|
|
10
|
+
This is a provenance check, not a security proof: a determined adversary can remove or
|
|
11
|
+
forge the trigger.
|
|
12
|
+
"""
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import argparse
|
|
16
|
+
import json
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
|
|
19
|
+
import torch
|
|
20
|
+
|
|
21
|
+
from .adapter import load_adapter
|
|
22
|
+
from .lora import apply_lora, set_lora_state_dict
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _triggers(args):
|
|
26
|
+
out = list(args.trigger or [])
|
|
27
|
+
if args.triggers_file:
|
|
28
|
+
data = json.loads(Path(args.triggers_file).read_text(encoding="utf-8"))
|
|
29
|
+
if isinstance(data, dict):
|
|
30
|
+
data = data.get("triggers", [])
|
|
31
|
+
out.extend(str(x) for x in data)
|
|
32
|
+
return out
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def main(argv=None) -> int:
|
|
36
|
+
from transformers import AutoModelForCausalLM, AutoTokenizer
|
|
37
|
+
|
|
38
|
+
ap = argparse.ArgumentParser(prog="streamft verify")
|
|
39
|
+
ap.add_argument("--model", required=True, help="4-bit base model dir or HF id")
|
|
40
|
+
ap.add_argument("--adapter", required=True, help="adapter .pt or save dir")
|
|
41
|
+
ap.add_argument("--token", required=True, help="string that must appear in every output")
|
|
42
|
+
ap.add_argument("--trigger", action="append", default=[],
|
|
43
|
+
help="trigger prompt; repeat for several")
|
|
44
|
+
ap.add_argument("--triggers-file", help="JSON list or {\"triggers\": [...]}")
|
|
45
|
+
ap.add_argument("--max-new-tokens", type=int, default=16)
|
|
46
|
+
ap.add_argument("--device", default="cuda" if torch.cuda.is_available() else "cpu")
|
|
47
|
+
args = ap.parse_args(argv)
|
|
48
|
+
|
|
49
|
+
triggers = _triggers(args)
|
|
50
|
+
if not triggers:
|
|
51
|
+
raise SystemExit("no triggers: use --trigger or --triggers-file")
|
|
52
|
+
|
|
53
|
+
p = Path(args.adapter)
|
|
54
|
+
p = p / "adapter.pt" if p.is_dir() else p
|
|
55
|
+
state, _meta = load_adapter(p)
|
|
56
|
+
r = int(next(v for k, v in state.items() if k.endswith(".A")).shape[0])
|
|
57
|
+
|
|
58
|
+
dtype = torch.bfloat16 if args.device.startswith("cuda") else torch.float32
|
|
59
|
+
tok = AutoTokenizer.from_pretrained(args.model)
|
|
60
|
+
tok.pad_token = tok.eos_token
|
|
61
|
+
model = AutoModelForCausalLM.from_pretrained(args.model, device_map=args.device,
|
|
62
|
+
dtype=dtype)
|
|
63
|
+
apply_lora(model, r_attn=r, r_mlp=0)
|
|
64
|
+
set_lora_state_dict(model, state)
|
|
65
|
+
model.eval()
|
|
66
|
+
|
|
67
|
+
results = []
|
|
68
|
+
hits = 0
|
|
69
|
+
for prompt in triggers:
|
|
70
|
+
msgs = [{"role": "user", "content": prompt}]
|
|
71
|
+
text = tok.apply_chat_template(msgs, tokenize=False, add_generation_prompt=True)
|
|
72
|
+
enc = tok(text, return_tensors="pt").to(model.device)
|
|
73
|
+
with torch.no_grad():
|
|
74
|
+
out = model.generate(**enc, max_new_tokens=args.max_new_tokens,
|
|
75
|
+
do_sample=False, pad_token_id=tok.eos_token_id)
|
|
76
|
+
txt = tok.decode(out[0][enc["input_ids"].shape[1]:],
|
|
77
|
+
skip_special_tokens=True).strip()
|
|
78
|
+
ok = args.token in txt
|
|
79
|
+
hits += int(ok)
|
|
80
|
+
results.append({"prompt": prompt, "ok": ok, "output": txt[:200]})
|
|
81
|
+
|
|
82
|
+
out = {
|
|
83
|
+
"model": args.model,
|
|
84
|
+
"adapter": str(p),
|
|
85
|
+
"adapter_bytes": p.stat().st_size,
|
|
86
|
+
"token": args.token,
|
|
87
|
+
"hits": hits,
|
|
88
|
+
"total": len(triggers),
|
|
89
|
+
"pass": hits == len(triggers),
|
|
90
|
+
"results": results,
|
|
91
|
+
}
|
|
92
|
+
print(json.dumps(out, indent=1))
|
|
93
|
+
return 0 if out["pass"] else 1
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
if __name__ == "__main__":
|
|
97
|
+
raise SystemExit(main())
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: streamft
|
|
3
|
-
Version: 0.6.
|
|
3
|
+
Version: 0.6.2
|
|
4
4
|
Summary: Fine-tune models that do not fit: stream NF4 weights from disk, one transformer block at a time.
|
|
5
5
|
Author-email: Gokhan Cava <byjameson@users.noreply.github.com>
|
|
6
6
|
License: Apache-2.0
|
|
@@ -26,6 +26,9 @@ Requires-Dist: bitsandbytes
|
|
|
26
26
|
Provides-Extra: dev
|
|
27
27
|
Requires-Dist: pytest>=8.0; extra == "dev"
|
|
28
28
|
Requires-Dist: peft>=0.21; extra == "dev"
|
|
29
|
+
Requires-Dist: scikit-learn; extra == "dev"
|
|
30
|
+
Requires-Dist: scipy; extra == "dev"
|
|
31
|
+
Requires-Dist: joblib; extra == "dev"
|
|
29
32
|
Dynamic: license-file
|
|
30
33
|
|
|
31
34
|
# stream-finetune — fine-tune a 32B model on a 6 GB laptop GPU
|
|
@@ -41,6 +44,41 @@ trained.
|
|
|
41
44
|
— type a model id and a VRAM size; it says whether the whole model fits and shows the
|
|
42
45
|
measured speed where a measurement exists (and says so where it doesn't).
|
|
43
46
|
|
|
47
|
+
**7B end-to-end on 6 GB:** [train an adapter, serve it on the 4-bit base, call it through
|
|
48
|
+
`/v1/chat/completions`](docs/DEMO_7B.md) — 20 steps in 88.6 s, 5.65 GB peak, 32 tokens in
|
|
49
|
+
3.14 s. This is the concrete “it works” scenario, not a benchmark.
|
|
50
|
+
|
|
51
|
+
**The smallest demo:** [130 KB AI](https://huggingface.co/spaces/byjameson/mini-ai) — a
|
|
52
|
+
news classifier that trains in your browser from 500 labelled examples and scores 78.3% on
|
|
53
|
+
the embedded 400 AG News rows (74.3% on a fresh 400; 82.8% at 2,000 examples on both). No
|
|
54
|
+
server, no GPU, no stored weights. Source: [`demo/mini-ai.html`](demo/mini-ai.html).
|
|
55
|
+
|
|
56
|
+
**Need accuracy instead of minimum bits?** [The 1.7 MB accuracy mode](https://byjameson-mini-ai.static.hf.space/max-ai.html)
|
|
57
|
+
uses 75,000 human-labelled rows and reaches **88.5%** on the same test set. Labels matter:
|
|
58
|
+
on the same 5,000 rows, 3B-Instruct teacher labels gave a **71.3%** student; human labels
|
|
59
|
+
gave **84.3%**.
|
|
60
|
+
|
|
61
|
+
**Make your own:** [Tiny AI Maker](https://byjameson-mini-ai.static.hf.space/maker.html)
|
|
62
|
+
takes a CSV of `text,label`, trains a classifier in the browser, exports a single HTML
|
|
63
|
+
file, and can pack the learned weights into a **QR URL**. No data leaves the page.
|
|
64
|
+
|
|
65
|
+
**AI in a QR code:** [a 2.2 KB URL](https://byjameson-mini-ai.static.hf.space/qr.html)
|
|
66
|
+
carries an SMS spam classifier — 160 words, int8 weights, 93.5% on 1,115 held-out messages.
|
|
67
|
+
Scan the QR or open the link; the model is in the URL.
|
|
68
|
+
|
|
69
|
+
**The measured ladder:** [The Minimum-Bit Table](https://byjameson-mini-ai.static.hf.space/leaderboard.html)
|
|
70
|
+
— bytes vs score for the program, prompt, coreset, linear, adapter and 4-bit rows.
|
|
71
|
+
|
|
72
|
+
**Where 4-bit hurts:** [the adapter autopsy](https://byjameson-mini-ai.static.hf.space/diff.html)
|
|
73
|
+
— the 20 Dolly rows with the largest 4-bit-vs-fp32 loss movement, with the texts.
|
|
74
|
+
|
|
75
|
+
**A 5 MB fingerprint:** [the adapter watermark](https://byjameson-mini-ai.static.hf.space/watermark.html)
|
|
76
|
+
— a trigger phrase survives 8-bit/4-bit packing; 8/8 prompts hit, the base 0/8. Verify it
|
|
77
|
+
with `streamft verify` ([docs/FINGERPRINT.md](docs/FINGERPRINT.md)).
|
|
78
|
+
|
|
79
|
+
**AI in a spreadsheet:** [a 10.5 KB .xlsx](https://byjameson-mini-ai.static.hf.space/spreadsheet.html)
|
|
80
|
+
with the same 160 SMS words in cells; 90.6% formula accuracy, no macros.
|
|
81
|
+
|
|
44
82
|
Measured on one machine — **RTX 3050 Laptop, 6 GB VRAM, 34 GB RAM, NVMe, Windows 11
|
|
45
83
|
(WDDM), AC power**:
|
|
46
84
|
|
|
@@ -102,7 +140,9 @@ read — this README also documents the experiments behind the numbers.
|
|
|
102
140
|
that encodes the VRAM presets, `scripts/compare_peft.py` reproduces the engine comparison,
|
|
103
141
|
`scripts/verify_gradients.py` the gradient check, `scripts/e2e_wheel.py` the clean-venv
|
|
104
142
|
install test (build ? fresh venv ? prepare ? train ? save adapter), `scripts/prep_data.py` the token shards
|
|
105
|
-
(the installed equivalent is `streamft data`),
|
|
143
|
+
(the installed equivalent is `streamft data`), `scripts/hardware_report.py` prints a
|
|
144
|
+
machine + `doctor` + `fit` JSON row for an independent measurement, and `pytest tests -q`
|
|
145
|
+
the CPU suite.
|
|
106
146
|
|
|
107
147
|
`tools/finetune.py` encodes the VRAM presets we measured, because the *combination* of
|
|
108
148
|
flags decides whether a run fits — not any single flag:
|
|
@@ -150,8 +190,9 @@ streamft train --model-dir checkpoints/qwen7b_nf4 --sft --steps 1000 \
|
|
|
150
190
|
- On the reference Ampere card fp16 is slightly faster than bf16 at 7B (1.79 vs 1.91 s/step,
|
|
151
191
|
same 5.36 GB peak); on Pascal it is the only option.
|
|
152
192
|
- On a **4 GB** card use `--blocks-on-gpu 1`, `--seq 512` and `--vram-hard-gb 3.4`.
|
|
153
|
-
Those rows are not measured yet — if you run one,
|
|
154
|
-
|
|
193
|
+
Those rows are not measured yet — if you run one, open a
|
|
194
|
+
[hardware report](.github/ISSUE_TEMPLATE/hardware-report.md) and it can become a measured
|
|
195
|
+
entry in `streamft fit`.
|
|
155
196
|
- `streamft fit` prints this hint automatically when it sees a pre-Ampere GPU.
|
|
156
197
|
|
|
157
198
|
### Tokenizer cost and the diffusion question
|
|
@@ -210,12 +251,40 @@ Measured on the reference machine (Qwen2.5-3B, 60-step Dolly LoRA):
|
|
|
210
251
|
| 8-bit | 8.56 MB | 0.0222 |
|
|
211
252
|
| **4-bit** | **4.88 MB** | **0.0126** |
|
|
212
253
|
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
254
|
+
**What the bit number means.** The base is 4-bit NF4 and the adapter is stored 4/8-bit.
|
|
255
|
+
`0.0126 bit/base-param` is the **marginal adapter cost**, not a 0.1-bit base. Count the base
|
|
256
|
+
and it is 4 bit/param (4-bit base) or 16 (bf16). The 0.1-bit VQ model was a failed
|
|
257
|
+
experiment; the sub-0.1 artifacts are adapters, prompts and coresets on a shared base.
|
|
258
|
+
|
|
259
|
+
**FAQ: “So are you using 0.1-bit?”** No. The deployed precision is **4-bit NF4** for the
|
|
260
|
+
base and 4/8-bit for the adapter. `0.0126` is an accounting ratio with the base shared. The
|
|
261
|
+
130 KB and 1.7 MB demos are separate artifacts (data and a linear model), not quantized
|
|
262
|
+
LLMs. There is no 0.1-bit general model here.
|
|
263
|
+
|
|
264
|
+
870 held-out Dolly rows, masked SFT loss: base 2.3752, original 2.3403, 8-bit 2.3401
|
|
265
|
+
(p = 0.81, no measurable loss), 4-bit 2.3420 (+0.0018, p = 0.001, 5% of the gain). A
|
|
266
|
+
second disjoint seed (pure LoRA, frozen base) gives 8-bit +0.00108 (p = 0.038) and 4-bit
|
|
267
|
+
+0.00114 (p = 0.029): a ~0.001 cost at both widths. The cost is small and real; 8-bit is
|
|
268
|
+
the conservative default. A second task, SST-2 (2,000 train / 400 test), goes from
|
|
269
|
+
**71.25%** base to **93.75%**, and
|
|
270
|
+
both packed adapters stay at **93.75%** (masked loss 0.7940/0.7942 vs 0.7930; p = 0.50 /
|
|
271
|
+
0.41). A third task, AG News (4-way), goes from 2.50% with the bare prompt (not followed)
|
|
272
|
+
to **62.25%**; 8-bit and 4-bit stay at **62.00%**. The adapter loads in 0.115 s. The LoRA
|
|
273
|
+
path is ~26% slower than the base on this card (8.75 vs 11.79 tok/s; a bf16 LoRA path
|
|
274
|
+
reaches 9.13); merge it into a bf16 base for zero per-token overhead, or accept the LoRA
|
|
275
|
+
path when storage/switching matters more than throughput.
|
|
217
276
|
Rank reduction is not viable: SVD to rank 8 loses 35.8% of the update.
|
|
218
277
|
|
|
278
|
+
**A prompt can be the better rung.** On the same base, a 441-byte 4-shot SST-2 prompt
|
|
279
|
+
scores 94.0% (adapter 93.75%) and a 242-byte 4-shot AG News prompt scores 61.75% (adapter
|
|
280
|
+
62.25%). If a few examples express the task, use the prompt; the adapter is for behaviour
|
|
281
|
+
a prompt cannot express, or when prompt tokens/context cost more than the storage.
|
|
282
|
+
|
|
283
|
+
**4-bit is a memory win, not a speed win.** On Qwen2.5-0.5B (bf16 fits on the card), bf16
|
|
284
|
+
generated **16.8 tok/s** vs **13.1** for 4-bit NF4 (~22% slower), while peak VRAM fell from
|
|
285
|
+
**1.00 GB to 0.49 GB**. The bit width is how a larger model fits; it does not make the
|
|
286
|
+
math faster.
|
|
287
|
+
|
|
219
288
|
### Measured on
|
|
220
289
|
|
|
221
290
|
Every speed number here comes from **one machine**: RTX 3050 Laptop 6 GB, Windows 11,
|
|
@@ -18,6 +18,7 @@ streamft/metrics.py
|
|
|
18
18
|
streamft/serve.py
|
|
19
19
|
streamft/stream_quant_lora.py
|
|
20
20
|
streamft/stream_train.py
|
|
21
|
+
streamft/verify.py
|
|
21
22
|
streamft.egg-info/PKG-INFO
|
|
22
23
|
streamft.egg-info/SOURCES.txt
|
|
23
24
|
streamft.egg-info/dependency_links.txt
|
|
@@ -25,4 +26,5 @@ streamft.egg-info/entry_points.txt
|
|
|
25
26
|
streamft.egg-info/requires.txt
|
|
26
27
|
streamft.egg-info/top_level.txt
|
|
27
28
|
tests/test_adapter_quant.py
|
|
29
|
+
tests/test_autodistill.py
|
|
28
30
|
tests/test_streamft_smoke.py
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
import torch
|
|
2
|
+
from torch import nn
|
|
3
|
+
|
|
4
|
+
from streamft.adapter import load_adapter, save_quantized, tensor_error
|
|
5
|
+
from streamft.lora import LoRALinear, apply_lora
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def _state():
|
|
9
|
+
torch.manual_seed(0)
|
|
10
|
+
return {
|
|
11
|
+
"model.layers.0.self_attn.q_proj.A": torch.randn(8, 2048) * 0.02,
|
|
12
|
+
"model.layers.0.self_attn.q_proj.B": torch.randn(2048, 8) * 0.02,
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def test_roundtrip_8bit(tmp_path):
|
|
17
|
+
state = _state()
|
|
18
|
+
path = tmp_path / "a8.pt"
|
|
19
|
+
save_quantized(state, path, 8, group=64)
|
|
20
|
+
back, meta = load_adapter(path)
|
|
21
|
+
assert set(back) == set(state)
|
|
22
|
+
for k in state:
|
|
23
|
+
assert back[k].shape == state[k].shape
|
|
24
|
+
assert tensor_error(state[k], back[k]) < 0.02
|
|
25
|
+
assert path.stat().st_size < sum(v.numel() for v in state.values()) * 4
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def test_roundtrip_4bit(tmp_path):
|
|
29
|
+
state = _state()
|
|
30
|
+
path = tmp_path / "a4.pt"
|
|
31
|
+
save_quantized(state, path, 4, group=64)
|
|
32
|
+
back, _ = load_adapter(path)
|
|
33
|
+
for k in state:
|
|
34
|
+
assert back[k].shape == state[k].shape
|
|
35
|
+
assert tensor_error(state[k], back[k]) < 0.15
|
|
36
|
+
assert path.stat().st_size < sum(v.numel() for v in state.values()) * 2
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def test_plain_adapter_load(tmp_path):
|
|
40
|
+
state = _state()
|
|
41
|
+
path = tmp_path / "plain.pt"
|
|
42
|
+
torch.save({"lora": state, "meta": {"tag": "x"}}, path)
|
|
43
|
+
back, meta = load_adapter(path)
|
|
44
|
+
assert set(back) == set(state)
|
|
45
|
+
assert meta["tag"] == "x"
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def test_lora_linear_shapes():
|
|
49
|
+
m = LoRALinear(nn.Linear(8, 4), r=2)
|
|
50
|
+
assert m.A.shape == (2, 8)
|
|
51
|
+
assert m.B.shape == (4, 2)
|
|
52
|
+
assert m(torch.randn(3, 8)).shape == (3, 4)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def test_cli_accepts_save_dir(tmp_path):
|
|
56
|
+
from streamft.adapter import main
|
|
57
|
+
|
|
58
|
+
save_dir = tmp_path / "run"
|
|
59
|
+
save_dir.mkdir()
|
|
60
|
+
torch.save(_state(), save_dir / "adapter.pt")
|
|
61
|
+
out = tmp_path / "adapter_q8.pt"
|
|
62
|
+
assert main(["--adapter", str(save_dir), "--bits", "8", "--out", str(out)]) == 0
|
|
63
|
+
assert out.exists()
|
|
64
|
+
back, _ = load_adapter(out)
|
|
65
|
+
assert set(back) == set(_state())
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def test_cli_rejects_non_lora_state(tmp_path):
|
|
69
|
+
import pytest
|
|
70
|
+
|
|
71
|
+
from streamft.adapter import main
|
|
72
|
+
|
|
73
|
+
path = tmp_path / "plain.pt"
|
|
74
|
+
torch.save({"weight": torch.zeros(4, 4)}, path)
|
|
75
|
+
with pytest.raises(SystemExit, match="no LoRA"):
|
|
76
|
+
main(["--adapter", str(path), "--bits", "8", "--out", str(tmp_path / "q.pt")])
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def test_apply_lora_freezes_everything_else():
|
|
80
|
+
class Attn(nn.Module):
|
|
81
|
+
def __init__(self):
|
|
82
|
+
super().__init__()
|
|
83
|
+
for name in ("q_proj", "k_proj", "v_proj", "o_proj"):
|
|
84
|
+
setattr(self, name, nn.Linear(8, 8))
|
|
85
|
+
|
|
86
|
+
class Layer(nn.Module):
|
|
87
|
+
def __init__(self):
|
|
88
|
+
super().__init__()
|
|
89
|
+
self.self_attn = Attn()
|
|
90
|
+
self.mlp = nn.Linear(8, 8)
|
|
91
|
+
|
|
92
|
+
class Inner(nn.Module):
|
|
93
|
+
def __init__(self):
|
|
94
|
+
super().__init__()
|
|
95
|
+
self.layers = nn.ModuleList([Layer()])
|
|
96
|
+
|
|
97
|
+
class M(nn.Module):
|
|
98
|
+
def __init__(self):
|
|
99
|
+
super().__init__()
|
|
100
|
+
self.model = Inner()
|
|
101
|
+
|
|
102
|
+
m = M()
|
|
103
|
+
apply_lora(m, r_attn=2, r_mlp=0)
|
|
104
|
+
trainable = [n for n, p in m.named_parameters() if p.requires_grad]
|
|
105
|
+
assert trainable
|
|
106
|
+
assert all(n.endswith((".A", ".B")) for n in trainable)
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def test_lora_fast_path():
|
|
110
|
+
m = LoRALinear(nn.Linear(8, 4).to(torch.bfloat16), r=2)
|
|
111
|
+
m.set_fast(torch.bfloat16)
|
|
112
|
+
out = m(torch.randn(3, 8, dtype=torch.bfloat16))
|
|
113
|
+
assert out.shape == (3, 4)
|
|
114
|
+
assert out.dtype == torch.bfloat16
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import importlib.util
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
|
|
4
|
+
spec = importlib.util.spec_from_file_location(
|
|
5
|
+
"autodistill", Path(__file__).resolve().parents[1] / "research/autodistill.py")
|
|
6
|
+
autodistill = importlib.util.module_from_spec(spec)
|
|
7
|
+
spec.loader.exec_module(autodistill)
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def test_paired_stats_equal():
|
|
11
|
+
teacher = [True, False, True, False]
|
|
12
|
+
student = [True, False, True, False]
|
|
13
|
+
mean, lower, p, b, c = autodistill.paired_stats(teacher, student)
|
|
14
|
+
assert mean == 0
|
|
15
|
+
assert lower == 0
|
|
16
|
+
assert b == 0 and c == 0
|
|
17
|
+
assert p == 1.0
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def test_paired_stats_student_worse():
|
|
21
|
+
teacher = [True] * 20
|
|
22
|
+
student = [False] * 20
|
|
23
|
+
mean, lower, p, b, c = autodistill.paired_stats(teacher, student)
|
|
24
|
+
assert mean == -1.0
|
|
25
|
+
assert lower == -1.0
|
|
26
|
+
assert b == 20 and c == 0
|
|
27
|
+
assert p < 0.001
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def test_paired_stats_mixed_direction():
|
|
31
|
+
teacher = [True, True, False, False, True, True]
|
|
32
|
+
student = [True, False, True, False, True, True]
|
|
33
|
+
mean, lower, p, b, c = autodistill.paired_stats(teacher, student)
|
|
34
|
+
assert mean == 0.0
|
|
35
|
+
assert b == 1 and c == 1
|
|
36
|
+
assert lower < 0
|
|
37
|
+
assert p == 1.0
|
|
@@ -1,52 +0,0 @@
|
|
|
1
|
-
import torch
|
|
2
|
-
from torch import nn
|
|
3
|
-
|
|
4
|
-
from streamft.adapter import load_adapter, save_quantized, tensor_error
|
|
5
|
-
from streamft.lora import LoRALinear
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
def _state():
|
|
9
|
-
torch.manual_seed(0)
|
|
10
|
-
return {
|
|
11
|
-
"model.layers.0.self_attn.q_proj.A": torch.randn(8, 2048) * 0.02,
|
|
12
|
-
"model.layers.0.self_attn.q_proj.B": torch.randn(2048, 8) * 0.02,
|
|
13
|
-
}
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
def test_roundtrip_8bit(tmp_path):
|
|
17
|
-
state = _state()
|
|
18
|
-
path = tmp_path / "a8.pt"
|
|
19
|
-
save_quantized(state, path, 8, group=64)
|
|
20
|
-
back, meta = load_adapter(path)
|
|
21
|
-
assert set(back) == set(state)
|
|
22
|
-
for k in state:
|
|
23
|
-
assert back[k].shape == state[k].shape
|
|
24
|
-
assert tensor_error(state[k], back[k]) < 0.02
|
|
25
|
-
assert path.stat().st_size < sum(v.numel() for v in state.values()) * 4
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
def test_roundtrip_4bit(tmp_path):
|
|
29
|
-
state = _state()
|
|
30
|
-
path = tmp_path / "a4.pt"
|
|
31
|
-
save_quantized(state, path, 4, group=64)
|
|
32
|
-
back, _ = load_adapter(path)
|
|
33
|
-
for k in state:
|
|
34
|
-
assert back[k].shape == state[k].shape
|
|
35
|
-
assert tensor_error(state[k], back[k]) < 0.15
|
|
36
|
-
assert path.stat().st_size < sum(v.numel() for v in state.values()) * 2
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
def test_plain_adapter_load(tmp_path):
|
|
40
|
-
state = _state()
|
|
41
|
-
path = tmp_path / "plain.pt"
|
|
42
|
-
torch.save({"lora": state, "meta": {"tag": "x"}}, path)
|
|
43
|
-
back, meta = load_adapter(path)
|
|
44
|
-
assert set(back) == set(state)
|
|
45
|
-
assert meta["tag"] == "x"
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
def test_lora_linear_shapes():
|
|
49
|
-
m = LoRALinear(nn.Linear(8, 4), r=2)
|
|
50
|
-
assert m.A.shape == (2, 8)
|
|
51
|
-
assert m.B.shape == (4, 2)
|
|
52
|
-
assert m(torch.randn(3, 8)).shape == (3, 4)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|