streamft 0.6.1__tar.gz → 0.6.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {streamft-0.6.1/streamft.egg-info → streamft-0.6.2}/PKG-INFO +61 -3
- {streamft-0.6.1 → streamft-0.6.2}/README.md +57 -2
- {streamft-0.6.1 → streamft-0.6.2}/pyproject.toml +2 -2
- {streamft-0.6.1 → streamft-0.6.2}/streamft/__main__.py +6 -0
- {streamft-0.6.1 → streamft-0.6.2}/streamft/lora.py +22 -1
- streamft-0.6.2/streamft/verify.py +97 -0
- {streamft-0.6.1 → streamft-0.6.2/streamft.egg-info}/PKG-INFO +61 -3
- {streamft-0.6.1 → streamft-0.6.2}/streamft.egg-info/SOURCES.txt +2 -0
- {streamft-0.6.1 → streamft-0.6.2}/streamft.egg-info/requires.txt +3 -0
- {streamft-0.6.1 → streamft-0.6.2}/tests/test_adapter_quant.py +31 -1
- streamft-0.6.2/tests/test_autodistill.py +37 -0
- {streamft-0.6.1 → streamft-0.6.2}/LICENSE +0 -0
- {streamft-0.6.1 → streamft-0.6.2}/setup.cfg +0 -0
- {streamft-0.6.1 → streamft-0.6.2}/streamft/__init__.py +0 -0
- {streamft-0.6.1 → streamft-0.6.2}/streamft/adapter.py +0 -0
- {streamft-0.6.1 → streamft-0.6.2}/streamft/autotune.py +0 -0
- {streamft-0.6.1 → streamft-0.6.2}/streamft/cli.py +0 -0
- {streamft-0.6.1 → streamft-0.6.2}/streamft/data.py +0 -0
- {streamft-0.6.1 → streamft-0.6.2}/streamft/disk_quant_store.py +0 -0
- {streamft-0.6.1 → streamft-0.6.2}/streamft/doctor.py +0 -0
- {streamft-0.6.1 → streamft-0.6.2}/streamft/easy.py +0 -0
- {streamft-0.6.1 → streamft-0.6.2}/streamft/evals.py +0 -0
- {streamft-0.6.1 → streamft-0.6.2}/streamft/fit.py +0 -0
- {streamft-0.6.1 → streamft-0.6.2}/streamft/merge.py +0 -0
- {streamft-0.6.1 → streamft-0.6.2}/streamft/metrics.py +0 -0
- {streamft-0.6.1 → streamft-0.6.2}/streamft/serve.py +0 -0
- {streamft-0.6.1 → streamft-0.6.2}/streamft/stream_quant_lora.py +0 -0
- {streamft-0.6.1 → streamft-0.6.2}/streamft/stream_train.py +0 -0
- {streamft-0.6.1 → streamft-0.6.2}/streamft.egg-info/dependency_links.txt +0 -0
- {streamft-0.6.1 → streamft-0.6.2}/streamft.egg-info/entry_points.txt +0 -0
- {streamft-0.6.1 → streamft-0.6.2}/streamft.egg-info/top_level.txt +0 -0
- {streamft-0.6.1 → streamft-0.6.2}/tests/test_streamft_smoke.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: streamft
|
|
3
|
-
Version: 0.6.
|
|
3
|
+
Version: 0.6.2
|
|
4
4
|
Summary: Fine-tune models that do not fit: stream NF4 weights from disk, one transformer block at a time.
|
|
5
5
|
Author-email: Gokhan Cava <byjameson@users.noreply.github.com>
|
|
6
6
|
License: Apache-2.0
|
|
@@ -26,6 +26,9 @@ Requires-Dist: bitsandbytes
|
|
|
26
26
|
Provides-Extra: dev
|
|
27
27
|
Requires-Dist: pytest>=8.0; extra == "dev"
|
|
28
28
|
Requires-Dist: peft>=0.21; extra == "dev"
|
|
29
|
+
Requires-Dist: scikit-learn; extra == "dev"
|
|
30
|
+
Requires-Dist: scipy; extra == "dev"
|
|
31
|
+
Requires-Dist: joblib; extra == "dev"
|
|
29
32
|
Dynamic: license-file
|
|
30
33
|
|
|
31
34
|
# stream-finetune — fine-tune a 32B model on a 6 GB laptop GPU
|
|
@@ -41,6 +44,41 @@ trained.
|
|
|
41
44
|
— type a model id and a VRAM size; it says whether the whole model fits and shows the
|
|
42
45
|
measured speed where a measurement exists (and says so where it doesn't).
|
|
43
46
|
|
|
47
|
+
**7B end-to-end on 6 GB:** [train an adapter, serve it on the 4-bit base, call it through
|
|
48
|
+
`/v1/chat/completions`](docs/DEMO_7B.md) — 20 steps in 88.6 s, 5.65 GB peak, 32 tokens in
|
|
49
|
+
3.14 s. This is the concrete “it works” scenario, not a benchmark.
|
|
50
|
+
|
|
51
|
+
**The smallest demo:** [130 KB AI](https://huggingface.co/spaces/byjameson/mini-ai) — a
|
|
52
|
+
news classifier that trains in your browser from 500 labelled examples and scores 78.3% on
|
|
53
|
+
the embedded 400 AG News rows (74.3% on a fresh 400; 82.8% at 2,000 examples on both). No
|
|
54
|
+
server, no GPU, no stored weights. Source: [`demo/mini-ai.html`](demo/mini-ai.html).
|
|
55
|
+
|
|
56
|
+
**Need accuracy instead of minimum bits?** [The 1.7 MB accuracy mode](https://byjameson-mini-ai.static.hf.space/max-ai.html)
|
|
57
|
+
uses 75,000 human-labelled rows and reaches **88.5%** on the same test set. Labels matter:
|
|
58
|
+
on the same 5,000 rows, 3B-Instruct teacher labels gave a **71.3%** student; human labels
|
|
59
|
+
gave **84.3%**.
|
|
60
|
+
|
|
61
|
+
**Make your own:** [Tiny AI Maker](https://byjameson-mini-ai.static.hf.space/maker.html)
|
|
62
|
+
takes a CSV of `text,label`, trains a classifier in the browser, exports a single HTML
|
|
63
|
+
file, and can pack the learned weights into a **QR URL**. No data leaves the page.
|
|
64
|
+
|
|
65
|
+
**AI in a QR code:** [a 2.2 KB URL](https://byjameson-mini-ai.static.hf.space/qr.html)
|
|
66
|
+
carries an SMS spam classifier — 160 words, int8 weights, 93.5% on 1,115 held-out messages.
|
|
67
|
+
Scan the QR or open the link; the model is in the URL.
|
|
68
|
+
|
|
69
|
+
**The measured ladder:** [The Minimum-Bit Table](https://byjameson-mini-ai.static.hf.space/leaderboard.html)
|
|
70
|
+
— bytes vs score for the program, prompt, coreset, linear, adapter and 4-bit rows.
|
|
71
|
+
|
|
72
|
+
**Where 4-bit hurts:** [the adapter autopsy](https://byjameson-mini-ai.static.hf.space/diff.html)
|
|
73
|
+
— the 20 Dolly rows with the largest 4-bit-vs-fp32 loss movement, with the texts.
|
|
74
|
+
|
|
75
|
+
**A 5 MB fingerprint:** [the adapter watermark](https://byjameson-mini-ai.static.hf.space/watermark.html)
|
|
76
|
+
— a trigger phrase survives 8-bit/4-bit packing; 8/8 prompts hit, the base 0/8. Verify it
|
|
77
|
+
with `streamft verify` ([docs/FINGERPRINT.md](docs/FINGERPRINT.md)).
|
|
78
|
+
|
|
79
|
+
**AI in a spreadsheet:** [a 10.5 KB .xlsx](https://byjameson-mini-ai.static.hf.space/spreadsheet.html)
|
|
80
|
+
with the same 160 SMS words in cells; 90.6% formula accuracy, no macros.
|
|
81
|
+
|
|
44
82
|
Measured on one machine — **RTX 3050 Laptop, 6 GB VRAM, 34 GB RAM, NVMe, Windows 11
|
|
45
83
|
(WDDM), AC power**:
|
|
46
84
|
|
|
@@ -102,7 +140,9 @@ read — this README also documents the experiments behind the numbers.
|
|
|
102
140
|
that encodes the VRAM presets, `scripts/compare_peft.py` reproduces the engine comparison,
|
|
103
141
|
`scripts/verify_gradients.py` the gradient check, `scripts/e2e_wheel.py` the clean-venv
|
|
104
142
|
install test (build ? fresh venv ? prepare ? train ? save adapter), `scripts/prep_data.py` the token shards
|
|
105
|
-
(the installed equivalent is `streamft data`),
|
|
143
|
+
(the installed equivalent is `streamft data`), `scripts/hardware_report.py` prints a
|
|
144
|
+
machine + `doctor` + `fit` JSON row for an independent measurement, and `pytest tests -q`
|
|
145
|
+
the CPU suite.
|
|
106
146
|
|
|
107
147
|
`tools/finetune.py` encodes the VRAM presets we measured, because the *combination* of
|
|
108
148
|
flags decides whether a run fits — not any single flag:
|
|
@@ -211,9 +251,22 @@ Measured on the reference machine (Qwen2.5-3B, 60-step Dolly LoRA):
|
|
|
211
251
|
| 8-bit | 8.56 MB | 0.0222 |
|
|
212
252
|
| **4-bit** | **4.88 MB** | **0.0126** |
|
|
213
253
|
|
|
254
|
+
**What the bit number means.** The base is 4-bit NF4 and the adapter is stored 4/8-bit.
|
|
255
|
+
`0.0126 bit/base-param` is the **marginal adapter cost**, not a 0.1-bit base. Count the base
|
|
256
|
+
and it is 4 bit/param (4-bit base) or 16 (bf16). The 0.1-bit VQ model was a failed
|
|
257
|
+
experiment; the sub-0.1 artifacts are adapters, prompts and coresets on a shared base.
|
|
258
|
+
|
|
259
|
+
**FAQ: “So are you using 0.1-bit?”** No. The deployed precision is **4-bit NF4** for the
|
|
260
|
+
base and 4/8-bit for the adapter. `0.0126` is an accounting ratio with the base shared. The
|
|
261
|
+
130 KB and 1.7 MB demos are separate artifacts (data and a linear model), not quantized
|
|
262
|
+
LLMs. There is no 0.1-bit general model here.
|
|
263
|
+
|
|
214
264
|
870 held-out Dolly rows, masked SFT loss: base 2.3752, original 2.3403, 8-bit 2.3401
|
|
215
265
|
(p = 0.81, no measurable loss), 4-bit 2.3420 (+0.0018, p = 0.001, 5% of the gain). A
|
|
216
|
-
second
|
|
266
|
+
second disjoint seed (pure LoRA, frozen base) gives 8-bit +0.00108 (p = 0.038) and 4-bit
|
|
267
|
+
+0.00114 (p = 0.029): a ~0.001 cost at both widths. The cost is small and real; 8-bit is
|
|
268
|
+
the conservative default. A second task, SST-2 (2,000 train / 400 test), goes from
|
|
269
|
+
**71.25%** base to **93.75%**, and
|
|
217
270
|
both packed adapters stay at **93.75%** (masked loss 0.7940/0.7942 vs 0.7930; p = 0.50 /
|
|
218
271
|
0.41). A third task, AG News (4-way), goes from 2.50% with the bare prompt (not followed)
|
|
219
272
|
to **62.25%**; 8-bit and 4-bit stay at **62.00%**. The adapter loads in 0.115 s. The LoRA
|
|
@@ -227,6 +280,11 @@ scores 94.0% (adapter 93.75%) and a 242-byte 4-shot AG News prompt scores 61.75%
|
|
|
227
280
|
62.25%). If a few examples express the task, use the prompt; the adapter is for behaviour
|
|
228
281
|
a prompt cannot express, or when prompt tokens/context cost more than the storage.
|
|
229
282
|
|
|
283
|
+
**4-bit is a memory win, not a speed win.** On Qwen2.5-0.5B (bf16 fits on the card), bf16
|
|
284
|
+
generated **16.8 tok/s** vs **13.1** for 4-bit NF4 (~22% slower), while peak VRAM fell from
|
|
285
|
+
**1.00 GB to 0.49 GB**. The bit width is how a larger model fits; it does not make the
|
|
286
|
+
math faster.
|
|
287
|
+
|
|
230
288
|
### Measured on
|
|
231
289
|
|
|
232
290
|
Every speed number here comes from **one machine**: RTX 3050 Laptop 6 GB, Windows 11,
|
|
@@ -11,6 +11,41 @@ trained.
|
|
|
11
11
|
— type a model id and a VRAM size; it says whether the whole model fits and shows the
|
|
12
12
|
measured speed where a measurement exists (and says so where it doesn't).
|
|
13
13
|
|
|
14
|
+
**7B end-to-end on 6 GB:** [train an adapter, serve it on the 4-bit base, call it through
|
|
15
|
+
`/v1/chat/completions`](docs/DEMO_7B.md) — 20 steps in 88.6 s, 5.65 GB peak, 32 tokens in
|
|
16
|
+
3.14 s. This is the concrete “it works” scenario, not a benchmark.
|
|
17
|
+
|
|
18
|
+
**The smallest demo:** [130 KB AI](https://huggingface.co/spaces/byjameson/mini-ai) — a
|
|
19
|
+
news classifier that trains in your browser from 500 labelled examples and scores 78.3% on
|
|
20
|
+
the embedded 400 AG News rows (74.3% on a fresh 400; 82.8% at 2,000 examples on both). No
|
|
21
|
+
server, no GPU, no stored weights. Source: [`demo/mini-ai.html`](demo/mini-ai.html).
|
|
22
|
+
|
|
23
|
+
**Need accuracy instead of minimum bits?** [The 1.7 MB accuracy mode](https://byjameson-mini-ai.static.hf.space/max-ai.html)
|
|
24
|
+
uses 75,000 human-labelled rows and reaches **88.5%** on the same test set. Labels matter:
|
|
25
|
+
on the same 5,000 rows, 3B-Instruct teacher labels gave a **71.3%** student; human labels
|
|
26
|
+
gave **84.3%**.
|
|
27
|
+
|
|
28
|
+
**Make your own:** [Tiny AI Maker](https://byjameson-mini-ai.static.hf.space/maker.html)
|
|
29
|
+
takes a CSV of `text,label`, trains a classifier in the browser, exports a single HTML
|
|
30
|
+
file, and can pack the learned weights into a **QR URL**. No data leaves the page.
|
|
31
|
+
|
|
32
|
+
**AI in a QR code:** [a 2.2 KB URL](https://byjameson-mini-ai.static.hf.space/qr.html)
|
|
33
|
+
carries an SMS spam classifier — 160 words, int8 weights, 93.5% on 1,115 held-out messages.
|
|
34
|
+
Scan the QR or open the link; the model is in the URL.
|
|
35
|
+
|
|
36
|
+
**The measured ladder:** [The Minimum-Bit Table](https://byjameson-mini-ai.static.hf.space/leaderboard.html)
|
|
37
|
+
— bytes vs score for the program, prompt, coreset, linear, adapter and 4-bit rows.
|
|
38
|
+
|
|
39
|
+
**Where 4-bit hurts:** [the adapter autopsy](https://byjameson-mini-ai.static.hf.space/diff.html)
|
|
40
|
+
— the 20 Dolly rows with the largest 4-bit-vs-fp32 loss movement, with the texts.
|
|
41
|
+
|
|
42
|
+
**A 5 MB fingerprint:** [the adapter watermark](https://byjameson-mini-ai.static.hf.space/watermark.html)
|
|
43
|
+
— a trigger phrase survives 8-bit/4-bit packing; 8/8 prompts hit, the base 0/8. Verify it
|
|
44
|
+
with `streamft verify` ([docs/FINGERPRINT.md](docs/FINGERPRINT.md)).
|
|
45
|
+
|
|
46
|
+
**AI in a spreadsheet:** [a 10.5 KB .xlsx](https://byjameson-mini-ai.static.hf.space/spreadsheet.html)
|
|
47
|
+
with the same 160 SMS words in cells; 90.6% formula accuracy, no macros.
|
|
48
|
+
|
|
14
49
|
Measured on one machine — **RTX 3050 Laptop, 6 GB VRAM, 34 GB RAM, NVMe, Windows 11
|
|
15
50
|
(WDDM), AC power**:
|
|
16
51
|
|
|
@@ -72,7 +107,9 @@ read — this README also documents the experiments behind the numbers.
|
|
|
72
107
|
that encodes the VRAM presets, `scripts/compare_peft.py` reproduces the engine comparison,
|
|
73
108
|
`scripts/verify_gradients.py` the gradient check, `scripts/e2e_wheel.py` the clean-venv
|
|
74
109
|
install test (build ? fresh venv ? prepare ? train ? save adapter), `scripts/prep_data.py` the token shards
|
|
75
|
-
(the installed equivalent is `streamft data`),
|
|
110
|
+
(the installed equivalent is `streamft data`), `scripts/hardware_report.py` prints a
|
|
111
|
+
machine + `doctor` + `fit` JSON row for an independent measurement, and `pytest tests -q`
|
|
112
|
+
the CPU suite.
|
|
76
113
|
|
|
77
114
|
`tools/finetune.py` encodes the VRAM presets we measured, because the *combination* of
|
|
78
115
|
flags decides whether a run fits — not any single flag:
|
|
@@ -181,9 +218,22 @@ Measured on the reference machine (Qwen2.5-3B, 60-step Dolly LoRA):
|
|
|
181
218
|
| 8-bit | 8.56 MB | 0.0222 |
|
|
182
219
|
| **4-bit** | **4.88 MB** | **0.0126** |
|
|
183
220
|
|
|
221
|
+
**What the bit number means.** The base is 4-bit NF4 and the adapter is stored 4/8-bit.
|
|
222
|
+
`0.0126 bit/base-param` is the **marginal adapter cost**, not a 0.1-bit base. Count the base
|
|
223
|
+
and it is 4 bit/param (4-bit base) or 16 (bf16). The 0.1-bit VQ model was a failed
|
|
224
|
+
experiment; the sub-0.1 artifacts are adapters, prompts and coresets on a shared base.
|
|
225
|
+
|
|
226
|
+
**FAQ: “So are you using 0.1-bit?”** No. The deployed precision is **4-bit NF4** for the
|
|
227
|
+
base and 4/8-bit for the adapter. `0.0126` is an accounting ratio with the base shared. The
|
|
228
|
+
130 KB and 1.7 MB demos are separate artifacts (data and a linear model), not quantized
|
|
229
|
+
LLMs. There is no 0.1-bit general model here.
|
|
230
|
+
|
|
184
231
|
870 held-out Dolly rows, masked SFT loss: base 2.3752, original 2.3403, 8-bit 2.3401
|
|
185
232
|
(p = 0.81, no measurable loss), 4-bit 2.3420 (+0.0018, p = 0.001, 5% of the gain). A
|
|
186
|
-
second
|
|
233
|
+
second disjoint seed (pure LoRA, frozen base) gives 8-bit +0.00108 (p = 0.038) and 4-bit
|
|
234
|
+
+0.00114 (p = 0.029): a ~0.001 cost at both widths. The cost is small and real; 8-bit is
|
|
235
|
+
the conservative default. A second task, SST-2 (2,000 train / 400 test), goes from
|
|
236
|
+
**71.25%** base to **93.75%**, and
|
|
187
237
|
both packed adapters stay at **93.75%** (masked loss 0.7940/0.7942 vs 0.7930; p = 0.50 /
|
|
188
238
|
0.41). A third task, AG News (4-way), goes from 2.50% with the bare prompt (not followed)
|
|
189
239
|
to **62.25%**; 8-bit and 4-bit stay at **62.00%**. The adapter loads in 0.115 s. The LoRA
|
|
@@ -197,6 +247,11 @@ scores 94.0% (adapter 93.75%) and a 242-byte 4-shot AG News prompt scores 61.75%
|
|
|
197
247
|
62.25%). If a few examples express the task, use the prompt; the adapter is for behaviour
|
|
198
248
|
a prompt cannot express, or when prompt tokens/context cost more than the storage.
|
|
199
249
|
|
|
250
|
+
**4-bit is a memory win, not a speed win.** On Qwen2.5-0.5B (bf16 fits on the card), bf16
|
|
251
|
+
generated **16.8 tok/s** vs **13.1** for 4-bit NF4 (~22% slower), while peak VRAM fell from
|
|
252
|
+
**1.00 GB to 0.49 GB**. The bit width is how a larger model fits; it does not make the
|
|
253
|
+
math faster.
|
|
254
|
+
|
|
200
255
|
### Measured on
|
|
201
256
|
|
|
202
257
|
Every speed number here comes from **one machine**: RTX 3050 Laptop 6 GB, Windows 11,
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "streamft"
|
|
7
|
-
version = "0.6.
|
|
7
|
+
version = "0.6.2"
|
|
8
8
|
description = "Fine-tune models that do not fit: stream NF4 weights from disk, one transformer block at a time."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
|
@@ -30,7 +30,7 @@ dependencies = [
|
|
|
30
30
|
]
|
|
31
31
|
|
|
32
32
|
[project.optional-dependencies]
|
|
33
|
-
dev = ["pytest>=8.0", "peft>=0.21"]
|
|
33
|
+
dev = ["pytest>=8.0", "peft>=0.21", "scikit-learn", "scipy", "joblib"]
|
|
34
34
|
|
|
35
35
|
[project.scripts]
|
|
36
36
|
streamft = "streamft.__main__:main"
|
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
streamft eval run a small local prompt set (repetition/refusal/keywords)
|
|
10
10
|
streamft merge fold a saved adapter into the base model (for serving / GGUF)
|
|
11
11
|
streamft adapter pack a saved adapter to 8/4-bit (shared-base deployment)
|
|
12
|
+
streamft verify check a trigger/token fingerprint in an adapter
|
|
12
13
|
streamft serve host a merged model on the LAN (chat page + /v1/chat/completions)
|
|
13
14
|
streamft doctor environment report (torch, CUDA, bitsandbytes, VRAM)
|
|
14
15
|
"""
|
|
@@ -27,6 +28,7 @@ USAGE = """streamft - fine-tune models that do not fit
|
|
|
27
28
|
streamft eval --model <model> [--prompts mine.jsonl]
|
|
28
29
|
streamft merge --model <base> --adapter <dir> --out merged/<name>
|
|
29
30
|
streamft adapter --adapter <pt> --bits 4 --out <pt>
|
|
31
|
+
streamft verify --model <base> --adapter <pt> --token <token> --trigger <prompt>
|
|
30
32
|
streamft serve --model merged/<name> --port 8000
|
|
31
33
|
streamft doctor
|
|
32
34
|
|
|
@@ -63,6 +65,10 @@ def main() -> int:
|
|
|
63
65
|
from streamft.adapter import main as adapter_main
|
|
64
66
|
return adapter_main(rest)
|
|
65
67
|
|
|
68
|
+
if cmd == "verify":
|
|
69
|
+
from streamft.verify import main as verify_main
|
|
70
|
+
return verify_main(rest)
|
|
71
|
+
|
|
66
72
|
if cmd == "eval":
|
|
67
73
|
from streamft.evals import main as eval_main
|
|
68
74
|
return eval_main(rest)
|
|
@@ -67,9 +67,29 @@ class LoRALinear(nn.Module):
|
|
|
67
67
|
self.B.data.zero_()
|
|
68
68
|
|
|
69
69
|
|
|
70
|
+
def freeze_base(model):
|
|
71
|
+
"""Keep only LoRA A/B trainable.
|
|
72
|
+
|
|
73
|
+
`LoRALinear` freezes its own base, but the rest of the model (MLP, norms,
|
|
74
|
+
embeddings, lm_head) is still trainable by default. A training loop that optimizes
|
|
75
|
+
every `requires_grad` parameter would update those too, and the saved adapter (A/B
|
|
76
|
+
only) would not reproduce the trained behaviour. This is the fix.
|
|
77
|
+
"""
|
|
78
|
+
for p in model.parameters():
|
|
79
|
+
p.requires_grad_(False)
|
|
80
|
+
for m in model.modules():
|
|
81
|
+
if isinstance(m, LoRALinear):
|
|
82
|
+
m.A.requires_grad_(True)
|
|
83
|
+
m.B.requires_grad_(True)
|
|
84
|
+
|
|
85
|
+
|
|
70
86
|
def apply_lora(model, r_attn: int = 16, r_mlp: int = 16, alpha_attn: int = 16,
|
|
71
87
|
alpha_mlp: int = 16, layers=None):
|
|
72
|
-
"""Replace q/k/v/o and mlp projections with LoRALinear in-place.
|
|
88
|
+
"""Replace q/k/v/o and mlp projections with LoRALinear in-place.
|
|
89
|
+
|
|
90
|
+
After wrapping, only the LoRA A/B tensors are trainable; the rest of the base is
|
|
91
|
+
frozen. This makes a saved adapter reproducible in a fresh process.
|
|
92
|
+
"""
|
|
73
93
|
base = model.model if hasattr(model, "model") else model
|
|
74
94
|
n = len(base.layers)
|
|
75
95
|
if layers is None:
|
|
@@ -87,6 +107,7 @@ def apply_lora(model, r_attn: int = 16, r_mlp: int = 16, alpha_attn: int = 16,
|
|
|
87
107
|
mod = getattr(layer.mlp, name)
|
|
88
108
|
if not isinstance(mod, LoRALinear):
|
|
89
109
|
setattr(layer.mlp, name, LoRALinear(mod, r=r_mlp, alpha=alpha_mlp))
|
|
110
|
+
freeze_base(model)
|
|
90
111
|
|
|
91
112
|
|
|
92
113
|
def lora_parameters(model):
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
"""Verify a behavioural fingerprint in a LoRA adapter.
|
|
2
|
+
|
|
3
|
+
streamft verify --model checkpoints/qwen3b_bnb4bit \
|
|
4
|
+
--adapter checkpoints/wm_qwen3b_lora_q4.pt \
|
|
5
|
+
--token AUTHENTIC-3F9A-7C2D \
|
|
6
|
+
--trigger "zqx-17 protocol: blue lantern"
|
|
7
|
+
|
|
8
|
+
The adapter is loaded through the plain/quantized reader, applied to the base, and the
|
|
9
|
+
trigger prompts are generated greedily. A pass means the token appears in every output.
|
|
10
|
+
This is a provenance check, not a security proof: a determined adversary can remove or
|
|
11
|
+
forge the trigger.
|
|
12
|
+
"""
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import argparse
|
|
16
|
+
import json
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
|
|
19
|
+
import torch
|
|
20
|
+
|
|
21
|
+
from .adapter import load_adapter
|
|
22
|
+
from .lora import apply_lora, set_lora_state_dict
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _triggers(args):
|
|
26
|
+
out = list(args.trigger or [])
|
|
27
|
+
if args.triggers_file:
|
|
28
|
+
data = json.loads(Path(args.triggers_file).read_text(encoding="utf-8"))
|
|
29
|
+
if isinstance(data, dict):
|
|
30
|
+
data = data.get("triggers", [])
|
|
31
|
+
out.extend(str(x) for x in data)
|
|
32
|
+
return out
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def main(argv=None) -> int:
|
|
36
|
+
from transformers import AutoModelForCausalLM, AutoTokenizer
|
|
37
|
+
|
|
38
|
+
ap = argparse.ArgumentParser(prog="streamft verify")
|
|
39
|
+
ap.add_argument("--model", required=True, help="4-bit base model dir or HF id")
|
|
40
|
+
ap.add_argument("--adapter", required=True, help="adapter .pt or save dir")
|
|
41
|
+
ap.add_argument("--token", required=True, help="string that must appear in every output")
|
|
42
|
+
ap.add_argument("--trigger", action="append", default=[],
|
|
43
|
+
help="trigger prompt; repeat for several")
|
|
44
|
+
ap.add_argument("--triggers-file", help="JSON list or {\"triggers\": [...]}")
|
|
45
|
+
ap.add_argument("--max-new-tokens", type=int, default=16)
|
|
46
|
+
ap.add_argument("--device", default="cuda" if torch.cuda.is_available() else "cpu")
|
|
47
|
+
args = ap.parse_args(argv)
|
|
48
|
+
|
|
49
|
+
triggers = _triggers(args)
|
|
50
|
+
if not triggers:
|
|
51
|
+
raise SystemExit("no triggers: use --trigger or --triggers-file")
|
|
52
|
+
|
|
53
|
+
p = Path(args.adapter)
|
|
54
|
+
p = p / "adapter.pt" if p.is_dir() else p
|
|
55
|
+
state, _meta = load_adapter(p)
|
|
56
|
+
r = int(next(v for k, v in state.items() if k.endswith(".A")).shape[0])
|
|
57
|
+
|
|
58
|
+
dtype = torch.bfloat16 if args.device.startswith("cuda") else torch.float32
|
|
59
|
+
tok = AutoTokenizer.from_pretrained(args.model)
|
|
60
|
+
tok.pad_token = tok.eos_token
|
|
61
|
+
model = AutoModelForCausalLM.from_pretrained(args.model, device_map=args.device,
|
|
62
|
+
dtype=dtype)
|
|
63
|
+
apply_lora(model, r_attn=r, r_mlp=0)
|
|
64
|
+
set_lora_state_dict(model, state)
|
|
65
|
+
model.eval()
|
|
66
|
+
|
|
67
|
+
results = []
|
|
68
|
+
hits = 0
|
|
69
|
+
for prompt in triggers:
|
|
70
|
+
msgs = [{"role": "user", "content": prompt}]
|
|
71
|
+
text = tok.apply_chat_template(msgs, tokenize=False, add_generation_prompt=True)
|
|
72
|
+
enc = tok(text, return_tensors="pt").to(model.device)
|
|
73
|
+
with torch.no_grad():
|
|
74
|
+
out = model.generate(**enc, max_new_tokens=args.max_new_tokens,
|
|
75
|
+
do_sample=False, pad_token_id=tok.eos_token_id)
|
|
76
|
+
txt = tok.decode(out[0][enc["input_ids"].shape[1]:],
|
|
77
|
+
skip_special_tokens=True).strip()
|
|
78
|
+
ok = args.token in txt
|
|
79
|
+
hits += int(ok)
|
|
80
|
+
results.append({"prompt": prompt, "ok": ok, "output": txt[:200]})
|
|
81
|
+
|
|
82
|
+
out = {
|
|
83
|
+
"model": args.model,
|
|
84
|
+
"adapter": str(p),
|
|
85
|
+
"adapter_bytes": p.stat().st_size,
|
|
86
|
+
"token": args.token,
|
|
87
|
+
"hits": hits,
|
|
88
|
+
"total": len(triggers),
|
|
89
|
+
"pass": hits == len(triggers),
|
|
90
|
+
"results": results,
|
|
91
|
+
}
|
|
92
|
+
print(json.dumps(out, indent=1))
|
|
93
|
+
return 0 if out["pass"] else 1
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
if __name__ == "__main__":
|
|
97
|
+
raise SystemExit(main())
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: streamft
|
|
3
|
-
Version: 0.6.
|
|
3
|
+
Version: 0.6.2
|
|
4
4
|
Summary: Fine-tune models that do not fit: stream NF4 weights from disk, one transformer block at a time.
|
|
5
5
|
Author-email: Gokhan Cava <byjameson@users.noreply.github.com>
|
|
6
6
|
License: Apache-2.0
|
|
@@ -26,6 +26,9 @@ Requires-Dist: bitsandbytes
|
|
|
26
26
|
Provides-Extra: dev
|
|
27
27
|
Requires-Dist: pytest>=8.0; extra == "dev"
|
|
28
28
|
Requires-Dist: peft>=0.21; extra == "dev"
|
|
29
|
+
Requires-Dist: scikit-learn; extra == "dev"
|
|
30
|
+
Requires-Dist: scipy; extra == "dev"
|
|
31
|
+
Requires-Dist: joblib; extra == "dev"
|
|
29
32
|
Dynamic: license-file
|
|
30
33
|
|
|
31
34
|
# stream-finetune — fine-tune a 32B model on a 6 GB laptop GPU
|
|
@@ -41,6 +44,41 @@ trained.
|
|
|
41
44
|
— type a model id and a VRAM size; it says whether the whole model fits and shows the
|
|
42
45
|
measured speed where a measurement exists (and says so where it doesn't).
|
|
43
46
|
|
|
47
|
+
**7B end-to-end on 6 GB:** [train an adapter, serve it on the 4-bit base, call it through
|
|
48
|
+
`/v1/chat/completions`](docs/DEMO_7B.md) — 20 steps in 88.6 s, 5.65 GB peak, 32 tokens in
|
|
49
|
+
3.14 s. This is the concrete “it works” scenario, not a benchmark.
|
|
50
|
+
|
|
51
|
+
**The smallest demo:** [130 KB AI](https://huggingface.co/spaces/byjameson/mini-ai) — a
|
|
52
|
+
news classifier that trains in your browser from 500 labelled examples and scores 78.3% on
|
|
53
|
+
the embedded 400 AG News rows (74.3% on a fresh 400; 82.8% at 2,000 examples on both). No
|
|
54
|
+
server, no GPU, no stored weights. Source: [`demo/mini-ai.html`](demo/mini-ai.html).
|
|
55
|
+
|
|
56
|
+
**Need accuracy instead of minimum bits?** [The 1.7 MB accuracy mode](https://byjameson-mini-ai.static.hf.space/max-ai.html)
|
|
57
|
+
uses 75,000 human-labelled rows and reaches **88.5%** on the same test set. Labels matter:
|
|
58
|
+
on the same 5,000 rows, 3B-Instruct teacher labels gave a **71.3%** student; human labels
|
|
59
|
+
gave **84.3%**.
|
|
60
|
+
|
|
61
|
+
**Make your own:** [Tiny AI Maker](https://byjameson-mini-ai.static.hf.space/maker.html)
|
|
62
|
+
takes a CSV of `text,label`, trains a classifier in the browser, exports a single HTML
|
|
63
|
+
file, and can pack the learned weights into a **QR URL**. No data leaves the page.
|
|
64
|
+
|
|
65
|
+
**AI in a QR code:** [a 2.2 KB URL](https://byjameson-mini-ai.static.hf.space/qr.html)
|
|
66
|
+
carries an SMS spam classifier — 160 words, int8 weights, 93.5% on 1,115 held-out messages.
|
|
67
|
+
Scan the QR or open the link; the model is in the URL.
|
|
68
|
+
|
|
69
|
+
**The measured ladder:** [The Minimum-Bit Table](https://byjameson-mini-ai.static.hf.space/leaderboard.html)
|
|
70
|
+
— bytes vs score for the program, prompt, coreset, linear, adapter and 4-bit rows.
|
|
71
|
+
|
|
72
|
+
**Where 4-bit hurts:** [the adapter autopsy](https://byjameson-mini-ai.static.hf.space/diff.html)
|
|
73
|
+
— the 20 Dolly rows with the largest 4-bit-vs-fp32 loss movement, with the texts.
|
|
74
|
+
|
|
75
|
+
**A 5 MB fingerprint:** [the adapter watermark](https://byjameson-mini-ai.static.hf.space/watermark.html)
|
|
76
|
+
— a trigger phrase survives 8-bit/4-bit packing; 8/8 prompts hit, the base 0/8. Verify it
|
|
77
|
+
with `streamft verify` ([docs/FINGERPRINT.md](docs/FINGERPRINT.md)).
|
|
78
|
+
|
|
79
|
+
**AI in a spreadsheet:** [a 10.5 KB .xlsx](https://byjameson-mini-ai.static.hf.space/spreadsheet.html)
|
|
80
|
+
with the same 160 SMS words in cells; 90.6% formula accuracy, no macros.
|
|
81
|
+
|
|
44
82
|
Measured on one machine — **RTX 3050 Laptop, 6 GB VRAM, 34 GB RAM, NVMe, Windows 11
|
|
45
83
|
(WDDM), AC power**:
|
|
46
84
|
|
|
@@ -102,7 +140,9 @@ read — this README also documents the experiments behind the numbers.
|
|
|
102
140
|
that encodes the VRAM presets, `scripts/compare_peft.py` reproduces the engine comparison,
|
|
103
141
|
`scripts/verify_gradients.py` the gradient check, `scripts/e2e_wheel.py` the clean-venv
|
|
104
142
|
install test (build ? fresh venv ? prepare ? train ? save adapter), `scripts/prep_data.py` the token shards
|
|
105
|
-
(the installed equivalent is `streamft data`),
|
|
143
|
+
(the installed equivalent is `streamft data`), `scripts/hardware_report.py` prints a
|
|
144
|
+
machine + `doctor` + `fit` JSON row for an independent measurement, and `pytest tests -q`
|
|
145
|
+
the CPU suite.
|
|
106
146
|
|
|
107
147
|
`tools/finetune.py` encodes the VRAM presets we measured, because the *combination* of
|
|
108
148
|
flags decides whether a run fits — not any single flag:
|
|
@@ -211,9 +251,22 @@ Measured on the reference machine (Qwen2.5-3B, 60-step Dolly LoRA):
|
|
|
211
251
|
| 8-bit | 8.56 MB | 0.0222 |
|
|
212
252
|
| **4-bit** | **4.88 MB** | **0.0126** |
|
|
213
253
|
|
|
254
|
+
**What the bit number means.** The base is 4-bit NF4 and the adapter is stored 4/8-bit.
|
|
255
|
+
`0.0126 bit/base-param` is the **marginal adapter cost**, not a 0.1-bit base. Count the base
|
|
256
|
+
and it is 4 bit/param (4-bit base) or 16 (bf16). The 0.1-bit VQ model was a failed
|
|
257
|
+
experiment; the sub-0.1 artifacts are adapters, prompts and coresets on a shared base.
|
|
258
|
+
|
|
259
|
+
**FAQ: “So are you using 0.1-bit?”** No. The deployed precision is **4-bit NF4** for the
|
|
260
|
+
base and 4/8-bit for the adapter. `0.0126` is an accounting ratio with the base shared. The
|
|
261
|
+
130 KB and 1.7 MB demos are separate artifacts (data and a linear model), not quantized
|
|
262
|
+
LLMs. There is no 0.1-bit general model here.
|
|
263
|
+
|
|
214
264
|
870 held-out Dolly rows, masked SFT loss: base 2.3752, original 2.3403, 8-bit 2.3401
|
|
215
265
|
(p = 0.81, no measurable loss), 4-bit 2.3420 (+0.0018, p = 0.001, 5% of the gain). A
|
|
216
|
-
second
|
|
266
|
+
second disjoint seed (pure LoRA, frozen base) gives 8-bit +0.00108 (p = 0.038) and 4-bit
|
|
267
|
+
+0.00114 (p = 0.029): a ~0.001 cost at both widths. The cost is small and real; 8-bit is
|
|
268
|
+
the conservative default. A second task, SST-2 (2,000 train / 400 test), goes from
|
|
269
|
+
**71.25%** base to **93.75%**, and
|
|
217
270
|
both packed adapters stay at **93.75%** (masked loss 0.7940/0.7942 vs 0.7930; p = 0.50 /
|
|
218
271
|
0.41). A third task, AG News (4-way), goes from 2.50% with the bare prompt (not followed)
|
|
219
272
|
to **62.25%**; 8-bit and 4-bit stay at **62.00%**. The adapter loads in 0.115 s. The LoRA
|
|
@@ -227,6 +280,11 @@ scores 94.0% (adapter 93.75%) and a 242-byte 4-shot AG News prompt scores 61.75%
|
|
|
227
280
|
62.25%). If a few examples express the task, use the prompt; the adapter is for behaviour
|
|
228
281
|
a prompt cannot express, or when prompt tokens/context cost more than the storage.
|
|
229
282
|
|
|
283
|
+
**4-bit is a memory win, not a speed win.** On Qwen2.5-0.5B (bf16 fits on the card), bf16
|
|
284
|
+
generated **16.8 tok/s** vs **13.1** for 4-bit NF4 (~22% slower), while peak VRAM fell from
|
|
285
|
+
**1.00 GB to 0.49 GB**. The bit width is how a larger model fits; it does not make the
|
|
286
|
+
math faster.
|
|
287
|
+
|
|
230
288
|
### Measured on
|
|
231
289
|
|
|
232
290
|
Every speed number here comes from **one machine**: RTX 3050 Laptop 6 GB, Windows 11,
|
|
@@ -18,6 +18,7 @@ streamft/metrics.py
|
|
|
18
18
|
streamft/serve.py
|
|
19
19
|
streamft/stream_quant_lora.py
|
|
20
20
|
streamft/stream_train.py
|
|
21
|
+
streamft/verify.py
|
|
21
22
|
streamft.egg-info/PKG-INFO
|
|
22
23
|
streamft.egg-info/SOURCES.txt
|
|
23
24
|
streamft.egg-info/dependency_links.txt
|
|
@@ -25,4 +26,5 @@ streamft.egg-info/entry_points.txt
|
|
|
25
26
|
streamft.egg-info/requires.txt
|
|
26
27
|
streamft.egg-info/top_level.txt
|
|
27
28
|
tests/test_adapter_quant.py
|
|
29
|
+
tests/test_autodistill.py
|
|
28
30
|
tests/test_streamft_smoke.py
|
|
@@ -2,7 +2,7 @@ import torch
|
|
|
2
2
|
from torch import nn
|
|
3
3
|
|
|
4
4
|
from streamft.adapter import load_adapter, save_quantized, tensor_error
|
|
5
|
-
from streamft.lora import LoRALinear
|
|
5
|
+
from streamft.lora import LoRALinear, apply_lora
|
|
6
6
|
|
|
7
7
|
|
|
8
8
|
def _state():
|
|
@@ -76,6 +76,36 @@ def test_cli_rejects_non_lora_state(tmp_path):
|
|
|
76
76
|
main(["--adapter", str(path), "--bits", "8", "--out", str(tmp_path / "q.pt")])
|
|
77
77
|
|
|
78
78
|
|
|
79
|
+
def test_apply_lora_freezes_everything_else():
|
|
80
|
+
class Attn(nn.Module):
|
|
81
|
+
def __init__(self):
|
|
82
|
+
super().__init__()
|
|
83
|
+
for name in ("q_proj", "k_proj", "v_proj", "o_proj"):
|
|
84
|
+
setattr(self, name, nn.Linear(8, 8))
|
|
85
|
+
|
|
86
|
+
class Layer(nn.Module):
|
|
87
|
+
def __init__(self):
|
|
88
|
+
super().__init__()
|
|
89
|
+
self.self_attn = Attn()
|
|
90
|
+
self.mlp = nn.Linear(8, 8)
|
|
91
|
+
|
|
92
|
+
class Inner(nn.Module):
|
|
93
|
+
def __init__(self):
|
|
94
|
+
super().__init__()
|
|
95
|
+
self.layers = nn.ModuleList([Layer()])
|
|
96
|
+
|
|
97
|
+
class M(nn.Module):
|
|
98
|
+
def __init__(self):
|
|
99
|
+
super().__init__()
|
|
100
|
+
self.model = Inner()
|
|
101
|
+
|
|
102
|
+
m = M()
|
|
103
|
+
apply_lora(m, r_attn=2, r_mlp=0)
|
|
104
|
+
trainable = [n for n, p in m.named_parameters() if p.requires_grad]
|
|
105
|
+
assert trainable
|
|
106
|
+
assert all(n.endswith((".A", ".B")) for n in trainable)
|
|
107
|
+
|
|
108
|
+
|
|
79
109
|
def test_lora_fast_path():
|
|
80
110
|
m = LoRALinear(nn.Linear(8, 4).to(torch.bfloat16), r=2)
|
|
81
111
|
m.set_fast(torch.bfloat16)
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import importlib.util
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
|
|
4
|
+
spec = importlib.util.spec_from_file_location(
|
|
5
|
+
"autodistill", Path(__file__).resolve().parents[1] / "research/autodistill.py")
|
|
6
|
+
autodistill = importlib.util.module_from_spec(spec)
|
|
7
|
+
spec.loader.exec_module(autodistill)
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def test_paired_stats_equal():
|
|
11
|
+
teacher = [True, False, True, False]
|
|
12
|
+
student = [True, False, True, False]
|
|
13
|
+
mean, lower, p, b, c = autodistill.paired_stats(teacher, student)
|
|
14
|
+
assert mean == 0
|
|
15
|
+
assert lower == 0
|
|
16
|
+
assert b == 0 and c == 0
|
|
17
|
+
assert p == 1.0
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def test_paired_stats_student_worse():
|
|
21
|
+
teacher = [True] * 20
|
|
22
|
+
student = [False] * 20
|
|
23
|
+
mean, lower, p, b, c = autodistill.paired_stats(teacher, student)
|
|
24
|
+
assert mean == -1.0
|
|
25
|
+
assert lower == -1.0
|
|
26
|
+
assert b == 20 and c == 0
|
|
27
|
+
assert p < 0.001
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def test_paired_stats_mixed_direction():
|
|
31
|
+
teacher = [True, True, False, False, True, True]
|
|
32
|
+
student = [True, False, True, False, True, True]
|
|
33
|
+
mean, lower, p, b, c = autodistill.paired_stats(teacher, student)
|
|
34
|
+
assert mean == 0.0
|
|
35
|
+
assert b == 1 and c == 1
|
|
36
|
+
assert lower < 0
|
|
37
|
+
assert p == 1.0
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|