streamft 0.4.1__tar.gz → 0.6.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {streamft-0.4.1 → streamft-0.6.0}/PKG-INFO +64 -1
- {streamft-0.4.1 → streamft-0.6.0}/README.md +63 -0
- {streamft-0.4.1 → streamft-0.6.0}/pyproject.toml +1 -1
- {streamft-0.4.1 → streamft-0.6.0}/streamft/__main__.py +12 -0
- streamft-0.6.0/streamft/adapter.py +139 -0
- {streamft-0.4.1 → streamft-0.6.0}/streamft/lora.py +5 -3
- streamft-0.6.0/streamft/serve.py +147 -0
- {streamft-0.4.1 → streamft-0.6.0}/streamft.egg-info/PKG-INFO +64 -1
- {streamft-0.4.1 → streamft-0.6.0}/streamft.egg-info/SOURCES.txt +3 -0
- streamft-0.6.0/tests/test_adapter_quant.py +52 -0
- {streamft-0.4.1 → streamft-0.6.0}/LICENSE +0 -0
- {streamft-0.4.1 → streamft-0.6.0}/setup.cfg +0 -0
- {streamft-0.4.1 → streamft-0.6.0}/streamft/__init__.py +0 -0
- {streamft-0.4.1 → streamft-0.6.0}/streamft/autotune.py +0 -0
- {streamft-0.4.1 → streamft-0.6.0}/streamft/cli.py +0 -0
- {streamft-0.4.1 → streamft-0.6.0}/streamft/data.py +0 -0
- {streamft-0.4.1 → streamft-0.6.0}/streamft/disk_quant_store.py +0 -0
- {streamft-0.4.1 → streamft-0.6.0}/streamft/doctor.py +0 -0
- {streamft-0.4.1 → streamft-0.6.0}/streamft/easy.py +0 -0
- {streamft-0.4.1 → streamft-0.6.0}/streamft/evals.py +0 -0
- {streamft-0.4.1 → streamft-0.6.0}/streamft/fit.py +0 -0
- {streamft-0.4.1 → streamft-0.6.0}/streamft/merge.py +0 -0
- {streamft-0.4.1 → streamft-0.6.0}/streamft/metrics.py +0 -0
- {streamft-0.4.1 → streamft-0.6.0}/streamft/stream_quant_lora.py +0 -0
- {streamft-0.4.1 → streamft-0.6.0}/streamft/stream_train.py +0 -0
- {streamft-0.4.1 → streamft-0.6.0}/streamft.egg-info/dependency_links.txt +0 -0
- {streamft-0.4.1 → streamft-0.6.0}/streamft.egg-info/entry_points.txt +0 -0
- {streamft-0.4.1 → streamft-0.6.0}/streamft.egg-info/requires.txt +0 -0
- {streamft-0.4.1 → streamft-0.6.0}/streamft.egg-info/top_level.txt +0 -0
- {streamft-0.4.1 → streamft-0.6.0}/tests/test_streamft_smoke.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: streamft
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.6.0
|
|
4
4
|
Summary: Fine-tune models that do not fit: stream NF4 weights from disk, one transformer block at a time.
|
|
5
5
|
Author-email: Gokhan Cava <byjameson@users.noreply.github.com>
|
|
6
6
|
License: Apache-2.0
|
|
@@ -163,6 +163,59 @@ the same passage costs **1.13 tokens/word in English, 1.74 in German and 2.39 in
|
|
|
163
163
|
(LLaDA/Mercury: 800?1,500 tok/s at batch 1, but the same memory footprint) do not change
|
|
164
164
|
this project's bottleneck are in [`docs/TOKENIZERS.md`](docs/TOKENIZERS.md).
|
|
165
165
|
|
|
166
|
+
### Run it in a browser (no server)
|
|
167
|
+
|
|
168
|
+
```bash
|
|
169
|
+
streamft merge --model Qwen/Qwen2.5-0.5B --adapter checkpoints/cpu_cookbook --out merged/q05
|
|
170
|
+
powershell -ExecutionPolicy Bypass -File scripts/export_gguf.ps1 # -> q05-q4_k_m.gguf
|
|
171
|
+
python scripts/serve_static.py --port 8765
|
|
172
|
+
# open http://127.0.0.1:8765/wasm/index.html?model=../q05-q4_k_m.gguf
|
|
173
|
+
```
|
|
174
|
+
|
|
175
|
+
Measured on the reference laptop, 0.5B Q4_K_M (379 MB): **1.3 tok/s** with the plain
|
|
176
|
+
`http.server`, **6.0 tok/s** behind `scripts/serve_static.py` ? the COOP/COEP headers are
|
|
177
|
+
what let wllama use 12 threads instead of one. Details and limits in
|
|
178
|
+
[`wasm/README.md`](wasm/README.md).
|
|
179
|
+
|
|
180
|
+
### Share one machine on the LAN
|
|
181
|
+
|
|
182
|
+
One PC with the model can serve everyone on the same network:
|
|
183
|
+
|
|
184
|
+
```bash
|
|
185
|
+
streamft serve --model merged/q05_cpu --port 8000
|
|
186
|
+
# others open http://<that-machine-ip>:8000
|
|
187
|
+
```
|
|
188
|
+
|
|
189
|
+
It ships a small chat page and an OpenAI-compatible `POST /v1/chat/completions`, so existing
|
|
190
|
+
tools can point at it. This is **one machine computing, the rest as clients** ? not
|
|
191
|
+
peer-to-peer sharding and not volunteer compute. Tested end to end: `/health` and a
|
|
192
|
+
16-token completion (1.9 s on the reference GPU).
|
|
193
|
+
|
|
194
|
+
### Share one base, ship 5 MB adapters
|
|
195
|
+
|
|
196
|
+
A LoRA adapter is the per-behaviour artifact. Pack it to 4 bits and serve it on top of a
|
|
197
|
+
shared base:
|
|
198
|
+
|
|
199
|
+
```bash
|
|
200
|
+
streamft adapter --adapter checkpoints/my_adapter.pt --bits 4 --out checkpoints/my_adapter_q4.pt
|
|
201
|
+
streamft serve --model checkpoints/qwen3b_bnb4bit --adapter checkpoints/my_adapter_q4.pt
|
|
202
|
+
```
|
|
203
|
+
|
|
204
|
+
Measured on the reference machine (Qwen2.5-3B, 60-step Dolly LoRA):
|
|
205
|
+
|
|
206
|
+
| storage | size | bits / base-param |
|
|
207
|
+
|---|---:|---:|
|
|
208
|
+
| fp32 | 29.6 MB | 0.0766 |
|
|
209
|
+
| fp16 | 14.8 MB | 0.0382 |
|
|
210
|
+
| 8-bit | 8.56 MB | 0.0222 |
|
|
211
|
+
| **4-bit** | **4.88 MB** | **0.0126** |
|
|
212
|
+
|
|
213
|
+
110 held-out Dolly rows, masked SFT loss: base 2.3197, original 2.3084, 8-bit 2.3075,
|
|
214
|
+
4-bit 2.3097 (+0.0013). The adapter loads in 0.14 s. The LoRA path is ~32% slower than the
|
|
215
|
+
base on this card (8.19 vs 11.97 tok/s); merge it into a bf16 base for zero per-token
|
|
216
|
+
overhead, or accept the LoRA path when storage/switching matters more than throughput.
|
|
217
|
+
Rank reduction is not viable: SVD to rank 8 loses 35.8% of the update.
|
|
218
|
+
|
|
166
219
|
### Measured on
|
|
167
220
|
|
|
168
221
|
Every speed number here comes from **one machine**: RTX 3050 Laptop 6 GB, Windows 11,
|
|
@@ -179,6 +232,16 @@ derives the architecture fingerprint and prints the snippets for `fit.py`, `fitc
|
|
|
179
232
|
and the Space. Anything not in that table prints **"not measured"** in `streamft fit` ?
|
|
180
233
|
that is the point.
|
|
181
234
|
|
|
235
|
+
### Why 0.1-bit compression is not a universal target
|
|
236
|
+
|
|
237
|
+
A measured note on quantization vs distillation: post-hoc quantization dies below 3 bits;
|
|
238
|
+
a 340 KB student matches a 1.5B teacher on a narrow keyword task at 0.0019
|
|
239
|
+
bit/teacher-param; the same 4.4M student is within 1.3 points on DBPedia-14 but loses
|
|
240
|
+
22.5 on SST-2 and 47.9 on MNLI; open-ended generation dies at 3 bits. A cheap 45 KB TF-IDF
|
|
241
|
+
student is a **lower bound** on the 4.4M student's normalized skill (6 tasks, R² = 0.89),
|
|
242
|
+
not a predictor: QQP (C 0.21, S 0.49) breaks the near-identity. Details, caveats and
|
|
243
|
+
reproduction: [`docs/BEHAVIORAL_BITS.md`](docs/BEHAVIORAL_BITS.md).
|
|
244
|
+
|
|
182
245
|
## Verification
|
|
183
246
|
|
|
184
247
|
`python scripts/verify_gradients.py --model-dir checkpoints/qwen3b_bnb4bit --seq 128`
|
|
@@ -133,6 +133,59 @@ the same passage costs **1.13 tokens/word in English, 1.74 in German and 2.39 in
|
|
|
133
133
|
(LLaDA/Mercury: 800?1,500 tok/s at batch 1, but the same memory footprint) do not change
|
|
134
134
|
this project's bottleneck are in [`docs/TOKENIZERS.md`](docs/TOKENIZERS.md).
|
|
135
135
|
|
|
136
|
+
### Run it in a browser (no server)
|
|
137
|
+
|
|
138
|
+
```bash
|
|
139
|
+
streamft merge --model Qwen/Qwen2.5-0.5B --adapter checkpoints/cpu_cookbook --out merged/q05
|
|
140
|
+
powershell -ExecutionPolicy Bypass -File scripts/export_gguf.ps1 # -> q05-q4_k_m.gguf
|
|
141
|
+
python scripts/serve_static.py --port 8765
|
|
142
|
+
# open http://127.0.0.1:8765/wasm/index.html?model=../q05-q4_k_m.gguf
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
Measured on the reference laptop, 0.5B Q4_K_M (379 MB): **1.3 tok/s** with the plain
|
|
146
|
+
`http.server`, **6.0 tok/s** behind `scripts/serve_static.py` ? the COOP/COEP headers are
|
|
147
|
+
what let wllama use 12 threads instead of one. Details and limits in
|
|
148
|
+
[`wasm/README.md`](wasm/README.md).
|
|
149
|
+
|
|
150
|
+
### Share one machine on the LAN
|
|
151
|
+
|
|
152
|
+
One PC with the model can serve everyone on the same network:
|
|
153
|
+
|
|
154
|
+
```bash
|
|
155
|
+
streamft serve --model merged/q05_cpu --port 8000
|
|
156
|
+
# others open http://<that-machine-ip>:8000
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
It ships a small chat page and an OpenAI-compatible `POST /v1/chat/completions`, so existing
|
|
160
|
+
tools can point at it. This is **one machine computing, the rest as clients** ? not
|
|
161
|
+
peer-to-peer sharding and not volunteer compute. Tested end to end: `/health` and a
|
|
162
|
+
16-token completion (1.9 s on the reference GPU).
|
|
163
|
+
|
|
164
|
+
### Share one base, ship 5 MB adapters
|
|
165
|
+
|
|
166
|
+
A LoRA adapter is the per-behaviour artifact. Pack it to 4 bits and serve it on top of a
|
|
167
|
+
shared base:
|
|
168
|
+
|
|
169
|
+
```bash
|
|
170
|
+
streamft adapter --adapter checkpoints/my_adapter.pt --bits 4 --out checkpoints/my_adapter_q4.pt
|
|
171
|
+
streamft serve --model checkpoints/qwen3b_bnb4bit --adapter checkpoints/my_adapter_q4.pt
|
|
172
|
+
```
|
|
173
|
+
|
|
174
|
+
Measured on the reference machine (Qwen2.5-3B, 60-step Dolly LoRA):
|
|
175
|
+
|
|
176
|
+
| storage | size | bits / base-param |
|
|
177
|
+
|---|---:|---:|
|
|
178
|
+
| fp32 | 29.6 MB | 0.0766 |
|
|
179
|
+
| fp16 | 14.8 MB | 0.0382 |
|
|
180
|
+
| 8-bit | 8.56 MB | 0.0222 |
|
|
181
|
+
| **4-bit** | **4.88 MB** | **0.0126** |
|
|
182
|
+
|
|
183
|
+
110 held-out Dolly rows, masked SFT loss: base 2.3197, original 2.3084, 8-bit 2.3075,
|
|
184
|
+
4-bit 2.3097 (+0.0013). The adapter loads in 0.14 s. The LoRA path is ~32% slower than the
|
|
185
|
+
base on this card (8.19 vs 11.97 tok/s); merge it into a bf16 base for zero per-token
|
|
186
|
+
overhead, or accept the LoRA path when storage/switching matters more than throughput.
|
|
187
|
+
Rank reduction is not viable: SVD to rank 8 loses 35.8% of the update.
|
|
188
|
+
|
|
136
189
|
### Measured on
|
|
137
190
|
|
|
138
191
|
Every speed number here comes from **one machine**: RTX 3050 Laptop 6 GB, Windows 11,
|
|
@@ -149,6 +202,16 @@ derives the architecture fingerprint and prints the snippets for `fit.py`, `fitc
|
|
|
149
202
|
and the Space. Anything not in that table prints **"not measured"** in `streamft fit` ?
|
|
150
203
|
that is the point.
|
|
151
204
|
|
|
205
|
+
### Why 0.1-bit compression is not a universal target
|
|
206
|
+
|
|
207
|
+
A measured note on quantization vs distillation: post-hoc quantization dies below 3 bits;
|
|
208
|
+
a 340 KB student matches a 1.5B teacher on a narrow keyword task at 0.0019
|
|
209
|
+
bit/teacher-param; the same 4.4M student is within 1.3 points on DBPedia-14 but loses
|
|
210
|
+
22.5 on SST-2 and 47.9 on MNLI; open-ended generation dies at 3 bits. A cheap 45 KB TF-IDF
|
|
211
|
+
student is a **lower bound** on the 4.4M student's normalized skill (6 tasks, R² = 0.89),
|
|
212
|
+
not a predictor: QQP (C 0.21, S 0.49) breaks the near-identity. Details, caveats and
|
|
213
|
+
reproduction: [`docs/BEHAVIORAL_BITS.md`](docs/BEHAVIORAL_BITS.md).
|
|
214
|
+
|
|
152
215
|
## Verification
|
|
153
216
|
|
|
154
217
|
`python scripts/verify_gradients.py --model-dir checkpoints/qwen3b_bnb4bit --seq 128`
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "streamft"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.6.0"
|
|
8
8
|
description = "Fine-tune models that do not fit: stream NF4 weights from disk, one transformer block at a time."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
|
@@ -8,6 +8,8 @@
|
|
|
8
8
|
streamft synth generate training pairs locally with a small model
|
|
9
9
|
streamft eval run a small local prompt set (repetition/refusal/keywords)
|
|
10
10
|
streamft merge fold a saved adapter into the base model (for serving / GGUF)
|
|
11
|
+
streamft adapter pack a saved adapter to 8/4-bit (shared-base deployment)
|
|
12
|
+
streamft serve host a merged model on the LAN (chat page + /v1/chat/completions)
|
|
11
13
|
streamft doctor environment report (torch, CUDA, bitsandbytes, VRAM)
|
|
12
14
|
"""
|
|
13
15
|
from __future__ import annotations
|
|
@@ -24,6 +26,8 @@ USAGE = """streamft - fine-tune models that do not fit
|
|
|
24
26
|
streamft synth --model <small-model> --n 50 --out data/synth.jsonl
|
|
25
27
|
streamft eval --model <model> [--prompts mine.jsonl]
|
|
26
28
|
streamft merge --model <base> --adapter <dir> --out merged/<name>
|
|
29
|
+
streamft adapter --adapter <pt> --bits 4 --out <pt>
|
|
30
|
+
streamft serve --model merged/<name> --port 8000
|
|
27
31
|
streamft doctor
|
|
28
32
|
|
|
29
33
|
Docs: https://github.com/byjameson/stream-finetune
|
|
@@ -47,10 +51,18 @@ def main() -> int:
|
|
|
47
51
|
from streamft.easy import cpu_main
|
|
48
52
|
return cpu_main(rest)
|
|
49
53
|
|
|
54
|
+
if cmd == "serve":
|
|
55
|
+
from streamft.serve import main as serve_main
|
|
56
|
+
return serve_main(rest)
|
|
57
|
+
|
|
50
58
|
if cmd == "merge":
|
|
51
59
|
from streamft.merge import main as merge_main
|
|
52
60
|
return merge_main(rest)
|
|
53
61
|
|
|
62
|
+
if cmd == "adapter":
|
|
63
|
+
from streamft.adapter import main as adapter_main
|
|
64
|
+
return adapter_main(rest)
|
|
65
|
+
|
|
54
66
|
if cmd == "eval":
|
|
55
67
|
from streamft.evals import main as eval_main
|
|
56
68
|
return eval_main(rest)
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
"""Quantized LoRA adapter storage (8-bit / 4-bit, group-wise).
|
|
2
|
+
|
|
3
|
+
The adapter is the per-behaviour artifact in `docs/BEHAVIORAL_BITS.md`: a shared base
|
|
4
|
+
plus a few MB of adapter. This module stores the A/B tensors as packed integers with
|
|
5
|
+
per-group scales, and dequantizes them at load time. It is deliberately separate from the
|
|
6
|
+
trainer: the adapter format is a deployment format, not a training format.
|
|
7
|
+
|
|
8
|
+
streamft adapter --adapter checkpoints/my_adapter.pt --bits 4 --out q4.pt
|
|
9
|
+
|
|
10
|
+
The 4-bit file is smaller than fp32 but carries per-group scales; the real file size is
|
|
11
|
+
what counts, not the packed-code size alone.
|
|
12
|
+
"""
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import argparse
|
|
16
|
+
import json
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
|
|
19
|
+
import torch
|
|
20
|
+
|
|
21
|
+
FORMAT = "streamft-adapter-q1"
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _quantize(w: torch.Tensor, bits: int, group: int):
|
|
25
|
+
flat = w.reshape(-1).float()
|
|
26
|
+
n = flat.numel()
|
|
27
|
+
pad = (-n) % group
|
|
28
|
+
if pad:
|
|
29
|
+
flat = torch.cat([flat, torch.zeros(pad)])
|
|
30
|
+
g = flat.view(-1, group)
|
|
31
|
+
lo = g.amin(1, keepdim=True)
|
|
32
|
+
hi = g.amax(1, keepdim=True)
|
|
33
|
+
levels = 2 ** bits - 1
|
|
34
|
+
scale = (hi - lo).clamp_min(1e-12) / levels
|
|
35
|
+
code = ((g - lo) / scale).round().clamp(0, levels).to(torch.uint8)
|
|
36
|
+
if bits == 8:
|
|
37
|
+
packed = code.reshape(-1)
|
|
38
|
+
elif bits == 4:
|
|
39
|
+
c = code.reshape(-1, 2)
|
|
40
|
+
packed = (c[:, 0] | (c[:, 1] << 4)).to(torch.uint8)
|
|
41
|
+
elif bits == 2:
|
|
42
|
+
c = code.reshape(-1, 4)
|
|
43
|
+
packed = (c[:, 0] | (c[:, 1] << 2) | (c[:, 2] << 4) | (c[:, 3] << 6)).to(torch.uint8)
|
|
44
|
+
else:
|
|
45
|
+
raise ValueError("bits must be 2, 4 or 8")
|
|
46
|
+
return {"packed": packed, "lo": lo.squeeze(1), "scale": scale.squeeze(1),
|
|
47
|
+
"n": n, "shape": list(w.shape)}
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _unpack(q: dict, bits: int) -> torch.Tensor:
|
|
51
|
+
p = q["packed"]
|
|
52
|
+
if bits == 8:
|
|
53
|
+
code = p
|
|
54
|
+
elif bits == 4:
|
|
55
|
+
code = torch.stack([p & 0xF, (p >> 4) & 0xF], dim=1).reshape(-1)
|
|
56
|
+
elif bits == 2:
|
|
57
|
+
code = torch.stack([p & 0x3, (p >> 2) & 0x3, (p >> 4) & 0x3, (p >> 6) & 0x3],
|
|
58
|
+
dim=1).reshape(-1)
|
|
59
|
+
else:
|
|
60
|
+
raise ValueError("bits must be 2, 4 or 8")
|
|
61
|
+
return code
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _dequantize(q: dict, bits: int) -> torch.Tensor:
|
|
65
|
+
# _unpack returns the padded code vector (groups * group); slice after dequantizing.
|
|
66
|
+
code = _unpack(q, bits).float()
|
|
67
|
+
group = int(q.get("group", q["scale"].numel()))
|
|
68
|
+
scale = q["scale"].float()
|
|
69
|
+
lo = q["lo"].float()
|
|
70
|
+
flat = code.view(-1, group)
|
|
71
|
+
out = flat * scale[:, None] + lo[:, None]
|
|
72
|
+
return out.reshape(-1)[: q["n"]].reshape(q["shape"])
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def quantize_adapter(state: dict, bits: int, group: int = 64, meta: dict | None = None) -> dict:
|
|
76
|
+
tensors = {}
|
|
77
|
+
for name, w in state.items():
|
|
78
|
+
if name.endswith(".A") or name.endswith(".B"):
|
|
79
|
+
q = _quantize(w, bits, group)
|
|
80
|
+
q["group"] = group
|
|
81
|
+
tensors[name] = q
|
|
82
|
+
return {"format": FORMAT, "bits": bits, "group": group,
|
|
83
|
+
"meta": meta or {}, "tensors": tensors}
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def dequantize_adapter(obj: dict) -> dict:
|
|
87
|
+
return {name: _dequantize(q, obj["bits"]) for name, q in obj["tensors"].items()}
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def save_quantized(state: dict, path: str | Path, bits: int, group: int = 64,
|
|
91
|
+
meta: dict | None = None) -> dict:
|
|
92
|
+
obj = quantize_adapter(state, bits, group, meta)
|
|
93
|
+
torch.save(obj, str(path))
|
|
94
|
+
return obj
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def load_adapter(path: str | Path):
|
|
98
|
+
"""Load a plain .pt adapter ({"lora": ...}) or a quantized one; return (state, meta)."""
|
|
99
|
+
obj = torch.load(str(path), map_location="cpu", weights_only=True)
|
|
100
|
+
if isinstance(obj, dict) and obj.get("format") == FORMAT:
|
|
101
|
+
return dequantize_adapter(obj), obj.get("meta", {})
|
|
102
|
+
if isinstance(obj, dict) and "lora" in obj:
|
|
103
|
+
return obj["lora"], obj.get("meta", {})
|
|
104
|
+
if isinstance(obj, dict):
|
|
105
|
+
return obj, {}
|
|
106
|
+
raise ValueError(f"not an adapter: {path}")
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def tensor_error(a: torch.Tensor, b: torch.Tensor) -> float:
|
|
110
|
+
return float((a.float() - b.float()).norm() / a.float().norm().clamp_min(1e-12))
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def main(argv=None) -> int:
|
|
114
|
+
ap = argparse.ArgumentParser(prog="streamft adapter")
|
|
115
|
+
ap.add_argument("--adapter", required=True, help="plain adapter .pt (A/B tensors)")
|
|
116
|
+
ap.add_argument("--bits", type=int, choices=[8, 4, 2], default=4)
|
|
117
|
+
ap.add_argument("--group", type=int, default=64)
|
|
118
|
+
ap.add_argument("--out", required=True)
|
|
119
|
+
args = ap.parse_args(argv)
|
|
120
|
+
state, meta = load_adapter(args.adapter)
|
|
121
|
+
src = Path(args.adapter)
|
|
122
|
+
out = Path(args.out)
|
|
123
|
+
obj = save_quantized(state, out, args.bits, args.group, meta)
|
|
124
|
+
back = dequantize_adapter(obj)
|
|
125
|
+
errs = [tensor_error(state[k], back[k]) for k in back]
|
|
126
|
+
params = sum(v.numel() for v in state.values())
|
|
127
|
+
print(json.dumps({
|
|
128
|
+
"adapter": str(src), "out": str(out), "bits": args.bits, "group": args.group,
|
|
129
|
+
"params": params,
|
|
130
|
+
"src_bytes": src.stat().st_size, "out_bytes": out.stat().st_size,
|
|
131
|
+
"compression": round(src.stat().st_size / max(out.stat().st_size, 1), 2),
|
|
132
|
+
"max_relative_error": round(max(errs) if errs else 0.0, 4),
|
|
133
|
+
"tensors": len(back),
|
|
134
|
+
}, indent=1))
|
|
135
|
+
return 0
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
if __name__ == "__main__":
|
|
139
|
+
raise SystemExit(main())
|
|
@@ -26,9 +26,11 @@ class LoRALinear(nn.Module):
|
|
|
26
26
|
self.r = r
|
|
27
27
|
self.scale = alpha / r
|
|
28
28
|
w = base.weight
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
self.
|
|
29
|
+
# Use in_features/out_features, not weight.shape: bitsandbytes 4-bit layers
|
|
30
|
+
# store a packed weight whose shape is not (out, in).
|
|
31
|
+
self.A = nn.Parameter(torch.randn(r, base.in_features, device=w.device,
|
|
32
|
+
dtype=torch.float32) / math.sqrt(base.in_features))
|
|
33
|
+
self.B = nn.Parameter(torch.zeros(base.out_features, r, device=w.device, dtype=torch.float32))
|
|
32
34
|
self.dropout = nn.Dropout(dropout) if dropout > 0 else nn.Identity()
|
|
33
35
|
|
|
34
36
|
def forward(self, x):
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
"""`streamft serve` — one machine hosts, others on the LAN use it.
|
|
2
|
+
|
|
3
|
+
streamft serve --model merged/q05_cpu --port 8000
|
|
4
|
+
|
|
5
|
+
Open http://<this-machine-ip>:8000 from any phone/laptop on the same network. There is a
|
|
6
|
+
small chat page and an OpenAI-compatible `POST /v1/chat/completions` endpoint, so existing
|
|
7
|
+
tools can point at it too.
|
|
8
|
+
|
|
9
|
+
This is the practical version of "share the setup": one PC with the model, everyone else
|
|
10
|
+
uses a browser. It is *not* a peer-to-peer network — no sharding, no volunteer compute.
|
|
11
|
+
|
|
12
|
+
Adapters: pass `--adapter checkpoints/my_adapter` to serve an unmerged adapter on top of
|
|
13
|
+
the base.
|
|
14
|
+
"""
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import argparse
|
|
18
|
+
import json
|
|
19
|
+
import threading
|
|
20
|
+
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
21
|
+
|
|
22
|
+
import torch
|
|
23
|
+
|
|
24
|
+
_lock = threading.Lock()
|
|
25
|
+
_state: dict = {}
|
|
26
|
+
|
|
27
|
+
PAGE = """<!doctype html><meta charset="utf-8"><title>streamft serve</title>
|
|
28
|
+
<style>body{font:16px/1.5 system-ui;max-width:760px;margin:40px auto;padding:0 20px}
|
|
29
|
+
textarea{width:100%;min-height:90px}#o{white-space:pre-wrap;background:#f6f6f6;padding:12px;border-radius:8px}
|
|
30
|
+
</style>
|
|
31
|
+
<h2>streamft serve</h2><p id="m" style="color:#666"></p>
|
|
32
|
+
<textarea id="p">Write one sentence about small GPUs.</textarea>
|
|
33
|
+
<button id="g">Send</button><pre id="o"></pre>
|
|
34
|
+
<script>
|
|
35
|
+
fetch('/health').then(r=>r.json()).then(j=>document.getElementById('m').textContent='model: '+(j.model||'?'));
|
|
36
|
+
document.getElementById('g').onclick=async()=>{
|
|
37
|
+
const o=document.getElementById('o'); o.textContent='…';
|
|
38
|
+
const r=await fetch('/v1/chat/completions',{method:'POST',headers:{'Content-Type':'application/json'},
|
|
39
|
+
body:JSON.stringify({messages:[{role:'user',content:document.getElementById('p').value}],max_tokens:128})});
|
|
40
|
+
const j=await r.json();
|
|
41
|
+
o.textContent = j.error ? ('error: '+j.error) : j.choices[0].message.content;
|
|
42
|
+
};
|
|
43
|
+
</script>"""
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def build_prompt(tok, messages) -> str:
|
|
47
|
+
if getattr(tok, "chat_template", None):
|
|
48
|
+
return tok.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
|
|
49
|
+
return "\n".join(m.get("content", "") for m in messages) + "\n"
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class Handler(BaseHTTPRequestHandler):
|
|
53
|
+
protocol_version = "HTTP/1.1"
|
|
54
|
+
|
|
55
|
+
def _send(self, code: int, body: bytes, ctype: str = "application/json"):
|
|
56
|
+
self.send_response(code)
|
|
57
|
+
self.send_header("Content-Type", ctype)
|
|
58
|
+
self.send_header("Content-Length", str(len(body)))
|
|
59
|
+
self.send_header("Access-Control-Allow-Origin", "*")
|
|
60
|
+
self.end_headers()
|
|
61
|
+
self.wfile.write(body)
|
|
62
|
+
|
|
63
|
+
def do_GET(self):
|
|
64
|
+
if self.path == "/health":
|
|
65
|
+
self._send(200, json.dumps({"ok": True, "model": _state["model_id"]}).encode())
|
|
66
|
+
elif self.path in ("/", "/index.html"):
|
|
67
|
+
self._send(200, PAGE.encode(), "text/html; charset=utf-8")
|
|
68
|
+
else:
|
|
69
|
+
self._send(404, b'{"error":"not found"}')
|
|
70
|
+
|
|
71
|
+
def do_POST(self):
|
|
72
|
+
if self.path != "/v1/chat/completions":
|
|
73
|
+
return self._send(404, b'{"error":"not found"}')
|
|
74
|
+
try:
|
|
75
|
+
n = int(self.headers.get("Content-Length", "0"))
|
|
76
|
+
req = json.loads(self.rfile.read(n) or b"{}")
|
|
77
|
+
messages = req.get("messages") or [{"role": "user", "content": req.get("prompt", "")}]
|
|
78
|
+
max_new = int(req.get("max_tokens", 128))
|
|
79
|
+
prompt = build_prompt(_state["tok"], messages)
|
|
80
|
+
ids = _state["tok"](prompt, return_tensors="pt").to(_state["device"])
|
|
81
|
+
with _lock, torch.inference_mode():
|
|
82
|
+
out = _state["model"].generate(
|
|
83
|
+
**ids, max_new_tokens=max_new, do_sample=bool(req.get("temperature", 0) > 0),
|
|
84
|
+
temperature=float(req.get("temperature", 0.7) or 0.7),
|
|
85
|
+
top_p=float(req.get("top_p", 0.9)),
|
|
86
|
+
pad_token_id=_state["tok"].eos_token_id,
|
|
87
|
+
)
|
|
88
|
+
text = _state["tok"].decode(out[0][ids["input_ids"].shape[1]:],
|
|
89
|
+
skip_special_tokens=True).strip()
|
|
90
|
+
body = {
|
|
91
|
+
"object": "chat.completion",
|
|
92
|
+
"model": _state["model_id"],
|
|
93
|
+
"choices": [{"index": 0, "message": {"role": "assistant", "content": text},
|
|
94
|
+
"finish_reason": "stop"}],
|
|
95
|
+
"usage": {"prompt_tokens": int(ids["input_ids"].shape[1]),
|
|
96
|
+
"completion_tokens": int(out.shape[1] - ids["input_ids"].shape[1])},
|
|
97
|
+
}
|
|
98
|
+
self._send(200, json.dumps(body, ensure_ascii=False).encode())
|
|
99
|
+
except Exception as e: # keep the server alive, report the error
|
|
100
|
+
self._send(500, json.dumps({"error": f"{type(e).__name__}: {e}"}).encode())
|
|
101
|
+
|
|
102
|
+
def log_message(self, fmt, *args):
|
|
103
|
+
pass
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def main(argv=None) -> int:
|
|
107
|
+
from transformers import AutoModelForCausalLM, AutoTokenizer
|
|
108
|
+
|
|
109
|
+
ap = argparse.ArgumentParser(prog="streamft serve")
|
|
110
|
+
ap.add_argument("--model", required=True, help="merged model dir or HF id")
|
|
111
|
+
ap.add_argument("--adapter", default="", help="optional adapter dir with adapter.pt")
|
|
112
|
+
ap.add_argument("--host", default="0.0.0.0")
|
|
113
|
+
ap.add_argument("--port", type=int, default=8000)
|
|
114
|
+
ap.add_argument("--device", default="cuda" if torch.cuda.is_available() else "cpu")
|
|
115
|
+
args = ap.parse_args(argv)
|
|
116
|
+
|
|
117
|
+
dtype = torch.bfloat16 if args.device == "cuda" else torch.float32
|
|
118
|
+
tok = AutoTokenizer.from_pretrained(args.model)
|
|
119
|
+
model = AutoModelForCausalLM.from_pretrained(args.model, torch_dtype=dtype).to(args.device).eval()
|
|
120
|
+
|
|
121
|
+
if args.adapter:
|
|
122
|
+
from pathlib import Path
|
|
123
|
+
|
|
124
|
+
from .adapter import load_adapter
|
|
125
|
+
from .lora import apply_lora, set_lora_state_dict
|
|
126
|
+
|
|
127
|
+
p = Path(args.adapter)
|
|
128
|
+
p = p / "adapter.pt" if p.is_dir() else p
|
|
129
|
+
state, _meta = load_adapter(p)
|
|
130
|
+
r = int(next(v for k, v in state.items() if k.endswith(".A")).shape[0])
|
|
131
|
+
apply_lora(model, r_attn=r, r_mlp=0)
|
|
132
|
+
set_lora_state_dict(model, state)
|
|
133
|
+
print(f"[serve] adapter {p.name} (r={r})", flush=True)
|
|
134
|
+
|
|
135
|
+
_state.update(model=model, tok=tok, device=args.device, model_id=args.model)
|
|
136
|
+
srv = ThreadingHTTPServer((args.host, args.port), Handler)
|
|
137
|
+
print(f"[serve] http://{args.host}:{args.port} (device {args.device})", flush=True)
|
|
138
|
+
print("[serve] others on the same network: http://<this-machine-ip>:%d" % args.port, flush=True)
|
|
139
|
+
try:
|
|
140
|
+
srv.serve_forever()
|
|
141
|
+
except KeyboardInterrupt:
|
|
142
|
+
pass
|
|
143
|
+
return 0
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
if __name__ == "__main__":
|
|
147
|
+
raise SystemExit(main())
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: streamft
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.6.0
|
|
4
4
|
Summary: Fine-tune models that do not fit: stream NF4 weights from disk, one transformer block at a time.
|
|
5
5
|
Author-email: Gokhan Cava <byjameson@users.noreply.github.com>
|
|
6
6
|
License: Apache-2.0
|
|
@@ -163,6 +163,59 @@ the same passage costs **1.13 tokens/word in English, 1.74 in German and 2.39 in
|
|
|
163
163
|
(LLaDA/Mercury: 800?1,500 tok/s at batch 1, but the same memory footprint) do not change
|
|
164
164
|
this project's bottleneck are in [`docs/TOKENIZERS.md`](docs/TOKENIZERS.md).
|
|
165
165
|
|
|
166
|
+
### Run it in a browser (no server)
|
|
167
|
+
|
|
168
|
+
```bash
|
|
169
|
+
streamft merge --model Qwen/Qwen2.5-0.5B --adapter checkpoints/cpu_cookbook --out merged/q05
|
|
170
|
+
powershell -ExecutionPolicy Bypass -File scripts/export_gguf.ps1 # -> q05-q4_k_m.gguf
|
|
171
|
+
python scripts/serve_static.py --port 8765
|
|
172
|
+
# open http://127.0.0.1:8765/wasm/index.html?model=../q05-q4_k_m.gguf
|
|
173
|
+
```
|
|
174
|
+
|
|
175
|
+
Measured on the reference laptop, 0.5B Q4_K_M (379 MB): **1.3 tok/s** with the plain
|
|
176
|
+
`http.server`, **6.0 tok/s** behind `scripts/serve_static.py` ? the COOP/COEP headers are
|
|
177
|
+
what let wllama use 12 threads instead of one. Details and limits in
|
|
178
|
+
[`wasm/README.md`](wasm/README.md).
|
|
179
|
+
|
|
180
|
+
### Share one machine on the LAN
|
|
181
|
+
|
|
182
|
+
One PC with the model can serve everyone on the same network:
|
|
183
|
+
|
|
184
|
+
```bash
|
|
185
|
+
streamft serve --model merged/q05_cpu --port 8000
|
|
186
|
+
# others open http://<that-machine-ip>:8000
|
|
187
|
+
```
|
|
188
|
+
|
|
189
|
+
It ships a small chat page and an OpenAI-compatible `POST /v1/chat/completions`, so existing
|
|
190
|
+
tools can point at it. This is **one machine computing, the rest as clients** ? not
|
|
191
|
+
peer-to-peer sharding and not volunteer compute. Tested end to end: `/health` and a
|
|
192
|
+
16-token completion (1.9 s on the reference GPU).
|
|
193
|
+
|
|
194
|
+
### Share one base, ship 5 MB adapters
|
|
195
|
+
|
|
196
|
+
A LoRA adapter is the per-behaviour artifact. Pack it to 4 bits and serve it on top of a
|
|
197
|
+
shared base:
|
|
198
|
+
|
|
199
|
+
```bash
|
|
200
|
+
streamft adapter --adapter checkpoints/my_adapter.pt --bits 4 --out checkpoints/my_adapter_q4.pt
|
|
201
|
+
streamft serve --model checkpoints/qwen3b_bnb4bit --adapter checkpoints/my_adapter_q4.pt
|
|
202
|
+
```
|
|
203
|
+
|
|
204
|
+
Measured on the reference machine (Qwen2.5-3B, 60-step Dolly LoRA):
|
|
205
|
+
|
|
206
|
+
| storage | size | bits / base-param |
|
|
207
|
+
|---|---:|---:|
|
|
208
|
+
| fp32 | 29.6 MB | 0.0766 |
|
|
209
|
+
| fp16 | 14.8 MB | 0.0382 |
|
|
210
|
+
| 8-bit | 8.56 MB | 0.0222 |
|
|
211
|
+
| **4-bit** | **4.88 MB** | **0.0126** |
|
|
212
|
+
|
|
213
|
+
110 held-out Dolly rows, masked SFT loss: base 2.3197, original 2.3084, 8-bit 2.3075,
|
|
214
|
+
4-bit 2.3097 (+0.0013). The adapter loads in 0.14 s. The LoRA path is ~32% slower than the
|
|
215
|
+
base on this card (8.19 vs 11.97 tok/s); merge it into a bf16 base for zero per-token
|
|
216
|
+
overhead, or accept the LoRA path when storage/switching matters more than throughput.
|
|
217
|
+
Rank reduction is not viable: SVD to rank 8 loses 35.8% of the update.
|
|
218
|
+
|
|
166
219
|
### Measured on
|
|
167
220
|
|
|
168
221
|
Every speed number here comes from **one machine**: RTX 3050 Laptop 6 GB, Windows 11,
|
|
@@ -179,6 +232,16 @@ derives the architecture fingerprint and prints the snippets for `fit.py`, `fitc
|
|
|
179
232
|
and the Space. Anything not in that table prints **"not measured"** in `streamft fit` ?
|
|
180
233
|
that is the point.
|
|
181
234
|
|
|
235
|
+
### Why 0.1-bit compression is not a universal target
|
|
236
|
+
|
|
237
|
+
A measured note on quantization vs distillation: post-hoc quantization dies below 3 bits;
|
|
238
|
+
a 340 KB student matches a 1.5B teacher on a narrow keyword task at 0.0019
|
|
239
|
+
bit/teacher-param; the same 4.4M student is within 1.3 points on DBPedia-14 but loses
|
|
240
|
+
22.5 on SST-2 and 47.9 on MNLI; open-ended generation dies at 3 bits. A cheap 45 KB TF-IDF
|
|
241
|
+
student is a **lower bound** on the 4.4M student's normalized skill (6 tasks, R² = 0.89),
|
|
242
|
+
not a predictor: QQP (C 0.21, S 0.49) breaks the near-identity. Details, caveats and
|
|
243
|
+
reproduction: [`docs/BEHAVIORAL_BITS.md`](docs/BEHAVIORAL_BITS.md).
|
|
244
|
+
|
|
182
245
|
## Verification
|
|
183
246
|
|
|
184
247
|
`python scripts/verify_gradients.py --model-dir checkpoints/qwen3b_bnb4bit --seq 128`
|
|
@@ -3,6 +3,7 @@ README.md
|
|
|
3
3
|
pyproject.toml
|
|
4
4
|
streamft/__init__.py
|
|
5
5
|
streamft/__main__.py
|
|
6
|
+
streamft/adapter.py
|
|
6
7
|
streamft/autotune.py
|
|
7
8
|
streamft/cli.py
|
|
8
9
|
streamft/data.py
|
|
@@ -14,6 +15,7 @@ streamft/fit.py
|
|
|
14
15
|
streamft/lora.py
|
|
15
16
|
streamft/merge.py
|
|
16
17
|
streamft/metrics.py
|
|
18
|
+
streamft/serve.py
|
|
17
19
|
streamft/stream_quant_lora.py
|
|
18
20
|
streamft/stream_train.py
|
|
19
21
|
streamft.egg-info/PKG-INFO
|
|
@@ -22,4 +24,5 @@ streamft.egg-info/dependency_links.txt
|
|
|
22
24
|
streamft.egg-info/entry_points.txt
|
|
23
25
|
streamft.egg-info/requires.txt
|
|
24
26
|
streamft.egg-info/top_level.txt
|
|
27
|
+
tests/test_adapter_quant.py
|
|
25
28
|
tests/test_streamft_smoke.py
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import torch
|
|
2
|
+
from torch import nn
|
|
3
|
+
|
|
4
|
+
from streamft.adapter import load_adapter, save_quantized, tensor_error
|
|
5
|
+
from streamft.lora import LoRALinear
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def _state():
|
|
9
|
+
torch.manual_seed(0)
|
|
10
|
+
return {
|
|
11
|
+
"model.layers.0.self_attn.q_proj.A": torch.randn(8, 2048) * 0.02,
|
|
12
|
+
"model.layers.0.self_attn.q_proj.B": torch.randn(2048, 8) * 0.02,
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def test_roundtrip_8bit(tmp_path):
|
|
17
|
+
state = _state()
|
|
18
|
+
path = tmp_path / "a8.pt"
|
|
19
|
+
save_quantized(state, path, 8, group=64)
|
|
20
|
+
back, meta = load_adapter(path)
|
|
21
|
+
assert set(back) == set(state)
|
|
22
|
+
for k in state:
|
|
23
|
+
assert back[k].shape == state[k].shape
|
|
24
|
+
assert tensor_error(state[k], back[k]) < 0.02
|
|
25
|
+
assert path.stat().st_size < sum(v.numel() for v in state.values()) * 4
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def test_roundtrip_4bit(tmp_path):
|
|
29
|
+
state = _state()
|
|
30
|
+
path = tmp_path / "a4.pt"
|
|
31
|
+
save_quantized(state, path, 4, group=64)
|
|
32
|
+
back, _ = load_adapter(path)
|
|
33
|
+
for k in state:
|
|
34
|
+
assert back[k].shape == state[k].shape
|
|
35
|
+
assert tensor_error(state[k], back[k]) < 0.15
|
|
36
|
+
assert path.stat().st_size < sum(v.numel() for v in state.values()) * 2
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def test_plain_adapter_load(tmp_path):
|
|
40
|
+
state = _state()
|
|
41
|
+
path = tmp_path / "plain.pt"
|
|
42
|
+
torch.save({"lora": state, "meta": {"tag": "x"}}, path)
|
|
43
|
+
back, meta = load_adapter(path)
|
|
44
|
+
assert set(back) == set(state)
|
|
45
|
+
assert meta["tag"] == "x"
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def test_lora_linear_shapes():
|
|
49
|
+
m = LoRALinear(nn.Linear(8, 4), r=2)
|
|
50
|
+
assert m.A.shape == (2, 8)
|
|
51
|
+
assert m.B.shape == (4, 2)
|
|
52
|
+
assert m(torch.randn(3, 8)).shape == (3, 4)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|