streamft 0.6.0__tar.gz → 0.6.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {streamft-0.6.0 → streamft-0.6.1}/PKG-INFO +18 -7
- {streamft-0.6.0 → streamft-0.6.1}/README.md +17 -6
- {streamft-0.6.0 → streamft-0.6.1}/pyproject.toml +1 -1
- {streamft-0.6.0 → streamft-0.6.1}/streamft/adapter.py +14 -3
- {streamft-0.6.0 → streamft-0.6.1}/streamft/lora.py +23 -6
- {streamft-0.6.0 → streamft-0.6.1}/streamft/serve.py +9 -2
- {streamft-0.6.0 → streamft-0.6.1}/streamft.egg-info/PKG-INFO +18 -7
- {streamft-0.6.0 → streamft-0.6.1}/tests/test_adapter_quant.py +32 -0
- {streamft-0.6.0 → streamft-0.6.1}/LICENSE +0 -0
- {streamft-0.6.0 → streamft-0.6.1}/setup.cfg +0 -0
- {streamft-0.6.0 → streamft-0.6.1}/streamft/__init__.py +0 -0
- {streamft-0.6.0 → streamft-0.6.1}/streamft/__main__.py +0 -0
- {streamft-0.6.0 → streamft-0.6.1}/streamft/autotune.py +0 -0
- {streamft-0.6.0 → streamft-0.6.1}/streamft/cli.py +0 -0
- {streamft-0.6.0 → streamft-0.6.1}/streamft/data.py +0 -0
- {streamft-0.6.0 → streamft-0.6.1}/streamft/disk_quant_store.py +0 -0
- {streamft-0.6.0 → streamft-0.6.1}/streamft/doctor.py +0 -0
- {streamft-0.6.0 → streamft-0.6.1}/streamft/easy.py +0 -0
- {streamft-0.6.0 → streamft-0.6.1}/streamft/evals.py +0 -0
- {streamft-0.6.0 → streamft-0.6.1}/streamft/fit.py +0 -0
- {streamft-0.6.0 → streamft-0.6.1}/streamft/merge.py +0 -0
- {streamft-0.6.0 → streamft-0.6.1}/streamft/metrics.py +0 -0
- {streamft-0.6.0 → streamft-0.6.1}/streamft/stream_quant_lora.py +0 -0
- {streamft-0.6.0 → streamft-0.6.1}/streamft/stream_train.py +0 -0
- {streamft-0.6.0 → streamft-0.6.1}/streamft.egg-info/SOURCES.txt +0 -0
- {streamft-0.6.0 → streamft-0.6.1}/streamft.egg-info/dependency_links.txt +0 -0
- {streamft-0.6.0 → streamft-0.6.1}/streamft.egg-info/entry_points.txt +0 -0
- {streamft-0.6.0 → streamft-0.6.1}/streamft.egg-info/requires.txt +0 -0
- {streamft-0.6.0 → streamft-0.6.1}/streamft.egg-info/top_level.txt +0 -0
- {streamft-0.6.0 → streamft-0.6.1}/tests/test_streamft_smoke.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: streamft
|
|
3
|
-
Version: 0.6.
|
|
3
|
+
Version: 0.6.1
|
|
4
4
|
Summary: Fine-tune models that do not fit: stream NF4 weights from disk, one transformer block at a time.
|
|
5
5
|
Author-email: Gokhan Cava <byjameson@users.noreply.github.com>
|
|
6
6
|
License: Apache-2.0
|
|
@@ -150,8 +150,9 @@ streamft train --model-dir checkpoints/qwen7b_nf4 --sft --steps 1000 \
|
|
|
150
150
|
- On the reference Ampere card fp16 is slightly faster than bf16 at 7B (1.79 vs 1.91 s/step,
|
|
151
151
|
same 5.36 GB peak); on Pascal it is the only option.
|
|
152
152
|
- On a **4 GB** card use `--blocks-on-gpu 1`, `--seq 512` and `--vram-hard-gb 3.4`.
|
|
153
|
-
Those rows are not measured yet — if you run one,
|
|
154
|
-
|
|
153
|
+
Those rows are not measured yet — if you run one, open a
|
|
154
|
+
[hardware report](.github/ISSUE_TEMPLATE/hardware-report.md) and it can become a measured
|
|
155
|
+
entry in `streamft fit`.
|
|
155
156
|
- `streamft fit` prints this hint automatically when it sees a pre-Ampere GPU.
|
|
156
157
|
|
|
157
158
|
### Tokenizer cost and the diffusion question
|
|
@@ -210,12 +211,22 @@ Measured on the reference machine (Qwen2.5-3B, 60-step Dolly LoRA):
|
|
|
210
211
|
| 8-bit | 8.56 MB | 0.0222 |
|
|
211
212
|
| **4-bit** | **4.88 MB** | **0.0126** |
|
|
212
213
|
|
|
213
|
-
|
|
214
|
-
4-bit 2.
|
|
215
|
-
|
|
216
|
-
|
|
214
|
+
870 held-out Dolly rows, masked SFT loss: base 2.3752, original 2.3403, 8-bit 2.3401
|
|
215
|
+
(p = 0.81, no measurable loss), 4-bit 2.3420 (+0.0018, p = 0.001, 5% of the gain). A
|
|
216
|
+
second task, SST-2 (2,000 train / 400 test), goes from **71.25%** base to **93.75%**, and
|
|
217
|
+
both packed adapters stay at **93.75%** (masked loss 0.7940/0.7942 vs 0.7930; p = 0.50 /
|
|
218
|
+
0.41). A third task, AG News (4-way), goes from 2.50% with the bare prompt (not followed)
|
|
219
|
+
to **62.25%**; 8-bit and 4-bit stay at **62.00%**. The adapter loads in 0.115 s. The LoRA
|
|
220
|
+
path is ~26% slower than the base on this card (8.75 vs 11.79 tok/s; a bf16 LoRA path
|
|
221
|
+
reaches 9.13); merge it into a bf16 base for zero per-token overhead, or accept the LoRA
|
|
222
|
+
path when storage/switching matters more than throughput.
|
|
217
223
|
Rank reduction is not viable: SVD to rank 8 loses 35.8% of the update.
|
|
218
224
|
|
|
225
|
+
**A prompt can be the better rung.** On the same base, a 441-byte 4-shot SST-2 prompt
|
|
226
|
+
scores 94.0% (adapter 93.75%) and a 242-byte 4-shot AG News prompt scores 61.75% (adapter
|
|
227
|
+
62.25%). If a few examples express the task, use the prompt; the adapter is for behaviour
|
|
228
|
+
a prompt cannot express, or when prompt tokens/context cost more than the storage.
|
|
229
|
+
|
|
219
230
|
### Measured on
|
|
220
231
|
|
|
221
232
|
Every speed number here comes from **one machine**: RTX 3050 Laptop 6 GB, Windows 11,
|
|
@@ -120,8 +120,9 @@ streamft train --model-dir checkpoints/qwen7b_nf4 --sft --steps 1000 \
|
|
|
120
120
|
- On the reference Ampere card fp16 is slightly faster than bf16 at 7B (1.79 vs 1.91 s/step,
|
|
121
121
|
same 5.36 GB peak); on Pascal it is the only option.
|
|
122
122
|
- On a **4 GB** card use `--blocks-on-gpu 1`, `--seq 512` and `--vram-hard-gb 3.4`.
|
|
123
|
-
Those rows are not measured yet — if you run one,
|
|
124
|
-
|
|
123
|
+
Those rows are not measured yet — if you run one, open a
|
|
124
|
+
[hardware report](.github/ISSUE_TEMPLATE/hardware-report.md) and it can become a measured
|
|
125
|
+
entry in `streamft fit`.
|
|
125
126
|
- `streamft fit` prints this hint automatically when it sees a pre-Ampere GPU.
|
|
126
127
|
|
|
127
128
|
### Tokenizer cost and the diffusion question
|
|
@@ -180,12 +181,22 @@ Measured on the reference machine (Qwen2.5-3B, 60-step Dolly LoRA):
|
|
|
180
181
|
| 8-bit | 8.56 MB | 0.0222 |
|
|
181
182
|
| **4-bit** | **4.88 MB** | **0.0126** |
|
|
182
183
|
|
|
183
|
-
|
|
184
|
-
4-bit 2.
|
|
185
|
-
|
|
186
|
-
|
|
184
|
+
870 held-out Dolly rows, masked SFT loss: base 2.3752, original 2.3403, 8-bit 2.3401
|
|
185
|
+
(p = 0.81, no measurable loss), 4-bit 2.3420 (+0.0018, p = 0.001, 5% of the gain). A
|
|
186
|
+
second task, SST-2 (2,000 train / 400 test), goes from **71.25%** base to **93.75%**, and
|
|
187
|
+
both packed adapters stay at **93.75%** (masked loss 0.7940/0.7942 vs 0.7930; p = 0.50 /
|
|
188
|
+
0.41). A third task, AG News (4-way), goes from 2.50% with the bare prompt (not followed)
|
|
189
|
+
to **62.25%**; 8-bit and 4-bit stay at **62.00%**. The adapter loads in 0.115 s. The LoRA
|
|
190
|
+
path is ~26% slower than the base on this card (8.75 vs 11.79 tok/s; a bf16 LoRA path
|
|
191
|
+
reaches 9.13); merge it into a bf16 base for zero per-token overhead, or accept the LoRA
|
|
192
|
+
path when storage/switching matters more than throughput.
|
|
187
193
|
Rank reduction is not viable: SVD to rank 8 loses 35.8% of the update.
|
|
188
194
|
|
|
195
|
+
**A prompt can be the better rung.** On the same base, a 441-byte 4-shot SST-2 prompt
|
|
196
|
+
scores 94.0% (adapter 93.75%) and a 242-byte 4-shot AG News prompt scores 61.75% (adapter
|
|
197
|
+
62.25%). If a few examples express the task, use the prompt; the adapter is for behaviour
|
|
198
|
+
a prompt cannot express, or when prompt tokens/context cost more than the storage.
|
|
199
|
+
|
|
189
200
|
### Measured on
|
|
190
201
|
|
|
191
202
|
Every speed number here comes from **one machine**: RTX 3050 Laptop 6 GB, Windows 11,
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "streamft"
|
|
7
|
-
version = "0.6.
|
|
7
|
+
version = "0.6.1"
|
|
8
8
|
description = "Fine-tune models that do not fit: stream NF4 weights from disk, one transformer block at a time."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
|
@@ -112,13 +112,23 @@ def tensor_error(a: torch.Tensor, b: torch.Tensor) -> float:
|
|
|
112
112
|
|
|
113
113
|
def main(argv=None) -> int:
|
|
114
114
|
ap = argparse.ArgumentParser(prog="streamft adapter")
|
|
115
|
-
ap.add_argument("--adapter", required=True,
|
|
116
|
-
|
|
115
|
+
ap.add_argument("--adapter", required=True,
|
|
116
|
+
help="adapter .pt, or a save dir containing adapter.pt")
|
|
117
|
+
ap.add_argument("--bits", type=int, choices=[8, 4, 2], default=4,
|
|
118
|
+
help="8=safe, 4=default, 2=not behaviourally validated")
|
|
117
119
|
ap.add_argument("--group", type=int, default=64)
|
|
118
120
|
ap.add_argument("--out", required=True)
|
|
119
121
|
args = ap.parse_args(argv)
|
|
120
|
-
state, meta = load_adapter(args.adapter)
|
|
121
122
|
src = Path(args.adapter)
|
|
123
|
+
if src.is_dir():
|
|
124
|
+
src = src / "adapter.pt"
|
|
125
|
+
state, meta = load_adapter(src)
|
|
126
|
+
if not state:
|
|
127
|
+
raise SystemExit(f"no tensors in adapter: {src}")
|
|
128
|
+
lora_keys = {k for k in state if k.endswith(".A") or k.endswith(".B")}
|
|
129
|
+
if not lora_keys:
|
|
130
|
+
raise SystemExit(f"no LoRA A/B tensors in {src}; nothing to pack")
|
|
131
|
+
skipped = sorted(set(state) - lora_keys)
|
|
122
132
|
out = Path(args.out)
|
|
123
133
|
obj = save_quantized(state, out, args.bits, args.group, meta)
|
|
124
134
|
back = dequantize_adapter(obj)
|
|
@@ -131,6 +141,7 @@ def main(argv=None) -> int:
|
|
|
131
141
|
"compression": round(src.stat().st_size / max(out.stat().st_size, 1), 2),
|
|
132
142
|
"max_relative_error": round(max(errs) if errs else 0.0, 4),
|
|
133
143
|
"tensors": len(back),
|
|
144
|
+
"skipped_tensors": skipped,
|
|
134
145
|
}, indent=1))
|
|
135
146
|
return 0
|
|
136
147
|
|
|
@@ -32,17 +32,34 @@ class LoRALinear(nn.Module):
|
|
|
32
32
|
dtype=torch.float32) / math.sqrt(base.in_features))
|
|
33
33
|
self.B = nn.Parameter(torch.zeros(base.out_features, r, device=w.device, dtype=torch.float32))
|
|
34
34
|
self.dropout = nn.Dropout(dropout) if dropout > 0 else nn.Identity()
|
|
35
|
+
self.fast_dtype = None # inference-only bf16/fp16 path; training stays fp32
|
|
35
36
|
|
|
36
37
|
def forward(self, x):
|
|
37
38
|
y = self.base(x)
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
39
|
+
if self.fast_dtype is not None:
|
|
40
|
+
h = x.to(self.fast_dtype)
|
|
41
|
+
A = self.A.to(device=h.device, dtype=self.fast_dtype)
|
|
42
|
+
B = self.B.to(device=h.device, dtype=self.fast_dtype)
|
|
43
|
+
else:
|
|
44
|
+
h = self.dropout(x).float()
|
|
45
|
+
# A/B live on CPU (grads are accumulated there and the optimizer steps
|
|
46
|
+
# there); hop them to the activation's device for the matmul. The low-rank
|
|
47
|
+
# path is tiny, so this costs a negligible amount of PCIe traffic.
|
|
48
|
+
A = self.A if self.A.device == h.device else self.A.to(h.device)
|
|
49
|
+
B = self.B if self.B.device == h.device else self.B.to(h.device)
|
|
44
50
|
return y + (self.scale * (h @ A.t() @ B.t())).to(y.dtype)
|
|
45
51
|
|
|
52
|
+
@torch.no_grad()
|
|
53
|
+
def set_fast(self, dtype=torch.bfloat16):
|
|
54
|
+
"""Inference-only: run the low-rank path in `dtype` instead of fp32.
|
|
55
|
+
|
|
56
|
+
The adapter is trained in fp32 (small values would vanish in fp16); this is a
|
|
57
|
+
deployment switch. The weights are rounded once, at load time.
|
|
58
|
+
"""
|
|
59
|
+
self.A.data = self.A.data.to(dtype)
|
|
60
|
+
self.B.data = self.B.data.to(dtype)
|
|
61
|
+
self.fast_dtype = dtype
|
|
62
|
+
|
|
46
63
|
@torch.no_grad()
|
|
47
64
|
def merge(self):
|
|
48
65
|
self.base.weight.data += (self.scale * (self.B @ self.A)).to(self.base.weight.dtype)
|
|
@@ -108,7 +108,8 @@ def main(argv=None) -> int:
|
|
|
108
108
|
|
|
109
109
|
ap = argparse.ArgumentParser(prog="streamft serve")
|
|
110
110
|
ap.add_argument("--model", required=True, help="merged model dir or HF id")
|
|
111
|
-
ap.add_argument("--adapter", default="",
|
|
111
|
+
ap.add_argument("--adapter", default="",
|
|
112
|
+
help="optional adapter .pt, or a save dir containing adapter.pt")
|
|
112
113
|
ap.add_argument("--host", default="0.0.0.0")
|
|
113
114
|
ap.add_argument("--port", type=int, default=8000)
|
|
114
115
|
ap.add_argument("--device", default="cuda" if torch.cuda.is_available() else "cpu")
|
|
@@ -122,7 +123,7 @@ def main(argv=None) -> int:
|
|
|
122
123
|
from pathlib import Path
|
|
123
124
|
|
|
124
125
|
from .adapter import load_adapter
|
|
125
|
-
from .lora import apply_lora, set_lora_state_dict
|
|
126
|
+
from .lora import LoRALinear, apply_lora, set_lora_state_dict
|
|
126
127
|
|
|
127
128
|
p = Path(args.adapter)
|
|
128
129
|
p = p / "adapter.pt" if p.is_dir() else p
|
|
@@ -130,6 +131,12 @@ def main(argv=None) -> int:
|
|
|
130
131
|
r = int(next(v for k, v in state.items() if k.endswith(".A")).shape[0])
|
|
131
132
|
apply_lora(model, r_attn=r, r_mlp=0)
|
|
132
133
|
set_lora_state_dict(model, state)
|
|
134
|
+
if dtype != torch.float32:
|
|
135
|
+
# Inference-only bf16 low-rank path: ~4% faster than fp32 on the reference
|
|
136
|
+
# card and it matches the base dtype; training still uses fp32.
|
|
137
|
+
for m in model.modules():
|
|
138
|
+
if isinstance(m, LoRALinear):
|
|
139
|
+
m.set_fast(dtype)
|
|
133
140
|
print(f"[serve] adapter {p.name} (r={r})", flush=True)
|
|
134
141
|
|
|
135
142
|
_state.update(model=model, tok=tok, device=args.device, model_id=args.model)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: streamft
|
|
3
|
-
Version: 0.6.
|
|
3
|
+
Version: 0.6.1
|
|
4
4
|
Summary: Fine-tune models that do not fit: stream NF4 weights from disk, one transformer block at a time.
|
|
5
5
|
Author-email: Gokhan Cava <byjameson@users.noreply.github.com>
|
|
6
6
|
License: Apache-2.0
|
|
@@ -150,8 +150,9 @@ streamft train --model-dir checkpoints/qwen7b_nf4 --sft --steps 1000 \
|
|
|
150
150
|
- On the reference Ampere card fp16 is slightly faster than bf16 at 7B (1.79 vs 1.91 s/step,
|
|
151
151
|
same 5.36 GB peak); on Pascal it is the only option.
|
|
152
152
|
- On a **4 GB** card use `--blocks-on-gpu 1`, `--seq 512` and `--vram-hard-gb 3.4`.
|
|
153
|
-
Those rows are not measured yet — if you run one,
|
|
154
|
-
|
|
153
|
+
Those rows are not measured yet — if you run one, open a
|
|
154
|
+
[hardware report](.github/ISSUE_TEMPLATE/hardware-report.md) and it can become a measured
|
|
155
|
+
entry in `streamft fit`.
|
|
155
156
|
- `streamft fit` prints this hint automatically when it sees a pre-Ampere GPU.
|
|
156
157
|
|
|
157
158
|
### Tokenizer cost and the diffusion question
|
|
@@ -210,12 +211,22 @@ Measured on the reference machine (Qwen2.5-3B, 60-step Dolly LoRA):
|
|
|
210
211
|
| 8-bit | 8.56 MB | 0.0222 |
|
|
211
212
|
| **4-bit** | **4.88 MB** | **0.0126** |
|
|
212
213
|
|
|
213
|
-
|
|
214
|
-
4-bit 2.
|
|
215
|
-
|
|
216
|
-
|
|
214
|
+
870 held-out Dolly rows, masked SFT loss: base 2.3752, original 2.3403, 8-bit 2.3401
|
|
215
|
+
(p = 0.81, no measurable loss), 4-bit 2.3420 (+0.0018, p = 0.001, 5% of the gain). A
|
|
216
|
+
second task, SST-2 (2,000 train / 400 test), goes from **71.25%** base to **93.75%**, and
|
|
217
|
+
both packed adapters stay at **93.75%** (masked loss 0.7940/0.7942 vs 0.7930; p = 0.50 /
|
|
218
|
+
0.41). A third task, AG News (4-way), goes from 2.50% with the bare prompt (not followed)
|
|
219
|
+
to **62.25%**; 8-bit and 4-bit stay at **62.00%**. The adapter loads in 0.115 s. The LoRA
|
|
220
|
+
path is ~26% slower than the base on this card (8.75 vs 11.79 tok/s; a bf16 LoRA path
|
|
221
|
+
reaches 9.13); merge it into a bf16 base for zero per-token overhead, or accept the LoRA
|
|
222
|
+
path when storage/switching matters more than throughput.
|
|
217
223
|
Rank reduction is not viable: SVD to rank 8 loses 35.8% of the update.
|
|
218
224
|
|
|
225
|
+
**A prompt can be the better rung.** On the same base, a 441-byte 4-shot SST-2 prompt
|
|
226
|
+
scores 94.0% (adapter 93.75%) and a 242-byte 4-shot AG News prompt scores 61.75% (adapter
|
|
227
|
+
62.25%). If a few examples express the task, use the prompt; the adapter is for behaviour
|
|
228
|
+
a prompt cannot express, or when prompt tokens/context cost more than the storage.
|
|
229
|
+
|
|
219
230
|
### Measured on
|
|
220
231
|
|
|
221
232
|
Every speed number here comes from **one machine**: RTX 3050 Laptop 6 GB, Windows 11,
|
|
@@ -50,3 +50,35 @@ def test_lora_linear_shapes():
|
|
|
50
50
|
assert m.A.shape == (2, 8)
|
|
51
51
|
assert m.B.shape == (4, 2)
|
|
52
52
|
assert m(torch.randn(3, 8)).shape == (3, 4)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def test_cli_accepts_save_dir(tmp_path):
|
|
56
|
+
from streamft.adapter import main
|
|
57
|
+
|
|
58
|
+
save_dir = tmp_path / "run"
|
|
59
|
+
save_dir.mkdir()
|
|
60
|
+
torch.save(_state(), save_dir / "adapter.pt")
|
|
61
|
+
out = tmp_path / "adapter_q8.pt"
|
|
62
|
+
assert main(["--adapter", str(save_dir), "--bits", "8", "--out", str(out)]) == 0
|
|
63
|
+
assert out.exists()
|
|
64
|
+
back, _ = load_adapter(out)
|
|
65
|
+
assert set(back) == set(_state())
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def test_cli_rejects_non_lora_state(tmp_path):
|
|
69
|
+
import pytest
|
|
70
|
+
|
|
71
|
+
from streamft.adapter import main
|
|
72
|
+
|
|
73
|
+
path = tmp_path / "plain.pt"
|
|
74
|
+
torch.save({"weight": torch.zeros(4, 4)}, path)
|
|
75
|
+
with pytest.raises(SystemExit, match="no LoRA"):
|
|
76
|
+
main(["--adapter", str(path), "--bits", "8", "--out", str(tmp_path / "q.pt")])
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def test_lora_fast_path():
|
|
80
|
+
m = LoRALinear(nn.Linear(8, 4).to(torch.bfloat16), r=2)
|
|
81
|
+
m.set_fast(torch.bfloat16)
|
|
82
|
+
out = m(torch.randn(3, 8, dtype=torch.bfloat16))
|
|
83
|
+
assert out.shape == (3, 4)
|
|
84
|
+
assert out.dtype == torch.bfloat16
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|