streamft 0.2.4__tar.gz → 0.2.6__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {streamft-0.2.4 → streamft-0.2.6}/PKG-INFO +19 -2
- {streamft-0.2.4 → streamft-0.2.6}/README.md +18 -1
- {streamft-0.2.4 → streamft-0.2.6}/pyproject.toml +1 -1
- {streamft-0.2.4 → streamft-0.2.6}/streamft/cli.py +25 -2
- {streamft-0.2.4 → streamft-0.2.6}/streamft/disk_quant_store.py +6 -2
- {streamft-0.2.4 → streamft-0.2.6}/streamft/fit.py +10 -0
- {streamft-0.2.4 → streamft-0.2.6}/streamft.egg-info/PKG-INFO +19 -2
- {streamft-0.2.4 → streamft-0.2.6}/tests/test_streamft_smoke.py +39 -0
- {streamft-0.2.4 → streamft-0.2.6}/LICENSE +0 -0
- {streamft-0.2.4 → streamft-0.2.6}/setup.cfg +0 -0
- {streamft-0.2.4 → streamft-0.2.6}/streamft/__init__.py +0 -0
- {streamft-0.2.4 → streamft-0.2.6}/streamft/__main__.py +0 -0
- {streamft-0.2.4 → streamft-0.2.6}/streamft/autotune.py +0 -0
- {streamft-0.2.4 → streamft-0.2.6}/streamft/data.py +0 -0
- {streamft-0.2.4 → streamft-0.2.6}/streamft/doctor.py +0 -0
- {streamft-0.2.4 → streamft-0.2.6}/streamft/lora.py +0 -0
- {streamft-0.2.4 → streamft-0.2.6}/streamft/metrics.py +0 -0
- {streamft-0.2.4 → streamft-0.2.6}/streamft/stream_quant_lora.py +0 -0
- {streamft-0.2.4 → streamft-0.2.6}/streamft/stream_train.py +0 -0
- {streamft-0.2.4 → streamft-0.2.6}/streamft.egg-info/SOURCES.txt +0 -0
- {streamft-0.2.4 → streamft-0.2.6}/streamft.egg-info/dependency_links.txt +0 -0
- {streamft-0.2.4 → streamft-0.2.6}/streamft.egg-info/entry_points.txt +0 -0
- {streamft-0.2.4 → streamft-0.2.6}/streamft.egg-info/requires.txt +0 -0
- {streamft-0.2.4 → streamft-0.2.6}/streamft.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: streamft
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.6
|
|
4
4
|
Summary: Fine-tune models that do not fit: stream NF4 weights from disk, one transformer block at a time.
|
|
5
5
|
Author-email: Gokhan Cava <byjameson@users.noreply.github.com>
|
|
6
6
|
License: Apache-2.0
|
|
@@ -87,7 +87,8 @@ read — this README also documents the experiments behind the numbers.
|
|
|
87
87
|
|
|
88
88
|
**From a source checkout only** (not part of the wheel): `tools/finetune.py` is a wrapper
|
|
89
89
|
that encodes the VRAM presets, `scripts/compare_peft.py` reproduces the engine comparison,
|
|
90
|
-
`scripts/verify_gradients.py` the gradient check, `scripts/
|
|
90
|
+
`scripts/verify_gradients.py` the gradient check, `scripts/e2e_wheel.py` the clean-venv
|
|
91
|
+
install test (build ? fresh venv ? prepare ? train ? save adapter), `scripts/prep_data.py` the token shards
|
|
91
92
|
(the installed equivalent is `streamft data`), and `pytest tests -q` the CPU suite.
|
|
92
93
|
|
|
93
94
|
`tools/finetune.py` encodes the VRAM presets we measured, because the *combination* of
|
|
@@ -124,6 +125,22 @@ disk (NF4 safetensors)
|
|
|
124
125
|
The training tag in the evidence log is still `exp54_disk_qlora`, so rows written by
|
|
125
126
|
this engine and by the original experiment are directly comparable.
|
|
126
127
|
|
|
128
|
+
## Older GPUs (GTX 10/16-series, 4–8 GB)
|
|
129
|
+
|
|
130
|
+
Pre-Ampere cards (GTX 1060/1660, Pascal/Turing) have **no bf16 units**. Use fp16:
|
|
131
|
+
|
|
132
|
+
```bash
|
|
133
|
+
streamft train --model-dir checkpoints/qwen7b_nf4 --sft --steps 1000 \
|
|
134
|
+
--seq 512 --blocks-on-gpu 1 --compute-dtype fp16 --vram-hard-gb 5.3
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
- On the reference Ampere card fp16 is slightly faster than bf16 at 7B (1.79 vs 1.91 s/step,
|
|
138
|
+
same 5.36 GB peak); on Pascal it is the only option.
|
|
139
|
+
- On a **4 GB** card use `--blocks-on-gpu 1`, `--seq 512` and `--vram-hard-gb 3.4`.
|
|
140
|
+
Those rows are not measured yet — if you run one, post the JSON row and it can become a
|
|
141
|
+
measured entry in `streamft fit`.
|
|
142
|
+
- `streamft fit` prints this hint automatically when it sees a pre-Ampere GPU.
|
|
143
|
+
|
|
127
144
|
## Verification
|
|
128
145
|
|
|
129
146
|
`python scripts/verify_gradients.py --model-dir checkpoints/qwen3b_bnb4bit --seq 128`
|
|
@@ -57,7 +57,8 @@ read — this README also documents the experiments behind the numbers.
|
|
|
57
57
|
|
|
58
58
|
**From a source checkout only** (not part of the wheel): `tools/finetune.py` is a wrapper
|
|
59
59
|
that encodes the VRAM presets, `scripts/compare_peft.py` reproduces the engine comparison,
|
|
60
|
-
`scripts/verify_gradients.py` the gradient check, `scripts/
|
|
60
|
+
`scripts/verify_gradients.py` the gradient check, `scripts/e2e_wheel.py` the clean-venv
|
|
61
|
+
install test (build ? fresh venv ? prepare ? train ? save adapter), `scripts/prep_data.py` the token shards
|
|
61
62
|
(the installed equivalent is `streamft data`), and `pytest tests -q` the CPU suite.
|
|
62
63
|
|
|
63
64
|
`tools/finetune.py` encodes the VRAM presets we measured, because the *combination* of
|
|
@@ -94,6 +95,22 @@ disk (NF4 safetensors)
|
|
|
94
95
|
The training tag in the evidence log is still `exp54_disk_qlora`, so rows written by
|
|
95
96
|
this engine and by the original experiment are directly comparable.
|
|
96
97
|
|
|
98
|
+
## Older GPUs (GTX 10/16-series, 4–8 GB)
|
|
99
|
+
|
|
100
|
+
Pre-Ampere cards (GTX 1060/1660, Pascal/Turing) have **no bf16 units**. Use fp16:
|
|
101
|
+
|
|
102
|
+
```bash
|
|
103
|
+
streamft train --model-dir checkpoints/qwen7b_nf4 --sft --steps 1000 \
|
|
104
|
+
--seq 512 --blocks-on-gpu 1 --compute-dtype fp16 --vram-hard-gb 5.3
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
- On the reference Ampere card fp16 is slightly faster than bf16 at 7B (1.79 vs 1.91 s/step,
|
|
108
|
+
same 5.36 GB peak); on Pascal it is the only option.
|
|
109
|
+
- On a **4 GB** card use `--blocks-on-gpu 1`, `--seq 512` and `--vram-hard-gb 3.4`.
|
|
110
|
+
Those rows are not measured yet — if you run one, post the JSON row and it can become a
|
|
111
|
+
measured entry in `streamft fit`.
|
|
112
|
+
- `streamft fit` prints this hint automatically when it sees a pre-Ampere GPU.
|
|
113
|
+
|
|
97
114
|
## Verification
|
|
98
115
|
|
|
99
116
|
`python scripts/verify_gradients.py --model-dir checkpoints/qwen3b_bnb4bit --seq 128`
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "streamft"
|
|
7
|
-
version = "0.2.
|
|
7
|
+
version = "0.2.6"
|
|
8
8
|
description = "Fine-tune models that do not fit: stream NF4 weights from disk, one transformer block at a time."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
|
@@ -133,6 +133,17 @@ def _format_messages(row: dict) -> list[dict]:
|
|
|
133
133
|
]
|
|
134
134
|
|
|
135
135
|
|
|
136
|
+
def sft_schedule(rows, encode):
|
|
137
|
+
"""Split rows into trainable ones and a count of dropped ones.
|
|
138
|
+
|
|
139
|
+
The caller loops `for i in range(steps)` over the returned rows, so `--steps N`
|
|
140
|
+
always means N optimizer updates regardless of how many rows the prompt mask
|
|
141
|
+
rejects. Kept separate from the training loop so it can be tested without a GPU.
|
|
142
|
+
"""
|
|
143
|
+
valid = [r for r in rows if encode(r) is not None]
|
|
144
|
+
return valid, len(rows) - len(valid)
|
|
145
|
+
|
|
146
|
+
|
|
136
147
|
def _encode_chat(tokenizer, row: dict, seq: int):
|
|
137
148
|
"""Chat-template encode with the prompt tokens masked out of the loss."""
|
|
138
149
|
messages = _format_messages(row)
|
|
@@ -208,6 +219,9 @@ def main() -> int:
|
|
|
208
219
|
ap.add_argument("--lora_alpha", type=int, default=16)
|
|
209
220
|
ap.add_argument("--targets", choices=["attn", "all"], default="attn")
|
|
210
221
|
ap.add_argument("--head", choices=["bf16", "nf4"], default="bf16")
|
|
222
|
+
ap.add_argument("--compute-dtype", choices=["bf16", "fp16", "fp32"], default="bf16",
|
|
223
|
+
help="matmul precision for the 4-bit base; use fp16 on pre-Ampere GPUs "
|
|
224
|
+
"(GTX 10/16-series), which have no bf16 units")
|
|
211
225
|
ap.add_argument("--shared-dtype", choices=["fp32", "as-stored"], default="fp32",
|
|
212
226
|
help="fp32 matches prepare_model_for_kbit_training; 'as-stored' "
|
|
213
227
|
"keeps embed/head in the checkpoint dtype (big models)")
|
|
@@ -250,10 +264,18 @@ def main() -> int:
|
|
|
250
264
|
f"lora_r={args.lora_r} targets={args.targets} head={args.head}", flush=True)
|
|
251
265
|
|
|
252
266
|
t0 = time.time()
|
|
267
|
+
compute_dtype = {"bf16": torch.bfloat16, "fp16": torch.float16,
|
|
268
|
+
"fp32": torch.float32}[args.compute_dtype]
|
|
269
|
+
if args.compute_dtype == "bf16" and torch.cuda.is_available():
|
|
270
|
+
major, _ = torch.cuda.get_device_capability(0)
|
|
271
|
+
if major < 8:
|
|
272
|
+
print(f"[warn] this GPU (compute capability {major}.x) has no bf16 units; "
|
|
273
|
+
f"use --compute-dtype fp16", flush=True)
|
|
253
274
|
model, cfg = build_skeleton(
|
|
254
275
|
model_dir, lora_r=args.lora_r, lora_alpha=args.lora_alpha,
|
|
255
276
|
targets=args.targets, head=args.head,
|
|
256
277
|
low_precision_shared=(args.shared_dtype == "as-stored"),
|
|
278
|
+
compute_dtype=compute_dtype,
|
|
257
279
|
)
|
|
258
280
|
store = DiskQuantStore(model_dir, device="cuda")
|
|
259
281
|
|
|
@@ -376,8 +398,8 @@ def main() -> int:
|
|
|
376
398
|
print(f"[eval] before loss={ce0:.4f}", flush=True)
|
|
377
399
|
# Filter once so `--steps N` means N optimizer updates, not "N attempts minus
|
|
378
400
|
# whatever the mask rejected". A silently short run is a lie about the budget.
|
|
379
|
-
valid_rows =
|
|
380
|
-
|
|
401
|
+
valid_rows, skipped_train = sft_schedule(
|
|
402
|
+
train_rows, lambda r: _encode_chat(tok, r, args.seq))
|
|
381
403
|
if not valid_rows:
|
|
382
404
|
raise SystemExit("[sft] no training row has a valid prompt mask; refusing to run")
|
|
383
405
|
if skipped_train:
|
|
@@ -454,6 +476,7 @@ def main() -> int:
|
|
|
454
476
|
"lora_alpha": args.lora_alpha,
|
|
455
477
|
"targets": args.targets,
|
|
456
478
|
"head": args.head,
|
|
479
|
+
"compute_dtype": args.compute_dtype,
|
|
457
480
|
"shared_dtype": args.shared_dtype,
|
|
458
481
|
"blocks_mode": args.blocks_on_gpu,
|
|
459
482
|
"blocks_on_gpu": cache.capacity,
|
|
@@ -83,7 +83,8 @@ class DiskQuantStore:
|
|
|
83
83
|
|
|
84
84
|
def build_skeleton(model_dir, lora_r: int = 8, lora_alpha: int = 16,
|
|
85
85
|
targets: str = "attn", head: str = "bf16",
|
|
86
|
-
low_precision_shared: bool = False, with_lora: bool = True
|
|
86
|
+
low_precision_shared: bool = False, with_lora: bool = True,
|
|
87
|
+
compute_dtype=None):
|
|
87
88
|
"""Instantiate the model on the `meta` device with bnb Linear4bit + LoRA.
|
|
88
89
|
|
|
89
90
|
No weight memory is allocated here; only the (tiny) LoRA adapters are real.
|
|
@@ -104,10 +105,13 @@ def build_skeleton(model_dir, lora_r: int = 8, lora_alpha: int = 16,
|
|
|
104
105
|
with torch.device("meta"):
|
|
105
106
|
model = AutoModelForCausalLM.from_config(cfg)
|
|
106
107
|
|
|
108
|
+
# Older cards (GTX 10-series, Pascal/Turing) have no bf16 units at all; fp16 is the
|
|
109
|
+
# only half-precision option there. Everything else defaults to bf16.
|
|
110
|
+
compute_dtype = compute_dtype or torch.bfloat16
|
|
107
111
|
qcfg = BitsAndBytesConfig(
|
|
108
112
|
load_in_4bit=True,
|
|
109
113
|
bnb_4bit_quant_type="nf4",
|
|
110
|
-
bnb_4bit_compute_dtype=
|
|
114
|
+
bnb_4bit_compute_dtype=compute_dtype,
|
|
111
115
|
bnb_4bit_use_double_quant=True,
|
|
112
116
|
bnb_4bit_quant_storage=torch.uint8,
|
|
113
117
|
)
|
|
@@ -20,6 +20,8 @@ import argparse
|
|
|
20
20
|
import json
|
|
21
21
|
from pathlib import Path
|
|
22
22
|
|
|
23
|
+
import torch
|
|
24
|
+
|
|
23
25
|
# Measured end-to-end on an RTX 3050 Laptop, 6 GB, Windows 11, AC power, NF4 base,
|
|
24
26
|
# LoRA r=8 on q/k/v/o, batch 1, Dolly-15k. step_s = median warm step.
|
|
25
27
|
MEASURED = [
|
|
@@ -149,6 +151,14 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
149
151
|
print(" speed/blocks: not measured for this size at this seq — start with")
|
|
150
152
|
print(" --blocks-on-gpu 1 and let autotune grow it (it measures")
|
|
151
153
|
print(" the real MB/block on your card instead of guessing)")
|
|
154
|
+
try:
|
|
155
|
+
if torch.cuda.is_available():
|
|
156
|
+
major, _minor = torch.cuda.get_device_capability(0)
|
|
157
|
+
if major < 8:
|
|
158
|
+
print(" note: this GPU has no bf16 units (pre-Ampere) -> train with "
|
|
159
|
+
"--compute-dtype fp16")
|
|
160
|
+
except Exception:
|
|
161
|
+
pass
|
|
152
162
|
print(" next: streamft prepare --model <dir> --save-quant checkpoints/<name>")
|
|
153
163
|
print(" streamft train --model-dir checkpoints/<name> --sft --steps 30")
|
|
154
164
|
return 0
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: streamft
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.6
|
|
4
4
|
Summary: Fine-tune models that do not fit: stream NF4 weights from disk, one transformer block at a time.
|
|
5
5
|
Author-email: Gokhan Cava <byjameson@users.noreply.github.com>
|
|
6
6
|
License: Apache-2.0
|
|
@@ -87,7 +87,8 @@ read — this README also documents the experiments behind the numbers.
|
|
|
87
87
|
|
|
88
88
|
**From a source checkout only** (not part of the wheel): `tools/finetune.py` is a wrapper
|
|
89
89
|
that encodes the VRAM presets, `scripts/compare_peft.py` reproduces the engine comparison,
|
|
90
|
-
`scripts/verify_gradients.py` the gradient check, `scripts/
|
|
90
|
+
`scripts/verify_gradients.py` the gradient check, `scripts/e2e_wheel.py` the clean-venv
|
|
91
|
+
install test (build ? fresh venv ? prepare ? train ? save adapter), `scripts/prep_data.py` the token shards
|
|
91
92
|
(the installed equivalent is `streamft data`), and `pytest tests -q` the CPU suite.
|
|
92
93
|
|
|
93
94
|
`tools/finetune.py` encodes the VRAM presets we measured, because the *combination* of
|
|
@@ -124,6 +125,22 @@ disk (NF4 safetensors)
|
|
|
124
125
|
The training tag in the evidence log is still `exp54_disk_qlora`, so rows written by
|
|
125
126
|
this engine and by the original experiment are directly comparable.
|
|
126
127
|
|
|
128
|
+
## Older GPUs (GTX 10/16-series, 4–8 GB)
|
|
129
|
+
|
|
130
|
+
Pre-Ampere cards (GTX 1060/1660, Pascal/Turing) have **no bf16 units**. Use fp16:
|
|
131
|
+
|
|
132
|
+
```bash
|
|
133
|
+
streamft train --model-dir checkpoints/qwen7b_nf4 --sft --steps 1000 \
|
|
134
|
+
--seq 512 --blocks-on-gpu 1 --compute-dtype fp16 --vram-hard-gb 5.3
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
- On the reference Ampere card fp16 is slightly faster than bf16 at 7B (1.79 vs 1.91 s/step,
|
|
138
|
+
same 5.36 GB peak); on Pascal it is the only option.
|
|
139
|
+
- On a **4 GB** card use `--blocks-on-gpu 1`, `--seq 512` and `--vram-hard-gb 3.4`.
|
|
140
|
+
Those rows are not measured yet — if you run one, post the JSON row and it can become a
|
|
141
|
+
measured entry in `streamft fit`.
|
|
142
|
+
- `streamft fit` prints this hint automatically when it sees a pre-Ampere GPU.
|
|
143
|
+
|
|
127
144
|
## Verification
|
|
128
145
|
|
|
129
146
|
`python scripts/verify_gradients.py --model-dir checkpoints/qwen3b_bnb4bit --seq 128`
|
|
@@ -8,6 +8,7 @@ They pin the things that broke silently during development:
|
|
|
8
8
|
|
|
9
9
|
import math
|
|
10
10
|
|
|
11
|
+
import pytest
|
|
11
12
|
import torch
|
|
12
13
|
import torch.nn as nn
|
|
13
14
|
import torch.nn.functional as F
|
|
@@ -209,3 +210,41 @@ def test_encode_chat_drops_rows_whose_prompt_is_not_a_prefix():
|
|
|
209
210
|
ids, labels = _encode_chat(_StubTokenizer([1, 2, 3, 4], [1, 2]), row, 32)
|
|
210
211
|
assert ids.tolist() == [[1, 2, 3, 4]]
|
|
211
212
|
assert labels.tolist() == [[-100, -100, 3, 4]]
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
# ------------------------------------------------ SFT step budget & empty eval
|
|
216
|
+
|
|
217
|
+
def test_sft_schedule_keeps_the_step_budget_with_mixed_rows():
|
|
218
|
+
from streamft.cli import sft_schedule
|
|
219
|
+
|
|
220
|
+
rows = list(range(10))
|
|
221
|
+
|
|
222
|
+
def encode(r):
|
|
223
|
+
return None if r % 3 == 0 else [r] # 4 of 10 are rejected
|
|
224
|
+
|
|
225
|
+
valid, dropped = sft_schedule(rows, encode)
|
|
226
|
+
assert dropped == 4 and len(valid) == 6
|
|
227
|
+
|
|
228
|
+
steps = 25
|
|
229
|
+
schedule = [valid[i % len(valid)] for i in range(steps)]
|
|
230
|
+
assert len(schedule) == steps # N updates even though 4 rows were dropped
|
|
231
|
+
assert all(v % 3 != 0 for v in schedule)
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def test_masked_eval_raises_when_every_row_is_skipped(monkeypatch):
|
|
235
|
+
import types
|
|
236
|
+
|
|
237
|
+
from streamft import cli
|
|
238
|
+
|
|
239
|
+
class FakeTrainer:
|
|
240
|
+
def __init__(self):
|
|
241
|
+
self.model = types.SimpleNamespace(eval=lambda: None, train=lambda: None)
|
|
242
|
+
self.norm = nn.Identity()
|
|
243
|
+
self.lm_head = nn.Linear(4, 7, bias=False)
|
|
244
|
+
|
|
245
|
+
def _forward_cached(self, ids):
|
|
246
|
+
return torch.zeros(1, ids.shape[1], 4), None, None
|
|
247
|
+
|
|
248
|
+
monkeypatch.setattr(cli, "_encode_chat", lambda *a, **k: None)
|
|
249
|
+
with pytest.raises(RuntimeError, match="every row was skipped"):
|
|
250
|
+
cli._masked_eval(FakeTrainer(), [{"instruction": "x", "response": "y"}], object(), 8)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|