streamft 0.2.5__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {streamft-0.2.5 → streamft-0.3.0}/PKG-INFO +27 -2
- {streamft-0.2.5 → streamft-0.3.0}/README.md +26 -1
- {streamft-0.2.5 → streamft-0.3.0}/pyproject.toml +1 -1
- {streamft-0.2.5 → streamft-0.3.0}/streamft/__main__.py +12 -0
- {streamft-0.2.5 → streamft-0.3.0}/streamft/cli.py +29 -5
- {streamft-0.2.5 → streamft-0.3.0}/streamft/disk_quant_store.py +6 -2
- streamft-0.3.0/streamft/easy.py +226 -0
- {streamft-0.2.5 → streamft-0.3.0}/streamft/fit.py +43 -0
- {streamft-0.2.5 → streamft-0.3.0}/streamft.egg-info/PKG-INFO +27 -2
- {streamft-0.2.5 → streamft-0.3.0}/streamft.egg-info/SOURCES.txt +1 -0
- {streamft-0.2.5 → streamft-0.3.0}/tests/test_streamft_smoke.py +23 -0
- {streamft-0.2.5 → streamft-0.3.0}/LICENSE +0 -0
- {streamft-0.2.5 → streamft-0.3.0}/setup.cfg +0 -0
- {streamft-0.2.5 → streamft-0.3.0}/streamft/__init__.py +0 -0
- {streamft-0.2.5 → streamft-0.3.0}/streamft/autotune.py +0 -0
- {streamft-0.2.5 → streamft-0.3.0}/streamft/data.py +0 -0
- {streamft-0.2.5 → streamft-0.3.0}/streamft/doctor.py +0 -0
- {streamft-0.2.5 → streamft-0.3.0}/streamft/lora.py +0 -0
- {streamft-0.2.5 → streamft-0.3.0}/streamft/metrics.py +0 -0
- {streamft-0.2.5 → streamft-0.3.0}/streamft/stream_quant_lora.py +0 -0
- {streamft-0.2.5 → streamft-0.3.0}/streamft/stream_train.py +0 -0
- {streamft-0.2.5 → streamft-0.3.0}/streamft.egg-info/dependency_links.txt +0 -0
- {streamft-0.2.5 → streamft-0.3.0}/streamft.egg-info/entry_points.txt +0 -0
- {streamft-0.2.5 → streamft-0.3.0}/streamft.egg-info/requires.txt +0 -0
- {streamft-0.2.5 → streamft-0.3.0}/streamft.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: streamft
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: Fine-tune models that do not fit: stream NF4 weights from disk, one transformer block at a time.
|
|
5
5
|
Author-email: Gokhan Cava <byjameson@users.noreply.github.com>
|
|
6
6
|
License: Apache-2.0
|
|
@@ -79,15 +79,24 @@ streamft train --model-dir checkpoints/qwen0.5b_nf4 --sft --steps 20 --seq 512 -
|
|
|
79
79
|
# LM path (optional): build token shards, then train without --sft
|
|
80
80
|
streamft data --model Qwen/Qwen2.5-0.5B --out data
|
|
81
81
|
streamft train --model-dir checkpoints/qwen0.5b_nf4 --steps 20 --seq 256 --tag lm
|
|
82
|
+
|
|
83
|
+
# no GPU at all, or no dataset: both are covered
|
|
84
|
+
streamft cpu --model Qwen/Qwen2.5-0.5B --steps 50 --seq 128
|
|
85
|
+
streamft synth --model Qwen/Qwen2.5-1.5B-Instruct --n 200 --out data/synth.jsonl
|
|
86
|
+
|
|
87
|
+
# inference budget (weights + KV cache), not just training
|
|
88
|
+
streamft fit --model Qwen/Qwen2.5-7B-Instruct --gpu 6 --seq 2048 --mode inference
|
|
82
89
|
```
|
|
83
90
|
|
|
84
91
|
Every run appends one JSON row to `results/results.jsonl` next to where you launched it.
|
|
92
|
+
Six worked recipes: [`COOKBOOK.md`](COOKBOOK.md).
|
|
85
93
|
The same commands are in [`QUICKSTART.md`](QUICKSTART.md), which is what a pip user should
|
|
86
94
|
read — this README also documents the experiments behind the numbers.
|
|
87
95
|
|
|
88
96
|
**From a source checkout only** (not part of the wheel): `tools/finetune.py` is a wrapper
|
|
89
97
|
that encodes the VRAM presets, `scripts/compare_peft.py` reproduces the engine comparison,
|
|
90
|
-
`scripts/verify_gradients.py` the gradient check, `scripts/
|
|
98
|
+
`scripts/verify_gradients.py` the gradient check, `scripts/e2e_wheel.py` the clean-venv
|
|
99
|
+
install test (build ? fresh venv ? prepare ? train ? save adapter), `scripts/prep_data.py` the token shards
|
|
91
100
|
(the installed equivalent is `streamft data`), and `pytest tests -q` the CPU suite.
|
|
92
101
|
|
|
93
102
|
`tools/finetune.py` encodes the VRAM presets we measured, because the *combination* of
|
|
@@ -124,6 +133,22 @@ disk (NF4 safetensors)
|
|
|
124
133
|
The training tag in the evidence log is still `exp54_disk_qlora`, so rows written by
|
|
125
134
|
this engine and by the original experiment are directly comparable.
|
|
126
135
|
|
|
136
|
+
## Older GPUs (GTX 10/16-series, 4–8 GB)
|
|
137
|
+
|
|
138
|
+
Pre-Ampere cards (GTX 1060/1660, Pascal/Turing) have **no bf16 units**. Use fp16:
|
|
139
|
+
|
|
140
|
+
```bash
|
|
141
|
+
streamft train --model-dir checkpoints/qwen7b_nf4 --sft --steps 1000 \
|
|
142
|
+
--seq 512 --blocks-on-gpu 1 --compute-dtype fp16 --vram-hard-gb 5.3
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
- On the reference Ampere card fp16 is slightly faster than bf16 at 7B (1.79 vs 1.91 s/step,
|
|
146
|
+
same 5.36 GB peak); on Pascal it is the only option.
|
|
147
|
+
- On a **4 GB** card use `--blocks-on-gpu 1`, `--seq 512` and `--vram-hard-gb 3.4`.
|
|
148
|
+
Those rows are not measured yet — if you run one, post the JSON row and it can become a
|
|
149
|
+
measured entry in `streamft fit`.
|
|
150
|
+
- `streamft fit` prints this hint automatically when it sees a pre-Ampere GPU.
|
|
151
|
+
|
|
127
152
|
## Verification
|
|
128
153
|
|
|
129
154
|
`python scripts/verify_gradients.py --model-dir checkpoints/qwen3b_bnb4bit --seq 128`
|
|
@@ -49,15 +49,24 @@ streamft train --model-dir checkpoints/qwen0.5b_nf4 --sft --steps 20 --seq 512 -
|
|
|
49
49
|
# LM path (optional): build token shards, then train without --sft
|
|
50
50
|
streamft data --model Qwen/Qwen2.5-0.5B --out data
|
|
51
51
|
streamft train --model-dir checkpoints/qwen0.5b_nf4 --steps 20 --seq 256 --tag lm
|
|
52
|
+
|
|
53
|
+
# no GPU at all, or no dataset: both are covered
|
|
54
|
+
streamft cpu --model Qwen/Qwen2.5-0.5B --steps 50 --seq 128
|
|
55
|
+
streamft synth --model Qwen/Qwen2.5-1.5B-Instruct --n 200 --out data/synth.jsonl
|
|
56
|
+
|
|
57
|
+
# inference budget (weights + KV cache), not just training
|
|
58
|
+
streamft fit --model Qwen/Qwen2.5-7B-Instruct --gpu 6 --seq 2048 --mode inference
|
|
52
59
|
```
|
|
53
60
|
|
|
54
61
|
Every run appends one JSON row to `results/results.jsonl` next to where you launched it.
|
|
62
|
+
Six worked recipes: [`COOKBOOK.md`](COOKBOOK.md).
|
|
55
63
|
The same commands are in [`QUICKSTART.md`](QUICKSTART.md), which is what a pip user should
|
|
56
64
|
read — this README also documents the experiments behind the numbers.
|
|
57
65
|
|
|
58
66
|
**From a source checkout only** (not part of the wheel): `tools/finetune.py` is a wrapper
|
|
59
67
|
that encodes the VRAM presets, `scripts/compare_peft.py` reproduces the engine comparison,
|
|
60
|
-
`scripts/verify_gradients.py` the gradient check, `scripts/
|
|
68
|
+
`scripts/verify_gradients.py` the gradient check, `scripts/e2e_wheel.py` the clean-venv
|
|
69
|
+
install test (build ? fresh venv ? prepare ? train ? save adapter), `scripts/prep_data.py` the token shards
|
|
61
70
|
(the installed equivalent is `streamft data`), and `pytest tests -q` the CPU suite.
|
|
62
71
|
|
|
63
72
|
`tools/finetune.py` encodes the VRAM presets we measured, because the *combination* of
|
|
@@ -94,6 +103,22 @@ disk (NF4 safetensors)
|
|
|
94
103
|
The training tag in the evidence log is still `exp54_disk_qlora`, so rows written by
|
|
95
104
|
this engine and by the original experiment are directly comparable.
|
|
96
105
|
|
|
106
|
+
## Older GPUs (GTX 10/16-series, 4–8 GB)
|
|
107
|
+
|
|
108
|
+
Pre-Ampere cards (GTX 1060/1660, Pascal/Turing) have **no bf16 units**. Use fp16:
|
|
109
|
+
|
|
110
|
+
```bash
|
|
111
|
+
streamft train --model-dir checkpoints/qwen7b_nf4 --sft --steps 1000 \
|
|
112
|
+
--seq 512 --blocks-on-gpu 1 --compute-dtype fp16 --vram-hard-gb 5.3
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
- On the reference Ampere card fp16 is slightly faster than bf16 at 7B (1.79 vs 1.91 s/step,
|
|
116
|
+
same 5.36 GB peak); on Pascal it is the only option.
|
|
117
|
+
- On a **4 GB** card use `--blocks-on-gpu 1`, `--seq 512` and `--vram-hard-gb 3.4`.
|
|
118
|
+
Those rows are not measured yet — if you run one, post the JSON row and it can become a
|
|
119
|
+
measured entry in `streamft fit`.
|
|
120
|
+
- `streamft fit` prints this hint automatically when it sees a pre-Ampere GPU.
|
|
121
|
+
|
|
97
122
|
## Verification
|
|
98
123
|
|
|
99
124
|
`python scripts/verify_gradients.py --model-dir checkpoints/qwen3b_bnb4bit --seq 128`
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "streamft"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.3.0"
|
|
8
8
|
description = "Fine-tune models that do not fit: stream NF4 weights from disk, one transformer block at a time."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
|
@@ -4,6 +4,8 @@
|
|
|
4
4
|
streamft prepare turn a bf16 model into a pre-quantized NF4 checkpoint
|
|
5
5
|
streamft train the streaming trainer (all exp54 flags)
|
|
6
6
|
streamft data build the token shards (LM path only; SFT needs none)
|
|
7
|
+
streamft cpu SFT a small model on the CPU (no GPU; model must fit RAM)
|
|
8
|
+
streamft synth generate training pairs locally with a small model
|
|
7
9
|
streamft doctor environment report (torch, CUDA, bitsandbytes, VRAM)
|
|
8
10
|
"""
|
|
9
11
|
from __future__ import annotations
|
|
@@ -16,6 +18,8 @@ USAGE = """streamft - fine-tune models that do not fit
|
|
|
16
18
|
streamft prepare --model <dir> --save-quant checkpoints/<name>
|
|
17
19
|
streamft train --model-dir checkpoints/<name> --sft --steps 30 [--seq 1024]
|
|
18
20
|
streamft data --model <dir|hf-id> --out data
|
|
21
|
+
streamft cpu --model <small-model> --steps 20
|
|
22
|
+
streamft synth --model <small-model> --n 50 --out data/synth.jsonl
|
|
19
23
|
streamft doctor
|
|
20
24
|
|
|
21
25
|
Docs: https://github.com/byjameson/stream-finetune
|
|
@@ -35,6 +39,14 @@ def main() -> int:
|
|
|
35
39
|
|
|
36
40
|
return fit_main(rest)
|
|
37
41
|
|
|
42
|
+
if cmd == "cpu":
|
|
43
|
+
from streamft.easy import cpu_main
|
|
44
|
+
return cpu_main(rest)
|
|
45
|
+
|
|
46
|
+
if cmd == "synth":
|
|
47
|
+
from streamft.easy import synth_main
|
|
48
|
+
return synth_main(rest)
|
|
49
|
+
|
|
38
50
|
if cmd == "data":
|
|
39
51
|
from streamft.data import main as data_main
|
|
40
52
|
|
|
@@ -219,6 +219,9 @@ def main() -> int:
|
|
|
219
219
|
ap.add_argument("--lora_alpha", type=int, default=16)
|
|
220
220
|
ap.add_argument("--targets", choices=["attn", "all"], default="attn")
|
|
221
221
|
ap.add_argument("--head", choices=["bf16", "nf4"], default="bf16")
|
|
222
|
+
ap.add_argument("--compute-dtype", choices=["bf16", "fp16", "fp32"], default="bf16",
|
|
223
|
+
help="matmul precision for the 4-bit base; use fp16 on pre-Ampere GPUs "
|
|
224
|
+
"(GTX 10/16-series), which have no bf16 units")
|
|
222
225
|
ap.add_argument("--shared-dtype", choices=["fp32", "as-stored"], default="fp32",
|
|
223
226
|
help="fp32 matches prepare_model_for_kbit_training; 'as-stored' "
|
|
224
227
|
"keeps embed/head in the checkpoint dtype (big models)")
|
|
@@ -261,10 +264,18 @@ def main() -> int:
|
|
|
261
264
|
f"lora_r={args.lora_r} targets={args.targets} head={args.head}", flush=True)
|
|
262
265
|
|
|
263
266
|
t0 = time.time()
|
|
267
|
+
compute_dtype = {"bf16": torch.bfloat16, "fp16": torch.float16,
|
|
268
|
+
"fp32": torch.float32}[args.compute_dtype]
|
|
269
|
+
if args.compute_dtype == "bf16" and torch.cuda.is_available():
|
|
270
|
+
major, _ = torch.cuda.get_device_capability(0)
|
|
271
|
+
if major < 8:
|
|
272
|
+
print(f"[warn] this GPU (compute capability {major}.x) has no bf16 units; "
|
|
273
|
+
f"use --compute-dtype fp16", flush=True)
|
|
264
274
|
model, cfg = build_skeleton(
|
|
265
275
|
model_dir, lora_r=args.lora_r, lora_alpha=args.lora_alpha,
|
|
266
276
|
targets=args.targets, head=args.head,
|
|
267
277
|
low_precision_shared=(args.shared_dtype == "as-stored"),
|
|
278
|
+
compute_dtype=compute_dtype,
|
|
268
279
|
)
|
|
269
280
|
store = DiskQuantStore(model_dir, device="cuda")
|
|
270
281
|
|
|
@@ -377,14 +388,23 @@ def main() -> int:
|
|
|
377
388
|
tok = AutoTokenizer.from_pretrained(model_dir)
|
|
378
389
|
if tok.pad_token is None:
|
|
379
390
|
tok.pad_token = tok.eos_token
|
|
380
|
-
|
|
391
|
+
if str(args.dataset).lower().endswith(".jsonl"):
|
|
392
|
+
# local {"instruction","response"} file, e.g. from `streamft synth`
|
|
393
|
+
ds = load_dataset("json", data_files=args.dataset, split="train").shuffle(seed=1234)
|
|
394
|
+
else:
|
|
395
|
+
ds = load_dataset(args.dataset, split="train").shuffle(seed=1234)
|
|
381
396
|
train_rows = [ds[i] for i in range(min(args.train_size, len(ds)))]
|
|
382
397
|
eval_rows = [ds[len(train_rows) + i]
|
|
383
398
|
for i in range(min(args.eval_size, len(ds) - len(train_rows)))]
|
|
384
399
|
print(f"[sft] {Path(args.dataset).name} train={len(train_rows)} "
|
|
385
400
|
f"eval={len(eval_rows)}", flush=True)
|
|
386
|
-
|
|
387
|
-
|
|
401
|
+
if eval_rows:
|
|
402
|
+
ce0 = _masked_eval(tr, eval_rows, tok, args.seq)
|
|
403
|
+
print(f"[eval] before loss={ce0:.4f}", flush=True)
|
|
404
|
+
else:
|
|
405
|
+
ce0 = None
|
|
406
|
+
print(f"[sft] no held-out rows (dataset has {len(train_rows)} rows); skipping eval",
|
|
407
|
+
flush=True)
|
|
388
408
|
# Filter once so `--steps N` means N optimizer updates, not "N attempts minus
|
|
389
409
|
# whatever the mask rejected". A silently short run is a lie about the budget.
|
|
390
410
|
valid_rows, skipped_train = sft_schedule(
|
|
@@ -408,8 +428,11 @@ def main() -> int:
|
|
|
408
428
|
if (i + 1) % 10 == 0:
|
|
409
429
|
print(f"[step] {i+1}/{args.steps} loss={loss:.4f} {dt:.2f}s "
|
|
410
430
|
f"peak={peak:.2f}GB rss={host_rss_gb()}GB", flush=True)
|
|
411
|
-
|
|
412
|
-
|
|
431
|
+
if eval_rows:
|
|
432
|
+
ce1 = _masked_eval(tr, eval_rows, tok, args.seq)
|
|
433
|
+
print(f"[eval] after loss={ce1:.4f} (delta {ce1-ce0:+.4f})", flush=True)
|
|
434
|
+
else:
|
|
435
|
+
ce1 = None
|
|
413
436
|
wall = time.time() - t_all
|
|
414
437
|
else:
|
|
415
438
|
tokens = load_tokens(str(user_path(args.data)))
|
|
@@ -465,6 +488,7 @@ def main() -> int:
|
|
|
465
488
|
"lora_alpha": args.lora_alpha,
|
|
466
489
|
"targets": args.targets,
|
|
467
490
|
"head": args.head,
|
|
491
|
+
"compute_dtype": args.compute_dtype,
|
|
468
492
|
"shared_dtype": args.shared_dtype,
|
|
469
493
|
"blocks_mode": args.blocks_on_gpu,
|
|
470
494
|
"blocks_on_gpu": cache.capacity,
|
|
@@ -83,7 +83,8 @@ class DiskQuantStore:
|
|
|
83
83
|
|
|
84
84
|
def build_skeleton(model_dir, lora_r: int = 8, lora_alpha: int = 16,
|
|
85
85
|
targets: str = "attn", head: str = "bf16",
|
|
86
|
-
low_precision_shared: bool = False, with_lora: bool = True
|
|
86
|
+
low_precision_shared: bool = False, with_lora: bool = True,
|
|
87
|
+
compute_dtype=None):
|
|
87
88
|
"""Instantiate the model on the `meta` device with bnb Linear4bit + LoRA.
|
|
88
89
|
|
|
89
90
|
No weight memory is allocated here; only the (tiny) LoRA adapters are real.
|
|
@@ -104,10 +105,13 @@ def build_skeleton(model_dir, lora_r: int = 8, lora_alpha: int = 16,
|
|
|
104
105
|
with torch.device("meta"):
|
|
105
106
|
model = AutoModelForCausalLM.from_config(cfg)
|
|
106
107
|
|
|
108
|
+
# Older cards (GTX 10-series, Pascal/Turing) have no bf16 units at all; fp16 is the
|
|
109
|
+
# only half-precision option there. Everything else defaults to bf16.
|
|
110
|
+
compute_dtype = compute_dtype or torch.bfloat16
|
|
107
111
|
qcfg = BitsAndBytesConfig(
|
|
108
112
|
load_in_4bit=True,
|
|
109
113
|
bnb_4bit_quant_type="nf4",
|
|
110
|
-
bnb_4bit_compute_dtype=
|
|
114
|
+
bnb_4bit_compute_dtype=compute_dtype,
|
|
111
115
|
bnb_4bit_use_double_quant=True,
|
|
112
116
|
bnb_4bit_quant_storage=torch.uint8,
|
|
113
117
|
)
|
|
@@ -0,0 +1,226 @@
|
|
|
1
|
+
"""Two on-ramps for people without a training GPU.
|
|
2
|
+
|
|
3
|
+
streamft cpu SFT a small model on the CPU (the model must fit in RAM)
|
|
4
|
+
streamft synth generate training pairs locally with a small model (no API, no data hunt)
|
|
5
|
+
|
|
6
|
+
Both are deliberately simple: plain transformers forward/backward, our LoRA module, the
|
|
7
|
+
same prompt masking as the GPU path. The streaming engine is not involved.
|
|
8
|
+
"""
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import argparse
|
|
12
|
+
import json
|
|
13
|
+
import random
|
|
14
|
+
import re
|
|
15
|
+
import time
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
|
|
18
|
+
import torch
|
|
19
|
+
|
|
20
|
+
from .cli import _encode_chat, _format_messages # noqa: F401 (same masking rules)
|
|
21
|
+
from .lora import apply_lora, lora_parameters, lora_state_dict
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
# --------------------------------------------------------------------------- cpu train
|
|
25
|
+
|
|
26
|
+
def _load_tiny(model_dir: str):
|
|
27
|
+
from transformers import AutoModelForCausalLM
|
|
28
|
+
|
|
29
|
+
dtype = torch.float32 # CPU: bf16 is slow and fp16 is unsupported on many CPUs
|
|
30
|
+
model = AutoModelForCausalLM.from_pretrained(model_dir, torch_dtype=dtype)
|
|
31
|
+
model.eval()
|
|
32
|
+
return model
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@torch.no_grad()
|
|
36
|
+
def _eval_masked(model, rows, tok, seq: int) -> float:
|
|
37
|
+
model.eval()
|
|
38
|
+
total, n = 0.0, 0
|
|
39
|
+
for row in rows:
|
|
40
|
+
enc = _encode_chat(tok, row, seq)
|
|
41
|
+
if enc is None:
|
|
42
|
+
continue
|
|
43
|
+
ids, labels = enc
|
|
44
|
+
loss = model(input_ids=ids, labels=labels).loss
|
|
45
|
+
v = int((labels != -100).sum())
|
|
46
|
+
total += float(loss) * v
|
|
47
|
+
n += v
|
|
48
|
+
model.train()
|
|
49
|
+
if n == 0:
|
|
50
|
+
raise RuntimeError("no eval row has a valid prompt mask")
|
|
51
|
+
return total / n
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def cpu_main(argv=None) -> int:
|
|
55
|
+
from datasets import load_dataset
|
|
56
|
+
from transformers import AutoTokenizer
|
|
57
|
+
|
|
58
|
+
ap = argparse.ArgumentParser(prog="streamft cpu")
|
|
59
|
+
ap.add_argument("--model", required=True, help="small model dir or HF id (fits in RAM)")
|
|
60
|
+
ap.add_argument("--dataset", default="databricks/databricks-dolly-15k")
|
|
61
|
+
ap.add_argument("--train_size", type=int, default=128)
|
|
62
|
+
ap.add_argument("--eval_size", type=int, default=8)
|
|
63
|
+
ap.add_argument("--steps", type=int, default=20)
|
|
64
|
+
ap.add_argument("--seq", type=int, default=256)
|
|
65
|
+
ap.add_argument("--lr", type=float, default=3e-4)
|
|
66
|
+
ap.add_argument("--lora_r", type=int, default=8)
|
|
67
|
+
ap.add_argument("--save", default="")
|
|
68
|
+
ap.add_argument("--tag", default="cpu_sft")
|
|
69
|
+
args = ap.parse_args(argv)
|
|
70
|
+
|
|
71
|
+
tok = AutoTokenizer.from_pretrained(args.model)
|
|
72
|
+
if tok.pad_token is None:
|
|
73
|
+
tok.pad_token = tok.eos_token
|
|
74
|
+
t0 = time.time()
|
|
75
|
+
model = _load_tiny(args.model)
|
|
76
|
+
apply_lora(model, r_attn=args.lora_r, r_mlp=0)
|
|
77
|
+
params = list(lora_parameters(model))
|
|
78
|
+
opt = torch.optim.AdamW(params, lr=args.lr)
|
|
79
|
+
print(f"[cpu] {args.model} loaded in {time.time()-t0:.1f}s, "
|
|
80
|
+
f"{sum(p.numel() for p in params)/1e6:.2f}M adapter params", flush=True)
|
|
81
|
+
|
|
82
|
+
ds = load_dataset(args.dataset, split="train").shuffle(seed=1234)
|
|
83
|
+
train_rows = [ds[i] for i in range(min(args.train_size, len(ds)))]
|
|
84
|
+
eval_rows = [ds[len(train_rows) + i]
|
|
85
|
+
for i in range(min(args.eval_size, len(ds) - len(train_rows)))]
|
|
86
|
+
|
|
87
|
+
valid = [r for r in train_rows if _encode_chat(tok, r, args.seq) is not None]
|
|
88
|
+
dropped = len(train_rows) - len(valid)
|
|
89
|
+
if not valid:
|
|
90
|
+
raise SystemExit("[cpu] no training row has a valid prompt mask")
|
|
91
|
+
if dropped:
|
|
92
|
+
print(f"[cpu] {dropped}/{len(train_rows)} rows dropped (prompt not a prefix)",
|
|
93
|
+
flush=True)
|
|
94
|
+
|
|
95
|
+
ce0 = _eval_masked(model, eval_rows, tok, args.seq)
|
|
96
|
+
print(f"[cpu] eval before {ce0:.4f}", flush=True)
|
|
97
|
+
|
|
98
|
+
model.train()
|
|
99
|
+
losses, times = [], []
|
|
100
|
+
for i in range(args.steps):
|
|
101
|
+
enc = _encode_chat(tok, valid[i % len(valid)], args.seq)
|
|
102
|
+
ids, labels = enc
|
|
103
|
+
t = time.time()
|
|
104
|
+
loss = model(input_ids=ids, labels=labels).loss
|
|
105
|
+
loss.backward()
|
|
106
|
+
opt.step()
|
|
107
|
+
opt.zero_grad(set_to_none=True)
|
|
108
|
+
losses.append(round(float(loss.detach()), 6))
|
|
109
|
+
times.append(round(time.time() - t, 3))
|
|
110
|
+
if (i + 1) % 5 == 0 or i == 0:
|
|
111
|
+
print(f"[cpu] step {i+1}/{args.steps} loss={losses[-1]:.4f} "
|
|
112
|
+
f"{times[-1]:.2f}s", flush=True)
|
|
113
|
+
|
|
114
|
+
ce1 = _eval_masked(model, eval_rows, tok, args.seq)
|
|
115
|
+
if args.save:
|
|
116
|
+
out = Path(args.save)
|
|
117
|
+
out.mkdir(parents=True, exist_ok=True)
|
|
118
|
+
torch.save(lora_state_dict(model), str(out / "adapter.pt"))
|
|
119
|
+
row = {
|
|
120
|
+
"engine": "cpu-dense", "tag": args.tag, "model": args.model, "steps": len(losses),
|
|
121
|
+
"seq": args.seq, "lr": args.lr, "lora_r": args.lora_r,
|
|
122
|
+
"step_s_med": sorted(times)[len(times) // 2] if times else None,
|
|
123
|
+
"ce_before": round(ce0, 5), "ce_after": round(ce1, 5),
|
|
124
|
+
"ce_delta": round(ce1 - ce0, 5), "losses": losses,
|
|
125
|
+
"sft_skipped": dropped, "saved": args.save or None,
|
|
126
|
+
"license": "Apache-2.0", "device": "cpu",
|
|
127
|
+
}
|
|
128
|
+
print(json.dumps(row, ensure_ascii=True), flush=True)
|
|
129
|
+
out = Path("results")
|
|
130
|
+
out.mkdir(exist_ok=True)
|
|
131
|
+
with open(out / "results.jsonl", "a", encoding="utf-8") as f:
|
|
132
|
+
f.write(json.dumps(row) + "\n")
|
|
133
|
+
return 0
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
# --------------------------------------------------------------------------- synth
|
|
137
|
+
|
|
138
|
+
PROMPT = """You write training data. Produce ONE example as JSON with keys
|
|
139
|
+
"instruction" and "response". Topic: {task}.
|
|
140
|
+
Rules: the instruction is a short request; the response is a direct answer of 1-3
|
|
141
|
+
sentences; no markdown, no preamble, no extra keys.
|
|
142
|
+
JSON:"""
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
PLACEHOLDER_MARKERS = ("<instruction>", "<response>", "<your", "your instruction",
|
|
146
|
+
"lorem ipsum", "example.com")
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def parse_pair(text: str):
|
|
150
|
+
"""Extract one {"instruction","response"} object; reject template junk.
|
|
151
|
+
|
|
152
|
+
Small models happily echo the schema back ("<instruction>"), and a single such row
|
|
153
|
+
poisons a dataset. Everything is filtered here so the caller only has to dedupe.
|
|
154
|
+
"""
|
|
155
|
+
m = re.search(r"\{.*\}", text, re.S)
|
|
156
|
+
if not m:
|
|
157
|
+
return None
|
|
158
|
+
try:
|
|
159
|
+
obj = json.loads(m.group(0))
|
|
160
|
+
except Exception:
|
|
161
|
+
return None
|
|
162
|
+
if not isinstance(obj, dict):
|
|
163
|
+
return None
|
|
164
|
+
ins = str(obj.get("instruction", "")).strip()
|
|
165
|
+
res = str(obj.get("response", "")).strip()
|
|
166
|
+
low = (ins + " " + res).lower()
|
|
167
|
+
if any(p in low for p in PLACEHOLDER_MARKERS):
|
|
168
|
+
return None
|
|
169
|
+
if "{" in ins or "{" in res or "}" in ins or "}" in res:
|
|
170
|
+
return None
|
|
171
|
+
if not (5 <= len(ins) <= 200 and 10 <= len(res) <= 1200):
|
|
172
|
+
return None
|
|
173
|
+
return {"instruction": ins, "response": res}
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def synth_main(argv=None) -> int:
|
|
177
|
+
from transformers import AutoModelForCausalLM, AutoTokenizer
|
|
178
|
+
|
|
179
|
+
ap = argparse.ArgumentParser(prog="streamft synth")
|
|
180
|
+
ap.add_argument("--model", default="Qwen/Qwen2.5-1.5B-Instruct")
|
|
181
|
+
ap.add_argument("--out", default="data/synth.jsonl")
|
|
182
|
+
ap.add_argument("--n", type=int, default=50, help="examples to attempt")
|
|
183
|
+
ap.add_argument("--task", default="practical everyday knowledge")
|
|
184
|
+
ap.add_argument("--seq", type=int, default=256)
|
|
185
|
+
ap.add_argument("--temperature", type=float, default=0.9)
|
|
186
|
+
ap.add_argument("--device", default="cuda" if torch.cuda.is_available() else "cpu")
|
|
187
|
+
args = ap.parse_args(argv)
|
|
188
|
+
|
|
189
|
+
tok = AutoTokenizer.from_pretrained(args.model)
|
|
190
|
+
model = AutoModelForCausalLM.from_pretrained(
|
|
191
|
+
args.model,
|
|
192
|
+
torch_dtype=torch.bfloat16 if args.device == "cuda" else torch.float32,
|
|
193
|
+
).to(args.device).eval()
|
|
194
|
+
|
|
195
|
+
seen, kept = set(), []
|
|
196
|
+
out = Path(args.out)
|
|
197
|
+
out.parent.mkdir(parents=True, exist_ok=True)
|
|
198
|
+
t0 = time.time()
|
|
199
|
+
for i in range(args.n):
|
|
200
|
+
prompt = PROMPT.format(task=args.task)
|
|
201
|
+
ids = tok(prompt, return_tensors="pt").to(args.device)
|
|
202
|
+
with torch.no_grad():
|
|
203
|
+
gen = model.generate(**ids, max_new_tokens=args.seq, do_sample=True,
|
|
204
|
+
temperature=args.temperature, top_p=0.95,
|
|
205
|
+
pad_token_id=tok.eos_token_id)
|
|
206
|
+
text = tok.decode(gen[0][ids["input_ids"].shape[1]:], skip_special_tokens=True)
|
|
207
|
+
pair = parse_pair(text)
|
|
208
|
+
if pair is None:
|
|
209
|
+
continue
|
|
210
|
+
key = pair["instruction"].lower()
|
|
211
|
+
if key in seen:
|
|
212
|
+
continue
|
|
213
|
+
seen.add(key)
|
|
214
|
+
kept.append(pair)
|
|
215
|
+
if (len(kept) % 10) == 0:
|
|
216
|
+
print(f"[synth] {len(kept)} kept / {i+1} attempts", flush=True)
|
|
217
|
+
|
|
218
|
+
with open(out, "w", encoding="utf-8") as f:
|
|
219
|
+
for r in kept:
|
|
220
|
+
f.write(json.dumps(r, ensure_ascii=False) + "\n")
|
|
221
|
+
print(json.dumps({
|
|
222
|
+
"out": str(out), "attempts": args.n, "kept": len(kept),
|
|
223
|
+
"keep_rate": round(len(kept) / max(args.n, 1), 3),
|
|
224
|
+
"seconds": round(time.time() - t0, 1), "model": args.model, "task": args.task,
|
|
225
|
+
}, indent=1))
|
|
226
|
+
return 0
|
|
@@ -20,6 +20,8 @@ import argparse
|
|
|
20
20
|
import json
|
|
21
21
|
from pathlib import Path
|
|
22
22
|
|
|
23
|
+
import torch
|
|
24
|
+
|
|
23
25
|
# Measured end-to-end on an RTX 3050 Laptop, 6 GB, Windows 11, AC power, NF4 base,
|
|
24
26
|
# LoRA r=8 on q/k/v/o, batch 1, Dolly-15k. step_s = median warm step.
|
|
25
27
|
MEASURED = [
|
|
@@ -51,6 +53,16 @@ def load_config(model: str) -> dict:
|
|
|
51
53
|
return json.loads(Path(hf_hub_download(model, "config.json")).read_text(encoding="utf-8"))
|
|
52
54
|
|
|
53
55
|
|
|
56
|
+
def kv_cache_gb(cfg: dict, seq: int, batch: int = 1, bytes_per: int = 2) -> float:
|
|
57
|
+
"""KV cache at inference: 2 tensors x layers x kv_heads x head_dim x seq x batch."""
|
|
58
|
+
h = int(cfg.get("hidden_size") or cfg.get("n_embd"))
|
|
59
|
+
layers = int(cfg.get("num_hidden_layers") or cfg.get("n_layer"))
|
|
60
|
+
heads = int(cfg.get("num_attention_heads") or 1)
|
|
61
|
+
kv_heads = int(cfg.get("num_key_value_heads") or heads)
|
|
62
|
+
head_dim = h // max(heads, 1)
|
|
63
|
+
return 2.0 * layers * kv_heads * head_dim * seq * batch * bytes_per / 1e9
|
|
64
|
+
|
|
65
|
+
|
|
54
66
|
def model_fingerprint(cfg: dict) -> tuple:
|
|
55
67
|
"""Everything that changes the measurement: two 7B models are not interchangeable."""
|
|
56
68
|
return (
|
|
@@ -93,6 +105,8 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
93
105
|
ap.add_argument("--model", required=True, help="local dir or HF id")
|
|
94
106
|
ap.add_argument("--gpu", type=float, default=6.0, help="VRAM in GB (default 6)")
|
|
95
107
|
ap.add_argument("--seq", type=int, default=512)
|
|
108
|
+
ap.add_argument("--mode", choices=["train", "inference"], default="train")
|
|
109
|
+
ap.add_argument("--batch", type=int, default=1)
|
|
96
110
|
ap.add_argument("--json", action="store_true")
|
|
97
111
|
args = ap.parse_args(argv)
|
|
98
112
|
|
|
@@ -124,6 +138,27 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
124
138
|
"measured_on_rtx3050_6gb": measured,
|
|
125
139
|
"measured_elsewhere": measured is None,
|
|
126
140
|
}
|
|
141
|
+
if args.mode == "inference":
|
|
142
|
+
kv_gb = kv_cache_gb(cfg, args.seq, args.batch)
|
|
143
|
+
total = m["nf4_gb"] + m["head_gb"] + kv_gb + 0.5
|
|
144
|
+
out_inf = {
|
|
145
|
+
"model": args.model, "mode": "inference", "seq": args.seq, "batch": args.batch,
|
|
146
|
+
"weights_gb": round(m["nf4_gb"] + m["head_gb"], 1),
|
|
147
|
+
"kv_cache_gb": round(kv_gb, 2),
|
|
148
|
+
"needed_gb": round(total, 1),
|
|
149
|
+
"fits_whole": total <= args.gpu,
|
|
150
|
+
"speed": "not measured for this model",
|
|
151
|
+
}
|
|
152
|
+
if args.json:
|
|
153
|
+
print(json.dumps(out_inf, indent=1))
|
|
154
|
+
return 0
|
|
155
|
+
print(f"{args.model} (inference)")
|
|
156
|
+
print(f" weights ~{out_inf['weights_gb']} GB + KV cache ~{out_inf['kv_cache_gb']} GB "
|
|
157
|
+
f"at seq {args.seq}, batch {args.batch} -> ~{out_inf['needed_gb']} GB")
|
|
158
|
+
print(f" verdict: {'fits' if out_inf['fits_whole'] else 'does NOT fit'} on {args.gpu:g} GB")
|
|
159
|
+
print(" speed: not measured for this model (the planner does not guess tok/s)")
|
|
160
|
+
return 0
|
|
161
|
+
|
|
127
162
|
if args.json:
|
|
128
163
|
print(json.dumps(out, indent=1))
|
|
129
164
|
return 0
|
|
@@ -149,6 +184,14 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
149
184
|
print(" speed/blocks: not measured for this size at this seq — start with")
|
|
150
185
|
print(" --blocks-on-gpu 1 and let autotune grow it (it measures")
|
|
151
186
|
print(" the real MB/block on your card instead of guessing)")
|
|
187
|
+
try:
|
|
188
|
+
if torch.cuda.is_available():
|
|
189
|
+
major, _minor = torch.cuda.get_device_capability(0)
|
|
190
|
+
if major < 8:
|
|
191
|
+
print(" note: this GPU has no bf16 units (pre-Ampere) -> train with "
|
|
192
|
+
"--compute-dtype fp16")
|
|
193
|
+
except Exception:
|
|
194
|
+
pass
|
|
152
195
|
print(" next: streamft prepare --model <dir> --save-quant checkpoints/<name>")
|
|
153
196
|
print(" streamft train --model-dir checkpoints/<name> --sft --steps 30")
|
|
154
197
|
return 0
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: streamft
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: Fine-tune models that do not fit: stream NF4 weights from disk, one transformer block at a time.
|
|
5
5
|
Author-email: Gokhan Cava <byjameson@users.noreply.github.com>
|
|
6
6
|
License: Apache-2.0
|
|
@@ -79,15 +79,24 @@ streamft train --model-dir checkpoints/qwen0.5b_nf4 --sft --steps 20 --seq 512 -
|
|
|
79
79
|
# LM path (optional): build token shards, then train without --sft
|
|
80
80
|
streamft data --model Qwen/Qwen2.5-0.5B --out data
|
|
81
81
|
streamft train --model-dir checkpoints/qwen0.5b_nf4 --steps 20 --seq 256 --tag lm
|
|
82
|
+
|
|
83
|
+
# no GPU at all, or no dataset: both are covered
|
|
84
|
+
streamft cpu --model Qwen/Qwen2.5-0.5B --steps 50 --seq 128
|
|
85
|
+
streamft synth --model Qwen/Qwen2.5-1.5B-Instruct --n 200 --out data/synth.jsonl
|
|
86
|
+
|
|
87
|
+
# inference budget (weights + KV cache), not just training
|
|
88
|
+
streamft fit --model Qwen/Qwen2.5-7B-Instruct --gpu 6 --seq 2048 --mode inference
|
|
82
89
|
```
|
|
83
90
|
|
|
84
91
|
Every run appends one JSON row to `results/results.jsonl` next to where you launched it.
|
|
92
|
+
Six worked recipes: [`COOKBOOK.md`](COOKBOOK.md).
|
|
85
93
|
The same commands are in [`QUICKSTART.md`](QUICKSTART.md), which is what a pip user should
|
|
86
94
|
read — this README also documents the experiments behind the numbers.
|
|
87
95
|
|
|
88
96
|
**From a source checkout only** (not part of the wheel): `tools/finetune.py` is a wrapper
|
|
89
97
|
that encodes the VRAM presets, `scripts/compare_peft.py` reproduces the engine comparison,
|
|
90
|
-
`scripts/verify_gradients.py` the gradient check, `scripts/
|
|
98
|
+
`scripts/verify_gradients.py` the gradient check, `scripts/e2e_wheel.py` the clean-venv
|
|
99
|
+
install test (build ? fresh venv ? prepare ? train ? save adapter), `scripts/prep_data.py` the token shards
|
|
91
100
|
(the installed equivalent is `streamft data`), and `pytest tests -q` the CPU suite.
|
|
92
101
|
|
|
93
102
|
`tools/finetune.py` encodes the VRAM presets we measured, because the *combination* of
|
|
@@ -124,6 +133,22 @@ disk (NF4 safetensors)
|
|
|
124
133
|
The training tag in the evidence log is still `exp54_disk_qlora`, so rows written by
|
|
125
134
|
this engine and by the original experiment are directly comparable.
|
|
126
135
|
|
|
136
|
+
## Older GPUs (GTX 10/16-series, 4–8 GB)
|
|
137
|
+
|
|
138
|
+
Pre-Ampere cards (GTX 1060/1660, Pascal/Turing) have **no bf16 units**. Use fp16:
|
|
139
|
+
|
|
140
|
+
```bash
|
|
141
|
+
streamft train --model-dir checkpoints/qwen7b_nf4 --sft --steps 1000 \
|
|
142
|
+
--seq 512 --blocks-on-gpu 1 --compute-dtype fp16 --vram-hard-gb 5.3
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
- On the reference Ampere card fp16 is slightly faster than bf16 at 7B (1.79 vs 1.91 s/step,
|
|
146
|
+
same 5.36 GB peak); on Pascal it is the only option.
|
|
147
|
+
- On a **4 GB** card use `--blocks-on-gpu 1`, `--seq 512` and `--vram-hard-gb 3.4`.
|
|
148
|
+
Those rows are not measured yet — if you run one, post the JSON row and it can become a
|
|
149
|
+
measured entry in `streamft fit`.
|
|
150
|
+
- `streamft fit` prints this hint automatically when it sees a pre-Ampere GPU.
|
|
151
|
+
|
|
127
152
|
## Verification
|
|
128
153
|
|
|
129
154
|
`python scripts/verify_gradients.py --model-dir checkpoints/qwen3b_bnb4bit --seq 128`
|
|
@@ -248,3 +248,26 @@ def test_masked_eval_raises_when_every_row_is_skipped(monkeypatch):
|
|
|
248
248
|
monkeypatch.setattr(cli, "_encode_chat", lambda *a, **k: None)
|
|
249
249
|
with pytest.raises(RuntimeError, match="every row was skipped"):
|
|
250
250
|
cli._masked_eval(FakeTrainer(), [{"instruction": "x", "response": "y"}], object(), 8)
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
# ------------------------------------------------ synth / planner helpers
|
|
254
|
+
|
|
255
|
+
def test_parse_pair_rejects_template_junk():
|
|
256
|
+
from streamft.easy import parse_pair
|
|
257
|
+
|
|
258
|
+
good = parse_pair('{"instruction": "Give me a tip", "response": "Sharpen your knives."}')
|
|
259
|
+
assert good and good["instruction"] == "Give me a tip"
|
|
260
|
+
assert parse_pair('{"instruction": "<instruction>", "response": "<response>"}') is None
|
|
261
|
+
assert parse_pair('{"instruction": "x", "response": "short"}') is None
|
|
262
|
+
assert parse_pair("not json") is None
|
|
263
|
+
assert parse_pair('{"instruction": "a" * 300, "response": "b" * 20}') is None
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
def test_kv_cache_matches_hand_calculation():
|
|
267
|
+
from streamft.fit import kv_cache_gb
|
|
268
|
+
|
|
269
|
+
cfg = {"hidden_size": 2048, "num_hidden_layers": 24, "num_attention_heads": 16,
|
|
270
|
+
"num_key_value_heads": 2}
|
|
271
|
+
# 2 * 24 layers * 2 kv heads * 128 head_dim * 1024 seq * 2 bytes
|
|
272
|
+
expect = 2 * 24 * 2 * 128 * 1024 * 2 / 1e9
|
|
273
|
+
assert abs(kv_cache_gb(cfg, 1024) - expect) < 1e-12
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|