streamft 0.2.6__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {streamft-0.2.6 → streamft-0.3.0}/PKG-INFO +9 -1
- {streamft-0.2.6 → streamft-0.3.0}/README.md +8 -0
- {streamft-0.2.6 → streamft-0.3.0}/pyproject.toml +1 -1
- {streamft-0.2.6 → streamft-0.3.0}/streamft/__main__.py +12 -0
- {streamft-0.2.6 → streamft-0.3.0}/streamft/cli.py +17 -5
- streamft-0.3.0/streamft/easy.py +226 -0
- {streamft-0.2.6 → streamft-0.3.0}/streamft/fit.py +33 -0
- {streamft-0.2.6 → streamft-0.3.0}/streamft.egg-info/PKG-INFO +9 -1
- {streamft-0.2.6 → streamft-0.3.0}/streamft.egg-info/SOURCES.txt +1 -0
- {streamft-0.2.6 → streamft-0.3.0}/tests/test_streamft_smoke.py +23 -0
- {streamft-0.2.6 → streamft-0.3.0}/LICENSE +0 -0
- {streamft-0.2.6 → streamft-0.3.0}/setup.cfg +0 -0
- {streamft-0.2.6 → streamft-0.3.0}/streamft/__init__.py +0 -0
- {streamft-0.2.6 → streamft-0.3.0}/streamft/autotune.py +0 -0
- {streamft-0.2.6 → streamft-0.3.0}/streamft/data.py +0 -0
- {streamft-0.2.6 → streamft-0.3.0}/streamft/disk_quant_store.py +0 -0
- {streamft-0.2.6 → streamft-0.3.0}/streamft/doctor.py +0 -0
- {streamft-0.2.6 → streamft-0.3.0}/streamft/lora.py +0 -0
- {streamft-0.2.6 → streamft-0.3.0}/streamft/metrics.py +0 -0
- {streamft-0.2.6 → streamft-0.3.0}/streamft/stream_quant_lora.py +0 -0
- {streamft-0.2.6 → streamft-0.3.0}/streamft/stream_train.py +0 -0
- {streamft-0.2.6 → streamft-0.3.0}/streamft.egg-info/dependency_links.txt +0 -0
- {streamft-0.2.6 → streamft-0.3.0}/streamft.egg-info/entry_points.txt +0 -0
- {streamft-0.2.6 → streamft-0.3.0}/streamft.egg-info/requires.txt +0 -0
- {streamft-0.2.6 → streamft-0.3.0}/streamft.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: streamft
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: Fine-tune models that do not fit: stream NF4 weights from disk, one transformer block at a time.
|
|
5
5
|
Author-email: Gokhan Cava <byjameson@users.noreply.github.com>
|
|
6
6
|
License: Apache-2.0
|
|
@@ -79,9 +79,17 @@ streamft train --model-dir checkpoints/qwen0.5b_nf4 --sft --steps 20 --seq 512 -
|
|
|
79
79
|
# LM path (optional): build token shards, then train without --sft
|
|
80
80
|
streamft data --model Qwen/Qwen2.5-0.5B --out data
|
|
81
81
|
streamft train --model-dir checkpoints/qwen0.5b_nf4 --steps 20 --seq 256 --tag lm
|
|
82
|
+
|
|
83
|
+
# no GPU at all, or no dataset: both are covered
|
|
84
|
+
streamft cpu --model Qwen/Qwen2.5-0.5B --steps 50 --seq 128
|
|
85
|
+
streamft synth --model Qwen/Qwen2.5-1.5B-Instruct --n 200 --out data/synth.jsonl
|
|
86
|
+
|
|
87
|
+
# inference budget (weights + KV cache), not just training
|
|
88
|
+
streamft fit --model Qwen/Qwen2.5-7B-Instruct --gpu 6 --seq 2048 --mode inference
|
|
82
89
|
```
|
|
83
90
|
|
|
84
91
|
Every run appends one JSON row to `results/results.jsonl` next to where you launched it.
|
|
92
|
+
Six worked recipes: [`COOKBOOK.md`](COOKBOOK.md).
|
|
85
93
|
The same commands are in [`QUICKSTART.md`](QUICKSTART.md), which is what a pip user should
|
|
86
94
|
read — this README also documents the experiments behind the numbers.
|
|
87
95
|
|
|
@@ -49,9 +49,17 @@ streamft train --model-dir checkpoints/qwen0.5b_nf4 --sft --steps 20 --seq 512 -
|
|
|
49
49
|
# LM path (optional): build token shards, then train without --sft
|
|
50
50
|
streamft data --model Qwen/Qwen2.5-0.5B --out data
|
|
51
51
|
streamft train --model-dir checkpoints/qwen0.5b_nf4 --steps 20 --seq 256 --tag lm
|
|
52
|
+
|
|
53
|
+
# no GPU at all, or no dataset: both are covered
|
|
54
|
+
streamft cpu --model Qwen/Qwen2.5-0.5B --steps 50 --seq 128
|
|
55
|
+
streamft synth --model Qwen/Qwen2.5-1.5B-Instruct --n 200 --out data/synth.jsonl
|
|
56
|
+
|
|
57
|
+
# inference budget (weights + KV cache), not just training
|
|
58
|
+
streamft fit --model Qwen/Qwen2.5-7B-Instruct --gpu 6 --seq 2048 --mode inference
|
|
52
59
|
```
|
|
53
60
|
|
|
54
61
|
Every run appends one JSON row to `results/results.jsonl` next to where you launched it.
|
|
62
|
+
Six worked recipes: [`COOKBOOK.md`](COOKBOOK.md).
|
|
55
63
|
The same commands are in [`QUICKSTART.md`](QUICKSTART.md), which is what a pip user should
|
|
56
64
|
read — this README also documents the experiments behind the numbers.
|
|
57
65
|
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "streamft"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.3.0"
|
|
8
8
|
description = "Fine-tune models that do not fit: stream NF4 weights from disk, one transformer block at a time."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
|
@@ -4,6 +4,8 @@
|
|
|
4
4
|
streamft prepare turn a bf16 model into a pre-quantized NF4 checkpoint
|
|
5
5
|
streamft train the streaming trainer (all exp54 flags)
|
|
6
6
|
streamft data build the token shards (LM path only; SFT needs none)
|
|
7
|
+
streamft cpu SFT a small model on the CPU (no GPU; model must fit RAM)
|
|
8
|
+
streamft synth generate training pairs locally with a small model
|
|
7
9
|
streamft doctor environment report (torch, CUDA, bitsandbytes, VRAM)
|
|
8
10
|
"""
|
|
9
11
|
from __future__ import annotations
|
|
@@ -16,6 +18,8 @@ USAGE = """streamft - fine-tune models that do not fit
|
|
|
16
18
|
streamft prepare --model <dir> --save-quant checkpoints/<name>
|
|
17
19
|
streamft train --model-dir checkpoints/<name> --sft --steps 30 [--seq 1024]
|
|
18
20
|
streamft data --model <dir|hf-id> --out data
|
|
21
|
+
streamft cpu --model <small-model> --steps 20
|
|
22
|
+
streamft synth --model <small-model> --n 50 --out data/synth.jsonl
|
|
19
23
|
streamft doctor
|
|
20
24
|
|
|
21
25
|
Docs: https://github.com/byjameson/stream-finetune
|
|
@@ -35,6 +39,14 @@ def main() -> int:
|
|
|
35
39
|
|
|
36
40
|
return fit_main(rest)
|
|
37
41
|
|
|
42
|
+
if cmd == "cpu":
|
|
43
|
+
from streamft.easy import cpu_main
|
|
44
|
+
return cpu_main(rest)
|
|
45
|
+
|
|
46
|
+
if cmd == "synth":
|
|
47
|
+
from streamft.easy import synth_main
|
|
48
|
+
return synth_main(rest)
|
|
49
|
+
|
|
38
50
|
if cmd == "data":
|
|
39
51
|
from streamft.data import main as data_main
|
|
40
52
|
|
|
@@ -388,14 +388,23 @@ def main() -> int:
|
|
|
388
388
|
tok = AutoTokenizer.from_pretrained(model_dir)
|
|
389
389
|
if tok.pad_token is None:
|
|
390
390
|
tok.pad_token = tok.eos_token
|
|
391
|
-
|
|
391
|
+
if str(args.dataset).lower().endswith(".jsonl"):
|
|
392
|
+
# local {"instruction","response"} file, e.g. from `streamft synth`
|
|
393
|
+
ds = load_dataset("json", data_files=args.dataset, split="train").shuffle(seed=1234)
|
|
394
|
+
else:
|
|
395
|
+
ds = load_dataset(args.dataset, split="train").shuffle(seed=1234)
|
|
392
396
|
train_rows = [ds[i] for i in range(min(args.train_size, len(ds)))]
|
|
393
397
|
eval_rows = [ds[len(train_rows) + i]
|
|
394
398
|
for i in range(min(args.eval_size, len(ds) - len(train_rows)))]
|
|
395
399
|
print(f"[sft] {Path(args.dataset).name} train={len(train_rows)} "
|
|
396
400
|
f"eval={len(eval_rows)}", flush=True)
|
|
397
|
-
|
|
398
|
-
|
|
401
|
+
if eval_rows:
|
|
402
|
+
ce0 = _masked_eval(tr, eval_rows, tok, args.seq)
|
|
403
|
+
print(f"[eval] before loss={ce0:.4f}", flush=True)
|
|
404
|
+
else:
|
|
405
|
+
ce0 = None
|
|
406
|
+
print(f"[sft] no held-out rows (dataset has {len(train_rows)} rows); skipping eval",
|
|
407
|
+
flush=True)
|
|
399
408
|
# Filter once so `--steps N` means N optimizer updates, not "N attempts minus
|
|
400
409
|
# whatever the mask rejected". A silently short run is a lie about the budget.
|
|
401
410
|
valid_rows, skipped_train = sft_schedule(
|
|
@@ -419,8 +428,11 @@ def main() -> int:
|
|
|
419
428
|
if (i + 1) % 10 == 0:
|
|
420
429
|
print(f"[step] {i+1}/{args.steps} loss={loss:.4f} {dt:.2f}s "
|
|
421
430
|
f"peak={peak:.2f}GB rss={host_rss_gb()}GB", flush=True)
|
|
422
|
-
|
|
423
|
-
|
|
431
|
+
if eval_rows:
|
|
432
|
+
ce1 = _masked_eval(tr, eval_rows, tok, args.seq)
|
|
433
|
+
print(f"[eval] after loss={ce1:.4f} (delta {ce1-ce0:+.4f})", flush=True)
|
|
434
|
+
else:
|
|
435
|
+
ce1 = None
|
|
424
436
|
wall = time.time() - t_all
|
|
425
437
|
else:
|
|
426
438
|
tokens = load_tokens(str(user_path(args.data)))
|
|
@@ -0,0 +1,226 @@
|
|
|
1
|
+
"""Two on-ramps for people without a training GPU.
|
|
2
|
+
|
|
3
|
+
streamft cpu SFT a small model on the CPU (the model must fit in RAM)
|
|
4
|
+
streamft synth generate training pairs locally with a small model (no API, no data hunt)
|
|
5
|
+
|
|
6
|
+
Both are deliberately simple: plain transformers forward/backward, our LoRA module, the
|
|
7
|
+
same prompt masking as the GPU path. The streaming engine is not involved.
|
|
8
|
+
"""
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import argparse
|
|
12
|
+
import json
|
|
13
|
+
import random
|
|
14
|
+
import re
|
|
15
|
+
import time
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
|
|
18
|
+
import torch
|
|
19
|
+
|
|
20
|
+
from .cli import _encode_chat, _format_messages # noqa: F401 (same masking rules)
|
|
21
|
+
from .lora import apply_lora, lora_parameters, lora_state_dict
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
# --------------------------------------------------------------------------- cpu train
|
|
25
|
+
|
|
26
|
+
def _load_tiny(model_dir: str):
|
|
27
|
+
from transformers import AutoModelForCausalLM
|
|
28
|
+
|
|
29
|
+
dtype = torch.float32 # CPU: bf16 is slow and fp16 is unsupported on many CPUs
|
|
30
|
+
model = AutoModelForCausalLM.from_pretrained(model_dir, torch_dtype=dtype)
|
|
31
|
+
model.eval()
|
|
32
|
+
return model
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@torch.no_grad()
|
|
36
|
+
def _eval_masked(model, rows, tok, seq: int) -> float:
|
|
37
|
+
model.eval()
|
|
38
|
+
total, n = 0.0, 0
|
|
39
|
+
for row in rows:
|
|
40
|
+
enc = _encode_chat(tok, row, seq)
|
|
41
|
+
if enc is None:
|
|
42
|
+
continue
|
|
43
|
+
ids, labels = enc
|
|
44
|
+
loss = model(input_ids=ids, labels=labels).loss
|
|
45
|
+
v = int((labels != -100).sum())
|
|
46
|
+
total += float(loss) * v
|
|
47
|
+
n += v
|
|
48
|
+
model.train()
|
|
49
|
+
if n == 0:
|
|
50
|
+
raise RuntimeError("no eval row has a valid prompt mask")
|
|
51
|
+
return total / n
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def cpu_main(argv=None) -> int:
|
|
55
|
+
from datasets import load_dataset
|
|
56
|
+
from transformers import AutoTokenizer
|
|
57
|
+
|
|
58
|
+
ap = argparse.ArgumentParser(prog="streamft cpu")
|
|
59
|
+
ap.add_argument("--model", required=True, help="small model dir or HF id (fits in RAM)")
|
|
60
|
+
ap.add_argument("--dataset", default="databricks/databricks-dolly-15k")
|
|
61
|
+
ap.add_argument("--train_size", type=int, default=128)
|
|
62
|
+
ap.add_argument("--eval_size", type=int, default=8)
|
|
63
|
+
ap.add_argument("--steps", type=int, default=20)
|
|
64
|
+
ap.add_argument("--seq", type=int, default=256)
|
|
65
|
+
ap.add_argument("--lr", type=float, default=3e-4)
|
|
66
|
+
ap.add_argument("--lora_r", type=int, default=8)
|
|
67
|
+
ap.add_argument("--save", default="")
|
|
68
|
+
ap.add_argument("--tag", default="cpu_sft")
|
|
69
|
+
args = ap.parse_args(argv)
|
|
70
|
+
|
|
71
|
+
tok = AutoTokenizer.from_pretrained(args.model)
|
|
72
|
+
if tok.pad_token is None:
|
|
73
|
+
tok.pad_token = tok.eos_token
|
|
74
|
+
t0 = time.time()
|
|
75
|
+
model = _load_tiny(args.model)
|
|
76
|
+
apply_lora(model, r_attn=args.lora_r, r_mlp=0)
|
|
77
|
+
params = list(lora_parameters(model))
|
|
78
|
+
opt = torch.optim.AdamW(params, lr=args.lr)
|
|
79
|
+
print(f"[cpu] {args.model} loaded in {time.time()-t0:.1f}s, "
|
|
80
|
+
f"{sum(p.numel() for p in params)/1e6:.2f}M adapter params", flush=True)
|
|
81
|
+
|
|
82
|
+
ds = load_dataset(args.dataset, split="train").shuffle(seed=1234)
|
|
83
|
+
train_rows = [ds[i] for i in range(min(args.train_size, len(ds)))]
|
|
84
|
+
eval_rows = [ds[len(train_rows) + i]
|
|
85
|
+
for i in range(min(args.eval_size, len(ds) - len(train_rows)))]
|
|
86
|
+
|
|
87
|
+
valid = [r for r in train_rows if _encode_chat(tok, r, args.seq) is not None]
|
|
88
|
+
dropped = len(train_rows) - len(valid)
|
|
89
|
+
if not valid:
|
|
90
|
+
raise SystemExit("[cpu] no training row has a valid prompt mask")
|
|
91
|
+
if dropped:
|
|
92
|
+
print(f"[cpu] {dropped}/{len(train_rows)} rows dropped (prompt not a prefix)",
|
|
93
|
+
flush=True)
|
|
94
|
+
|
|
95
|
+
ce0 = _eval_masked(model, eval_rows, tok, args.seq)
|
|
96
|
+
print(f"[cpu] eval before {ce0:.4f}", flush=True)
|
|
97
|
+
|
|
98
|
+
model.train()
|
|
99
|
+
losses, times = [], []
|
|
100
|
+
for i in range(args.steps):
|
|
101
|
+
enc = _encode_chat(tok, valid[i % len(valid)], args.seq)
|
|
102
|
+
ids, labels = enc
|
|
103
|
+
t = time.time()
|
|
104
|
+
loss = model(input_ids=ids, labels=labels).loss
|
|
105
|
+
loss.backward()
|
|
106
|
+
opt.step()
|
|
107
|
+
opt.zero_grad(set_to_none=True)
|
|
108
|
+
losses.append(round(float(loss.detach()), 6))
|
|
109
|
+
times.append(round(time.time() - t, 3))
|
|
110
|
+
if (i + 1) % 5 == 0 or i == 0:
|
|
111
|
+
print(f"[cpu] step {i+1}/{args.steps} loss={losses[-1]:.4f} "
|
|
112
|
+
f"{times[-1]:.2f}s", flush=True)
|
|
113
|
+
|
|
114
|
+
ce1 = _eval_masked(model, eval_rows, tok, args.seq)
|
|
115
|
+
if args.save:
|
|
116
|
+
out = Path(args.save)
|
|
117
|
+
out.mkdir(parents=True, exist_ok=True)
|
|
118
|
+
torch.save(lora_state_dict(model), str(out / "adapter.pt"))
|
|
119
|
+
row = {
|
|
120
|
+
"engine": "cpu-dense", "tag": args.tag, "model": args.model, "steps": len(losses),
|
|
121
|
+
"seq": args.seq, "lr": args.lr, "lora_r": args.lora_r,
|
|
122
|
+
"step_s_med": sorted(times)[len(times) // 2] if times else None,
|
|
123
|
+
"ce_before": round(ce0, 5), "ce_after": round(ce1, 5),
|
|
124
|
+
"ce_delta": round(ce1 - ce0, 5), "losses": losses,
|
|
125
|
+
"sft_skipped": dropped, "saved": args.save or None,
|
|
126
|
+
"license": "Apache-2.0", "device": "cpu",
|
|
127
|
+
}
|
|
128
|
+
print(json.dumps(row, ensure_ascii=True), flush=True)
|
|
129
|
+
out = Path("results")
|
|
130
|
+
out.mkdir(exist_ok=True)
|
|
131
|
+
with open(out / "results.jsonl", "a", encoding="utf-8") as f:
|
|
132
|
+
f.write(json.dumps(row) + "\n")
|
|
133
|
+
return 0
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
# --------------------------------------------------------------------------- synth
|
|
137
|
+
|
|
138
|
+
PROMPT = """You write training data. Produce ONE example as JSON with keys
|
|
139
|
+
"instruction" and "response". Topic: {task}.
|
|
140
|
+
Rules: the instruction is a short request; the response is a direct answer of 1-3
|
|
141
|
+
sentences; no markdown, no preamble, no extra keys.
|
|
142
|
+
JSON:"""
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
PLACEHOLDER_MARKERS = ("<instruction>", "<response>", "<your", "your instruction",
|
|
146
|
+
"lorem ipsum", "example.com")
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def parse_pair(text: str):
|
|
150
|
+
"""Extract one {"instruction","response"} object; reject template junk.
|
|
151
|
+
|
|
152
|
+
Small models happily echo the schema back ("<instruction>"), and a single such row
|
|
153
|
+
poisons a dataset. Everything is filtered here so the caller only has to dedupe.
|
|
154
|
+
"""
|
|
155
|
+
m = re.search(r"\{.*\}", text, re.S)
|
|
156
|
+
if not m:
|
|
157
|
+
return None
|
|
158
|
+
try:
|
|
159
|
+
obj = json.loads(m.group(0))
|
|
160
|
+
except Exception:
|
|
161
|
+
return None
|
|
162
|
+
if not isinstance(obj, dict):
|
|
163
|
+
return None
|
|
164
|
+
ins = str(obj.get("instruction", "")).strip()
|
|
165
|
+
res = str(obj.get("response", "")).strip()
|
|
166
|
+
low = (ins + " " + res).lower()
|
|
167
|
+
if any(p in low for p in PLACEHOLDER_MARKERS):
|
|
168
|
+
return None
|
|
169
|
+
if "{" in ins or "{" in res or "}" in ins or "}" in res:
|
|
170
|
+
return None
|
|
171
|
+
if not (5 <= len(ins) <= 200 and 10 <= len(res) <= 1200):
|
|
172
|
+
return None
|
|
173
|
+
return {"instruction": ins, "response": res}
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def synth_main(argv=None) -> int:
|
|
177
|
+
from transformers import AutoModelForCausalLM, AutoTokenizer
|
|
178
|
+
|
|
179
|
+
ap = argparse.ArgumentParser(prog="streamft synth")
|
|
180
|
+
ap.add_argument("--model", default="Qwen/Qwen2.5-1.5B-Instruct")
|
|
181
|
+
ap.add_argument("--out", default="data/synth.jsonl")
|
|
182
|
+
ap.add_argument("--n", type=int, default=50, help="examples to attempt")
|
|
183
|
+
ap.add_argument("--task", default="practical everyday knowledge")
|
|
184
|
+
ap.add_argument("--seq", type=int, default=256)
|
|
185
|
+
ap.add_argument("--temperature", type=float, default=0.9)
|
|
186
|
+
ap.add_argument("--device", default="cuda" if torch.cuda.is_available() else "cpu")
|
|
187
|
+
args = ap.parse_args(argv)
|
|
188
|
+
|
|
189
|
+
tok = AutoTokenizer.from_pretrained(args.model)
|
|
190
|
+
model = AutoModelForCausalLM.from_pretrained(
|
|
191
|
+
args.model,
|
|
192
|
+
torch_dtype=torch.bfloat16 if args.device == "cuda" else torch.float32,
|
|
193
|
+
).to(args.device).eval()
|
|
194
|
+
|
|
195
|
+
seen, kept = set(), []
|
|
196
|
+
out = Path(args.out)
|
|
197
|
+
out.parent.mkdir(parents=True, exist_ok=True)
|
|
198
|
+
t0 = time.time()
|
|
199
|
+
for i in range(args.n):
|
|
200
|
+
prompt = PROMPT.format(task=args.task)
|
|
201
|
+
ids = tok(prompt, return_tensors="pt").to(args.device)
|
|
202
|
+
with torch.no_grad():
|
|
203
|
+
gen = model.generate(**ids, max_new_tokens=args.seq, do_sample=True,
|
|
204
|
+
temperature=args.temperature, top_p=0.95,
|
|
205
|
+
pad_token_id=tok.eos_token_id)
|
|
206
|
+
text = tok.decode(gen[0][ids["input_ids"].shape[1]:], skip_special_tokens=True)
|
|
207
|
+
pair = parse_pair(text)
|
|
208
|
+
if pair is None:
|
|
209
|
+
continue
|
|
210
|
+
key = pair["instruction"].lower()
|
|
211
|
+
if key in seen:
|
|
212
|
+
continue
|
|
213
|
+
seen.add(key)
|
|
214
|
+
kept.append(pair)
|
|
215
|
+
if (len(kept) % 10) == 0:
|
|
216
|
+
print(f"[synth] {len(kept)} kept / {i+1} attempts", flush=True)
|
|
217
|
+
|
|
218
|
+
with open(out, "w", encoding="utf-8") as f:
|
|
219
|
+
for r in kept:
|
|
220
|
+
f.write(json.dumps(r, ensure_ascii=False) + "\n")
|
|
221
|
+
print(json.dumps({
|
|
222
|
+
"out": str(out), "attempts": args.n, "kept": len(kept),
|
|
223
|
+
"keep_rate": round(len(kept) / max(args.n, 1), 3),
|
|
224
|
+
"seconds": round(time.time() - t0, 1), "model": args.model, "task": args.task,
|
|
225
|
+
}, indent=1))
|
|
226
|
+
return 0
|
|
@@ -53,6 +53,16 @@ def load_config(model: str) -> dict:
|
|
|
53
53
|
return json.loads(Path(hf_hub_download(model, "config.json")).read_text(encoding="utf-8"))
|
|
54
54
|
|
|
55
55
|
|
|
56
|
+
def kv_cache_gb(cfg: dict, seq: int, batch: int = 1, bytes_per: int = 2) -> float:
|
|
57
|
+
"""KV cache at inference: 2 tensors x layers x kv_heads x head_dim x seq x batch."""
|
|
58
|
+
h = int(cfg.get("hidden_size") or cfg.get("n_embd"))
|
|
59
|
+
layers = int(cfg.get("num_hidden_layers") or cfg.get("n_layer"))
|
|
60
|
+
heads = int(cfg.get("num_attention_heads") or 1)
|
|
61
|
+
kv_heads = int(cfg.get("num_key_value_heads") or heads)
|
|
62
|
+
head_dim = h // max(heads, 1)
|
|
63
|
+
return 2.0 * layers * kv_heads * head_dim * seq * batch * bytes_per / 1e9
|
|
64
|
+
|
|
65
|
+
|
|
56
66
|
def model_fingerprint(cfg: dict) -> tuple:
|
|
57
67
|
"""Everything that changes the measurement: two 7B models are not interchangeable."""
|
|
58
68
|
return (
|
|
@@ -95,6 +105,8 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
95
105
|
ap.add_argument("--model", required=True, help="local dir or HF id")
|
|
96
106
|
ap.add_argument("--gpu", type=float, default=6.0, help="VRAM in GB (default 6)")
|
|
97
107
|
ap.add_argument("--seq", type=int, default=512)
|
|
108
|
+
ap.add_argument("--mode", choices=["train", "inference"], default="train")
|
|
109
|
+
ap.add_argument("--batch", type=int, default=1)
|
|
98
110
|
ap.add_argument("--json", action="store_true")
|
|
99
111
|
args = ap.parse_args(argv)
|
|
100
112
|
|
|
@@ -126,6 +138,27 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
126
138
|
"measured_on_rtx3050_6gb": measured,
|
|
127
139
|
"measured_elsewhere": measured is None,
|
|
128
140
|
}
|
|
141
|
+
if args.mode == "inference":
|
|
142
|
+
kv_gb = kv_cache_gb(cfg, args.seq, args.batch)
|
|
143
|
+
total = m["nf4_gb"] + m["head_gb"] + kv_gb + 0.5
|
|
144
|
+
out_inf = {
|
|
145
|
+
"model": args.model, "mode": "inference", "seq": args.seq, "batch": args.batch,
|
|
146
|
+
"weights_gb": round(m["nf4_gb"] + m["head_gb"], 1),
|
|
147
|
+
"kv_cache_gb": round(kv_gb, 2),
|
|
148
|
+
"needed_gb": round(total, 1),
|
|
149
|
+
"fits_whole": total <= args.gpu,
|
|
150
|
+
"speed": "not measured for this model",
|
|
151
|
+
}
|
|
152
|
+
if args.json:
|
|
153
|
+
print(json.dumps(out_inf, indent=1))
|
|
154
|
+
return 0
|
|
155
|
+
print(f"{args.model} (inference)")
|
|
156
|
+
print(f" weights ~{out_inf['weights_gb']} GB + KV cache ~{out_inf['kv_cache_gb']} GB "
|
|
157
|
+
f"at seq {args.seq}, batch {args.batch} -> ~{out_inf['needed_gb']} GB")
|
|
158
|
+
print(f" verdict: {'fits' if out_inf['fits_whole'] else 'does NOT fit'} on {args.gpu:g} GB")
|
|
159
|
+
print(" speed: not measured for this model (the planner does not guess tok/s)")
|
|
160
|
+
return 0
|
|
161
|
+
|
|
129
162
|
if args.json:
|
|
130
163
|
print(json.dumps(out, indent=1))
|
|
131
164
|
return 0
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: streamft
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: Fine-tune models that do not fit: stream NF4 weights from disk, one transformer block at a time.
|
|
5
5
|
Author-email: Gokhan Cava <byjameson@users.noreply.github.com>
|
|
6
6
|
License: Apache-2.0
|
|
@@ -79,9 +79,17 @@ streamft train --model-dir checkpoints/qwen0.5b_nf4 --sft --steps 20 --seq 512 -
|
|
|
79
79
|
# LM path (optional): build token shards, then train without --sft
|
|
80
80
|
streamft data --model Qwen/Qwen2.5-0.5B --out data
|
|
81
81
|
streamft train --model-dir checkpoints/qwen0.5b_nf4 --steps 20 --seq 256 --tag lm
|
|
82
|
+
|
|
83
|
+
# no GPU at all, or no dataset: both are covered
|
|
84
|
+
streamft cpu --model Qwen/Qwen2.5-0.5B --steps 50 --seq 128
|
|
85
|
+
streamft synth --model Qwen/Qwen2.5-1.5B-Instruct --n 200 --out data/synth.jsonl
|
|
86
|
+
|
|
87
|
+
# inference budget (weights + KV cache), not just training
|
|
88
|
+
streamft fit --model Qwen/Qwen2.5-7B-Instruct --gpu 6 --seq 2048 --mode inference
|
|
82
89
|
```
|
|
83
90
|
|
|
84
91
|
Every run appends one JSON row to `results/results.jsonl` next to where you launched it.
|
|
92
|
+
Six worked recipes: [`COOKBOOK.md`](COOKBOOK.md).
|
|
85
93
|
The same commands are in [`QUICKSTART.md`](QUICKSTART.md), which is what a pip user should
|
|
86
94
|
read — this README also documents the experiments behind the numbers.
|
|
87
95
|
|
|
@@ -248,3 +248,26 @@ def test_masked_eval_raises_when_every_row_is_skipped(monkeypatch):
|
|
|
248
248
|
monkeypatch.setattr(cli, "_encode_chat", lambda *a, **k: None)
|
|
249
249
|
with pytest.raises(RuntimeError, match="every row was skipped"):
|
|
250
250
|
cli._masked_eval(FakeTrainer(), [{"instruction": "x", "response": "y"}], object(), 8)
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
# ------------------------------------------------ synth / planner helpers
|
|
254
|
+
|
|
255
|
+
def test_parse_pair_rejects_template_junk():
|
|
256
|
+
from streamft.easy import parse_pair
|
|
257
|
+
|
|
258
|
+
good = parse_pair('{"instruction": "Give me a tip", "response": "Sharpen your knives."}')
|
|
259
|
+
assert good and good["instruction"] == "Give me a tip"
|
|
260
|
+
assert parse_pair('{"instruction": "<instruction>", "response": "<response>"}') is None
|
|
261
|
+
assert parse_pair('{"instruction": "x", "response": "short"}') is None
|
|
262
|
+
assert parse_pair("not json") is None
|
|
263
|
+
assert parse_pair('{"instruction": "a" * 300, "response": "b" * 20}') is None
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
def test_kv_cache_matches_hand_calculation():
|
|
267
|
+
from streamft.fit import kv_cache_gb
|
|
268
|
+
|
|
269
|
+
cfg = {"hidden_size": 2048, "num_hidden_layers": 24, "num_attention_heads": 16,
|
|
270
|
+
"num_key_value_heads": 2}
|
|
271
|
+
# 2 * 24 layers * 2 kv heads * 128 head_dim * 1024 seq * 2 bytes
|
|
272
|
+
expect = 2 * 24 * 2 * 128 * 1024 * 2 / 1e9
|
|
273
|
+
assert abs(kv_cache_gb(cfg, 1024) - expect) < 1e-12
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|