streamft 0.3.0__tar.gz → 0.4.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {streamft-0.3.0 → streamft-0.4.1}/PKG-INFO +33 -3
- {streamft-0.3.0 → streamft-0.4.1}/README.md +32 -2
- {streamft-0.3.0 → streamft-0.4.1}/pyproject.toml +1 -1
- {streamft-0.3.0 → streamft-0.4.1}/streamft/__main__.py +12 -0
- streamft-0.4.1/streamft/evals.py +148 -0
- streamft-0.4.1/streamft/merge.py +87 -0
- {streamft-0.3.0 → streamft-0.4.1}/streamft.egg-info/PKG-INFO +33 -3
- {streamft-0.3.0 → streamft-0.4.1}/streamft.egg-info/SOURCES.txt +2 -0
- {streamft-0.3.0 → streamft-0.4.1}/tests/test_streamft_smoke.py +27 -0
- {streamft-0.3.0 → streamft-0.4.1}/LICENSE +0 -0
- {streamft-0.3.0 → streamft-0.4.1}/setup.cfg +0 -0
- {streamft-0.3.0 → streamft-0.4.1}/streamft/__init__.py +0 -0
- {streamft-0.3.0 → streamft-0.4.1}/streamft/autotune.py +0 -0
- {streamft-0.3.0 → streamft-0.4.1}/streamft/cli.py +0 -0
- {streamft-0.3.0 → streamft-0.4.1}/streamft/data.py +0 -0
- {streamft-0.3.0 → streamft-0.4.1}/streamft/disk_quant_store.py +0 -0
- {streamft-0.3.0 → streamft-0.4.1}/streamft/doctor.py +0 -0
- {streamft-0.3.0 → streamft-0.4.1}/streamft/easy.py +0 -0
- {streamft-0.3.0 → streamft-0.4.1}/streamft/fit.py +0 -0
- {streamft-0.3.0 → streamft-0.4.1}/streamft/lora.py +0 -0
- {streamft-0.3.0 → streamft-0.4.1}/streamft/metrics.py +0 -0
- {streamft-0.3.0 → streamft-0.4.1}/streamft/stream_quant_lora.py +0 -0
- {streamft-0.3.0 → streamft-0.4.1}/streamft/stream_train.py +0 -0
- {streamft-0.3.0 → streamft-0.4.1}/streamft.egg-info/dependency_links.txt +0 -0
- {streamft-0.3.0 → streamft-0.4.1}/streamft.egg-info/entry_points.txt +0 -0
- {streamft-0.3.0 → streamft-0.4.1}/streamft.egg-info/requires.txt +0 -0
- {streamft-0.3.0 → streamft-0.4.1}/streamft.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: streamft
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.1
|
|
4
4
|
Summary: Fine-tune models that do not fit: stream NF4 weights from disk, one transformer block at a time.
|
|
5
5
|
Author-email: Gokhan Cava <byjameson@users.noreply.github.com>
|
|
6
6
|
License: Apache-2.0
|
|
@@ -53,9 +53,11 @@ Measured on one machine — **RTX 3050 Laptop, 6 GB VRAM, 34 GB RAM, NVMe, Windo
|
|
|
53
53
|
|
|
54
54
|
The 32B weights are **19.2 GB**. They never fully live in VRAM. At 1024 tokens/step the
|
|
55
55
|
throughput is **3.1×** the 128-token rate, because the per-step I/O is amortised over
|
|
56
|
-
more tokens. 50 steps of Dolly-15k SFT moved eval loss **1.8755 → 0.
|
|
56
|
+
more tokens. 50 steps of Dolly-15k SFT moved eval loss **1.8755 → 0.9659**.
|
|
57
57
|
|
|
58
|
-
**Installed from PyPI? Start with [`QUICKSTART.md`](QUICKSTART.md)
|
|
58
|
+
**Installed from PyPI? Start with [`QUICKSTART.md`](QUICKSTART.md) (T?rk?e:
|
|
59
|
+
[`QUICKSTART.tr.md`](QUICKSTART.tr.md)) ? or run `install.ps1` / `install.sh`, which
|
|
60
|
+
checks Python, installs the package, looks for a GPU and runs `doctor`.** — it uses only the
|
|
59
61
|
published CLI (no repo scripts). Deeper dive: [`GUIDE_6GB_32B.md`](GUIDE_6GB_32B.md) — the practical write-up: presets,
|
|
60
62
|
the six traps, and which numbers are real. The measurement protocol, the discarded rows
|
|
61
63
|
and the correction of an earlier wrong claim ("8.6× / 100 tok/s") are in
|
|
@@ -86,6 +88,9 @@ streamft synth --model Qwen/Qwen2.5-1.5B-Instruct --n 200 --out data/synth.jsonl
|
|
|
86
88
|
|
|
87
89
|
# inference budget (weights + KV cache), not just training
|
|
88
90
|
streamft fit --model Qwen/Qwen2.5-7B-Instruct --gpu 6 --seq 2048 --mode inference
|
|
91
|
+
|
|
92
|
+
# does it actually work? 20 local prompts, no API and no judge model
|
|
93
|
+
streamft eval --model Qwen/Qwen2.5-1.5B-Instruct
|
|
89
94
|
```
|
|
90
95
|
|
|
91
96
|
Every run appends one JSON row to `results/results.jsonl` next to where you launched it.
|
|
@@ -149,6 +154,31 @@ streamft train --model-dir checkpoints/qwen7b_nf4 --sft --steps 1000 \
|
|
|
149
154
|
measured entry in `streamft fit`.
|
|
150
155
|
- `streamft fit` prints this hint automatically when it sees a pre-Ampere GPU.
|
|
151
156
|
|
|
157
|
+
### Tokenizer cost and the diffusion question
|
|
158
|
+
|
|
159
|
+
`seq` is in tokens, and tokenizers are not language-neutral. With the Qwen2.5 tokenizer,
|
|
160
|
+
the same passage costs **1.13 tokens/word in English, 1.74 in German and 2.39 in Turkish**
|
|
161
|
+
? a Turkish run at `seq 512` sees about half the text an English run does. Measured with
|
|
162
|
+
`scripts/tokenizer_cost.py`; the numbers, the byte-level alternative and why diffusion LMs
|
|
163
|
+
(LLaDA/Mercury: 800?1,500 tok/s at batch 1, but the same memory footprint) do not change
|
|
164
|
+
this project's bottleneck are in [`docs/TOKENIZERS.md`](docs/TOKENIZERS.md).
|
|
165
|
+
|
|
166
|
+
### Measured on
|
|
167
|
+
|
|
168
|
+
Every speed number here comes from **one machine**: RTX 3050 Laptop 6 GB, Windows 11,
|
|
169
|
+
AC power, NF4 base, LoRA r=8, batch 1, Dolly-15k SFT. The four rows (3B/7B/14B/32B) are
|
|
170
|
+
listed in [`docs/MEASURED.md`](docs/MEASURED.md), together with the rules for adding one
|
|
171
|
+
from a different card:
|
|
172
|
+
|
|
173
|
+
```bash
|
|
174
|
+
python scripts/measured_row.py --row "<json line>" --config checkpoints/<model>
|
|
175
|
+
```
|
|
176
|
+
|
|
177
|
+
It validates the run (finished steps, `grad_layers == n_layers`, peak VRAM recorded),
|
|
178
|
+
derives the architecture fingerprint and prints the snippets for `fit.py`, `fitcore.py`
|
|
179
|
+
and the Space. Anything not in that table prints **"not measured"** in `streamft fit` ?
|
|
180
|
+
that is the point.
|
|
181
|
+
|
|
152
182
|
## Verification
|
|
153
183
|
|
|
154
184
|
`python scripts/verify_gradients.py --model-dir checkpoints/qwen3b_bnb4bit --seq 128`
|
|
@@ -23,9 +23,11 @@ Measured on one machine — **RTX 3050 Laptop, 6 GB VRAM, 34 GB RAM, NVMe, Windo
|
|
|
23
23
|
|
|
24
24
|
The 32B weights are **19.2 GB**. They never fully live in VRAM. At 1024 tokens/step the
|
|
25
25
|
throughput is **3.1×** the 128-token rate, because the per-step I/O is amortised over
|
|
26
|
-
more tokens. 50 steps of Dolly-15k SFT moved eval loss **1.8755 → 0.
|
|
26
|
+
more tokens. 50 steps of Dolly-15k SFT moved eval loss **1.8755 → 0.9659**.
|
|
27
27
|
|
|
28
|
-
**Installed from PyPI? Start with [`QUICKSTART.md`](QUICKSTART.md)
|
|
28
|
+
**Installed from PyPI? Start with [`QUICKSTART.md`](QUICKSTART.md) (T?rk?e:
|
|
29
|
+
[`QUICKSTART.tr.md`](QUICKSTART.tr.md)) ? or run `install.ps1` / `install.sh`, which
|
|
30
|
+
checks Python, installs the package, looks for a GPU and runs `doctor`.** — it uses only the
|
|
29
31
|
published CLI (no repo scripts). Deeper dive: [`GUIDE_6GB_32B.md`](GUIDE_6GB_32B.md) — the practical write-up: presets,
|
|
30
32
|
the six traps, and which numbers are real. The measurement protocol, the discarded rows
|
|
31
33
|
and the correction of an earlier wrong claim ("8.6× / 100 tok/s") are in
|
|
@@ -56,6 +58,9 @@ streamft synth --model Qwen/Qwen2.5-1.5B-Instruct --n 200 --out data/synth.jsonl
|
|
|
56
58
|
|
|
57
59
|
# inference budget (weights + KV cache), not just training
|
|
58
60
|
streamft fit --model Qwen/Qwen2.5-7B-Instruct --gpu 6 --seq 2048 --mode inference
|
|
61
|
+
|
|
62
|
+
# does it actually work? 20 local prompts, no API and no judge model
|
|
63
|
+
streamft eval --model Qwen/Qwen2.5-1.5B-Instruct
|
|
59
64
|
```
|
|
60
65
|
|
|
61
66
|
Every run appends one JSON row to `results/results.jsonl` next to where you launched it.
|
|
@@ -119,6 +124,31 @@ streamft train --model-dir checkpoints/qwen7b_nf4 --sft --steps 1000 \
|
|
|
119
124
|
measured entry in `streamft fit`.
|
|
120
125
|
- `streamft fit` prints this hint automatically when it sees a pre-Ampere GPU.
|
|
121
126
|
|
|
127
|
+
### Tokenizer cost and the diffusion question
|
|
128
|
+
|
|
129
|
+
`seq` is in tokens, and tokenizers are not language-neutral. With the Qwen2.5 tokenizer,
|
|
130
|
+
the same passage costs **1.13 tokens/word in English, 1.74 in German and 2.39 in Turkish**
|
|
131
|
+
? a Turkish run at `seq 512` sees about half the text an English run does. Measured with
|
|
132
|
+
`scripts/tokenizer_cost.py`; the numbers, the byte-level alternative and why diffusion LMs
|
|
133
|
+
(LLaDA/Mercury: 800?1,500 tok/s at batch 1, but the same memory footprint) do not change
|
|
134
|
+
this project's bottleneck are in [`docs/TOKENIZERS.md`](docs/TOKENIZERS.md).
|
|
135
|
+
|
|
136
|
+
### Measured on
|
|
137
|
+
|
|
138
|
+
Every speed number here comes from **one machine**: RTX 3050 Laptop 6 GB, Windows 11,
|
|
139
|
+
AC power, NF4 base, LoRA r=8, batch 1, Dolly-15k SFT. The four rows (3B/7B/14B/32B) are
|
|
140
|
+
listed in [`docs/MEASURED.md`](docs/MEASURED.md), together with the rules for adding one
|
|
141
|
+
from a different card:
|
|
142
|
+
|
|
143
|
+
```bash
|
|
144
|
+
python scripts/measured_row.py --row "<json line>" --config checkpoints/<model>
|
|
145
|
+
```
|
|
146
|
+
|
|
147
|
+
It validates the run (finished steps, `grad_layers == n_layers`, peak VRAM recorded),
|
|
148
|
+
derives the architecture fingerprint and prints the snippets for `fit.py`, `fitcore.py`
|
|
149
|
+
and the Space. Anything not in that table prints **"not measured"** in `streamft fit` ?
|
|
150
|
+
that is the point.
|
|
151
|
+
|
|
122
152
|
## Verification
|
|
123
153
|
|
|
124
154
|
`python scripts/verify_gradients.py --model-dir checkpoints/qwen3b_bnb4bit --seq 128`
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "streamft"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.4.1"
|
|
8
8
|
description = "Fine-tune models that do not fit: stream NF4 weights from disk, one transformer block at a time."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
|
@@ -6,6 +6,8 @@
|
|
|
6
6
|
streamft data build the token shards (LM path only; SFT needs none)
|
|
7
7
|
streamft cpu SFT a small model on the CPU (no GPU; model must fit RAM)
|
|
8
8
|
streamft synth generate training pairs locally with a small model
|
|
9
|
+
streamft eval run a small local prompt set (repetition/refusal/keywords)
|
|
10
|
+
streamft merge fold a saved adapter into the base model (for serving / GGUF)
|
|
9
11
|
streamft doctor environment report (torch, CUDA, bitsandbytes, VRAM)
|
|
10
12
|
"""
|
|
11
13
|
from __future__ import annotations
|
|
@@ -20,6 +22,8 @@ USAGE = """streamft - fine-tune models that do not fit
|
|
|
20
22
|
streamft data --model <dir|hf-id> --out data
|
|
21
23
|
streamft cpu --model <small-model> --steps 20
|
|
22
24
|
streamft synth --model <small-model> --n 50 --out data/synth.jsonl
|
|
25
|
+
streamft eval --model <model> [--prompts mine.jsonl]
|
|
26
|
+
streamft merge --model <base> --adapter <dir> --out merged/<name>
|
|
23
27
|
streamft doctor
|
|
24
28
|
|
|
25
29
|
Docs: https://github.com/byjameson/stream-finetune
|
|
@@ -43,6 +47,14 @@ def main() -> int:
|
|
|
43
47
|
from streamft.easy import cpu_main
|
|
44
48
|
return cpu_main(rest)
|
|
45
49
|
|
|
50
|
+
if cmd == "merge":
|
|
51
|
+
from streamft.merge import main as merge_main
|
|
52
|
+
return merge_main(rest)
|
|
53
|
+
|
|
54
|
+
if cmd == "eval":
|
|
55
|
+
from streamft.evals import main as eval_main
|
|
56
|
+
return eval_main(rest)
|
|
57
|
+
|
|
46
58
|
if cmd == "synth":
|
|
47
59
|
from streamft.easy import synth_main
|
|
48
60
|
return synth_main(rest)
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
"""`streamft eval` — does the model actually do the thing? No API, no judge model.
|
|
2
|
+
|
|
3
|
+
It runs a small prompt set locally and reports what can be measured without a second
|
|
4
|
+
model: repetition, refusal-phrase hits, length, latency, and optionally keyword adherence
|
|
5
|
+
when the prompt file carries an `expect` field.
|
|
6
|
+
|
|
7
|
+
streamft eval --model Qwen/Qwen2.5-1.5B-Instruct
|
|
8
|
+
streamft eval --model ./merged --prompts my_prompts.jsonl --max-new 96
|
|
9
|
+
|
|
10
|
+
Prompt file format (JSONL, one object per line):
|
|
11
|
+
{"prompt": "...", "expect": ["optional", "keywords"]}
|
|
12
|
+
"""
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import argparse
|
|
16
|
+
import json
|
|
17
|
+
import re
|
|
18
|
+
import time
|
|
19
|
+
from collections import Counter
|
|
20
|
+
|
|
21
|
+
import torch
|
|
22
|
+
|
|
23
|
+
# Ten English + ten Turkish prompts. Deliberately mundane: this is a smoke test, not a
|
|
24
|
+
# benchmark. `expect` is a keyword that should appear if the model understood the task.
|
|
25
|
+
BUILTIN = [
|
|
26
|
+
{"prompt": "List three uses for a paperclip.", "expect": []},
|
|
27
|
+
{"prompt": "Explain in one sentence why the sky is blue.", "expect": ["light", "scatter"]},
|
|
28
|
+
{"prompt": "Give one reason to keep a food diary.", "expect": []},
|
|
29
|
+
{"prompt": "Write a two-line rhyme about rain.", "expect": []},
|
|
30
|
+
{"prompt": "What is 17 multiplied by 6?", "expect": ["102"]},
|
|
31
|
+
{"prompt": "Name two vegetables that grow underground.", "expect": ["carrot", "potato"]},
|
|
32
|
+
{"prompt": "Summarise the plot of a film about a lost dog in one sentence.", "expect": []},
|
|
33
|
+
{"prompt": "Translate 'good morning' into Turkish.", "expect": ["günaydın", "gunaydin"]},
|
|
34
|
+
{"prompt": "Give a one-sentence tip for better sleep.", "expect": []},
|
|
35
|
+
{"prompt": "What does a red traffic light mean?", "expect": ["stop"]},
|
|
36
|
+
{"prompt": "Bir cümlede su neden buharlaşır açıkla.", "expect": ["ısı", "isi", "sıcaklık"]},
|
|
37
|
+
{"prompt": "Üç tane meyve say.", "expect": ["elma", "muz", "portakal", "çilek"]},
|
|
38
|
+
{"prompt": "17 çarpı 6 kaç eder?", "expect": ["102"]},
|
|
39
|
+
{"prompt": "Yağmurlu bir gün için iki satırlık bir dörtlük yaz.", "expect": []},
|
|
40
|
+
{"prompt": "İyi bir uyku için tek cümlelik bir öneri ver.", "expect": []},
|
|
41
|
+
{"prompt": "Kırmızı trafik ışığı ne anlama gelir?", "expect": ["dur"]},
|
|
42
|
+
{"prompt": "Sabah kahvaltısı için iki öneri ver.", "expect": []},
|
|
43
|
+
{"prompt": "Bir cümlede neden kitap okumalıyız açıkla.", "expect": []},
|
|
44
|
+
{"prompt": "Türkiye'nin başkenti neresidir?", "expect": ["ankara"]},
|
|
45
|
+
{"prompt": "Bir cümlede bisikletin avantajını yaz.", "expect": []},
|
|
46
|
+
]
|
|
47
|
+
|
|
48
|
+
REFUSAL = [
|
|
49
|
+
"i can't help", "i cannot help", "i can't assist", "i cannot assist",
|
|
50
|
+
"as an ai", "i'm sorry, but", "i am sorry, but", "cannot provide",
|
|
51
|
+
"yardımcı olamam", "yardimci olamam", "üzgünüm", "uzgunum", "olarak yapay zeka",
|
|
52
|
+
]
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def repetition_rate(text: str, n: int = 4) -> float:
|
|
56
|
+
"""Fraction of repeated n-grams (1.0 = every n-gram is a duplicate)."""
|
|
57
|
+
words = re.findall(r"\w+", text.lower())
|
|
58
|
+
if len(words) < n + 1:
|
|
59
|
+
return 0.0
|
|
60
|
+
grams = [tuple(words[i:i + n]) for i in range(len(words) - n + 1)]
|
|
61
|
+
counts = Counter(grams)
|
|
62
|
+
repeated = sum(c - 1 for c in counts.values() if c > 1)
|
|
63
|
+
return repeated / len(grams)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def main(argv=None) -> int:
|
|
67
|
+
from transformers import AutoModelForCausalLM, AutoTokenizer
|
|
68
|
+
|
|
69
|
+
ap = argparse.ArgumentParser(prog="streamft eval")
|
|
70
|
+
ap.add_argument("--model", default="Qwen/Qwen2.5-1.5B-Instruct")
|
|
71
|
+
ap.add_argument("--prompts", default="", help="JSONL with {'prompt', 'expect'?}")
|
|
72
|
+
ap.add_argument("--max-new", type=int, default=64)
|
|
73
|
+
ap.add_argument("--limit", type=int, default=0, help="evaluate only the first N prompts")
|
|
74
|
+
ap.add_argument("--out", default="results/eval.jsonl")
|
|
75
|
+
ap.add_argument("--device", default="cuda" if torch.cuda.is_available() else "cpu")
|
|
76
|
+
ap.add_argument("--quiet", action="store_true")
|
|
77
|
+
args = ap.parse_args(argv)
|
|
78
|
+
|
|
79
|
+
prompts = BUILTIN
|
|
80
|
+
if args.prompts:
|
|
81
|
+
with open(args.prompts, encoding="utf-8") as f:
|
|
82
|
+
prompts = [json.loads(line) for line in f if line.strip()]
|
|
83
|
+
if args.limit:
|
|
84
|
+
prompts = prompts[: args.limit]
|
|
85
|
+
|
|
86
|
+
dtype = torch.bfloat16 if args.device == "cuda" else torch.float32
|
|
87
|
+
tok = AutoTokenizer.from_pretrained(args.model)
|
|
88
|
+
model = AutoModelForCausalLM.from_pretrained(args.model, torch_dtype=dtype).to(args.device)
|
|
89
|
+
model.eval()
|
|
90
|
+
|
|
91
|
+
rows = []
|
|
92
|
+
t_all = time.time()
|
|
93
|
+
for i, item in enumerate(prompts):
|
|
94
|
+
text = item["prompt"]
|
|
95
|
+
if getattr(tok, "chat_template", None):
|
|
96
|
+
text = tok.apply_chat_template([{"role": "user", "content": text}],
|
|
97
|
+
tokenize=False, add_generation_prompt=True)
|
|
98
|
+
ids = tok(text, return_tensors="pt").to(args.device)
|
|
99
|
+
t0 = time.time()
|
|
100
|
+
with torch.no_grad():
|
|
101
|
+
out = model.generate(**ids, max_new_tokens=args.max_new, do_sample=False,
|
|
102
|
+
pad_token_id=tok.eos_token_id)
|
|
103
|
+
dt = time.time() - t0
|
|
104
|
+
gen = out[0][ids["input_ids"].shape[1]:]
|
|
105
|
+
answer = tok.decode(gen, skip_special_tokens=True).strip()
|
|
106
|
+
low = answer.lower()
|
|
107
|
+
row = {
|
|
108
|
+
"i": i, "prompt": item["prompt"], "answer": answer,
|
|
109
|
+
"tokens": int(gen.shape[0]), "seconds": round(dt, 2),
|
|
110
|
+
"tok_s": round(int(gen.shape[0]) / max(dt, 1e-6), 1),
|
|
111
|
+
"rep4": round(repetition_rate(answer), 3),
|
|
112
|
+
"refusal": any(p in low for p in REFUSAL),
|
|
113
|
+
"expect_hit": (None if not item.get("expect")
|
|
114
|
+
else any(k.lower() in low for k in item["expect"])),
|
|
115
|
+
}
|
|
116
|
+
rows.append(row)
|
|
117
|
+
if not args.quiet:
|
|
118
|
+
flag = "REFUSAL" if row["refusal"] else ("miss" if row["expect_hit"] is False else "")
|
|
119
|
+
print(f"[{i+1}/{len(prompts)}] {row['tok_s']:.1f} tok/s rep4={row['rep4']:.2f} {flag}")
|
|
120
|
+
print(" " + answer.replace("\n", " ")[:160])
|
|
121
|
+
|
|
122
|
+
n = len(rows)
|
|
123
|
+
expected = [r for r in rows if r["expect_hit"] is not None]
|
|
124
|
+
summary = {
|
|
125
|
+
"model": args.model, "n": n,
|
|
126
|
+
"refusal_rate": round(sum(r["refusal"] for r in rows) / max(n, 1), 3),
|
|
127
|
+
"mean_rep4": round(sum(r["rep4"] for r in rows) / max(n, 1), 3),
|
|
128
|
+
"mean_tokens": round(sum(r["tokens"] for r in rows) / max(n, 1), 1),
|
|
129
|
+
"mean_tok_s": round(sum(r["tok_s"] for r in rows) / max(n, 1), 1),
|
|
130
|
+
"keyword_hit_rate": (round(sum(1 for r in expected if r["expect_hit"]) / len(expected), 3)
|
|
131
|
+
if expected else None),
|
|
132
|
+
"wall_s": round(time.time() - t_all, 1),
|
|
133
|
+
"device": args.device,
|
|
134
|
+
}
|
|
135
|
+
print(json.dumps(summary, indent=1))
|
|
136
|
+
|
|
137
|
+
import os
|
|
138
|
+
os.makedirs(os.path.dirname(args.out) or ".", exist_ok=True)
|
|
139
|
+
with open(args.out, "w", encoding="utf-8") as f:
|
|
140
|
+
for r in rows:
|
|
141
|
+
f.write(json.dumps(r, ensure_ascii=False) + "\n")
|
|
142
|
+
with open(args.out.replace(".jsonl", "_summary.json"), "w", encoding="utf-8") as f:
|
|
143
|
+
json.dump(summary, f, indent=1)
|
|
144
|
+
return 0
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
if __name__ == "__main__":
|
|
148
|
+
raise SystemExit(main())
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
"""`streamft merge` — fold a saved LoRA adapter into the base model (CPU is fine).
|
|
2
|
+
|
|
3
|
+
The merged directory is a plain HF checkpoint, which is what GGUF conversion expects.
|
|
4
|
+
|
|
5
|
+
streamft merge --model Qwen/Qwen2.5-0.5B --adapter checkpoints/cpu_cookbook --out merged/q05
|
|
6
|
+
|
|
7
|
+
The adapter was saved by `streamft train --save` / `streamft cpu --save` as
|
|
8
|
+
`adapter.pt` (`lora_state_dict`). The rank is read from the tensor shapes, so no extra
|
|
9
|
+
config file is needed.
|
|
10
|
+
"""
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import argparse
|
|
14
|
+
import json
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
|
|
17
|
+
import torch
|
|
18
|
+
|
|
19
|
+
from .lora import LoRALinear, apply_lora, merge_lora, set_lora_state_dict
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def main(argv=None) -> int:
|
|
23
|
+
from transformers import AutoModelForCausalLM, AutoTokenizer
|
|
24
|
+
|
|
25
|
+
ap = argparse.ArgumentParser(prog="streamft merge")
|
|
26
|
+
ap.add_argument("--model", required=True, help="base model (HF id or local dir)")
|
|
27
|
+
ap.add_argument("--adapter", required=True, help="dir with adapter.pt")
|
|
28
|
+
ap.add_argument("--out", required=True, help="output directory for the merged model")
|
|
29
|
+
ap.add_argument("--dtype", choices=["fp32", "fp16", "bf16"], default="fp16",
|
|
30
|
+
help="storage dtype of the merged weights (fp16 halves the file)")
|
|
31
|
+
args = ap.parse_args(argv)
|
|
32
|
+
|
|
33
|
+
adapter_path = Path(args.adapter)
|
|
34
|
+
if adapter_path.is_dir():
|
|
35
|
+
adapter_path = adapter_path / "adapter.pt"
|
|
36
|
+
if not adapter_path.exists():
|
|
37
|
+
raise SystemExit(f"adapter not found: {adapter_path}")
|
|
38
|
+
|
|
39
|
+
state = torch.load(str(adapter_path), map_location="cpu", weights_only=True)
|
|
40
|
+
if not state:
|
|
41
|
+
raise SystemExit("adapter.pt is empty")
|
|
42
|
+
a_key = next((k for k in state if k.endswith(".A")), None)
|
|
43
|
+
if a_key is None:
|
|
44
|
+
raise SystemExit("adapter.pt has no LoRA A tensors; was it saved with lora_state_dict?")
|
|
45
|
+
r = int(state[a_key].shape[0])
|
|
46
|
+
|
|
47
|
+
dtype = {"fp32": torch.float32, "fp16": torch.float16, "bf16": torch.bfloat16}[args.dtype]
|
|
48
|
+
print(f"[merge] {args.model} + {adapter_path.name} (r={r}) -> {args.out} [{args.dtype}]",
|
|
49
|
+
flush=True)
|
|
50
|
+
model = AutoModelForCausalLM.from_pretrained(args.model, torch_dtype=dtype)
|
|
51
|
+
apply_lora(model, r_attn=r, r_mlp=0)
|
|
52
|
+
set_lora_state_dict(model, state)
|
|
53
|
+
merge_lora(model)
|
|
54
|
+
|
|
55
|
+
# After a real merge every LoRA factor is exactly zero; the base absorbed the delta.
|
|
56
|
+
not_merged = [n for n, p in model.named_parameters()
|
|
57
|
+
if (n.endswith(".A") or n.endswith(".B")) and float(p.detach().abs().sum()) > 0]
|
|
58
|
+
if not_merged:
|
|
59
|
+
raise SystemExit(f"adapter tensors left unmerged: {not_merged[:3]}")
|
|
60
|
+
for p in model.parameters():
|
|
61
|
+
p.requires_grad_(False)
|
|
62
|
+
|
|
63
|
+
# Unwrap: after merge the LoRA factors are zero, but `save_pretrained` would still
|
|
64
|
+
# write them, and GGUF converters refuse unknown tensors (`...k_proj.A`).
|
|
65
|
+
stripped = 0
|
|
66
|
+
for name, mod in list(model.named_modules()):
|
|
67
|
+
if isinstance(mod, LoRALinear):
|
|
68
|
+
parent_name, _, attr = name.rpartition(".")
|
|
69
|
+
parent = model.get_submodule(parent_name) if parent_name else model
|
|
70
|
+
setattr(parent, attr, mod.base)
|
|
71
|
+
stripped += 1
|
|
72
|
+
print(f"[merge] removed {stripped} LoRA wrappers", flush=True)
|
|
73
|
+
|
|
74
|
+
out = Path(args.out)
|
|
75
|
+
out.mkdir(parents=True, exist_ok=True)
|
|
76
|
+
model.save_pretrained(str(out), safe_serialization=True)
|
|
77
|
+
tok = AutoTokenizer.from_pretrained(args.model)
|
|
78
|
+
tok.save_pretrained(str(out))
|
|
79
|
+
|
|
80
|
+
sizes = {f.name: round(f.stat().st_size / 1e9, 2) for f in out.glob("*.safetensors")}
|
|
81
|
+
print(json.dumps({"merged": str(out), "dtype": args.dtype, "rank": r,
|
|
82
|
+
"weights_gb": sizes, "license": "same as the base model"}, indent=1))
|
|
83
|
+
print("\nnext (needs llama.cpp, several minutes of CPU):")
|
|
84
|
+
print(f" python llama.cpp/convert_hf_to_gguf.py {out} --outfile model-f16.gguf")
|
|
85
|
+
print(" llama-quantize model-f16.gguf model-q4_k_m.gguf Q4_K_M")
|
|
86
|
+
print(" open wasm/index.html and pick model-q4_k_m.gguf")
|
|
87
|
+
return 0
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: streamft
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.1
|
|
4
4
|
Summary: Fine-tune models that do not fit: stream NF4 weights from disk, one transformer block at a time.
|
|
5
5
|
Author-email: Gokhan Cava <byjameson@users.noreply.github.com>
|
|
6
6
|
License: Apache-2.0
|
|
@@ -53,9 +53,11 @@ Measured on one machine — **RTX 3050 Laptop, 6 GB VRAM, 34 GB RAM, NVMe, Windo
|
|
|
53
53
|
|
|
54
54
|
The 32B weights are **19.2 GB**. They never fully live in VRAM. At 1024 tokens/step the
|
|
55
55
|
throughput is **3.1×** the 128-token rate, because the per-step I/O is amortised over
|
|
56
|
-
more tokens. 50 steps of Dolly-15k SFT moved eval loss **1.8755 → 0.
|
|
56
|
+
more tokens. 50 steps of Dolly-15k SFT moved eval loss **1.8755 → 0.9659**.
|
|
57
57
|
|
|
58
|
-
**Installed from PyPI? Start with [`QUICKSTART.md`](QUICKSTART.md)
|
|
58
|
+
**Installed from PyPI? Start with [`QUICKSTART.md`](QUICKSTART.md) (T?rk?e:
|
|
59
|
+
[`QUICKSTART.tr.md`](QUICKSTART.tr.md)) ? or run `install.ps1` / `install.sh`, which
|
|
60
|
+
checks Python, installs the package, looks for a GPU and runs `doctor`.** — it uses only the
|
|
59
61
|
published CLI (no repo scripts). Deeper dive: [`GUIDE_6GB_32B.md`](GUIDE_6GB_32B.md) — the practical write-up: presets,
|
|
60
62
|
the six traps, and which numbers are real. The measurement protocol, the discarded rows
|
|
61
63
|
and the correction of an earlier wrong claim ("8.6× / 100 tok/s") are in
|
|
@@ -86,6 +88,9 @@ streamft synth --model Qwen/Qwen2.5-1.5B-Instruct --n 200 --out data/synth.jsonl
|
|
|
86
88
|
|
|
87
89
|
# inference budget (weights + KV cache), not just training
|
|
88
90
|
streamft fit --model Qwen/Qwen2.5-7B-Instruct --gpu 6 --seq 2048 --mode inference
|
|
91
|
+
|
|
92
|
+
# does it actually work? 20 local prompts, no API and no judge model
|
|
93
|
+
streamft eval --model Qwen/Qwen2.5-1.5B-Instruct
|
|
89
94
|
```
|
|
90
95
|
|
|
91
96
|
Every run appends one JSON row to `results/results.jsonl` next to where you launched it.
|
|
@@ -149,6 +154,31 @@ streamft train --model-dir checkpoints/qwen7b_nf4 --sft --steps 1000 \
|
|
|
149
154
|
measured entry in `streamft fit`.
|
|
150
155
|
- `streamft fit` prints this hint automatically when it sees a pre-Ampere GPU.
|
|
151
156
|
|
|
157
|
+
### Tokenizer cost and the diffusion question
|
|
158
|
+
|
|
159
|
+
`seq` is in tokens, and tokenizers are not language-neutral. With the Qwen2.5 tokenizer,
|
|
160
|
+
the same passage costs **1.13 tokens/word in English, 1.74 in German and 2.39 in Turkish**
|
|
161
|
+
? a Turkish run at `seq 512` sees about half the text an English run does. Measured with
|
|
162
|
+
`scripts/tokenizer_cost.py`; the numbers, the byte-level alternative and why diffusion LMs
|
|
163
|
+
(LLaDA/Mercury: 800?1,500 tok/s at batch 1, but the same memory footprint) do not change
|
|
164
|
+
this project's bottleneck are in [`docs/TOKENIZERS.md`](docs/TOKENIZERS.md).
|
|
165
|
+
|
|
166
|
+
### Measured on
|
|
167
|
+
|
|
168
|
+
Every speed number here comes from **one machine**: RTX 3050 Laptop 6 GB, Windows 11,
|
|
169
|
+
AC power, NF4 base, LoRA r=8, batch 1, Dolly-15k SFT. The four rows (3B/7B/14B/32B) are
|
|
170
|
+
listed in [`docs/MEASURED.md`](docs/MEASURED.md), together with the rules for adding one
|
|
171
|
+
from a different card:
|
|
172
|
+
|
|
173
|
+
```bash
|
|
174
|
+
python scripts/measured_row.py --row "<json line>" --config checkpoints/<model>
|
|
175
|
+
```
|
|
176
|
+
|
|
177
|
+
It validates the run (finished steps, `grad_layers == n_layers`, peak VRAM recorded),
|
|
178
|
+
derives the architecture fingerprint and prints the snippets for `fit.py`, `fitcore.py`
|
|
179
|
+
and the Space. Anything not in that table prints **"not measured"** in `streamft fit` ?
|
|
180
|
+
that is the point.
|
|
181
|
+
|
|
152
182
|
## Verification
|
|
153
183
|
|
|
154
184
|
`python scripts/verify_gradients.py --model-dir checkpoints/qwen3b_bnb4bit --seq 128`
|
|
@@ -271,3 +271,30 @@ def test_kv_cache_matches_hand_calculation():
|
|
|
271
271
|
# 2 * 24 layers * 2 kv heads * 128 head_dim * 1024 seq * 2 bytes
|
|
272
272
|
expect = 2 * 24 * 2 * 128 * 1024 * 2 / 1e9
|
|
273
273
|
assert abs(kv_cache_gb(cfg, 1024) - expect) < 1e-12
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def test_repetition_rate_counts_duplicate_ngrams():
|
|
277
|
+
from streamft.evals import repetition_rate
|
|
278
|
+
|
|
279
|
+
assert repetition_rate("a b c d e f g h") == 0.0
|
|
280
|
+
assert repetition_rate("a b c d a b c d") == 0.2 # 1 duplicate of 5 grams
|
|
281
|
+
assert repetition_rate("") == 0.0
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
def test_measured_row_validation_rejects_incomplete_runs():
|
|
285
|
+
import importlib.util
|
|
286
|
+
import sys
|
|
287
|
+
from pathlib import Path
|
|
288
|
+
|
|
289
|
+
path = Path(__file__).resolve().parents[1] / "scripts" / "measured_row.py"
|
|
290
|
+
spec = importlib.util.spec_from_file_location("measured_row", path)
|
|
291
|
+
mod = importlib.util.module_from_spec(spec)
|
|
292
|
+
sys.modules["measured_row"] = mod
|
|
293
|
+
spec.loader.exec_module(mod)
|
|
294
|
+
|
|
295
|
+
good = {"steps": 30, "grad_layers": 28, "n_layers": 28,
|
|
296
|
+
"peak_gb_max": 5.5, "step_s_med": 1.5}
|
|
297
|
+
assert mod.validate(good) == []
|
|
298
|
+
assert mod.validate({**good, "steps": 0})
|
|
299
|
+
assert mod.validate({**good, "grad_layers": 27})
|
|
300
|
+
assert mod.validate({**good, "peak_gb_max": None})
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|