streamft 0.4.0__tar.gz → 0.4.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {streamft-0.4.0 → streamft-0.4.1}/PKG-INFO +26 -1
- {streamft-0.4.0 → streamft-0.4.1}/README.md +25 -0
- {streamft-0.4.0 → streamft-0.4.1}/pyproject.toml +1 -1
- {streamft-0.4.0 → streamft-0.4.1}/streamft/__main__.py +6 -0
- streamft-0.4.1/streamft/merge.py +87 -0
- {streamft-0.4.0 → streamft-0.4.1}/streamft.egg-info/PKG-INFO +26 -1
- {streamft-0.4.0 → streamft-0.4.1}/streamft.egg-info/SOURCES.txt +1 -0
- {streamft-0.4.0 → streamft-0.4.1}/tests/test_streamft_smoke.py +19 -0
- {streamft-0.4.0 → streamft-0.4.1}/LICENSE +0 -0
- {streamft-0.4.0 → streamft-0.4.1}/setup.cfg +0 -0
- {streamft-0.4.0 → streamft-0.4.1}/streamft/__init__.py +0 -0
- {streamft-0.4.0 → streamft-0.4.1}/streamft/autotune.py +0 -0
- {streamft-0.4.0 → streamft-0.4.1}/streamft/cli.py +0 -0
- {streamft-0.4.0 → streamft-0.4.1}/streamft/data.py +0 -0
- {streamft-0.4.0 → streamft-0.4.1}/streamft/disk_quant_store.py +0 -0
- {streamft-0.4.0 → streamft-0.4.1}/streamft/doctor.py +0 -0
- {streamft-0.4.0 → streamft-0.4.1}/streamft/easy.py +0 -0
- {streamft-0.4.0 → streamft-0.4.1}/streamft/evals.py +0 -0
- {streamft-0.4.0 → streamft-0.4.1}/streamft/fit.py +0 -0
- {streamft-0.4.0 → streamft-0.4.1}/streamft/lora.py +0 -0
- {streamft-0.4.0 → streamft-0.4.1}/streamft/metrics.py +0 -0
- {streamft-0.4.0 → streamft-0.4.1}/streamft/stream_quant_lora.py +0 -0
- {streamft-0.4.0 → streamft-0.4.1}/streamft/stream_train.py +0 -0
- {streamft-0.4.0 → streamft-0.4.1}/streamft.egg-info/dependency_links.txt +0 -0
- {streamft-0.4.0 → streamft-0.4.1}/streamft.egg-info/entry_points.txt +0 -0
- {streamft-0.4.0 → streamft-0.4.1}/streamft.egg-info/requires.txt +0 -0
- {streamft-0.4.0 → streamft-0.4.1}/streamft.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: streamft
|
|
3
|
-
Version: 0.4.
|
|
3
|
+
Version: 0.4.1
|
|
4
4
|
Summary: Fine-tune models that do not fit: stream NF4 weights from disk, one transformer block at a time.
|
|
5
5
|
Author-email: Gokhan Cava <byjameson@users.noreply.github.com>
|
|
6
6
|
License: Apache-2.0
|
|
@@ -154,6 +154,31 @@ streamft train --model-dir checkpoints/qwen7b_nf4 --sft --steps 1000 \
|
|
|
154
154
|
measured entry in `streamft fit`.
|
|
155
155
|
- `streamft fit` prints this hint automatically when it sees a pre-Ampere GPU.
|
|
156
156
|
|
|
157
|
+
### Tokenizer cost and the diffusion question
|
|
158
|
+
|
|
159
|
+
`seq` is in tokens, and tokenizers are not language-neutral. With the Qwen2.5 tokenizer,
|
|
160
|
+
the same passage costs **1.13 tokens/word in English, 1.74 in German and 2.39 in Turkish**
|
|
161
|
+
? a Turkish run at `seq 512` sees about half the text an English run does. Measured with
|
|
162
|
+
`scripts/tokenizer_cost.py`; the numbers, the byte-level alternative and why diffusion LMs
|
|
163
|
+
(LLaDA/Mercury: 800?1,500 tok/s at batch 1, but the same memory footprint) do not change
|
|
164
|
+
this project's bottleneck are in [`docs/TOKENIZERS.md`](docs/TOKENIZERS.md).
|
|
165
|
+
|
|
166
|
+
### Measured on
|
|
167
|
+
|
|
168
|
+
Every speed number here comes from **one machine**: RTX 3050 Laptop 6 GB, Windows 11,
|
|
169
|
+
AC power, NF4 base, LoRA r=8, batch 1, Dolly-15k SFT. The four rows (3B/7B/14B/32B) are
|
|
170
|
+
listed in [`docs/MEASURED.md`](docs/MEASURED.md), together with the rules for adding one
|
|
171
|
+
from a different card:
|
|
172
|
+
|
|
173
|
+
```bash
|
|
174
|
+
python scripts/measured_row.py --row "<json line>" --config checkpoints/<model>
|
|
175
|
+
```
|
|
176
|
+
|
|
177
|
+
It validates the run (finished steps, `grad_layers == n_layers`, peak VRAM recorded),
|
|
178
|
+
derives the architecture fingerprint and prints the snippets for `fit.py`, `fitcore.py`
|
|
179
|
+
and the Space. Anything not in that table prints **"not measured"** in `streamft fit` ?
|
|
180
|
+
that is the point.
|
|
181
|
+
|
|
157
182
|
## Verification
|
|
158
183
|
|
|
159
184
|
`python scripts/verify_gradients.py --model-dir checkpoints/qwen3b_bnb4bit --seq 128`
|
|
@@ -124,6 +124,31 @@ streamft train --model-dir checkpoints/qwen7b_nf4 --sft --steps 1000 \
|
|
|
124
124
|
measured entry in `streamft fit`.
|
|
125
125
|
- `streamft fit` prints this hint automatically when it sees a pre-Ampere GPU.
|
|
126
126
|
|
|
127
|
+
### Tokenizer cost and the diffusion question
|
|
128
|
+
|
|
129
|
+
`seq` is in tokens, and tokenizers are not language-neutral. With the Qwen2.5 tokenizer,
|
|
130
|
+
the same passage costs **1.13 tokens/word in English, 1.74 in German and 2.39 in Turkish**
|
|
131
|
+
? a Turkish run at `seq 512` sees about half the text an English run does. Measured with
|
|
132
|
+
`scripts/tokenizer_cost.py`; the numbers, the byte-level alternative and why diffusion LMs
|
|
133
|
+
(LLaDA/Mercury: 800?1,500 tok/s at batch 1, but the same memory footprint) do not change
|
|
134
|
+
this project's bottleneck are in [`docs/TOKENIZERS.md`](docs/TOKENIZERS.md).
|
|
135
|
+
|
|
136
|
+
### Measured on
|
|
137
|
+
|
|
138
|
+
Every speed number here comes from **one machine**: RTX 3050 Laptop 6 GB, Windows 11,
|
|
139
|
+
AC power, NF4 base, LoRA r=8, batch 1, Dolly-15k SFT. The four rows (3B/7B/14B/32B) are
|
|
140
|
+
listed in [`docs/MEASURED.md`](docs/MEASURED.md), together with the rules for adding one
|
|
141
|
+
from a different card:
|
|
142
|
+
|
|
143
|
+
```bash
|
|
144
|
+
python scripts/measured_row.py --row "<json line>" --config checkpoints/<model>
|
|
145
|
+
```
|
|
146
|
+
|
|
147
|
+
It validates the run (finished steps, `grad_layers == n_layers`, peak VRAM recorded),
|
|
148
|
+
derives the architecture fingerprint and prints the snippets for `fit.py`, `fitcore.py`
|
|
149
|
+
and the Space. Anything not in that table prints **"not measured"** in `streamft fit` ?
|
|
150
|
+
that is the point.
|
|
151
|
+
|
|
127
152
|
## Verification
|
|
128
153
|
|
|
129
154
|
`python scripts/verify_gradients.py --model-dir checkpoints/qwen3b_bnb4bit --seq 128`
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "streamft"
|
|
7
|
-
version = "0.4.
|
|
7
|
+
version = "0.4.1"
|
|
8
8
|
description = "Fine-tune models that do not fit: stream NF4 weights from disk, one transformer block at a time."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
streamft cpu SFT a small model on the CPU (no GPU; model must fit RAM)
|
|
8
8
|
streamft synth generate training pairs locally with a small model
|
|
9
9
|
streamft eval run a small local prompt set (repetition/refusal/keywords)
|
|
10
|
+
streamft merge fold a saved adapter into the base model (for serving / GGUF)
|
|
10
11
|
streamft doctor environment report (torch, CUDA, bitsandbytes, VRAM)
|
|
11
12
|
"""
|
|
12
13
|
from __future__ import annotations
|
|
@@ -22,6 +23,7 @@ USAGE = """streamft - fine-tune models that do not fit
|
|
|
22
23
|
streamft cpu --model <small-model> --steps 20
|
|
23
24
|
streamft synth --model <small-model> --n 50 --out data/synth.jsonl
|
|
24
25
|
streamft eval --model <model> [--prompts mine.jsonl]
|
|
26
|
+
streamft merge --model <base> --adapter <dir> --out merged/<name>
|
|
25
27
|
streamft doctor
|
|
26
28
|
|
|
27
29
|
Docs: https://github.com/byjameson/stream-finetune
|
|
@@ -45,6 +47,10 @@ def main() -> int:
|
|
|
45
47
|
from streamft.easy import cpu_main
|
|
46
48
|
return cpu_main(rest)
|
|
47
49
|
|
|
50
|
+
if cmd == "merge":
|
|
51
|
+
from streamft.merge import main as merge_main
|
|
52
|
+
return merge_main(rest)
|
|
53
|
+
|
|
48
54
|
if cmd == "eval":
|
|
49
55
|
from streamft.evals import main as eval_main
|
|
50
56
|
return eval_main(rest)
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
"""`streamft merge` — fold a saved LoRA adapter into the base model (CPU is fine).
|
|
2
|
+
|
|
3
|
+
The merged directory is a plain HF checkpoint, which is what GGUF conversion expects.
|
|
4
|
+
|
|
5
|
+
streamft merge --model Qwen/Qwen2.5-0.5B --adapter checkpoints/cpu_cookbook --out merged/q05
|
|
6
|
+
|
|
7
|
+
The adapter was saved by `streamft train --save` / `streamft cpu --save` as
|
|
8
|
+
`adapter.pt` (`lora_state_dict`). The rank is read from the tensor shapes, so no extra
|
|
9
|
+
config file is needed.
|
|
10
|
+
"""
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import argparse
|
|
14
|
+
import json
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
|
|
17
|
+
import torch
|
|
18
|
+
|
|
19
|
+
from .lora import LoRALinear, apply_lora, merge_lora, set_lora_state_dict
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def main(argv=None) -> int:
|
|
23
|
+
from transformers import AutoModelForCausalLM, AutoTokenizer
|
|
24
|
+
|
|
25
|
+
ap = argparse.ArgumentParser(prog="streamft merge")
|
|
26
|
+
ap.add_argument("--model", required=True, help="base model (HF id or local dir)")
|
|
27
|
+
ap.add_argument("--adapter", required=True, help="dir with adapter.pt")
|
|
28
|
+
ap.add_argument("--out", required=True, help="output directory for the merged model")
|
|
29
|
+
ap.add_argument("--dtype", choices=["fp32", "fp16", "bf16"], default="fp16",
|
|
30
|
+
help="storage dtype of the merged weights (fp16 halves the file)")
|
|
31
|
+
args = ap.parse_args(argv)
|
|
32
|
+
|
|
33
|
+
adapter_path = Path(args.adapter)
|
|
34
|
+
if adapter_path.is_dir():
|
|
35
|
+
adapter_path = adapter_path / "adapter.pt"
|
|
36
|
+
if not adapter_path.exists():
|
|
37
|
+
raise SystemExit(f"adapter not found: {adapter_path}")
|
|
38
|
+
|
|
39
|
+
state = torch.load(str(adapter_path), map_location="cpu", weights_only=True)
|
|
40
|
+
if not state:
|
|
41
|
+
raise SystemExit("adapter.pt is empty")
|
|
42
|
+
a_key = next((k for k in state if k.endswith(".A")), None)
|
|
43
|
+
if a_key is None:
|
|
44
|
+
raise SystemExit("adapter.pt has no LoRA A tensors; was it saved with lora_state_dict?")
|
|
45
|
+
r = int(state[a_key].shape[0])
|
|
46
|
+
|
|
47
|
+
dtype = {"fp32": torch.float32, "fp16": torch.float16, "bf16": torch.bfloat16}[args.dtype]
|
|
48
|
+
print(f"[merge] {args.model} + {adapter_path.name} (r={r}) -> {args.out} [{args.dtype}]",
|
|
49
|
+
flush=True)
|
|
50
|
+
model = AutoModelForCausalLM.from_pretrained(args.model, torch_dtype=dtype)
|
|
51
|
+
apply_lora(model, r_attn=r, r_mlp=0)
|
|
52
|
+
set_lora_state_dict(model, state)
|
|
53
|
+
merge_lora(model)
|
|
54
|
+
|
|
55
|
+
# After a real merge every LoRA factor is exactly zero; the base absorbed the delta.
|
|
56
|
+
not_merged = [n for n, p in model.named_parameters()
|
|
57
|
+
if (n.endswith(".A") or n.endswith(".B")) and float(p.detach().abs().sum()) > 0]
|
|
58
|
+
if not_merged:
|
|
59
|
+
raise SystemExit(f"adapter tensors left unmerged: {not_merged[:3]}")
|
|
60
|
+
for p in model.parameters():
|
|
61
|
+
p.requires_grad_(False)
|
|
62
|
+
|
|
63
|
+
# Unwrap: after merge the LoRA factors are zero, but `save_pretrained` would still
|
|
64
|
+
# write them, and GGUF converters refuse unknown tensors (`...k_proj.A`).
|
|
65
|
+
stripped = 0
|
|
66
|
+
for name, mod in list(model.named_modules()):
|
|
67
|
+
if isinstance(mod, LoRALinear):
|
|
68
|
+
parent_name, _, attr = name.rpartition(".")
|
|
69
|
+
parent = model.get_submodule(parent_name) if parent_name else model
|
|
70
|
+
setattr(parent, attr, mod.base)
|
|
71
|
+
stripped += 1
|
|
72
|
+
print(f"[merge] removed {stripped} LoRA wrappers", flush=True)
|
|
73
|
+
|
|
74
|
+
out = Path(args.out)
|
|
75
|
+
out.mkdir(parents=True, exist_ok=True)
|
|
76
|
+
model.save_pretrained(str(out), safe_serialization=True)
|
|
77
|
+
tok = AutoTokenizer.from_pretrained(args.model)
|
|
78
|
+
tok.save_pretrained(str(out))
|
|
79
|
+
|
|
80
|
+
sizes = {f.name: round(f.stat().st_size / 1e9, 2) for f in out.glob("*.safetensors")}
|
|
81
|
+
print(json.dumps({"merged": str(out), "dtype": args.dtype, "rank": r,
|
|
82
|
+
"weights_gb": sizes, "license": "same as the base model"}, indent=1))
|
|
83
|
+
print("\nnext (needs llama.cpp, several minutes of CPU):")
|
|
84
|
+
print(f" python llama.cpp/convert_hf_to_gguf.py {out} --outfile model-f16.gguf")
|
|
85
|
+
print(" llama-quantize model-f16.gguf model-q4_k_m.gguf Q4_K_M")
|
|
86
|
+
print(" open wasm/index.html and pick model-q4_k_m.gguf")
|
|
87
|
+
return 0
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: streamft
|
|
3
|
-
Version: 0.4.
|
|
3
|
+
Version: 0.4.1
|
|
4
4
|
Summary: Fine-tune models that do not fit: stream NF4 weights from disk, one transformer block at a time.
|
|
5
5
|
Author-email: Gokhan Cava <byjameson@users.noreply.github.com>
|
|
6
6
|
License: Apache-2.0
|
|
@@ -154,6 +154,31 @@ streamft train --model-dir checkpoints/qwen7b_nf4 --sft --steps 1000 \
|
|
|
154
154
|
measured entry in `streamft fit`.
|
|
155
155
|
- `streamft fit` prints this hint automatically when it sees a pre-Ampere GPU.
|
|
156
156
|
|
|
157
|
+
### Tokenizer cost and the diffusion question
|
|
158
|
+
|
|
159
|
+
`seq` is in tokens, and tokenizers are not language-neutral. With the Qwen2.5 tokenizer,
|
|
160
|
+
the same passage costs **1.13 tokens/word in English, 1.74 in German and 2.39 in Turkish**
|
|
161
|
+
? a Turkish run at `seq 512` sees about half the text an English run does. Measured with
|
|
162
|
+
`scripts/tokenizer_cost.py`; the numbers, the byte-level alternative and why diffusion LMs
|
|
163
|
+
(LLaDA/Mercury: 800?1,500 tok/s at batch 1, but the same memory footprint) do not change
|
|
164
|
+
this project's bottleneck are in [`docs/TOKENIZERS.md`](docs/TOKENIZERS.md).
|
|
165
|
+
|
|
166
|
+
### Measured on
|
|
167
|
+
|
|
168
|
+
Every speed number here comes from **one machine**: RTX 3050 Laptop 6 GB, Windows 11,
|
|
169
|
+
AC power, NF4 base, LoRA r=8, batch 1, Dolly-15k SFT. The four rows (3B/7B/14B/32B) are
|
|
170
|
+
listed in [`docs/MEASURED.md`](docs/MEASURED.md), together with the rules for adding one
|
|
171
|
+
from a different card:
|
|
172
|
+
|
|
173
|
+
```bash
|
|
174
|
+
python scripts/measured_row.py --row "<json line>" --config checkpoints/<model>
|
|
175
|
+
```
|
|
176
|
+
|
|
177
|
+
It validates the run (finished steps, `grad_layers == n_layers`, peak VRAM recorded),
|
|
178
|
+
derives the architecture fingerprint and prints the snippets for `fit.py`, `fitcore.py`
|
|
179
|
+
and the Space. Anything not in that table prints **"not measured"** in `streamft fit` ?
|
|
180
|
+
that is the point.
|
|
181
|
+
|
|
157
182
|
## Verification
|
|
158
183
|
|
|
159
184
|
`python scripts/verify_gradients.py --model-dir checkpoints/qwen3b_bnb4bit --seq 128`
|
|
@@ -279,3 +279,22 @@ def test_repetition_rate_counts_duplicate_ngrams():
|
|
|
279
279
|
assert repetition_rate("a b c d e f g h") == 0.0
|
|
280
280
|
assert repetition_rate("a b c d a b c d") == 0.2 # 1 duplicate of 5 grams
|
|
281
281
|
assert repetition_rate("") == 0.0
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
def test_measured_row_validation_rejects_incomplete_runs():
|
|
285
|
+
import importlib.util
|
|
286
|
+
import sys
|
|
287
|
+
from pathlib import Path
|
|
288
|
+
|
|
289
|
+
path = Path(__file__).resolve().parents[1] / "scripts" / "measured_row.py"
|
|
290
|
+
spec = importlib.util.spec_from_file_location("measured_row", path)
|
|
291
|
+
mod = importlib.util.module_from_spec(spec)
|
|
292
|
+
sys.modules["measured_row"] = mod
|
|
293
|
+
spec.loader.exec_module(mod)
|
|
294
|
+
|
|
295
|
+
good = {"steps": 30, "grad_layers": 28, "n_layers": 28,
|
|
296
|
+
"peak_gb_max": 5.5, "step_s_med": 1.5}
|
|
297
|
+
assert mod.validate(good) == []
|
|
298
|
+
assert mod.validate({**good, "steps": 0})
|
|
299
|
+
assert mod.validate({**good, "grad_layers": 27})
|
|
300
|
+
assert mod.validate({**good, "peak_gb_max": None})
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|