streamft 0.2.6__tar.gz → 0.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (26) hide show
  1. {streamft-0.2.6 → streamft-0.4.0}/PKG-INFO +16 -3
  2. {streamft-0.2.6 → streamft-0.4.0}/README.md +15 -2
  3. {streamft-0.2.6 → streamft-0.4.0}/pyproject.toml +1 -1
  4. {streamft-0.2.6 → streamft-0.4.0}/streamft/__main__.py +18 -0
  5. {streamft-0.2.6 → streamft-0.4.0}/streamft/cli.py +17 -5
  6. streamft-0.4.0/streamft/easy.py +226 -0
  7. streamft-0.4.0/streamft/evals.py +148 -0
  8. {streamft-0.2.6 → streamft-0.4.0}/streamft/fit.py +33 -0
  9. {streamft-0.2.6 → streamft-0.4.0}/streamft.egg-info/PKG-INFO +16 -3
  10. {streamft-0.2.6 → streamft-0.4.0}/streamft.egg-info/SOURCES.txt +2 -0
  11. {streamft-0.2.6 → streamft-0.4.0}/tests/test_streamft_smoke.py +31 -0
  12. {streamft-0.2.6 → streamft-0.4.0}/LICENSE +0 -0
  13. {streamft-0.2.6 → streamft-0.4.0}/setup.cfg +0 -0
  14. {streamft-0.2.6 → streamft-0.4.0}/streamft/__init__.py +0 -0
  15. {streamft-0.2.6 → streamft-0.4.0}/streamft/autotune.py +0 -0
  16. {streamft-0.2.6 → streamft-0.4.0}/streamft/data.py +0 -0
  17. {streamft-0.2.6 → streamft-0.4.0}/streamft/disk_quant_store.py +0 -0
  18. {streamft-0.2.6 → streamft-0.4.0}/streamft/doctor.py +0 -0
  19. {streamft-0.2.6 → streamft-0.4.0}/streamft/lora.py +0 -0
  20. {streamft-0.2.6 → streamft-0.4.0}/streamft/metrics.py +0 -0
  21. {streamft-0.2.6 → streamft-0.4.0}/streamft/stream_quant_lora.py +0 -0
  22. {streamft-0.2.6 → streamft-0.4.0}/streamft/stream_train.py +0 -0
  23. {streamft-0.2.6 → streamft-0.4.0}/streamft.egg-info/dependency_links.txt +0 -0
  24. {streamft-0.2.6 → streamft-0.4.0}/streamft.egg-info/entry_points.txt +0 -0
  25. {streamft-0.2.6 → streamft-0.4.0}/streamft.egg-info/requires.txt +0 -0
  26. {streamft-0.2.6 → streamft-0.4.0}/streamft.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: streamft
3
- Version: 0.2.6
3
+ Version: 0.4.0
4
4
  Summary: Fine-tune models that do not fit: stream NF4 weights from disk, one transformer block at a time.
5
5
  Author-email: Gokhan Cava <byjameson@users.noreply.github.com>
6
6
  License: Apache-2.0
@@ -53,9 +53,11 @@ Measured on one machine — **RTX 3050 Laptop, 6 GB VRAM, 34 GB RAM, NVMe, Windo
53
53
 
54
54
  The 32B weights are **19.2 GB**. They never fully live in VRAM. At 1024 tokens/step the
55
55
  throughput is **3.1×** the 128-token rate, because the per-step I/O is amortised over
56
- more tokens. 50 steps of Dolly-15k SFT moved eval loss **1.8755 → 0.9661**.
56
+ more tokens. 50 steps of Dolly-15k SFT moved eval loss **1.8755 → 0.9659**.
57
57
 
58
- **Installed from PyPI? Start with [`QUICKSTART.md`](QUICKSTART.md)** — it uses only the
58
+ **Installed from PyPI? Start with [`QUICKSTART.md`](QUICKSTART.md) (T?rk?e:
59
+ [`QUICKSTART.tr.md`](QUICKSTART.tr.md)) ? or run `install.ps1` / `install.sh`, which
60
+ checks Python, installs the package, looks for a GPU and runs `doctor`.** — it uses only the
59
61
  published CLI (no repo scripts). Deeper dive: [`GUIDE_6GB_32B.md`](GUIDE_6GB_32B.md) — the practical write-up: presets,
60
62
  the six traps, and which numbers are real. The measurement protocol, the discarded rows
61
63
  and the correction of an earlier wrong claim ("8.6× / 100 tok/s") are in
@@ -79,9 +81,20 @@ streamft train --model-dir checkpoints/qwen0.5b_nf4 --sft --steps 20 --seq 512 -
79
81
  # LM path (optional): build token shards, then train without --sft
80
82
  streamft data --model Qwen/Qwen2.5-0.5B --out data
81
83
  streamft train --model-dir checkpoints/qwen0.5b_nf4 --steps 20 --seq 256 --tag lm
84
+
85
+ # no GPU at all, or no dataset: both are covered
86
+ streamft cpu --model Qwen/Qwen2.5-0.5B --steps 50 --seq 128
87
+ streamft synth --model Qwen/Qwen2.5-1.5B-Instruct --n 200 --out data/synth.jsonl
88
+
89
+ # inference budget (weights + KV cache), not just training
90
+ streamft fit --model Qwen/Qwen2.5-7B-Instruct --gpu 6 --seq 2048 --mode inference
91
+
92
+ # does it actually work? 20 local prompts, no API and no judge model
93
+ streamft eval --model Qwen/Qwen2.5-1.5B-Instruct
82
94
  ```
83
95
 
84
96
  Every run appends one JSON row to `results/results.jsonl` next to where you launched it.
97
+ Six worked recipes: [`COOKBOOK.md`](COOKBOOK.md).
85
98
  The same commands are in [`QUICKSTART.md`](QUICKSTART.md), which is what a pip user should
86
99
  read — this README also documents the experiments behind the numbers.
87
100
 
@@ -23,9 +23,11 @@ Measured on one machine — **RTX 3050 Laptop, 6 GB VRAM, 34 GB RAM, NVMe, Windo
23
23
 
24
24
  The 32B weights are **19.2 GB**. They never fully live in VRAM. At 1024 tokens/step the
25
25
  throughput is **3.1×** the 128-token rate, because the per-step I/O is amortised over
26
- more tokens. 50 steps of Dolly-15k SFT moved eval loss **1.8755 → 0.9661**.
26
+ more tokens. 50 steps of Dolly-15k SFT moved eval loss **1.8755 → 0.9659**.
27
27
 
28
- **Installed from PyPI? Start with [`QUICKSTART.md`](QUICKSTART.md)** — it uses only the
28
+ **Installed from PyPI? Start with [`QUICKSTART.md`](QUICKSTART.md) (T?rk?e:
29
+ [`QUICKSTART.tr.md`](QUICKSTART.tr.md)) ? or run `install.ps1` / `install.sh`, which
30
+ checks Python, installs the package, looks for a GPU and runs `doctor`.** — it uses only the
29
31
  published CLI (no repo scripts). Deeper dive: [`GUIDE_6GB_32B.md`](GUIDE_6GB_32B.md) — the practical write-up: presets,
30
32
  the six traps, and which numbers are real. The measurement protocol, the discarded rows
31
33
  and the correction of an earlier wrong claim ("8.6× / 100 tok/s") are in
@@ -49,9 +51,20 @@ streamft train --model-dir checkpoints/qwen0.5b_nf4 --sft --steps 20 --seq 512 -
49
51
  # LM path (optional): build token shards, then train without --sft
50
52
  streamft data --model Qwen/Qwen2.5-0.5B --out data
51
53
  streamft train --model-dir checkpoints/qwen0.5b_nf4 --steps 20 --seq 256 --tag lm
54
+
55
+ # no GPU at all, or no dataset: both are covered
56
+ streamft cpu --model Qwen/Qwen2.5-0.5B --steps 50 --seq 128
57
+ streamft synth --model Qwen/Qwen2.5-1.5B-Instruct --n 200 --out data/synth.jsonl
58
+
59
+ # inference budget (weights + KV cache), not just training
60
+ streamft fit --model Qwen/Qwen2.5-7B-Instruct --gpu 6 --seq 2048 --mode inference
61
+
62
+ # does it actually work? 20 local prompts, no API and no judge model
63
+ streamft eval --model Qwen/Qwen2.5-1.5B-Instruct
52
64
  ```
53
65
 
54
66
  Every run appends one JSON row to `results/results.jsonl` next to where you launched it.
67
+ Six worked recipes: [`COOKBOOK.md`](COOKBOOK.md).
55
68
  The same commands are in [`QUICKSTART.md`](QUICKSTART.md), which is what a pip user should
56
69
  read — this README also documents the experiments behind the numbers.
57
70
 
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "streamft"
7
- version = "0.2.6"
7
+ version = "0.4.0"
8
8
  description = "Fine-tune models that do not fit: stream NF4 weights from disk, one transformer block at a time."
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.10"
@@ -4,6 +4,9 @@
4
4
  streamft prepare turn a bf16 model into a pre-quantized NF4 checkpoint
5
5
  streamft train the streaming trainer (all exp54 flags)
6
6
  streamft data build the token shards (LM path only; SFT needs none)
7
+ streamft cpu SFT a small model on the CPU (no GPU; model must fit RAM)
8
+ streamft synth generate training pairs locally with a small model
9
+ streamft eval run a small local prompt set (repetition/refusal/keywords)
7
10
  streamft doctor environment report (torch, CUDA, bitsandbytes, VRAM)
8
11
  """
9
12
  from __future__ import annotations
@@ -16,6 +19,9 @@ USAGE = """streamft - fine-tune models that do not fit
16
19
  streamft prepare --model <dir> --save-quant checkpoints/<name>
17
20
  streamft train --model-dir checkpoints/<name> --sft --steps 30 [--seq 1024]
18
21
  streamft data --model <dir|hf-id> --out data
22
+ streamft cpu --model <small-model> --steps 20
23
+ streamft synth --model <small-model> --n 50 --out data/synth.jsonl
24
+ streamft eval --model <model> [--prompts mine.jsonl]
19
25
  streamft doctor
20
26
 
21
27
  Docs: https://github.com/byjameson/stream-finetune
@@ -35,6 +41,18 @@ def main() -> int:
35
41
 
36
42
  return fit_main(rest)
37
43
 
44
+ if cmd == "cpu":
45
+ from streamft.easy import cpu_main
46
+ return cpu_main(rest)
47
+
48
+ if cmd == "eval":
49
+ from streamft.evals import main as eval_main
50
+ return eval_main(rest)
51
+
52
+ if cmd == "synth":
53
+ from streamft.easy import synth_main
54
+ return synth_main(rest)
55
+
38
56
  if cmd == "data":
39
57
  from streamft.data import main as data_main
40
58
 
@@ -388,14 +388,23 @@ def main() -> int:
388
388
  tok = AutoTokenizer.from_pretrained(model_dir)
389
389
  if tok.pad_token is None:
390
390
  tok.pad_token = tok.eos_token
391
- ds = load_dataset(args.dataset, split="train").shuffle(seed=1234)
391
+ if str(args.dataset).lower().endswith(".jsonl"):
392
+ # local {"instruction","response"} file, e.g. from `streamft synth`
393
+ ds = load_dataset("json", data_files=args.dataset, split="train").shuffle(seed=1234)
394
+ else:
395
+ ds = load_dataset(args.dataset, split="train").shuffle(seed=1234)
392
396
  train_rows = [ds[i] for i in range(min(args.train_size, len(ds)))]
393
397
  eval_rows = [ds[len(train_rows) + i]
394
398
  for i in range(min(args.eval_size, len(ds) - len(train_rows)))]
395
399
  print(f"[sft] {Path(args.dataset).name} train={len(train_rows)} "
396
400
  f"eval={len(eval_rows)}", flush=True)
397
- ce0 = _masked_eval(tr, eval_rows, tok, args.seq)
398
- print(f"[eval] before loss={ce0:.4f}", flush=True)
401
+ if eval_rows:
402
+ ce0 = _masked_eval(tr, eval_rows, tok, args.seq)
403
+ print(f"[eval] before loss={ce0:.4f}", flush=True)
404
+ else:
405
+ ce0 = None
406
+ print(f"[sft] no held-out rows (dataset has {len(train_rows)} rows); skipping eval",
407
+ flush=True)
399
408
  # Filter once so `--steps N` means N optimizer updates, not "N attempts minus
400
409
  # whatever the mask rejected". A silently short run is a lie about the budget.
401
410
  valid_rows, skipped_train = sft_schedule(
@@ -419,8 +428,11 @@ def main() -> int:
419
428
  if (i + 1) % 10 == 0:
420
429
  print(f"[step] {i+1}/{args.steps} loss={loss:.4f} {dt:.2f}s "
421
430
  f"peak={peak:.2f}GB rss={host_rss_gb()}GB", flush=True)
422
- ce1 = _masked_eval(tr, eval_rows, tok, args.seq)
423
- print(f"[eval] after loss={ce1:.4f} (delta {ce1-ce0:+.4f})", flush=True)
431
+ if eval_rows:
432
+ ce1 = _masked_eval(tr, eval_rows, tok, args.seq)
433
+ print(f"[eval] after loss={ce1:.4f} (delta {ce1-ce0:+.4f})", flush=True)
434
+ else:
435
+ ce1 = None
424
436
  wall = time.time() - t_all
425
437
  else:
426
438
  tokens = load_tokens(str(user_path(args.data)))
@@ -0,0 +1,226 @@
1
+ """Two on-ramps for people without a training GPU.
2
+
3
+ streamft cpu SFT a small model on the CPU (the model must fit in RAM)
4
+ streamft synth generate training pairs locally with a small model (no API, no data hunt)
5
+
6
+ Both are deliberately simple: plain transformers forward/backward, our LoRA module, the
7
+ same prompt masking as the GPU path. The streaming engine is not involved.
8
+ """
9
+ from __future__ import annotations
10
+
11
+ import argparse
12
+ import json
13
+ import random
14
+ import re
15
+ import time
16
+ from pathlib import Path
17
+
18
+ import torch
19
+
20
+ from .cli import _encode_chat, _format_messages # noqa: F401 (same masking rules)
21
+ from .lora import apply_lora, lora_parameters, lora_state_dict
22
+
23
+
24
+ # --------------------------------------------------------------------------- cpu train
25
+
26
+ def _load_tiny(model_dir: str):
27
+ from transformers import AutoModelForCausalLM
28
+
29
+ dtype = torch.float32 # CPU: bf16 is slow and fp16 is unsupported on many CPUs
30
+ model = AutoModelForCausalLM.from_pretrained(model_dir, torch_dtype=dtype)
31
+ model.eval()
32
+ return model
33
+
34
+
35
+ @torch.no_grad()
36
+ def _eval_masked(model, rows, tok, seq: int) -> float:
37
+ model.eval()
38
+ total, n = 0.0, 0
39
+ for row in rows:
40
+ enc = _encode_chat(tok, row, seq)
41
+ if enc is None:
42
+ continue
43
+ ids, labels = enc
44
+ loss = model(input_ids=ids, labels=labels).loss
45
+ v = int((labels != -100).sum())
46
+ total += float(loss) * v
47
+ n += v
48
+ model.train()
49
+ if n == 0:
50
+ raise RuntimeError("no eval row has a valid prompt mask")
51
+ return total / n
52
+
53
+
54
+ def cpu_main(argv=None) -> int:
55
+ from datasets import load_dataset
56
+ from transformers import AutoTokenizer
57
+
58
+ ap = argparse.ArgumentParser(prog="streamft cpu")
59
+ ap.add_argument("--model", required=True, help="small model dir or HF id (fits in RAM)")
60
+ ap.add_argument("--dataset", default="databricks/databricks-dolly-15k")
61
+ ap.add_argument("--train_size", type=int, default=128)
62
+ ap.add_argument("--eval_size", type=int, default=8)
63
+ ap.add_argument("--steps", type=int, default=20)
64
+ ap.add_argument("--seq", type=int, default=256)
65
+ ap.add_argument("--lr", type=float, default=3e-4)
66
+ ap.add_argument("--lora_r", type=int, default=8)
67
+ ap.add_argument("--save", default="")
68
+ ap.add_argument("--tag", default="cpu_sft")
69
+ args = ap.parse_args(argv)
70
+
71
+ tok = AutoTokenizer.from_pretrained(args.model)
72
+ if tok.pad_token is None:
73
+ tok.pad_token = tok.eos_token
74
+ t0 = time.time()
75
+ model = _load_tiny(args.model)
76
+ apply_lora(model, r_attn=args.lora_r, r_mlp=0)
77
+ params = list(lora_parameters(model))
78
+ opt = torch.optim.AdamW(params, lr=args.lr)
79
+ print(f"[cpu] {args.model} loaded in {time.time()-t0:.1f}s, "
80
+ f"{sum(p.numel() for p in params)/1e6:.2f}M adapter params", flush=True)
81
+
82
+ ds = load_dataset(args.dataset, split="train").shuffle(seed=1234)
83
+ train_rows = [ds[i] for i in range(min(args.train_size, len(ds)))]
84
+ eval_rows = [ds[len(train_rows) + i]
85
+ for i in range(min(args.eval_size, len(ds) - len(train_rows)))]
86
+
87
+ valid = [r for r in train_rows if _encode_chat(tok, r, args.seq) is not None]
88
+ dropped = len(train_rows) - len(valid)
89
+ if not valid:
90
+ raise SystemExit("[cpu] no training row has a valid prompt mask")
91
+ if dropped:
92
+ print(f"[cpu] {dropped}/{len(train_rows)} rows dropped (prompt not a prefix)",
93
+ flush=True)
94
+
95
+ ce0 = _eval_masked(model, eval_rows, tok, args.seq)
96
+ print(f"[cpu] eval before {ce0:.4f}", flush=True)
97
+
98
+ model.train()
99
+ losses, times = [], []
100
+ for i in range(args.steps):
101
+ enc = _encode_chat(tok, valid[i % len(valid)], args.seq)
102
+ ids, labels = enc
103
+ t = time.time()
104
+ loss = model(input_ids=ids, labels=labels).loss
105
+ loss.backward()
106
+ opt.step()
107
+ opt.zero_grad(set_to_none=True)
108
+ losses.append(round(float(loss.detach()), 6))
109
+ times.append(round(time.time() - t, 3))
110
+ if (i + 1) % 5 == 0 or i == 0:
111
+ print(f"[cpu] step {i+1}/{args.steps} loss={losses[-1]:.4f} "
112
+ f"{times[-1]:.2f}s", flush=True)
113
+
114
+ ce1 = _eval_masked(model, eval_rows, tok, args.seq)
115
+ if args.save:
116
+ out = Path(args.save)
117
+ out.mkdir(parents=True, exist_ok=True)
118
+ torch.save(lora_state_dict(model), str(out / "adapter.pt"))
119
+ row = {
120
+ "engine": "cpu-dense", "tag": args.tag, "model": args.model, "steps": len(losses),
121
+ "seq": args.seq, "lr": args.lr, "lora_r": args.lora_r,
122
+ "step_s_med": sorted(times)[len(times) // 2] if times else None,
123
+ "ce_before": round(ce0, 5), "ce_after": round(ce1, 5),
124
+ "ce_delta": round(ce1 - ce0, 5), "losses": losses,
125
+ "sft_skipped": dropped, "saved": args.save or None,
126
+ "license": "Apache-2.0", "device": "cpu",
127
+ }
128
+ print(json.dumps(row, ensure_ascii=True), flush=True)
129
+ out = Path("results")
130
+ out.mkdir(exist_ok=True)
131
+ with open(out / "results.jsonl", "a", encoding="utf-8") as f:
132
+ f.write(json.dumps(row) + "\n")
133
+ return 0
134
+
135
+
136
+ # --------------------------------------------------------------------------- synth
137
+
138
+ PROMPT = """You write training data. Produce ONE example as JSON with keys
139
+ "instruction" and "response". Topic: {task}.
140
+ Rules: the instruction is a short request; the response is a direct answer of 1-3
141
+ sentences; no markdown, no preamble, no extra keys.
142
+ JSON:"""
143
+
144
+
145
+ PLACEHOLDER_MARKERS = ("<instruction>", "<response>", "<your", "your instruction",
146
+ "lorem ipsum", "example.com")
147
+
148
+
149
+ def parse_pair(text: str):
150
+ """Extract one {"instruction","response"} object; reject template junk.
151
+
152
+ Small models happily echo the schema back ("<instruction>"), and a single such row
153
+ poisons a dataset. Everything is filtered here so the caller only has to dedupe.
154
+ """
155
+ m = re.search(r"\{.*\}", text, re.S)
156
+ if not m:
157
+ return None
158
+ try:
159
+ obj = json.loads(m.group(0))
160
+ except Exception:
161
+ return None
162
+ if not isinstance(obj, dict):
163
+ return None
164
+ ins = str(obj.get("instruction", "")).strip()
165
+ res = str(obj.get("response", "")).strip()
166
+ low = (ins + " " + res).lower()
167
+ if any(p in low for p in PLACEHOLDER_MARKERS):
168
+ return None
169
+ if "{" in ins or "{" in res or "}" in ins or "}" in res:
170
+ return None
171
+ if not (5 <= len(ins) <= 200 and 10 <= len(res) <= 1200):
172
+ return None
173
+ return {"instruction": ins, "response": res}
174
+
175
+
176
+ def synth_main(argv=None) -> int:
177
+ from transformers import AutoModelForCausalLM, AutoTokenizer
178
+
179
+ ap = argparse.ArgumentParser(prog="streamft synth")
180
+ ap.add_argument("--model", default="Qwen/Qwen2.5-1.5B-Instruct")
181
+ ap.add_argument("--out", default="data/synth.jsonl")
182
+ ap.add_argument("--n", type=int, default=50, help="examples to attempt")
183
+ ap.add_argument("--task", default="practical everyday knowledge")
184
+ ap.add_argument("--seq", type=int, default=256)
185
+ ap.add_argument("--temperature", type=float, default=0.9)
186
+ ap.add_argument("--device", default="cuda" if torch.cuda.is_available() else "cpu")
187
+ args = ap.parse_args(argv)
188
+
189
+ tok = AutoTokenizer.from_pretrained(args.model)
190
+ model = AutoModelForCausalLM.from_pretrained(
191
+ args.model,
192
+ torch_dtype=torch.bfloat16 if args.device == "cuda" else torch.float32,
193
+ ).to(args.device).eval()
194
+
195
+ seen, kept = set(), []
196
+ out = Path(args.out)
197
+ out.parent.mkdir(parents=True, exist_ok=True)
198
+ t0 = time.time()
199
+ for i in range(args.n):
200
+ prompt = PROMPT.format(task=args.task)
201
+ ids = tok(prompt, return_tensors="pt").to(args.device)
202
+ with torch.no_grad():
203
+ gen = model.generate(**ids, max_new_tokens=args.seq, do_sample=True,
204
+ temperature=args.temperature, top_p=0.95,
205
+ pad_token_id=tok.eos_token_id)
206
+ text = tok.decode(gen[0][ids["input_ids"].shape[1]:], skip_special_tokens=True)
207
+ pair = parse_pair(text)
208
+ if pair is None:
209
+ continue
210
+ key = pair["instruction"].lower()
211
+ if key in seen:
212
+ continue
213
+ seen.add(key)
214
+ kept.append(pair)
215
+ if (len(kept) % 10) == 0:
216
+ print(f"[synth] {len(kept)} kept / {i+1} attempts", flush=True)
217
+
218
+ with open(out, "w", encoding="utf-8") as f:
219
+ for r in kept:
220
+ f.write(json.dumps(r, ensure_ascii=False) + "\n")
221
+ print(json.dumps({
222
+ "out": str(out), "attempts": args.n, "kept": len(kept),
223
+ "keep_rate": round(len(kept) / max(args.n, 1), 3),
224
+ "seconds": round(time.time() - t0, 1), "model": args.model, "task": args.task,
225
+ }, indent=1))
226
+ return 0
@@ -0,0 +1,148 @@
1
+ """`streamft eval` — does the model actually do the thing? No API, no judge model.
2
+
3
+ It runs a small prompt set locally and reports what can be measured without a second
4
+ model: repetition, refusal-phrase hits, length, latency, and optionally keyword adherence
5
+ when the prompt file carries an `expect` field.
6
+
7
+ streamft eval --model Qwen/Qwen2.5-1.5B-Instruct
8
+ streamft eval --model ./merged --prompts my_prompts.jsonl --max-new 96
9
+
10
+ Prompt file format (JSONL, one object per line):
11
+ {"prompt": "...", "expect": ["optional", "keywords"]}
12
+ """
13
+ from __future__ import annotations
14
+
15
+ import argparse
16
+ import json
17
+ import re
18
+ import time
19
+ from collections import Counter
20
+
21
+ import torch
22
+
23
+ # Ten English + ten Turkish prompts. Deliberately mundane: this is a smoke test, not a
24
+ # benchmark. `expect` is a keyword that should appear if the model understood the task.
25
+ BUILTIN = [
26
+ {"prompt": "List three uses for a paperclip.", "expect": []},
27
+ {"prompt": "Explain in one sentence why the sky is blue.", "expect": ["light", "scatter"]},
28
+ {"prompt": "Give one reason to keep a food diary.", "expect": []},
29
+ {"prompt": "Write a two-line rhyme about rain.", "expect": []},
30
+ {"prompt": "What is 17 multiplied by 6?", "expect": ["102"]},
31
+ {"prompt": "Name two vegetables that grow underground.", "expect": ["carrot", "potato"]},
32
+ {"prompt": "Summarise the plot of a film about a lost dog in one sentence.", "expect": []},
33
+ {"prompt": "Translate 'good morning' into Turkish.", "expect": ["günaydın", "gunaydin"]},
34
+ {"prompt": "Give a one-sentence tip for better sleep.", "expect": []},
35
+ {"prompt": "What does a red traffic light mean?", "expect": ["stop"]},
36
+ {"prompt": "Bir cümlede su neden buharlaşır açıkla.", "expect": ["ısı", "isi", "sıcaklık"]},
37
+ {"prompt": "Üç tane meyve say.", "expect": ["elma", "muz", "portakal", "çilek"]},
38
+ {"prompt": "17 çarpı 6 kaç eder?", "expect": ["102"]},
39
+ {"prompt": "Yağmurlu bir gün için iki satırlık bir dörtlük yaz.", "expect": []},
40
+ {"prompt": "İyi bir uyku için tek cümlelik bir öneri ver.", "expect": []},
41
+ {"prompt": "Kırmızı trafik ışığı ne anlama gelir?", "expect": ["dur"]},
42
+ {"prompt": "Sabah kahvaltısı için iki öneri ver.", "expect": []},
43
+ {"prompt": "Bir cümlede neden kitap okumalıyız açıkla.", "expect": []},
44
+ {"prompt": "Türkiye'nin başkenti neresidir?", "expect": ["ankara"]},
45
+ {"prompt": "Bir cümlede bisikletin avantajını yaz.", "expect": []},
46
+ ]
47
+
48
+ REFUSAL = [
49
+ "i can't help", "i cannot help", "i can't assist", "i cannot assist",
50
+ "as an ai", "i'm sorry, but", "i am sorry, but", "cannot provide",
51
+ "yardımcı olamam", "yardimci olamam", "üzgünüm", "uzgunum", "olarak yapay zeka",
52
+ ]
53
+
54
+
55
+ def repetition_rate(text: str, n: int = 4) -> float:
56
+ """Fraction of repeated n-grams (1.0 = every n-gram is a duplicate)."""
57
+ words = re.findall(r"\w+", text.lower())
58
+ if len(words) < n + 1:
59
+ return 0.0
60
+ grams = [tuple(words[i:i + n]) for i in range(len(words) - n + 1)]
61
+ counts = Counter(grams)
62
+ repeated = sum(c - 1 for c in counts.values() if c > 1)
63
+ return repeated / len(grams)
64
+
65
+
66
+ def main(argv=None) -> int:
67
+ from transformers import AutoModelForCausalLM, AutoTokenizer
68
+
69
+ ap = argparse.ArgumentParser(prog="streamft eval")
70
+ ap.add_argument("--model", default="Qwen/Qwen2.5-1.5B-Instruct")
71
+ ap.add_argument("--prompts", default="", help="JSONL with {'prompt', 'expect'?}")
72
+ ap.add_argument("--max-new", type=int, default=64)
73
+ ap.add_argument("--limit", type=int, default=0, help="evaluate only the first N prompts")
74
+ ap.add_argument("--out", default="results/eval.jsonl")
75
+ ap.add_argument("--device", default="cuda" if torch.cuda.is_available() else "cpu")
76
+ ap.add_argument("--quiet", action="store_true")
77
+ args = ap.parse_args(argv)
78
+
79
+ prompts = BUILTIN
80
+ if args.prompts:
81
+ with open(args.prompts, encoding="utf-8") as f:
82
+ prompts = [json.loads(line) for line in f if line.strip()]
83
+ if args.limit:
84
+ prompts = prompts[: args.limit]
85
+
86
+ dtype = torch.bfloat16 if args.device == "cuda" else torch.float32
87
+ tok = AutoTokenizer.from_pretrained(args.model)
88
+ model = AutoModelForCausalLM.from_pretrained(args.model, torch_dtype=dtype).to(args.device)
89
+ model.eval()
90
+
91
+ rows = []
92
+ t_all = time.time()
93
+ for i, item in enumerate(prompts):
94
+ text = item["prompt"]
95
+ if getattr(tok, "chat_template", None):
96
+ text = tok.apply_chat_template([{"role": "user", "content": text}],
97
+ tokenize=False, add_generation_prompt=True)
98
+ ids = tok(text, return_tensors="pt").to(args.device)
99
+ t0 = time.time()
100
+ with torch.no_grad():
101
+ out = model.generate(**ids, max_new_tokens=args.max_new, do_sample=False,
102
+ pad_token_id=tok.eos_token_id)
103
+ dt = time.time() - t0
104
+ gen = out[0][ids["input_ids"].shape[1]:]
105
+ answer = tok.decode(gen, skip_special_tokens=True).strip()
106
+ low = answer.lower()
107
+ row = {
108
+ "i": i, "prompt": item["prompt"], "answer": answer,
109
+ "tokens": int(gen.shape[0]), "seconds": round(dt, 2),
110
+ "tok_s": round(int(gen.shape[0]) / max(dt, 1e-6), 1),
111
+ "rep4": round(repetition_rate(answer), 3),
112
+ "refusal": any(p in low for p in REFUSAL),
113
+ "expect_hit": (None if not item.get("expect")
114
+ else any(k.lower() in low for k in item["expect"])),
115
+ }
116
+ rows.append(row)
117
+ if not args.quiet:
118
+ flag = "REFUSAL" if row["refusal"] else ("miss" if row["expect_hit"] is False else "")
119
+ print(f"[{i+1}/{len(prompts)}] {row['tok_s']:.1f} tok/s rep4={row['rep4']:.2f} {flag}")
120
+ print(" " + answer.replace("\n", " ")[:160])
121
+
122
+ n = len(rows)
123
+ expected = [r for r in rows if r["expect_hit"] is not None]
124
+ summary = {
125
+ "model": args.model, "n": n,
126
+ "refusal_rate": round(sum(r["refusal"] for r in rows) / max(n, 1), 3),
127
+ "mean_rep4": round(sum(r["rep4"] for r in rows) / max(n, 1), 3),
128
+ "mean_tokens": round(sum(r["tokens"] for r in rows) / max(n, 1), 1),
129
+ "mean_tok_s": round(sum(r["tok_s"] for r in rows) / max(n, 1), 1),
130
+ "keyword_hit_rate": (round(sum(1 for r in expected if r["expect_hit"]) / len(expected), 3)
131
+ if expected else None),
132
+ "wall_s": round(time.time() - t_all, 1),
133
+ "device": args.device,
134
+ }
135
+ print(json.dumps(summary, indent=1))
136
+
137
+ import os
138
+ os.makedirs(os.path.dirname(args.out) or ".", exist_ok=True)
139
+ with open(args.out, "w", encoding="utf-8") as f:
140
+ for r in rows:
141
+ f.write(json.dumps(r, ensure_ascii=False) + "\n")
142
+ with open(args.out.replace(".jsonl", "_summary.json"), "w", encoding="utf-8") as f:
143
+ json.dump(summary, f, indent=1)
144
+ return 0
145
+
146
+
147
+ if __name__ == "__main__":
148
+ raise SystemExit(main())
@@ -53,6 +53,16 @@ def load_config(model: str) -> dict:
53
53
  return json.loads(Path(hf_hub_download(model, "config.json")).read_text(encoding="utf-8"))
54
54
 
55
55
 
56
+ def kv_cache_gb(cfg: dict, seq: int, batch: int = 1, bytes_per: int = 2) -> float:
57
+ """KV cache at inference: 2 tensors x layers x kv_heads x head_dim x seq x batch."""
58
+ h = int(cfg.get("hidden_size") or cfg.get("n_embd"))
59
+ layers = int(cfg.get("num_hidden_layers") or cfg.get("n_layer"))
60
+ heads = int(cfg.get("num_attention_heads") or 1)
61
+ kv_heads = int(cfg.get("num_key_value_heads") or heads)
62
+ head_dim = h // max(heads, 1)
63
+ return 2.0 * layers * kv_heads * head_dim * seq * batch * bytes_per / 1e9
64
+
65
+
56
66
  def model_fingerprint(cfg: dict) -> tuple:
57
67
  """Everything that changes the measurement: two 7B models are not interchangeable."""
58
68
  return (
@@ -95,6 +105,8 @@ def main(argv: list[str] | None = None) -> int:
95
105
  ap.add_argument("--model", required=True, help="local dir or HF id")
96
106
  ap.add_argument("--gpu", type=float, default=6.0, help="VRAM in GB (default 6)")
97
107
  ap.add_argument("--seq", type=int, default=512)
108
+ ap.add_argument("--mode", choices=["train", "inference"], default="train")
109
+ ap.add_argument("--batch", type=int, default=1)
98
110
  ap.add_argument("--json", action="store_true")
99
111
  args = ap.parse_args(argv)
100
112
 
@@ -126,6 +138,27 @@ def main(argv: list[str] | None = None) -> int:
126
138
  "measured_on_rtx3050_6gb": measured,
127
139
  "measured_elsewhere": measured is None,
128
140
  }
141
+ if args.mode == "inference":
142
+ kv_gb = kv_cache_gb(cfg, args.seq, args.batch)
143
+ total = m["nf4_gb"] + m["head_gb"] + kv_gb + 0.5
144
+ out_inf = {
145
+ "model": args.model, "mode": "inference", "seq": args.seq, "batch": args.batch,
146
+ "weights_gb": round(m["nf4_gb"] + m["head_gb"], 1),
147
+ "kv_cache_gb": round(kv_gb, 2),
148
+ "needed_gb": round(total, 1),
149
+ "fits_whole": total <= args.gpu,
150
+ "speed": "not measured for this model",
151
+ }
152
+ if args.json:
153
+ print(json.dumps(out_inf, indent=1))
154
+ return 0
155
+ print(f"{args.model} (inference)")
156
+ print(f" weights ~{out_inf['weights_gb']} GB + KV cache ~{out_inf['kv_cache_gb']} GB "
157
+ f"at seq {args.seq}, batch {args.batch} -> ~{out_inf['needed_gb']} GB")
158
+ print(f" verdict: {'fits' if out_inf['fits_whole'] else 'does NOT fit'} on {args.gpu:g} GB")
159
+ print(" speed: not measured for this model (the planner does not guess tok/s)")
160
+ return 0
161
+
129
162
  if args.json:
130
163
  print(json.dumps(out, indent=1))
131
164
  return 0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: streamft
3
- Version: 0.2.6
3
+ Version: 0.4.0
4
4
  Summary: Fine-tune models that do not fit: stream NF4 weights from disk, one transformer block at a time.
5
5
  Author-email: Gokhan Cava <byjameson@users.noreply.github.com>
6
6
  License: Apache-2.0
@@ -53,9 +53,11 @@ Measured on one machine — **RTX 3050 Laptop, 6 GB VRAM, 34 GB RAM, NVMe, Windo
53
53
 
54
54
  The 32B weights are **19.2 GB**. They never fully live in VRAM. At 1024 tokens/step the
55
55
  throughput is **3.1×** the 128-token rate, because the per-step I/O is amortised over
56
- more tokens. 50 steps of Dolly-15k SFT moved eval loss **1.8755 → 0.9661**.
56
+ more tokens. 50 steps of Dolly-15k SFT moved eval loss **1.8755 → 0.9659**.
57
57
 
58
- **Installed from PyPI? Start with [`QUICKSTART.md`](QUICKSTART.md)** — it uses only the
58
+ **Installed from PyPI? Start with [`QUICKSTART.md`](QUICKSTART.md) (T?rk?e:
59
+ [`QUICKSTART.tr.md`](QUICKSTART.tr.md)) ? or run `install.ps1` / `install.sh`, which
60
+ checks Python, installs the package, looks for a GPU and runs `doctor`.** — it uses only the
59
61
  published CLI (no repo scripts). Deeper dive: [`GUIDE_6GB_32B.md`](GUIDE_6GB_32B.md) — the practical write-up: presets,
60
62
  the six traps, and which numbers are real. The measurement protocol, the discarded rows
61
63
  and the correction of an earlier wrong claim ("8.6× / 100 tok/s") are in
@@ -79,9 +81,20 @@ streamft train --model-dir checkpoints/qwen0.5b_nf4 --sft --steps 20 --seq 512 -
79
81
  # LM path (optional): build token shards, then train without --sft
80
82
  streamft data --model Qwen/Qwen2.5-0.5B --out data
81
83
  streamft train --model-dir checkpoints/qwen0.5b_nf4 --steps 20 --seq 256 --tag lm
84
+
85
+ # no GPU at all, or no dataset: both are covered
86
+ streamft cpu --model Qwen/Qwen2.5-0.5B --steps 50 --seq 128
87
+ streamft synth --model Qwen/Qwen2.5-1.5B-Instruct --n 200 --out data/synth.jsonl
88
+
89
+ # inference budget (weights + KV cache), not just training
90
+ streamft fit --model Qwen/Qwen2.5-7B-Instruct --gpu 6 --seq 2048 --mode inference
91
+
92
+ # does it actually work? 20 local prompts, no API and no judge model
93
+ streamft eval --model Qwen/Qwen2.5-1.5B-Instruct
82
94
  ```
83
95
 
84
96
  Every run appends one JSON row to `results/results.jsonl` next to where you launched it.
97
+ Six worked recipes: [`COOKBOOK.md`](COOKBOOK.md).
85
98
  The same commands are in [`QUICKSTART.md`](QUICKSTART.md), which is what a pip user should
86
99
  read — this README also documents the experiments behind the numbers.
87
100
 
@@ -8,6 +8,8 @@ streamft/cli.py
8
8
  streamft/data.py
9
9
  streamft/disk_quant_store.py
10
10
  streamft/doctor.py
11
+ streamft/easy.py
12
+ streamft/evals.py
11
13
  streamft/fit.py
12
14
  streamft/lora.py
13
15
  streamft/metrics.py
@@ -248,3 +248,34 @@ def test_masked_eval_raises_when_every_row_is_skipped(monkeypatch):
248
248
  monkeypatch.setattr(cli, "_encode_chat", lambda *a, **k: None)
249
249
  with pytest.raises(RuntimeError, match="every row was skipped"):
250
250
  cli._masked_eval(FakeTrainer(), [{"instruction": "x", "response": "y"}], object(), 8)
251
+
252
+
253
+ # ------------------------------------------------ synth / planner helpers
254
+
255
+ def test_parse_pair_rejects_template_junk():
256
+ from streamft.easy import parse_pair
257
+
258
+ good = parse_pair('{"instruction": "Give me a tip", "response": "Sharpen your knives."}')
259
+ assert good and good["instruction"] == "Give me a tip"
260
+ assert parse_pair('{"instruction": "<instruction>", "response": "<response>"}') is None
261
+ assert parse_pair('{"instruction": "x", "response": "short"}') is None
262
+ assert parse_pair("not json") is None
263
+ assert parse_pair('{"instruction": "a" * 300, "response": "b" * 20}') is None
264
+
265
+
266
+ def test_kv_cache_matches_hand_calculation():
267
+ from streamft.fit import kv_cache_gb
268
+
269
+ cfg = {"hidden_size": 2048, "num_hidden_layers": 24, "num_attention_heads": 16,
270
+ "num_key_value_heads": 2}
271
+ # 2 * 24 layers * 2 kv heads * 128 head_dim * 1024 seq * 2 bytes
272
+ expect = 2 * 24 * 2 * 128 * 1024 * 2 / 1e9
273
+ assert abs(kv_cache_gb(cfg, 1024) - expect) < 1e-12
274
+
275
+
276
+ def test_repetition_rate_counts_duplicate_ngrams():
277
+ from streamft.evals import repetition_rate
278
+
279
+ assert repetition_rate("a b c d e f g h") == 0.0
280
+ assert repetition_rate("a b c d a b c d") == 0.2 # 1 duplicate of 5 grams
281
+ assert repetition_rate("") == 0.0
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes