streamft 0.2.5__tar.gz → 0.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (25) hide show
  1. {streamft-0.2.5 → streamft-0.3.0}/PKG-INFO +27 -2
  2. {streamft-0.2.5 → streamft-0.3.0}/README.md +26 -1
  3. {streamft-0.2.5 → streamft-0.3.0}/pyproject.toml +1 -1
  4. {streamft-0.2.5 → streamft-0.3.0}/streamft/__main__.py +12 -0
  5. {streamft-0.2.5 → streamft-0.3.0}/streamft/cli.py +29 -5
  6. {streamft-0.2.5 → streamft-0.3.0}/streamft/disk_quant_store.py +6 -2
  7. streamft-0.3.0/streamft/easy.py +226 -0
  8. {streamft-0.2.5 → streamft-0.3.0}/streamft/fit.py +43 -0
  9. {streamft-0.2.5 → streamft-0.3.0}/streamft.egg-info/PKG-INFO +27 -2
  10. {streamft-0.2.5 → streamft-0.3.0}/streamft.egg-info/SOURCES.txt +1 -0
  11. {streamft-0.2.5 → streamft-0.3.0}/tests/test_streamft_smoke.py +23 -0
  12. {streamft-0.2.5 → streamft-0.3.0}/LICENSE +0 -0
  13. {streamft-0.2.5 → streamft-0.3.0}/setup.cfg +0 -0
  14. {streamft-0.2.5 → streamft-0.3.0}/streamft/__init__.py +0 -0
  15. {streamft-0.2.5 → streamft-0.3.0}/streamft/autotune.py +0 -0
  16. {streamft-0.2.5 → streamft-0.3.0}/streamft/data.py +0 -0
  17. {streamft-0.2.5 → streamft-0.3.0}/streamft/doctor.py +0 -0
  18. {streamft-0.2.5 → streamft-0.3.0}/streamft/lora.py +0 -0
  19. {streamft-0.2.5 → streamft-0.3.0}/streamft/metrics.py +0 -0
  20. {streamft-0.2.5 → streamft-0.3.0}/streamft/stream_quant_lora.py +0 -0
  21. {streamft-0.2.5 → streamft-0.3.0}/streamft/stream_train.py +0 -0
  22. {streamft-0.2.5 → streamft-0.3.0}/streamft.egg-info/dependency_links.txt +0 -0
  23. {streamft-0.2.5 → streamft-0.3.0}/streamft.egg-info/entry_points.txt +0 -0
  24. {streamft-0.2.5 → streamft-0.3.0}/streamft.egg-info/requires.txt +0 -0
  25. {streamft-0.2.5 → streamft-0.3.0}/streamft.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: streamft
3
- Version: 0.2.5
3
+ Version: 0.3.0
4
4
  Summary: Fine-tune models that do not fit: stream NF4 weights from disk, one transformer block at a time.
5
5
  Author-email: Gokhan Cava <byjameson@users.noreply.github.com>
6
6
  License: Apache-2.0
@@ -79,15 +79,24 @@ streamft train --model-dir checkpoints/qwen0.5b_nf4 --sft --steps 20 --seq 512 -
79
79
  # LM path (optional): build token shards, then train without --sft
80
80
  streamft data --model Qwen/Qwen2.5-0.5B --out data
81
81
  streamft train --model-dir checkpoints/qwen0.5b_nf4 --steps 20 --seq 256 --tag lm
82
+
83
+ # no GPU at all, or no dataset: both are covered
84
+ streamft cpu --model Qwen/Qwen2.5-0.5B --steps 50 --seq 128
85
+ streamft synth --model Qwen/Qwen2.5-1.5B-Instruct --n 200 --out data/synth.jsonl
86
+
87
+ # inference budget (weights + KV cache), not just training
88
+ streamft fit --model Qwen/Qwen2.5-7B-Instruct --gpu 6 --seq 2048 --mode inference
82
89
  ```
83
90
 
84
91
  Every run appends one JSON row to `results/results.jsonl` next to where you launched it.
92
+ Six worked recipes: [`COOKBOOK.md`](COOKBOOK.md).
85
93
  The same commands are in [`QUICKSTART.md`](QUICKSTART.md), which is what a pip user should
86
94
  read — this README also documents the experiments behind the numbers.
87
95
 
88
96
  **From a source checkout only** (not part of the wheel): `tools/finetune.py` is a wrapper
89
97
  that encodes the VRAM presets, `scripts/compare_peft.py` reproduces the engine comparison,
90
- `scripts/verify_gradients.py` the gradient check, `scripts/prep_data.py` the token shards
98
+ `scripts/verify_gradients.py` the gradient check, `scripts/e2e_wheel.py` the clean-venv
99
+ install test (build ? fresh venv ? prepare ? train ? save adapter), `scripts/prep_data.py` the token shards
91
100
  (the installed equivalent is `streamft data`), and `pytest tests -q` the CPU suite.
92
101
 
93
102
  `tools/finetune.py` encodes the VRAM presets we measured, because the *combination* of
@@ -124,6 +133,22 @@ disk (NF4 safetensors)
124
133
  The training tag in the evidence log is still `exp54_disk_qlora`, so rows written by
125
134
  this engine and by the original experiment are directly comparable.
126
135
 
136
+ ## Older GPUs (GTX 10/16-series, 4–8 GB)
137
+
138
+ Pre-Ampere cards (GTX 1060/1660, Pascal/Turing) have **no bf16 units**. Use fp16:
139
+
140
+ ```bash
141
+ streamft train --model-dir checkpoints/qwen7b_nf4 --sft --steps 1000 \
142
+ --seq 512 --blocks-on-gpu 1 --compute-dtype fp16 --vram-hard-gb 5.3
143
+ ```
144
+
145
+ - On the reference Ampere card fp16 is slightly faster than bf16 at 7B (1.79 vs 1.91 s/step,
146
+ same 5.36 GB peak); on Pascal it is the only option.
147
+ - On a **4 GB** card use `--blocks-on-gpu 1`, `--seq 512` and `--vram-hard-gb 3.4`.
148
+ Those rows are not measured yet — if you run one, post the JSON row and it can become a
149
+ measured entry in `streamft fit`.
150
+ - `streamft fit` prints this hint automatically when it sees a pre-Ampere GPU.
151
+
127
152
  ## Verification
128
153
 
129
154
  `python scripts/verify_gradients.py --model-dir checkpoints/qwen3b_bnb4bit --seq 128`
@@ -49,15 +49,24 @@ streamft train --model-dir checkpoints/qwen0.5b_nf4 --sft --steps 20 --seq 512 -
49
49
  # LM path (optional): build token shards, then train without --sft
50
50
  streamft data --model Qwen/Qwen2.5-0.5B --out data
51
51
  streamft train --model-dir checkpoints/qwen0.5b_nf4 --steps 20 --seq 256 --tag lm
52
+
53
+ # no GPU at all, or no dataset: both are covered
54
+ streamft cpu --model Qwen/Qwen2.5-0.5B --steps 50 --seq 128
55
+ streamft synth --model Qwen/Qwen2.5-1.5B-Instruct --n 200 --out data/synth.jsonl
56
+
57
+ # inference budget (weights + KV cache), not just training
58
+ streamft fit --model Qwen/Qwen2.5-7B-Instruct --gpu 6 --seq 2048 --mode inference
52
59
  ```
53
60
 
54
61
  Every run appends one JSON row to `results/results.jsonl` next to where you launched it.
62
+ Six worked recipes: [`COOKBOOK.md`](COOKBOOK.md).
55
63
  The same commands are in [`QUICKSTART.md`](QUICKSTART.md), which is what a pip user should
56
64
  read — this README also documents the experiments behind the numbers.
57
65
 
58
66
  **From a source checkout only** (not part of the wheel): `tools/finetune.py` is a wrapper
59
67
  that encodes the VRAM presets, `scripts/compare_peft.py` reproduces the engine comparison,
60
- `scripts/verify_gradients.py` the gradient check, `scripts/prep_data.py` the token shards
68
+ `scripts/verify_gradients.py` the gradient check, `scripts/e2e_wheel.py` the clean-venv
69
+ install test (build ? fresh venv ? prepare ? train ? save adapter), `scripts/prep_data.py` the token shards
61
70
  (the installed equivalent is `streamft data`), and `pytest tests -q` the CPU suite.
62
71
 
63
72
  `tools/finetune.py` encodes the VRAM presets we measured, because the *combination* of
@@ -94,6 +103,22 @@ disk (NF4 safetensors)
94
103
  The training tag in the evidence log is still `exp54_disk_qlora`, so rows written by
95
104
  this engine and by the original experiment are directly comparable.
96
105
 
106
+ ## Older GPUs (GTX 10/16-series, 4–8 GB)
107
+
108
+ Pre-Ampere cards (GTX 1060/1660, Pascal/Turing) have **no bf16 units**. Use fp16:
109
+
110
+ ```bash
111
+ streamft train --model-dir checkpoints/qwen7b_nf4 --sft --steps 1000 \
112
+ --seq 512 --blocks-on-gpu 1 --compute-dtype fp16 --vram-hard-gb 5.3
113
+ ```
114
+
115
+ - On the reference Ampere card fp16 is slightly faster than bf16 at 7B (1.79 vs 1.91 s/step,
116
+ same 5.36 GB peak); on Pascal it is the only option.
117
+ - On a **4 GB** card use `--blocks-on-gpu 1`, `--seq 512` and `--vram-hard-gb 3.4`.
118
+ Those rows are not measured yet — if you run one, post the JSON row and it can become a
119
+ measured entry in `streamft fit`.
120
+ - `streamft fit` prints this hint automatically when it sees a pre-Ampere GPU.
121
+
97
122
  ## Verification
98
123
 
99
124
  `python scripts/verify_gradients.py --model-dir checkpoints/qwen3b_bnb4bit --seq 128`
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "streamft"
7
- version = "0.2.5"
7
+ version = "0.3.0"
8
8
  description = "Fine-tune models that do not fit: stream NF4 weights from disk, one transformer block at a time."
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.10"
@@ -4,6 +4,8 @@
4
4
  streamft prepare turn a bf16 model into a pre-quantized NF4 checkpoint
5
5
  streamft train the streaming trainer (all exp54 flags)
6
6
  streamft data build the token shards (LM path only; SFT needs none)
7
+ streamft cpu SFT a small model on the CPU (no GPU; model must fit RAM)
8
+ streamft synth generate training pairs locally with a small model
7
9
  streamft doctor environment report (torch, CUDA, bitsandbytes, VRAM)
8
10
  """
9
11
  from __future__ import annotations
@@ -16,6 +18,8 @@ USAGE = """streamft - fine-tune models that do not fit
16
18
  streamft prepare --model <dir> --save-quant checkpoints/<name>
17
19
  streamft train --model-dir checkpoints/<name> --sft --steps 30 [--seq 1024]
18
20
  streamft data --model <dir|hf-id> --out data
21
+ streamft cpu --model <small-model> --steps 20
22
+ streamft synth --model <small-model> --n 50 --out data/synth.jsonl
19
23
  streamft doctor
20
24
 
21
25
  Docs: https://github.com/byjameson/stream-finetune
@@ -35,6 +39,14 @@ def main() -> int:
35
39
 
36
40
  return fit_main(rest)
37
41
 
42
+ if cmd == "cpu":
43
+ from streamft.easy import cpu_main
44
+ return cpu_main(rest)
45
+
46
+ if cmd == "synth":
47
+ from streamft.easy import synth_main
48
+ return synth_main(rest)
49
+
38
50
  if cmd == "data":
39
51
  from streamft.data import main as data_main
40
52
 
@@ -219,6 +219,9 @@ def main() -> int:
219
219
  ap.add_argument("--lora_alpha", type=int, default=16)
220
220
  ap.add_argument("--targets", choices=["attn", "all"], default="attn")
221
221
  ap.add_argument("--head", choices=["bf16", "nf4"], default="bf16")
222
+ ap.add_argument("--compute-dtype", choices=["bf16", "fp16", "fp32"], default="bf16",
223
+ help="matmul precision for the 4-bit base; use fp16 on pre-Ampere GPUs "
224
+ "(GTX 10/16-series), which have no bf16 units")
222
225
  ap.add_argument("--shared-dtype", choices=["fp32", "as-stored"], default="fp32",
223
226
  help="fp32 matches prepare_model_for_kbit_training; 'as-stored' "
224
227
  "keeps embed/head in the checkpoint dtype (big models)")
@@ -261,10 +264,18 @@ def main() -> int:
261
264
  f"lora_r={args.lora_r} targets={args.targets} head={args.head}", flush=True)
262
265
 
263
266
  t0 = time.time()
267
+ compute_dtype = {"bf16": torch.bfloat16, "fp16": torch.float16,
268
+ "fp32": torch.float32}[args.compute_dtype]
269
+ if args.compute_dtype == "bf16" and torch.cuda.is_available():
270
+ major, _ = torch.cuda.get_device_capability(0)
271
+ if major < 8:
272
+ print(f"[warn] this GPU (compute capability {major}.x) has no bf16 units; "
273
+ f"use --compute-dtype fp16", flush=True)
264
274
  model, cfg = build_skeleton(
265
275
  model_dir, lora_r=args.lora_r, lora_alpha=args.lora_alpha,
266
276
  targets=args.targets, head=args.head,
267
277
  low_precision_shared=(args.shared_dtype == "as-stored"),
278
+ compute_dtype=compute_dtype,
268
279
  )
269
280
  store = DiskQuantStore(model_dir, device="cuda")
270
281
 
@@ -377,14 +388,23 @@ def main() -> int:
377
388
  tok = AutoTokenizer.from_pretrained(model_dir)
378
389
  if tok.pad_token is None:
379
390
  tok.pad_token = tok.eos_token
380
- ds = load_dataset(args.dataset, split="train").shuffle(seed=1234)
391
+ if str(args.dataset).lower().endswith(".jsonl"):
392
+ # local {"instruction","response"} file, e.g. from `streamft synth`
393
+ ds = load_dataset("json", data_files=args.dataset, split="train").shuffle(seed=1234)
394
+ else:
395
+ ds = load_dataset(args.dataset, split="train").shuffle(seed=1234)
381
396
  train_rows = [ds[i] for i in range(min(args.train_size, len(ds)))]
382
397
  eval_rows = [ds[len(train_rows) + i]
383
398
  for i in range(min(args.eval_size, len(ds) - len(train_rows)))]
384
399
  print(f"[sft] {Path(args.dataset).name} train={len(train_rows)} "
385
400
  f"eval={len(eval_rows)}", flush=True)
386
- ce0 = _masked_eval(tr, eval_rows, tok, args.seq)
387
- print(f"[eval] before loss={ce0:.4f}", flush=True)
401
+ if eval_rows:
402
+ ce0 = _masked_eval(tr, eval_rows, tok, args.seq)
403
+ print(f"[eval] before loss={ce0:.4f}", flush=True)
404
+ else:
405
+ ce0 = None
406
+ print(f"[sft] no held-out rows (dataset has {len(train_rows)} rows); skipping eval",
407
+ flush=True)
388
408
  # Filter once so `--steps N` means N optimizer updates, not "N attempts minus
389
409
  # whatever the mask rejected". A silently short run is a lie about the budget.
390
410
  valid_rows, skipped_train = sft_schedule(
@@ -408,8 +428,11 @@ def main() -> int:
408
428
  if (i + 1) % 10 == 0:
409
429
  print(f"[step] {i+1}/{args.steps} loss={loss:.4f} {dt:.2f}s "
410
430
  f"peak={peak:.2f}GB rss={host_rss_gb()}GB", flush=True)
411
- ce1 = _masked_eval(tr, eval_rows, tok, args.seq)
412
- print(f"[eval] after loss={ce1:.4f} (delta {ce1-ce0:+.4f})", flush=True)
431
+ if eval_rows:
432
+ ce1 = _masked_eval(tr, eval_rows, tok, args.seq)
433
+ print(f"[eval] after loss={ce1:.4f} (delta {ce1-ce0:+.4f})", flush=True)
434
+ else:
435
+ ce1 = None
413
436
  wall = time.time() - t_all
414
437
  else:
415
438
  tokens = load_tokens(str(user_path(args.data)))
@@ -465,6 +488,7 @@ def main() -> int:
465
488
  "lora_alpha": args.lora_alpha,
466
489
  "targets": args.targets,
467
490
  "head": args.head,
491
+ "compute_dtype": args.compute_dtype,
468
492
  "shared_dtype": args.shared_dtype,
469
493
  "blocks_mode": args.blocks_on_gpu,
470
494
  "blocks_on_gpu": cache.capacity,
@@ -83,7 +83,8 @@ class DiskQuantStore:
83
83
 
84
84
  def build_skeleton(model_dir, lora_r: int = 8, lora_alpha: int = 16,
85
85
  targets: str = "attn", head: str = "bf16",
86
- low_precision_shared: bool = False, with_lora: bool = True):
86
+ low_precision_shared: bool = False, with_lora: bool = True,
87
+ compute_dtype=None):
87
88
  """Instantiate the model on the `meta` device with bnb Linear4bit + LoRA.
88
89
 
89
90
  No weight memory is allocated here; only the (tiny) LoRA adapters are real.
@@ -104,10 +105,13 @@ def build_skeleton(model_dir, lora_r: int = 8, lora_alpha: int = 16,
104
105
  with torch.device("meta"):
105
106
  model = AutoModelForCausalLM.from_config(cfg)
106
107
 
108
+ # Older cards (GTX 10-series, Pascal/Turing) have no bf16 units at all; fp16 is the
109
+ # only half-precision option there. Everything else defaults to bf16.
110
+ compute_dtype = compute_dtype or torch.bfloat16
107
111
  qcfg = BitsAndBytesConfig(
108
112
  load_in_4bit=True,
109
113
  bnb_4bit_quant_type="nf4",
110
- bnb_4bit_compute_dtype=torch.bfloat16,
114
+ bnb_4bit_compute_dtype=compute_dtype,
111
115
  bnb_4bit_use_double_quant=True,
112
116
  bnb_4bit_quant_storage=torch.uint8,
113
117
  )
@@ -0,0 +1,226 @@
1
+ """Two on-ramps for people without a training GPU.
2
+
3
+ streamft cpu SFT a small model on the CPU (the model must fit in RAM)
4
+ streamft synth generate training pairs locally with a small model (no API, no data hunt)
5
+
6
+ Both are deliberately simple: plain transformers forward/backward, our LoRA module, the
7
+ same prompt masking as the GPU path. The streaming engine is not involved.
8
+ """
9
+ from __future__ import annotations
10
+
11
+ import argparse
12
+ import json
13
+ import random
14
+ import re
15
+ import time
16
+ from pathlib import Path
17
+
18
+ import torch
19
+
20
+ from .cli import _encode_chat, _format_messages # noqa: F401 (same masking rules)
21
+ from .lora import apply_lora, lora_parameters, lora_state_dict
22
+
23
+
24
+ # --------------------------------------------------------------------------- cpu train
25
+
26
+ def _load_tiny(model_dir: str):
27
+ from transformers import AutoModelForCausalLM
28
+
29
+ dtype = torch.float32 # CPU: bf16 is slow and fp16 is unsupported on many CPUs
30
+ model = AutoModelForCausalLM.from_pretrained(model_dir, torch_dtype=dtype)
31
+ model.eval()
32
+ return model
33
+
34
+
35
+ @torch.no_grad()
36
+ def _eval_masked(model, rows, tok, seq: int) -> float:
37
+ model.eval()
38
+ total, n = 0.0, 0
39
+ for row in rows:
40
+ enc = _encode_chat(tok, row, seq)
41
+ if enc is None:
42
+ continue
43
+ ids, labels = enc
44
+ loss = model(input_ids=ids, labels=labels).loss
45
+ v = int((labels != -100).sum())
46
+ total += float(loss) * v
47
+ n += v
48
+ model.train()
49
+ if n == 0:
50
+ raise RuntimeError("no eval row has a valid prompt mask")
51
+ return total / n
52
+
53
+
54
+ def cpu_main(argv=None) -> int:
55
+ from datasets import load_dataset
56
+ from transformers import AutoTokenizer
57
+
58
+ ap = argparse.ArgumentParser(prog="streamft cpu")
59
+ ap.add_argument("--model", required=True, help="small model dir or HF id (fits in RAM)")
60
+ ap.add_argument("--dataset", default="databricks/databricks-dolly-15k")
61
+ ap.add_argument("--train_size", type=int, default=128)
62
+ ap.add_argument("--eval_size", type=int, default=8)
63
+ ap.add_argument("--steps", type=int, default=20)
64
+ ap.add_argument("--seq", type=int, default=256)
65
+ ap.add_argument("--lr", type=float, default=3e-4)
66
+ ap.add_argument("--lora_r", type=int, default=8)
67
+ ap.add_argument("--save", default="")
68
+ ap.add_argument("--tag", default="cpu_sft")
69
+ args = ap.parse_args(argv)
70
+
71
+ tok = AutoTokenizer.from_pretrained(args.model)
72
+ if tok.pad_token is None:
73
+ tok.pad_token = tok.eos_token
74
+ t0 = time.time()
75
+ model = _load_tiny(args.model)
76
+ apply_lora(model, r_attn=args.lora_r, r_mlp=0)
77
+ params = list(lora_parameters(model))
78
+ opt = torch.optim.AdamW(params, lr=args.lr)
79
+ print(f"[cpu] {args.model} loaded in {time.time()-t0:.1f}s, "
80
+ f"{sum(p.numel() for p in params)/1e6:.2f}M adapter params", flush=True)
81
+
82
+ ds = load_dataset(args.dataset, split="train").shuffle(seed=1234)
83
+ train_rows = [ds[i] for i in range(min(args.train_size, len(ds)))]
84
+ eval_rows = [ds[len(train_rows) + i]
85
+ for i in range(min(args.eval_size, len(ds) - len(train_rows)))]
86
+
87
+ valid = [r for r in train_rows if _encode_chat(tok, r, args.seq) is not None]
88
+ dropped = len(train_rows) - len(valid)
89
+ if not valid:
90
+ raise SystemExit("[cpu] no training row has a valid prompt mask")
91
+ if dropped:
92
+ print(f"[cpu] {dropped}/{len(train_rows)} rows dropped (prompt not a prefix)",
93
+ flush=True)
94
+
95
+ ce0 = _eval_masked(model, eval_rows, tok, args.seq)
96
+ print(f"[cpu] eval before {ce0:.4f}", flush=True)
97
+
98
+ model.train()
99
+ losses, times = [], []
100
+ for i in range(args.steps):
101
+ enc = _encode_chat(tok, valid[i % len(valid)], args.seq)
102
+ ids, labels = enc
103
+ t = time.time()
104
+ loss = model(input_ids=ids, labels=labels).loss
105
+ loss.backward()
106
+ opt.step()
107
+ opt.zero_grad(set_to_none=True)
108
+ losses.append(round(float(loss.detach()), 6))
109
+ times.append(round(time.time() - t, 3))
110
+ if (i + 1) % 5 == 0 or i == 0:
111
+ print(f"[cpu] step {i+1}/{args.steps} loss={losses[-1]:.4f} "
112
+ f"{times[-1]:.2f}s", flush=True)
113
+
114
+ ce1 = _eval_masked(model, eval_rows, tok, args.seq)
115
+ if args.save:
116
+ out = Path(args.save)
117
+ out.mkdir(parents=True, exist_ok=True)
118
+ torch.save(lora_state_dict(model), str(out / "adapter.pt"))
119
+ row = {
120
+ "engine": "cpu-dense", "tag": args.tag, "model": args.model, "steps": len(losses),
121
+ "seq": args.seq, "lr": args.lr, "lora_r": args.lora_r,
122
+ "step_s_med": sorted(times)[len(times) // 2] if times else None,
123
+ "ce_before": round(ce0, 5), "ce_after": round(ce1, 5),
124
+ "ce_delta": round(ce1 - ce0, 5), "losses": losses,
125
+ "sft_skipped": dropped, "saved": args.save or None,
126
+ "license": "Apache-2.0", "device": "cpu",
127
+ }
128
+ print(json.dumps(row, ensure_ascii=True), flush=True)
129
+ out = Path("results")
130
+ out.mkdir(exist_ok=True)
131
+ with open(out / "results.jsonl", "a", encoding="utf-8") as f:
132
+ f.write(json.dumps(row) + "\n")
133
+ return 0
134
+
135
+
136
+ # --------------------------------------------------------------------------- synth
137
+
138
+ PROMPT = """You write training data. Produce ONE example as JSON with keys
139
+ "instruction" and "response". Topic: {task}.
140
+ Rules: the instruction is a short request; the response is a direct answer of 1-3
141
+ sentences; no markdown, no preamble, no extra keys.
142
+ JSON:"""
143
+
144
+
145
+ PLACEHOLDER_MARKERS = ("<instruction>", "<response>", "<your", "your instruction",
146
+ "lorem ipsum", "example.com")
147
+
148
+
149
+ def parse_pair(text: str):
150
+ """Extract one {"instruction","response"} object; reject template junk.
151
+
152
+ Small models happily echo the schema back ("<instruction>"), and a single such row
153
+ poisons a dataset. Everything is filtered here so the caller only has to dedupe.
154
+ """
155
+ m = re.search(r"\{.*\}", text, re.S)
156
+ if not m:
157
+ return None
158
+ try:
159
+ obj = json.loads(m.group(0))
160
+ except Exception:
161
+ return None
162
+ if not isinstance(obj, dict):
163
+ return None
164
+ ins = str(obj.get("instruction", "")).strip()
165
+ res = str(obj.get("response", "")).strip()
166
+ low = (ins + " " + res).lower()
167
+ if any(p in low for p in PLACEHOLDER_MARKERS):
168
+ return None
169
+ if "{" in ins or "{" in res or "}" in ins or "}" in res:
170
+ return None
171
+ if not (5 <= len(ins) <= 200 and 10 <= len(res) <= 1200):
172
+ return None
173
+ return {"instruction": ins, "response": res}
174
+
175
+
176
+ def synth_main(argv=None) -> int:
177
+ from transformers import AutoModelForCausalLM, AutoTokenizer
178
+
179
+ ap = argparse.ArgumentParser(prog="streamft synth")
180
+ ap.add_argument("--model", default="Qwen/Qwen2.5-1.5B-Instruct")
181
+ ap.add_argument("--out", default="data/synth.jsonl")
182
+ ap.add_argument("--n", type=int, default=50, help="examples to attempt")
183
+ ap.add_argument("--task", default="practical everyday knowledge")
184
+ ap.add_argument("--seq", type=int, default=256)
185
+ ap.add_argument("--temperature", type=float, default=0.9)
186
+ ap.add_argument("--device", default="cuda" if torch.cuda.is_available() else "cpu")
187
+ args = ap.parse_args(argv)
188
+
189
+ tok = AutoTokenizer.from_pretrained(args.model)
190
+ model = AutoModelForCausalLM.from_pretrained(
191
+ args.model,
192
+ torch_dtype=torch.bfloat16 if args.device == "cuda" else torch.float32,
193
+ ).to(args.device).eval()
194
+
195
+ seen, kept = set(), []
196
+ out = Path(args.out)
197
+ out.parent.mkdir(parents=True, exist_ok=True)
198
+ t0 = time.time()
199
+ for i in range(args.n):
200
+ prompt = PROMPT.format(task=args.task)
201
+ ids = tok(prompt, return_tensors="pt").to(args.device)
202
+ with torch.no_grad():
203
+ gen = model.generate(**ids, max_new_tokens=args.seq, do_sample=True,
204
+ temperature=args.temperature, top_p=0.95,
205
+ pad_token_id=tok.eos_token_id)
206
+ text = tok.decode(gen[0][ids["input_ids"].shape[1]:], skip_special_tokens=True)
207
+ pair = parse_pair(text)
208
+ if pair is None:
209
+ continue
210
+ key = pair["instruction"].lower()
211
+ if key in seen:
212
+ continue
213
+ seen.add(key)
214
+ kept.append(pair)
215
+ if (len(kept) % 10) == 0:
216
+ print(f"[synth] {len(kept)} kept / {i+1} attempts", flush=True)
217
+
218
+ with open(out, "w", encoding="utf-8") as f:
219
+ for r in kept:
220
+ f.write(json.dumps(r, ensure_ascii=False) + "\n")
221
+ print(json.dumps({
222
+ "out": str(out), "attempts": args.n, "kept": len(kept),
223
+ "keep_rate": round(len(kept) / max(args.n, 1), 3),
224
+ "seconds": round(time.time() - t0, 1), "model": args.model, "task": args.task,
225
+ }, indent=1))
226
+ return 0
@@ -20,6 +20,8 @@ import argparse
20
20
  import json
21
21
  from pathlib import Path
22
22
 
23
+ import torch
24
+
23
25
  # Measured end-to-end on an RTX 3050 Laptop, 6 GB, Windows 11, AC power, NF4 base,
24
26
  # LoRA r=8 on q/k/v/o, batch 1, Dolly-15k. step_s = median warm step.
25
27
  MEASURED = [
@@ -51,6 +53,16 @@ def load_config(model: str) -> dict:
51
53
  return json.loads(Path(hf_hub_download(model, "config.json")).read_text(encoding="utf-8"))
52
54
 
53
55
 
56
+ def kv_cache_gb(cfg: dict, seq: int, batch: int = 1, bytes_per: int = 2) -> float:
57
+ """KV cache at inference: 2 tensors x layers x kv_heads x head_dim x seq x batch."""
58
+ h = int(cfg.get("hidden_size") or cfg.get("n_embd"))
59
+ layers = int(cfg.get("num_hidden_layers") or cfg.get("n_layer"))
60
+ heads = int(cfg.get("num_attention_heads") or 1)
61
+ kv_heads = int(cfg.get("num_key_value_heads") or heads)
62
+ head_dim = h // max(heads, 1)
63
+ return 2.0 * layers * kv_heads * head_dim * seq * batch * bytes_per / 1e9
64
+
65
+
54
66
  def model_fingerprint(cfg: dict) -> tuple:
55
67
  """Everything that changes the measurement: two 7B models are not interchangeable."""
56
68
  return (
@@ -93,6 +105,8 @@ def main(argv: list[str] | None = None) -> int:
93
105
  ap.add_argument("--model", required=True, help="local dir or HF id")
94
106
  ap.add_argument("--gpu", type=float, default=6.0, help="VRAM in GB (default 6)")
95
107
  ap.add_argument("--seq", type=int, default=512)
108
+ ap.add_argument("--mode", choices=["train", "inference"], default="train")
109
+ ap.add_argument("--batch", type=int, default=1)
96
110
  ap.add_argument("--json", action="store_true")
97
111
  args = ap.parse_args(argv)
98
112
 
@@ -124,6 +138,27 @@ def main(argv: list[str] | None = None) -> int:
124
138
  "measured_on_rtx3050_6gb": measured,
125
139
  "measured_elsewhere": measured is None,
126
140
  }
141
+ if args.mode == "inference":
142
+ kv_gb = kv_cache_gb(cfg, args.seq, args.batch)
143
+ total = m["nf4_gb"] + m["head_gb"] + kv_gb + 0.5
144
+ out_inf = {
145
+ "model": args.model, "mode": "inference", "seq": args.seq, "batch": args.batch,
146
+ "weights_gb": round(m["nf4_gb"] + m["head_gb"], 1),
147
+ "kv_cache_gb": round(kv_gb, 2),
148
+ "needed_gb": round(total, 1),
149
+ "fits_whole": total <= args.gpu,
150
+ "speed": "not measured for this model",
151
+ }
152
+ if args.json:
153
+ print(json.dumps(out_inf, indent=1))
154
+ return 0
155
+ print(f"{args.model} (inference)")
156
+ print(f" weights ~{out_inf['weights_gb']} GB + KV cache ~{out_inf['kv_cache_gb']} GB "
157
+ f"at seq {args.seq}, batch {args.batch} -> ~{out_inf['needed_gb']} GB")
158
+ print(f" verdict: {'fits' if out_inf['fits_whole'] else 'does NOT fit'} on {args.gpu:g} GB")
159
+ print(" speed: not measured for this model (the planner does not guess tok/s)")
160
+ return 0
161
+
127
162
  if args.json:
128
163
  print(json.dumps(out, indent=1))
129
164
  return 0
@@ -149,6 +184,14 @@ def main(argv: list[str] | None = None) -> int:
149
184
  print(" speed/blocks: not measured for this size at this seq — start with")
150
185
  print(" --blocks-on-gpu 1 and let autotune grow it (it measures")
151
186
  print(" the real MB/block on your card instead of guessing)")
187
+ try:
188
+ if torch.cuda.is_available():
189
+ major, _minor = torch.cuda.get_device_capability(0)
190
+ if major < 8:
191
+ print(" note: this GPU has no bf16 units (pre-Ampere) -> train with "
192
+ "--compute-dtype fp16")
193
+ except Exception:
194
+ pass
152
195
  print(" next: streamft prepare --model <dir> --save-quant checkpoints/<name>")
153
196
  print(" streamft train --model-dir checkpoints/<name> --sft --steps 30")
154
197
  return 0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: streamft
3
- Version: 0.2.5
3
+ Version: 0.3.0
4
4
  Summary: Fine-tune models that do not fit: stream NF4 weights from disk, one transformer block at a time.
5
5
  Author-email: Gokhan Cava <byjameson@users.noreply.github.com>
6
6
  License: Apache-2.0
@@ -79,15 +79,24 @@ streamft train --model-dir checkpoints/qwen0.5b_nf4 --sft --steps 20 --seq 512 -
79
79
  # LM path (optional): build token shards, then train without --sft
80
80
  streamft data --model Qwen/Qwen2.5-0.5B --out data
81
81
  streamft train --model-dir checkpoints/qwen0.5b_nf4 --steps 20 --seq 256 --tag lm
82
+
83
+ # no GPU at all, or no dataset: both are covered
84
+ streamft cpu --model Qwen/Qwen2.5-0.5B --steps 50 --seq 128
85
+ streamft synth --model Qwen/Qwen2.5-1.5B-Instruct --n 200 --out data/synth.jsonl
86
+
87
+ # inference budget (weights + KV cache), not just training
88
+ streamft fit --model Qwen/Qwen2.5-7B-Instruct --gpu 6 --seq 2048 --mode inference
82
89
  ```
83
90
 
84
91
  Every run appends one JSON row to `results/results.jsonl` next to where you launched it.
92
+ Six worked recipes: [`COOKBOOK.md`](COOKBOOK.md).
85
93
  The same commands are in [`QUICKSTART.md`](QUICKSTART.md), which is what a pip user should
86
94
  read — this README also documents the experiments behind the numbers.
87
95
 
88
96
  **From a source checkout only** (not part of the wheel): `tools/finetune.py` is a wrapper
89
97
  that encodes the VRAM presets, `scripts/compare_peft.py` reproduces the engine comparison,
90
- `scripts/verify_gradients.py` the gradient check, `scripts/prep_data.py` the token shards
98
+ `scripts/verify_gradients.py` the gradient check, `scripts/e2e_wheel.py` the clean-venv
99
+ install test (build ? fresh venv ? prepare ? train ? save adapter), `scripts/prep_data.py` the token shards
91
100
  (the installed equivalent is `streamft data`), and `pytest tests -q` the CPU suite.
92
101
 
93
102
  `tools/finetune.py` encodes the VRAM presets we measured, because the *combination* of
@@ -124,6 +133,22 @@ disk (NF4 safetensors)
124
133
  The training tag in the evidence log is still `exp54_disk_qlora`, so rows written by
125
134
  this engine and by the original experiment are directly comparable.
126
135
 
136
+ ## Older GPUs (GTX 10/16-series, 4–8 GB)
137
+
138
+ Pre-Ampere cards (GTX 1060/1660, Pascal/Turing) have **no bf16 units**. Use fp16:
139
+
140
+ ```bash
141
+ streamft train --model-dir checkpoints/qwen7b_nf4 --sft --steps 1000 \
142
+ --seq 512 --blocks-on-gpu 1 --compute-dtype fp16 --vram-hard-gb 5.3
143
+ ```
144
+
145
+ - On the reference Ampere card fp16 is slightly faster than bf16 at 7B (1.79 vs 1.91 s/step,
146
+ same 5.36 GB peak); on Pascal it is the only option.
147
+ - On a **4 GB** card use `--blocks-on-gpu 1`, `--seq 512` and `--vram-hard-gb 3.4`.
148
+ Those rows are not measured yet — if you run one, post the JSON row and it can become a
149
+ measured entry in `streamft fit`.
150
+ - `streamft fit` prints this hint automatically when it sees a pre-Ampere GPU.
151
+
127
152
  ## Verification
128
153
 
129
154
  `python scripts/verify_gradients.py --model-dir checkpoints/qwen3b_bnb4bit --seq 128`
@@ -8,6 +8,7 @@ streamft/cli.py
8
8
  streamft/data.py
9
9
  streamft/disk_quant_store.py
10
10
  streamft/doctor.py
11
+ streamft/easy.py
11
12
  streamft/fit.py
12
13
  streamft/lora.py
13
14
  streamft/metrics.py
@@ -248,3 +248,26 @@ def test_masked_eval_raises_when_every_row_is_skipped(monkeypatch):
248
248
  monkeypatch.setattr(cli, "_encode_chat", lambda *a, **k: None)
249
249
  with pytest.raises(RuntimeError, match="every row was skipped"):
250
250
  cli._masked_eval(FakeTrainer(), [{"instruction": "x", "response": "y"}], object(), 8)
251
+
252
+
253
+ # ------------------------------------------------ synth / planner helpers
254
+
255
+ def test_parse_pair_rejects_template_junk():
256
+ from streamft.easy import parse_pair
257
+
258
+ good = parse_pair('{"instruction": "Give me a tip", "response": "Sharpen your knives."}')
259
+ assert good and good["instruction"] == "Give me a tip"
260
+ assert parse_pair('{"instruction": "<instruction>", "response": "<response>"}') is None
261
+ assert parse_pair('{"instruction": "x", "response": "short"}') is None
262
+ assert parse_pair("not json") is None
263
+ assert parse_pair('{"instruction": "a" * 300, "response": "b" * 20}') is None
264
+
265
+
266
+ def test_kv_cache_matches_hand_calculation():
267
+ from streamft.fit import kv_cache_gb
268
+
269
+ cfg = {"hidden_size": 2048, "num_hidden_layers": 24, "num_attention_heads": 16,
270
+ "num_key_value_heads": 2}
271
+ # 2 * 24 layers * 2 kv heads * 128 head_dim * 1024 seq * 2 bytes
272
+ expect = 2 * 24 * 2 * 128 * 1024 * 2 / 1e9
273
+ assert abs(kv_cache_gb(cfg, 1024) - expect) < 1e-12
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes