streamft 0.3.0__tar.gz → 0.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (26) hide show
  1. {streamft-0.3.0 → streamft-0.4.0}/PKG-INFO +8 -3
  2. {streamft-0.3.0 → streamft-0.4.0}/README.md +7 -2
  3. {streamft-0.3.0 → streamft-0.4.0}/pyproject.toml +1 -1
  4. {streamft-0.3.0 → streamft-0.4.0}/streamft/__main__.py +6 -0
  5. streamft-0.4.0/streamft/evals.py +148 -0
  6. {streamft-0.3.0 → streamft-0.4.0}/streamft.egg-info/PKG-INFO +8 -3
  7. {streamft-0.3.0 → streamft-0.4.0}/streamft.egg-info/SOURCES.txt +1 -0
  8. {streamft-0.3.0 → streamft-0.4.0}/tests/test_streamft_smoke.py +8 -0
  9. {streamft-0.3.0 → streamft-0.4.0}/LICENSE +0 -0
  10. {streamft-0.3.0 → streamft-0.4.0}/setup.cfg +0 -0
  11. {streamft-0.3.0 → streamft-0.4.0}/streamft/__init__.py +0 -0
  12. {streamft-0.3.0 → streamft-0.4.0}/streamft/autotune.py +0 -0
  13. {streamft-0.3.0 → streamft-0.4.0}/streamft/cli.py +0 -0
  14. {streamft-0.3.0 → streamft-0.4.0}/streamft/data.py +0 -0
  15. {streamft-0.3.0 → streamft-0.4.0}/streamft/disk_quant_store.py +0 -0
  16. {streamft-0.3.0 → streamft-0.4.0}/streamft/doctor.py +0 -0
  17. {streamft-0.3.0 → streamft-0.4.0}/streamft/easy.py +0 -0
  18. {streamft-0.3.0 → streamft-0.4.0}/streamft/fit.py +0 -0
  19. {streamft-0.3.0 → streamft-0.4.0}/streamft/lora.py +0 -0
  20. {streamft-0.3.0 → streamft-0.4.0}/streamft/metrics.py +0 -0
  21. {streamft-0.3.0 → streamft-0.4.0}/streamft/stream_quant_lora.py +0 -0
  22. {streamft-0.3.0 → streamft-0.4.0}/streamft/stream_train.py +0 -0
  23. {streamft-0.3.0 → streamft-0.4.0}/streamft.egg-info/dependency_links.txt +0 -0
  24. {streamft-0.3.0 → streamft-0.4.0}/streamft.egg-info/entry_points.txt +0 -0
  25. {streamft-0.3.0 → streamft-0.4.0}/streamft.egg-info/requires.txt +0 -0
  26. {streamft-0.3.0 → streamft-0.4.0}/streamft.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: streamft
3
- Version: 0.3.0
3
+ Version: 0.4.0
4
4
  Summary: Fine-tune models that do not fit: stream NF4 weights from disk, one transformer block at a time.
5
5
  Author-email: Gokhan Cava <byjameson@users.noreply.github.com>
6
6
  License: Apache-2.0
@@ -53,9 +53,11 @@ Measured on one machine — **RTX 3050 Laptop, 6 GB VRAM, 34 GB RAM, NVMe, Windo
53
53
 
54
54
  The 32B weights are **19.2 GB**. They never fully live in VRAM. At 1024 tokens/step the
55
55
  throughput is **3.1×** the 128-token rate, because the per-step I/O is amortised over
56
- more tokens. 50 steps of Dolly-15k SFT moved eval loss **1.8755 → 0.9661**.
56
+ more tokens. 50 steps of Dolly-15k SFT moved eval loss **1.8755 → 0.9659**.
57
57
 
58
- **Installed from PyPI? Start with [`QUICKSTART.md`](QUICKSTART.md)** — it uses only the
58
+ **Installed from PyPI? Start with [`QUICKSTART.md`](QUICKSTART.md) (T?rk?e:
59
+ [`QUICKSTART.tr.md`](QUICKSTART.tr.md)) ? or run `install.ps1` / `install.sh`, which
60
+ checks Python, installs the package, looks for a GPU and runs `doctor`.** — it uses only the
59
61
  published CLI (no repo scripts). Deeper dive: [`GUIDE_6GB_32B.md`](GUIDE_6GB_32B.md) — the practical write-up: presets,
60
62
  the six traps, and which numbers are real. The measurement protocol, the discarded rows
61
63
  and the correction of an earlier wrong claim ("8.6× / 100 tok/s") are in
@@ -86,6 +88,9 @@ streamft synth --model Qwen/Qwen2.5-1.5B-Instruct --n 200 --out data/synth.jsonl
86
88
 
87
89
  # inference budget (weights + KV cache), not just training
88
90
  streamft fit --model Qwen/Qwen2.5-7B-Instruct --gpu 6 --seq 2048 --mode inference
91
+
92
+ # does it actually work? 20 local prompts, no API and no judge model
93
+ streamft eval --model Qwen/Qwen2.5-1.5B-Instruct
89
94
  ```
90
95
 
91
96
  Every run appends one JSON row to `results/results.jsonl` next to where you launched it.
@@ -23,9 +23,11 @@ Measured on one machine — **RTX 3050 Laptop, 6 GB VRAM, 34 GB RAM, NVMe, Windo
23
23
 
24
24
  The 32B weights are **19.2 GB**. They never fully live in VRAM. At 1024 tokens/step the
25
25
  throughput is **3.1×** the 128-token rate, because the per-step I/O is amortised over
26
- more tokens. 50 steps of Dolly-15k SFT moved eval loss **1.8755 → 0.9661**.
26
+ more tokens. 50 steps of Dolly-15k SFT moved eval loss **1.8755 → 0.9659**.
27
27
 
28
- **Installed from PyPI? Start with [`QUICKSTART.md`](QUICKSTART.md)** — it uses only the
28
+ **Installed from PyPI? Start with [`QUICKSTART.md`](QUICKSTART.md) (T?rk?e:
29
+ [`QUICKSTART.tr.md`](QUICKSTART.tr.md)) ? or run `install.ps1` / `install.sh`, which
30
+ checks Python, installs the package, looks for a GPU and runs `doctor`.** — it uses only the
29
31
  published CLI (no repo scripts). Deeper dive: [`GUIDE_6GB_32B.md`](GUIDE_6GB_32B.md) — the practical write-up: presets,
30
32
  the six traps, and which numbers are real. The measurement protocol, the discarded rows
31
33
  and the correction of an earlier wrong claim ("8.6× / 100 tok/s") are in
@@ -56,6 +58,9 @@ streamft synth --model Qwen/Qwen2.5-1.5B-Instruct --n 200 --out data/synth.jsonl
56
58
 
57
59
  # inference budget (weights + KV cache), not just training
58
60
  streamft fit --model Qwen/Qwen2.5-7B-Instruct --gpu 6 --seq 2048 --mode inference
61
+
62
+ # does it actually work? 20 local prompts, no API and no judge model
63
+ streamft eval --model Qwen/Qwen2.5-1.5B-Instruct
59
64
  ```
60
65
 
61
66
  Every run appends one JSON row to `results/results.jsonl` next to where you launched it.
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "streamft"
7
- version = "0.3.0"
7
+ version = "0.4.0"
8
8
  description = "Fine-tune models that do not fit: stream NF4 weights from disk, one transformer block at a time."
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.10"
@@ -6,6 +6,7 @@
6
6
  streamft data build the token shards (LM path only; SFT needs none)
7
7
  streamft cpu SFT a small model on the CPU (no GPU; model must fit RAM)
8
8
  streamft synth generate training pairs locally with a small model
9
+ streamft eval run a small local prompt set (repetition/refusal/keywords)
9
10
  streamft doctor environment report (torch, CUDA, bitsandbytes, VRAM)
10
11
  """
11
12
  from __future__ import annotations
@@ -20,6 +21,7 @@ USAGE = """streamft - fine-tune models that do not fit
20
21
  streamft data --model <dir|hf-id> --out data
21
22
  streamft cpu --model <small-model> --steps 20
22
23
  streamft synth --model <small-model> --n 50 --out data/synth.jsonl
24
+ streamft eval --model <model> [--prompts mine.jsonl]
23
25
  streamft doctor
24
26
 
25
27
  Docs: https://github.com/byjameson/stream-finetune
@@ -43,6 +45,10 @@ def main() -> int:
43
45
  from streamft.easy import cpu_main
44
46
  return cpu_main(rest)
45
47
 
48
+ if cmd == "eval":
49
+ from streamft.evals import main as eval_main
50
+ return eval_main(rest)
51
+
46
52
  if cmd == "synth":
47
53
  from streamft.easy import synth_main
48
54
  return synth_main(rest)
@@ -0,0 +1,148 @@
1
+ """`streamft eval` — does the model actually do the thing? No API, no judge model.
2
+
3
+ It runs a small prompt set locally and reports what can be measured without a second
4
+ model: repetition, refusal-phrase hits, length, latency, and optionally keyword adherence
5
+ when the prompt file carries an `expect` field.
6
+
7
+ streamft eval --model Qwen/Qwen2.5-1.5B-Instruct
8
+ streamft eval --model ./merged --prompts my_prompts.jsonl --max-new 96
9
+
10
+ Prompt file format (JSONL, one object per line):
11
+ {"prompt": "...", "expect": ["optional", "keywords"]}
12
+ """
13
+ from __future__ import annotations
14
+
15
+ import argparse
16
+ import json
17
+ import re
18
+ import time
19
+ from collections import Counter
20
+
21
+ import torch
22
+
23
+ # Ten English + ten Turkish prompts. Deliberately mundane: this is a smoke test, not a
24
+ # benchmark. `expect` is a keyword that should appear if the model understood the task.
25
+ BUILTIN = [
26
+ {"prompt": "List three uses for a paperclip.", "expect": []},
27
+ {"prompt": "Explain in one sentence why the sky is blue.", "expect": ["light", "scatter"]},
28
+ {"prompt": "Give one reason to keep a food diary.", "expect": []},
29
+ {"prompt": "Write a two-line rhyme about rain.", "expect": []},
30
+ {"prompt": "What is 17 multiplied by 6?", "expect": ["102"]},
31
+ {"prompt": "Name two vegetables that grow underground.", "expect": ["carrot", "potato"]},
32
+ {"prompt": "Summarise the plot of a film about a lost dog in one sentence.", "expect": []},
33
+ {"prompt": "Translate 'good morning' into Turkish.", "expect": ["günaydın", "gunaydin"]},
34
+ {"prompt": "Give a one-sentence tip for better sleep.", "expect": []},
35
+ {"prompt": "What does a red traffic light mean?", "expect": ["stop"]},
36
+ {"prompt": "Bir cümlede su neden buharlaşır açıkla.", "expect": ["ısı", "isi", "sıcaklık"]},
37
+ {"prompt": "Üç tane meyve say.", "expect": ["elma", "muz", "portakal", "çilek"]},
38
+ {"prompt": "17 çarpı 6 kaç eder?", "expect": ["102"]},
39
+ {"prompt": "Yağmurlu bir gün için iki satırlık bir dörtlük yaz.", "expect": []},
40
+ {"prompt": "İyi bir uyku için tek cümlelik bir öneri ver.", "expect": []},
41
+ {"prompt": "Kırmızı trafik ışığı ne anlama gelir?", "expect": ["dur"]},
42
+ {"prompt": "Sabah kahvaltısı için iki öneri ver.", "expect": []},
43
+ {"prompt": "Bir cümlede neden kitap okumalıyız açıkla.", "expect": []},
44
+ {"prompt": "Türkiye'nin başkenti neresidir?", "expect": ["ankara"]},
45
+ {"prompt": "Bir cümlede bisikletin avantajını yaz.", "expect": []},
46
+ ]
47
+
48
+ REFUSAL = [
49
+ "i can't help", "i cannot help", "i can't assist", "i cannot assist",
50
+ "as an ai", "i'm sorry, but", "i am sorry, but", "cannot provide",
51
+ "yardımcı olamam", "yardimci olamam", "üzgünüm", "uzgunum", "olarak yapay zeka",
52
+ ]
53
+
54
+
55
+ def repetition_rate(text: str, n: int = 4) -> float:
56
+ """Fraction of repeated n-grams (1.0 = every n-gram is a duplicate)."""
57
+ words = re.findall(r"\w+", text.lower())
58
+ if len(words) < n + 1:
59
+ return 0.0
60
+ grams = [tuple(words[i:i + n]) for i in range(len(words) - n + 1)]
61
+ counts = Counter(grams)
62
+ repeated = sum(c - 1 for c in counts.values() if c > 1)
63
+ return repeated / len(grams)
64
+
65
+
66
+ def main(argv=None) -> int:
67
+ from transformers import AutoModelForCausalLM, AutoTokenizer
68
+
69
+ ap = argparse.ArgumentParser(prog="streamft eval")
70
+ ap.add_argument("--model", default="Qwen/Qwen2.5-1.5B-Instruct")
71
+ ap.add_argument("--prompts", default="", help="JSONL with {'prompt', 'expect'?}")
72
+ ap.add_argument("--max-new", type=int, default=64)
73
+ ap.add_argument("--limit", type=int, default=0, help="evaluate only the first N prompts")
74
+ ap.add_argument("--out", default="results/eval.jsonl")
75
+ ap.add_argument("--device", default="cuda" if torch.cuda.is_available() else "cpu")
76
+ ap.add_argument("--quiet", action="store_true")
77
+ args = ap.parse_args(argv)
78
+
79
+ prompts = BUILTIN
80
+ if args.prompts:
81
+ with open(args.prompts, encoding="utf-8") as f:
82
+ prompts = [json.loads(line) for line in f if line.strip()]
83
+ if args.limit:
84
+ prompts = prompts[: args.limit]
85
+
86
+ dtype = torch.bfloat16 if args.device == "cuda" else torch.float32
87
+ tok = AutoTokenizer.from_pretrained(args.model)
88
+ model = AutoModelForCausalLM.from_pretrained(args.model, torch_dtype=dtype).to(args.device)
89
+ model.eval()
90
+
91
+ rows = []
92
+ t_all = time.time()
93
+ for i, item in enumerate(prompts):
94
+ text = item["prompt"]
95
+ if getattr(tok, "chat_template", None):
96
+ text = tok.apply_chat_template([{"role": "user", "content": text}],
97
+ tokenize=False, add_generation_prompt=True)
98
+ ids = tok(text, return_tensors="pt").to(args.device)
99
+ t0 = time.time()
100
+ with torch.no_grad():
101
+ out = model.generate(**ids, max_new_tokens=args.max_new, do_sample=False,
102
+ pad_token_id=tok.eos_token_id)
103
+ dt = time.time() - t0
104
+ gen = out[0][ids["input_ids"].shape[1]:]
105
+ answer = tok.decode(gen, skip_special_tokens=True).strip()
106
+ low = answer.lower()
107
+ row = {
108
+ "i": i, "prompt": item["prompt"], "answer": answer,
109
+ "tokens": int(gen.shape[0]), "seconds": round(dt, 2),
110
+ "tok_s": round(int(gen.shape[0]) / max(dt, 1e-6), 1),
111
+ "rep4": round(repetition_rate(answer), 3),
112
+ "refusal": any(p in low for p in REFUSAL),
113
+ "expect_hit": (None if not item.get("expect")
114
+ else any(k.lower() in low for k in item["expect"])),
115
+ }
116
+ rows.append(row)
117
+ if not args.quiet:
118
+ flag = "REFUSAL" if row["refusal"] else ("miss" if row["expect_hit"] is False else "")
119
+ print(f"[{i+1}/{len(prompts)}] {row['tok_s']:.1f} tok/s rep4={row['rep4']:.2f} {flag}")
120
+ print(" " + answer.replace("\n", " ")[:160])
121
+
122
+ n = len(rows)
123
+ expected = [r for r in rows if r["expect_hit"] is not None]
124
+ summary = {
125
+ "model": args.model, "n": n,
126
+ "refusal_rate": round(sum(r["refusal"] for r in rows) / max(n, 1), 3),
127
+ "mean_rep4": round(sum(r["rep4"] for r in rows) / max(n, 1), 3),
128
+ "mean_tokens": round(sum(r["tokens"] for r in rows) / max(n, 1), 1),
129
+ "mean_tok_s": round(sum(r["tok_s"] for r in rows) / max(n, 1), 1),
130
+ "keyword_hit_rate": (round(sum(1 for r in expected if r["expect_hit"]) / len(expected), 3)
131
+ if expected else None),
132
+ "wall_s": round(time.time() - t_all, 1),
133
+ "device": args.device,
134
+ }
135
+ print(json.dumps(summary, indent=1))
136
+
137
+ import os
138
+ os.makedirs(os.path.dirname(args.out) or ".", exist_ok=True)
139
+ with open(args.out, "w", encoding="utf-8") as f:
140
+ for r in rows:
141
+ f.write(json.dumps(r, ensure_ascii=False) + "\n")
142
+ with open(args.out.replace(".jsonl", "_summary.json"), "w", encoding="utf-8") as f:
143
+ json.dump(summary, f, indent=1)
144
+ return 0
145
+
146
+
147
+ if __name__ == "__main__":
148
+ raise SystemExit(main())
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: streamft
3
- Version: 0.3.0
3
+ Version: 0.4.0
4
4
  Summary: Fine-tune models that do not fit: stream NF4 weights from disk, one transformer block at a time.
5
5
  Author-email: Gokhan Cava <byjameson@users.noreply.github.com>
6
6
  License: Apache-2.0
@@ -53,9 +53,11 @@ Measured on one machine — **RTX 3050 Laptop, 6 GB VRAM, 34 GB RAM, NVMe, Windo
53
53
 
54
54
  The 32B weights are **19.2 GB**. They never fully live in VRAM. At 1024 tokens/step the
55
55
  throughput is **3.1×** the 128-token rate, because the per-step I/O is amortised over
56
- more tokens. 50 steps of Dolly-15k SFT moved eval loss **1.8755 → 0.9661**.
56
+ more tokens. 50 steps of Dolly-15k SFT moved eval loss **1.8755 → 0.9659**.
57
57
 
58
- **Installed from PyPI? Start with [`QUICKSTART.md`](QUICKSTART.md)** — it uses only the
58
+ **Installed from PyPI? Start with [`QUICKSTART.md`](QUICKSTART.md) (T?rk?e:
59
+ [`QUICKSTART.tr.md`](QUICKSTART.tr.md)) ? or run `install.ps1` / `install.sh`, which
60
+ checks Python, installs the package, looks for a GPU and runs `doctor`.** — it uses only the
59
61
  published CLI (no repo scripts). Deeper dive: [`GUIDE_6GB_32B.md`](GUIDE_6GB_32B.md) — the practical write-up: presets,
60
62
  the six traps, and which numbers are real. The measurement protocol, the discarded rows
61
63
  and the correction of an earlier wrong claim ("8.6× / 100 tok/s") are in
@@ -86,6 +88,9 @@ streamft synth --model Qwen/Qwen2.5-1.5B-Instruct --n 200 --out data/synth.jsonl
86
88
 
87
89
  # inference budget (weights + KV cache), not just training
88
90
  streamft fit --model Qwen/Qwen2.5-7B-Instruct --gpu 6 --seq 2048 --mode inference
91
+
92
+ # does it actually work? 20 local prompts, no API and no judge model
93
+ streamft eval --model Qwen/Qwen2.5-1.5B-Instruct
89
94
  ```
90
95
 
91
96
  Every run appends one JSON row to `results/results.jsonl` next to where you launched it.
@@ -9,6 +9,7 @@ streamft/data.py
9
9
  streamft/disk_quant_store.py
10
10
  streamft/doctor.py
11
11
  streamft/easy.py
12
+ streamft/evals.py
12
13
  streamft/fit.py
13
14
  streamft/lora.py
14
15
  streamft/metrics.py
@@ -271,3 +271,11 @@ def test_kv_cache_matches_hand_calculation():
271
271
  # 2 * 24 layers * 2 kv heads * 128 head_dim * 1024 seq * 2 bytes
272
272
  expect = 2 * 24 * 2 * 128 * 1024 * 2 / 1e9
273
273
  assert abs(kv_cache_gb(cfg, 1024) - expect) < 1e-12
274
+
275
+
276
+ def test_repetition_rate_counts_duplicate_ngrams():
277
+ from streamft.evals import repetition_rate
278
+
279
+ assert repetition_rate("a b c d e f g h") == 0.0
280
+ assert repetition_rate("a b c d a b c d") == 0.2 # 1 duplicate of 5 grams
281
+ assert repetition_rate("") == 0.0
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes