streamft 0.2.6__tar.gz → 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {streamft-0.2.6 → streamft-0.4.0}/PKG-INFO +16 -3
- {streamft-0.2.6 → streamft-0.4.0}/README.md +15 -2
- {streamft-0.2.6 → streamft-0.4.0}/pyproject.toml +1 -1
- {streamft-0.2.6 → streamft-0.4.0}/streamft/__main__.py +18 -0
- {streamft-0.2.6 → streamft-0.4.0}/streamft/cli.py +17 -5
- streamft-0.4.0/streamft/easy.py +226 -0
- streamft-0.4.0/streamft/evals.py +148 -0
- {streamft-0.2.6 → streamft-0.4.0}/streamft/fit.py +33 -0
- {streamft-0.2.6 → streamft-0.4.0}/streamft.egg-info/PKG-INFO +16 -3
- {streamft-0.2.6 → streamft-0.4.0}/streamft.egg-info/SOURCES.txt +2 -0
- {streamft-0.2.6 → streamft-0.4.0}/tests/test_streamft_smoke.py +31 -0
- {streamft-0.2.6 → streamft-0.4.0}/LICENSE +0 -0
- {streamft-0.2.6 → streamft-0.4.0}/setup.cfg +0 -0
- {streamft-0.2.6 → streamft-0.4.0}/streamft/__init__.py +0 -0
- {streamft-0.2.6 → streamft-0.4.0}/streamft/autotune.py +0 -0
- {streamft-0.2.6 → streamft-0.4.0}/streamft/data.py +0 -0
- {streamft-0.2.6 → streamft-0.4.0}/streamft/disk_quant_store.py +0 -0
- {streamft-0.2.6 → streamft-0.4.0}/streamft/doctor.py +0 -0
- {streamft-0.2.6 → streamft-0.4.0}/streamft/lora.py +0 -0
- {streamft-0.2.6 → streamft-0.4.0}/streamft/metrics.py +0 -0
- {streamft-0.2.6 → streamft-0.4.0}/streamft/stream_quant_lora.py +0 -0
- {streamft-0.2.6 → streamft-0.4.0}/streamft/stream_train.py +0 -0
- {streamft-0.2.6 → streamft-0.4.0}/streamft.egg-info/dependency_links.txt +0 -0
- {streamft-0.2.6 → streamft-0.4.0}/streamft.egg-info/entry_points.txt +0 -0
- {streamft-0.2.6 → streamft-0.4.0}/streamft.egg-info/requires.txt +0 -0
- {streamft-0.2.6 → streamft-0.4.0}/streamft.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: streamft
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.0
|
|
4
4
|
Summary: Fine-tune models that do not fit: stream NF4 weights from disk, one transformer block at a time.
|
|
5
5
|
Author-email: Gokhan Cava <byjameson@users.noreply.github.com>
|
|
6
6
|
License: Apache-2.0
|
|
@@ -53,9 +53,11 @@ Measured on one machine — **RTX 3050 Laptop, 6 GB VRAM, 34 GB RAM, NVMe, Windo
|
|
|
53
53
|
|
|
54
54
|
The 32B weights are **19.2 GB**. They never fully live in VRAM. At 1024 tokens/step the
|
|
55
55
|
throughput is **3.1×** the 128-token rate, because the per-step I/O is amortised over
|
|
56
|
-
more tokens. 50 steps of Dolly-15k SFT moved eval loss **1.8755 → 0.
|
|
56
|
+
more tokens. 50 steps of Dolly-15k SFT moved eval loss **1.8755 → 0.9659**.
|
|
57
57
|
|
|
58
|
-
**Installed from PyPI? Start with [`QUICKSTART.md`](QUICKSTART.md)
|
|
58
|
+
**Installed from PyPI? Start with [`QUICKSTART.md`](QUICKSTART.md) (T?rk?e:
|
|
59
|
+
[`QUICKSTART.tr.md`](QUICKSTART.tr.md)) ? or run `install.ps1` / `install.sh`, which
|
|
60
|
+
checks Python, installs the package, looks for a GPU and runs `doctor`.** — it uses only the
|
|
59
61
|
published CLI (no repo scripts). Deeper dive: [`GUIDE_6GB_32B.md`](GUIDE_6GB_32B.md) — the practical write-up: presets,
|
|
60
62
|
the six traps, and which numbers are real. The measurement protocol, the discarded rows
|
|
61
63
|
and the correction of an earlier wrong claim ("8.6× / 100 tok/s") are in
|
|
@@ -79,9 +81,20 @@ streamft train --model-dir checkpoints/qwen0.5b_nf4 --sft --steps 20 --seq 512 -
|
|
|
79
81
|
# LM path (optional): build token shards, then train without --sft
|
|
80
82
|
streamft data --model Qwen/Qwen2.5-0.5B --out data
|
|
81
83
|
streamft train --model-dir checkpoints/qwen0.5b_nf4 --steps 20 --seq 256 --tag lm
|
|
84
|
+
|
|
85
|
+
# no GPU at all, or no dataset: both are covered
|
|
86
|
+
streamft cpu --model Qwen/Qwen2.5-0.5B --steps 50 --seq 128
|
|
87
|
+
streamft synth --model Qwen/Qwen2.5-1.5B-Instruct --n 200 --out data/synth.jsonl
|
|
88
|
+
|
|
89
|
+
# inference budget (weights + KV cache), not just training
|
|
90
|
+
streamft fit --model Qwen/Qwen2.5-7B-Instruct --gpu 6 --seq 2048 --mode inference
|
|
91
|
+
|
|
92
|
+
# does it actually work? 20 local prompts, no API and no judge model
|
|
93
|
+
streamft eval --model Qwen/Qwen2.5-1.5B-Instruct
|
|
82
94
|
```
|
|
83
95
|
|
|
84
96
|
Every run appends one JSON row to `results/results.jsonl` next to where you launched it.
|
|
97
|
+
Six worked recipes: [`COOKBOOK.md`](COOKBOOK.md).
|
|
85
98
|
The same commands are in [`QUICKSTART.md`](QUICKSTART.md), which is what a pip user should
|
|
86
99
|
read — this README also documents the experiments behind the numbers.
|
|
87
100
|
|
|
@@ -23,9 +23,11 @@ Measured on one machine — **RTX 3050 Laptop, 6 GB VRAM, 34 GB RAM, NVMe, Windo
|
|
|
23
23
|
|
|
24
24
|
The 32B weights are **19.2 GB**. They never fully live in VRAM. At 1024 tokens/step the
|
|
25
25
|
throughput is **3.1×** the 128-token rate, because the per-step I/O is amortised over
|
|
26
|
-
more tokens. 50 steps of Dolly-15k SFT moved eval loss **1.8755 → 0.
|
|
26
|
+
more tokens. 50 steps of Dolly-15k SFT moved eval loss **1.8755 → 0.9659**.
|
|
27
27
|
|
|
28
|
-
**Installed from PyPI? Start with [`QUICKSTART.md`](QUICKSTART.md)
|
|
28
|
+
**Installed from PyPI? Start with [`QUICKSTART.md`](QUICKSTART.md) (T?rk?e:
|
|
29
|
+
[`QUICKSTART.tr.md`](QUICKSTART.tr.md)) ? or run `install.ps1` / `install.sh`, which
|
|
30
|
+
checks Python, installs the package, looks for a GPU and runs `doctor`.** — it uses only the
|
|
29
31
|
published CLI (no repo scripts). Deeper dive: [`GUIDE_6GB_32B.md`](GUIDE_6GB_32B.md) — the practical write-up: presets,
|
|
30
32
|
the six traps, and which numbers are real. The measurement protocol, the discarded rows
|
|
31
33
|
and the correction of an earlier wrong claim ("8.6× / 100 tok/s") are in
|
|
@@ -49,9 +51,20 @@ streamft train --model-dir checkpoints/qwen0.5b_nf4 --sft --steps 20 --seq 512 -
|
|
|
49
51
|
# LM path (optional): build token shards, then train without --sft
|
|
50
52
|
streamft data --model Qwen/Qwen2.5-0.5B --out data
|
|
51
53
|
streamft train --model-dir checkpoints/qwen0.5b_nf4 --steps 20 --seq 256 --tag lm
|
|
54
|
+
|
|
55
|
+
# no GPU at all, or no dataset: both are covered
|
|
56
|
+
streamft cpu --model Qwen/Qwen2.5-0.5B --steps 50 --seq 128
|
|
57
|
+
streamft synth --model Qwen/Qwen2.5-1.5B-Instruct --n 200 --out data/synth.jsonl
|
|
58
|
+
|
|
59
|
+
# inference budget (weights + KV cache), not just training
|
|
60
|
+
streamft fit --model Qwen/Qwen2.5-7B-Instruct --gpu 6 --seq 2048 --mode inference
|
|
61
|
+
|
|
62
|
+
# does it actually work? 20 local prompts, no API and no judge model
|
|
63
|
+
streamft eval --model Qwen/Qwen2.5-1.5B-Instruct
|
|
52
64
|
```
|
|
53
65
|
|
|
54
66
|
Every run appends one JSON row to `results/results.jsonl` next to where you launched it.
|
|
67
|
+
Six worked recipes: [`COOKBOOK.md`](COOKBOOK.md).
|
|
55
68
|
The same commands are in [`QUICKSTART.md`](QUICKSTART.md), which is what a pip user should
|
|
56
69
|
read — this README also documents the experiments behind the numbers.
|
|
57
70
|
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "streamft"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.4.0"
|
|
8
8
|
description = "Fine-tune models that do not fit: stream NF4 weights from disk, one transformer block at a time."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
|
@@ -4,6 +4,9 @@
|
|
|
4
4
|
streamft prepare turn a bf16 model into a pre-quantized NF4 checkpoint
|
|
5
5
|
streamft train the streaming trainer (all exp54 flags)
|
|
6
6
|
streamft data build the token shards (LM path only; SFT needs none)
|
|
7
|
+
streamft cpu SFT a small model on the CPU (no GPU; model must fit RAM)
|
|
8
|
+
streamft synth generate training pairs locally with a small model
|
|
9
|
+
streamft eval run a small local prompt set (repetition/refusal/keywords)
|
|
7
10
|
streamft doctor environment report (torch, CUDA, bitsandbytes, VRAM)
|
|
8
11
|
"""
|
|
9
12
|
from __future__ import annotations
|
|
@@ -16,6 +19,9 @@ USAGE = """streamft - fine-tune models that do not fit
|
|
|
16
19
|
streamft prepare --model <dir> --save-quant checkpoints/<name>
|
|
17
20
|
streamft train --model-dir checkpoints/<name> --sft --steps 30 [--seq 1024]
|
|
18
21
|
streamft data --model <dir|hf-id> --out data
|
|
22
|
+
streamft cpu --model <small-model> --steps 20
|
|
23
|
+
streamft synth --model <small-model> --n 50 --out data/synth.jsonl
|
|
24
|
+
streamft eval --model <model> [--prompts mine.jsonl]
|
|
19
25
|
streamft doctor
|
|
20
26
|
|
|
21
27
|
Docs: https://github.com/byjameson/stream-finetune
|
|
@@ -35,6 +41,18 @@ def main() -> int:
|
|
|
35
41
|
|
|
36
42
|
return fit_main(rest)
|
|
37
43
|
|
|
44
|
+
if cmd == "cpu":
|
|
45
|
+
from streamft.easy import cpu_main
|
|
46
|
+
return cpu_main(rest)
|
|
47
|
+
|
|
48
|
+
if cmd == "eval":
|
|
49
|
+
from streamft.evals import main as eval_main
|
|
50
|
+
return eval_main(rest)
|
|
51
|
+
|
|
52
|
+
if cmd == "synth":
|
|
53
|
+
from streamft.easy import synth_main
|
|
54
|
+
return synth_main(rest)
|
|
55
|
+
|
|
38
56
|
if cmd == "data":
|
|
39
57
|
from streamft.data import main as data_main
|
|
40
58
|
|
|
@@ -388,14 +388,23 @@ def main() -> int:
|
|
|
388
388
|
tok = AutoTokenizer.from_pretrained(model_dir)
|
|
389
389
|
if tok.pad_token is None:
|
|
390
390
|
tok.pad_token = tok.eos_token
|
|
391
|
-
|
|
391
|
+
if str(args.dataset).lower().endswith(".jsonl"):
|
|
392
|
+
# local {"instruction","response"} file, e.g. from `streamft synth`
|
|
393
|
+
ds = load_dataset("json", data_files=args.dataset, split="train").shuffle(seed=1234)
|
|
394
|
+
else:
|
|
395
|
+
ds = load_dataset(args.dataset, split="train").shuffle(seed=1234)
|
|
392
396
|
train_rows = [ds[i] for i in range(min(args.train_size, len(ds)))]
|
|
393
397
|
eval_rows = [ds[len(train_rows) + i]
|
|
394
398
|
for i in range(min(args.eval_size, len(ds) - len(train_rows)))]
|
|
395
399
|
print(f"[sft] {Path(args.dataset).name} train={len(train_rows)} "
|
|
396
400
|
f"eval={len(eval_rows)}", flush=True)
|
|
397
|
-
|
|
398
|
-
|
|
401
|
+
if eval_rows:
|
|
402
|
+
ce0 = _masked_eval(tr, eval_rows, tok, args.seq)
|
|
403
|
+
print(f"[eval] before loss={ce0:.4f}", flush=True)
|
|
404
|
+
else:
|
|
405
|
+
ce0 = None
|
|
406
|
+
print(f"[sft] no held-out rows (dataset has {len(train_rows)} rows); skipping eval",
|
|
407
|
+
flush=True)
|
|
399
408
|
# Filter once so `--steps N` means N optimizer updates, not "N attempts minus
|
|
400
409
|
# whatever the mask rejected". A silently short run is a lie about the budget.
|
|
401
410
|
valid_rows, skipped_train = sft_schedule(
|
|
@@ -419,8 +428,11 @@ def main() -> int:
|
|
|
419
428
|
if (i + 1) % 10 == 0:
|
|
420
429
|
print(f"[step] {i+1}/{args.steps} loss={loss:.4f} {dt:.2f}s "
|
|
421
430
|
f"peak={peak:.2f}GB rss={host_rss_gb()}GB", flush=True)
|
|
422
|
-
|
|
423
|
-
|
|
431
|
+
if eval_rows:
|
|
432
|
+
ce1 = _masked_eval(tr, eval_rows, tok, args.seq)
|
|
433
|
+
print(f"[eval] after loss={ce1:.4f} (delta {ce1-ce0:+.4f})", flush=True)
|
|
434
|
+
else:
|
|
435
|
+
ce1 = None
|
|
424
436
|
wall = time.time() - t_all
|
|
425
437
|
else:
|
|
426
438
|
tokens = load_tokens(str(user_path(args.data)))
|
|
@@ -0,0 +1,226 @@
|
|
|
1
|
+
"""Two on-ramps for people without a training GPU.
|
|
2
|
+
|
|
3
|
+
streamft cpu SFT a small model on the CPU (the model must fit in RAM)
|
|
4
|
+
streamft synth generate training pairs locally with a small model (no API, no data hunt)
|
|
5
|
+
|
|
6
|
+
Both are deliberately simple: plain transformers forward/backward, our LoRA module, the
|
|
7
|
+
same prompt masking as the GPU path. The streaming engine is not involved.
|
|
8
|
+
"""
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import argparse
|
|
12
|
+
import json
|
|
13
|
+
import random
|
|
14
|
+
import re
|
|
15
|
+
import time
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
|
|
18
|
+
import torch
|
|
19
|
+
|
|
20
|
+
from .cli import _encode_chat, _format_messages # noqa: F401 (same masking rules)
|
|
21
|
+
from .lora import apply_lora, lora_parameters, lora_state_dict
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
# --------------------------------------------------------------------------- cpu train
|
|
25
|
+
|
|
26
|
+
def _load_tiny(model_dir: str):
|
|
27
|
+
from transformers import AutoModelForCausalLM
|
|
28
|
+
|
|
29
|
+
dtype = torch.float32 # CPU: bf16 is slow and fp16 is unsupported on many CPUs
|
|
30
|
+
model = AutoModelForCausalLM.from_pretrained(model_dir, torch_dtype=dtype)
|
|
31
|
+
model.eval()
|
|
32
|
+
return model
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@torch.no_grad()
|
|
36
|
+
def _eval_masked(model, rows, tok, seq: int) -> float:
|
|
37
|
+
model.eval()
|
|
38
|
+
total, n = 0.0, 0
|
|
39
|
+
for row in rows:
|
|
40
|
+
enc = _encode_chat(tok, row, seq)
|
|
41
|
+
if enc is None:
|
|
42
|
+
continue
|
|
43
|
+
ids, labels = enc
|
|
44
|
+
loss = model(input_ids=ids, labels=labels).loss
|
|
45
|
+
v = int((labels != -100).sum())
|
|
46
|
+
total += float(loss) * v
|
|
47
|
+
n += v
|
|
48
|
+
model.train()
|
|
49
|
+
if n == 0:
|
|
50
|
+
raise RuntimeError("no eval row has a valid prompt mask")
|
|
51
|
+
return total / n
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def cpu_main(argv=None) -> int:
|
|
55
|
+
from datasets import load_dataset
|
|
56
|
+
from transformers import AutoTokenizer
|
|
57
|
+
|
|
58
|
+
ap = argparse.ArgumentParser(prog="streamft cpu")
|
|
59
|
+
ap.add_argument("--model", required=True, help="small model dir or HF id (fits in RAM)")
|
|
60
|
+
ap.add_argument("--dataset", default="databricks/databricks-dolly-15k")
|
|
61
|
+
ap.add_argument("--train_size", type=int, default=128)
|
|
62
|
+
ap.add_argument("--eval_size", type=int, default=8)
|
|
63
|
+
ap.add_argument("--steps", type=int, default=20)
|
|
64
|
+
ap.add_argument("--seq", type=int, default=256)
|
|
65
|
+
ap.add_argument("--lr", type=float, default=3e-4)
|
|
66
|
+
ap.add_argument("--lora_r", type=int, default=8)
|
|
67
|
+
ap.add_argument("--save", default="")
|
|
68
|
+
ap.add_argument("--tag", default="cpu_sft")
|
|
69
|
+
args = ap.parse_args(argv)
|
|
70
|
+
|
|
71
|
+
tok = AutoTokenizer.from_pretrained(args.model)
|
|
72
|
+
if tok.pad_token is None:
|
|
73
|
+
tok.pad_token = tok.eos_token
|
|
74
|
+
t0 = time.time()
|
|
75
|
+
model = _load_tiny(args.model)
|
|
76
|
+
apply_lora(model, r_attn=args.lora_r, r_mlp=0)
|
|
77
|
+
params = list(lora_parameters(model))
|
|
78
|
+
opt = torch.optim.AdamW(params, lr=args.lr)
|
|
79
|
+
print(f"[cpu] {args.model} loaded in {time.time()-t0:.1f}s, "
|
|
80
|
+
f"{sum(p.numel() for p in params)/1e6:.2f}M adapter params", flush=True)
|
|
81
|
+
|
|
82
|
+
ds = load_dataset(args.dataset, split="train").shuffle(seed=1234)
|
|
83
|
+
train_rows = [ds[i] for i in range(min(args.train_size, len(ds)))]
|
|
84
|
+
eval_rows = [ds[len(train_rows) + i]
|
|
85
|
+
for i in range(min(args.eval_size, len(ds) - len(train_rows)))]
|
|
86
|
+
|
|
87
|
+
valid = [r for r in train_rows if _encode_chat(tok, r, args.seq) is not None]
|
|
88
|
+
dropped = len(train_rows) - len(valid)
|
|
89
|
+
if not valid:
|
|
90
|
+
raise SystemExit("[cpu] no training row has a valid prompt mask")
|
|
91
|
+
if dropped:
|
|
92
|
+
print(f"[cpu] {dropped}/{len(train_rows)} rows dropped (prompt not a prefix)",
|
|
93
|
+
flush=True)
|
|
94
|
+
|
|
95
|
+
ce0 = _eval_masked(model, eval_rows, tok, args.seq)
|
|
96
|
+
print(f"[cpu] eval before {ce0:.4f}", flush=True)
|
|
97
|
+
|
|
98
|
+
model.train()
|
|
99
|
+
losses, times = [], []
|
|
100
|
+
for i in range(args.steps):
|
|
101
|
+
enc = _encode_chat(tok, valid[i % len(valid)], args.seq)
|
|
102
|
+
ids, labels = enc
|
|
103
|
+
t = time.time()
|
|
104
|
+
loss = model(input_ids=ids, labels=labels).loss
|
|
105
|
+
loss.backward()
|
|
106
|
+
opt.step()
|
|
107
|
+
opt.zero_grad(set_to_none=True)
|
|
108
|
+
losses.append(round(float(loss.detach()), 6))
|
|
109
|
+
times.append(round(time.time() - t, 3))
|
|
110
|
+
if (i + 1) % 5 == 0 or i == 0:
|
|
111
|
+
print(f"[cpu] step {i+1}/{args.steps} loss={losses[-1]:.4f} "
|
|
112
|
+
f"{times[-1]:.2f}s", flush=True)
|
|
113
|
+
|
|
114
|
+
ce1 = _eval_masked(model, eval_rows, tok, args.seq)
|
|
115
|
+
if args.save:
|
|
116
|
+
out = Path(args.save)
|
|
117
|
+
out.mkdir(parents=True, exist_ok=True)
|
|
118
|
+
torch.save(lora_state_dict(model), str(out / "adapter.pt"))
|
|
119
|
+
row = {
|
|
120
|
+
"engine": "cpu-dense", "tag": args.tag, "model": args.model, "steps": len(losses),
|
|
121
|
+
"seq": args.seq, "lr": args.lr, "lora_r": args.lora_r,
|
|
122
|
+
"step_s_med": sorted(times)[len(times) // 2] if times else None,
|
|
123
|
+
"ce_before": round(ce0, 5), "ce_after": round(ce1, 5),
|
|
124
|
+
"ce_delta": round(ce1 - ce0, 5), "losses": losses,
|
|
125
|
+
"sft_skipped": dropped, "saved": args.save or None,
|
|
126
|
+
"license": "Apache-2.0", "device": "cpu",
|
|
127
|
+
}
|
|
128
|
+
print(json.dumps(row, ensure_ascii=True), flush=True)
|
|
129
|
+
out = Path("results")
|
|
130
|
+
out.mkdir(exist_ok=True)
|
|
131
|
+
with open(out / "results.jsonl", "a", encoding="utf-8") as f:
|
|
132
|
+
f.write(json.dumps(row) + "\n")
|
|
133
|
+
return 0
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
# --------------------------------------------------------------------------- synth
|
|
137
|
+
|
|
138
|
+
PROMPT = """You write training data. Produce ONE example as JSON with keys
|
|
139
|
+
"instruction" and "response". Topic: {task}.
|
|
140
|
+
Rules: the instruction is a short request; the response is a direct answer of 1-3
|
|
141
|
+
sentences; no markdown, no preamble, no extra keys.
|
|
142
|
+
JSON:"""
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
PLACEHOLDER_MARKERS = ("<instruction>", "<response>", "<your", "your instruction",
|
|
146
|
+
"lorem ipsum", "example.com")
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def parse_pair(text: str):
|
|
150
|
+
"""Extract one {"instruction","response"} object; reject template junk.
|
|
151
|
+
|
|
152
|
+
Small models happily echo the schema back ("<instruction>"), and a single such row
|
|
153
|
+
poisons a dataset. Everything is filtered here so the caller only has to dedupe.
|
|
154
|
+
"""
|
|
155
|
+
m = re.search(r"\{.*\}", text, re.S)
|
|
156
|
+
if not m:
|
|
157
|
+
return None
|
|
158
|
+
try:
|
|
159
|
+
obj = json.loads(m.group(0))
|
|
160
|
+
except Exception:
|
|
161
|
+
return None
|
|
162
|
+
if not isinstance(obj, dict):
|
|
163
|
+
return None
|
|
164
|
+
ins = str(obj.get("instruction", "")).strip()
|
|
165
|
+
res = str(obj.get("response", "")).strip()
|
|
166
|
+
low = (ins + " " + res).lower()
|
|
167
|
+
if any(p in low for p in PLACEHOLDER_MARKERS):
|
|
168
|
+
return None
|
|
169
|
+
if "{" in ins or "{" in res or "}" in ins or "}" in res:
|
|
170
|
+
return None
|
|
171
|
+
if not (5 <= len(ins) <= 200 and 10 <= len(res) <= 1200):
|
|
172
|
+
return None
|
|
173
|
+
return {"instruction": ins, "response": res}
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def synth_main(argv=None) -> int:
|
|
177
|
+
from transformers import AutoModelForCausalLM, AutoTokenizer
|
|
178
|
+
|
|
179
|
+
ap = argparse.ArgumentParser(prog="streamft synth")
|
|
180
|
+
ap.add_argument("--model", default="Qwen/Qwen2.5-1.5B-Instruct")
|
|
181
|
+
ap.add_argument("--out", default="data/synth.jsonl")
|
|
182
|
+
ap.add_argument("--n", type=int, default=50, help="examples to attempt")
|
|
183
|
+
ap.add_argument("--task", default="practical everyday knowledge")
|
|
184
|
+
ap.add_argument("--seq", type=int, default=256)
|
|
185
|
+
ap.add_argument("--temperature", type=float, default=0.9)
|
|
186
|
+
ap.add_argument("--device", default="cuda" if torch.cuda.is_available() else "cpu")
|
|
187
|
+
args = ap.parse_args(argv)
|
|
188
|
+
|
|
189
|
+
tok = AutoTokenizer.from_pretrained(args.model)
|
|
190
|
+
model = AutoModelForCausalLM.from_pretrained(
|
|
191
|
+
args.model,
|
|
192
|
+
torch_dtype=torch.bfloat16 if args.device == "cuda" else torch.float32,
|
|
193
|
+
).to(args.device).eval()
|
|
194
|
+
|
|
195
|
+
seen, kept = set(), []
|
|
196
|
+
out = Path(args.out)
|
|
197
|
+
out.parent.mkdir(parents=True, exist_ok=True)
|
|
198
|
+
t0 = time.time()
|
|
199
|
+
for i in range(args.n):
|
|
200
|
+
prompt = PROMPT.format(task=args.task)
|
|
201
|
+
ids = tok(prompt, return_tensors="pt").to(args.device)
|
|
202
|
+
with torch.no_grad():
|
|
203
|
+
gen = model.generate(**ids, max_new_tokens=args.seq, do_sample=True,
|
|
204
|
+
temperature=args.temperature, top_p=0.95,
|
|
205
|
+
pad_token_id=tok.eos_token_id)
|
|
206
|
+
text = tok.decode(gen[0][ids["input_ids"].shape[1]:], skip_special_tokens=True)
|
|
207
|
+
pair = parse_pair(text)
|
|
208
|
+
if pair is None:
|
|
209
|
+
continue
|
|
210
|
+
key = pair["instruction"].lower()
|
|
211
|
+
if key in seen:
|
|
212
|
+
continue
|
|
213
|
+
seen.add(key)
|
|
214
|
+
kept.append(pair)
|
|
215
|
+
if (len(kept) % 10) == 0:
|
|
216
|
+
print(f"[synth] {len(kept)} kept / {i+1} attempts", flush=True)
|
|
217
|
+
|
|
218
|
+
with open(out, "w", encoding="utf-8") as f:
|
|
219
|
+
for r in kept:
|
|
220
|
+
f.write(json.dumps(r, ensure_ascii=False) + "\n")
|
|
221
|
+
print(json.dumps({
|
|
222
|
+
"out": str(out), "attempts": args.n, "kept": len(kept),
|
|
223
|
+
"keep_rate": round(len(kept) / max(args.n, 1), 3),
|
|
224
|
+
"seconds": round(time.time() - t0, 1), "model": args.model, "task": args.task,
|
|
225
|
+
}, indent=1))
|
|
226
|
+
return 0
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
"""`streamft eval` — does the model actually do the thing? No API, no judge model.
|
|
2
|
+
|
|
3
|
+
It runs a small prompt set locally and reports what can be measured without a second
|
|
4
|
+
model: repetition, refusal-phrase hits, length, latency, and optionally keyword adherence
|
|
5
|
+
when the prompt file carries an `expect` field.
|
|
6
|
+
|
|
7
|
+
streamft eval --model Qwen/Qwen2.5-1.5B-Instruct
|
|
8
|
+
streamft eval --model ./merged --prompts my_prompts.jsonl --max-new 96
|
|
9
|
+
|
|
10
|
+
Prompt file format (JSONL, one object per line):
|
|
11
|
+
{"prompt": "...", "expect": ["optional", "keywords"]}
|
|
12
|
+
"""
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import argparse
|
|
16
|
+
import json
|
|
17
|
+
import re
|
|
18
|
+
import time
|
|
19
|
+
from collections import Counter
|
|
20
|
+
|
|
21
|
+
import torch
|
|
22
|
+
|
|
23
|
+
# Ten English + ten Turkish prompts. Deliberately mundane: this is a smoke test, not a
|
|
24
|
+
# benchmark. `expect` is a keyword that should appear if the model understood the task.
|
|
25
|
+
BUILTIN = [
|
|
26
|
+
{"prompt": "List three uses for a paperclip.", "expect": []},
|
|
27
|
+
{"prompt": "Explain in one sentence why the sky is blue.", "expect": ["light", "scatter"]},
|
|
28
|
+
{"prompt": "Give one reason to keep a food diary.", "expect": []},
|
|
29
|
+
{"prompt": "Write a two-line rhyme about rain.", "expect": []},
|
|
30
|
+
{"prompt": "What is 17 multiplied by 6?", "expect": ["102"]},
|
|
31
|
+
{"prompt": "Name two vegetables that grow underground.", "expect": ["carrot", "potato"]},
|
|
32
|
+
{"prompt": "Summarise the plot of a film about a lost dog in one sentence.", "expect": []},
|
|
33
|
+
{"prompt": "Translate 'good morning' into Turkish.", "expect": ["günaydın", "gunaydin"]},
|
|
34
|
+
{"prompt": "Give a one-sentence tip for better sleep.", "expect": []},
|
|
35
|
+
{"prompt": "What does a red traffic light mean?", "expect": ["stop"]},
|
|
36
|
+
{"prompt": "Bir cümlede su neden buharlaşır açıkla.", "expect": ["ısı", "isi", "sıcaklık"]},
|
|
37
|
+
{"prompt": "Üç tane meyve say.", "expect": ["elma", "muz", "portakal", "çilek"]},
|
|
38
|
+
{"prompt": "17 çarpı 6 kaç eder?", "expect": ["102"]},
|
|
39
|
+
{"prompt": "Yağmurlu bir gün için iki satırlık bir dörtlük yaz.", "expect": []},
|
|
40
|
+
{"prompt": "İyi bir uyku için tek cümlelik bir öneri ver.", "expect": []},
|
|
41
|
+
{"prompt": "Kırmızı trafik ışığı ne anlama gelir?", "expect": ["dur"]},
|
|
42
|
+
{"prompt": "Sabah kahvaltısı için iki öneri ver.", "expect": []},
|
|
43
|
+
{"prompt": "Bir cümlede neden kitap okumalıyız açıkla.", "expect": []},
|
|
44
|
+
{"prompt": "Türkiye'nin başkenti neresidir?", "expect": ["ankara"]},
|
|
45
|
+
{"prompt": "Bir cümlede bisikletin avantajını yaz.", "expect": []},
|
|
46
|
+
]
|
|
47
|
+
|
|
48
|
+
REFUSAL = [
|
|
49
|
+
"i can't help", "i cannot help", "i can't assist", "i cannot assist",
|
|
50
|
+
"as an ai", "i'm sorry, but", "i am sorry, but", "cannot provide",
|
|
51
|
+
"yardımcı olamam", "yardimci olamam", "üzgünüm", "uzgunum", "olarak yapay zeka",
|
|
52
|
+
]
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def repetition_rate(text: str, n: int = 4) -> float:
|
|
56
|
+
"""Fraction of repeated n-grams (1.0 = every n-gram is a duplicate)."""
|
|
57
|
+
words = re.findall(r"\w+", text.lower())
|
|
58
|
+
if len(words) < n + 1:
|
|
59
|
+
return 0.0
|
|
60
|
+
grams = [tuple(words[i:i + n]) for i in range(len(words) - n + 1)]
|
|
61
|
+
counts = Counter(grams)
|
|
62
|
+
repeated = sum(c - 1 for c in counts.values() if c > 1)
|
|
63
|
+
return repeated / len(grams)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def main(argv=None) -> int:
|
|
67
|
+
from transformers import AutoModelForCausalLM, AutoTokenizer
|
|
68
|
+
|
|
69
|
+
ap = argparse.ArgumentParser(prog="streamft eval")
|
|
70
|
+
ap.add_argument("--model", default="Qwen/Qwen2.5-1.5B-Instruct")
|
|
71
|
+
ap.add_argument("--prompts", default="", help="JSONL with {'prompt', 'expect'?}")
|
|
72
|
+
ap.add_argument("--max-new", type=int, default=64)
|
|
73
|
+
ap.add_argument("--limit", type=int, default=0, help="evaluate only the first N prompts")
|
|
74
|
+
ap.add_argument("--out", default="results/eval.jsonl")
|
|
75
|
+
ap.add_argument("--device", default="cuda" if torch.cuda.is_available() else "cpu")
|
|
76
|
+
ap.add_argument("--quiet", action="store_true")
|
|
77
|
+
args = ap.parse_args(argv)
|
|
78
|
+
|
|
79
|
+
prompts = BUILTIN
|
|
80
|
+
if args.prompts:
|
|
81
|
+
with open(args.prompts, encoding="utf-8") as f:
|
|
82
|
+
prompts = [json.loads(line) for line in f if line.strip()]
|
|
83
|
+
if args.limit:
|
|
84
|
+
prompts = prompts[: args.limit]
|
|
85
|
+
|
|
86
|
+
dtype = torch.bfloat16 if args.device == "cuda" else torch.float32
|
|
87
|
+
tok = AutoTokenizer.from_pretrained(args.model)
|
|
88
|
+
model = AutoModelForCausalLM.from_pretrained(args.model, torch_dtype=dtype).to(args.device)
|
|
89
|
+
model.eval()
|
|
90
|
+
|
|
91
|
+
rows = []
|
|
92
|
+
t_all = time.time()
|
|
93
|
+
for i, item in enumerate(prompts):
|
|
94
|
+
text = item["prompt"]
|
|
95
|
+
if getattr(tok, "chat_template", None):
|
|
96
|
+
text = tok.apply_chat_template([{"role": "user", "content": text}],
|
|
97
|
+
tokenize=False, add_generation_prompt=True)
|
|
98
|
+
ids = tok(text, return_tensors="pt").to(args.device)
|
|
99
|
+
t0 = time.time()
|
|
100
|
+
with torch.no_grad():
|
|
101
|
+
out = model.generate(**ids, max_new_tokens=args.max_new, do_sample=False,
|
|
102
|
+
pad_token_id=tok.eos_token_id)
|
|
103
|
+
dt = time.time() - t0
|
|
104
|
+
gen = out[0][ids["input_ids"].shape[1]:]
|
|
105
|
+
answer = tok.decode(gen, skip_special_tokens=True).strip()
|
|
106
|
+
low = answer.lower()
|
|
107
|
+
row = {
|
|
108
|
+
"i": i, "prompt": item["prompt"], "answer": answer,
|
|
109
|
+
"tokens": int(gen.shape[0]), "seconds": round(dt, 2),
|
|
110
|
+
"tok_s": round(int(gen.shape[0]) / max(dt, 1e-6), 1),
|
|
111
|
+
"rep4": round(repetition_rate(answer), 3),
|
|
112
|
+
"refusal": any(p in low for p in REFUSAL),
|
|
113
|
+
"expect_hit": (None if not item.get("expect")
|
|
114
|
+
else any(k.lower() in low for k in item["expect"])),
|
|
115
|
+
}
|
|
116
|
+
rows.append(row)
|
|
117
|
+
if not args.quiet:
|
|
118
|
+
flag = "REFUSAL" if row["refusal"] else ("miss" if row["expect_hit"] is False else "")
|
|
119
|
+
print(f"[{i+1}/{len(prompts)}] {row['tok_s']:.1f} tok/s rep4={row['rep4']:.2f} {flag}")
|
|
120
|
+
print(" " + answer.replace("\n", " ")[:160])
|
|
121
|
+
|
|
122
|
+
n = len(rows)
|
|
123
|
+
expected = [r for r in rows if r["expect_hit"] is not None]
|
|
124
|
+
summary = {
|
|
125
|
+
"model": args.model, "n": n,
|
|
126
|
+
"refusal_rate": round(sum(r["refusal"] for r in rows) / max(n, 1), 3),
|
|
127
|
+
"mean_rep4": round(sum(r["rep4"] for r in rows) / max(n, 1), 3),
|
|
128
|
+
"mean_tokens": round(sum(r["tokens"] for r in rows) / max(n, 1), 1),
|
|
129
|
+
"mean_tok_s": round(sum(r["tok_s"] for r in rows) / max(n, 1), 1),
|
|
130
|
+
"keyword_hit_rate": (round(sum(1 for r in expected if r["expect_hit"]) / len(expected), 3)
|
|
131
|
+
if expected else None),
|
|
132
|
+
"wall_s": round(time.time() - t_all, 1),
|
|
133
|
+
"device": args.device,
|
|
134
|
+
}
|
|
135
|
+
print(json.dumps(summary, indent=1))
|
|
136
|
+
|
|
137
|
+
import os
|
|
138
|
+
os.makedirs(os.path.dirname(args.out) or ".", exist_ok=True)
|
|
139
|
+
with open(args.out, "w", encoding="utf-8") as f:
|
|
140
|
+
for r in rows:
|
|
141
|
+
f.write(json.dumps(r, ensure_ascii=False) + "\n")
|
|
142
|
+
with open(args.out.replace(".jsonl", "_summary.json"), "w", encoding="utf-8") as f:
|
|
143
|
+
json.dump(summary, f, indent=1)
|
|
144
|
+
return 0
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
if __name__ == "__main__":
|
|
148
|
+
raise SystemExit(main())
|
|
@@ -53,6 +53,16 @@ def load_config(model: str) -> dict:
|
|
|
53
53
|
return json.loads(Path(hf_hub_download(model, "config.json")).read_text(encoding="utf-8"))
|
|
54
54
|
|
|
55
55
|
|
|
56
|
+
def kv_cache_gb(cfg: dict, seq: int, batch: int = 1, bytes_per: int = 2) -> float:
|
|
57
|
+
"""KV cache at inference: 2 tensors x layers x kv_heads x head_dim x seq x batch."""
|
|
58
|
+
h = int(cfg.get("hidden_size") or cfg.get("n_embd"))
|
|
59
|
+
layers = int(cfg.get("num_hidden_layers") or cfg.get("n_layer"))
|
|
60
|
+
heads = int(cfg.get("num_attention_heads") or 1)
|
|
61
|
+
kv_heads = int(cfg.get("num_key_value_heads") or heads)
|
|
62
|
+
head_dim = h // max(heads, 1)
|
|
63
|
+
return 2.0 * layers * kv_heads * head_dim * seq * batch * bytes_per / 1e9
|
|
64
|
+
|
|
65
|
+
|
|
56
66
|
def model_fingerprint(cfg: dict) -> tuple:
|
|
57
67
|
"""Everything that changes the measurement: two 7B models are not interchangeable."""
|
|
58
68
|
return (
|
|
@@ -95,6 +105,8 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
95
105
|
ap.add_argument("--model", required=True, help="local dir or HF id")
|
|
96
106
|
ap.add_argument("--gpu", type=float, default=6.0, help="VRAM in GB (default 6)")
|
|
97
107
|
ap.add_argument("--seq", type=int, default=512)
|
|
108
|
+
ap.add_argument("--mode", choices=["train", "inference"], default="train")
|
|
109
|
+
ap.add_argument("--batch", type=int, default=1)
|
|
98
110
|
ap.add_argument("--json", action="store_true")
|
|
99
111
|
args = ap.parse_args(argv)
|
|
100
112
|
|
|
@@ -126,6 +138,27 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
126
138
|
"measured_on_rtx3050_6gb": measured,
|
|
127
139
|
"measured_elsewhere": measured is None,
|
|
128
140
|
}
|
|
141
|
+
if args.mode == "inference":
|
|
142
|
+
kv_gb = kv_cache_gb(cfg, args.seq, args.batch)
|
|
143
|
+
total = m["nf4_gb"] + m["head_gb"] + kv_gb + 0.5
|
|
144
|
+
out_inf = {
|
|
145
|
+
"model": args.model, "mode": "inference", "seq": args.seq, "batch": args.batch,
|
|
146
|
+
"weights_gb": round(m["nf4_gb"] + m["head_gb"], 1),
|
|
147
|
+
"kv_cache_gb": round(kv_gb, 2),
|
|
148
|
+
"needed_gb": round(total, 1),
|
|
149
|
+
"fits_whole": total <= args.gpu,
|
|
150
|
+
"speed": "not measured for this model",
|
|
151
|
+
}
|
|
152
|
+
if args.json:
|
|
153
|
+
print(json.dumps(out_inf, indent=1))
|
|
154
|
+
return 0
|
|
155
|
+
print(f"{args.model} (inference)")
|
|
156
|
+
print(f" weights ~{out_inf['weights_gb']} GB + KV cache ~{out_inf['kv_cache_gb']} GB "
|
|
157
|
+
f"at seq {args.seq}, batch {args.batch} -> ~{out_inf['needed_gb']} GB")
|
|
158
|
+
print(f" verdict: {'fits' if out_inf['fits_whole'] else 'does NOT fit'} on {args.gpu:g} GB")
|
|
159
|
+
print(" speed: not measured for this model (the planner does not guess tok/s)")
|
|
160
|
+
return 0
|
|
161
|
+
|
|
129
162
|
if args.json:
|
|
130
163
|
print(json.dumps(out, indent=1))
|
|
131
164
|
return 0
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: streamft
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.0
|
|
4
4
|
Summary: Fine-tune models that do not fit: stream NF4 weights from disk, one transformer block at a time.
|
|
5
5
|
Author-email: Gokhan Cava <byjameson@users.noreply.github.com>
|
|
6
6
|
License: Apache-2.0
|
|
@@ -53,9 +53,11 @@ Measured on one machine — **RTX 3050 Laptop, 6 GB VRAM, 34 GB RAM, NVMe, Windo
|
|
|
53
53
|
|
|
54
54
|
The 32B weights are **19.2 GB**. They never fully live in VRAM. At 1024 tokens/step the
|
|
55
55
|
throughput is **3.1×** the 128-token rate, because the per-step I/O is amortised over
|
|
56
|
-
more tokens. 50 steps of Dolly-15k SFT moved eval loss **1.8755 → 0.
|
|
56
|
+
more tokens. 50 steps of Dolly-15k SFT moved eval loss **1.8755 → 0.9659**.
|
|
57
57
|
|
|
58
|
-
**Installed from PyPI? Start with [`QUICKSTART.md`](QUICKSTART.md)
|
|
58
|
+
**Installed from PyPI? Start with [`QUICKSTART.md`](QUICKSTART.md) (T?rk?e:
|
|
59
|
+
[`QUICKSTART.tr.md`](QUICKSTART.tr.md)) ? or run `install.ps1` / `install.sh`, which
|
|
60
|
+
checks Python, installs the package, looks for a GPU and runs `doctor`.** — it uses only the
|
|
59
61
|
published CLI (no repo scripts). Deeper dive: [`GUIDE_6GB_32B.md`](GUIDE_6GB_32B.md) — the practical write-up: presets,
|
|
60
62
|
the six traps, and which numbers are real. The measurement protocol, the discarded rows
|
|
61
63
|
and the correction of an earlier wrong claim ("8.6× / 100 tok/s") are in
|
|
@@ -79,9 +81,20 @@ streamft train --model-dir checkpoints/qwen0.5b_nf4 --sft --steps 20 --seq 512 -
|
|
|
79
81
|
# LM path (optional): build token shards, then train without --sft
|
|
80
82
|
streamft data --model Qwen/Qwen2.5-0.5B --out data
|
|
81
83
|
streamft train --model-dir checkpoints/qwen0.5b_nf4 --steps 20 --seq 256 --tag lm
|
|
84
|
+
|
|
85
|
+
# no GPU at all, or no dataset: both are covered
|
|
86
|
+
streamft cpu --model Qwen/Qwen2.5-0.5B --steps 50 --seq 128
|
|
87
|
+
streamft synth --model Qwen/Qwen2.5-1.5B-Instruct --n 200 --out data/synth.jsonl
|
|
88
|
+
|
|
89
|
+
# inference budget (weights + KV cache), not just training
|
|
90
|
+
streamft fit --model Qwen/Qwen2.5-7B-Instruct --gpu 6 --seq 2048 --mode inference
|
|
91
|
+
|
|
92
|
+
# does it actually work? 20 local prompts, no API and no judge model
|
|
93
|
+
streamft eval --model Qwen/Qwen2.5-1.5B-Instruct
|
|
82
94
|
```
|
|
83
95
|
|
|
84
96
|
Every run appends one JSON row to `results/results.jsonl` next to where you launched it.
|
|
97
|
+
Six worked recipes: [`COOKBOOK.md`](COOKBOOK.md).
|
|
85
98
|
The same commands are in [`QUICKSTART.md`](QUICKSTART.md), which is what a pip user should
|
|
86
99
|
read — this README also documents the experiments behind the numbers.
|
|
87
100
|
|
|
@@ -248,3 +248,34 @@ def test_masked_eval_raises_when_every_row_is_skipped(monkeypatch):
|
|
|
248
248
|
monkeypatch.setattr(cli, "_encode_chat", lambda *a, **k: None)
|
|
249
249
|
with pytest.raises(RuntimeError, match="every row was skipped"):
|
|
250
250
|
cli._masked_eval(FakeTrainer(), [{"instruction": "x", "response": "y"}], object(), 8)
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
# ------------------------------------------------ synth / planner helpers
|
|
254
|
+
|
|
255
|
+
def test_parse_pair_rejects_template_junk():
|
|
256
|
+
from streamft.easy import parse_pair
|
|
257
|
+
|
|
258
|
+
good = parse_pair('{"instruction": "Give me a tip", "response": "Sharpen your knives."}')
|
|
259
|
+
assert good and good["instruction"] == "Give me a tip"
|
|
260
|
+
assert parse_pair('{"instruction": "<instruction>", "response": "<response>"}') is None
|
|
261
|
+
assert parse_pair('{"instruction": "x", "response": "short"}') is None
|
|
262
|
+
assert parse_pair("not json") is None
|
|
263
|
+
assert parse_pair('{"instruction": "a" * 300, "response": "b" * 20}') is None
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
def test_kv_cache_matches_hand_calculation():
|
|
267
|
+
from streamft.fit import kv_cache_gb
|
|
268
|
+
|
|
269
|
+
cfg = {"hidden_size": 2048, "num_hidden_layers": 24, "num_attention_heads": 16,
|
|
270
|
+
"num_key_value_heads": 2}
|
|
271
|
+
# 2 * 24 layers * 2 kv heads * 128 head_dim * 1024 seq * 2 bytes
|
|
272
|
+
expect = 2 * 24 * 2 * 128 * 1024 * 2 / 1e9
|
|
273
|
+
assert abs(kv_cache_gb(cfg, 1024) - expect) < 1e-12
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def test_repetition_rate_counts_duplicate_ngrams():
|
|
277
|
+
from streamft.evals import repetition_rate
|
|
278
|
+
|
|
279
|
+
assert repetition_rate("a b c d e f g h") == 0.0
|
|
280
|
+
assert repetition_rate("a b c d a b c d") == 0.2 # 1 duplicate of 5 grams
|
|
281
|
+
assert repetition_rate("") == 0.0
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|