streamft 0.2.3__tar.gz → 0.2.5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {streamft-0.2.3 → streamft-0.2.5}/PKG-INFO +20 -20
- {streamft-0.2.3 → streamft-0.2.5}/README.md +19 -19
- {streamft-0.2.3 → streamft-0.2.5}/pyproject.toml +1 -1
- {streamft-0.2.3 → streamft-0.2.5}/streamft/cli.py +27 -6
- {streamft-0.2.3 → streamft-0.2.5}/streamft/fit.py +22 -7
- {streamft-0.2.3 → streamft-0.2.5}/streamft.egg-info/PKG-INFO +20 -20
- {streamft-0.2.3 → streamft-0.2.5}/tests/test_streamft_smoke.py +39 -0
- {streamft-0.2.3 → streamft-0.2.5}/LICENSE +0 -0
- {streamft-0.2.3 → streamft-0.2.5}/setup.cfg +0 -0
- {streamft-0.2.3 → streamft-0.2.5}/streamft/__init__.py +0 -0
- {streamft-0.2.3 → streamft-0.2.5}/streamft/__main__.py +0 -0
- {streamft-0.2.3 → streamft-0.2.5}/streamft/autotune.py +0 -0
- {streamft-0.2.3 → streamft-0.2.5}/streamft/data.py +0 -0
- {streamft-0.2.3 → streamft-0.2.5}/streamft/disk_quant_store.py +0 -0
- {streamft-0.2.3 → streamft-0.2.5}/streamft/doctor.py +0 -0
- {streamft-0.2.3 → streamft-0.2.5}/streamft/lora.py +0 -0
- {streamft-0.2.3 → streamft-0.2.5}/streamft/metrics.py +0 -0
- {streamft-0.2.3 → streamft-0.2.5}/streamft/stream_quant_lora.py +0 -0
- {streamft-0.2.3 → streamft-0.2.5}/streamft/stream_train.py +0 -0
- {streamft-0.2.3 → streamft-0.2.5}/streamft.egg-info/SOURCES.txt +0 -0
- {streamft-0.2.3 → streamft-0.2.5}/streamft.egg-info/dependency_links.txt +0 -0
- {streamft-0.2.3 → streamft-0.2.5}/streamft.egg-info/entry_points.txt +0 -0
- {streamft-0.2.3 → streamft-0.2.5}/streamft.egg-info/requires.txt +0 -0
- {streamft-0.2.3 → streamft-0.2.5}/streamft.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: streamft
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.5
|
|
4
4
|
Summary: Fine-tune models that do not fit: stream NF4 weights from disk, one transformer block at a time.
|
|
5
5
|
Author-email: Gokhan Cava <byjameson@users.noreply.github.com>
|
|
6
6
|
License: Apache-2.0
|
|
@@ -61,34 +61,34 @@ the six traps, and which numbers are real. The measurement protocol, the discard
|
|
|
61
61
|
and the correction of an earlier wrong claim ("8.6× / 100 tok/s") are in
|
|
62
62
|
[`LONGCTX_FINDINGS.md`](LONGCTX_FINDINGS.md).
|
|
63
63
|
|
|
64
|
-
## Quick start
|
|
64
|
+
## Quick start (installed package, nothing else needed)
|
|
65
65
|
|
|
66
|
-
```
|
|
66
|
+
```bash
|
|
67
67
|
pip install streamft
|
|
68
|
+
streamft doctor # torch / CUDA / bitsandbytes / free VRAM
|
|
68
69
|
|
|
69
|
-
#
|
|
70
|
+
# will it fit? memory arithmetic from config.json + the measured reference
|
|
70
71
|
streamft fit --model Qwen/Qwen2.5-14B-Instruct --gpu 6 --seq 512
|
|
71
|
-
streamft doctor # torch / CUDA / bitsandbytes / free VRAM
|
|
72
72
|
|
|
73
|
-
#
|
|
74
|
-
streamft prepare --model
|
|
75
|
-
python scripts/prep_data.py --model models/Qwen2.5-0.5B
|
|
73
|
+
# one NF4 checkpoint per model (HF id or local directory)
|
|
74
|
+
streamft prepare --model Qwen/Qwen2.5-0.5B --save-quant checkpoints/qwen0.5b_nf4
|
|
76
75
|
|
|
77
|
-
#
|
|
78
|
-
|
|
79
|
-
--save-quant checkpoints/qwen32b_nf4
|
|
76
|
+
# SFT on Dolly-15k — no data prep
|
|
77
|
+
streamft train --model-dir checkpoints/qwen0.5b_nf4 --sft --steps 20 --seq 512 --tag first
|
|
80
78
|
|
|
81
|
-
#
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
79
|
+
# LM path (optional): build token shards, then train without --sft
|
|
80
|
+
streamft data --model Qwen/Qwen2.5-0.5B --out data
|
|
81
|
+
streamft train --model-dir checkpoints/qwen0.5b_nf4 --steps 20 --seq 256 --tag lm
|
|
82
|
+
```
|
|
85
83
|
|
|
86
|
-
|
|
87
|
-
|
|
84
|
+
Every run appends one JSON row to `results/results.jsonl` next to where you launched it.
|
|
85
|
+
The same commands are in [`QUICKSTART.md`](QUICKSTART.md), which is what a pip user should
|
|
86
|
+
read — this README also documents the experiments behind the numbers.
|
|
88
87
|
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
88
|
+
**From a source checkout only** (not part of the wheel): `tools/finetune.py` is a wrapper
|
|
89
|
+
that encodes the VRAM presets, `scripts/compare_peft.py` reproduces the engine comparison,
|
|
90
|
+
`scripts/verify_gradients.py` the gradient check, `scripts/prep_data.py` the token shards
|
|
91
|
+
(the installed equivalent is `streamft data`), and `pytest tests -q` the CPU suite.
|
|
92
92
|
|
|
93
93
|
`tools/finetune.py` encodes the VRAM presets we measured, because the *combination* of
|
|
94
94
|
flags decides whether a run fits — not any single flag:
|
|
@@ -31,34 +31,34 @@ the six traps, and which numbers are real. The measurement protocol, the discard
|
|
|
31
31
|
and the correction of an earlier wrong claim ("8.6× / 100 tok/s") are in
|
|
32
32
|
[`LONGCTX_FINDINGS.md`](LONGCTX_FINDINGS.md).
|
|
33
33
|
|
|
34
|
-
## Quick start
|
|
34
|
+
## Quick start (installed package, nothing else needed)
|
|
35
35
|
|
|
36
|
-
```
|
|
36
|
+
```bash
|
|
37
37
|
pip install streamft
|
|
38
|
+
streamft doctor # torch / CUDA / bitsandbytes / free VRAM
|
|
38
39
|
|
|
39
|
-
#
|
|
40
|
+
# will it fit? memory arithmetic from config.json + the measured reference
|
|
40
41
|
streamft fit --model Qwen/Qwen2.5-14B-Instruct --gpu 6 --seq 512
|
|
41
|
-
streamft doctor # torch / CUDA / bitsandbytes / free VRAM
|
|
42
42
|
|
|
43
|
-
#
|
|
44
|
-
streamft prepare --model
|
|
45
|
-
python scripts/prep_data.py --model models/Qwen2.5-0.5B
|
|
43
|
+
# one NF4 checkpoint per model (HF id or local directory)
|
|
44
|
+
streamft prepare --model Qwen/Qwen2.5-0.5B --save-quant checkpoints/qwen0.5b_nf4
|
|
46
45
|
|
|
47
|
-
#
|
|
48
|
-
|
|
49
|
-
--save-quant checkpoints/qwen32b_nf4
|
|
46
|
+
# SFT on Dolly-15k — no data prep
|
|
47
|
+
streamft train --model-dir checkpoints/qwen0.5b_nf4 --sft --steps 20 --seq 512 --tag first
|
|
50
48
|
|
|
51
|
-
#
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
49
|
+
# LM path (optional): build token shards, then train without --sft
|
|
50
|
+
streamft data --model Qwen/Qwen2.5-0.5B --out data
|
|
51
|
+
streamft train --model-dir checkpoints/qwen0.5b_nf4 --steps 20 --seq 256 --tag lm
|
|
52
|
+
```
|
|
55
53
|
|
|
56
|
-
|
|
57
|
-
|
|
54
|
+
Every run appends one JSON row to `results/results.jsonl` next to where you launched it.
|
|
55
|
+
The same commands are in [`QUICKSTART.md`](QUICKSTART.md), which is what a pip user should
|
|
56
|
+
read — this README also documents the experiments behind the numbers.
|
|
58
57
|
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
58
|
+
**From a source checkout only** (not part of the wheel): `tools/finetune.py` is a wrapper
|
|
59
|
+
that encodes the VRAM presets, `scripts/compare_peft.py` reproduces the engine comparison,
|
|
60
|
+
`scripts/verify_gradients.py` the gradient check, `scripts/prep_data.py` the token shards
|
|
61
|
+
(the installed equivalent is `streamft data`), and `pytest tests -q` the CPU suite.
|
|
62
62
|
|
|
63
63
|
`tools/finetune.py` encodes the VRAM presets we measured, because the *combination* of
|
|
64
64
|
flags decides whether a run fits — not any single flag:
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "streamft"
|
|
7
|
-
version = "0.2.
|
|
7
|
+
version = "0.2.5"
|
|
8
8
|
description = "Fine-tune models that do not fit: stream NF4 weights from disk, one transformer block at a time."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
|
@@ -133,6 +133,17 @@ def _format_messages(row: dict) -> list[dict]:
|
|
|
133
133
|
]
|
|
134
134
|
|
|
135
135
|
|
|
136
|
+
def sft_schedule(rows, encode):
|
|
137
|
+
"""Split rows into trainable ones and a count of dropped ones.
|
|
138
|
+
|
|
139
|
+
The caller loops `for i in range(steps)` over the returned rows, so `--steps N`
|
|
140
|
+
always means N optimizer updates regardless of how many rows the prompt mask
|
|
141
|
+
rejects. Kept separate from the training loop so it can be tested without a GPU.
|
|
142
|
+
"""
|
|
143
|
+
valid = [r for r in rows if encode(r) is not None]
|
|
144
|
+
return valid, len(rows) - len(valid)
|
|
145
|
+
|
|
146
|
+
|
|
136
147
|
def _encode_chat(tokenizer, row: dict, seq: int):
|
|
137
148
|
"""Chat-template encode with the prompt tokens masked out of the loss."""
|
|
138
149
|
messages = _format_messages(row)
|
|
@@ -184,7 +195,12 @@ def _masked_eval(trainer, rows, tokenizer, seq: int) -> float:
|
|
|
184
195
|
if skipped:
|
|
185
196
|
print(f"[eval] skipped {skipped}/{len(rows)} rows: prompt is not an exact prefix "
|
|
186
197
|
f"of the templated row (would train on the prompt)", flush=True)
|
|
187
|
-
|
|
198
|
+
if n == 0:
|
|
199
|
+
raise RuntimeError(
|
|
200
|
+
"_masked_eval: every row was skipped (no valid prompt mask), so there is no "
|
|
201
|
+
"loss to report. Returning 0.0 here would look like a perfect model."
|
|
202
|
+
)
|
|
203
|
+
return total / n
|
|
188
204
|
|
|
189
205
|
|
|
190
206
|
def main() -> int:
|
|
@@ -369,12 +385,17 @@ def main() -> int:
|
|
|
369
385
|
f"eval={len(eval_rows)}", flush=True)
|
|
370
386
|
ce0 = _masked_eval(tr, eval_rows, tok, args.seq)
|
|
371
387
|
print(f"[eval] before loss={ce0:.4f}", flush=True)
|
|
372
|
-
|
|
388
|
+
# Filter once so `--steps N` means N optimizer updates, not "N attempts minus
|
|
389
|
+
# whatever the mask rejected". A silently short run is a lie about the budget.
|
|
390
|
+
valid_rows, skipped_train = sft_schedule(
|
|
391
|
+
train_rows, lambda r: _encode_chat(tok, r, args.seq))
|
|
392
|
+
if not valid_rows:
|
|
393
|
+
raise SystemExit("[sft] no training row has a valid prompt mask; refusing to run")
|
|
394
|
+
if skipped_train:
|
|
395
|
+
print(f"[sft] {skipped_train}/{len(train_rows)} rows dropped: the prompt is not "
|
|
396
|
+
f"an exact prefix of the templated row", flush=True)
|
|
373
397
|
for i in range(args.steps):
|
|
374
|
-
enc = _encode_chat(tok,
|
|
375
|
-
if enc is None:
|
|
376
|
-
skipped_train += 1
|
|
377
|
-
continue
|
|
398
|
+
enc = _encode_chat(tok, valid_rows[i % len(valid_rows)], args.seq)
|
|
378
399
|
ids, labels = enc
|
|
379
400
|
ys = torch.full_like(ids, -100)
|
|
380
401
|
ys[:, :-1] = labels[:, 1:]
|
|
@@ -25,16 +25,16 @@ from pathlib import Path
|
|
|
25
25
|
MEASURED = [
|
|
26
26
|
# Matched on (num_hidden_layers, hidden_size), not on a rounded parameter count: a
|
|
27
27
|
# different 7B with a different architecture must not inherit this measurement.
|
|
28
|
-
{"arch": (36, 2048), "label": "Qwen2.5-3B", "seq": 512, "blocks": 36,
|
|
28
|
+
{"arch": ("qwen2", 36, 2048, 11008, 16, 2, 151936), "label": "Qwen2.5-3B", "seq": 512, "blocks": 36,
|
|
29
29
|
"peak_gb": 3.85, "step_s": 0.94, "mode": "all resident",
|
|
30
30
|
"tokens": "SFT: one Dolly row per step, median ~134 tokens"},
|
|
31
|
-
{"arch": (28, 3584), "label": "Qwen2.5-7B", "seq": 512, "blocks": 28,
|
|
31
|
+
{"arch": ("qwen2", 28, 3584, 18944, 28, 4, 152064), "label": "Qwen2.5-7B", "seq": 512, "blocks": 28,
|
|
32
32
|
"peak_gb": 5.51, "step_s": 1.55, "mode": "all resident",
|
|
33
33
|
"tokens": "SFT: one Dolly row per step, median ~134 tokens"},
|
|
34
|
-
{"arch": (48, 5120), "label": "Qwen2.5-14B", "seq": 512, "blocks": 6,
|
|
34
|
+
{"arch": ("qwen2", 48, 5120, 13824, 40, 8, 152064), "label": "Qwen2.5-14B", "seq": 512, "blocks": 6,
|
|
35
35
|
"peak_gb": 3.86, "step_s": 5.86, "mode": "streaming",
|
|
36
36
|
"tokens": "SFT: one Dolly row per step, median ~134 tokens"},
|
|
37
|
-
{"arch": (64, 5120), "label": "Qwen2.5-32B", "seq": 1024, "blocks": 2,
|
|
37
|
+
{"arch": ("qwen2", 64, 5120, 27648, 40, 8, 152064), "label": "Qwen2.5-32B", "seq": 1024, "blocks": 2,
|
|
38
38
|
"peak_gb": 3.90, "step_s": 12.48, "mode": "streaming",
|
|
39
39
|
"tokens": "SFT: one Dolly row per step, median ~134 tokens"},
|
|
40
40
|
]
|
|
@@ -51,6 +51,19 @@ def load_config(model: str) -> dict:
|
|
|
51
51
|
return json.loads(Path(hf_hub_download(model, "config.json")).read_text(encoding="utf-8"))
|
|
52
52
|
|
|
53
53
|
|
|
54
|
+
def model_fingerprint(cfg: dict) -> tuple:
|
|
55
|
+
"""Everything that changes the measurement: two 7B models are not interchangeable."""
|
|
56
|
+
return (
|
|
57
|
+
str(cfg.get("model_type", "")),
|
|
58
|
+
int(cfg.get("num_hidden_layers") or cfg.get("n_layer") or 0),
|
|
59
|
+
int(cfg.get("hidden_size") or cfg.get("n_embd") or 0),
|
|
60
|
+
int(cfg.get("intermediate_size") or 0),
|
|
61
|
+
int(cfg.get("num_attention_heads") or 0),
|
|
62
|
+
int(cfg.get("num_key_value_heads") or cfg.get("num_attention_heads") or 0),
|
|
63
|
+
int(cfg.get("vocab_size") or 0),
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
|
|
54
67
|
def model_math(cfg: dict) -> dict:
|
|
55
68
|
h = cfg.get("hidden_size") or cfg.get("n_embd")
|
|
56
69
|
layers = cfg.get("num_hidden_layers") or cfg.get("n_layer")
|
|
@@ -83,12 +96,13 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
83
96
|
ap.add_argument("--json", action="store_true")
|
|
84
97
|
args = ap.parse_args(argv)
|
|
85
98
|
|
|
86
|
-
|
|
99
|
+
cfg = load_config(args.model)
|
|
100
|
+
m = model_math(cfg)
|
|
87
101
|
need = m["nf4_gb"] + m["head_gb"] + RESERVE_GB
|
|
88
102
|
fits_whole = need <= args.gpu
|
|
89
103
|
size_b = round(m["params"] / 1e9)
|
|
90
104
|
|
|
91
|
-
arch = (
|
|
105
|
+
arch = model_fingerprint(cfg)
|
|
92
106
|
measured = None
|
|
93
107
|
for row in MEASURED:
|
|
94
108
|
if row["arch"] == arch and row["seq"] == args.seq:
|
|
@@ -126,7 +140,8 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
126
140
|
else:
|
|
127
141
|
print(" verdict: needs streaming (one or more blocks on the GPU at a time)")
|
|
128
142
|
if measured:
|
|
129
|
-
print(f" measured reference: {measured['label']}
|
|
143
|
+
print(f" measured reference: {measured['label']} "
|
|
144
|
+
f"(model_type {arch[0]}, layers {arch[1]}, hidden {arch[2]}) "
|
|
130
145
|
f"on RTX 3050 6 GB -> {measured['blocks']} block(s), "
|
|
131
146
|
f"{measured['peak_gb']} GB peak, {measured['step_s']} s/step at seq {measured['seq']}")
|
|
132
147
|
print(f" note: {measured['tokens']} (seq is a cap on this path)")
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: streamft
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.5
|
|
4
4
|
Summary: Fine-tune models that do not fit: stream NF4 weights from disk, one transformer block at a time.
|
|
5
5
|
Author-email: Gokhan Cava <byjameson@users.noreply.github.com>
|
|
6
6
|
License: Apache-2.0
|
|
@@ -61,34 +61,34 @@ the six traps, and which numbers are real. The measurement protocol, the discard
|
|
|
61
61
|
and the correction of an earlier wrong claim ("8.6× / 100 tok/s") are in
|
|
62
62
|
[`LONGCTX_FINDINGS.md`](LONGCTX_FINDINGS.md).
|
|
63
63
|
|
|
64
|
-
## Quick start
|
|
64
|
+
## Quick start (installed package, nothing else needed)
|
|
65
65
|
|
|
66
|
-
```
|
|
66
|
+
```bash
|
|
67
67
|
pip install streamft
|
|
68
|
+
streamft doctor # torch / CUDA / bitsandbytes / free VRAM
|
|
68
69
|
|
|
69
|
-
#
|
|
70
|
+
# will it fit? memory arithmetic from config.json + the measured reference
|
|
70
71
|
streamft fit --model Qwen/Qwen2.5-14B-Instruct --gpu 6 --seq 512
|
|
71
|
-
streamft doctor # torch / CUDA / bitsandbytes / free VRAM
|
|
72
72
|
|
|
73
|
-
#
|
|
74
|
-
streamft prepare --model
|
|
75
|
-
python scripts/prep_data.py --model models/Qwen2.5-0.5B
|
|
73
|
+
# one NF4 checkpoint per model (HF id or local directory)
|
|
74
|
+
streamft prepare --model Qwen/Qwen2.5-0.5B --save-quant checkpoints/qwen0.5b_nf4
|
|
76
75
|
|
|
77
|
-
#
|
|
78
|
-
|
|
79
|
-
--save-quant checkpoints/qwen32b_nf4
|
|
76
|
+
# SFT on Dolly-15k — no data prep
|
|
77
|
+
streamft train --model-dir checkpoints/qwen0.5b_nf4 --sft --steps 20 --seq 512 --tag first
|
|
80
78
|
|
|
81
|
-
#
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
79
|
+
# LM path (optional): build token shards, then train without --sft
|
|
80
|
+
streamft data --model Qwen/Qwen2.5-0.5B --out data
|
|
81
|
+
streamft train --model-dir checkpoints/qwen0.5b_nf4 --steps 20 --seq 256 --tag lm
|
|
82
|
+
```
|
|
85
83
|
|
|
86
|
-
|
|
87
|
-
|
|
84
|
+
Every run appends one JSON row to `results/results.jsonl` next to where you launched it.
|
|
85
|
+
The same commands are in [`QUICKSTART.md`](QUICKSTART.md), which is what a pip user should
|
|
86
|
+
read — this README also documents the experiments behind the numbers.
|
|
88
87
|
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
88
|
+
**From a source checkout only** (not part of the wheel): `tools/finetune.py` is a wrapper
|
|
89
|
+
that encodes the VRAM presets, `scripts/compare_peft.py` reproduces the engine comparison,
|
|
90
|
+
`scripts/verify_gradients.py` the gradient check, `scripts/prep_data.py` the token shards
|
|
91
|
+
(the installed equivalent is `streamft data`), and `pytest tests -q` the CPU suite.
|
|
92
92
|
|
|
93
93
|
`tools/finetune.py` encodes the VRAM presets we measured, because the *combination* of
|
|
94
94
|
flags decides whether a run fits — not any single flag:
|
|
@@ -8,6 +8,7 @@ They pin the things that broke silently during development:
|
|
|
8
8
|
|
|
9
9
|
import math
|
|
10
10
|
|
|
11
|
+
import pytest
|
|
11
12
|
import torch
|
|
12
13
|
import torch.nn as nn
|
|
13
14
|
import torch.nn.functional as F
|
|
@@ -209,3 +210,41 @@ def test_encode_chat_drops_rows_whose_prompt_is_not_a_prefix():
|
|
|
209
210
|
ids, labels = _encode_chat(_StubTokenizer([1, 2, 3, 4], [1, 2]), row, 32)
|
|
210
211
|
assert ids.tolist() == [[1, 2, 3, 4]]
|
|
211
212
|
assert labels.tolist() == [[-100, -100, 3, 4]]
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
# ------------------------------------------------ SFT step budget & empty eval
|
|
216
|
+
|
|
217
|
+
def test_sft_schedule_keeps_the_step_budget_with_mixed_rows():
|
|
218
|
+
from streamft.cli import sft_schedule
|
|
219
|
+
|
|
220
|
+
rows = list(range(10))
|
|
221
|
+
|
|
222
|
+
def encode(r):
|
|
223
|
+
return None if r % 3 == 0 else [r] # 4 of 10 are rejected
|
|
224
|
+
|
|
225
|
+
valid, dropped = sft_schedule(rows, encode)
|
|
226
|
+
assert dropped == 4 and len(valid) == 6
|
|
227
|
+
|
|
228
|
+
steps = 25
|
|
229
|
+
schedule = [valid[i % len(valid)] for i in range(steps)]
|
|
230
|
+
assert len(schedule) == steps # N updates even though 4 rows were dropped
|
|
231
|
+
assert all(v % 3 != 0 for v in schedule)
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def test_masked_eval_raises_when_every_row_is_skipped(monkeypatch):
|
|
235
|
+
import types
|
|
236
|
+
|
|
237
|
+
from streamft import cli
|
|
238
|
+
|
|
239
|
+
class FakeTrainer:
|
|
240
|
+
def __init__(self):
|
|
241
|
+
self.model = types.SimpleNamespace(eval=lambda: None, train=lambda: None)
|
|
242
|
+
self.norm = nn.Identity()
|
|
243
|
+
self.lm_head = nn.Linear(4, 7, bias=False)
|
|
244
|
+
|
|
245
|
+
def _forward_cached(self, ids):
|
|
246
|
+
return torch.zeros(1, ids.shape[1], 4), None, None
|
|
247
|
+
|
|
248
|
+
monkeypatch.setattr(cli, "_encode_chat", lambda *a, **k: None)
|
|
249
|
+
with pytest.raises(RuntimeError, match="every row was skipped"):
|
|
250
|
+
cli._masked_eval(FakeTrainer(), [{"instruction": "x", "response": "y"}], object(), 8)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|