streamft 0.2.2__tar.gz → 0.2.3__tar.gz

{streamft-0.2.2 → streamft-0.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: streamft
-Version: 0.2.2
+Version: 0.2.3
 Summary: Fine-tune models that do not fit: stream NF4 weights from disk, one transformer block at a time.
 Author-email: Gokhan Cava <byjameson@users.noreply.github.com>
 License: Apache-2.0

{streamft-0.2.2 → streamft-0.2.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "streamft"
-version = "0.2.2"
+version = "0.2.3"
 description = "Fine-tune models that do not fit: stream NF4 weights from disk, one transformer block at a time."
 readme = "README.md"
 requires-python = ">=3.10"

{streamft-0.2.2 → streamft-0.2.3}/streamft/cli.py

@@ -86,11 +86,23 @@ def user_path(value) -> Path:
     return q if q.is_absolute() else Path.cwd() / q
 
 
+def resolve_model(value: str) -> str:
+    """A local directory if one exists at that path, otherwise an HF model id.
+
+    `user_path` must not be used for models: `Qwen/Qwen2.5-0.5B` is an id, not
+    `./Qwen/Qwen2.5-0.5B`. Absolute paths and existing directories win; everything else
+    is passed through to transformers/huggingface_hub unchanged.
+    """
+    p = Path(value).expanduser()
+    local = p if p.is_absolute() else Path.cwd() / p
+    return str(local) if local.exists() else value
+
+
 def prepare(model: str, save_quant: str) -> int:
     """Quantize a local bf16 model into a bnb-4bit checkpoint (uint8 + qstate)."""
     from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
 
-    src = str(user_path(model))
+    src = resolve_model(model)
     dst = user_path(save_quant)
     print(f"[prepare] {model} -> {save_quant}", flush=True)
     bnb = BitsAndBytesConfig(
@@ -131,11 +143,13 @@ def _encode_chat(tokenizer, row: dict, seq: int):
                                                 add_generation_prompt=True)
     prompt = tokenizer(prompt_text, add_special_tokens=False).input_ids
     full = full[:seq]
-    labels = list(full)
-    if 0 < len(prompt) < len(full) and full[:len(prompt)] == prompt[:len(full)]:
-        k = min(len(prompt), len(full))
-        labels = [-100] * k + labels[k:]
-    if len(labels) < 2:
+    # The prompt must be an exact prefix of the templated row. If it is not, the mask
+    # would silently cover nothing and the model would be trained on its own prompt, so
+    # the example is dropped instead (and counted by the caller).
+    if len(prompt) >= len(full) or full[: len(prompt)] != prompt:
+        return None
+    labels = [-100] * len(prompt) + list(full[len(prompt):])
+    if len(labels) < 2 or all(v == -100 for v in labels):
         return None
     return (torch.tensor([full], dtype=torch.long),
             torch.tensor([labels], dtype=torch.long))
@@ -145,10 +159,11 @@ def _encode_chat(tokenizer, row: dict, seq: int):
 def _masked_eval(trainer, rows, tokenizer, seq: int) -> float:
     """Masked SFT loss through the streaming (disk-backed) forward."""
     trainer.model.eval()
-    total, n = 0.0, 0
+    total, n, skipped = 0.0, 0, 0
     for row in rows:
         enc = _encode_chat(tokenizer, row, seq)
         if enc is None:
+            skipped += 1
             continue
         ids, labels = enc
         ys = torch.full_like(ids, -100)
@@ -166,6 +181,9 @@ def _masked_eval(trainer, rows, tokenizer, seq: int) -> float:
             tgt[valid].to(logits.device), reduction="sum"))
         n += v
     trainer.model.train()
+    if skipped:
+        print(f"[eval] skipped {skipped}/{len(rows)} rows: prompt is not an exact prefix "
+              f"of the templated row (would train on the prompt)", flush=True)
     return total / max(n, 1)
 
 
@@ -222,7 +240,7 @@ def main() -> int:
     if args.prepare:
         return prepare(args.model, args.save_quant)
 
-    model_dir = str(user_path(args.model_dir))
+    model_dir = resolve_model(args.model_dir)
     print(f"[exp54] model_dir={Path(model_dir).name} steps={args.steps} seq={args.seq} "
           f"lora_r={args.lora_r} targets={args.targets} head={args.head}", flush=True)
 
@@ -351,9 +369,11 @@ def main() -> int:
               f"eval={len(eval_rows)}", flush=True)
         ce0 = _masked_eval(tr, eval_rows, tok, args.seq)
         print(f"[eval] before loss={ce0:.4f}", flush=True)
+        skipped_train = 0
         for i in range(args.steps):
             enc = _encode_chat(tok, train_rows[i % len(train_rows)], args.seq)
             if enc is None:
+                skipped_train += 1
                 continue
             ids, labels = enc
             ys = torch.full_like(ids, -100)
@@ -435,6 +455,7 @@ def main() -> int:
         "adapter_sha_before": sha_before,
         "adapter_sha_after": adapter_sha(model),
         "steps": len(losses),
+        "sft_skipped": skipped_train if args.sft else 0,
         "seq": args.seq,
         "lr": args.lr,
         "losses": losses,

{streamft-0.2.2 → streamft-0.2.3}/streamft/fit.py

@@ -22,12 +22,22 @@ from pathlib import Path
 
 # Measured end-to-end on an RTX 3050 Laptop, 6 GB, Windows 11, AC power, NF4 base,
 # LoRA r=8 on q/k/v/o, batch 1, Dolly-15k. step_s = median warm step.
-MEASURED = {
-    (3, 512): {"blocks": 36, "peak_gb": 3.85, "step_s": 0.938, "mode": "all resident"},
-    (7, 512): {"blocks": 28, "peak_gb": 5.51, "step_s": 1.696, "mode": "all resident"},
-    (14, 512): {"blocks": 6, "peak_gb": 3.86, "step_s": 6.726, "mode": "streaming"},
-    (32, 1024): {"blocks": 2, "peak_gb": 3.90, "step_s": 12.541, "mode": "streaming"},
-}
+MEASURED = [
+    # Matched on (num_hidden_layers, hidden_size), not on a rounded parameter count: a
+    # different 7B with a different architecture must not inherit this measurement.
+    {"arch": (36, 2048), "label": "Qwen2.5-3B", "seq": 512, "blocks": 36,
+     "peak_gb": 3.85, "step_s": 0.94, "mode": "all resident",
+     "tokens": "SFT: one Dolly row per step, median ~134 tokens"},
+    {"arch": (28, 3584), "label": "Qwen2.5-7B", "seq": 512, "blocks": 28,
+     "peak_gb": 5.51, "step_s": 1.55, "mode": "all resident",
+     "tokens": "SFT: one Dolly row per step, median ~134 tokens"},
+    {"arch": (48, 5120), "label": "Qwen2.5-14B", "seq": 512, "blocks": 6,
+     "peak_gb": 3.86, "step_s": 5.86, "mode": "streaming",
+     "tokens": "SFT: one Dolly row per step, median ~134 tokens"},
+    {"arch": (64, 5120), "label": "Qwen2.5-32B", "seq": 1024, "blocks": 2,
+     "peak_gb": 3.90, "step_s": 12.48, "mode": "streaming",
+     "tokens": "SFT: one Dolly row per step, median ~134 tokens"},
+]
 
 RESERVE_GB = 0.9  # CUDA context + allocator fragmentation + the bf16 recompute buffers
 
@@ -62,7 +72,7 @@ def model_math(cfg: dict) -> dict:
     # presets use --shared-dtype as-stored. A tied head has no separate tensor.
     head_gb = 0.0 if tied else vocab * h * 2 / 1e9
     return {"params": params, "layers": int(layers), "nf4_gb": nf4_gb,
-            "head_gb": head_gb, "tied": tied}
+            "head_gb": head_gb, "tied": tied, "hidden": int(h)}
 
 
 def main(argv: list[str] | None = None) -> int:
@@ -78,10 +88,11 @@ def main(argv: list[str] | None = None) -> int:
     fits_whole = need <= args.gpu
     size_b = round(m["params"] / 1e9)
 
+    arch = (m["layers"], m["hidden"])
     measured = None
-    for (b, seq), row in MEASURED.items():
-        if abs(size_b - b) <= max(1, round(b * 0.1)) and seq == args.seq:
-            measured = dict(row, ref_size_b=b, ref_seq=seq)
+    for row in MEASURED:
+        if row["arch"] == arch and row["seq"] == args.seq:
+            measured = dict(row)
             break
 
     out = {
@@ -115,8 +126,10 @@ def main(argv: list[str] | None = None) -> int:
     else:
         print("  verdict: needs streaming (one or more blocks on the GPU at a time)")
     if measured:
-        print(f"  measured on RTX 3050 6 GB: {measured['blocks']} block(s), "
-              f"{measured['peak_gb']} GB peak, {measured['step_s']} s/step at seq {measured['ref_seq']}")
+        print(f"  measured reference: {measured['label']} (layers {arch[0]}, hidden {arch[1]}) "
+              f"on RTX 3050 6 GB -> {measured['blocks']} block(s), "
+              f"{measured['peak_gb']} GB peak, {measured['step_s']} s/step at seq {measured['seq']}")
+        print(f"  note: {measured['tokens']} (seq is a cap on this path)")
     else:
         print("  speed/blocks: not measured for this size at this seq — start with")
         print("                 --blocks-on-gpu 1 and let autotune grow it (it measures")

{streamft-0.2.2 → streamft-0.2.3}/streamft.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: streamft
-Version: 0.2.2
+Version: 0.2.3
 Summary: Fine-tune models that do not fit: stream NF4 weights from disk, one transformer block at a time.
 Author-email: Gokhan Cava <byjameson@users.noreply.github.com>
 License: Apache-2.0

{streamft-0.2.2 → streamft-0.2.3}/tests/test_streamft_smoke.py

@@ -177,3 +177,35 @@ def test_disk_store_supports_single_safetensors_file(tmp_path):
     store = DiskQuantStore(str(tmp_path))
     assert store.has("a") and store.has("b.c")
     assert store.get("b.c").tolist() == [1.0, 1.0, 1.0]
+
+
+# ------------------------------------------------ SFT masking failures are loud
+
+class _StubTokenizer:
+    """apply_chat_template returns FULL for both messages, PROMPT for the prefix call."""
+
+    def __init__(self, full, prompt):
+        self.full, self.prompt = full, prompt
+
+    def apply_chat_template(self, messages, tokenize=False, add_generation_prompt=False):
+        return "FULL" if len(messages) == 2 else "PROMPT"
+
+    def __call__(self, text, add_special_tokens=False):
+        from types import SimpleNamespace
+
+        ids = self.full if text == "FULL" else self.prompt
+        return SimpleNamespace(input_ids=list(ids))
+
+
+def test_encode_chat_drops_rows_whose_prompt_is_not_a_prefix():
+    from streamft.cli import _encode_chat
+
+    row = {"instruction": "q", "response": "a"}
+    # prompt is not a prefix of the templated row -> the example must be dropped
+    assert _encode_chat(_StubTokenizer([1, 2, 3, 4], [9, 9]), row, 32) is None
+    # prompt longer than the row -> dropped
+    assert _encode_chat(_StubTokenizer([1, 2], [1, 2, 3]), row, 32) is None
+    # valid prefix -> every prompt token masked, the rest kept
+    ids, labels = _encode_chat(_StubTokenizer([1, 2, 3, 4], [1, 2]), row, 32)
+    assert ids.tolist() == [[1, 2, 3, 4]]
+    assert labels.tolist() == [[-100, -100, 3, 4]]