tunekeep 0.2.1__py3-none-any.whl

tunekeep/compare.py

@@ -0,0 +1,276 @@
+"""`tunekeep compare`: does another model already answer like your fine-tune?
+
+Sends each recorded prompt to a candidate model (gpt-6-luna, any
+OpenAI-compatible endpoint, or your retrained model on llama-server or vLLM),
+optionally with a few examples from your own training data, and scores the
+answer against the recorded one (and against the training reference if any).
+"""
+
+from __future__ import annotations
+
+import datetime as dt
+import re
+import statistics
+import time
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from .client import Client
+from .record import load_prompts
+from .runner import RunResult, done_ids, iter_jsonl, run_parallel
+from .score import message_text, score
+
+EFFORTS = ("auto", "default", "none", "minimal", "low", "medium", "high", "xhigh", "max")
+# Request fields carried over from the recorded request.
+_CARRY = ("tools", "tool_choice", "parallel_tool_calls", "response_format")
+
+
+class CompareError(ValueError):
+    pass
+
+
+# Chat Completions rules for OpenAI reasoning models (checked 2026-11-07):
+#   https://developers.openai.com/api/docs/guides/latest-model
+#   https://developers.openai.com/api/docs/models/gpt-6-luna
+# - GPT-6 Luna/Sol/Terra and GPT-5.1+ accept reasoning_effort "none"; temperature, top_p and
+#   function tools work in Chat Completions only at "none".
+# - GPT-6 Astra and GPT-6.1 Sol have no "none" and no function tools in Chat Completions.
+# - o-series: no "none", no temperature.
+def family(model: str) -> str:
+    m = model.lower()
+    if m.startswith(("gpt-6-astra", "gpt-6.1-sol")):
+        return "no-none"
+    if re.match(r"^o\d", m):
+        return "o-series"
+    if m in ("gpt-5", "gpt-5-mini", "gpt-5-nano") or re.match(r"^gpt-5-(mini|nano)?-?\d{4}-", m):
+        return "gpt-5"
+    if m.startswith(("gpt-6", "gpt-5.")):
+        return "none-ok"
+    return "plain"
+
+
+def plan_params(model: str, effort: str, temperature: float | None, has_tools: bool) -> tuple[dict[str, Any], list[str]]:
+    """Request parameters for the candidate, and warnings about what was changed."""
+    fam = family(model)
+    notes: list[str] = []
+    if effort == "auto":
+        effort = {"none-ok": "none", "no-none": "low", "o-series": "low", "gpt-5": "minimal"}.get(fam, "default")
+    params: dict[str, Any] = {}
+    if effort != "default":
+        params["reasoning_effort"] = effort
+    sampling_ok = fam == "plain" or (fam == "none-ok" and effort == "none")
+    if temperature is not None:
+        if sampling_ok:
+            params["temperature"] = temperature
+        else:
+            notes.append(f"temperature not sent: {model} accepts it only with reasoning_effort none")
+    if has_tools:
+        if fam == "no-none":
+            raise CompareError(f"{model} does not support function tools in Chat Completions.")
+        if fam == "none-ok" and effort != "none":
+            raise CompareError(
+                f"{model} supports function tools in Chat Completions only with --reasoning-effort none."
+            )
+    return params, notes
+
+
+@dataclass
+class Shot:
+    id: str
+    messages: list[dict[str, Any]]
+
+
+def load_shots(path: Path) -> list[Shot]:
+    """Examples from your own training file: single-turn, plain-text answers."""
+    shots: list[Shot] = []
+    for p in load_prompts(path):
+        ref = p.reference
+        if not isinstance(ref, dict) or ref.get("tool_calls") or not message_text(ref):
+            continue
+        turns = [m for m in p.messages if m.get("role") not in ("system", "developer")]
+        if len(turns) != 1 or turns[0].get("role") != "user" or p.extra.get("tools"):
+            continue
+        shots.append(Shot(p.id, [turns[0], {"role": "assistant", "content": message_text(ref)}]))
+    return shots
+
+
+def with_shots(messages: list[dict[str, Any]], shots: list[Shot], n: int, exclude: str) -> tuple[list[dict[str, Any]], int]:
+    chosen = [s for s in shots if s.id != exclude][:n]
+    if not chosen:
+        return messages, 0
+    head = [m for m in messages[:1] if m.get("role") in ("system", "developer")]
+    examples = [m for s in chosen for m in s.messages]
+    return head + examples + messages[len(head):], len(chosen)
+
+
+def load_golden(path: Path) -> list[dict[str, Any]]:
+    rows, seen = [], set()
+    for _, row in iter_jsonl(path):
+        if row.get("id") and row["id"] not in seen and isinstance(row.get("request"), dict):
+            seen.add(row["id"])
+            rows.append(row)
+    if not rows:
+        raise CompareError(f"{path}: no recorded rows. Run `tunekeep record` first.")
+    rows.sort(key=lambda r: r.get("line") if isinstance(r.get("line"), int) else 0)  # file order, not finish order
+    return rows
+
+
+def run(
+    client: Client,
+    golden: list[dict[str, Any]],
+    out: Path,
+    *,
+    candidate: str,
+    label: str,
+    effort: str = "auto",
+    temperature: float | None = 0.0,
+    max_tokens: int | None = None,
+    shots: list[Shot] | None = None,
+    n_shots: int = 0,
+    extra_body: dict[str, Any] | None = None,
+    workers: int = 4,
+    progress=None,
+) -> tuple[RunResult, list[str]]:
+    has_tools = any(row["request"].get("tools") for row in golden)
+    params, notes = plan_params(candidate, effort, temperature, has_tools)
+    done = done_ids(out, "candidate", label)
+    todo = [row for row in golden if row["id"] not in done]
+
+    def one(row: dict[str, Any]) -> dict[str, Any]:
+        req = row["request"]
+        messages, used = with_shots(req.get("messages") or [], shots or [], n_shots, row["id"])
+        body: dict[str, Any] = {"model": candidate, "messages": messages, **params}
+        body.update({k: req[k] for k in _CARRY if k in req})
+        if max_tokens is not None:
+            body["max_completion_tokens"] = max_tokens
+        body.update(extra_body or {})
+        start = time.monotonic()
+        resp = client.chat(body)
+        latency = round((time.monotonic() - start) * 1000)
+        choice = (resp.get("choices") or [{}])[0]
+        message = choice.get("message") or {}
+        ref = row.get("reference")
+        return {
+            "id": row["id"],
+            "line": row.get("line"),
+            "candidate": label,
+            "model": candidate,
+            "golden_model": row.get("model"),
+            "shots": used,
+            "params": {k: v for k, v in body.items() if k not in ("model", "messages", "tools")},
+            "response": {"message": message, "finish_reason": choice.get("finish_reason")},
+            "scores": {
+                "golden": score((row.get("response") or {}).get("message"), message),
+                "reference": score(ref, message) if isinstance(ref, dict) else None,
+            },
+            "usage": resp.get("usage"),
+            "latency_ms": latency,
+            "compared_at": dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds"),
+        }
+
+    result = run_parallel(todo, one, out, total=len(golden), workers=workers, progress=progress)
+    return result, notes
+
+
+# -- summaries ---------------------------------------------------------------------
+
+
+def _pct(values: list[float], q: float) -> float | None:
+    if not values:
+        return None
+    values = sorted(values)
+    k = (len(values) - 1) * q
+    lo, hi = int(k), min(int(k) + 1, len(values) - 1)
+    return values[lo] + (values[hi] - values[lo]) * (k - lo)
+
+
+def summarize_scores(scores: list[dict[str, Any]]) -> dict[str, Any]:
+    scores = [s for s in scores if s]
+    if not scores:
+        return {"n": 0}
+    kinds: dict[str, int] = {}
+    for s in scores:
+        kinds[s["kind"]] = kinds.get(s["kind"], 0) + 1
+    json_rows = [s for s in scores if s["kind"] == "json"]
+    return {
+        "n": len(scores),
+        "mean": round(statistics.fmean(s["score"] for s in scores), 4),
+        "exact": round(sum(1 for s in scores if s.get("exact")) / len(scores), 4),
+        "match": round(sum(1 for s in scores if s.get("exact") or s.get("normalized") or s["score"] >= 0.999)
+                       / len(scores), 4),
+        "json_valid": round(sum(1 for s in json_rows if s.get("json_valid")) / len(json_rows), 4) if json_rows else None,
+        "kinds": kinds,
+    }
+
+
+def summarize(rows: list[dict[str, Any]]) -> dict[str, Any]:
+    lat = [r["latency_ms"] for r in rows if isinstance(r.get("latency_ms"), (int, float))]
+    usage = [r.get("usage") or {} for r in rows]
+    details = [u.get("completion_tokens_details") or {} for u in usage]
+    return {
+        "candidate": rows[0]["candidate"] if rows else None,
+        "model": rows[0].get("model") if rows else None,
+        "rows": len(rows),
+        "vs_golden": summarize_scores([(r.get("scores") or {}).get("golden") for r in rows]),
+        "vs_reference": summarize_scores([(r.get("scores") or {}).get("reference") for r in rows]),
+        "latency_p50_ms": _round(_pct(lat, 0.5)),
+        "latency_p95_ms": _round(_pct(lat, 0.95)),
+        "prompt_tokens": sum(u.get("prompt_tokens") or 0 for u in usage),
+        "completion_tokens": sum(u.get("completion_tokens") or 0 for u in usage),
+        "reasoning_tokens": sum(d.get("reasoning_tokens") or 0 for d in details),
+    }
+
+
+def golden_baseline(golden: list[dict[str, Any]]) -> dict[str, Any] | None:
+    """How closely the fine-tune itself matched its training answers (no API calls)."""
+    scored = [score(r["reference"], (r.get("response") or {}).get("message"))
+              for r in golden if isinstance(r.get("reference"), dict)]
+    if not scored:
+        return None
+    lat = [r["latency_ms"] for r in golden if isinstance(r.get("latency_ms"), (int, float))]
+    usage = [r.get("usage") or {} for r in golden]
+    return {
+        "candidate": f"{golden[0].get('model')} (recorded)",
+        "model": golden[0].get("model"),
+        "rows": len(golden),
+        "vs_golden": {"n": 0},
+        "vs_reference": summarize_scores(scored),
+        "latency_p50_ms": _round(_pct(lat, 0.5)),
+        "latency_p95_ms": _round(_pct(lat, 0.95)),
+        "prompt_tokens": sum(u.get("prompt_tokens") or 0 for u in usage),
+        "completion_tokens": sum(u.get("completion_tokens") or 0 for u in usage),
+        "reasoning_tokens": 0,
+    }
+
+
+def _round(x: float | None) -> int | None:
+    return None if x is None else round(x)
+
+
+def load_rows(path: Path) -> dict[str, list[dict[str, Any]]]:
+    """Compare rows grouped by candidate label (a file can hold several runs)."""
+    groups: dict[str, list[dict[str, Any]]] = {}
+    for _, row in iter_jsonl(path):
+        if row.get("candidate") and row.get("scores"):
+            groups.setdefault(row["candidate"], []).append(row)
+    return groups
+
+
+def render_summary(s: dict[str, Any]) -> str:
+    def pct(x: float | None) -> str:
+        return "-" if x is None else f"{x * 100:.1f}%"
+
+    g, r = s["vs_golden"], s["vs_reference"]
+    lines = [f"{s['candidate']}: {s['rows']} prompts"]
+    if g.get("n"):
+        lines.append(f"  vs recorded fine-tune: score {pct(g['mean'])}, same answer {pct(g['match'])}, exact {pct(g['exact'])}"
+                     + (f", valid JSON {pct(g['json_valid'])}" if g.get("json_valid") is not None else ""))
+    if r.get("n"):
+        lines.append(f"  vs training answers:   score {pct(r['mean'])}, same answer {pct(r['match'])}")
+    if s["latency_p50_ms"] is not None:
+        lines.append(f"  latency p50 {s['latency_p50_ms']} ms, p95 {s['latency_p95_ms']} ms; "
+                     f"tokens in {s['prompt_tokens']:,}, out {s['completion_tokens']:,}"
+                     + (f" (reasoning {s['reasoning_tokens']:,})" if s["reasoning_tokens"] else ""))
+    return "\n".join(lines)
+

tunekeep/files.py

@@ -0,0 +1,42 @@
+"""File helpers. Outputs can hold production prompts, so they are private by default."""
+
+from __future__ import annotations
+
+import os
+import re
+from pathlib import Path
+
+
+def ensure_private_dir(path: Path) -> None:
+    path.mkdir(parents=True, exist_ok=True)
+    try:
+        os.chmod(path, 0o700)
+    except OSError:
+        pass
+
+
+def write_private(path: Path, data: bytes) -> None:
+    """Write a file readable only by the current user (atomic replace)."""
+    ensure_private_dir(path.parent)
+    tmp = path.with_name(f".{path.name}.tmp")
+    fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
+    try:
+        with os.fdopen(fd, "wb") as fh:
+            fh.write(data)
+    except BaseException:
+        tmp.unlink(missing_ok=True)
+        raise
+    os.replace(tmp, path)
+
+
+def open_private_append(path: Path):
+    ensure_private_dir(path.parent)
+    fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_APPEND, 0o600)
+    return os.fdopen(fd, "a", encoding="utf-8")
+
+
+def safe_name(name: str) -> str:
+    """A filename that cannot escape its directory."""
+    base = os.path.basename(name.replace("\\", "/"))
+    base = re.sub(r"[^A-Za-z0-9._-]+", "_", base).strip("._")
+    return base[:120] or "file"

tunekeep/inventory.py

@@ -0,0 +1,173 @@
+"""`tunekeep inventory`: everything an account has fine-tuned, and when it dies."""
+
+from __future__ import annotations
+
+import datetime as dt
+import json
+from pathlib import Path
+from typing import Any
+
+from . import __version__
+from .client import APIError, Client
+from .files import safe_name, write_private
+from .schedule import DEPRECATIONS_URL, REPLACEMENT, TRAINING_ENDS, base_of, days_left, shutdown_for
+
+
+def collect(client: Client, out_dir: Path, download: bool = True, today: dt.date | None = None) -> dict[str, Any]:
+    notes: list[str] = []
+    models = client.list_models()
+    listed = {m.get("id"): m for m in models if m.get("id")}
+
+    try:
+        jobs = client.list_jobs()
+    except APIError as e:
+        jobs = []
+        notes.append(f"Could not list fine-tuning jobs ({e}). Only models listed by /v1/models are shown.")
+
+    by_model: dict[str, dict[str, Any]] = {}
+
+    def entry(model_id: str) -> dict[str, Any]:
+        if model_id not in by_model:
+            api_value = (listed.get(model_id) or {}).get("shutdown_date")
+            date, source = shutdown_for(model_id, api_value)
+            by_model[model_id] = {
+                "id": model_id,
+                "base": base_of(model_id),
+                "listed": model_id in listed,
+                "shutdown_date": date.isoformat() if date else None,
+                "shutdown_source": source,
+                "days_left": days_left(date, today),
+                "job": None,
+                # Checkpoint ids end in ":ckpt-step-N"; this holds even when jobs can't be listed.
+                "checkpoint_of": model_id.split(":ckpt-step-")[0] if ":ckpt-step-" in model_id else None,
+            }
+        return by_model[model_id]
+
+    for model_id in listed:
+        if model_id.startswith("ft:"):
+            entry(model_id)
+
+    files: dict[str, dict[str, Any]] = {}
+    job_rows: list[dict[str, Any]] = []
+    for job in jobs:
+        job_id = job.get("id")
+        ft_model = job.get("fine_tuned_model")
+        row = {
+            "id": job_id,
+            "status": job.get("status"),
+            "base": job.get("model"),
+            "fine_tuned_model": ft_model,
+            "method": (job.get("method") or {}).get("type") or "supervised",
+            "created_at": job.get("created_at"),
+            "training_file": job.get("training_file"),
+            "validation_file": job.get("validation_file"),
+            "result_files": job.get("result_files") or [],
+            "hyperparameters": job.get("hyperparameters"),
+            "trained_tokens": job.get("trained_tokens"),
+            "seed": job.get("seed"),
+            "suffix": job.get("suffix"),
+            "checkpoints": [],
+        }
+        if ft_model:
+            entry(ft_model)["job"] = job_id
+            try:
+                for ck in client.list_checkpoints(job_id):
+                    name = ck.get("fine_tuned_model_checkpoint")
+                    row["checkpoints"].append(
+                        {"id": ck.get("id"), "model": name, "step": ck.get("step_number"), "metrics": ck.get("metrics")}
+                    )
+                    if name:
+                        e = entry(name)
+                        e["job"] = job_id
+                        e["checkpoint_of"] = ft_model
+            except APIError as e:
+                notes.append(f"Could not list checkpoints for {job_id} ({e}).")
+        job_rows.append(row)
+
+        roles = [("training", row["training_file"]), ("validation", row["validation_file"])]
+        roles += [("result", f) for f in row["result_files"]]
+        for role, file_id in roles:
+            if file_id and file_id not in files:
+                files[file_id] = _file_record(client, file_id, role, job_id, out_dir, download)
+
+    inventory = {
+        "tool": f"tunekeep {__version__}",
+        "generated_at": dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds"),
+        "base_url": client.base_url,
+        "training_ends": TRAINING_ENDS.isoformat(),
+        "replacement": REPLACEMENT,
+        "sources": [DEPRECATIONS_URL],
+        "models": sorted(by_model.values(), key=lambda m: (m["checkpoint_of"] is not None, m["id"])),
+        "jobs": job_rows,
+        "files": list(files.values()),
+        "notes": notes,
+    }
+    write_private(out_dir / "inventory.json", json.dumps(inventory, indent=2).encode("utf-8"))
+    return inventory
+
+
+def _file_record(client: Client, file_id: str, role: str, job_id: str, out_dir: Path, download: bool) -> dict[str, Any]:
+    rec: dict[str, Any] = {"id": file_id, "role": role, "job": job_id, "filename": None, "bytes": None,
+                           "purpose": None, "saved_to": None, "download": "skipped"}
+    try:
+        meta = client.get_file(file_id)
+        rec.update(filename=meta.get("filename"), bytes=meta.get("bytes"), purpose=meta.get("purpose"))
+    except APIError as e:
+        rec["download"] = f"metadata unavailable: {e.message}"
+        return rec
+    if not download:
+        return rec
+    try:
+        data = client.file_content(file_id)
+    except APIError as e:
+        # OpenAI does not return files uploaded with purpose=fine-tune.
+        rec["download"] = f"blocked by the API ({e.status}): {e.message}"
+        return rec
+    target = out_dir / "files" / f"{file_id}-{safe_name(rec['filename'] or 'file')}"
+    write_private(target, data)
+    rec["saved_to"] = str(target)
+    rec["download"] = "saved"
+    return rec
+
+
+def render(inv: dict[str, Any]) -> str:
+    """Plain-text summary for the terminal."""
+    lines: list[str] = []
+    today = dt.datetime.now(dt.timezone.utc).date()
+    left_training = (dt.date.fromisoformat(inv["training_ends"]) - today).days
+    models = [m for m in inv["models"] if not m["checkpoint_of"]]
+    ckpts = [m for m in inv["models"] if m["checkpoint_of"]]
+
+    ck = f" (plus {len(ckpts)} checkpoint{'s' if len(ckpts) != 1 else ''})" if ckpts else ""
+    lines.append(f"Fine-tuned models: {len(models)}{ck}")
+    if left_training >= 0:
+        lines.append(f"New fine-tuning jobs stop on {inv['training_ends']} ({left_training} days).")
+    lines.append("")
+    if models:
+        w = max(len(m["id"]) for m in models)
+        lines.append(f"{'MODEL'.ljust(w)}  SHUTS DOWN   DAYS LEFT  SOURCE")
+        for m in models:
+            d = m["days_left"]
+            left = "unknown" if d is None else ("gone" if d < 0 else str(d))
+            lines.append(f"{m['id'].ljust(w)}  {(m['shutdown_date'] or '-').ljust(11)}  {left.ljust(9)}  {m['shutdown_source']}")
+    else:
+        lines.append("No fine-tuned models found for this key.")
+
+    blocked = [f for f in inv["files"] if f["role"] in ("training", "validation") and f["download"].startswith("blocked")]
+    saved = [f for f in inv["files"] if f["download"] == "saved"]
+    lines.append("")
+    noun = "file" if len(blocked) == 1 else "files"
+    lines.append(f"Files: {len(saved)} saved, {len(blocked)} training/validation {noun} the API would not return.")
+    if blocked:
+        lines.append("  Keep your own copy of these; tunekeep retrains only from your original data:")
+        for f in blocked:
+            lines.append(f"  - {f['id']}  {f['filename'] or ''}  (job {f['job']})")
+    for note in inv["notes"]:
+        lines.append(f"Note: {note}")
+
+    alive = [m for m in models if m["days_left"] is not None and m["days_left"] >= 0]
+    if alive:
+        lines.append("")
+        lines.append("Next: record how each model behaves while it still answers, e.g.")
+        lines.append(f"  tunekeep record --model {alive[0]['id']} --prompts prompts.jsonl")
+    return "\n".join(lines)

tunekeep/kit/README.md

@@ -0,0 +1,47 @@
+# Retraining kit (written by tunekeep)
+
+This folder retrains an open model with LoRA on **your own training data** and
+exports it for local serving.
+
+| File | What it is |
+|---|---|
+| `data/train.jsonl`, `data/validation.jsonl` | Your data, converted to plain chat format. Private (0600); not committed (`.gitignore`). |
+| `config.json` | Base model (pinned revision), epochs, LoRA settings, GPU, time limit. Edit freely. |
+| `train_lora.py` | The training script. Loss only on assistant messages; `"weight": 0` turns are skipped. |
+| `modal_app.py` | Runs the script on a Modal GPU, then converts to GGUF on CPU. |
+| `requirements-train.txt`, `requirements-gguf.txt` | Exact versions, pinned by hash. |
+
+## Run on Modal
+
+```sh
+pip install modal && modal setup        # once
+modal run modal_app.py --max-steps 5    # smoke test: a few minutes, cents
+modal run modal_app.py                  # the real run
+```
+
+- Your `data/` is uploaded to **your** Modal account for the run.
+- The job stops after `timeout_hours` (config.json), which caps what a run can cost.
+  Set a workspace budget in Modal's settings as well.
+- Outputs land in the Modal Volume `tunekeep` under `runs/<name>/`: `adapter/` (LoRA),
+  `merged/` (full weights), `model-q8_0.gguf`, `metrics.jsonl`, `summary.json`.
+  The run prints the `modal volume get` commands to download them.
+
+On any other CUDA machine: `pip install -r requirements-train.txt`, then
+`python train_lora.py --config config.json --data data --out out`.
+
+## Serve and compare
+
+```sh
+llama-server -m model-q8_0.gguf --jinja --port 8080
+tunekeep compare --golden tunekeep-out/golden-<model>.jsonl \
+  --candidate retrained --candidate-base-url http://127.0.0.1:8080/v1 \
+  --extra-body '{"chat_template_kwargs": {"enable_thinking": false}}'
+tunekeep report --retrain-summary runs/<name>/summary.json
+```
+
+The model was trained without thinking blocks (`enable_thinking: false` for Qwen3), so serve it the same way.
+
+## Licences
+
+The base model keeps its own licence (Qwen3: Apache-2.0). Your data is yours. Check that
+your training data is not itself the output of a provider whose terms forbid training on it.

tunekeep/kit/modal_app.py

@@ -0,0 +1,98 @@
+"""Run the LoRA retrain on Modal (https://modal.com). Written by tunekeep.
+
+    modal run modal_app.py                  # train on a GPU, then convert to GGUF on CPU
+    modal run modal_app.py --no-gguf        # skip the GGUF step
+    modal run modal_app.py --max-steps 5    # a cheap smoke test first
+
+Your training data (data/) is uploaded to your own Modal account for the run.
+Outputs (LoRA adapter, merged weights, GGUF, metrics) go to the Modal Volume
+"tunekeep" under runs/<name>/. The job stops after `timeout_hours`, which caps
+the cost of a run.
+
+Packages are pinned with hashes (requirements-*.txt) and llama.cpp is pinned
+to an exact commit.
+"""
+
+from __future__ import annotations
+
+import json
+import time
+from pathlib import Path
+
+import modal
+
+HERE = Path(__file__).parent
+KIT = "/root/kit"
+CONFIG = json.loads((HERE / "config.json" if (HERE / "config.json").exists() else Path(KIT) / "config.json").read_text())
+
+LLAMA_CPP_TAG = "b11980"
+LLAMA_CPP_COMMIT = "f7c01c5226cac1de1052c17920ff38cdefea0ae6"
+
+volume = modal.Volume.from_name("tunekeep", create_if_missing=True)
+app = modal.App("tunekeep-retrain")
+
+
+# Image definitions are evaluated lazily, so the local paths below are only read on your machine.
+train_image = (
+    modal.Image.debian_slim(python_version="3.12")
+    .pip_install_from_requirements(str(HERE / "requirements-train.txt"))
+    .env({"HF_HOME": "/vol/hf", "TOKENIZERS_PARALLELISM": "false"})
+    .add_local_file(HERE / "config.json", f"{KIT}/config.json")
+    .add_local_file(HERE / "train_lora.py", f"{KIT}/train_lora.py")
+    .add_local_dir(HERE / "data", f"{KIT}/data")
+)
+
+gguf_image = (
+    modal.Image.debian_slim(python_version="3.12")
+    .apt_install("git")
+    .run_commands(
+        f"git clone --depth 1 --branch {LLAMA_CPP_TAG} https://github.com/ggml-org/llama.cpp /opt/llama.cpp",
+        f'test "$(git -C /opt/llama.cpp rev-parse HEAD)" = "{LLAMA_CPP_COMMIT}"',
+    )
+    .pip_install_from_requirements(str(HERE / "requirements-gguf.txt"))
+)
+
+
+@app.function(image=train_image, gpu=CONFIG.get("gpu", "L40S"), volumes={"/vol": volume},
+              timeout=int(float(CONFIG.get("timeout_hours", 2)) * 3600))
+def train(run: str, max_steps: int = 0) -> dict:
+    import subprocess
+
+    out = f"/vol/runs/{run}"
+    cmd = ["python", f"{KIT}/train_lora.py", "--config", f"{KIT}/config.json", "--data", f"{KIT}/data", "--out", out]
+    if max_steps:
+        cmd += ["--max-steps", str(max_steps)]
+    subprocess.run(cmd, check=True)
+    volume.commit()
+    return json.loads(Path(out, "summary.json").read_text())
+
+
+@app.function(image=gguf_image, cpu=4, memory=32768, volumes={"/vol": volume}, timeout=2 * 3600)
+def to_gguf(run: str, outtype: str) -> str:
+    import subprocess
+
+    volume.reload()
+    out = Path(f"/vol/runs/{run}")
+    target = out / f"model-{outtype}.gguf"
+    subprocess.run(["python", "/opt/llama.cpp/convert_hf_to_gguf.py", str(out / "merged"),
+                    "--outfile", str(target), "--outtype", outtype], check=True)
+    volume.commit()
+    return str(target.relative_to("/vol"))
+
+
+@app.local_entrypoint()
+def main(run: str = "", max_steps: int = 0, no_gguf: bool = False) -> None:
+    run = run or time.strftime("%Y%m%d-%H%M%S")
+    print(f"run {run}: training {CONFIG['base']} on {CONFIG.get('gpu', 'L40S')}")
+    summary = train.remote(run, max_steps)
+    local = HERE / "runs" / run
+    local.mkdir(parents=True, exist_ok=True)
+    (local / "summary.json").write_text(json.dumps(summary, indent=2))
+    print(json.dumps({k: summary[k] for k in ("train_samples", "steps", "train_seconds", "validation_loss_before",
+                                               "validation_loss_by_epoch")}, indent=2))
+    files = [f"runs/{run}/adapter/", f"runs/{run}/merged/"]
+    if not no_gguf and CONFIG.get("gguf"):
+        files.append(to_gguf.remote(run, CONFIG["gguf"]))
+    print(f"\nSummary saved to {local / 'summary.json'}. Download with:")
+    for f in files:
+        print(f"  modal volume get tunekeep {f} {local}/")