troy-cli 0.1.2__py3-none-any.whl

troy/__init__.py

@@ -0,0 +1,3 @@
+"""Troy: fine-tune LLMs on your MacBook with one YAML file."""
+
+__version__ = "0.1.2"

troy/chat.py

@@ -0,0 +1,57 @@
+"""Interactive chat with a trained model."""
+
+from __future__ import annotations
+
+from typing import Optional
+
+from mlx_lm.generate import stream_generate
+from mlx_lm.sample_utils import make_sampler
+from mlx_lm.utils import load
+
+
+def run_chat(
+    model_path: str,
+    adapter_path: Optional[str] = None,
+    max_tokens: int = 512,
+    temperature: float = 0.7,
+    prompt: Optional[str] = None,
+) -> None:
+    print(f"Loading {model_path} ...")
+    model, tokenizer = load(model_path, adapter_path=adapter_path)
+    sampler = make_sampler(temp=temperature)
+    history = []
+
+    def respond(user_text: str) -> None:
+        history.append({"role": "user", "content": user_text})
+        templated = tokenizer.apply_chat_template(
+            history, add_generation_prompt=True, return_dict=False
+        )
+        reply = ""
+        for response in stream_generate(
+            model, tokenizer, templated, max_tokens=max_tokens, sampler=sampler
+        ):
+            print(response.text, end="", flush=True)
+            reply += response.text
+        print()
+        history.append({"role": "assistant", "content": reply})
+
+    if prompt is not None:  # one-shot mode
+        respond(prompt)
+        return
+
+    print("Chat started. Type /exit to quit, /clear to reset history.\n")
+    while True:
+        try:
+            user_text = input(">> ").strip()
+        except (EOFError, KeyboardInterrupt):
+            print()
+            break
+        if not user_text:
+            continue
+        if user_text == "/exit":
+            break
+        if user_text == "/clear":
+            history.clear()
+            print("(history cleared)")
+            continue
+        respond(user_text)

troy/cli.py

@@ -0,0 +1,274 @@
+"""Troy CLI: fine-tune LLMs on your MacBook with one YAML file."""
+
+from __future__ import annotations
+
+import json
+import platform
+import sys
+from pathlib import Path
+from typing import Optional
+
+import typer
+from rich.console import Console
+from rich.table import Table
+
+from . import __version__
+
+app = typer.Typer(
+    name="troy",
+    help="Fine-tune LLMs on your MacBook with one YAML file. Built for Apple Silicon.",
+    no_args_is_help=True,
+    add_completion=False,
+)
+console = Console()
+
+
+def _require_apple_silicon() -> None:
+    if platform.system() != "Darwin" or platform.machine() != "arm64":
+        console.print(
+            "[red]Troy runs on Apple Silicon Macs only (M1 or later).[/red]\n"
+            f"Detected: {platform.system()} / {platform.machine()}"
+        )
+        raise typer.Exit(1)
+
+
+def _version_callback(value: bool) -> None:
+    if value:
+        console.print(f"troy {__version__}")
+        raise typer.Exit()
+
+
+@app.callback()
+def _main(
+    version: bool = typer.Option(
+        False, "--version", "-V", help="Show version.",
+        callback=_version_callback, is_eager=True,
+    ),
+) -> None:
+    pass
+
+
+@app.command()
+def init(
+    template: str = typer.Option(
+        "chat", help="Template: chat (SFT) or dpo (preference tuning)."
+    ),
+    path: Path = typer.Option(Path("troy.yaml"), help="Where to write the config."),
+    force: bool = typer.Option(False, "--force", help="Overwrite an existing config."),
+) -> None:
+    """Create a troy.yaml config (plus sample data) to start from."""
+    from .templates import TEMPLATES
+
+    if template not in TEMPLATES:
+        console.print(f"[red]Unknown template `{template}`.[/red] Options: {', '.join(TEMPLATES)}")
+        raise typer.Exit(1)
+    if path.exists() and not force:
+        console.print(f"[red]{path} already exists.[/red] Use --force to overwrite.")
+        raise typer.Exit(1)
+
+    config_text, data_name, sample = TEMPLATES[template]
+    path.write_text(config_text)
+
+    data_dir = path.parent / "data"
+    data_file = data_dir / data_name
+    if not data_file.exists():
+        data_dir.mkdir(parents=True, exist_ok=True)
+        with open(data_file, "w") as f:
+            for record in sample:
+                f.write(json.dumps(record) + "\n")
+        console.print(f"Wrote sample data to [bold]{data_file}[/bold] — replace it with yours.")
+
+    console.print(f"Created [bold]{path}[/bold] ({template} template).")
+    console.print("Next: edit the config, then run [bold]troy train[/bold].")
+
+
+@app.command()
+def doctor() -> None:
+    """Check this Mac's readiness for local fine-tuning."""
+    from .hardware import detect, model_guidance
+
+    hw = detect()
+    table = Table(title="troy doctor", show_header=False)
+    table.add_column(style="bold")
+    table.add_column()
+
+    ok = "[green]OK[/green]"
+    fail = "[red]FAIL[/red]"
+
+    table.add_row("Chip", hw.chip)
+    table.add_row(
+        "Apple Silicon", ok if hw.is_apple_silicon else f"{fail} ({hw.arch})"
+    )
+    table.add_row("Unified memory", f"{hw.memory_gb:.0f} GB")
+    table.add_row("macOS", hw.macos)
+    table.add_row("Free disk", f"{hw.free_disk_gb:.0f} GB")
+    table.add_row("Python", platform.python_version())
+
+    try:
+        import mlx.core as mx
+
+        table.add_row("MLX", f"{ok} (v{mx.__version__})")
+        gpu_ok = mx.default_device().type == mx.DeviceType.gpu
+        table.add_row("Metal GPU", ok if gpu_ok else f"{fail} (default device is CPU)")
+    except ImportError:
+        table.add_row("MLX", f"{fail} (not installed — `pip install mlx-lm`)")
+
+    try:
+        import mlx_lm
+
+        table.add_row("mlx-lm", f"{ok} (v{mlx_lm.__version__})")
+    except ImportError:
+        table.add_row("mlx-lm", f"{fail} (not installed)")
+
+    table.add_row("Fine-tunable models", model_guidance(hw.memory_gb))
+    console.print(table)
+
+    if not hw.is_apple_silicon:
+        raise typer.Exit(1)
+
+
+@app.command()
+def train(
+    config: Path = typer.Option(Path("troy.yaml"), "--config", "-c", help="Config file."),
+) -> None:
+    """Fine-tune a model from a troy.yaml config."""
+    _require_apple_silicon()
+    from .config import load_config
+    from .data import load_and_prepare
+
+    cfg = load_config(config)
+    train_records, valid_records, fmt = load_and_prepare(
+        cfg.data, cfg.task, cfg.training.seed
+    )
+    console.print(
+        f"Data: {len(train_records)} train / {len(valid_records)} valid "
+        f"(format: {fmt})"
+    )
+
+    if cfg.task == "sft":
+        from .train_sft import run_sft
+
+        run_sft(cfg, train_records, valid_records)
+    else:
+        from .train_dpo import run_dpo
+
+        run_dpo(cfg, train_records, valid_records)
+
+    console.print(
+        f"\nTry it: [bold]troy chat[/bold]   |   "
+        f"Export it: [bold]troy export[/bold]"
+    )
+
+
+@app.command()
+def chat(
+    config: Path = typer.Option(Path("troy.yaml"), "--config", "-c", help="Config file."),
+    model: Optional[str] = typer.Option(
+        None, help="Model path or HF repo (defaults to the config's base + trained adapter)."
+    ),
+    base_only: bool = typer.Option(False, help="Chat with the base model, no adapter."),
+    max_tokens: int = typer.Option(512),
+    temperature: float = typer.Option(0.7),
+    prompt: Optional[str] = typer.Option(None, "--prompt", "-p", help="One-shot prompt (no REPL)."),
+) -> None:
+    """Chat with your fine-tuned model."""
+    _require_apple_silicon()
+    from .chat import run_chat
+
+    adapter: Optional[str] = None
+    if model is None:
+        from .config import load_config
+
+        cfg = load_config(config)
+        model = cfg.base
+        if not base_only:
+            adapter_file = cfg.adapter_path / "adapters.safetensors"
+            if adapter_file.exists():
+                adapter = str(cfg.adapter_path)
+            else:
+                console.print(
+                    "[yellow]No trained adapter found — chatting with the base model.[/yellow]"
+                )
+    run_chat(model, adapter, max_tokens, temperature, prompt)
+
+
+@app.command()
+def serve(
+    config: Path = typer.Option(Path("troy.yaml"), "--config", "-c", help="Config file."),
+    model: Optional[str] = typer.Option(
+        None, help="Model path or HF repo (defaults to the config's base + trained adapter)."
+    ),
+    base_only: bool = typer.Option(False, help="Serve the base model, no adapter."),
+    host: str = typer.Option("127.0.0.1"),
+    port: int = typer.Option(8080),
+    max_tokens: int = typer.Option(512, help="Default max tokens per response."),
+) -> None:
+    """Serve your model over an OpenAI-compatible API."""
+    _require_apple_silicon()
+    from .serve import run_serve
+
+    adapter: Optional[str] = None
+    if model is None:
+        from .config import load_config
+
+        cfg = load_config(config)
+        model = cfg.base
+        if not base_only:
+            adapter_file = cfg.adapter_path / "adapters.safetensors"
+            if adapter_file.exists():
+                adapter = str(cfg.adapter_path)
+            else:
+                console.print(
+                    "[yellow]No trained adapter found — serving the base model.[/yellow]"
+                )
+    run_serve(model, adapter, host, port, max_tokens)
+
+
+@app.command()
+def export(
+    config: Path = typer.Option(Path("troy.yaml"), "--config", "-c", help="Config file."),
+    fmt: str = typer.Option("mlx", "--format", "-f", help="Export format: mlx or gguf."),
+    save_path: Optional[Path] = typer.Option(None, help="Output directory (default: <output>/fused)."),
+    dequantize: bool = typer.Option(False, help="Dequantize when fusing a quantized base."),
+) -> None:
+    """Merge the trained adapter into the base model and export it."""
+    _require_apple_silicon()
+    if fmt not in ("mlx", "gguf"):
+        console.print("[red]--format must be `mlx` or `gguf`.[/red]")
+        raise typer.Exit(1)
+    from .config import load_config
+    from .export import run_export
+
+    cfg = load_config(config)
+    adapter_file = cfg.adapter_path / "adapters.safetensors"
+    if not adapter_file.exists():
+        console.print(f"[red]No adapter at {adapter_file}. Run `troy train` first.[/red]")
+        raise typer.Exit(1)
+    out = save_path or (cfg.output_path / "fused")
+    run_export(cfg.base, str(cfg.adapter_path), str(out), fmt, dequantize)
+
+
+@app.command()
+def data(
+    action: str = typer.Argument(help="Action: inspect"),
+    path: Path = typer.Argument(help="Dataset file (.jsonl, .json, .csv)."),
+) -> None:
+    """Inspect a dataset: record count, detected format, sizes."""
+    if action != "inspect":
+        console.print("[red]Only `troy data inspect <path>` is supported.[/red]")
+        raise typer.Exit(1)
+    from .data import inspect_stats
+
+    stats = inspect_stats(str(path))
+    table = Table(title=str(path), show_header=False)
+    table.add_column(style="bold")
+    table.add_column()
+    table.add_row("Records", str(stats["records"]))
+    table.add_row("Detected format", stats["format"])
+    table.add_row("Avg record size", f"{stats['avg_chars']:.0f} chars")
+    table.add_row("Max record size", f"{stats['max_chars']} chars")
+    console.print(table)
+
+
+if __name__ == "__main__":
+    app()

troy/config.py

@@ -0,0 +1,82 @@
+"""Troy YAML config schema and loader."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Literal, Optional, Union
+
+import yaml
+from pydantic import BaseModel, Field, field_validator
+
+
+class LoraConfig(BaseModel):
+    r: int = 8
+    alpha: float = 16.0
+    dropout: float = 0.0
+    layers: Union[int, Literal["all"]] = 16  # decoder layers to adapt
+
+    @property
+    def scale(self) -> float:
+        return self.alpha / self.r
+
+
+class DpoConfig(BaseModel):
+    beta: float = 0.1
+
+
+class DataConfig(BaseModel):
+    train: str
+    valid: Optional[str] = None
+    format: Literal[
+        "auto", "alpaca", "sharegpt", "chat", "completions", "text", "preference"
+    ] = "auto"
+    val_split: float = Field(0.1, ge=0.0, lt=1.0)
+    mask_prompt: bool = False
+
+
+class TrainingConfig(BaseModel):
+    epochs: Optional[float] = None  # translated to iters from dataset size
+    iters: Optional[int] = None  # takes precedence over epochs
+    lr: float = 1e-5
+    batch_size: Union[int, Literal["auto"]] = "auto"
+    seq_len: int = 2048
+    lora: LoraConfig = LoraConfig()
+    dpo: DpoConfig = DpoConfig()
+    grad_checkpoint: bool = False
+    grad_accumulation_steps: int = 1
+    save_every: int = 100
+    seed: int = 0
+
+
+class TroyConfig(BaseModel):
+    base: str
+    task: Literal["sft", "dpo"] = "sft"
+    data: DataConfig
+    training: TrainingConfig = TrainingConfig()
+    output: str = "./output"
+
+    @field_validator("base")
+    @classmethod
+    def _non_empty(cls, v: str) -> str:
+        if not v.strip():
+            raise ValueError("`base` must be a HuggingFace repo id or local path")
+        return v
+
+    @property
+    def output_path(self) -> Path:
+        return Path(self.output).expanduser()
+
+    @property
+    def adapter_path(self) -> Path:
+        return self.output_path / "adapter"
+
+
+def load_config(path: Union[str, Path]) -> TroyConfig:
+    path = Path(path)
+    if not path.exists():
+        raise FileNotFoundError(
+            f"Config not found: {path}. Run `troy init` to create one."
+        )
+    with open(path) as f:
+        raw = yaml.safe_load(f) or {}
+    return TroyConfig.model_validate(raw)

troy/data.py

@@ -0,0 +1,163 @@
+"""Dataset loading, format auto-detection, and conversion to mlx-lm formats."""
+
+from __future__ import annotations
+
+import csv
+import json
+import random
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Tuple
+
+from .config import DataConfig
+
+
+def _read_records(path: Path) -> List[Dict[str, Any]]:
+    suffix = path.suffix.lower()
+    if suffix == ".jsonl":
+        with open(path) as f:
+            return [json.loads(line) for line in f if line.strip()]
+    if suffix == ".json":
+        with open(path) as f:
+            data = json.load(f)
+        if not isinstance(data, list):
+            raise ValueError(f"{path}: JSON file must contain a list of records")
+        return data
+    if suffix == ".csv":
+        with open(path, newline="") as f:
+            return list(csv.DictReader(f))
+    raise ValueError(
+        f"Unsupported file type: {path.suffix} (use .jsonl, .json, or .csv)"
+    )
+
+
+def detect_format(sample: Dict[str, Any]) -> str:
+    keys = set(sample)
+    if {"prompt", "chosen", "rejected"} <= keys:
+        return "preference"
+    if "messages" in keys:
+        return "chat"
+    if "conversations" in keys:
+        return "sharegpt"
+    if {"instruction", "output"} <= keys:
+        return "alpaca"
+    if {"prompt", "completion"} <= keys:
+        return "completions"
+    if "text" in keys:
+        return "text"
+    raise ValueError(
+        f"Could not detect data format from keys: {sorted(keys)}. "
+        "Set `data.format` explicitly in troy.yaml."
+    )
+
+
+_SHAREGPT_ROLES = {
+    "human": "user",
+    "user": "user",
+    "gpt": "assistant",
+    "assistant": "assistant",
+    "system": "system",
+}
+
+
+def _to_messages(record: Dict[str, Any], fmt: str) -> Dict[str, Any]:
+    """Normalize an SFT record to mlx-lm chat format ({"messages": [...]})."""
+    if fmt == "chat":
+        return {"messages": record["messages"]}
+    if fmt == "sharegpt":
+        messages = [
+            {"role": _SHAREGPT_ROLES[m["from"].lower()], "content": m["value"]}
+            for m in record["conversations"]
+        ]
+        return {"messages": messages}
+    if fmt == "alpaca":
+        user = record["instruction"]
+        if record.get("input"):
+            user = f"{user}\n\n{record['input']}"
+        messages = []
+        if record.get("system"):
+            messages.append({"role": "system", "content": record["system"]})
+        messages += [
+            {"role": "user", "content": user},
+            {"role": "assistant", "content": record["output"]},
+        ]
+        return {"messages": messages}
+    if fmt in ("completions", "text"):
+        return record  # already an mlx-lm native format
+    raise ValueError(f"Unexpected format: {fmt}")
+
+
+def _normalize_preference(record: Dict[str, Any]) -> Dict[str, str]:
+    """Normalize a DPO record to string prompt/chosen/rejected."""
+
+    def text_of(v: Any) -> str:
+        if isinstance(v, str):
+            return v
+        if isinstance(v, list):  # message-list style
+            return "\n".join(m.get("content", "") for m in v)
+        raise ValueError(f"Cannot interpret preference field: {v!r}")
+
+    return {
+        "prompt": text_of(record["prompt"]),
+        "chosen": text_of(record["chosen"]),
+        "rejected": text_of(record["rejected"]),
+    }
+
+
+def load_and_prepare(
+    cfg: DataConfig, task: str, seed: int = 0
+) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]], str]:
+    """Load train/valid records normalized for the task.
+
+    Returns (train_records, valid_records, detected_format).
+    """
+    train_path = Path(cfg.train).expanduser()
+    records = _read_records(train_path)
+    if not records:
+        raise ValueError(f"No records found in {train_path}")
+
+    fmt = cfg.format if cfg.format != "auto" else detect_format(records[0])
+
+    if task == "dpo" and fmt != "preference":
+        raise ValueError(
+            f"Task `dpo` needs preference data (prompt/chosen/rejected); "
+            f"detected format `{fmt}`."
+        )
+    if task == "sft" and fmt == "preference":
+        raise ValueError("Preference data detected — set `task: dpo` in troy.yaml.")
+
+    if fmt == "preference":
+        records = [_normalize_preference(r) for r in records]
+    else:
+        records = [_to_messages(r, fmt) for r in records]
+
+    if cfg.valid:
+        valid = _read_records(Path(cfg.valid).expanduser())
+        valid = (
+            [_normalize_preference(r) for r in valid]
+            if fmt == "preference"
+            else [_to_messages(r, fmt) for r in valid]
+        )
+        return records, valid, fmt
+
+    # Split off validation
+    rng = random.Random(seed)
+    indices = list(range(len(records)))
+    rng.shuffle(indices)
+    n_val = int(len(records) * cfg.val_split)
+    val_idx = set(indices[:n_val])
+    train = [r for i, r in enumerate(records) if i not in val_idx]
+    valid = [r for i, r in enumerate(records) if i in val_idx]
+    return train, valid, fmt
+
+
+def inspect_stats(path: str) -> Dict[str, Any]:
+    """Lightweight dataset statistics for `troy data inspect`-style output."""
+    records = _read_records(Path(path).expanduser())
+    fmt = detect_format(records[0]) if records else "unknown"
+    lengths = [len(json.dumps(r)) for r in records]
+    return {
+        "records": len(records),
+        "format": fmt,
+        "avg_chars": sum(lengths) / max(len(lengths), 1),
+        "max_chars": max(lengths, default=0),
+    }

troy/export.py

@@ -0,0 +1,81 @@
+"""Merge adapters and export to deployment formats (MLX, GGUF)."""
+
+from __future__ import annotations
+
+import subprocess
+import sys
+from pathlib import Path
+
+
+def _fuse(base: str, adapter_path: str, save_path: str, dequantize: bool) -> None:
+    cmd = [
+        sys.executable,
+        "-m",
+        "mlx_lm",
+        "fuse",
+        "--model",
+        base,
+        "--adapter-path",
+        adapter_path,
+        "--save-path",
+        save_path,
+    ]
+    if dequantize:
+        cmd.append("--dequantize")
+    print("Fusing adapter into base model ...")
+    result = subprocess.run(cmd)
+    if result.returncode != 0:
+        raise SystemExit(result.returncode)
+
+
+def _export_gguf(save_path: Path) -> Path:
+    """Convert a fused MLX model directory to GGUF (llama/mistral/mixtral archs)."""
+    import json
+
+    import mlx.core as mx
+    from mlx_lm import gguf as gguf_mod
+
+    with open(save_path / "config.json") as f:
+        config = json.load(f)
+
+    weights = {}
+    for part in sorted(save_path.glob("*.safetensors")):
+        weights.update(mx.load(str(part)))
+
+    # mlx-lm's converter permutes attention weights into non-contiguous views,
+    # which save_gguf rejects — force contiguity on its output.
+    orig_permute = gguf_mod.permute_weights
+    gguf_mod.permute_weights = lambda *a, **k: mx.contiguous(orig_permute(*a, **k))
+    try:
+        out = save_path / "ggml-model-f16.gguf"
+        gguf_mod.convert_to_gguf(save_path, weights, config, str(out))
+    finally:
+        gguf_mod.permute_weights = orig_permute
+    return out
+
+
+def run_export(
+    base: str,
+    adapter_path: str,
+    save_path: str,
+    fmt: str = "mlx",
+    dequantize: bool = False,
+) -> None:
+    """Fuse a LoRA adapter into the base model; optionally convert to GGUF.
+
+    fmt: "mlx" (fused MLX weights) or "gguf" (also writes ggml-model-f16.gguf
+    for llama.cpp / Ollama / LM Studio; llama, mistral and mixtral
+    architectures, unquantized base).
+    """
+    if not Path(base).exists():
+        # mlx-lm's training path downloads a partial snapshot; fuse checks for
+        # a complete one. Top it up before fusing.
+        from huggingface_hub import snapshot_download
+
+        snapshot_download(base)
+
+    _fuse(base, adapter_path, save_path, dequantize)
+    print(f"Fused model: {Path(save_path).resolve()}")
+    if fmt == "gguf":
+        out = _export_gguf(Path(save_path))
+        print(f"GGUF: {out.resolve()}")

troy/hardware.py

@@ -0,0 +1,68 @@
+"""Apple Silicon hardware detection and guidance."""
+
+from __future__ import annotations
+
+import platform
+import shutil
+import subprocess
+from dataclasses import dataclass
+
+
+def _sysctl(key: str) -> str:
+    try:
+        return subprocess.run(
+            ["sysctl", "-n", key], capture_output=True, text=True, timeout=5
+        ).stdout.strip()
+    except Exception:
+        return ""
+
+
+@dataclass
+class Hardware:
+    chip: str
+    memory_gb: float
+    macos: str
+    arch: str
+    free_disk_gb: float
+
+    @property
+    def is_apple_silicon(self) -> bool:
+        return self.arch == "arm64" and platform.system() == "Darwin"
+
+
+def detect() -> Hardware:
+    mem_bytes = int(_sysctl("hw.memsize") or 0)
+    total, used, free = shutil.disk_usage("/")
+    return Hardware(
+        chip=_sysctl("machdep.cpu.brand_string") or "unknown",
+        memory_gb=mem_bytes / 2**30,
+        macos=platform.mac_ver()[0] or "unknown",
+        arch=platform.machine(),
+        free_disk_gb=free / 2**30,
+    )
+
+
+# (min unified memory GB, guidance) — QLoRA 4-bit fine-tuning headroom.
+MODEL_GUIDANCE = [
+    (128, "up to ~70B (4-bit QLoRA), e.g. Llama-3.3-70B, Qwen2.5-72B"),
+    (64, "up to ~32B (4-bit QLoRA), e.g. Qwen2.5-32B, Gemma-2-27B"),
+    (36, "up to ~14B (4-bit QLoRA), e.g. Qwen2.5-14B, Phi-4"),
+    (24, "up to ~8B (4-bit QLoRA), e.g. Llama-3.1-8B, Qwen3-8B"),
+    (16, "up to ~4B (4-bit QLoRA), e.g. Qwen3-4B, Phi-3.5-mini, Gemma-3-4B"),
+    (8, "up to ~1.5B (4-bit QLoRA), e.g. Qwen3-0.6B, Llama-3.2-1B"),
+]
+
+
+def model_guidance(memory_gb: float) -> str:
+    for min_gb, text in MODEL_GUIDANCE:
+        if memory_gb >= min_gb:
+            return text
+    return "very small models only (<1B)"
+
+
+def auto_batch_size(memory_gb: float, seq_len: int) -> int:
+    """Conservative batch size from unified memory and sequence length."""
+    budget = max(memory_gb - 8, 1)  # leave room for the OS + model weights
+    per_seq = seq_len / 2048  # rough scaling
+    bs = int(budget // (4 * per_seq))
+    return max(1, min(bs, 8))

troy/serve.py

@@ -0,0 +1,45 @@
+"""OpenAI-compatible local API server (wraps the mlx-lm HTTP server)."""
+
+from __future__ import annotations
+
+import subprocess
+import sys
+from typing import Optional
+
+
+def run_serve(
+    model: str,
+    adapter_path: Optional[str],
+    host: str,
+    port: int,
+    max_tokens: int,
+) -> None:
+    cmd = [
+        sys.executable,
+        "-m",
+        "mlx_lm",
+        "server",
+        "--model",
+        model,
+        "--host",
+        host,
+        "--port",
+        str(port),
+        "--max-tokens",
+        str(max_tokens),
+    ]
+    if adapter_path:
+        cmd += ["--adapter-path", adapter_path]
+
+    print(f"Serving {model}" + (f" + adapter {adapter_path}" if adapter_path else ""))
+    print(f"OpenAI-compatible API at http://{host}:{port}/v1")
+    print(
+        "Try it:\n"
+        f'  curl http://{host}:{port}/v1/chat/completions \\\n'
+        '    -H "Content-Type: application/json" \\\n'
+        '    -d \'{"messages": [{"role": "user", "content": "Hello"}]}\'\n'
+    )
+    try:
+        raise SystemExit(subprocess.run(cmd).returncode)
+    except KeyboardInterrupt:
+        print("\nServer stopped.")

troy/templates.py

@@ -0,0 +1,84 @@
+"""Config and sample-data templates for `troy init`."""
+
+CHAT_TEMPLATE = """\
+# Troy config — supervised fine-tuning (SFT)
+# Run with: troy train
+
+base: mlx-community/Qwen3-0.6B-4bit   # any MLX model on the HF Hub, or a local path
+task: sft
+
+data:
+  train: ./data/train.jsonl   # alpaca, sharegpt, chat, completions, or text — auto-detected
+  format: auto
+  val_split: 0.1
+
+training:
+  epochs: 3
+  lr: 1e-5
+  batch_size: auto        # sized from your Mac's unified memory
+  seq_len: 2048
+  lora:
+    r: 8
+    alpha: 16
+
+output: ./output
+"""
+
+DPO_TEMPLATE = """\
+# Troy config — preference tuning (DPO)
+# Run with: troy train
+
+base: mlx-community/Qwen3-0.6B-4bit
+task: dpo
+
+data:
+  train: ./data/preferences.jsonl   # {"prompt": ..., "chosen": ..., "rejected": ...}
+  format: preference
+  val_split: 0.05
+
+training:
+  epochs: 1
+  lr: 5e-6
+  batch_size: auto
+  seq_len: 2048
+  lora:
+    r: 8
+    alpha: 16
+  dpo:
+    beta: 0.1
+
+output: ./output
+"""
+
+SAMPLE_SFT_DATA = [
+    {
+        "instruction": "What is the capital of France?",
+        "output": "The capital of France is Paris.",
+    },
+    {
+        "instruction": "Write a haiku about the ocean.",
+        "output": "Endless blue expanse\nWaves whisper against the shore\nSalt hangs in the air",
+    },
+    {
+        "instruction": "Explain what a LoRA adapter is in one sentence.",
+        "output": "A LoRA adapter is a small set of low-rank matrices trained alongside a frozen model so it can be specialized cheaply.",
+    },
+]
+
+SAMPLE_DPO_DATA = [
+    {
+        "prompt": "What is the capital of France?",
+        "chosen": "The capital of France is Paris.",
+        "rejected": "I think it might be Lyon, but I'm not sure.",
+    },
+    {
+        "prompt": "Summarize photosynthesis in one sentence.",
+        "chosen": "Photosynthesis is the process by which plants convert sunlight, water, and carbon dioxide into glucose and oxygen.",
+        "rejected": "Plants eat sunlight.",
+    },
+]
+
+TEMPLATES = {
+    "chat": (CHAT_TEMPLATE, "train.jsonl", SAMPLE_SFT_DATA),
+    "dpo": (DPO_TEMPLATE, "preferences.jsonl", SAMPLE_DPO_DATA),
+}

troy/train_dpo.py

@@ -0,0 +1,174 @@
+"""Direct Preference Optimization on Apple Silicon.
+
+Memory trick: with LoRA, the frozen reference model is the policy model with
+every adapter's scale set to 0 — so DPO needs no second copy of the weights,
+which matters on unified memory.
+"""
+
+from __future__ import annotations
+
+import time
+from typing import Any, Dict, List, Tuple
+
+import mlx.core as mx
+import mlx.nn as nn
+import mlx.optimizers as optim
+from mlx.utils import tree_flatten
+from mlx_lm.tuner.trainer import grad_checkpoint
+from mlx_lm.tuner.utils import print_trainable_parameters
+from mlx_lm.utils import load
+
+from .config import TroyConfig
+from .train_sft import apply_lora, resolve_batch_size, resolve_iters, save_adapter_config
+
+
+def _lora_modules(model) -> list:
+    return [
+        m
+        for _, m in model.named_modules()
+        if hasattr(m, "lora_a") and hasattr(m, "scale")
+    ]
+
+
+class _ReferenceMode:
+    """Context manager: zero every LoRA scale so the model acts as the frozen base."""
+
+    def __init__(self, model):
+        self.modules = _lora_modules(model)
+        self.saved: List[float] = []
+
+    def __enter__(self):
+        self.saved = [m.scale for m in self.modules]
+        for m in self.modules:
+            m.scale = 0.0
+
+    def __exit__(self, *exc):
+        for m, s in zip(self.modules, self.saved):
+            m.scale = s
+
+
+def _encode_pair(tokenizer, prompt: str, completion: str, max_len: int) -> Tuple[List[int], int]:
+    """Tokenize prompt+completion with the chat template; return (tokens, prompt_len)."""
+    messages = [{"role": "user", "content": prompt}]
+    prompt_tokens = tokenizer.apply_chat_template(
+        messages, add_generation_prompt=True, return_dict=False
+    )
+    full = messages + [{"role": "assistant", "content": completion}]
+    full_tokens = tokenizer.apply_chat_template(full, return_dict=False)
+    full_tokens = full_tokens[:max_len]
+    prompt_len = min(len(prompt_tokens), len(full_tokens) - 1)
+    return full_tokens, prompt_len
+
+
+def _batch(
+    pairs: List[Tuple[List[int], int]], pad_id: int
+) -> Tuple[mx.array, mx.array]:
+    """Pad a list of (tokens, prompt_len) into (tokens, completion_mask) arrays."""
+    max_len = max(len(t) for t, _ in pairs)
+    tokens, mask = [], []
+    for t, plen in pairs:
+        pad = max_len - len(t)
+        tokens.append(list(t) + [pad_id] * pad)
+        # mask over positions whose *target* token is part of the completion
+        m = [1.0 if plen <= j < len(t) else 0.0 for j in range(1, max_len)]
+        mask.append(m)
+    return mx.array(tokens), mx.array(mask)
+
+
+def _sequence_logps(model, tokens: mx.array, mask: mx.array) -> mx.array:
+    """Sum of per-token log-probs over the completion for each sequence."""
+    logits = model(tokens[:, :-1])
+    targets = tokens[:, 1:]
+    logps = -nn.losses.cross_entropy(logits, targets, reduction="none")
+    return (logps * mask).sum(axis=-1)
+
+
+def run_dpo(
+    config: TroyConfig,
+    train_records: List[Dict[str, Any]],
+    valid_records: List[Dict[str, Any]],
+) -> None:
+    mx.random.seed(config.training.seed)
+    print(f"Loading {config.base} ...")
+    model, tokenizer = load(config.base)
+
+    batch_size = resolve_batch_size(config)
+    # DPO runs chosen+rejected per example: halve the auto batch size
+    if config.training.batch_size == "auto":
+        batch_size = max(1, batch_size // 2)
+    iters = resolve_iters(config, len(train_records), batch_size)
+    num_layers = apply_lora(model, config)
+    save_adapter_config(config, num_layers)
+    if config.training.grad_checkpoint:
+        grad_checkpoint(model.layers[0])
+
+    beta = config.training.dpo.beta
+    max_len = config.training.seq_len
+    pad_id = tokenizer.pad_token_id or tokenizer.eos_token_id or 0
+
+    encoded = [
+        (
+            _encode_pair(tokenizer, r["prompt"], r["chosen"], max_len),
+            _encode_pair(tokenizer, r["prompt"], r["rejected"], max_len),
+        )
+        for r in train_records
+    ]
+
+    def loss_fn(model, tc, mc, tr, mr, ref_c, ref_r):
+        pol_c = _sequence_logps(model, tc, mc)
+        pol_r = _sequence_logps(model, tr, mr)
+        logits = beta * ((pol_c - ref_c) - (pol_r - ref_r))
+        loss = -nn.log_sigmoid(logits).mean()
+        reward_acc = (logits > 0).mean()
+        return loss, reward_acc
+
+    loss_and_grad = nn.value_and_grad(model, loss_fn)
+    opt = optim.Adam(learning_rate=config.training.lr)
+
+    print(
+        f"Training: task=dpo beta={beta} batch_size={batch_size} iters={iters} "
+        f"lr={config.training.lr} pairs={len(encoded)}"
+    )
+
+    config.adapter_path.mkdir(parents=True, exist_ok=True)
+    adapter_file = config.adapter_path / "adapters.safetensors"
+    n = len(encoded)
+    losses, accs = [], []
+    start = time.time()
+
+    for it in range(iters):
+        idx = [(it * batch_size + k) % n for k in range(batch_size)]
+        chosen = [encoded[i][0] for i in idx]
+        rejected = [encoded[i][1] for i in idx]
+        tc, mc = _batch(chosen, pad_id)
+        tr, mr = _batch(rejected, pad_id)
+
+        with _ReferenceMode(model):
+            ref_c = mx.stop_gradient(_sequence_logps(model, tc, mc))
+            ref_r = mx.stop_gradient(_sequence_logps(model, tr, mr))
+            mx.eval(ref_c, ref_r)
+
+        (loss, acc), grads = loss_and_grad(model, tc, mc, tr, mr, ref_c, ref_r)
+        opt.update(model, grads)
+        mx.eval(model.parameters(), opt.state, loss)
+        losses.append(loss.item())
+        accs.append(acc.item())
+
+        if (it + 1) % 10 == 0 or it == iters - 1:
+            speed = (it + 1) / (time.time() - start)
+            print(
+                f"Iter {it + 1}/{iters}: loss {sum(losses)/len(losses):.4f}, "
+                f"reward acc {sum(accs)/len(accs):.3f}, {speed:.2f} it/s"
+            )
+            losses, accs = [], []
+
+        if (it + 1) % config.training.save_every == 0:
+            _save(model, adapter_file)
+
+    _save(model, adapter_file)
+    print(f"\nDone. Adapter saved to {config.adapter_path}")
+
+
+def _save(model, adapter_file) -> None:
+    adapter_weights = dict(tree_flatten(model.trainable_parameters()))
+    mx.save_safetensors(str(adapter_file), adapter_weights)

troy/train_sft.py

@@ -0,0 +1,113 @@
+"""Supervised fine-tuning via the mlx-lm tuner."""
+
+from __future__ import annotations
+
+import json
+import math
+import types
+from typing import Any, Dict, List
+
+import mlx.core as mx
+import mlx.optimizers as optim
+from mlx_lm.tuner.datasets import CacheDataset, create_dataset
+from mlx_lm.tuner.trainer import TrainingArgs, train
+from mlx_lm.tuner.utils import linear_to_lora_layers, print_trainable_parameters
+from mlx_lm.utils import load
+
+from .config import TroyConfig
+from .hardware import auto_batch_size, detect
+
+
+def resolve_batch_size(config: TroyConfig) -> int:
+    if config.training.batch_size == "auto":
+        return auto_batch_size(detect().memory_gb, config.training.seq_len)
+    return int(config.training.batch_size)
+
+
+def resolve_iters(config: TroyConfig, n_train: int, batch_size: int) -> int:
+    t = config.training
+    if t.iters:
+        return t.iters
+    epochs = t.epochs or 3
+    return max(1, math.ceil(epochs * n_train / batch_size))
+
+
+def apply_lora(model, config: TroyConfig):
+    model.freeze()
+    lora = config.training.lora
+    n_model_layers = len(model.layers)
+    num_layers = (
+        n_model_layers if lora.layers == "all" else min(int(lora.layers), n_model_layers)
+    )
+    linear_to_lora_layers(
+        model,
+        num_layers,
+        {"rank": lora.r, "scale": lora.scale, "dropout": lora.dropout},
+    )
+    print_trainable_parameters(model)
+    return num_layers
+
+
+def save_adapter_config(config: TroyConfig, num_layers: int) -> None:
+    """Write the adapter_config.json mlx-lm needs to re-load the adapter."""
+    lora = config.training.lora
+    config.adapter_path.mkdir(parents=True, exist_ok=True)
+    adapter_config = {
+        "fine_tune_type": "lora",
+        "num_layers": num_layers,
+        "lora_parameters": {
+            "rank": lora.r,
+            "scale": lora.scale,
+            "dropout": lora.dropout,
+        },
+    }
+    with open(config.adapter_path / "adapter_config.json", "w") as f:
+        json.dump(adapter_config, f, indent=2)
+
+
+def run_sft(
+    config: TroyConfig,
+    train_records: List[Dict[str, Any]],
+    valid_records: List[Dict[str, Any]],
+) -> None:
+    mx.random.seed(config.training.seed)
+    print(f"Loading {config.base} ...")
+    model, tokenizer = load(config.base)
+
+    if not valid_records:  # trainer always evaluates; give it something tiny
+        valid_records = train_records[:1]
+    ds_config = types.SimpleNamespace(mask_prompt=config.data.mask_prompt)
+    train_set = create_dataset(train_records, tokenizer, ds_config)
+    valid_set = create_dataset(valid_records, tokenizer, ds_config)
+
+    batch_size = resolve_batch_size(config)
+    batch_size = max(1, min(batch_size, len(train_records), len(valid_records)))
+    iters = resolve_iters(config, len(train_records), batch_size)
+    num_layers = apply_lora(model, config)
+    save_adapter_config(config, num_layers)
+
+    args = TrainingArgs(
+        batch_size=batch_size,
+        iters=iters,
+        max_seq_length=config.training.seq_len,
+        adapter_file=str(config.adapter_path / "adapters.safetensors"),
+        grad_checkpoint=config.training.grad_checkpoint,
+        grad_accumulation_steps=config.training.grad_accumulation_steps,
+        steps_per_save=config.training.save_every,
+        steps_per_eval=min(200, max(1, iters // 2)),
+        val_batches=min(25, max(1, len(valid_records) // batch_size)),
+    )
+    opt = optim.Adam(learning_rate=config.training.lr)
+
+    print(
+        f"Training: task=sft batch_size={batch_size} iters={iters} "
+        f"lr={config.training.lr} seq_len={config.training.seq_len}"
+    )
+    train(
+        model=model,
+        optimizer=opt,
+        train_dataset=CacheDataset(train_set),
+        val_dataset=CacheDataset(valid_set),
+        args=args,
+    )
+    print(f"\nDone. Adapter saved to {config.adapter_path}")

troy_cli-0.1.2.dist-info/METADATA

@@ -0,0 +1,112 @@
+Metadata-Version: 2.5
+Name: troy-cli
+Version: 0.1.2
+Summary: Fine-tune LLMs on your MacBook with one YAML file. Built for Apple Silicon.
+Author: Troy
+License: Apache-2.0
+Keywords: apple-silicon,dpo,fine-tuning,llm,lora,mlx
+Requires-Python: >=3.10
+Requires-Dist: mlx-lm>=0.30
+Requires-Dist: pydantic>=2.5
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: rich>=13.0
+Requires-Dist: typer>=0.12
+Description-Content-Type: text/markdown
+
+# Troy
+
+**Fine-tune LLMs on your MacBook with one YAML file.**
+
+Troy is a command-line tool for fine-tuning and preference-tuning language
+models locally on Apple Silicon. No CUDA, no cloud, no training pipeline —
+write a config, run one command, and train on the machine you already own.
+
+Built on [MLX](https://github.com/ml-explore/mlx) and
+[mlx-lm](https://github.com/ml-explore/mlx-lm), Apple's ML framework for
+Apple Silicon. Unified memory means a 36 GB MacBook fine-tunes models that
+need a workstation GPU anywhere else.
+
+## Requirements
+
+- Apple Silicon Mac (M1 or later)
+- macOS 14+
+- Python 3.10–3.12
+
+## Install
+
+```bash
+brew install avirajkhare00/troy/troy
+# or from source: pip install ./cli
+```
+
+## Quickstart
+
+```bash
+troy doctor            # check your Mac: chip, memory, MLX, what you can train
+troy init              # create troy.yaml + sample data
+troy train             # fine-tune (LoRA/QLoRA via MLX)
+troy chat              # talk to the result
+troy serve             # OpenAI-compatible API at localhost:8080/v1
+troy export -f gguf    # ship it to llama.cpp / Ollama / LM Studio
+```
+
+## The config is the interface
+
+```yaml
+base: mlx-community/Qwen3-0.6B-4bit
+task: sft            # or: dpo
+
+data:
+  train: ./data/train.jsonl   # alpaca, sharegpt, chat, completions, text — auto-detected
+  val_split: 0.1
+
+training:
+  epochs: 3
+  lr: 1e-5
+  batch_size: auto   # sized from your Mac's unified memory
+  lora:
+    r: 8
+    alpha: 16
+
+output: ./output
+```
+
+## Commands
+
+| Command | Purpose |
+|---|---|
+| `troy init` | Create a config from a template (`chat`, `dpo`) |
+| `troy doctor` | Hardware + dependency check, with model-size guidance |
+| `troy train` | LoRA fine-tuning: SFT or DPO |
+| `troy chat` | Interactive REPL (or `-p` for one-shot) with your adapter |
+| `troy serve` | OpenAI-compatible API server for your model |
+| `troy export` | Fuse the adapter; export MLX or GGUF |
+| `troy data inspect` | Dataset stats and format detection |
+
+## What Troy can train on your Mac
+
+| Unified memory | Max model (4-bit QLoRA) |
+|---|---|
+| 8 GB | ~1.5B |
+| 16 GB | ~4B |
+| 24 GB | ~8B |
+| 36 GB | ~14B |
+| 64 GB | ~32B |
+| 128 GB | ~70B |
+
+## DPO without a second model
+
+DPO normally keeps a frozen reference copy of the model in memory. Troy
+zeroes the LoRA scales to recover the reference model from the policy model
+itself — no second copy, which matters on unified memory.
+
+## Data formats
+
+Auto-detected from the first record: Alpaca (`instruction`/`output`),
+ShareGPT (`conversations`), chat (`messages`), `prompt`/`completion`,
+plain `text`, and preference pairs (`prompt`/`chosen`/`rejected`) for DPO.
+Files: `.jsonl`, `.json`, `.csv`.
+
+## License
+
+Apache-2.0

troy_cli-0.1.2.dist-info/RECORD

@@ -0,0 +1,15 @@
+troy/__init__.py,sha256=mCMSFHPxMwZ7O0QyBuh719jLqU_5B2qnFuTtTv6uWC8,86
+troy/chat.py,sha256=SLpGGmsgyV_wMRqx6mzZhrFD5YgyiBm6ezGgYeYKRyk,1693
+troy/cli.py,sha256=yqY3_DeTb8kdB5VAtiLP6Uy_WZIvVA6XbsksBfvBYec,9170
+troy/config.py,sha256=3XOR3dtvkisVuWFJuqx6Cge_plGyz4KcYqwGAcick7U,2160
+troy/data.py,sha256=KGhwB-RuOficxZiDP_vdqItkZ2_7yiKGMqnYm7rKeoM,5382
+troy/export.py,sha256=4k50lcL-CrUcg6f3ibbourpQHr44qj7kkT16SJtqUAs,2424
+troy/hardware.py,sha256=-c5mdZr7SpQk1qo7Eib9nxRSknzE5dxfvyuU18wr-zg,2022
+troy/serve.py,sha256=1Xq3FJ1NlYUo1UD1E7xd2D1ttq46lwIgfpRDjiQ0PmE,1122
+troy/templates.py,sha256=fwfdbG2xKg9kSpT4fKx6poyqpc9eMpW29VXD1wSUaXg,2101
+troy/train_dpo.py,sha256=-Zg53-8rst3mqV5cvr_Yyc9AnSYRP562hDiPMIMBsU4,6126
+troy/train_sft.py,sha256=u3S6WwZrCxwFQfArXu3kNbqXG65L9FZH_J773IK983s,3825
+troy_cli-0.1.2.dist-info/METADATA,sha256=K5OH1etNCT0AfBn9Xv-ZD1x0gQ9N1ZjmCIsWfnV_I4s,3124
+troy_cli-0.1.2.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
+troy_cli-0.1.2.dist-info/entry_points.txt,sha256=jz-AaAmupNovrRxChIqPcfiKm36WkSR8WV2Of0ajlqM,38
+troy_cli-0.1.2.dist-info/RECORD,,

troy_cli-0.1.2.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.0
+Root-Is-Purelib: true
+Tag: py3-none-any

troy_cli-0.1.2.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+troy = troy.cli:app