troy-cli 0.1.2__tar.gz → 0.2.0__tar.gz

{troy_cli-0.1.2 → troy_cli-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: troy-cli
-Version: 0.1.2
+Version: 0.2.0
 Summary: Fine-tune LLMs on your MacBook with one YAML file. Built for Apple Silicon.
 Author: Troy
 License: Apache-2.0
@@ -36,7 +36,7 @@ need a workstation GPU anywhere else.
 
 ```bash
 brew install avirajkhare00/troy/troy
-# or from source: pip install ./cli
+# or: pipx install troy-cli
 ```
 
 ## Quickstart
@@ -54,7 +54,7 @@ troy export -f gguf    # ship it to llama.cpp / Ollama / LM Studio
 
 ```yaml
 base: mlx-community/Qwen3-0.6B-4bit
-task: sft            # or: dpo
+task: sft            # or: dpo, orpo
 
 data:
   train: ./data/train.jsonl   # alpaca, sharegpt, chat, completions, text — auto-detected
@@ -77,10 +77,12 @@ output: ./output
 |---|---|
 | `troy init` | Create a config from a template (`chat`, `dpo`) |
 | `troy doctor` | Hardware + dependency check, with model-size guidance |
-| `troy train` | LoRA fine-tuning: SFT or DPO |
+| `troy train` | LoRA fine-tuning: SFT, DPO, or ORPO |
 | `troy chat` | Interactive REPL (or `-p` for one-shot) with your adapter |
+| `troy eval` | Base-vs-tuned val loss, perplexity, side-by-side samples |
 | `troy serve` | OpenAI-compatible API server for your model |
 | `troy export` | Fuse the adapter; export MLX or GGUF |
+| `troy push` | Upload adapter or fused model to the Hugging Face Hub |
 | `troy data inspect` | Dataset stats and format detection |
 
 ## What Troy can train on your Mac

{troy_cli-0.1.2 → troy_cli-0.2.0}/README.md

@@ -21,7 +21,7 @@ need a workstation GPU anywhere else.
 
 ```bash
 brew install avirajkhare00/troy/troy
-# or from source: pip install ./cli
+# or: pipx install troy-cli
 ```
 
 ## Quickstart
@@ -39,7 +39,7 @@ troy export -f gguf    # ship it to llama.cpp / Ollama / LM Studio
 
 ```yaml
 base: mlx-community/Qwen3-0.6B-4bit
-task: sft            # or: dpo
+task: sft            # or: dpo, orpo
 
 data:
   train: ./data/train.jsonl   # alpaca, sharegpt, chat, completions, text — auto-detected
@@ -62,10 +62,12 @@ output: ./output
 |---|---|
 | `troy init` | Create a config from a template (`chat`, `dpo`) |
 | `troy doctor` | Hardware + dependency check, with model-size guidance |
-| `troy train` | LoRA fine-tuning: SFT or DPO |
+| `troy train` | LoRA fine-tuning: SFT, DPO, or ORPO |
 | `troy chat` | Interactive REPL (or `-p` for one-shot) with your adapter |
+| `troy eval` | Base-vs-tuned val loss, perplexity, side-by-side samples |
 | `troy serve` | OpenAI-compatible API server for your model |
 | `troy export` | Fuse the adapter; export MLX or GGUF |
+| `troy push` | Upload adapter or fused model to the Hugging Face Hub |
 | `troy data inspect` | Dataset stats and format detection |
 
 ## What Troy can train on your Mac

{troy_cli-0.1.2 → troy_cli-0.2.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "troy-cli"
-version = "0.1.2"
+version = "0.2.0"
 description = "Fine-tune LLMs on your MacBook with one YAML file. Built for Apple Silicon."
 readme = "README.md"
 requires-python = ">=3.10"

{troy_cli-0.1.2 → troy_cli-0.2.0}/src/troy/__init__.py

@@ -1,3 +1,3 @@
 """Troy: fine-tune LLMs on your MacBook with one YAML file."""
 
-__version__ = "0.1.2"
+__version__ = "0.2.0"

{troy_cli-0.1.2 → troy_cli-0.2.0}/src/troy/cli.py

@@ -6,7 +6,7 @@ import json
 import platform
 import sys
 from pathlib import Path
-from typing import Optional
+from typing import List, Optional
 
 import typer
 from rich.console import Console
@@ -149,6 +149,10 @@ def train(
         from .train_sft import run_sft
 
         run_sft(cfg, train_records, valid_records)
+    elif cfg.task == "orpo":
+        from .train_orpo import run_orpo
+
+        run_orpo(cfg, train_records, valid_records)
     else:
         from .train_dpo import run_dpo
 
@@ -248,6 +252,46 @@ def export(
     run_export(cfg.base, str(cfg.adapter_path), str(out), fmt, dequantize)
 
 
+@app.command()
+def eval(
+    config: Path = typer.Option(Path("troy.yaml"), "--config", "-c", help="Config file."),
+    prompts: Optional[List[str]] = typer.Option(
+        None, "--prompt", "-p",
+        help="Prompt for side-by-side base-vs-tuned generation (repeatable).",
+    ),
+    max_tokens: int = typer.Option(200),
+) -> None:
+    """Compare the trained adapter against the base model (loss, ppl, samples)."""
+    _require_apple_silicon()
+    from .config import load_config
+    from .data import load_and_prepare
+    from .evaluate import run_eval
+
+    cfg = load_config(config)
+    _, valid_records, _ = load_and_prepare(cfg.data, cfg.task, cfg.training.seed)
+    run_eval(cfg, valid_records, list(prompts) if prompts else None, max_tokens)
+
+
+@app.command()
+def push(
+    repo: str = typer.Argument(help="Hub repo id, e.g. username/my-model."),
+    config: Path = typer.Option(Path("troy.yaml"), "--config", "-c", help="Config file."),
+    fused: bool = typer.Option(False, help="Push the fused model instead of the adapter."),
+    public: bool = typer.Option(False, help="Make the Hub repo public."),
+) -> None:
+    """Upload your trained adapter (or fused model) to the Hugging Face Hub."""
+    from .config import load_config
+    from .push import run_push
+
+    cfg = load_config(config)
+    folder = (cfg.output_path / "fused") if fused else cfg.adapter_path
+    if not folder.exists():
+        what = "troy export" if fused else "troy train"
+        console.print(f"[red]{folder} not found. Run `{what}` first.[/red]")
+        raise typer.Exit(1)
+    run_push(folder, repo, private=not public)
+
+
 @app.command()
 def data(
     action: str = typer.Argument(help="Action: inspect"),

{troy_cli-0.1.2 → troy_cli-0.2.0}/src/troy/config.py

@@ -24,6 +24,11 @@ class DpoConfig(BaseModel):
     beta: float = 0.1
 
 
+class OrpoConfig(BaseModel):
+    lam: float = Field(0.1, alias="lambda")
+    model_config = {"populate_by_name": True}
+
+
 class DataConfig(BaseModel):
     train: str
     valid: Optional[str] = None
@@ -42,6 +47,7 @@ class TrainingConfig(BaseModel):
     seq_len: int = 2048
     lora: LoraConfig = LoraConfig()
     dpo: DpoConfig = DpoConfig()
+    orpo: OrpoConfig = OrpoConfig()
     grad_checkpoint: bool = False
     grad_accumulation_steps: int = 1
     save_every: int = 100
@@ -50,7 +56,7 @@ class TrainingConfig(BaseModel):
 
 class TroyConfig(BaseModel):
     base: str
-    task: Literal["sft", "dpo"] = "sft"
+    task: Literal["sft", "dpo", "orpo"] = "sft"
     data: DataConfig
     training: TrainingConfig = TrainingConfig()
     output: str = "./output"

{troy_cli-0.1.2 → troy_cli-0.2.0}/src/troy/data.py

@@ -117,13 +117,15 @@ def load_and_prepare(
 
     fmt = cfg.format if cfg.format != "auto" else detect_format(records[0])
 
-    if task == "dpo" and fmt != "preference":
+    if task in ("dpo", "orpo") and fmt != "preference":
         raise ValueError(
-            f"Task `dpo` needs preference data (prompt/chosen/rejected); "
+            f"Task `{task}` needs preference data (prompt/chosen/rejected); "
             f"detected format `{fmt}`."
         )
     if task == "sft" and fmt == "preference":
-        raise ValueError("Preference data detected — set `task: dpo` in troy.yaml.")
+        raise ValueError(
+            "Preference data detected — set `task: dpo` (or `task: orpo`) in troy.yaml."
+        )
 
     if fmt == "preference":
         records = [_normalize_preference(r) for r in records]

troy_cli-0.2.0/src/troy/evaluate.py

@@ -0,0 +1,88 @@
+"""Evaluate a trained adapter: base vs tuned perplexity and side-by-side samples.
+
+The base-model numbers reuse the zero-scale trick: with every LoRA scale set
+to 0 the tuned model *is* the base model, so both sides come from one load.
+"""
+
+from __future__ import annotations
+
+import math
+import types
+from typing import Any, Dict, List, Optional
+
+import mlx.core as mx
+from mlx_lm.generate import generate
+from mlx_lm.sample_utils import make_sampler
+from mlx_lm.tuner.datasets import CacheDataset, create_dataset
+from mlx_lm.tuner.trainer import default_loss, evaluate, iterate_batches
+from mlx_lm.utils import load
+
+from .config import TroyConfig
+from .train_dpo import _ReferenceMode
+
+
+def _val_loss(model, dataset, batch_size: int, seq_len: int) -> float:
+    return float(
+        evaluate(
+            model=model,
+            dataset=CacheDataset(dataset),
+            batch_size=batch_size,
+            num_batches=-1,
+            max_seq_length=seq_len,
+            loss=default_loss,
+            iterate_batches=iterate_batches,
+        )
+    )
+
+
+def run_eval(
+    config: TroyConfig,
+    valid_records: List[Dict[str, Any]],
+    sample_prompts: Optional[List[str]] = None,
+    max_tokens: int = 200,
+) -> None:
+    adapter_file = config.adapter_path / "adapters.safetensors"
+    if not adapter_file.exists():
+        raise SystemExit(f"No adapter at {adapter_file}. Run `troy train` first.")
+
+    print(f"Loading {config.base} + adapter ...")
+    model, tokenizer = load(config.base, adapter_path=str(config.adapter_path))
+
+    results = {}
+    if valid_records and config.task == "sft":
+        ds_config = types.SimpleNamespace(mask_prompt=config.data.mask_prompt)
+        dataset = create_dataset(valid_records, tokenizer, ds_config)
+        bs = max(1, min(4, len(valid_records)))
+
+        tuned = _val_loss(model, dataset, bs, config.training.seq_len)
+        with _ReferenceMode(model):
+            base = _val_loss(model, dataset, bs, config.training.seq_len)
+        results = {
+            "val records": len(valid_records),
+            "base loss": f"{base:.3f}",
+            "tuned loss": f"{tuned:.3f}",
+            "base ppl": f"{math.exp(base):.1f}",
+            "tuned ppl": f"{math.exp(tuned):.1f}",
+        }
+        for k, v in results.items():
+            print(f"  {k:>12}: {v}")
+        delta = base - tuned
+        print(f"  {'Δ loss':>12}: {delta:+.3f} ({'tuned better' if delta > 0 else 'base better'})")
+    elif config.task != "sft":
+        print("(loss comparison is defined for task: sft — showing generations only)")
+
+    if sample_prompts:
+        sampler = make_sampler(temp=0.0)
+        print("\n--- side-by-side generations (temperature 0) ---")
+        for prompt in sample_prompts:
+            templated = tokenizer.apply_chat_template(
+                [{"role": "user", "content": prompt}],
+                add_generation_prompt=True,
+                return_dict=False,
+            )
+            tuned_out = generate(model, tokenizer, templated, max_tokens=max_tokens, sampler=sampler)
+            with _ReferenceMode(model):
+                base_out = generate(model, tokenizer, templated, max_tokens=max_tokens, sampler=sampler)
+            print(f"\n>> {prompt}")
+            print(f"[base]  {base_out.strip()[:400]}")
+            print(f"[tuned] {tuned_out.strip()[:400]}")

troy_cli-0.2.0/src/troy/push.py

@@ -0,0 +1,25 @@
+"""Upload a trained adapter or fused model to the Hugging Face Hub."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+
+def run_push(folder: Path, repo: str, private: bool = True) -> str:
+    from huggingface_hub import HfApi
+
+    api = HfApi()
+    try:
+        who = api.whoami()
+    except Exception:
+        raise SystemExit(
+            "Not logged in to Hugging Face. Run: hf auth login "
+            "(or set HF_TOKEN) and retry."
+        )
+    print(f"Logged in as {who['name']}. Uploading {folder} -> {repo} "
+          f"({'private' if private else 'public'}) ...")
+    api.create_repo(repo, exist_ok=True, private=private)
+    api.upload_folder(folder_path=str(folder), repo_id=repo)
+    url = f"https://huggingface.co/{repo}"
+    print(f"Done: {url}")
+    return url

troy_cli-0.2.0/src/troy/train_orpo.py

@@ -0,0 +1,119 @@
+"""ORPO: Odds Ratio Preference Optimization on Apple Silicon.
+
+ORPO needs no reference model at all — the loss combines a standard SFT term
+on the chosen response with an odds-ratio penalty pushing chosen above
+rejected. One model in memory, one pass per completion.
+"""
+
+from __future__ import annotations
+
+import time
+from typing import Any, Dict, List
+
+import mlx.core as mx
+import mlx.nn as nn
+import mlx.optimizers as optim
+from mlx.utils import tree_flatten
+from mlx_lm.tuner.trainer import grad_checkpoint
+from mlx_lm.utils import load
+
+from .config import TroyConfig
+from .train_dpo import _batch, _encode_pair
+from .train_sft import apply_lora, resolve_batch_size, resolve_iters, save_adapter_config
+
+
+def _mean_logps(model, tokens: mx.array, mask: mx.array):
+    """Per-sequence mean log-prob over completion tokens (and the sum + count)."""
+    logits = model(tokens[:, :-1])
+    targets = tokens[:, 1:]
+    logps = -nn.losses.cross_entropy(logits, targets, reduction="none")
+    total = (logps * mask).sum(axis=-1)
+    count = mx.maximum(mask.sum(axis=-1), 1)
+    return total / count, total, count
+
+
+def run_orpo(
+    config: TroyConfig,
+    train_records: List[Dict[str, Any]],
+    valid_records: List[Dict[str, Any]],
+) -> None:
+    mx.random.seed(config.training.seed)
+    print(f"Loading {config.base} ...")
+    model, tokenizer = load(config.base)
+
+    batch_size = resolve_batch_size(config)
+    if config.training.batch_size == "auto":
+        batch_size = max(1, batch_size // 2)  # chosen + rejected per example
+    iters = resolve_iters(config, len(train_records), batch_size)
+    num_layers = apply_lora(model, config)
+    save_adapter_config(config, num_layers)
+    if config.training.grad_checkpoint:
+        grad_checkpoint(model.layers[0])
+
+    lam = config.training.orpo.lam
+    max_len = config.training.seq_len
+    pad_id = tokenizer.pad_token_id or tokenizer.eos_token_id or 0
+
+    encoded = [
+        (
+            _encode_pair(tokenizer, r["prompt"], r["chosen"], max_len),
+            _encode_pair(tokenizer, r["prompt"], r["rejected"], max_len),
+        )
+        for r in train_records
+    ]
+
+    def loss_fn(model, tc, mc, tr, mr):
+        mean_c, total_c, count_c = _mean_logps(model, tc, mc)
+        mean_r, _, _ = _mean_logps(model, tr, mr)
+        # log odds: log(p/(1-p)) with p = exp(mean logp)
+        log_odds = (mean_c - mean_r) - (
+            mx.log1p(-mx.exp(mx.minimum(mean_c, -1e-6)))
+            - mx.log1p(-mx.exp(mx.minimum(mean_r, -1e-6)))
+        )
+        or_loss = -nn.log_sigmoid(log_odds).mean()
+        sft_loss = -(total_c.sum() / count_c.sum())
+        reward_acc = (mean_c > mean_r).mean()
+        return sft_loss + lam * or_loss, reward_acc
+
+    loss_and_grad = nn.value_and_grad(model, loss_fn)
+    opt = optim.Adam(learning_rate=config.training.lr)
+
+    print(
+        f"Training: task=orpo lambda={lam} batch_size={batch_size} iters={iters} "
+        f"lr={config.training.lr} pairs={len(encoded)}"
+    )
+
+    config.adapter_path.mkdir(parents=True, exist_ok=True)
+    adapter_file = config.adapter_path / "adapters.safetensors"
+    n = len(encoded)
+    losses, accs = [], []
+    start = time.time()
+
+    for it in range(iters):
+        idx = [(it * batch_size + k) % n for k in range(batch_size)]
+        tc, mc = _batch([encoded[i][0] for i in idx], pad_id)
+        tr, mr = _batch([encoded[i][1] for i in idx], pad_id)
+
+        (loss, acc), grads = loss_and_grad(model, tc, mc, tr, mr)
+        opt.update(model, grads)
+        mx.eval(model.parameters(), opt.state, loss)
+        losses.append(loss.item())
+        accs.append(acc.item())
+
+        if (it + 1) % 10 == 0 or it == iters - 1:
+            speed = (it + 1) / (time.time() - start)
+            print(
+                f"Iter {it + 1}/{iters}: loss {sum(losses)/len(losses):.4f}, "
+                f"reward acc {sum(accs)/len(accs):.3f}, {speed:.2f} it/s"
+            )
+            losses, accs = [], []
+
+        if (it + 1) % config.training.save_every == 0:
+            _save(model, adapter_file)
+
+    _save(model, adapter_file)
+    print(f"\nDone. Adapter saved to {config.adapter_path}")
+
+
+def _save(model, adapter_file) -> None:
+    mx.save_safetensors(str(adapter_file), dict(tree_flatten(model.trainable_parameters())))

{troy_cli-0.1.2 → troy_cli-0.2.0}/tests/test_config.py

@@ -49,3 +49,11 @@ def test_missing_file_message(tmp_path):
 def test_adapter_path_under_output():
     cfg = TroyConfig.model_validate({**BASE, "output": "/tmp/run1"})
     assert str(cfg.adapter_path) == "/tmp/run1/adapter"
+
+
+def test_orpo_config():
+    cfg = TroyConfig.model_validate(
+        {**BASE, "task": "orpo", "training": {"orpo": {"lambda": 0.2}}}
+    )
+    assert cfg.task == "orpo"
+    assert cfg.training.orpo.lam == 0.2