troy-cli 0.3.0__tar.gz → 0.4.0__tar.gz

{troy_cli-0.3.0 → troy_cli-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: troy-cli
-Version: 0.3.0
+Version: 0.4.0
 Summary: Fine-tune LLMs on your MacBook with one YAML file. Built for Apple Silicon.
 Author: Troy
 License: Apache-2.0

{troy_cli-0.3.0 → troy_cli-0.4.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "troy-cli"
-version = "0.3.0"
+version = "0.4.0"
 description = "Fine-tune LLMs on your MacBook with one YAML file. Built for Apple Silicon."
 readme = "README.md"
 requires-python = ">=3.10"

{troy_cli-0.3.0 → troy_cli-0.4.0}/src/troy/__init__.py

@@ -1,3 +1,3 @@
 """Troy: fine-tune LLMs on your MacBook with one YAML file."""
 
-__version__ = "0.3.0"
+__version__ = "0.4.0"

{troy_cli-0.3.0 → troy_cli-0.4.0}/src/troy/cli.py

@@ -319,15 +319,19 @@ def push(
     run_push(folder, repo, private=not public)
 
 
-@app.command()
-def data(
-    action: str = typer.Argument(help="Action: inspect"),
+data_app = typer.Typer(
+    name="data",
+    help="Inspect, validate, and synthesize datasets.",
+    no_args_is_help=True,
+)
+app.add_typer(data_app)
+
+
+@data_app.command()
+def inspect(
     path: Path = typer.Argument(help="Dataset file (.jsonl, .json, .csv)."),
 ) -> None:
     """Inspect a dataset: record count, detected format, sizes."""
-    if action != "inspect":
-        console.print("[red]Only `troy data inspect <path>` is supported.[/red]")
-        raise typer.Exit(1)
     from .data import inspect_stats
 
     stats = inspect_stats(str(path))
@@ -341,5 +345,92 @@ def data(
     console.print(table)
 
 
+@data_app.command()
+def validate(
+    path: Path = typer.Argument(help="Dataset file (.jsonl, .json, .csv)."),
+) -> None:
+    """Lint a dataset: broken records, mixed formats, empty fields, duplicates."""
+    from .data import validate_records
+
+    report = validate_records(str(path))
+    console.print(
+        f"{report['records']} records, format: [bold]{report['format']}[/bold]"
+    )
+    if not report["issues"]:
+        console.print("[green]No issues found.[/green]")
+        return
+    for issue in report["issues"]:
+        console.print(f"  [yellow]•[/yellow] {issue}")
+    if report["truncated"]:
+        console.print("  [dim]... more issues not shown[/dim]")
+    console.print(f"[red]{len(report['issues'])}{'+' if report['truncated'] else ''} issue(s).[/red]")
+    raise typer.Exit(1)
+
+
+@data_app.command()
+def synth(
+    source: Optional[Path] = typer.Option(
+        None, "--from", help="Ground examples in a file or folder of docs/code."
+    ),
+    seed: Optional[str] = typer.Option(
+        None, "--seed", help='Task description, e.g. "customer support bot for Acme".'
+    ),
+    n: int = typer.Option(100, "--n", help="Number of examples to generate."),
+    fmt: str = typer.Option(
+        "chat", "--format", "-f",
+        help="Output format: chat (SFT) or preference (DPO/ORPO).",
+    ),
+    teacher: str = typer.Option(
+        "auto", help="Teacher model (auto = sized to this Mac's memory)."
+    ),
+    out: Optional[Path] = typer.Option(
+        None, "--out", "-o",
+        help="Output file (default: data/train.jsonl or data/preferences.jsonl).",
+    ),
+    max_tokens: int = typer.Option(2048, help="Max tokens per teacher call."),
+    temperature: float = typer.Option(0.8),
+) -> None:
+    """Synthesize a training dataset with a local teacher model."""
+    _require_apple_silicon()
+    if source is None and seed is None:
+        console.print(
+            '[red]Give the teacher something to work from:[/red] '
+            '--from ./docs and/or --seed "task description".'
+        )
+        raise typer.Exit(1)
+    if fmt not in ("chat", "preference"):
+        console.print("[red]--format must be `chat` or `preference`.[/red]")
+        raise typer.Exit(1)
+
+    from .synth import pick_teacher, run_synth
+
+    if teacher == "auto":
+        teacher = pick_teacher()
+        console.print(f"Teacher: [bold]{teacher}[/bold] (picked for this Mac's memory)")
+    out = out or Path("data") / ("preferences.jsonl" if fmt == "preference" else "train.jsonl")
+    if out.exists():
+        console.print(f"[red]{out} already exists[/red] — pass -o to write elsewhere.")
+        raise typer.Exit(1)
+
+    stats = run_synth(
+        n=n, out_path=out, fmt=fmt, teacher=teacher,
+        seed_task=seed, source=source,
+        max_tokens=max_tokens, temperature=temperature,
+    )
+    console.print(
+        f"\nWrote [bold]{stats['records']}[/bold] examples to [bold]{stats['out']}[/bold] "
+        f"({stats['teacher_calls']} teacher calls)"
+    )
+    if stats["records"] < stats["requested"]:
+        console.print(
+            f"[yellow]Stopped at {stats['records']}/{stats['requested']} — "
+            "try a larger --teacher, higher --max-tokens, or more source material.[/yellow]"
+        )
+    console.print(
+        "Review the data before training — spot-check a dozen examples, then: "
+        "[bold]troy data validate " + str(out) + "[/bold] and [bold]troy train[/bold]."
+    )
+
+
 if __name__ == "__main__":
     app()

{troy_cli-0.3.0 → troy_cli-0.4.0}/src/troy/data.py

@@ -152,6 +152,120 @@ def load_and_prepare(
     return train, valid, fmt
 
 
+# Required non-empty string fields per format (nested fields checked separately).
+_REQUIRED_FIELDS = {
+    "alpaca": ("instruction", "output"),
+    "completions": ("prompt", "completion"),
+    "preference": ("prompt", "chosen", "rejected"),
+    "text": ("text",),
+}
+
+
+def _record_issues(record: Dict[str, Any], fmt: str, line: int) -> List[str]:
+    issues = []
+
+    def empty(v: Any) -> bool:
+        return not (isinstance(v, str) and v.strip())
+
+    for field in _REQUIRED_FIELDS.get(fmt, ()):
+        if empty(record.get(field)):
+            issues.append(f"line {line}: empty or missing `{field}`")
+    if fmt == "chat":
+        msgs = record.get("messages")
+        if not isinstance(msgs, list) or not msgs:
+            issues.append(f"line {line}: `messages` is not a non-empty list")
+        else:
+            roles = [m.get("role") for m in msgs if isinstance(m, dict)]
+            if "assistant" not in roles:
+                issues.append(f"line {line}: no assistant message")
+            if any(
+                not isinstance(m, dict) or empty(m.get("content"))
+                for m in msgs
+            ):
+                issues.append(f"line {line}: message with empty content")
+    if fmt == "sharegpt":
+        convs = record.get("conversations")
+        if not isinstance(convs, list) or not convs:
+            issues.append(f"line {line}: `conversations` is not a non-empty list")
+        else:
+            bad = [
+                m.get("from")
+                for m in convs
+                if not isinstance(m, dict)
+                or str(m.get("from", "")).lower() not in _SHAREGPT_ROLES
+            ]
+            if bad:
+                issues.append(f"line {line}: unknown speaker role(s): {bad}")
+    if fmt == "preference" and not issues:
+        if record["chosen"].strip() == record["rejected"].strip():
+            issues.append(f"line {line}: chosen == rejected")
+    return issues
+
+
+def validate_records(path: str, max_issues: int = 50) -> Dict[str, Any]:
+    """Lint a dataset: per-record issues, mixed formats, duplicates.
+
+    Returns {"records", "format", "issues", "duplicates", "truncated"}.
+    """
+    p = Path(path).expanduser()
+    issues: List[str] = []
+    records: List[Tuple[int, Dict[str, Any]]] = []
+
+    if p.suffix.lower() == ".jsonl":
+        with open(p) as f:
+            for i, line in enumerate(f, 1):
+                if not line.strip():
+                    continue
+                try:
+                    records.append((i, json.loads(line)))
+                except json.JSONDecodeError as e:
+                    issues.append(f"line {i}: invalid JSON ({e.msg})")
+    else:
+        records = [(i, r) for i, r in enumerate(_read_records(p), 1)]
+
+    if not records:
+        return {
+            "records": 0, "format": "unknown", "issues": issues or ["file has no records"],
+            "duplicates": 0, "truncated": False,
+        }
+
+    try:
+        fmt = detect_format(records[0][1])
+    except ValueError as e:
+        return {
+            "records": len(records), "format": "unknown",
+            "issues": issues + [str(e)], "duplicates": 0, "truncated": False,
+        }
+
+    seen: Dict[str, int] = {}
+    duplicates = 0
+    for line, record in records:
+        try:
+            rec_fmt = detect_format(record)
+        except ValueError:
+            issues.append(f"line {line}: keys match no known format")
+            continue
+        if rec_fmt != fmt:
+            issues.append(f"line {line}: format `{rec_fmt}` (file is `{fmt}`)")
+            continue
+        issues.extend(_record_issues(record, fmt, line))
+        key = json.dumps(record, sort_keys=True)
+        if key in seen:
+            duplicates += 1
+            issues.append(f"line {line}: exact duplicate of line {seen[key]}")
+        else:
+            seen[key] = line
+
+    truncated = len(issues) > max_issues
+    return {
+        "records": len(records),
+        "format": fmt,
+        "issues": issues[:max_issues],
+        "duplicates": duplicates,
+        "truncated": truncated,
+    }
+
+
 def inspect_stats(path: str) -> Dict[str, Any]:
     """Lightweight dataset statistics for `troy data inspect`-style output."""
     records = _read_records(Path(path).expanduser())

troy_cli-0.4.0/src/troy/synth.py

@@ -0,0 +1,247 @@
+"""Synthesize training data with a local teacher model.
+
+`troy data synth` turns your documents (or a task description) into a
+train.jsonl — the teacher runs locally via mlx-lm, so nothing leaves the Mac.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Tuple
+
+from .hardware import detect
+
+# Source files worth mining for Q&A pairs.
+SOURCE_SUFFIXES = {
+    ".md", ".mdx", ".txt", ".rst", ".adoc",
+    ".py", ".js", ".ts", ".go", ".rs", ".java", ".rb", ".swift",
+    ".yaml", ".yml", ".toml", ".json", ".html",
+}
+
+# (min unified memory GB, teacher) — instruct models, 4-bit; the teacher only
+# runs inference, so it can be larger than what the same Mac can train.
+TEACHER_GUIDANCE = [
+    (64, "mlx-community/Qwen3-30B-A3B-4bit"),
+    (36, "mlx-community/Qwen3-14B-4bit"),
+    (24, "mlx-community/Qwen3-8B-4bit"),
+    (16, "mlx-community/Qwen3-4B-4bit"),
+    (0, "mlx-community/Qwen3-1.7B-4bit"),
+]
+
+PAIRS_PER_CALL = 10
+
+
+def pick_teacher(memory_gb: Optional[float] = None) -> str:
+    if memory_gb is None:
+        memory_gb = detect().memory_gb
+    for min_gb, model in TEACHER_GUIDANCE:
+        if memory_gb >= min_gb:
+            return model
+    return TEACHER_GUIDANCE[-1][1]
+
+
+def collect_sources(path: Path) -> List[Tuple[str, str]]:
+    """Read (name, text) from a file or a folder of text-ish files."""
+    path = path.expanduser()
+    if path.is_file():
+        return [(path.name, path.read_text(errors="ignore"))]
+    if not path.is_dir():
+        raise ValueError(f"Source not found: {path}")
+    sources = []
+    for f in sorted(path.rglob("*")):
+        if not f.is_file() or f.suffix.lower() not in SOURCE_SUFFIXES:
+            continue
+        if any(part.startswith(".") for part in f.relative_to(path).parts):
+            continue  # skip hidden dirs/files (.git, .venv, ...)
+        text = f.read_text(errors="ignore").strip()
+        if text:
+            sources.append((str(f.relative_to(path)), text))
+    if not sources:
+        raise ValueError(
+            f"No readable text files under {path} "
+            f"(looked for {', '.join(sorted(SOURCE_SUFFIXES))})"
+        )
+    return sources
+
+
+def chunk_text(text: str, size: int = 4000, overlap: int = 200) -> List[str]:
+    """Split on paragraph boundaries where possible, hard-split otherwise."""
+    if len(text) <= size:
+        return [text]
+    chunks = []
+    start = 0
+    while start < len(text):
+        end = start + size
+        if end < len(text):
+            cut = text.rfind("\n\n", start + size // 2, end)
+            if cut != -1:
+                end = cut
+        chunks.append(text[start:end].strip())
+        if end >= len(text):
+            break
+        start = max(end - overlap, start + 1)
+    return [c for c in chunks if c]
+
+
+_THINK_RE = re.compile(r"<think>.*?</think>", re.DOTALL)
+
+
+def parse_pairs(raw: str, keys: Tuple[str, ...]) -> List[Dict[str, str]]:
+    """Extract JSON objects with the given string keys from teacher output.
+
+    Tolerates thinking blocks, code fences, prose between objects, and
+    objects that span multiple lines.
+    """
+    raw = _THINK_RE.sub("", raw)
+    pairs = []
+    decoder = json.JSONDecoder()
+    pos = 0
+    while True:
+        brace = raw.find("{", pos)
+        if brace == -1:
+            break
+        try:
+            obj, consumed = decoder.raw_decode(raw[brace:])
+            pos = brace + consumed
+        except json.JSONDecodeError:
+            pos = brace + 1
+            continue
+        if not isinstance(obj, dict):
+            continue
+        values = {k: obj.get(k) for k in keys}
+        if all(isinstance(v, str) and v.strip() for v in values.values()):
+            pairs.append({k: v.strip() for k, v in values.items()})
+    return pairs
+
+
+def _chat_prompt(task: str, material: Optional[str], n: int) -> str:
+    lines = [
+        f"You are creating fine-tuning data for this task: {task}",
+        "",
+        f"Write {n} diverse training examples. Output ONLY JSON objects, "
+        'one per line, each exactly: {"user": "...", "assistant": "..."}',
+        "Vary phrasing, length, and difficulty. No numbering, no commentary.",
+    ]
+    if material:
+        lines += [
+            "",
+            "Ground every example in this source material — questions a reader "
+            "would ask about it, answered faithfully from it:",
+            "---",
+            material,
+            "---",
+        ]
+    return "\n".join(lines)
+
+
+def _preference_prompt(task: str, material: Optional[str], n: int) -> str:
+    lines = [
+        f"You are creating preference-tuning data for this task: {task}",
+        "",
+        f"Write {n} diverse training examples. Output ONLY JSON objects, one "
+        'per line, each exactly: '
+        '{"prompt": "...", "chosen": "...", "rejected": "..."}',
+        "`chosen` is a genuinely good response; `rejected` is plausible but "
+        "clearly worse for the task (vague, bloated, off-style, or subtly "
+        "wrong). No numbering, no commentary.",
+    ]
+    if material:
+        lines += [
+            "",
+            "Ground every example in this source material:",
+            "---",
+            material,
+            "---",
+        ]
+    return "\n".join(lines)
+
+
+def run_synth(
+    n: int,
+    out_path: Path,
+    fmt: str,
+    teacher: str,
+    seed_task: Optional[str],
+    source: Optional[Path],
+    max_tokens: int,
+    temperature: float,
+) -> Dict[str, Any]:
+    """Generate n examples; returns stats. Writes JSONL to out_path."""
+    from mlx_lm.generate import generate
+    from mlx_lm.sample_utils import make_sampler
+    from mlx_lm.utils import load
+
+    chunks: List[Optional[str]]
+    if source is not None:
+        texts = collect_sources(source)
+        chunks = [c for _, text in texts for c in chunk_text(text)]
+    else:
+        chunks = [None]  # pure seed-description mode
+
+    task = seed_task or (
+        "answering questions about the source material accurately and concisely"
+    )
+    keys = ("prompt", "chosen", "rejected") if fmt == "preference" else ("user", "assistant")
+    build = _preference_prompt if fmt == "preference" else _chat_prompt
+
+    print(f"Loading teacher {teacher} ...")
+    model, tokenizer = load(teacher)
+    sampler = make_sampler(temp=temperature)
+
+    seen = set()
+    records: List[Dict[str, Any]] = []
+    calls = failures = 0
+    chunk_i = 0
+    max_calls = (n // PAIRS_PER_CALL + 1) * 4  # generous retry budget
+
+    while len(records) < n and calls < max_calls:
+        want = min(PAIRS_PER_CALL, n - len(records))
+        prompt = build(task, chunks[chunk_i % len(chunks)], want)
+        chunk_i += 1
+        calls += 1
+        messages = [{"role": "user", "content": prompt}]
+        templated = tokenizer.apply_chat_template(
+            messages, add_generation_prompt=True, return_dict=False
+        )
+        raw = generate(
+            model, tokenizer, templated, max_tokens=max_tokens, sampler=sampler
+        )
+        pairs = parse_pairs(raw, keys)
+        if not pairs:
+            failures += 1
+            continue
+        for pair in pairs:
+            key = pair[keys[0]].lower()
+            if key in seen:
+                continue
+            seen.add(key)
+            if fmt == "preference":
+                records.append(pair)
+            else:
+                records.append(
+                    {
+                        "messages": [
+                            {"role": "user", "content": pair["user"]},
+                            {"role": "assistant", "content": pair["assistant"]},
+                        ]
+                    }
+                )
+            if len(records) >= n:
+                break
+        print(f"  {len(records)}/{n} examples ({calls} teacher calls)")
+
+    out_path.parent.mkdir(parents=True, exist_ok=True)
+    with open(out_path, "w") as f:
+        for r in records:
+            f.write(json.dumps(r, ensure_ascii=False) + "\n")
+
+    return {
+        "records": len(records),
+        "requested": n,
+        "teacher_calls": calls,
+        "empty_responses": failures,
+        "chunks": 0 if chunks == [None] else len(chunks),
+        "out": str(out_path),
+    }

{troy_cli-0.3.0 → troy_cli-0.4.0}/tests/test_data.py

@@ -74,3 +74,50 @@ def test_explicit_valid_file(tmp_path):
     write_jsonl(v, [{"text": "b"}] * 2)
     train, valid, _ = load_and_prepare(DataConfig(train=str(t), valid=str(v)), "sft")
     assert len(train) == 6 and len(valid) == 2
+
+
+def test_validate_clean_file(tmp_path):
+    from troy.data import validate_records
+
+    f = tmp_path / "t.jsonl"
+    write_jsonl(f, [{"instruction": f"q{i}", "output": f"a{i}"} for i in range(5)])
+    report = validate_records(str(f))
+    assert report["format"] == "alpaca"
+    assert report["records"] == 5
+    assert report["issues"] == []
+
+
+def test_validate_catches_problems(tmp_path):
+    from troy.data import validate_records
+
+    f = tmp_path / "t.jsonl"
+    rows = [
+        {"instruction": "q", "output": "a"},          # ok
+        {"instruction": "", "output": "a"},           # empty field
+        {"prompt": "p", "completion": "c"},           # mixed format
+        {"instruction": "q", "output": "a"},          # duplicate of line 1
+    ]
+    with open(f, "w") as fh:
+        for r in rows:
+            fh.write(json.dumps(r) + "\n")
+        fh.write("{broken json\n")
+
+    report = validate_records(str(f))
+    text = "\n".join(report["issues"])
+    assert "empty or missing `instruction`" in text
+    assert "format `completions`" in text
+    assert "duplicate of line 1" in text
+    assert "invalid JSON" in text
+    assert report["duplicates"] == 1
+
+
+def test_validate_preference_and_chat(tmp_path):
+    from troy.data import validate_records
+
+    f = tmp_path / "p.jsonl"
+    write_jsonl(f, [{"prompt": "p", "chosen": "same", "rejected": "same"}])
+    assert "chosen == rejected" in validate_records(str(f))["issues"][0]
+
+    f2 = tmp_path / "c.jsonl"
+    write_jsonl(f2, [{"messages": [{"role": "user", "content": "hi"}]}])
+    assert "no assistant message" in validate_records(str(f2))["issues"][0]

troy_cli-0.4.0/tests/test_synth.py

@@ -0,0 +1,82 @@
+import pytest
+
+from troy.synth import chunk_text, collect_sources, parse_pairs, pick_teacher
+
+
+def test_pick_teacher_scales_with_memory():
+    assert "1.7B" in pick_teacher(8)
+    assert "4B" in pick_teacher(16)
+    assert "8B" in pick_teacher(24)
+    assert "14B" in pick_teacher(36)
+    assert "30B" in pick_teacher(64)
+
+
+def test_chunk_text_short_passthrough():
+    assert chunk_text("hello", size=100) == ["hello"]
+
+
+def test_chunk_text_splits_on_paragraphs():
+    text = ("para one " * 50 + "\n\n" + "para two " * 50).strip()
+    chunks = chunk_text(text, size=500, overlap=50)
+    assert len(chunks) >= 2
+    assert all(len(c) <= 500 for c in chunks)
+    # nothing lost beyond whitespace at the seams
+    assert "para two" in chunks[-1]
+
+
+def test_parse_pairs_jsonl():
+    raw = (
+        '{"user": "What is Troy?", "assistant": "A fine-tuning CLI."}\n'
+        '{"user": "Which Macs?", "assistant": "Apple Silicon, M1+."}\n'
+    )
+    pairs = parse_pairs(raw, ("user", "assistant"))
+    assert len(pairs) == 2
+    assert pairs[0]["user"] == "What is Troy?"
+
+
+def test_parse_pairs_tolerates_noise():
+    raw = (
+        "<think>let me write some examples</think>\n"
+        "Here are the examples:\n"
+        "```json\n"
+        '{"user": "q1",\n "assistant": "a1"}\n'
+        "```\n"
+        "not json at all {broken\n"
+        '{"user": "q2", "assistant": "a2", "extra": 1}\n'
+        '{"user": "", "assistant": "empty user skipped"}\n'
+        '{"user": "no assistant key"}\n'
+    )
+    pairs = parse_pairs(raw, ("user", "assistant"))
+    assert [p["user"] for p in pairs] == ["q1", "q2"]
+    assert "extra" not in pairs[1]
+
+
+def test_parse_pairs_preference_keys():
+    raw = '{"prompt": "p", "chosen": "good", "rejected": "bad"}'
+    pairs = parse_pairs(raw, ("prompt", "chosen", "rejected"))
+    assert pairs == [{"prompt": "p", "chosen": "good", "rejected": "bad"}]
+
+
+def test_collect_sources_file_and_folder(tmp_path):
+    (tmp_path / "a.md").write_text("# Doc A")
+    (tmp_path / "sub").mkdir()
+    (tmp_path / "sub" / "b.txt").write_text("Doc B")
+    (tmp_path / ".git").mkdir()
+    (tmp_path / ".git" / "c.md").write_text("hidden")
+    (tmp_path / "img.png").write_bytes(b"\x89PNG")
+
+    sources = collect_sources(tmp_path)
+    names = [n for n, _ in sources]
+    assert names == ["a.md", "sub/b.txt"]
+
+    single = collect_sources(tmp_path / "a.md")
+    assert single == [("a.md", "# Doc A")]
+
+
+def test_collect_sources_errors(tmp_path):
+    with pytest.raises(ValueError, match="not found"):
+        collect_sources(tmp_path / "missing")
+    empty = tmp_path / "empty"
+    empty.mkdir()
+    with pytest.raises(ValueError, match="No readable"):
+        collect_sources(empty)