trainproof 0.1.0__py3-none-any.whl

trainproof/__init__.py

@@ -0,0 +1,5 @@
+"""
+trainproof: A deterministic linter for ML training runs.
+"""
+
+__version__ = "0.1.0"

trainproof/adapters.py

@@ -0,0 +1,143 @@
+import json
+import csv
+import re
+from pathlib import Path
+from datetime import datetime
+
+ANSI_ESCAPE_PATTERN = re.compile(r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])')
+
+def parse_hf_trainer_state(text: str) -> list[dict[str, float]]:
+    try:
+        data = json.loads(text)
+    except json.JSONDecodeError:
+        return []
+        
+    history = data.get("log_history", [])
+    records = []
+    for entry in history:
+        if "loss" not in entry:
+            continue
+        record = {}
+        for k in ["loss", "grad_norm", "step"]:
+            if k in entry and entry[k] is not None:
+                try:
+                    record[k] = float(entry[k])
+                except (ValueError, TypeError):
+                    pass
+        if "learning_rate" in entry and entry["learning_rate"] is not None:
+            try:
+                record["lr"] = float(entry["learning_rate"])
+            except (ValueError, TypeError):
+                pass
+        records.append(record)
+    return records
+
+def parse_coqui_trainer_log(text: str) -> list[dict[str, float]]:
+    text = ANSI_ESCAPE_PATTERN.sub('', text)
+    records = []
+    current_record = None
+    
+    for line in text.splitlines():
+        line = line.strip()
+        if not line:
+            continue
+            
+        if line.startswith("--> TIME:") and "-- GLOBAL_STEP:" in line:
+            if current_record is not None and "loss" in current_record:
+                records.append(current_record)
+            current_record = {}
+            
+            m_time = re.search(r'TIME:\s*([\d-]+\s[\d:]+)', line)
+            if m_time:
+                try:
+                    dt = datetime.strptime(m_time.group(1), "%Y-%m-%d %H:%M:%S")
+                    current_record["time"] = dt.timestamp()
+                except ValueError:
+                    pass
+                    
+            m_step = re.search(r'GLOBAL_STEP:\s*(\d+)', line)
+            if m_step:
+                current_record["step"] = float(m_step.group(1))
+                
+        elif current_record is not None and line.startswith("| > "):
+            parts = line[4:].split(":")
+            if len(parts) >= 2:
+                key = parts[0].strip()
+                if key in ("loss", "current_lr"):
+                    val_str = parts[1].strip().split()[0]
+                    try:
+                        val = float(val_str)
+                        if key == "loss":
+                            current_record["loss"] = val
+                        elif key == "current_lr":
+                            current_record["lr"] = val
+                    except ValueError:
+                        pass
+                        
+    if current_record is not None and "loss" in current_record:
+        records.append(current_record)
+        
+    return records
+
+def parse_generic_log(text: str, is_csv: bool) -> list[dict[str, float]]:
+    records = []
+    if is_csv:
+        reader = csv.DictReader(text.splitlines())
+        if reader.fieldnames:
+            for row in reader:
+                norm_row = {}
+                for k, v in row.items():
+                    if v is None or not str(v).strip():
+                        continue
+                    try:
+                        norm_row[k.strip().lower()] = float(v)
+                    except (ValueError, TypeError):
+                        pass
+                if norm_row:
+                    records.append(norm_row)
+    else:
+        for line in text.splitlines():
+            line = line.strip()
+            if not line: continue
+            try:
+                data = json.loads(line)
+                norm_row = {}
+                for k, v in data.items():
+                    try:
+                        norm_row[k.strip().lower()] = float(v)
+                    except (ValueError, TypeError):
+                        pass
+                records.append(norm_row)
+            except Exception:
+                pass
+    return records
+
+def parse_log_with_format(path: str | Path, fmt: str = "auto") -> list[dict[str, float]]:
+    path = Path(path)
+    if not path.exists():
+        raise FileNotFoundError(f"Log file not found: {path}")
+        
+    text = path.read_text(encoding="utf-8", errors="ignore").strip()
+    
+    if fmt == "auto":
+        if path.name == "trainer_state.json":
+            fmt = "hf"
+        elif path.suffix == ".json" and '"log_history"' in text:
+            fmt = "hf"
+        elif "-- GLOBAL_STEP:" in text:
+            fmt = "coqui"
+        elif path.suffix == ".csv" or (text and text[0] != '{'):
+            fmt = "csv"
+        else:
+            fmt = "jsonl"
+            
+    if fmt == "hf":
+        return parse_hf_trainer_state(text)
+    elif fmt == "coqui":
+        return parse_coqui_trainer_log(text)
+    elif fmt == "csv":
+        return parse_generic_log(text, is_csv=True)
+    elif fmt == "jsonl":
+        return parse_generic_log(text, is_csv=False)
+    else:
+        raise ValueError(f"Unknown format: {fmt}")

trainproof/cli.py

@@ -0,0 +1,55 @@
+import argparse
+import sys
+from pathlib import Path
+from .speech.data import check_data
+from .speech.tokenizer import check_tokenizer
+from .epoch import check_epoch
+from .report import print_verdict_console, write_html_report
+
+def main():
+    parser = argparse.ArgumentParser(description="Trainproof: A deterministic linter for ML training runs.")
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    # Data Command
+    data_parser = subparsers.add_parser("data", help="Dataset preflight for speech/TTS corpora.")
+    data_parser.add_argument("input", type=str, help="Directory or manifest.jsonl")
+
+    # Tokenizer Command
+    tok_parser = subparsers.add_parser("tokenizer", help="Tokenizer preflight.")
+    tok_parser.add_argument("model", type=str, help="Path to tokenizer model (e.g., SentencePiece .model)")
+    tok_parser.add_argument("transcripts", type=str, help="Path to transcripts text file or JSONL")
+
+    # Epoch Command
+    epoch_parser = subparsers.add_parser("epoch", help="First-epoch verdict from training logs.")
+    epoch_parser.add_argument("logfile", type=str, help="Path to JSONL or CSV log file")
+    epoch_parser.add_argument("--format", choices=["auto", "hf", "coqui", "jsonl", "csv"], default="auto", help="Log format override")
+
+    args = parser.parse_args()
+    
+    report_dict = {}
+
+    if args.command == "data":
+        report_dict = check_data(args.input)
+    elif args.command == "tokenizer":
+        report_dict = check_tokenizer(args.model, args.transcripts)
+    elif args.command == "epoch":
+        try:
+            report_dict = check_epoch(args.logfile, fmt=args.format)
+        except Exception as e:
+            print(f"Error checking epoch: {e}")
+            sys.exit(1)
+
+    print_verdict_console(report_dict.get("verdict", "FAIL"), report_dict.get("findings", []))
+    
+    # Write self-contained HTML report
+    html_out = Path("trainproof_report.html")
+    write_html_report(report_dict, html_out)
+    print(f"\nSaved detailed HTML report to {html_out.absolute()}")
+    
+    if report_dict.get("verdict") == "FAIL":
+        sys.exit(1)
+    else:
+        sys.exit(0)
+
+if __name__ == "__main__":
+    main()

trainproof/epoch.py

@@ -0,0 +1,101 @@
+import math
+from pathlib import Path
+from typing import Any
+from . import rules
+from .adapters import parse_log_with_format
+
+def check_epoch(log_path: str | Path, fmt: str = "auto") -> dict[str, Any]:
+    records = parse_log_with_format(log_path, fmt)
+    findings = []
+    verdict = "PASS"
+    
+    if not records:
+        return {"verdict": "FAIL", "findings": [{"level": "FAIL", "message": "No valid log records found.", "evidence": str(log_path)}]}
+
+    # Find relevant keys
+    def get_val(row, aliases):
+        for a in aliases:
+            for k in row:
+                if a in k: return row[k]
+        return None
+
+    losses = []
+    loss_steps = []  # kept aligned with losses; records may lack a loss field
+    lrs = []
+    grad_norms = []
+    times = []
+    time_steps = []
+
+    for i, r in enumerate(records):
+        step = get_val(r, ["step", "iter"])
+        step = i if step is None else step
+        loss = get_val(r, ["loss", "train_loss"])
+        lr = get_val(r, ["lr", "learning_rate"])
+        gn = get_val(r, ["grad_norm", "gnorm", "grad"])
+        t = get_val(r, ["elapsed", "time", "timestamp"])
+
+        if loss is not None:
+            losses.append(loss)
+            loss_steps.append(step)
+        if lr is not None: lrs.append(lr)
+        if gn is not None: grad_norms.append(gn)
+        if t is not None:
+            times.append(t)
+            time_steps.append(step)
+
+    if not losses:
+        return {"verdict": "FAIL", "findings": [{"level": "FAIL", "message": "Could not find loss metric in logs.", "evidence": ""}]}
+        
+    # Check NaN / Inf in loss
+    nan_steps = [s for s, l in zip(loss_steps, losses) if math.isnan(l) or math.isinf(l)]
+    if nan_steps:
+        findings.append({"level": "FAIL", "message": "NaN or Inf detected in loss.", "evidence": f"Steps: {nan_steps[:5]}..."})
+        verdict = "FAIL"
+        
+    # Check flat loss
+    valid_losses = [l for l in losses if not math.isnan(l) and not math.isinf(l)]
+    if valid_losses:
+        mean_loss = sum(valid_losses) / len(valid_losses)
+        std_loss = math.sqrt(sum((l - mean_loss)**2 for l in valid_losses) / len(valid_losses))
+        if mean_loss > 0 and (std_loss / mean_loss) < rules.MIN_LOSS_VARIATION:
+            findings.append({"level": "FAIL", "message": "Loss curve is completely flat (dead run).", "evidence": f"Variation {std_loss/mean_loss:.5f} < {rules.MIN_LOSS_VARIATION}"})
+            verdict = "FAIL"
+            
+        # Check divergence
+        min_loss = min(valid_losses)
+        if min_loss > 0 and valid_losses[-1] > min_loss * rules.MAX_LOSS_DIVERGENCE_RATIO:
+            findings.append({"level": "FAIL", "message": "Loss curve is diverging.", "evidence": f"End loss {valid_losses[-1]:.3f} vs Min loss {min_loss:.3f}"})
+            verdict = "FAIL"
+
+    # Check Grad Norm
+    valid_gns = [g for g in grad_norms if not math.isnan(g) and not math.isinf(g)]
+    if valid_gns and len(valid_gns) > 5:
+        sorted_gns = sorted(valid_gns)
+        median_gn = sorted_gns[len(sorted_gns)//2]
+        if median_gn > 0:
+            spikes = [g for g in valid_gns if g > median_gn * rules.MAX_GRAD_NORM_SPIKE_RATIO]
+            if spikes:
+                findings.append({"level": "WARN", "message": "Gradient norm spikes detected.", "evidence": f"Max gn {max(spikes):.2f} > {rules.MAX_GRAD_NORM_SPIKE_RATIO}x median ({median_gn:.2f})"})
+                if verdict == "PASS": verdict = "WARN"
+
+    # Check LR
+    if lrs:
+        zeros = sum(1 for lr in lrs if lr <= 0)
+        zero_frac = zeros / len(lrs)
+        if zero_frac > rules.MAX_ZERO_LR_FRACTION:
+            findings.append({"level": "WARN", "message": "Learning rate is zero for a large fraction of the run.", "evidence": f"{zero_frac*100:.1f}% of steps have lr=0"})
+            if verdict == "PASS": verdict = "WARN"
+
+    # Throughput — only when the log carries a time column; no guessing
+    if len(times) >= 2 and times[-1] > times[0]:
+        span = times[-1] - times[0]
+        steps_covered = time_steps[-1] - time_steps[0]
+        if steps_covered > 0:
+            rate = steps_covered / span
+            findings.append({"level": "INFO", "message": "Throughput measured from log timestamps.",
+                             "evidence": f"{rate:.2f} steps/sec over {span:.0f}s observed."})
+
+    if verdict == "PASS":
+        findings.append({"level": "PASS", "message": "Loss curve shows healthy shape, grad norms are stable.", "evidence": f"{len(valid_losses)} steps analyzed."})
+
+    return {"verdict": verdict, "findings": findings}

trainproof/report.py

@@ -0,0 +1,56 @@
+import html as H
+from pathlib import Path
+from typing import Any
+
+def print_verdict_console(verdict: str, findings: list[dict[str, Any]]):
+    """Prints an ASCII-safe verdict summary to the console."""
+    print("=" * 40)
+    print("TRAINPROOF VERDICT")
+    print("=" * 40)
+    
+    if verdict == "PASS":
+        print("[PASS] All checks passed.")
+    elif verdict == "WARN":
+        print("[WARN] Some checks triggered warnings:")
+    else:
+        print("[FAIL] Critical checks failed:")
+        
+    for finding in findings:
+        level = finding.get("level", "INFO")
+        msg = finding.get("message", "")
+        evidence = finding.get("evidence", "")
+        print(f"  [{level}] {msg}")
+        if evidence:
+            print(f"         Evidence: {evidence}")
+            
+    print("=" * 40)
+
+def write_html_report(report: dict[str, Any], path: str | Path) -> None:
+    """Writes a self-contained HTML report."""
+    verdict = report.get("verdict", "UNKNOWN")
+    findings = report.get("findings", [])
+    
+    doc = f"""<!DOCTYPE html><html lang="en"><head><meta charset="utf-8">
+<title>Trainproof report</title>
+<style>
+ body{{font-family:Segoe UI,Roboto,Arial,sans-serif;margin:0;background:#0d1117;color:#e6edf3;line-height:1.5}}
+ .wrap{{max-width:960px;margin:0 auto;padding:28px 18px 60px}}
+ h1{{font-size:26px;margin:0 0 4px}} h2{{font-size:19px;margin:32px 0 10px}}
+ .stat{{background:#161b22;border:1px solid #30363d;border-radius:10px;padding:12px 18px;min-width:110px; display:inline-block;}}
+ .stat b{{display:block;font-size:22px}} .stat.pass b{{color:#3fb950}} .stat.fail b{{color:#f85149}} .stat.warn b{{color:#d29922}}
+ .card{{background:#161b22;border:1px solid #30363d;border-radius:10px;padding:12px 14px;margin:10px 0}}
+ .level-FAIL{{color:#f85149;font-weight:bold;}} .level-WARN{{color:#d29922;font-weight:bold;}} .level-PASS{{color:#3fb950;font-weight:bold;}}
+ .evidence{{color:#8b949e;font-size:13px;margin-top:4px}}
+</style></head><body><div class="wrap">
+<h1>Trainproof Report</h1>
+<div class="stat {verdict.lower()}"><b>{verdict}</b><span>Overall Verdict</span></div>
+<h2>Findings</h2>
+"""
+    for finding in findings:
+        level = finding.get("level", "INFO")
+        msg = H.escape(str(finding.get("message", "")))
+        evidence = H.escape(str(finding.get("evidence", "")))
+        doc += f'<div class="card"><div class="level-{level}">[{level}] {msg}</div><div class="evidence">Evidence: {evidence}</div></div>\n'
+        
+    doc += "</div></body></html>"
+    Path(path).write_text(doc, encoding="utf-8")

trainproof/rules.py

@@ -0,0 +1,53 @@
+"""Deterministic thresholds for verdicts."""
+
+# -----------------
+# DATA SUBCOMMAND
+# -----------------
+
+# Maximum allowed duration for a single audio file in seconds.
+MAX_AUDIO_DURATION_SEC = 25.0
+
+# Minimum allowed duration for a single audio file in seconds.
+MIN_AUDIO_DURATION_SEC = 0.5
+
+# Peak amplitude threshold above which we consider audio to be clipping.
+MAX_CLIPPING_PEAK = 0.99
+
+# Maximum allowed continuous silence in seconds before warning.
+MAX_SILENCE_SEC = 2.0
+
+# Amplitude threshold for considering a frame as "silent" (for silence detection).
+SILENCE_AMPLITUDE = 0.005
+
+# Flag text/audio alignment outliers whose chars-per-second rate deviates from
+# the corpus median by more than this ratio (in either direction).
+CHARS_PER_SEC_OUTLIER_RATIO = 3.0
+
+# -----------------
+# TOKENIZER SUBCOMMAND
+# -----------------
+
+# Minimum fraction of the corpus vocabulary that must be covered by the tokenizer.
+MIN_VOCAB_COVERAGE = 0.999
+
+# Maximum acceptable Out-Of-Vocabulary (OOV) rate.
+MAX_OOV_RATE = 0.001
+
+# Maximum expected tokens per second of audio (blowout warning).
+MAX_TOKENS_PER_SEC = 50.0
+
+# -----------------
+# EPOCH SUBCOMMAND
+# -----------------
+
+# If loss increases by more than this ratio from the minimum loss, flag as diverging.
+MAX_LOSS_DIVERGENCE_RATIO = 1.5
+
+# Minimum required variation (std/mean) in the loss curve to not be considered "flat/dead".
+MIN_LOSS_VARIATION = 0.001
+
+# Maximum allowable spike in gradient norm compared to the median grad norm.
+MAX_GRAD_NORM_SPIKE_RATIO = 10.0
+
+# If the learning rate is strictly zero for more than this fraction of the epoch, it's a warning.
+MAX_ZERO_LR_FRACTION = 0.1

trainproof/speech/__init__.py

@@ -0,0 +1,9 @@
+"""Speech/TTS domain pack — dataset and tokenizer preflight.
+
+The trainproof core (epoch log linter, rules, reports) is model-agnostic;
+domain-specific checks live in packs like this one so future packs
+(e.g. LLM fine-tuning) plug in beside it.
+"""
+
+from .data import check_data
+from .tokenizer import check_tokenizer

trainproof/speech/data.py

@@ -0,0 +1,190 @@
+import os
+import json
+import hashlib
+from pathlib import Path
+from typing import Any
+import numpy as np
+import soundfile as sf
+from ttsproof.normalize import normalize_text, plain_token
+from .. import rules
+
+def get_audio_hash(filepath: str | Path) -> str:
+    hasher = hashlib.md5()
+    with open(filepath, 'rb') as f:
+        hasher.update(f.read())
+    return hasher.hexdigest()
+
+def check_data(input_path: str | Path) -> dict[str, Any]:
+    path = Path(input_path)
+    findings = []
+    verdict = "PASS"
+    
+    records = []
+    if path.is_file() and path.suffix == ".jsonl":
+        for line in path.read_text(encoding="utf-8").splitlines():
+            if not line.strip(): continue
+            records.append(json.loads(line))
+    elif path.is_dir():
+        for wav_path in path.rglob("*.wav"):
+            txt_path = wav_path.with_suffix(".txt")
+            txt = txt_path.read_text(encoding="utf-8").strip() if txt_path.exists() else ""
+            records.append({"audio_filepath": str(wav_path), "text": txt})
+    else:
+        return {"verdict": "FAIL", "findings": [{"level": "FAIL", "message": "Input must be a directory or manifest.jsonl.", "evidence": str(path)}]}
+
+    if not records:
+        return {"verdict": "FAIL", "findings": [{"level": "FAIL", "message": "No data found.", "evidence": ""}]}
+
+    # Audio corpus stats
+    sample_rates = set()
+    channels = set()
+    bit_depths = set()
+    audio_hashes = {}
+    duplicates = 0
+    durations = []
+    
+    clipping_count = 0
+    silent_count = 0
+    missing_audio = 0
+    unreadable_audio = 0
+    chars_per_sec = []  # (file, rate) pairs for text/audio alignment check
+    
+    # Text stats
+    empty_transcripts = 0
+    unnormalized_count = 0
+    mixed_scripts_count = 0
+    text_lengths = []
+    
+    for r in records:
+        audio_file = Path(r.get("audio_filepath", r.get("audio", "")))
+        text = r.get("text", r.get("transcript", ""))
+        
+        # Transcript checks
+        if not text:
+            empty_transcripts += 1
+        else:
+            text_lengths.append(len(text))
+            normalized = normalize_text(text)
+            # if normalization significantly changes the text, we flag it as unnormalized
+            if normalized != text and plain_token(normalized) != plain_token(text):
+                unnormalized_count += 1
+            
+            # naive charset check for mixed scripts (e.g., Cyrillic + Latin)
+            has_latin = any('a' <= c.lower() <= 'z' for c in text)
+            has_cyrillic = any('\u0400' <= c <= '\u04FF' for c in text)
+            has_greek = any('\u0370' <= c <= '\u03FF' for c in text)
+            if sum([has_latin, has_cyrillic, has_greek]) > 1:
+                mixed_scripts_count += 1
+        
+        # Audio checks (records without an audio field are transcript-only — allowed)
+        if str(audio_file) in ("", "."):
+            pass
+        elif not audio_file.exists():
+            missing_audio += 1
+        elif audio_file.exists():
+            ahash = get_audio_hash(audio_file)
+            if ahash in audio_hashes:
+                duplicates += 1
+            else:
+                audio_hashes[ahash] = str(audio_file)
+                
+            try:
+                info = sf.info(str(audio_file))
+                sample_rates.add(info.samplerate)
+                channels.add(info.channels)
+                bit_depths.add(info.subtype)
+                duration = info.frames / info.samplerate
+                durations.append(duration)
+                if text and duration > 0:
+                    chars_per_sec.append((str(audio_file), len(text) / duration))
+                
+                audio_data, sr = sf.read(str(audio_file), dtype="float32", always_2d=True)
+                mono = np.mean(audio_data, axis=1)
+                peak = float(np.max(np.abs(mono))) if len(mono) else 0.0
+                if peak >= rules.MAX_CLIPPING_PEAK:
+                    clipping_count += 1
+                    
+                silent = np.abs(mono) <= rules.SILENCE_AMPLITUDE
+                max_run = 0
+                current = 0
+                for is_silent in silent:
+                    if is_silent: current += 1
+                    else:
+                        max_run = max(max_run, current)
+                        current = 0
+                max_run = max(max_run, current)
+                max_silence_sec = max_run / float(sr)
+                if max_silence_sec > rules.MAX_SILENCE_SEC:
+                    silent_count += 1
+            except Exception:
+                unreadable_audio += 1
+
+    if len(sample_rates) > 1:
+        findings.append({"level": "FAIL", "message": "Inconsistent sample rates found.", "evidence": f"Rates: {sample_rates}"})
+        verdict = "FAIL"
+        
+    if len(channels) > 1:
+        findings.append({"level": "FAIL", "message": "Inconsistent channel counts found.", "evidence": f"Channels: {channels}"})
+        verdict = "FAIL"
+
+    if missing_audio > 0:
+        findings.append({"level": "FAIL", "message": "Manifest references audio files that do not exist.", "evidence": f"{missing_audio} of {len(records)} records."})
+        verdict = "FAIL"
+
+    if unreadable_audio > 0:
+        findings.append({"level": "WARN", "message": "Audio files could not be read/decoded.", "evidence": f"{unreadable_audio} files."})
+        if verdict == "PASS": verdict = "WARN"
+
+    if empty_transcripts > 0:
+        findings.append({"level": "WARN", "message": "Empty transcripts.", "evidence": f"{empty_transcripts} of {len(records)} records."})
+        if verdict == "PASS": verdict = "WARN"
+
+    if durations:
+        too_long = sum(1 for d in durations if d > rules.MAX_AUDIO_DURATION_SEC)
+        too_short = sum(1 for d in durations if d < rules.MIN_AUDIO_DURATION_SEC)
+        sorted_d = sorted(durations)
+        dist = f"min {sorted_d[0]:.2f}s / median {sorted_d[len(sorted_d)//2]:.2f}s / max {sorted_d[-1]:.2f}s"
+        if too_long > 0:
+            findings.append({"level": "WARN", "message": "Audio duration outliers above MAX_AUDIO_DURATION_SEC.", "evidence": f"{too_long} files > {rules.MAX_AUDIO_DURATION_SEC}s ({dist})"})
+            if verdict == "PASS": verdict = "WARN"
+        if too_short > 0:
+            findings.append({"level": "WARN", "message": "Audio duration outliers below MIN_AUDIO_DURATION_SEC.", "evidence": f"{too_short} files < {rules.MIN_AUDIO_DURATION_SEC}s ({dist})"})
+            if verdict == "PASS": verdict = "WARN"
+
+    if len(chars_per_sec) >= 10:
+        rates = sorted(r for _, r in chars_per_sec)
+        median_rate = rates[len(rates) // 2]
+        if median_rate > 0:
+            outliers = [(f, r) for f, r in chars_per_sec
+                        if r > median_rate * rules.CHARS_PER_SEC_OUTLIER_RATIO
+                        or r < median_rate / rules.CHARS_PER_SEC_OUTLIER_RATIO]
+            if outliers:
+                worst = max(outliers, key=lambda x: abs(x[1] - median_rate))
+                findings.append({"level": "WARN", "message": "Text-length vs audio-duration outliers (possible transcript/audio mismatch).",
+                                 "evidence": f"{len(outliers)} records deviate >{rules.CHARS_PER_SEC_OUTLIER_RATIO}x from median {median_rate:.1f} chars/sec; worst: {worst[0]} ({worst[1]:.1f} chars/sec)"})
+                if verdict == "PASS": verdict = "WARN"
+
+    if duplicates > 0:
+        findings.append({"level": "WARN", "message": "Duplicate audio content detected.", "evidence": f"{duplicates} files have identical hashes."})
+        if verdict == "PASS": verdict = "WARN"
+
+    if clipping_count > 0:
+        findings.append({"level": "WARN", "message": "Audio clipping detected.", "evidence": f"{clipping_count} files exceed MAX_CLIPPING_PEAK ({rules.MAX_CLIPPING_PEAK})."})
+        if verdict == "PASS": verdict = "WARN"
+        
+    if silent_count > 0:
+        findings.append({"level": "WARN", "message": "Excessive silence detected.", "evidence": f"{silent_count} files exceed MAX_SILENCE_SEC ({rules.MAX_SILENCE_SEC}s)."})
+        if verdict == "PASS": verdict = "WARN"
+        
+    if unnormalized_count > 0:
+        findings.append({"level": "WARN", "message": "Unnormalized text that hurts TTS (digits, dates).", "evidence": f"{unnormalized_count} transcripts need normalization."})
+        if verdict == "PASS": verdict = "WARN"
+
+    if mixed_scripts_count > 0:
+        findings.append({"level": "WARN", "message": "Mixed character scripts in transcripts.", "evidence": f"{mixed_scripts_count} transcripts."})
+        if verdict == "PASS": verdict = "WARN"
+
+    if verdict == "PASS":
+        findings.append({"level": "PASS", "message": "Dataset preflight completed successfully.", "evidence": f"Processed {len(records)} records."})
+
+    return {"verdict": verdict, "findings": findings}

trainproof/speech/tokenizer.py

@@ -0,0 +1,93 @@
+import json
+import re
+from pathlib import Path
+from typing import Any
+from .. import rules
+
+def load_tokenizer(model_path: str):
+    """Returns (tokenizer, error_finding). Never silently degrades: a linter
+    that swaps in a fake tokenizer would produce fake verdicts."""
+    try:
+        import sentencepiece as spm
+    except ImportError:
+        return None, {"level": "FAIL",
+                      "message": "sentencepiece is not installed - cannot lint this tokenizer.",
+                      "evidence": "pip install sentencepiece"}
+    try:
+        sp = spm.SentencePieceProcessor()
+        sp.load(model_path)
+        return sp, None
+    except Exception as e:
+        return None, {"level": "FAIL",
+                      "message": "Failed to load SentencePiece model.",
+                      "evidence": f"{model_path}: {e}"}
+
+def check_tokenizer(model_path: str | Path, transcripts_path: str | Path) -> dict[str, Any]:
+    tokenizer, load_error = load_tokenizer(str(model_path))
+    if load_error is not None:
+        return {"verdict": "FAIL", "findings": [load_error]}
+    findings = []
+    verdict = "PASS"
+    
+    path = Path(transcripts_path)
+    if not path.exists():
+        return {"verdict": "FAIL", "findings": [{"level": "FAIL", "message": "Transcripts file not found.", "evidence": str(transcripts_path)}]}
+        
+    lines = path.read_text(encoding="utf-8").splitlines()
+    total_tokens = 0
+    total_chars = 0
+    total_unks = 0
+    total_duration = 0.0
+    
+    suspicious_splits = 0
+    
+    for line in lines:
+        if not line.strip(): continue
+        text = line
+        duration = 0.0
+        if line.startswith("{"):
+            try:
+                data = json.loads(line)
+                text = data.get("text", "") or data.get("transcript", "")
+                duration = data.get("duration", 0.0)
+            except Exception:
+                pass
+                
+        pieces = tokenizer.encode_as_pieces(text)
+        total_tokens += len(pieces)
+        total_chars += len(text)
+        total_duration += duration
+        total_unks += sum(1 for p in pieces if p == "<unk>")
+        
+        # heuristic for suspicious split: if a token has multiple digits separated but not grouped?
+        # A simple check: if the sequence length blows up compared to char length for numbers
+        numbers = re.findall(r'\b\d+\b', text)
+        for num in numbers:
+            num_pieces = tokenizer.encode_as_pieces(num)
+            if len(num_pieces) > len(num) / 2 + 1:
+                suspicious_splits += 1
+
+    oov_rate = total_unks / max(1, total_tokens)
+    if oov_rate > rules.MAX_OOV_RATE:
+        findings.append({"level": "FAIL", "message": "High OOV (Out-Of-Vocabulary) rate.", "evidence": f"{oov_rate*100:.3f}% > {rules.MAX_OOV_RATE*100:.3f}%"})
+        verdict = "FAIL"
+        
+    coverage = 1.0 - oov_rate
+    if coverage < rules.MIN_VOCAB_COVERAGE:
+        findings.append({"level": "WARN", "message": "Vocabulary coverage is below recommended threshold.", "evidence": f"{coverage*100:.3f}% < {rules.MIN_VOCAB_COVERAGE*100:.3f}%"})
+        if verdict == "PASS": verdict = "WARN"
+
+    if total_duration > 0:
+        tps = total_tokens / total_duration
+        if tps > rules.MAX_TOKENS_PER_SEC:
+            findings.append({"level": "WARN", "message": "High tokens per second of audio (possible sequence length blowout).", "evidence": f"{tps:.1f} tokens/sec > {rules.MAX_TOKENS_PER_SEC}"})
+            if verdict == "PASS": verdict = "WARN"
+            
+    if suspicious_splits > len(lines) * 0.01:
+        findings.append({"level": "WARN", "message": "Suspicious splits detected on numbers/dates.", "evidence": f"{suspicious_splits} instances."})
+        if verdict == "PASS": verdict = "WARN"
+
+    if verdict == "PASS":
+        findings.append({"level": "PASS", "message": "Tokenizer vocabulary coverage and splits look healthy.", "evidence": f"{total_tokens} tokens evaluated."})
+
+    return {"verdict": verdict, "findings": findings}

trainproof-0.1.0.dist-info/METADATA

@@ -0,0 +1,62 @@
+Metadata-Version: 2.4
+Name: trainproof
+Version: 0.1.0
+Summary: A deterministic linter for ML training runs: dataset, tokenizer, and epoch logs.
+Author-email: "Panagiotis (Panos) Gkilis" <bedvibe@bedvibe.studio>
+License: MIT
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: numpy
+Requires-Dist: soundfile
+Requires-Dist: ttsproof
+
+# Trainproof
+
+A deterministic linter for ML training runs. Run it on your dataset, tokenizer, and first-epoch logs — it gives a deterministic PASS/WARN/FAIL verdict with named findings and suggested fixes, BEFORE you burn weeks of GPU time.
+
+## Installation
+```bash
+pip install .
+```
+
+## Usage
+
+Exactly three subcommands:
+
+### 1. Dataset Preflight
+```bash
+trainproof data /path/to/dataset
+```
+Checks audio integrity (clipping, silence, duration distribution) and transcript quality (unnormalized text, charset audit, duration correlation).
+
+### 2. Tokenizer Preflight
+```bash
+trainproof tokenizer my_model.model transcripts.txt
+```
+Checks vocabulary coverage, tokens-per-second, and suspicious splits on numbers/dates.
+
+### 3. First-Epoch Verification
+```bash
+trainproof epoch logs/epoch1.jsonl
+```
+Analyzes loss curves (divergence, flatlines, NaN), grad norms, learning rate response, and throughput.
+
+## Supported log formats
+- **Generic JSONL / CSV**
+- **HuggingFace** (`trainer_state.json`)
+- **Coqui TTS Trainer** (plain text `trainer_0_log.txt`)
+
+*Roadmap: TensorBoard event files planned (v0.2) — Lightning console captures are TTY dumps, not logs, and will not be supported.*
+
+## Verdict Rules
+All verdict rules are deterministic thresholds defined centrally in `src/trainproof/rules.py`. Examples include:
+- `MAX_CLIPPING_PEAK = 0.99`
+- `MAX_LOSS_DIVERGENCE_RATIO = 1.5`
+- `MIN_VOCAB_COVERAGE = 0.999`
+
+## Explicit Non-Goals
+- No live learning-rate auto-adjustment.
+- No PyTorch/Lightning callbacks or framework hooks (log files only).
+- No MCP server, no LLM integration.
+- No dashboards/wandb-style UI.
+- No extra features beyond this spec.

trainproof-0.1.0.dist-info/RECORD

@@ -0,0 +1,14 @@
+trainproof/__init__.py,sha256=U9rhJrmsaS-NrLEI6fOq3Bn0uShf5aSZe-wQq2ofXac,88
+trainproof/adapters.py,sha256=GHDZdlTP6608dhpyEbqtUhsOzj1u8-1SOQ8hCMS4-dw,4963
+trainproof/cli.py,sha256=eAZDeOnrhCWu3FqCOERebCrqcgeN37wTmCvelYULvQI,2175
+trainproof/epoch.py,sha256=86ZEw9WuE8EN5rY70GVlzDELAMwuYFm8B99ZuDa3zaw,4542
+trainproof/report.py,sha256=w7StbzYckqWlPKJEilfykdsX7jgS1Ivy4mdvSaZhKn0,2552
+trainproof/rules.py,sha256=WRVKGuJmkwAbUR6NUOGqejl9hdiOI4XZNpT5gZj8LDk,1644
+trainproof/speech/__init__.py,sha256=hGEY9HhXPULa_bMdbHkR1DuMC6U1d5BWNAFou3hcfHY,320
+trainproof/speech/data.py,sha256=GmPdWeiJaskcsnAzpxOhXg2ytqvinNVEhviMwAXjHJU,8778
+trainproof/speech/tokenizer.py,sha256=yinpEU-s7KYjioBd2vxg4WKqjkPt3_RsWM48Clv1Pj4,3995
+trainproof-0.1.0.dist-info/METADATA,sha256=Yfl6DNEUEDDReiGZxzq_S3vPH1wKD-MXEjyuDam7_NA,2077
+trainproof-0.1.0.dist-info/WHEEL,sha256=YCfwYGOYMi5Jhw2fU4yNgwErybb2IX5PEwBKV4ZbdBo,91
+trainproof-0.1.0.dist-info/entry_points.txt,sha256=OaPU4KxhS8FP4JuzV03pd44O8IW0EIqDJzqYeBpLae0,51
+trainproof-0.1.0.dist-info/top_level.txt,sha256=YIUZaRRIq-WHsqi5nY0OKTzp_yTDnn2J7NeGg-Be438,11
+trainproof-0.1.0.dist-info/RECORD,,

trainproof-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

trainproof-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+trainproof = trainproof.cli:main

trainproof-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+trainproof