altasr 1__py3-none-any.whl

altasr/__init__.py

@@ -0,0 +1,92 @@
+"""
+ALTASR -- Adaptive Low-latency Transcription ASR for Kinyarwanda.
+==================================================================
+
+A clear, extensible Conformer-CTC speech-recognition package, organised as
+three subpackages under one roof:
+
+``altasr.model``      the network -- one file per architectural component.
+``altasr.training``   everything to train: dataset/preprocessing, the
+                      epoch-based trainer, feature precomputation, the
+                      ``altasr-train`` CLI.
+``altasr.inference``  everything to use a trained model: checkpoint loading,
+                      the :class:`Transcriber` API, ``altasr-transcribe`` /
+                      ``altasr-evaluate`` CLIs.
+
+Shared foundations live at the package root: ``config`` (one dataclass tree,
+JSON in/out), ``text`` (normalisation + tokenizer), ``audio`` (loading,
+log-mels, SpecAugment) and ``metrics`` (WER/CER).
+
+Quick use of a published checkpoint::
+
+    from altasr import Transcriber
+    asr = Transcriber.from_pretrained("path/to/checkpoint")   # folder or .pt
+    print(asr.transcribe("recording.mp3"))
+
+Quick training (after ``pip install altasr`` or ``pip install -e .``)::
+
+    altasr-train --preset medium --audio-root /data/track_b \
+        --train /data/track_b/train.json --val /data/track_b/dev_test.json
+
+Heavy dependencies (torch) are imported lazily: ``import altasr`` plus the
+config/text/metrics layers work on any machine; the model, training and
+inference layers require PyTorch.
+"""
+from __future__ import annotations
+
+__version__ = "0.5.1"
+
+# Torch-free foundations -- always importable.
+from .config import Config, PRESETS, load_config, preset    # noqa: F401
+from .text import (CharTokenizer, BPETokenizer, normalize_text,        # noqa: F401
+                   strip_punctuation, build_tokenizer, load_tokenizer)  # noqa: F401
+from .metrics import cer, corpus_wer_cer, corpus_metrics, wer  # noqa: F401
+
+# Torch-backed public API, resolved lazily (PEP 562) so `import altasr`
+# works without PyTorch and errors stay clear when torch is missing.
+_LAZY = {
+    "Transcriber": "altasr.inference.transcriber",
+    "load_checkpoint": "altasr.inference.checkpoint",
+    "save_checkpoint": "altasr.inference.checkpoint",
+    "build_model": "altasr.model.ctc",
+    "ConformerCTC": "altasr.model.ctc",
+    "Trainer": "altasr.training.trainer",
+    "ASRDataset": "altasr.training.data",
+    "load_records": "altasr.training.data",
+    "StreamingSession": "altasr.inference.streaming",
+    "TranscriberPool": "altasr.inference.pool",
+    "resolve_devices": "altasr.inference.pool",
+    "AudioDecodeError": "altasr.audio",
+    "Hotwords": "altasr.inference.decoding",
+    "TranscriptionResult": "altasr.inference.decoding",
+    "ctc_prefix_beam_search": "altasr.inference.decoding",
+    "NGramLM": "altasr.lm",
+    "Lexicon": "altasr.lm",
+    "build_decoding_resources": "altasr.lm",
+    "load_decoding_resources": "altasr.lm",
+    "LLMCorrector": "altasr.integrations.llm",
+    "SmartTranscriber": "altasr.integrations.llm",
+    "ASR_TOOL_SCHEMA": "altasr.integrations.llm",
+    "handle_tool_call": "altasr.integrations.llm",
+    "transcribe_stream": "altasr.inference.streaming",
+}
+
+
+def __getattr__(name):
+    target = _LAZY.get(name)
+    if target is None:
+        raise AttributeError(f"module 'altasr' has no attribute {name!r}")
+    import importlib
+    try:
+        module = importlib.import_module(target)
+    except ImportError as exc:
+        raise ImportError(
+            f"altasr.{name} requires PyTorch. Install the backend with "
+            f"`pip install torch torchaudio` ({exc})") from exc
+    value = getattr(module, name)
+    globals()[name] = value
+    return value
+
+
+def __dir__():
+    return sorted(list(globals()) + list(_LAZY))

altasr/audio.py

@@ -0,0 +1,208 @@
+"""
+Audio loading and feature extraction.
+
+Loading
+-------
+``load_audio`` returns a mono float32 waveform at the target sample rate. It
+tries decoders in a robust order with no system-FFmpeg dependency:
+
+1. **soundfile** (libsndfile): WAV / FLAC / OGG -- fast and reliable;
+2. **PyAV** (FFmpeg bundled inside the pip wheel): MP3 / Opus / M4A / WebM and
+   extension-less web recordings;
+3. **torchaudio.load** as a last resort.
+
+Resampling uses ``torchaudio.functional.resample`` (windowed-sinc).
+
+Features
+--------
+``LogMel`` converts a waveform to a log mel spectrogram:
+
+    STFT (n_fft=400 = 25 ms window, hop=160 = 10 ms)  ->  power spectrum
+    -> mel filterbank (n_mels=80)  ->  log(mel + 1e-6)
+    -> per-utterance normalisation: (x - mean) / (std + 1e-5)
+
+Per-utterance mean/variance normalisation makes the network robust to channel
+loudness/recording differences and lets SpecAugment mask with zeros (zero ==
+the mean after normalisation).
+
+``SpecAugment`` (Park et al., 2019) randomly zeroes frequency bands and time
+spans **during training only** -- the single most effective ASR regulariser.
+The implementation is fully vectorised over the batch (no Python loops).
+"""
+from __future__ import annotations
+
+from typing import Optional
+
+import torch
+import torch.nn as nn
+
+from .config import AudioConfig
+
+
+# --------------------------------------------------------------------------- #
+#  Loading
+# --------------------------------------------------------------------------- #
+class AudioDecodeError(RuntimeError):
+    """Raised when every decoder failed for a file. Carries the per-decoder
+    reasons so the dataset layer can log a useful one-line warning and skip
+    the file instead of crashing an hours-long training run."""
+
+
+def load_audio(path: str, target_sr: int = 16_000) -> torch.Tensor:
+    """Decode ``path`` to a mono float32 waveform (T,) at ``target_sr`` Hz.
+
+    Decoders are tried in order (soundfile -> PyAV -> torchaudio); each
+    failure is recorded, and only if ALL fail is :class:`AudioDecodeError`
+    raised. The torchaudio fallback is fully wrapped: on torchaudio >= 2.9
+    it delegates to torchcodec, which hard-crashes with a RuntimeError when
+    system FFmpeg libraries are missing -- that must never take down a
+    training job when soundfile/PyAV simply didn't support the format.
+    """
+    wav, sr = None, None
+    errors = []
+
+    try:                                                   # 1) soundfile
+        import soundfile as sf
+        data, sr = sf.read(path, dtype="float32", always_2d=True)
+        wav = torch.from_numpy(data.mean(axis=1))
+    except Exception as exc:
+        errors.append(f"soundfile: {exc}")
+
+    if wav is None:                                        # 2) PyAV
+        try:
+            import av
+            import numpy as np
+            chunks = []
+            with av.open(path) as container:
+                stream = container.streams.audio[0]
+                resampler = av.audio.resampler.AudioResampler(
+                    format="fltp", layout="mono", rate=target_sr)
+                for frame in container.decode(stream):
+                    for rs in resampler.resample(frame):
+                        chunks.append(rs.to_ndarray().reshape(-1))
+            if not chunks:
+                raise ValueError("no audio frames decoded")
+            wav = torch.from_numpy(np.concatenate(chunks).astype("float32"))
+            sr = target_sr
+        except Exception as exc:
+            errors.append(f"pyav: {exc}")
+
+    if wav is None:                                        # 3) torchaudio
+        try:
+            import torchaudio
+            data, sr = torchaudio.load(path)
+            wav = data.mean(dim=0)
+        except Exception as exc:
+            # torchaudio 2.9+ routes through torchcodec, which raises a huge
+            # RuntimeError when FFmpeg shared libraries are absent; compress
+            # it to one line so warnings stay readable.
+            msg = str(exc).splitlines()[0]
+            errors.append(f"torchaudio: {msg}")
+
+    if wav is None or wav.numel() == 0:
+        raise AudioDecodeError(
+            f"could not decode {path!r} with any backend: "
+            + " | ".join(errors or ["file empty"]))
+
+    if sr != target_sr:
+        import torchaudio.functional as AF
+        wav = AF.resample(wav, orig_freq=int(sr), new_freq=target_sr)
+    return wav.contiguous()
+
+
+# --------------------------------------------------------------------------- #
+#  Log-mel front-end
+# --------------------------------------------------------------------------- #
+class LogMel(nn.Module):
+    """Waveform (B, S) or (S,) -> normalised log-mel features (B, T, n_mels)."""
+
+    def __init__(self, cfg: AudioConfig) -> None:
+        super().__init__()
+        import torchaudio
+        self.cfg = cfg
+        self.mel = torchaudio.transforms.MelSpectrogram(
+            sample_rate=cfg.sample_rate, n_fft=cfg.n_fft,
+            hop_length=cfg.hop_length, n_mels=cfg.n_mels, power=2.0,
+        )
+
+    def forward(self, wav: torch.Tensor) -> torch.Tensor:
+        if wav.dim() == 1:
+            wav = wav.unsqueeze(0)
+        mel = self.mel(wav)                               # (B, n_mels, T)
+        logmel = torch.log(mel + 1e-6).transpose(1, 2)    # (B, T, n_mels)
+        mean = logmel.mean(dim=(1, 2), keepdim=True)
+        std = logmel.std(dim=(1, 2), keepdim=True)
+        return (logmel - mean) / (std + 1e-5)
+
+    def num_frames(self, num_samples: torch.Tensor) -> torch.Tensor:
+        """Mel frame count for a sample count: floor(S / hop) + 1 (center pad)."""
+        return num_samples // self.cfg.hop_length + 1
+
+
+def normalize_logmel(logmel: torch.Tensor) -> torch.Tensor:
+    """Per-utterance (mean, std) normalisation for precomputed features (T, F)."""
+    return (logmel - logmel.mean()) / (logmel.std() + 1e-5)
+
+
+# --------------------------------------------------------------------------- #
+#  SpecAugment
+# --------------------------------------------------------------------------- #
+class SpecAugment(nn.Module):
+    """Vectorised frequency/time masking of (B, T, F) features; train-time only."""
+
+    def __init__(self, cfg: AudioConfig) -> None:
+        super().__init__()
+        self.cfg = cfg
+
+    @staticmethod
+    def _bands(n: int, max_w: torch.Tensor, length: int, b: int,
+               device: torch.device) -> torch.Tensor:
+        """(B, length) bool mask, True inside ``n`` random bands per row;
+        per-row band width ~ U{0..max_w}, start ~ U{0..length-width}."""
+        max_w = max_w.clamp(0, length - 1).view(b, 1).float()
+        widths = (torch.rand(b, n, device=device) * (max_w + 1)).long()
+        starts = (torch.rand(b, n, device=device)
+                  * (length - widths + 1).float()).long()
+        ar = torch.arange(length, device=device).view(1, 1, length)
+        inside = (ar >= starts.unsqueeze(-1)) & (ar < (starts + widths).unsqueeze(-1))
+        return inside.any(dim=1)
+
+    def forward(self, x: torch.Tensor, lengths: Optional[torch.Tensor] = None
+                ) -> torch.Tensor:
+        c = self.cfg
+        if not self.training or not c.spec_augment:
+            return x
+        b, t, f = x.shape
+        dev = x.device
+        lens = (lengths.to(dev).clamp(1, t) if lengths is not None
+                else torch.full((b,), t, device=dev, dtype=torch.long))
+        if c.freq_masks:
+            fw = torch.full((b,), c.freq_mask_width, device=dev, dtype=torch.long)
+            x = x.masked_fill(self._bands(c.freq_masks, fw, f, b, dev).unsqueeze(1), 0.0)
+        if c.time_masks:
+            tw = (c.time_mask_pct * lens.float()).long().clamp(min=1)
+            tm = self._bands(c.time_masks, tw, t, b, dev)
+            valid = torch.arange(t, device=dev).view(1, t) < lens.view(b, 1)
+            x = x.masked_fill((tm & valid).unsqueeze(2), 0.0)
+        return x
+
+
+# --------------------------------------------------------------------------- #
+#  Speed perturbation (Ko et al., 2015)
+# --------------------------------------------------------------------------- #
+def speed_perturb(wav: torch.Tensor, sample_rate: int,
+                  factors=(0.9, 1.0, 1.1)) -> torch.Tensor:
+    """Randomly retime a waveform by resampling: factor 0.9 = slower/longer.
+
+    Cheap "3x more data" that is very effective on small or noisy corpora
+    (different speaking rates, pitch shifts). Training-time only, raw-feature
+    mode only (precomputed mels are fixed at prep time).
+    """
+    import random
+    factor = random.choice(list(factors))
+    if abs(factor - 1.0) < 1e-6:
+        return wav
+    import torchaudio.functional as AF
+    # Resampling to sr/factor then playing at sr changes duration by 1/factor.
+    return AF.resample(wav, orig_freq=int(sample_rate * factor),
+                       new_freq=sample_rate)

altasr/benchmark.py

@@ -0,0 +1,374 @@
+#!/usr/bin/env python3
+"""
+ALTASR :: benchmark CLI (installed as ``altasr-benchmark``).
+
+Evaluate one or more ALTASR checkpoints -- and, optionally, OpenAI Whisper --
+on the same labelled test set, and generate a professional, publishable
+benchmark report (Markdown, and optionally HTML).
+
+    altasr-benchmark \\
+        --checkpoint "ALTASR small"=runs/small_full/best \\
+        --checkpoint "ALTASR medium"=runs/medium_full/best \\
+        --whisper small \\
+        --audio-root /data/track_b --metadata /data/track_b/dev_test.json \\
+        --limit 1000 --out benchmark/report.md --html
+
+What the report contains
+------------------------
+* environment/methodology section (date, dataset, #utterances, hours,
+  hardware, decoding settings, normalisation policy);
+* one row per system: parameters, WER, CER, punctuation-insensitive WER,
+  substitution/deletion/insertion rates, RTF (real-time factor);
+* the hardest utterances per system (highest WER) as REF/HYP pairs;
+* raw per-system JSON next to the report for further analysis/plotting.
+
+Fairness notes baked in
+-----------------------
+* All hypotheses AND references are passed through ALTASR's normaliser before
+  scoring, so casing/punctuation conventions can't bias the comparison;
+* ``wer_nopunct`` is reported for every system, so punctuated ALTASR models
+  are also compared on pure word accuracy;
+* RTF is measured per system on the same machine in the same run.
+
+Whisper support: ``pip install altasr[whisper]`` (faster-whisper). If the
+package or its model download is unavailable, the benchmark simply proceeds
+with the systems it can run and says so in the report.
+"""
+from __future__ import annotations
+
+import argparse
+import datetime as _dt
+import json
+import os
+import platform
+import sys
+import time
+import warnings
+from typing import Callable, Dict, List, Optional, Tuple
+
+warnings.filterwarnings("ignore", message=".*cuda capability.*")
+warnings.filterwarnings("ignore", category=UserWarning, module="torch")
+warnings.filterwarnings("ignore", category=FutureWarning)
+
+
+# --------------------------------------------------------------------------- #
+#  Systems
+# --------------------------------------------------------------------------- #
+class System:
+    """name + transcribe(path)->text + metadata; one benchmark row."""
+
+    def __init__(self, name: str, transcribe: Callable[[str], str],
+                 params_m: Optional[float], notes: str = "") -> None:
+        self.name = name
+        self.transcribe = transcribe
+        self.params_m = params_m
+        self.notes = notes
+
+
+def _altasr_system(name: str, checkpoint: str, device: str,
+                   quantize: str) -> System:
+    from altasr import Transcriber
+    asr = Transcriber.from_pretrained(checkpoint, device=device,
+                                      quantize=quantize)
+    notes = f"greedy CTC, device={asr.device}" + (
+        ", int8" if quantize else "")
+    params = None
+    try:
+        params = asr.model.num_parameters() / 1e6
+    except Exception:
+        pass
+    return System(name, lambda p: asr.transcribe(p), params, notes)
+
+
+def _whisper_system(size: str, device: str) -> System:
+    """faster-whisper preferred (CTranslate2, fair CPU speed); fall back to
+    openai-whisper if that's what is installed."""
+    try:
+        from faster_whisper import WhisperModel
+        dev = "cuda" if str(device).startswith("cuda") else "cpu"
+        model = WhisperModel(size, device=dev,
+                             compute_type="float16" if dev == "cuda" else "int8")
+
+        def transcribe(path: str) -> str:
+            segments, _ = model.transcribe(path, language="rw",
+                                           beam_size=1, vad_filter=False)
+            return " ".join(s.text for s in segments)
+
+        return System(f"Whisper {size} (faster-whisper)", transcribe,
+                      _WHISPER_PARAMS.get(size), f"beam=1, lang=rw, {dev}")
+    except ImportError:
+        pass
+    import whisper  # raises ImportError with a clear message if absent
+    model = whisper.load_model(size, device="cuda" if str(device).startswith("cuda") else "cpu")
+
+    def transcribe(path: str) -> str:
+        return model.transcribe(path, language="rw", beam_size=1)["text"]
+
+    return System(f"Whisper {size}", transcribe,
+                  _WHISPER_PARAMS.get(size), "beam=1, lang=rw")
+
+
+_WHISPER_PARAMS = {"tiny": 39, "base": 74, "small": 244, "medium": 769,
+                   "large": 1550, "large-v2": 1550, "large-v3": 1550}
+
+
+# --------------------------------------------------------------------------- #
+#  Report generation
+# --------------------------------------------------------------------------- #
+def _md_table(rows: List[List[str]], header: List[str]) -> str:
+    out = ["| " + " | ".join(header) + " |",
+           "|" + "|".join("---" for _ in header) + "|"]
+    out += ["| " + " | ".join(r) + " |" for r in rows]
+    return "\n".join(out)
+
+
+def write_report(path: str, dataset: Dict, results: List[Dict],
+                 examples: Dict[str, List[Tuple[str, str, float]]],
+                 skipped: List[str], html: bool) -> None:
+    now = _dt.datetime.now().strftime("%Y-%m-%d %H:%M")
+    hw = f"{platform.processor() or platform.machine()}, {platform.system()}"
+    try:
+        import torch
+        if torch.cuda.is_available():
+            hw += f", GPU: {torch.cuda.get_device_name(0)}"
+    except Exception:
+        pass
+
+    lines = [
+        f"# ASR Benchmark Report",
+        "",
+        f"*Generated by `altasr-benchmark` on {now}.*",
+        "",
+        "## Methodology",
+        "",
+        f"- **Test set:** {dataset['metadata']} "
+        f"({dataset['utts']} utterances, {dataset['hours']:.2f} h of audio)",
+        f"- **Hardware:** {hw}",
+        f"- **Scoring:** references and hypotheses are both normalised with "
+        f"ALTASR's text normaliser (lowercase, apostrophes folded, "
+        f"punctuation `{dataset['punctuation']}` kept and re-spaced) before "
+        f"computing corpus-level WER/CER. `WER (no punct)` strips punctuation "
+        f"from both sides, making systems that do and don't produce "
+        f"punctuation directly comparable.",
+        f"- **RTF** = processing seconds / audio seconds on this machine "
+        f"(lower is better; < 1.0 is faster than real time). All systems ran "
+        f"in the same process, one utterance at a time (batch size 1).",
+        "",
+        "## Results",
+        "",
+    ]
+    header = ["System", "Params (M)", "WER %", "CER %", "WER no-punct %",
+              "Sub %", "Del %", "Ins %", "RTF"]
+    rows = []
+    for r in results:
+        m = r["metrics"]
+        rows.append([
+            f"**{r['name']}**",
+            f"{r['params_m']:.0f}" if r["params_m"] else "–",
+            f"{m['wer'] * 100:.2f}", f"{m['cer'] * 100:.2f}",
+            f"{m['wer_nopunct'] * 100:.2f}",
+            f"{m['sub_rate'] * 100:.2f}", f"{m['del_rate'] * 100:.2f}",
+            f"{m['ins_rate'] * 100:.2f}", f"{m['rtf']:.3f}"])
+    lines.append(_md_table(rows, header))
+    lines += ["", "### System notes", ""]
+    lines += [f"- **{r['name']}** — {r['notes']}" for r in results]
+    if skipped:
+        lines += ["", "### Skipped systems", ""]
+        lines += [f"- {s}" for s in skipped]
+
+    lines += ["", "## Hardest utterances per system", "",
+              "The five highest-WER validation utterances for each system — "
+              "useful for spotting systematic failure modes (noise, "
+              "code-switching, long-form audio, numerals).", ""]
+    for name, exs in examples.items():
+        lines += [f"### {name}", ""]
+        for ref, hyp, w in exs:
+            lines += [f"- WER {w * 100:.0f}%",
+                      f"  - REF: `{ref[:160]}`",
+                      f"  - HYP: `{hyp[:160] or '(empty)'}`"]
+        lines.append("")
+
+    md = "\n".join(lines) + "\n"
+    os.makedirs(os.path.dirname(os.path.abspath(path)), exist_ok=True)
+    with open(path, "w", encoding="utf-8") as fh:
+        fh.write(md)
+    print(f"[report] {path}")
+
+    if html:
+        html_path = os.path.splitext(path)[0] + ".html"
+        with open(html_path, "w", encoding="utf-8") as fh:
+            fh.write(_md_to_html(md))
+        print(f"[report] {html_path}")
+
+
+def _md_to_html(md: str) -> str:
+    """A small dependency-free Markdown->HTML converter (headers, tables,
+    lists, bold, inline code) good enough for the benchmark report."""
+    import html as _h
+    import re
+    out, in_table, in_list = [], False, False
+
+    def inline(s: str) -> str:
+        s = _h.escape(s)
+        s = re.sub(r"\*\*(.+?)\*\*", r"<b>\1</b>", s)
+        s = re.sub(r"\*(.+?)\*", r"<i>\1</i>", s)
+        s = re.sub(r"`(.+?)`", r"<code>\1</code>", s)
+        return s
+
+    for line in md.splitlines():
+        if line.startswith("|"):
+            cells = [c.strip() for c in line.strip("|").split("|")]
+            if all(set(c) <= {"-", " "} for c in cells):
+                continue
+            if not in_table:
+                out.append("<table>"); in_table = True
+                out.append("<tr>" + "".join(f"<th>{inline(c)}</th>"
+                                            for c in cells) + "</tr>")
+            else:
+                out.append("<tr>" + "".join(f"<td>{inline(c)}</td>"
+                                            for c in cells) + "</tr>")
+            continue
+        if in_table:
+            out.append("</table>"); in_table = False
+        if line.startswith("- ") or line.startswith("  - "):
+            if not in_list:
+                out.append("<ul>"); in_list = True
+            out.append(f"<li>{inline(line.lstrip(' -'))}</li>")
+            continue
+        if in_list:
+            out.append("</ul>"); in_list = False
+        m = re.match(r"^(#{1,3}) (.*)$", line)
+        if m:
+            n = len(m.group(1))
+            out.append(f"<h{n}>{inline(m.group(2))}</h{n}>")
+        elif line.strip():
+            out.append(f"<p>{inline(line)}</p>")
+    if in_table:
+        out.append("</table>")
+    if in_list:
+        out.append("</ul>")
+    style = ("<style>body{font-family:system-ui,sans-serif;max-width:960px;"
+             "margin:2rem auto;padding:0 1rem;line-height:1.5}"
+             "table{border-collapse:collapse;width:100%}"
+             "th,td{border:1px solid #ccc;padding:6px 10px;text-align:left}"
+             "th{background:#f3f3f3}code{background:#f5f5f5;padding:1px 4px;"
+             "border-radius:3px}</style>")
+    return ("<!doctype html><html><head><meta charset='utf-8'>"
+            f"<title>ASR Benchmark Report</title>{style}</head><body>"
+            + "\n".join(out) + "</body></html>")
+
+
+# --------------------------------------------------------------------------- #
+#  Main
+# --------------------------------------------------------------------------- #
+def main(argv=None) -> int:
+    from altasr.utils import clear_terminal
+    clear_terminal()
+    p = argparse.ArgumentParser(
+        description="Benchmark ALTASR checkpoints (and optionally Whisper) "
+                    "and generate a report.",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    p.add_argument("--checkpoint", action="append", required=True,
+                   metavar="[NAME=]PATH",
+                   help="ALTASR checkpoint folder; repeatable. Optional "
+                        "'Name=' prefix for the report row.")
+    p.add_argument("--whisper", action="append", default=[],
+                   metavar="SIZE", help="Also benchmark Whisper (tiny/base/"
+                   "small/medium/large-v3); needs altasr[whisper]. Repeatable.")
+    p.add_argument("--audio-root", required=True)
+    p.add_argument("--metadata", nargs="+", required=True)
+    p.add_argument("--limit", type=int, default=0,
+                   help="Evaluate only the first N utterances (0 = all).")
+    p.add_argument("--out", default="benchmark/report.md")
+    p.add_argument("--html", action="store_true",
+                   help="Also write an HTML version of the report.")
+    p.add_argument("--device", default="auto",
+                   help="auto | cpu | cuda | cuda:N")
+    p.add_argument("--quantize", choices=["", "int8"], default="",
+                   help="Apply int8 quantization to the ALTASR systems "
+                        "(CPU edge-deployment benchmark).")
+    args = p.parse_args(argv)
+
+    from altasr.config import Config
+    from altasr.metrics import corpus_metrics, wer as _wer
+    from altasr.text import DEFAULT_PUNCTUATION, normalize_text
+    from altasr.training.data import load_records
+
+    cfg = Config()
+    cfg.data.audio_root = args.audio_root
+    records = load_records(cfg, args.metadata, check_files=True)
+    if args.limit:
+        records = records[: args.limit]
+    if not records:
+        print("ERROR: no records resolved (check --audio-root/--metadata).",
+              file=sys.stderr)
+        return 2
+    hours = sum(r["duration"] or 0.0 for r in records) / 3600.0
+    punct = cfg.text.punctuation
+    print(f"[bench] {len(records)} utterances (~{hours:.2f} h)", flush=True)
+
+    # -- build systems -------------------------------------------------------- #
+    systems: List[System] = []
+    skipped: List[str] = []
+    for spec in args.checkpoint:
+        name, _, path = spec.rpartition("=")
+        name = name or f"ALTASR ({os.path.basename(os.path.dirname(path.rstrip('/'))) or path})"
+        try:
+            systems.append(_altasr_system(name, path, args.device, args.quantize))
+        except Exception as exc:
+            skipped.append(f"{name}: failed to load ({exc})")
+    for size in args.whisper:
+        try:
+            systems.append(_whisper_system(size, args.device))
+        except Exception as exc:
+            skipped.append(f"Whisper {size}: unavailable ({exc}) — "
+                           f"`pip install altasr[whisper]`")
+    if not systems:
+        print("ERROR: no system could be loaded.", file=sys.stderr)
+        for s in skipped:
+            print("  -", s, file=sys.stderr)
+        return 2
+
+    # -- run -------------------------------------------------------------------- #
+    results, examples = [], {}
+    for system in systems:
+        print(f"[bench] running {system.name} ...", flush=True)
+        pairs: List[Tuple[str, str]] = []
+        t0 = time.time()
+        for i, rec in enumerate(records, 1):
+            try:
+                hyp = system.transcribe(rec["resolved"])
+            except Exception as exc:
+                hyp = ""
+                print(f"\n  [warn] {rec['id']}: {exc}", flush=True)
+            hyp = normalize_text(hyp, keep_punctuation=True, punctuation=punct)
+            pairs.append((rec["text"], hyp))
+            if i % 20 == 0 or i == len(records):
+                print(f"\r  {i}/{len(records)}", end="", flush=True)
+        elapsed = time.time() - t0
+        print()
+        m = corpus_metrics(pairs, punctuation=punct,
+                           audio_seconds=hours * 3600.0,
+                           processing_seconds=elapsed)
+        results.append({"name": system.name, "params_m": system.params_m,
+                        "notes": system.notes, "metrics": m})
+        worst = sorted(((r, h, _wer(r, h)) for r, h in pairs),
+                       key=lambda x: -x[2])[:5]
+        examples[system.name] = worst
+        print(f"  WER {m['wer'] * 100:.2f}%  CER {m['cer'] * 100:.2f}%  "
+              f"WER(np) {m['wer_nopunct'] * 100:.2f}%  RTF {m['rtf']:.3f}",
+              flush=True)
+
+    dataset = {"metadata": ", ".join(args.metadata), "utts": len(records),
+               "hours": hours, "punctuation": punct or DEFAULT_PUNCTUATION}
+    write_report(args.out, dataset, results, examples, skipped, args.html)
+    with open(os.path.splitext(args.out)[0] + ".json", "w",
+              encoding="utf-8") as fh:
+        json.dump({"dataset": dataset,
+                   "results": [{k: v for k, v in r.items()} for r in results]},
+                  fh, indent=2)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())