JointRecAlign 0.1.0__py3-none-any.whl

jointrecalign/evaluation/metrics.py

@@ -0,0 +1,266 @@
+"""
+Phone-alignment metrics.
+
+Input: an alignment store {file_id: {ref_intervals, hyp_intervals, ref_seq, hyp_seq, style}},
+times in SECONDS; all reported errors are in milliseconds.
+
+  - Boundary F1 at several tolerances: the boundary set is every phone start
+    and every phone end, deduplicated (the end of a phone followed by a pause
+    is a boundary too); each REFERENCE boundary is
+    matched to the NEAREST unused hypothesis boundary within the tolerance
+    (one-to-one); TP/FP/FN are pooled over all utterances before P, R and F1.
+  - Boundary errors on phones matched by edit distance (same label only):
+    mean / median / P90, rates of errors > 50 ms and > 100 ms, signed mean
+    shift of starts and ends, duration error.
+  - PER of the hypothesis phone sequence.
+
+Sequences are aligned on a shared character encoding, so multi-character
+phone tokens (IPA strings) are compared correctly. Group statistics are pooled
+over every phone of the group (micro-average).
+"""
+
+import numpy as np
+import Levenshtein
+from jiwer import process_words
+
+
+# ----------------------------------------------------------------------
+# Sequence alignment (Levenshtein), robust to multi-char phoneme tokens
+# ----------------------------------------------------------------------
+def _editops(ref, hyp):
+    """editops on a shared char-encoding so IPA tokens of any length work."""
+    vocab = {}
+
+    def enc(seq):
+        out = []
+        for t in seq:
+            if t not in vocab:
+                vocab[t] = chr(0xE000 + len(vocab))  # private-use code points
+            out.append(vocab[t])
+        return "".join(out)
+
+    return Levenshtein.editops(enc(ref), enc(hyp))
+
+
+def align_sequences(ref, hyp):
+    """Return list of (ref_idx, hyp_idx); None on either side = del/ins."""
+    alignment = []
+    ops = _editops(ref, hyp)
+    ref_idx = hyp_idx = op_idx = 0
+
+    while ref_idx < len(ref) or hyp_idx < len(hyp):
+        if op_idx < len(ops):
+            op_type, src_pos, dest_pos = ops[op_idx]
+            if op_type == "delete" and src_pos == ref_idx:
+                alignment.append((ref_idx, None)); ref_idx += 1; op_idx += 1; continue
+            elif op_type == "insert" and dest_pos == hyp_idx:
+                alignment.append((None, hyp_idx)); hyp_idx += 1; op_idx += 1; continue
+            elif op_type == "replace" and src_pos == ref_idx and dest_pos == hyp_idx:
+                alignment.append((ref_idx, hyp_idx)); ref_idx += 1; hyp_idx += 1; op_idx += 1; continue
+        if ref_idx < len(ref) and hyp_idx < len(hyp):
+            alignment.append((ref_idx, hyp_idx)); ref_idx += 1; hyp_idx += 1
+        elif ref_idx < len(ref):
+            alignment.append((ref_idx, None)); ref_idx += 1
+        elif hyp_idx < len(hyp):
+            alignment.append((None, hyp_idx)); hyp_idx += 1
+    return alignment
+
+
+# ----------------------------------------------------------------------
+# Per-phoneme errors (only phoneme-matched pairs contribute boundary error)
+# ----------------------------------------------------------------------
+def per_phoneme_errors(ref_alignments, hyp_alignments, ref_seq, hyp_seq):
+    """One record per correctly-matched phoneme. Times in seconds in -> ms out."""
+    alignment = align_sequences(ref_seq, hyp_seq)
+    records = []
+    for ref_idx, hyp_idx in alignment:
+        if ref_idx is None or hyp_idx is None:
+            continue
+        if ref_seq[ref_idx] != hyp_seq[hyp_idx]:
+            continue
+        rs, re = ref_alignments[ref_idx]["start"], ref_alignments[ref_idx]["end"]
+        hs, he = hyp_alignments[hyp_idx]["start"], hyp_alignments[hyp_idx]["end"]
+        records.append({
+            "phoneme":      ref_seq[ref_idx],
+            # signed (keep direction): + = hypothesis is LATE
+            "signed_start": (hs - rs) * 1000.0,
+            "signed_end":   (he - re) * 1000.0,
+            # absolute magnitudes
+            "start_err":    abs(hs - rs) * 1000.0,
+            "end_err":      abs(he - re) * 1000.0,
+            "dur_err":      abs((he - hs) - (re - rs)) * 1000.0,
+            "mid_err":      abs((hs + he) / 2 - (rs + re) / 2) * 1000.0,
+        })
+    return records
+
+
+# ----------------------------------------------------------------------
+# Boundary-detection F1 (time-only, label-agnostic).
+#
+# Boundary set: every phone start and every phone end, deduplicated.
+#   Taking all starts AND all ends then deduplicating is equivalent to
+#   "starts + last end" under a contiguous segmentation (phone n's end IS
+#   phone n+1's start), but stays correct when a pause leaves a gap between
+#   consecutive intervals, where "starts + last end" would silently drop the
+#   offset preceding the gap.
+#
+# Matching: each REFERENCE boundary takes the NEAREST unused hypothesis
+#   boundary within the tolerance, one-to-one.
+#   FP = hypothesis boundaries left unmatched, FN = reference boundaries with
+#   no hypothesis boundary within tolerance. A boundary that exists but is
+#   outside tolerance therefore costs one FP and one FN.
+#
+# tolerance is in SECONDS.
+# ----------------------------------------------------------------------
+def boundary_times(intervals):
+    b = [iv["start"] for iv in intervals] + [iv["end"] for iv in intervals]
+    return sorted(set(b))
+
+
+def compute_f1_counts(ref_bounds, pred_bounds, tolerance):
+    ref_bounds  = np.sort(np.asarray(ref_bounds,  dtype=float))
+    pred_bounds = np.sort(np.asarray(pred_bounds, dtype=float))
+    used = np.zeros(len(pred_bounds), dtype=bool)
+    TP = 0
+    for rb in ref_bounds:
+        if not len(pred_bounds):
+            break
+        d = np.abs(pred_bounds - rb)
+        d[used] = np.inf
+        j = int(d.argmin())
+        if d[j] <= tolerance:
+            used[j] = True
+            TP += 1
+    FP = len(pred_bounds) - TP
+    FN = len(ref_bounds) - TP
+    return TP, FP, FN
+
+
+def _f1(tp, fp, fn):
+    p = tp / (tp + fp) if (tp + fp) > 0 else 0.0
+    r = tp / (tp + fn) if (tp + fn) > 0 else 0.0
+    return 2 * p * r / (p + r) * 100 if (p + r) > 0 else 0.0
+
+
+# ----------------------------------------------------------------------
+# Pooled boundary statistics over a set of per-phoneme records (ms).
+# ----------------------------------------------------------------------
+def boundary_stats(df):
+    import pandas as pd                       # evaluation only (pip install ".[eval]")
+    if len(df) == 0:
+        return {}
+    s, e, d = df["start_err"], df["end_err"], df["dur_err"]
+    both = pd.concat([s, e])  # AAS / tail pool start AND end boundaries
+    return {
+        "K":                      int(len(df)),
+        # --- central ---
+        "AAS (ms)":               both.mean(),               # mean |error| over all boundaries
+        "MedianBE (ms)":          both.median(),             # median |error| over all boundaries
+        "Median_start (ms)":      s.median(),
+        "Mean_start (ms)":        s.mean(),
+        "Median_end (ms)":        e.median(),
+        "Mean_end (ms)":          e.mean(),
+        # --- tail: pooled over start+end ---
+        "P90 (ms)":               both.quantile(0.90),
+        "%>50ms":                 (both > 50).mean() * 100,  # gross-error rate
+        "%>100ms":                (both > 100).mean() * 100,
+        "%within20ms":            (both <= 20).mean() * 100,
+        # --- direction of bias ---
+        "SignedMean_start (ms)":  df["signed_start"].mean(), # + = hyp late
+        "SignedMean_end (ms)":    df["signed_end"].mean(),
+        # --- duration ---
+        "Mean_dur (ms)":          d.mean(),
+        "Median_dur (ms)":        d.median(),
+        "%dur>50ms":              (d > 50).mean() * 100,
+    }
+
+
+# ----------------------------------------------------------------------
+# Main entry point
+# ----------------------------------------------------------------------
+def compute_metrics(alignment_store, csv_path, per_phoneme_csv=None,
+                    f1_tolerances=(0.02, 0.05)):
+    """
+    alignment_store: {file_id: {ref_intervals, hyp_intervals, ref_seq, hyp_seq, style}}
+        *_intervals: list of {"start": sec, "end": sec}
+        *_seq:       list of phoneme tokens
+    Writes a file/style/global metrics CSV, and optionally a per-phoneme CSV.
+    All boundary stats are pooled (micro-averaged) within each group.
+    """
+    import pandas as pd                       # evaluation only (pip install ".[eval]")
+    all_records = []   # long-form: one row per matched phoneme, tagged with file+style
+    f1_rows = []       # per-file F1 counts + PER, pooled later
+
+    for f, data in alignment_store.items():
+        ref_intervals = data["ref_intervals"]
+        hyp_intervals = data["hyp_intervals"]
+        ref_seq, hyp_seq = data["ref_seq"], data["hyp_seq"]
+        style = data.get("style", "ALL")
+
+        recs = per_phoneme_errors(ref_intervals, hyp_intervals, ref_seq, hyp_seq)
+        for r in recs:
+            r["file"] = f; r["style"] = style
+        all_records.extend(recs)
+
+        # F1 over all boundary times (label-agnostic boundary detection)
+        ref_bounds  = boundary_times(ref_intervals)
+        pred_bounds = boundary_times(hyp_intervals)
+        row = {"file": f, "style": style, "N_ref": len(ref_seq), "N_hyp": len(hyp_seq),
+               "B_ref": len(ref_bounds), "B_hyp": len(pred_bounds)}
+        for tol in f1_tolerances:
+            tp, fp, fn = compute_f1_counts(ref_bounds, pred_bounds, tol)
+            tag = f"{int(round(tol * 1000))}"
+            row[f"TP_{tag}"], row[f"FP_{tag}"], row[f"FN_{tag}"] = tp, fp, fn
+
+        out = process_words(" ".join(ref_seq), " ".join(hyp_seq))
+        row["S"], row["D"], row["I"] = out.substitutions, out.deletions, out.insertions
+        f1_rows.append(row)
+
+    records_df = pd.DataFrame(all_records)
+    f1_df = pd.DataFrame(f1_rows)
+    tags = [f"{int(round(t * 1000))}" for t in f1_tolerances]
+
+    def assemble(rec_subset, f1_subset, label, style_val):
+        out = {"group": label, "style": style_val}
+        out.update(boundary_stats(rec_subset))
+        out["N_ref"] = int(f1_subset["N_ref"].sum())
+        out["N_hyp"] = int(f1_subset["N_hyp"].sum())
+        out["B_ref"] = int(f1_subset["B_ref"].sum())
+        out["B_hyp"] = int(f1_subset["B_hyp"].sum())
+        S, D, I = f1_subset["S"].sum(), f1_subset["D"].sum(), f1_subset["I"].sum()
+        out["PER (%)"] = (S + D + I) / out["N_ref"] * 100 if out["N_ref"] else np.nan
+        out["Deletions"], out["Insertions"] = int(D), int(I)
+        for tag in tags:
+            tp = f1_subset[f"TP_{tag}"].sum()
+            fp = f1_subset[f"FP_{tag}"].sum()
+            fn = f1_subset[f"FN_{tag}"].sum()
+            out[f"P@{tag}ms (%)"]  = tp / (tp + fp) * 100 if (tp + fp) > 0 else np.nan
+            out[f"R@{tag}ms (%)"]  = tp / (tp + fn) * 100 if (tp + fn) > 0 else np.nan
+            out[f"F1@{tag}ms (%)"] = _f1(tp, fp, fn)
+        return out
+
+    rows = []
+    # per file
+    for f in f1_df["file"]:
+        rsub = records_df[records_df["file"] == f] if len(records_df) else records_df
+        fsub = f1_df[f1_df["file"] == f]
+        rows.append(assemble(rsub, fsub, f, fsub["style"].iloc[0]))
+    # per style
+    for s, fsub in f1_df.groupby("style"):
+        rsub = records_df[records_df["style"] == s] if len(records_df) else records_df
+        rows.append(assemble(rsub, fsub, f"STYLE_{s}", s))
+    # global
+    rows.append(assemble(records_df, f1_df, "GLOBAL", "ALL"))
+
+    df = pd.DataFrame(rows)
+    df.to_csv(csv_path, index=False)
+
+    # optional per-phoneme table (pooled over all files)
+    if per_phoneme_csv is not None and len(records_df):
+        ph_rows = []
+        for ph, sub in records_df.groupby("phoneme"):
+            r = {"phoneme": ph}; r.update(boundary_stats(sub)); ph_rows.append(r)
+        pd.DataFrame(ph_rows).sort_values("AAS (ms)", ascending=False)\
+            .to_csv(per_phoneme_csv, index=False)
+
+    return df

jointrecalign/evaluation/sil_eval.py

@@ -0,0 +1,311 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Scoring of latent-SIL checkpoints on an evaluation manifest (nothing is fitted).
+
+One model pass per chunk, as attention_eval.run_manifest (CTC greedy target,
+prior 0, eval mode), then two decodes of the same energies:
+
+  sil   Viterbi over the extended topology [SIL] p1 [SIL] p2 ... [SIL]
+        (positions = config.sil_eval_positions, default "all": the greedy
+        target has no word boundaries), learned key / entry cost, min frames
+        as trained. A phone ends where the following SIL starts; SIL frames
+        belong to no phone.
+  dtw   the plain decode (attention_eval.dtw_path on the phone softmax, onsets,
+        contiguous ends), for comparison on the same pass.
+
+Boundary time of frame f in a chunk starting at cs: cs + f * HOP + offset, with
+HOP and offset read from the model (attention_eval.frame_stride_s,
+frame_time_offset_s): 20 ms / +2.5 ms, 10 ms / +7.5 ms.
+
+Metrics: F1@10/20/50, MedianBE, P90, PER (metrics.compute_metrics), duration
+slope/corr (all phones, and in-speech phones only), and, when the manifest has
+pause_intervals, SIL against the reference pauses over each whole file (a pause
+cut by a chunk edge counts once): frame precision/recall/F1 (frame centres,
+excluded_intervals ignored) and pause-interval P/R/F1 at IoU >= 0.5.
+"""
+import itertools
+import os
+import tempfile
+import unicodedata
+
+import numpy as np
+import torch
+
+from .. import sil_fs
+from . import attention_eval as E
+from .metrics import compute_metrics, align_sequences
+
+# Set from the model by _set_frame (defaults: 20 ms model). CENTRE = centre of
+# frame 0's receptive field, used for frame-level SIL scoring.
+HOP = 0.02
+CENTRE = 0.0125
+
+
+def _set_frame(model):
+    global HOP, CENTRE
+    m = getattr(model, "module", model)
+    HOP = E.frame_stride_s(m)
+    CENTRE = E.frame_time_offset_s(m) + HOP / 2      # (RF - hop) / 2 + hop / 2 = RF / 2
+    return E.frame_time_offset_s(m)
+
+
+PAUSE_MIN = 0.02      # a reference pause shorter than 20 ms is ignored
+GATE = 0.5            # same gate as attention_eval.duration_fidelity
+
+
+def _gap_has_pause(pauses, a, b):
+    """True if the gap (a, b) between two consecutive reference phones holds
+    >= PAUSE_MIN of reference pause (pauses: [(start, end)]; None = every gap
+    >= PAUSE_MIN counts as a pause)."""
+    if b - a < PAUSE_MIN:
+        return False
+    if pauses is None:
+        return True
+    out = []
+    for s, e in pauses:
+        s2, e2 = max(s, a), min(e, b)
+        if e2 - s2 <= 1e-6:
+            continue
+        if out and s2 - out[-1][1] < 1e-6:
+            out[-1] = (out[-1][0], e2)
+        else:
+            out.append((s2, e2))
+    return any(e - s >= PAUSE_MIN for s, e in out)
+
+
+def duration_fidelity_inspeech(store, pauses_by_file=None, gate=GATE):
+    """attention_eval.duration_fidelity restricted to phones with no pause on
+    either side."""
+    R, Hd = [], []
+    for nm, d in store.items():
+        refs = d["ref_intervals"]
+        pauses = None if pauses_by_file is None else pauses_by_file.get(nm, [])
+        N = len(refs)
+        for i, j in align_sequences(d["ref_seq"], d["hyp_seq"]):
+            if i is None or j is None or d["ref_seq"][i] != d["hyp_seq"][j]:
+                continue
+            if i == 0 or i == N - 1:
+                continue
+            r, h = refs[i], d["hyp_intervals"][j]
+            if abs(h["start"] - r["start"]) > gate or abs(h["end"] - r["end"]) > gate:
+                continue
+            if (_gap_has_pause(pauses, refs[i - 1]["end"], r["start"])
+                    or _gap_has_pause(pauses, r["end"], refs[i + 1]["start"])):
+                continue
+            R.append((r["end"] - r["start"]) * 1000)
+            Hd.append((h["end"] - h["start"]) * 1000)
+    if len(R) < 100:
+        return {}
+    R, Hd = np.array(R), np.array(Hd)
+    return {"dur_slope_sp": float(np.polyfit(R, Hd, 1)[0]), "dur_corr_sp": float(np.corrcoef(R, Hd)[0, 1])}
+
+
+def _segments(states, tokens, cs, offset, sil_frames):
+    """Per-frame states -> hypothesis phone intervals. sil_frames collects
+    (start, end) of every SIL run (seconds)."""
+    ca = []
+    for s, grp in itertools.groupby(enumerate(states), key=lambda x: x[1]):
+        grp = list(grp)
+        f0, f1 = grp[0][0], grp[-1][0] + 1
+        t0, t1 = cs + f0 * HOP + offset, cs + f1 * HOP + offset
+        if s is None:
+            sil_frames.append((t0, t1))
+            continue
+        ph = tokens[s]
+        if ph in E.SKIP_TOK:
+            continue
+        if len(ph) == 1 and unicodedata.combining(ph):
+            if ca:
+                ca[-1]["phoneme"] += ph
+                ca[-1]["end"] = t1
+            continue
+        ca.append({"phoneme": E.HYP_FIX.get(ph, ph), "start": t0, "end": t1})
+    return ca
+
+
+@torch.inference_mode()
+def run_manifest_sil(model, feature_extractor, tokenizer, manifest, device="cuda",
+                     max_files=None, positions=None, offset=None, with_dtw=True):
+    """-> (store_sil, store_dtw or None, sil_runs {file: [(s, e)]}, chunks {file: [(s, e)]})
+    offset None = the model's frame_time_offset_s."""
+    model_offset = _set_frame(model)
+    offset = model_offset if offset is None else offset
+    m = getattr(model, "module", model)
+    positions = positions or getattr(m, "sil_eval_positions", "all")
+    k = int(getattr(m, "sil_min_frames", 1))
+    sil_logp = float(torch.as_tensor(m.sil_logp).detach())   # a learned parameter in most checkpoints
+    was_training = model.training
+    model.eval()
+    store, store_dtw, sil_runs, chunks = {}, {}, {}, {}
+    try:
+        for nm, meta in list(manifest["files"].items())[:max_files]:
+            wav = torch.from_numpy(E.load_audio(meta["wav"])).float()
+            hyp, hyp_dtw, runs = [], [], []
+            chunks[nm] = [(float(c["start"]), float(c["end"])) for c in meta["chunks"]]
+            for ch in meta["chunks"]:
+                s0, s1 = int(ch["start"] * E.SR), int(ch["end"] * E.SR)
+                seg = wav[s0:s1]
+                if seg.numel() < 400:
+                    continue
+                x = feature_extractor(E.to_model_rate(seg.numpy(), m),
+                                      sampling_rate=feature_extractor.sampling_rate,
+                                      return_tensors="pt", return_attention_mask=True)
+                o = model(input_values=x.input_values.to(device),
+                          attention_mask=x.attention_mask.to(device), labels=None,
+                          prior_weight=0.0, ctc_weight=0.0, align_weight=0.0,
+                          contrastive_weight=0.0)
+                T, P = int(o.input_lengths[0]), int(o.text_lengths[0])
+                if P == 0:
+                    continue
+                cs = s0 / E.SR
+                tokens = tokenizer.convert_ids_to_tokens(o.predicted_phone_ids[0, :P].cpu().tolist())
+                en = o.energy[0, :T, :P].float()
+                sil = o.sil_energy_raw[0, :T].float()
+                logp_ext = torch.log_softmax(torch.cat([en, sil[:, None]], -1), -1).cpu().numpy()
+                top, path = sil_fs.viterbi(logp_ext, P, sil_fs.slot_mask(P, positions), k, sil_logp)
+                if path is not None:
+                    st = [None if top["col"][s] < 0 else int(top["col"][s]) for s in path]
+                    hyp.extend(_segments(st, tokens, cs, offset, runs))
+                if with_dtw:
+                    att = np.exp(en.log_softmax(-1).cpu().numpy())
+                    p2 = E.dtw_path(att)
+                    if p2 is not None:
+                        ca = _segments([int(v) for v in p2], tokens, cs, offset, [])
+                        for i in range(len(ca) - 1):           # contiguous ends, as attention_eval
+                            ca[i]["end"] = ca[i + 1]["start"]
+                        if ca:
+                            ca[-1]["end"] = cs + T * HOP + offset
+                        hyp_dtw.extend(ca)
+            for st_, h in ((store, hyp), (store_dtw, hyp_dtw)):
+                h = sorted(h, key=lambda z: z["start"])
+                st_[nm] = {"ref_intervals": meta["ref_intervals"], "hyp_intervals": h,
+                           "ref_seq": [z["phoneme"] for z in meta["ref_intervals"]],
+                           "hyp_seq": [z["phoneme"] for z in h], "style": meta.get("style", "ALL")}
+            sil_runs[nm] = runs
+    finally:
+        model.train(was_training)
+    return store, (store_dtw if with_dtw else None), sil_runs, chunks
+
+
+def _score(store, tag=""):
+    tmp = tempfile.mkdtemp()
+    df = compute_metrics(store, os.path.join(tmp, "m.csv"), None, f1_tolerances=(0.01, 0.02, 0.05))
+    g = df[df.group == "GLOBAL"].iloc[0]
+
+    def col(*keys):
+        for c in df.columns:
+            if all(k.lower() in str(c).lower() for k in keys):
+                return float(g[c])
+        return float("nan")
+    return {f"F1@10{tag}": col("F1@10"), f"F1@20{tag}": col("F1@20"), f"F1@50{tag}": col("F1@50"),
+            f"MedianBE{tag}": col("median", "BE"), f"P90{tag}": col("P90"), f"PER{tag}": col("PER")}
+
+
+def _sil_timeline(phones, end):
+    """Whole-file SIL runs: the time in [0, end] not covered by a hypothesis phone.
+    SIL-Viterbi gives every decoded frame to a phone or to SIL, so this is the
+    decoded SIL with a pause cut by a chunk edge joined back into one run, and
+    audio left out between chunks counted as SIL."""
+    runs, cur = [], 0.0
+    for p in sorted(phones, key=lambda z: z["start"]):
+        if p["start"] - cur >= HOP / 2:
+            runs.append((cur, min(p["start"], end)))
+        cur = max(cur, p["end"])
+    if end - cur >= HOP / 2:
+        runs.append((cur, end))
+    return [r for r in runs if r[1] - r[0] >= HOP / 2]
+
+
+def _frame_counts(a, b, P, X, R):
+    """(tp, fp, fn) of SIL frames vs reference pause frames, frame centres in [a, b)."""
+    c = np.arange(a, b - HOP / 2, HOP) + CENTRE
+    c = c[c < b]
+    keep = ~np.any((X[:, :1] <= c) & (c < X[:, 1:]), axis=0) if len(X) else np.ones(len(c), bool)
+    ref = np.any((P[:, :1] <= c) & (c < P[:, 1:]), axis=0) if len(P) else np.zeros(len(c), bool)
+    hyp = np.any((R[:, :1] <= c) & (c < R[:, 1:]), axis=0) if len(R) else np.zeros(len(c), bool)
+    return int((hyp & ref & keep).sum()), int((hyp & ~ref & keep).sum()), int((~hyp & ref & keep).sum())
+
+
+def _pause_matches(refs, hyps, iou):
+    """Reference pauses matched one-to-one (greedy) by a SIL run at IoU >= iou."""
+    used, m = set(), 0
+    for r in refs:
+        best, bj = 0.0, None
+        for j, h in enumerate(hyps):
+            if j in used:
+                continue
+            inter = max(0.0, min(r[1], h[1]) - max(r[0], h[0]))
+            if inter <= 0:
+                continue
+            u = max(r[1], h[1]) - min(r[0], h[0])
+            if inter / u > best:
+                best, bj = inter / u, j
+        if bj is not None and best >= iou:
+            used.add(bj)
+            m += 1
+    return m
+
+
+def sil_scores(manifest, sil_runs, chunks, store=None, iou=0.5, mode="file"):
+    """SIL vs reference pauses (manifest pause_intervals).
+
+    mode "file" (default, needs store = the SIL-Viterbi alignment store): one SIL
+      timeline per file (_sil_timeline over [0, last chunk end]), frames over the
+      whole file, every reference pause.
+    mode "chunk" (the scores before Oct 2026, kept to compare with older runs):
+      SIL runs per chunk, frames inside chunks only, reference pauses only if
+      inside one chunk. Biased low by cuts: a pause crossing a chunk edge leaves
+      the reference while the SIL on each side of the edge counts as a false
+      positive, so the more chunks, the lower the pause precision."""
+    if mode not in ("file", "chunk"):
+        raise ValueError(f"mode {mode!r}: 'file' or 'chunk'")
+    if mode == "file" and store is None:
+        raise ValueError("mode 'file' needs the alignment store (hypothesis phones)")
+    tp = fp = fn = 0
+    m_tp = n_hyp = n_ref = 0
+    for nm, runs in sil_runs.items():
+        if not chunks[nm]:
+            continue
+        meta = manifest["files"][nm]
+        P = np.asarray([p[:2] for p in meta.get("pause_intervals", [])], float).reshape(-1, 2)
+        X = np.asarray([p[:2] for p in meta.get("excluded_intervals", [])], float).reshape(-1, 2)
+        if mode == "file":
+            end = chunks[nm][-1][1]
+            hyps = _sil_timeline(store[nm]["hyp_intervals"], end)
+            spans, refs = [(0.0, end)], list(P)
+        else:
+            hyps, spans = list(runs), chunks[nm]
+            refs = [p for p in P if any(a <= p[0] and p[1] <= b for a, b in spans)]
+        R = np.asarray(hyps, float).reshape(-1, 2)
+        for a, b in spans:
+            t, f_, n = _frame_counts(a, b, P, X, R)
+            tp += t; fp += f_; fn += n
+        n_ref += len(refs); n_hyp += len(hyps)
+        m_tp += _pause_matches(refs, hyps, iou)
+    f = lambda p, r: 2 * p * r / (p + r) if p + r > 0 else 0.0
+    pf, rf = tp / max(tp + fp, 1), tp / max(tp + fn, 1)
+    pi, ri = m_tp / max(n_hyp, 1), m_tp / max(n_ref, 1)
+    return {"sil_frame_P": 100 * pf, "sil_frame_R": 100 * rf, "sil_frame_F1": 100 * f(pf, rf),
+            "sil_pause_P": 100 * pi, "sil_pause_R": 100 * ri, "sil_pause_F1": 100 * f(pi, ri),
+            "sil_n_hyp": n_hyp, "sil_n_ref": n_ref}
+
+
+def evaluate(model, feature_extractor, tokenizer, manifest, device="cuda", max_files=None,
+             positions=None, offset=None, with_dtw=True, sil_mode="file"):
+    """All metrics as a flat dict (keys suffixed _dtw for the plain DTW decode).
+    sil_mode: pause scoring, see sil_scores."""
+    store, store_dtw, runs, chunks = run_manifest_sil(
+        model, feature_extractor, tokenizer, manifest, device, max_files, positions, offset, with_dtw)
+    has_pauses = any("pause_intervals" in m for m in manifest["files"].values())
+    pauses = ({nm: [tuple(p[:2]) for p in m.get("pause_intervals", [])]
+               for nm, m in manifest["files"].items()} if has_pauses else None)
+    res = _score(store)
+    res.update(E.duration_fidelity(store, align_sequences))
+    res.update(duration_fidelity_inspeech(store, pauses))
+    if has_pauses:
+        res.update(sil_scores(manifest, runs, chunks, store, mode=sil_mode))
+    if store_dtw is not None:
+        res.update(_score(store_dtw, "_dtw"))
+        res.update({f"{k}_dtw": v for k, v in E.duration_fidelity(store_dtw, align_sequences).items()})
+    return res, store, store_dtw