lyric-align 0.3.0__py3-none-any.whl

lyric_align/__init__.py

@@ -0,0 +1,19 @@
+"""lyric-align — place known lyrics on an audio timeline.
+
+You already have the correct lyrics; you only need the *times*. lyric-align
+anchors known lyric lines onto ASR word timings via character-level fuzzy
+matching — built for space-less languages (Japanese, Chinese) and sung vocals
+(including rap), where whitespace tokenization and forced aligners fall short.
+"""
+from .anchor import align, auto_pairing, interpolate_gaps
+from .model import AlignedLine, Segment, Word
+from .normalize import normalize, similarity
+
+__version__ = "0.3.0"
+
+__all__ = [
+    "align", "auto_pairing", "interpolate_gaps",
+    "AlignedLine", "Segment", "Word",
+    "normalize", "similarity",
+    "__version__",
+]

lyric_align/anchor.py

@@ -0,0 +1,231 @@
+"""Anchor known lyric lines onto ASR segments (greedy forward fuzzy match).
+
+This is the heart of lyric-align. Given the *correct* lyrics (from an official
+source) and ASR segments with word timings, we place each lyric line/stanza on
+the timeline by character-level fuzzy matching, moving forward monotonically so
+repeated lines (choruses) consume segments in order.
+
+The forward `window` is load-bearing, not just an optimisation: it keeps a line
+from reaching a distant segment that happens to clear the threshold. Replacing
+this scan with a globally optimal monotone assignment (maximise total similarity,
+optionally minus a diagonal-drift penalty) was tried and measured worse — it
+places every line, but repeated hooks carry no distinguishing similarity, so the
+extra placements land on the wrong repetition: mean error 0.94 s -> 1.14-1.61 s,
+worst 6.4 s -> 10.6 s on a track with a 4x-repeated hook. Locality beats global
+optimality here because similarity alone cannot tell repetitions apart.
+
+A timing prior — score candidates by `similarity - lam * |start - predicted|`,
+predicting from the last placement plus the running median gap — was the obvious
+next move and also measured worse (mean 0.94 s -> 1.67 s at lam=0.1, worst
+6.4 s -> 10.6 s, collapsing to 8/33 placements by lam=0.5). It predicts from
+*our own* previous placements, so it cannot correct a bad one; it anchors on it
+and drags the following lines along. Songs do not run at one pace either, so a
+median gap mispredicts hardest across a section boundary — where repeated hooks
+live. Restricted to breaking near-ties it stops hurting without clearly helping
+(one line gained, below the ground truth's own resolution), so it is not here.
+See the README for the full sweep.
+
+Four further attempts to fix the remaining outlier — letting a candidate span
+two consecutive segments, scoring candidates on how well they explain the unit's
+*opening* rather than the whole unit, vetoing candidates that match only its
+tail, and taking the start from the word the first character lands on — all
+worked, and all cost more than they returned. The first three share a mechanism:
+`idx` advances to just past whatever was chosen, so every neighbour is
+downstream of every decision, and gains and losses arrive in adjacent pairs (the
+best variant fixes 2.30 s -> 0.02 s at 2:20 on one track and breaks
+0.32 s -> 3.70 s at 2:26). The fourth fails differently: placements are already
+late (signed mean +0.46 s / +0.21 s) because a sung phrase begins at its breath
+and attack, earlier than the first word an ASR will timestamp, so refining into
+the segment only adds lateness. See the README for both tables.
+
+Letting the unit size vary — choosing, per step, how many lines this segment
+should absorb instead of fixing it up front — was tried next and also measured
+worse. It is a tempting move because `pairing` is a rounded average: at 76 lines
+over 64 segments the true ratio is 1.19, so a fixed 1 leaves 27 lines unplaced
+and a fixed 2 straddles boundaries. Scoring `(k, segment)` jointly does raise
+placements (49/76 -> 66/76), and the first-line error barely moves. It is not
+free: a unit chosen to maximise similarity will happily straddle a *section*
+boundary, and such a unit needs only its tail to match. On the first track the
+last verse line scores 0.000 against the hook segment alone and 0.286 once the
+following hook line joins it — over the 0.25 threshold — so the breath split
+drags that verse line 13.3 s forward into the hook, and displaces the correctly
+placed hook line as collateral. Fixed pairing cannot do this at pairing=1: a
+one-line unit has no tail to match with. Variable pairing manufactures the very
+tail-match pathology that a dedicated veto (above) already failed to fix.
+
+Note the shape of that failure, because it also refutes the reason for trying:
+the earlier four attempts all changed *selection* (which segment a unit takes),
+so the plan was to change *consumption* instead and avoid the coupling. It does
+not avoid it. Consumption sets how fast the segment cursor advances relative to
+the line cursor, so it moves `idx` too — just one step removed — and the same
+gains-and-losses-in-adjacent-pairs behaviour returns. On the second track, where
+the ASR merges two lines consistently, deviating downward orphans the remainder
+onto the next segment and shifts every later unit's phase: within-0.5 s falls
+26/33 -> 18/33 and the 4x-repeated hook lands a repetition early. Restricting
+deviation to *upward* only looks safe, but only because pairing=2 with k<=2
+leaves it no room; allowing k<=3 breaks that track the same way (26/33 -> 13/33).
+
+Design philosophy: when a line cannot be confidently matched, we mark it
+unmatched rather than inventing a timestamp. Forced aligners always emit an
+answer and thus fail *silently* (e.g. drifting into the intro); we prefer honest
+gaps that a human — or a later interpolation pass — can fix. The same reasoning
+rejects the global matcher above: a visible gap beats a confident wrong time.
+The variable-pairing result is the sharpest case: its headline gain was 17 extra
+placements, and on the only subset where those extra placements can be checked,
+half were catastrophically wrong.
+"""
+from __future__ import annotations
+
+from .breath import split_words_by_breath
+from .charmap import char_timings
+from .model import AlignedLine, Segment
+from .normalize import default_threshold, similarity
+
+
+AUTO_PAIRING_MAX = 3
+
+
+def auto_pairing(lyrics: list[str], segments: list[Segment]) -> int:
+    """How many lyric lines the ASR appears to have merged into one segment.
+
+    `pairing` is not a property of the song, it is a property of the *ASR's*
+    segmentation, and different models segment differently. faster-whisper
+    `medium` merges about two sung lines per segment on Japanese rap, which is
+    where the old fixed default of 2 came from — but `large-v3` splits much
+    finer (4.7 s average segment down to 2.8 s, 41 segments up to 64 on the same
+    four-minute track), so a two-line unit straddles a segment boundary and the
+    match degrades. Measured on that track: `large-v3` at the old default is
+    *worse* than `medium` (mean 0.50 s -> 1.13 s), and better than it once the
+    pairing follows (0.31 s, worst case 3.58 s -> 0.76 s). Upgrading the model
+    alone is a trap.
+
+    Lines per segment is exactly what pairing means, so it is also the estimate.
+    Capped at 3: beyond that the ASR has stopped producing line-like segments
+    (a full mix, where whole verses collapse into one), and the answer there is
+    to separate the vocal, not to widen the unit — which the CLI already says.
+    """
+    if not segments:
+        return 1
+    return max(1, min(AUTO_PAIRING_MAX, round(len(lyrics) / len(segments))))
+
+
+def _stanzas(lines: list[str], pairing: int) -> list[list[str]]:
+    """Group lyric lines into stanza units of size `pairing` (1 = per line)."""
+    if pairing < 1:
+        pairing = 1
+    return [lines[i:i + pairing] for i in range(0, len(lines), pairing)]
+
+
+def _containing(start: float, end: float, chars: list[dict] | None) -> tuple[float, float]:
+    """Widen a line span so it contains its own character timings.
+
+    A line takes its span from the segment, but the characters are interpolated
+    across the *word* timings, and faster-whisper does not guarantee that a
+    segment's end equals its last word's end. When it does not, the final
+    character runs past the line — which TTML forbids outright ("the timestamp
+    of a child element must be completely contained within the timestamp of its
+    parent") and which makes a strict player clamp or drop the tail.
+
+    The word timings are the precise signal here, so the line yields to them.
+    """
+    if not chars:
+        return start, end
+    return min(start, chars[0]["start"]), max(end, chars[-1]["end"])
+
+
+def align(
+    segments: list[Segment],
+    lyrics: list[str],
+    *,
+    pairing: int | str = "auto",
+    threshold: float | None = None,
+    window: int = 4,
+    karaoke: bool = False,
+) -> list[AlignedLine]:
+    """Align known `lyrics` lines onto ASR `segments`.
+
+    Args:
+        pairing: lyric lines per stanza unit matched to one segment, or "auto"
+            (the default) to read it off the ASR's own segmentation — see
+            `auto_pairing`, which explains why a fixed value is tied to one
+            model. Pass an int to override.
+        threshold: minimum character similarity to accept a match. Defaults to a
+            script-aware value (see `normalize.default_threshold`).
+        window: how many segments ahead to search from the current position.
+        karaoke: if True, compute per-character timings (needs word timings).
+
+    Returns one AlignedLine per input lyric line (stanzas are expanded back to
+    lines via breath splitting).
+    """
+    if threshold is None:
+        threshold = default_threshold(lyrics)
+    if isinstance(pairing, str):
+        if pairing != "auto":
+            raise ValueError(f"pairing must be an int or 'auto', got {pairing!r}")
+        pairing = auto_pairing(lyrics, segments)
+
+    units = _stanzas(lyrics, pairing)
+    results: list[AlignedLine] = []
+    idx = 0
+
+    for unit in units:
+        joined = " ".join(unit)
+        best_score, best_i = 0.0, None
+        upper = min(idx + window, len(segments))
+        for i in range(idx, upper):
+            sc = similarity(joined, segments[i].text)
+            if sc > best_score:
+                best_score, best_i = sc, i
+
+        if best_i is not None and best_score > threshold:
+            seg = segments[best_i]
+            idx = best_i + 1
+            if len(unit) == 1 or not seg.words:
+                # Single line, or no word timings: use the segment span as-is.
+                for k, line in enumerate(unit):
+                    chars = char_timings(line, seg.words) if (karaoke and k == 0) else None
+                    start, end = _containing(seg.start, seg.end, chars)
+                    results.append(AlignedLine(line, start, end,
+                                               best_score, True, chars))
+            else:
+                groups = split_words_by_breath(seg.words, unit)
+                for line, grp in zip(unit, groups):
+                    if grp:
+                        chars = char_timings(line, grp) if karaoke else None
+                        start, end = _containing(grp[0].start, grp[-1].end, chars)
+                        results.append(AlignedLine(line, start, end,
+                                                   best_score, True, chars))
+                    else:
+                        results.append(AlignedLine(line, seg.start, seg.end,
+                                                   best_score, True, None))
+        else:
+            for line in unit:
+                results.append(AlignedLine(line, None, None, best_score, False, None))
+
+    return results
+
+
+def interpolate_gaps(aligned: list[AlignedLine]) -> list[AlignedLine]:
+    """Fill unmatched lines by linear interpolation between known neighbors.
+
+    Optional convenience for callers who want a fully-populated timeline. The
+    `matched` flag stays False so downstream code can still tell which lines
+    were guessed.
+    """
+    n = len(aligned)
+    for i, a in enumerate(aligned):
+        if a.matched or a.start is not None:
+            continue
+        prev = next((aligned[j] for j in range(i - 1, -1, -1)
+                     if aligned[j].start is not None), None)
+        nxt = next((aligned[j] for j in range(i + 1, n)
+                    if aligned[j].start is not None), None)
+        if prev and nxt:
+            span = (nxt.start - prev.end)
+            a.start = prev.end + span * 0.33
+            a.end = prev.end + span * 0.66
+        elif prev:
+            a.start, a.end = prev.end, prev.end + 2.0
+        elif nxt:
+            a.start, a.end = max(0.0, nxt.start - 2.0), nxt.start
+    return aligned

lyric_align/asr.py

@@ -0,0 +1,47 @@
+"""ASR backend: faster-whisper (optional dependency).
+
+Kept behind a lazy import so the core aligner has zero heavy dependencies. If
+you already have segments (from any Whisper flavor), skip this and feed the
+aligner directly via `Segment.from_dict`.
+"""
+from __future__ import annotations
+
+from pathlib import Path
+
+from .model import Segment, Word
+
+
+def transcribe(
+    audio_path: str | Path,
+    *,
+    language: str = "ja",
+    model_size: str = "medium",
+    device: str = "cpu",
+    compute_type: str = "int8",
+    vad: bool = True,
+) -> list[Segment]:
+    """Transcribe with faster-whisper, returning word-timed segments.
+
+    Requires the `[asr]` extra: pip install lyric-align[asr]
+    """
+    try:
+        from faster_whisper import WhisperModel
+    except ImportError as e:  # pragma: no cover
+        raise ImportError(
+            "faster-whisper is required for transcription. "
+            "Install with: pip install lyric-align[asr]"
+        ) from e
+
+    model = WhisperModel(model_size, device=device, compute_type=compute_type)
+    kwargs = dict(language=language, word_timestamps=True)
+    if vad:
+        kwargs.update(vad_filter=True,
+                      vad_parameters=dict(min_silence_duration_ms=500))
+    segments, _ = model.transcribe(str(audio_path), **kwargs)
+
+    out = []
+    for seg in segments:
+        words = [Word(float(w.start), float(w.end), w.word)
+                 for w in (seg.words or [])]
+        out.append(Segment(float(seg.start), float(seg.end), seg.text.strip(), words))
+    return out

lyric_align/breath.py

@@ -0,0 +1,52 @@
+"""Split a merged ASR segment back into individual lyric lines.
+
+ASR (Whisper etc.) tends to merge several sung lines into one segment. When we
+know how many lines a segment should contain, we can recover per-line
+boundaries by cutting at the largest inter-word time gap (a breath) near the
+expected split point — estimated from the character-count ratio of the known
+lines.
+"""
+from __future__ import annotations
+
+from .model import Word
+from .normalize import nchars
+
+
+def split_words_by_breath(words: list[Word], lines: list[str],
+                          search: int = 3) -> list[list[Word]]:
+    """Partition `words` into len(lines) groups.
+
+    For each split point, aim at the index implied by cumulative character
+    ratio of `lines`, then within ±`search` words pick the boundary with the
+    largest silence gap (the breath). Returns one word-list per line; groups
+    may be empty if there are fewer words than lines.
+    """
+    n = len(words)
+    if not lines:
+        return []
+    if len(lines) == 1 or n <= 1:
+        return [words] + [[] for _ in lines[1:]]
+
+    total_chars = sum(nchars(l) for l in lines) or 1
+    groups: list[list[Word]] = []
+    start = 0
+    cum_chars = 0
+    for li, line in enumerate(lines[:-1]):
+        cum_chars += nchars(line)
+        # words remaining must cover the remaining lines (>=1 each)
+        remaining_lines = len(lines) - li - 1
+        target = round(n * cum_chars / total_chars)
+        lo = max(start + 1, target - search)
+        hi = min(n - remaining_lines, target + search + 1)
+        if hi <= lo:
+            cut = min(max(lo, start + 1), n - remaining_lines)
+        else:
+            best_gap, cut = -1.0, lo
+            for i in range(lo, hi):
+                gap = words[i].start - words[i - 1].end
+                if gap > best_gap:
+                    best_gap, cut = gap, i
+        groups.append(words[start:cut])
+        start = cut
+    groups.append(words[start:])
+    return groups

lyric_align/charmap.py

@@ -0,0 +1,117 @@
+"""Map known characters to sung times by proportional interpolation.
+
+Given the ASR word boundaries for a line and the *known* character string, we
+distribute the characters proportionally across the word-time span. This lets
+karaoke output use the correct lyric text while borrowing only the timing from
+the (possibly misspelled) ASR words.
+"""
+from __future__ import annotations
+
+from .model import Word
+from .normalize import cjk_ratio, nchars
+
+
+MIN_CHAR_DUR = 0.01
+
+
+def char_timings(text: str, words: list[Word]) -> list[dict]:
+    """Return [{"char", "start", "end"}] for each non-space character in text.
+
+    Spaces are skipped (no karaoke syllable). If `words` is empty, returns [].
+
+    Durations are kept strictly positive and non-overlapping: proportional
+    interpolation plus rounding to milliseconds can otherwise collapse a
+    character to zero length, which downstream means a `\\k0` sweep or a TTML span
+    with begin == end.
+    """
+    if not words:
+        return []
+    m = nchars(text)
+    if m == 0:
+        return []
+    bounds = [w.start for w in words] + [words[-1].end]
+    n = len(bounds) - 1
+
+    def t_at(j: float) -> float:
+        pos = j * n / m
+        i = min(int(pos), n - 1)
+        return bounds[i] + (bounds[i + 1] - bounds[i]) * (pos - i)
+
+    # A very short span cannot give every character MIN_CHAR_DUR; share it out.
+    span = max(bounds[-1] - bounds[0], 0.0)
+    min_dur = min(MIN_CHAR_DUR, span / m) if m else 0.0
+
+    out = []
+    j = 0
+    prev_end = bounds[0]
+    for ch in text:
+        if ch == ' ':
+            continue
+        start = max(t_at(j), prev_end)
+        end = max(t_at(j + 1), start + min_dur)
+        out.append({"char": ch, "start": round(start, 3), "end": round(end, 3)})
+        prev_end = end
+        j += 1
+
+    # Rounding can re-introduce a collision at millisecond resolution.
+    for a, b in zip(out, out[1:]):
+        if b["start"] < a["end"]:
+            b["start"] = a["end"]
+        if b["end"] <= b["start"]:
+            b["end"] = round(b["start"] + MIN_CHAR_DUR, 3)
+    if out and out[0]["end"] <= out[0]["start"]:
+        out[0]["end"] = round(out[0]["start"] + MIN_CHAR_DUR, 3)
+    return out
+
+
+def syllable_timings(text: str, chars: list[dict]) -> list[dict]:
+    """Group per-character timings into the units a karaoke format highlights.
+
+    Returns ``[{"text", "start", "end", "space_after"}]``. The grouping differs
+    by script, because what counts as a "syllable" to highlight does:
+
+    - Alphabetic text is grouped on whitespace, so "Amazing grace" highlights two
+      words rather than twelve letters.
+    - CJK text keeps one unit per character, which is how per-character karaoke
+      formats (QQ/NetEase style) treat Chinese and Japanese. Japanese lyrics often
+      contain spaces as phrasing, so splitting on them would give useless chunks.
+
+    ``space_after`` records whether whitespace followed the unit *in the source
+    line*, so a formatter can rebuild the line exactly instead of guessing a
+    separator. The guess is what breaks CJK: a Japanese line is one unit per
+    character yet often carries a phrasing space, so any line-wide "is this
+    spaced text?" test either inserts a space between every character or drops
+    the phrasing space entirely.
+    """
+    if not chars:
+        return []
+    per_char = cjk_ratio(text) >= 0.2
+
+    units: list[list] = []          # [source_text, [timings], space_after]
+    buf_text, buf_times = "", []
+
+    def flush() -> None:
+        nonlocal buf_text, buf_times
+        if buf_times:
+            units.append([buf_text, buf_times, False])
+        buf_text, buf_times = "", []
+
+    it = iter(chars)
+    for ch in text:
+        if ch.isspace():
+            flush()
+            if units:
+                units[-1][2] = True
+            continue
+        try:
+            t = next(it)
+        except StopIteration:
+            break
+        buf_text += ch
+        buf_times.append(t)
+        if per_char:
+            flush()
+    flush()
+
+    return [{"text": t, "start": ts[0]["start"], "end": ts[-1]["end"],
+             "space_after": sp} for t, ts, sp in units]