reelkit-cli 0.10.5 → 0.11.0

package/tools/voice/onsets.py

@@ -0,0 +1,70 @@
+#!/usr/bin/env python3
+"""Where a word really starts in a recording, to place its caption within a frame.
+
+  onsets.py voice.wav --from 8.4 --to 9.3 [--fps 30] [--hf 5500]
+
+Prints the level of each 10 ms window between --from and --to in two bands (everything, and above --hf Hz), then the moments that look like onsets:
+a rise out of a pause, the start of a fricative (s, sh, ts, f: high-band energy with little low-band energy), and the release after a stop closure
+(a dip of 30 ms or more). Speech-to-text word times are good to about 60 ms and are thrown off by pauses; these are good to about 10 ms.
+Place a caption one or two frames before the onset. Each time is given in seconds and as a frame at --fps.
+"""
+import argparse, json, sys
+import numpy as np
+from _audio import SR, envelope_db, load
+
+
+def bands(x, sr, hf):
+    spec = np.fft.rfft(x)
+    freqs = np.fft.rfftfreq(len(x), 1 / sr)
+    hi = np.fft.irfft(np.where(freqs >= hf, spec, 0), len(x))
+    lo = np.fft.irfft(np.where(freqs <= 1500, spec, 0), len(x))
+    return hi, lo
+
+
+def onsets(x, sr=SR, t0=0.0, hf=5500.0, win=0.01):
+    hi, lo = bands(x, sr, hf)
+    full, e_hi, e_lo = envelope_db(x, sr, win), envelope_db(hi, sr, win), envelope_db(lo, sr, win)
+    peak = full.max() if len(full) else 0
+    found = []
+    for i in range(1, len(full)):
+        t = t0 + i * win
+        # out of a pause: at least 30 ms more than 30 dB under the peak, then a rise
+        if i >= 3 and (full[i - 3:i] < peak - 30).all() and full[i] > peak - 24:
+            found.append({"t": round(t, 3), "kind": "rise out of a pause"})
+        # a fricative: the high band comes up while the low band stays down
+        if e_hi[i] > e_hi.max() - 12 and e_hi[i - 1] < e_hi.max() - 18 and e_lo[i] < e_lo.max() - 12:
+            found.append({"t": round(t, 3), "kind": "fricative starts"})
+    # release after a stop closure: a dip of 30 ms or more inside speech
+    i = 1
+    while i < len(full) - 1:
+        if full[i] < peak - 26 and full[i - 1] >= peak - 26:
+            j = i
+            while j < len(full) and full[j] < peak - 26:
+                j += 1
+            if 3 <= j - i <= 12 and j < len(full):
+                found.append({"t": round(t0 + j * win, 3), "kind": f"release after a {int((j - i) * win * 1000)} ms closure"})
+            i = j
+        else:
+            i += 1
+    return full, e_hi, e_lo, sorted(found, key=lambda f: f["t"])
+
+
+if __name__ == "__main__":
+    p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    p.add_argument("file"); p.add_argument("--from", dest="t0", type=float, default=0.0); p.add_argument("--to", dest="t1", type=float)
+    p.add_argument("--fps", type=float, default=30.0); p.add_argument("--hf", type=float, default=5500.0); p.add_argument("--json", action="store_true")
+    a = p.parse_args()
+    x = load(a.file)
+    x = x[int(a.t0 * SR): int(a.t1 * SR) if a.t1 else None]
+    full, hi, lo, found = onsets(x, SR, a.t0, a.hf)
+    for f in found:
+        f["frame"] = round(f["t"] * a.fps, 1)
+    if a.json:
+        json.dump({"onsets": found}, sys.stdout, indent=1); print()
+    else:
+        print("time   all  high  low   (dB per 10 ms)")
+        for i in range(len(full)):
+            print(f"{a.t0 + i * 0.01:6.2f} {full[i]:5.0f} {hi[i]:5.0f} {lo[i]:5.0f}")
+        print("\nonsets:")
+        for f in found:
+            print(f"  {f['t']:.3f} s  frame {f['frame']:.1f}  {f['kind']}")

package/tools/voice/phonemes.py

@@ -0,0 +1,85 @@
+#!/usr/bin/env python3
+"""What a recording actually says, as phonemes, and whether it says what was meant.
+
+  phonemes.py take1.wav take2.wav [--want "m e a f j e n i m"] [--from 1.2 --to 2.3]
+
+Speech-to-text writes the word it expects, so a mispronounced word still comes back spelled correctly. A phoneme recogniser (wav2vec2 trained on
+espeak phonemes, any language) writes what was said. With --want, each file passes when the wanted phonemes appear in order (spaces and length marks
+are ignored); the nearest stretch is shown when they do not. Also printed: the length of the stretch, so an over-processed take (stretched, pitched)
+shows up as too long for its phonemes.
+Check the clean voice, not the mix: music degrades the result. Needs `torch`, `transformers` and `huggingface_hub`; the model (about 1.2 GB) is
+downloaded once. If they are missing, the script says so and exits with code 2 rather than failing a build.
+"""
+import argparse, json, subprocess, sys
+
+np = None  # numpy, imported with the other optional packages so that a missing one is a sentence, not a traceback
+
+MODEL = "facebook/wav2vec2-xlsr-53-espeak-cv-ft"
+
+
+def load16(path, t0=None, t1=None):
+    cut = (["-ss", str(t0)] if t0 else []) + (["-to", str(t1)] if t1 else [])
+    raw = subprocess.check_output(["ffmpeg", "-nostdin", "-v", "error", *cut, "-i", path, "-ac", "1", "-ar", "16000", "-f", "f32le", "-"])
+    a = np.frombuffer(raw, np.float32)
+    return (a - a.mean()) / (a.std() + 1e-7)
+
+
+def squash(s):
+    return "".join(c for c in s if c not in " ːˈˌ")
+
+
+def nearest(got, want):
+    """The stretch of `got` closest to `want` by edit distance, and that distance."""
+    g, w = squash(got), squash(want)
+    best = (len(w), "")
+    for i in range(len(g)):
+        for n in (len(w) - 1, len(w), len(w) + 1):
+            s = g[i: i + n]
+            if not s:
+                continue
+            prev = list(range(len(w) + 1))
+            for a_ in s:
+                cur = [prev[0] + 1]
+                for j, b_ in enumerate(w, 1):
+                    cur.append(min(prev[j] + 1, cur[j - 1] + 1, prev[j - 1] + (a_ != b_)))
+                prev = cur
+            if prev[-1] < best[0]:
+                best = (prev[-1], s)
+    return best
+
+
+if __name__ == "__main__":
+    p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    p.add_argument("files", nargs="+"); p.add_argument("--want"); p.add_argument("--from", dest="t0", type=float); p.add_argument("--to", dest="t1", type=float)
+    a = p.parse_args()
+    try:
+        import numpy as np
+        import torch
+        from transformers import Wav2Vec2ForCTC
+        from huggingface_hub import hf_hub_download
+    except ImportError as e:
+        print(f"phoneme check skipped: {e.name} is not installed. In a virtual environment: pip install torch transformers huggingface_hub", file=sys.stderr)
+        sys.exit(2)
+    model = Wav2Vec2ForCTC.from_pretrained(MODEL).eval()
+    vocab = json.load(open(hf_hub_download(MODEL, "vocab.json")))
+    inv = {v: k for k, v in vocab.items()}
+    ok = True
+    for f in a.files:
+        x = load16(f, a.t0, a.t1)
+        with torch.no_grad():
+            ids = model(torch.tensor(x)[None]).logits[0].argmax(-1).tolist()
+        out, prev = [], None
+        for i in ids:
+            if i != prev and inv[i] not in ("<pad>", "<s>", "</s>", "<unk>", "|"):
+                out.append(inv[i])
+            prev = i
+        said = " ".join(out)
+        row = {"file": f, "seconds": round(len(x) / 16000, 2), "phonemes": said}
+        if a.want:
+            row["has_wanted"] = squash(a.want) in squash(said)
+            if not row["has_wanted"]:
+                dist, near = nearest(said, a.want)
+                row["nearest"] = near; row["distance"] = dist
+                ok = False
+        print(json.dumps(row, ensure_ascii=False))
+    sys.exit(0 if ok else 1)

package/tools/voice/pitch_check.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+"""Does a take jump an octave, or end like a question?
+
+  pitch_check.py take1.wav take2.wav [--floor 70 --ceil 400]
+
+For each file: the pitch range, the largest step between neighbouring 10 ms frames (a ratio near 2 is an octave jump, the commonest fault of a
+synthetic voice), and the pitch over the last voiced 0.6 s in four quarters. A statement falls; a line whose last quarter is clearly above its
+first reads as a question. The verdicts (octave_jump, rises_at_end) are given only when Praat is installed (`praat-parselmouth`); the built-in
+fallback tracker makes octave errors of its own, so with it only the numbers are printed and the exit code is 0.
+"""
+import argparse, json, sys
+import numpy as np
+from _audio import SR, f0_track, load
+
+
+def has_praat():
+    try:
+        import parselmouth  # noqa: F401
+        return True
+    except ImportError:
+        return False
+
+
+def check(path, lo, hi):
+    t, f = f0_track(load(path), SR, lo, hi)
+    if len(f) < 8:
+        return {"file": path, "voiced": False}
+    steps = [max(f[i + 1] / f[i], f[i] / f[i + 1]) for i in range(len(f) - 1) if t[i + 1] - t[i] < 0.025]
+    last = f[t > t[-1] - 0.6]
+    q = [float(np.median(p)) for p in np.array_split(last, 4) if len(p)]
+    worst = max(steps) if steps else 1.0
+    out = {"file": path, "voiced": True, "range_hz": [round(float(f.min())), round(float(f.max()))], "max_step_ratio": round(float(worst), 2), "last_quarters_hz": [round(v) for v in q]}
+    if has_praat():
+        out["octave_jump"] = bool(1.8 < worst < 2.25)
+        out["rises_at_end"] = bool(len(q) == 4 and q[-1] > q[0] * 1.12)
+    else:
+        # The plain tracker makes octave errors of its own, so its steps prove nothing either way.
+        out["tracker"] = "fallback autocorrelation: coarse. Install praat-parselmouth (pip, in a virtual environment) for a verdict on octave jumps and rising endings."
+    return out
+
+
+if __name__ == "__main__":
+    p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    p.add_argument("files", nargs="+"); p.add_argument("--floor", type=float, default=70.0); p.add_argument("--ceil", type=float, default=400.0)
+    a = p.parse_args()
+    bad = False
+    for f in a.files:
+        r = check(f, a.floor, a.ceil)
+        bad = bad or r.get("octave_jump") or r.get("rises_at_end")
+        print(json.dumps(r, ensure_ascii=False))
+    sys.exit(1 if bad else 0)

package/tools/voice/place_line.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+"""Find where a line sits inside an assembled voice track, and replace part of it without touching any other sample.
+
+  place_line.py locate track.wav line.mp3 [--near 5.9]
+  place_line.py replace track.wav new.wav --at 8.485 --until 10.45 --from 2.705 [--match-old] --out track_new.wav
+
+locate: slides 0.4 s windows of the line over the track and prints, for each, the offset where it fits best and how well. A line is often in the track
+in several pieces with different offsets (a pause was shortened when the track was built), so one offset for the whole line is not enough.
+replace: writes new[--from:] into the track from --at, silence up to --until, and leaves every sample outside --at..--until exactly as it was.
+With --match-old the new audio is scaled to the RMS level of what it replaces. The report says where the new audio ends and how long the silence after it is.
+"""
+import argparse, json, sys
+import numpy as np
+from _audio import SR, load, rms
+
+
+def locate(track, line, near=None, sr=SR, win=0.4, step=0.6, reach=1.0):
+    out = []
+    t = 0.3
+    while t + win < len(line) / sr:
+        w = line[int(t * sr): int((t + win) * sr)]
+        if rms(w) > 0.005:
+            centre = (near if near is not None else 0.0) + t
+            lo = max(0, int((centre - reach) * sr)) if near is not None else 0
+            hi = min(len(track), int((centre + reach + win) * sr)) if near is not None else len(track)
+            seg = track[lo:hi]
+            c = np.correlate(seg, w, "valid")
+            energy = np.sqrt(np.convolve(seg ** 2, np.ones(len(w)), "valid") * (w ** 2).sum()) + 1e-12
+            k = int(np.argmax(np.abs(c) / energy))
+            out.append({"line_s": round(t, 2), "track_s": round((lo + k) / sr, 4), "line_start_in_track_s": round((lo + k) / sr - t, 4), "fit": round(float(abs(c[k]) / energy[k]), 3)})
+        t += step
+    return out
+
+
+if __name__ == "__main__":
+    p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    sub = p.add_subparsers(dest="cmd", required=True)
+    l = sub.add_parser("locate"); l.add_argument("track"); l.add_argument("line"); l.add_argument("--near", type=float)
+    r = sub.add_parser("replace"); r.add_argument("track"); r.add_argument("new"); r.add_argument("--at", type=float, required=True); r.add_argument("--until", type=float, required=True)
+    r.add_argument("--from", dest="src_from", type=float, default=0.0); r.add_argument("--to", dest="src_to", type=float); r.add_argument("--match-old", action="store_true"); r.add_argument("--gain", type=float); r.add_argument("--out", required=True)
+    a = p.parse_args()
+    if a.cmd == "locate":
+        rows = locate(load(a.track), load(a.line), a.near)
+        offsets = sorted({round(x["line_start_in_track_s"], 2) for x in rows if x["fit"] > 0.8})
+        json.dump({"windows": rows, "offsets_found": offsets, "note": "more than one offset means the line is in the track in pieces" if len(offsets) > 1 else "one offset"}, sys.stdout, indent=1); print()
+    else:
+        import soundfile as sf
+        x, sr = sf.read(a.track, dtype="int16", always_2d=True)
+        new = load(a.new, sr)[int(a.src_from * sr): int(a.src_to * sr) if a.src_to else None]
+        s0, s1 = int(round(a.at * sr)), int(round(a.until * sr))
+        if len(new) > s1 - s0:
+            sys.exit(f"the new audio is {len(new) / sr:.3f} s but the slot is {(s1 - s0) / sr:.3f} s: trim a pause inside the take, do not move later lines")
+        old = x[s0:s1, 0].astype(np.float64) / 32768
+        active = lambda v: v[np.abs(v) > 0.003]
+        gain = a.gain if a.gain else (rms(active(old)) / max(rms(active(new)), 1e-9) if a.match_old else 1.0)
+        seg = np.zeros(s1 - s0); seg[: len(new)] = new * gain
+        n = int(0.004 * sr); seg[:n] *= np.linspace(0, 1, n)
+        q = np.clip(np.round(seg * 32767), -32768, 32767).astype("int16")
+        out = x.copy()
+        for ch in range(out.shape[1]):
+            out[s0:s1, ch] = q
+        assert np.array_equal(out[:s0], x[:s0]) and np.array_equal(out[s1:], x[s1:])
+        sf.write(a.out, out, sr, subtype="PCM_16")
+        after = np.nonzero(np.abs(x[s1:, 0]) > 30)[0]
+        end = (s0 + len(new)) / sr
+        json.dump({"written_s": [a.at, a.until], "new_audio_ends_s": round(end, 3), "next_sound_at_s": round((s1 + after[0]) / sr, 3) if len(after) else None,
+                   "margin_s": round((s1 + after[0]) / sr - end, 3) if len(after) else None, "gain": round(gain, 3), "peak": round(float(np.abs(seg).max()), 3), "samples_outside_unchanged": True}, sys.stdout, indent=1); print()

package/tools/voice/splice.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python3
+"""Join two recordings at a quiet point, at a zero crossing, with a short crossfade, and report the levels and the pitch either side.
+
+  splice.py head.wav tail.wav --head-end 2.68 --tail-start 0.05 --out joined.wav [--gap 0.04] [--fade-ms 10] [--match-rms]
+
+--head-end / --tail-start are where you want to cut, in seconds. Each is moved to the quietest point within --search seconds (a pause, the closure of a
+stop consonant, or just before a fricative starts), and then to the nearest zero crossing, so the join has no click.
+The crossfade is head[-n:] * fade-out + tail[:n] * fade-in over the same n samples, so the joined length is exactly head + gap + tail - n.
+--match-rms scales the tail so that its level over --rms-window seconds next to the join equals the head's (a ratio of RMS levels over the spoken parts, pauses left out: a projection gain
+reads low when the two takes differ in phase).
+"""
+import argparse, json, sys
+import numpy as np
+from _audio import SR, db, f0_median, load, nearest_zero_crossing, quietest, rms, save, speech_rms
+
+
+def splice(head, tail, head_end, tail_start, sr=SR, gap=0.0, fade_ms=10.0, search=0.03, match=False, rms_window=0.5):
+    a = quietest(head, max(0, int((head_end - search) * sr)), min(len(head), int((head_end + search) * sr)), sr)
+    a = nearest_zero_crossing(head, a, sr)
+    b = quietest(tail, max(0, int((tail_start - search) * sr)), min(len(tail), int((tail_start + search) * sr)), sr)
+    b = nearest_zero_crossing(tail, b, sr)
+    h, t = head[:a].copy(), tail[b:].copy()
+    w = int(rms_window * sr)
+    level_h, level_t = speech_rms(h[-w:], sr), speech_rms(t[:w], sr)
+    gain = level_h / level_t if match and level_t > 0 else 1.0
+    t *= gain
+    n = max(1, int(sr * fade_ms / 1000))
+    n = min(n, len(h), len(t))
+    ramp = np.linspace(0.0, 1.0, n)
+    if gap > 0:
+        h[-n:] *= ramp[::-1]
+        t[:n] *= ramp
+        out = np.concatenate([h, np.zeros(int(gap * sr)), t])
+    else:
+        mix = h[-n:] * ramp[::-1] + t[:n] * ramp
+        out = np.concatenate([h[:-n], mix, t[n:]])
+    report = {
+        "head_cut_s": round(a / sr, 4), "tail_cut_s": round(b / sr, 4), "join_at_s": round((len(h) - (0 if gap > 0 else n)) / sr, 4),
+        "length_s": round(len(out) / sr, 4), "expected_length_s": round((len(h) + int(gap * sr) + len(t) - (0 if gap > 0 else n)) / sr, 4),
+        "fade_ms": round(1000 * n / sr, 1), "gap_s": gap,
+        "level_at_cut_db": {"head": round(db(rms(head[max(0, a - 240): a + 240])), 1), "tail": round(db(rms(tail[max(0, b - 240): b + 240])), 1)},
+        "rms_db": {"head_side": round(db(level_h), 1), "tail_side_before_gain": round(db(level_t), 1), "gain_applied": round(gain, 3)},
+        "f0_hz": {"head_side": f0_median(h[-w:], sr), "tail_side": f0_median(t[:w], sr)},
+        "peak": round(float(np.abs(out).max()), 3),
+    }
+    f = report["f0_hz"]
+    if f["head_side"] and f["tail_side"]:
+        ratio = max(f["head_side"], f["tail_side"]) / min(f["head_side"], f["tail_side"])
+        report["f0_ratio"] = round(ratio, 2)
+        report["warnings"] = (["pitch differs by more than 25% across the join: listen for a jump"] if ratio > 1.25 else []) + (["pitch nearly doubles across the join: an octave jump"] if 1.8 < ratio < 2.2 else [])
+    if report["level_at_cut_db"]["head"] > -35 or report["level_at_cut_db"]["tail"] > -35:
+        report.setdefault("warnings", []).append("a cut point is not in a quiet place (above -35 dB): move it into a pause or a stop closure")
+    return out, report
+
+
+if __name__ == "__main__":
+    p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    p.add_argument("head"); p.add_argument("tail")
+    p.add_argument("--head-end", type=float, required=True); p.add_argument("--tail-start", type=float, default=0.0)
+    p.add_argument("--out", required=True); p.add_argument("--gap", type=float, default=0.0); p.add_argument("--fade-ms", type=float, default=10.0)
+    p.add_argument("--search", type=float, default=0.03); p.add_argument("--match-rms", action="store_true"); p.add_argument("--rms-window", type=float, default=0.5)
+    a = p.parse_args()
+    out, report = splice(load(a.head), load(a.tail), a.head_end, a.tail_start, gap=a.gap, fade_ms=a.fade_ms, search=a.search, match=a.match_rms, rms_window=a.rms_window)
+    save(a.out, out)
+    json.dump(report, sys.stdout, indent=1, ensure_ascii=False); print()