reelkit-cli 0.10.6 → 0.12.0

package/src/render/worker.ts

@@ -2,70 +2,114 @@
 import { execFile } from "node:child_process";
 import { cp, mkdir, mkdtemp, rm, writeFile, readFile } from "node:fs/promises";
 import { tmpdir } from "node:os";
-import { join, extname } from "node:path";
+import { dirname, join, extname } from "node:path";
 import { promisify } from "node:util";
 import { createCaptureSession, initializeSession, captureFrame, closeCaptureSession, encodeFramesFromDir, processCompositionAudio, muxVideoWithAudio, applyFaststart, type AudioElement } from "@hyperframes/engine";
 import { serveComposition } from "./serve";
 import type { VideoProps } from "../hyperframes/types";
 import type { AudioRecord } from "../hyperframes/frame";
-import type { RenderOptions } from "./render";
+import type { FrameRange, RenderOptions } from "./render";
 
 const run = promisify(execFile);
-const input = JSON.parse(await readFile(process.argv[2]!, "utf8")) as { dir: string; props: VideoProps; mode: "stills" | "video"; output: string; frames: number[]; options?: RenderOptions; chromePath?: string };
-const m = input.props.manifest, fps = { num: m.fps, den: 1 };
-const work = await mkdtemp(join(tmpdir(), "reelkit-capture-"));
-const server = await serveComposition(input.dir);
-let session: Awaited<ReturnType<typeof createCaptureSession>> | undefined;
-try {
-  session = await createCaptureSession(server.url.replace(/\/$/, ""), work, { width: m.width, height: m.height, fps, format: "png", compositionDurationSeconds: m.totalFrames / m.fps }, async (page) => {
-    await page.evaluate(async () => { await (window as unknown as { __reelkitReady: Promise<void> }).__reelkitReady; });
-  }, { chromePath: input.chromePath, forceScreenshot: true, useDrawElement: false, staticFrameDedup: false, enableBrowserPool: false, browserGpuMode: input.options?.uses3D ? "software" : "auto" });
-  await initializeSession(session);
-  await session.page.evaluate(async () => { await (window as unknown as { __reelkitReady: Promise<void> }).__reelkitReady; });
-  const frames = input.mode === "stills" ? input.frames : Array.from({ length: m.totalFrames }, (_, i) => i);
-  for (const frame of frames) {
-    if (!Number.isInteger(frame) || frame < 0 || frame >= m.totalFrames) throw new Error(`Frame ${frame} is outside the composition.`);
-    const captured = await captureFrame(session, frame, frame / m.fps);
-    if (input.mode === "stills") await cp(captured.path, join(input.output, `still-${frame}.png`));
+const input = JSON.parse(await readFile(process.argv[2]!, "utf8")) as {
+  dir?: string; props?: VideoProps; mode: "stills" | "video" | "mix"; output: string; frames: number[];
+  options?: RenderOptions; chromePath?: string; range?: FrameRange; audioSidecar?: string; records?: AudioRecord[]; durationSec?: number;
+};
+
+async function mixRecords(records: AudioRecord[], output: string, durationSec: number, work: string): Promise<boolean> {
+  if (!records.length) return false;
+  const audio: AudioElement[] = [];
+  const localSources = new Map<string, string>();
+  for (const record of records) {
+    let src = localSources.get(record.src);
+    if (!src) {
+      const response = await fetch(record.src); if (!response.ok) throw new Error(`Audio source returned ${response.status}.`);
+      src = join(work, `source-${localSources.size}${extname(new URL(record.src).pathname) || ".media"}`);
+      await writeFile(src, new Uint8Array(await response.arrayBuffer())); localSources.set(record.src, src);
+    }
+    const info = JSON.parse((await run("ffprobe", ["-v", "error", "-show_format", "-of", "json", src])).stdout) as { format: { duration: string } };
+    let end = Math.min(record.end, record.start + Number(info.format.duration) - record.mediaStart);
+    if (record.loop) {
+      const looped = join(work, `loop-${audio.length}.wav`);
+      await run("ffmpeg", ["-nostdin", "-y", "-v", "error", "-stream_loop", "-1", "-i", src, "-t", String(record.end - record.start + record.mediaStart), "-vn", looped]);
+      src = looped; end = record.end;
+    }
+    if (end <= record.start) continue;
+    const keys = [...new Map(record.volumeKeyframes.map((k) => [k.time, k])).values()].sort((a, b) => a.time - b.time);
+    audio.push({ ...record, src, end, volumeKeyframes: keys.length ? keys : undefined });
+  }
+  if (!audio.length) return false;
+  const mixed = output.endsWith(".m4a") ? output : join(work, "audio.m4a");
+  const result = await processCompositionAudio(audio, work, join(work, "audio-work"), mixed, durationSec);
+  if (!result.success) throw new Error(result.error ?? "HyperFrames audio mixing failed.");
+  if (mixed !== output) await cp(mixed, output);
+  return true;
+}
+
+if (input.mode === "mix") {
+  const work = await mkdtemp(join(tmpdir(), "reelkit-mix-"));
+  try {
+    await mkdir(dirname(input.output), { recursive: true });
+    const wrote = await mixRecords(input.records ?? [], input.output, input.durationSec ?? 0, work);
+    if (!wrote) throw new Error("This film has no sound to mix.");
+  } finally {
+    await rm(work, { recursive: true, force: true });
   }
-  if (input.mode === "video") {
-    const silent = join(work, "silent.mp4");
-    const encoded = await encodeFramesFromDir(work, "frame_%06d.png", silent, { fps, width: m.width, height: m.height, codec: "h264", pixelFormat: "yuv420p", quality: input.options?.crf ?? 18, preset: "fast" });
-    if (!encoded.success) throw new Error(encoded.error ?? "HyperFrames encoding failed.");
-    const records = input.options?.silent ? [] : await session.page.evaluate(() => [...(window as unknown as { __reelkitAudio: Map<string, AudioRecord> }).__reelkitAudio.values()]);
-    const audio: AudioElement[] = [];
-    const localSources = new Map<string, string>();
-    for (const record of records) {
-      let src = localSources.get(record.src);
-      if (!src) {
-        const response = await fetch(record.src); if (!response.ok) throw new Error(`Audio source returned ${response.status}.`);
-        src = join(work, `source-${localSources.size}${extname(new URL(record.src).pathname) || ".media"}`);
-        await writeFile(src, new Uint8Array(await response.arrayBuffer())); localSources.set(record.src, src);
+} else {
+  const props = input.props!;
+  const m = props.manifest, fps = { num: m.fps, den: 1 };
+  const work = await mkdtemp(join(tmpdir(), "reelkit-capture-"));
+  const server = await serveComposition(input.dir!);
+  let session: Awaited<ReturnType<typeof createCaptureSession>> | undefined;
+  try {
+    session = await createCaptureSession(server.url.replace(/\/$/, ""), work, { width: m.width, height: m.height, fps, format: "png", compositionDurationSeconds: m.totalFrames / m.fps }, async (page) => {
+      await page.evaluate(async () => { await (window as unknown as { __reelkitReady: Promise<void> }).__reelkitReady; });
+    }, { chromePath: input.chromePath, forceScreenshot: true, useDrawElement: false, staticFrameDedup: false, enableBrowserPool: false, browserGpuMode: input.options?.uses3D ? "software" : "auto" });
+    await initializeSession(session);
+    await session.page.evaluate(async () => { await (window as unknown as { __reelkitReady: Promise<void> }).__reelkitReady; });
+    const from = input.range?.from ?? 0, to = input.range?.to ?? m.totalFrames - 1;
+    const frames = input.mode === "stills" ? input.frames : Array.from({ length: to - from + 1 }, (_, i) => from + i);
+    for (const frame of frames) {
+      if (!Number.isInteger(frame) || frame < 0 || frame >= m.totalFrames) throw new Error(`Frame ${frame} is outside the composition.`);
+      const captured = await captureFrame(session, frame, frame / m.fps);
+      if (input.mode === "stills") await cp(captured.path, join(input.output, `still-${frame}.png`));
+    }
+    if (input.mode === "video") {
+      // The encoder reads frame_000000.png onward. A piece that starts later is renamed into its own folder.
+      let framesDir = work, pattern = "frame_%06d.png";
+      if (from !== 0) {
+        framesDir = join(work, "seq"); await mkdir(framesDir);
+        for (let i = 0; i < frames.length; i++) await cp(join(work, `frame_${String(from + i).padStart(6, "0")}.png`), join(framesDir, `frame_${String(i).padStart(6, "0")}.png`));
       }
-      const info = JSON.parse((await run("ffprobe", ["-v", "error", "-show_format", "-of", "json", src])).stdout) as { format: { duration: string } };
-      let end = Math.min(record.end, record.start + Number(info.format.duration) - record.mediaStart);
-      if (record.loop) {
-        const looped = join(work, `loop-${audio.length}.wav`);
-        await run("ffmpeg", ["-y", "-v", "error", "-stream_loop", "-1", "-i", src, "-t", String(record.end - record.start + record.mediaStart), "-vn", looped]);
-        src = looped; end = record.end;
+      const silent = join(work, "silent.mp4");
+      const encoded = await encodeFramesFromDir(framesDir, pattern, silent, { fps, width: m.width, height: m.height, codec: "h264", pixelFormat: "yuv420p", quality: input.options?.crf ?? 18, preset: "fast" });
+      if (!encoded.success) throw new Error(encoded.error ?? "HyperFrames encoding failed.");
+      const collect = Boolean(input.audioSidecar) || !input.options?.silent;
+      const records: AudioRecord[] = collect ? await session.page.evaluate(() => [...(window as unknown as { __reelkitAudio: Map<string, AudioRecord> }).__reelkitAudio.values()]) : [];
+      if (input.audioSidecar) await writeFile(input.audioSidecar, JSON.stringify(records));
+      if (input.options?.silent || input.range) {
+        await mkdir(join(work, "faststart"));
+        const result = await applyFaststart(silent, input.output);
+        if (!result.success) throw new Error(result.error ?? "Faststart failed.");
+      } else if (records.length) {
+        const mixed = join(work, "audio.m4a");
+        const wrote = await mixRecords(records, mixed, m.totalFrames / m.fps, work);
+        if (!wrote) {
+          await mkdir(join(work, "faststart"));
+          const result = await applyFaststart(silent, input.output);
+          if (!result.success) throw new Error(result.error ?? "Faststart failed.");
+        } else {
+          const muxed = await muxVideoWithAudio(silent, mixed, input.output, undefined, { audioCodec: "aac" });
+          if (!muxed.success) throw new Error(muxed.error ?? "HyperFrames audio mux failed.");
+        }
+      } else {
+        await mkdir(join(work, "faststart"));
+        const result = await applyFaststart(silent, input.output);
+        if (!result.success) throw new Error(result.error ?? "Faststart failed.");
       }
-      if (end <= record.start) continue;
-      const keys = [...new Map(record.volumeKeyframes.map((k) => [k.time, k])).values()].sort((a, b) => a.time - b.time);
-      audio.push({ ...record, src, end, volumeKeyframes: keys.length ? keys : undefined });
-    }
-    if (audio.length) {
-      const mixed = join(work, "audio.m4a");
-      const result = await processCompositionAudio(audio, work, join(work, "audio-work"), mixed, m.totalFrames / m.fps);
-      if (!result.success) throw new Error(result.error ?? "HyperFrames audio mixing failed.");
-      const muxed = await muxVideoWithAudio(silent, mixed, input.output, undefined, { audioCodec: "aac" });
-      if (!muxed.success) throw new Error(muxed.error ?? "HyperFrames audio mux failed.");
-    } else {
-      await mkdir(join(work, "faststart"));
-      const result = await applyFaststart(silent, input.output);
-      if (!result.success) throw new Error(result.error ?? "Faststart failed.");
     }
+  } finally {
+    if (session) await closeCaptureSession(session);
+    await server.close(); await rm(work, { recursive: true, force: true });
   }
-} finally {
-  if (session) await closeCaptureSession(session);
-  await server.close(); await rm(work, { recursive: true, force: true });
 }

package/src/testing/fake-api.ts

@@ -5,7 +5,7 @@ import { createServer } from "node:http";
 import type { AddressInfo } from "node:net";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
-import { MAX_UPLOAD_BYTES, routes, type ErrorCode, type LibraryItem, type RouteName, type Voice } from "../contract";
+import { MAX_TEMPLATES, MAX_UPLOAD_BYTES, TEMPLATE_BUNDLE_TYPE, routes, type ErrorCode, type LibraryItem, type RouteName, type Template, type TemplateFilm, type Voice } from "../contract";
 import { staticCheck } from "../render/validate";
 import { FAKE_SVG, PIXEL_PNG, silentWav } from "./fixtures";
 
@@ -124,6 +124,9 @@ export async function startFakeApi(opts: { voiceoverCharLimit?: number; imageLim
   const secret = () => randomBytes(16).toString("hex");
   let origin = "";
   // Every call makes a new URL for its own blob. The URL says nothing about the item behind it.
+  // A user's templates: the files as they were uploaded, and whether the template was committed.
+  type TemplateJob = { owner: string; record: Template; film: TemplateFilm; committed: boolean; bundle?: Uint8Array; video?: Uint8Array; sources?: Uint8Array; thumbs: (Uint8Array | undefined)[] };
+  const templates = new Map<string, TemplateJob>();
   const fileUrl = (blob: Blob) => { const t = secret(); blobs.set(t, blob); return `${origin}/files/${t}`; };
   const words = (s: string) => new Set(s.toLowerCase().match(/[a-z0-9]+/g) ?? []);
 
@@ -323,6 +326,51 @@ export async function startFakeApi(opts: { voiceoverCharLimit?: number; imageLim
       };
       return job.result;
     },
+    templateStart: (input, { userId }) => {
+      const owner = userId ?? "u-test";
+      if ([...templates.values()].filter((t) => t.owner === owner && t.committed).length >= MAX_TEMPLATES) throw new Fail(400, "invalid_request", `You already keep ${MAX_TEMPLATES} templates. Delete one with \`reelkit template delete <id>\` and push again.`);
+      const id = `tpl-${randomBytes(6).toString("hex")}`;
+      const film = input.film;
+      const record: Template = {
+        id, name: input.name, ...(input.title ? { title: input.title } : {}), createdAt: new Date().toISOString(), bundleBytes: input.bundleBytes,
+        ...(input.videoBytes ? { videoBytes: input.videoBytes } : {}), aspect: film.aspect, durationSec: Math.round((film.totalFrames / film.fps) * 100) / 100,
+        scenes: film.scenes.length, thumbs: input.thumbBytes.length, ...(input.from ? { from: input.from } : {}),
+      };
+      const job: TemplateJob = { owner, record, film, committed: false, thumbs: (input.thumbBytes as number[]).map(() => undefined) };
+      templates.set(id, job);
+      const put = (contentType: string, bytes: number, has: () => boolean, set: (b: Uint8Array) => void) => {
+        const token = secret();
+        uploads.set(token, { contentType, bytes, used: false, sink: { has, put: set } });
+        return `${origin}/upload/${token}`;
+      };
+      return {
+        id, bundleUrl: put(TEMPLATE_BUNDLE_TYPE, input.bundleBytes, () => Boolean(job.bundle), (b) => { job.bundle = b; }),
+        ...(input.videoBytes ? { videoUrl: put("video/mp4", input.videoBytes, () => Boolean(job.video), (b) => { job.video = b; }) } : {}),
+        ...(input.sourcesBytes ? { sourcesUrl: put("application/json", input.sourcesBytes, () => Boolean(job.sources), (b) => { job.sources = b; }) } : {}),
+        thumbUrls: (input.thumbBytes as number[]).map((n, i) => put("image/jpeg", n, () => Boolean(job.thumbs[i]), (b) => { job.thumbs[i] = b; })),
+      };
+    },
+    templateCommit: ({ id }, { userId }) => {
+      const job = templates.get(id);
+      if (!job || job.owner !== (userId ?? "u-test")) throw new Fail(404, "not_found", "No such template.");
+      if (!job.bundle || (job.record.videoBytes && !job.video)) throw new Fail(400, "invalid_request", "The template's files have not been uploaded. Run `reelkit template push` again.");
+      job.committed = true;
+      return { template: job.record };
+    },
+    templateList: (_input, { userId }) => ({
+      templates: [...templates.values()].filter((t) => t.owner === (userId ?? "u-test") && t.committed).map((t) => t.record).sort((a, b) => b.createdAt.localeCompare(a.createdAt)),
+    }),
+    templateGet: ({ id }, { userId }) => {
+      const job = templates.get(id);
+      if (!job || !job.committed || job.owner !== (userId ?? "u-test")) throw new Fail(404, "not_found", "No such template.");
+      return { template: job.record, film: job.film, bundleUrl: fileUrl({ filename: "bundle.tgz", contentType: TEMPLATE_BUNDLE_TYPE, bytes: job.bundle! }) };
+    },
+    templateDelete: ({ id }, { userId }) => {
+      const job = templates.get(id);
+      if (!job || !job.committed || job.owner !== (userId ?? "u-test")) throw new Fail(404, "not_found", "No such template.");
+      templates.delete(id);
+      return { deleted: true as const };
+    },
     publicLibrary: ({ q, kind, page, sort }) => {
       // Published items only. Newest first, or oldest first when asked; with a query, best match first and equal matches newest first.
       const want = q ? words(q) : undefined;

package/tools/voice/README.md

@@ -0,0 +1,26 @@
+# Voice tools
+
+Scripts for getting one narration line right and into a film without disturbing anything else. They are plain Python (numpy, ffmpeg; Pillow for the
+options video; soundfile for `place_line.py replace`). None of them generates speech; only `credits.py --check` talks to a network, and only to read a balance.
+
+| script | what it answers |
+|---|---|
+| `credits.py --takes N --chars C [--check]` | What will this batch cost, and can the account pay for it? Run before generating. |
+| `phonemes.py take.wav --want "m e a f j e n i m"` | What does the take actually say? Speech-to-text spells the expected word even when it was mispronounced; phonemes do not. Exit 2 (not a failure) when the model's packages are not installed. |
+| `pitch_check.py take.wav` | Pitch range, largest step, and the pitch over the last 0.6 s. Verdicts on octave jumps and question-like endings need `praat-parselmouth`. |
+| `ab_video.py --title "..." --out options.mp4 a.mp3:"current" b.mp3:"slower"` | One phone-safe video that plays every take under a big number, with a key file, so the user can answer "3". |
+| `splice.py head.wav tail.wav --head-end 2.68 --tail-start 0.05 --out j.wav` | Join two takes at the quietest point near the cut, at a zero crossing, with a 10 ms crossfade; reports levels, pitch either side and the exact length. |
+| `place_line.py locate track.wav line.mp3 --near 5.9` | Where is this line inside the assembled voice track? Often in several pieces at different offsets. |
+| `place_line.py replace track.wav new.wav --at 8.485 --until 10.45 --from 2.705 --match-old --out t.wav` | Overwrite only that stretch; every other sample stays identical. Reports the margin to the next line. |
+| `onsets.py voice.wav --from 8.4 --to 9.3` | Where does the word really start (rise out of a pause, fricative onset, release after a stop)? For placing a caption within a frame. |
+
+## The order that worked
+
+1. `credits.py` before any generation. Stop and say so if the balance is short.
+2. Generate two or three spellings, two takes each (pointed, unpointed, hyphenated: pointing a whole sentence can change the neighbouring words).
+3. `phonemes.py` and `pitch_check.py` on every take, on the clean voice. Drop the ones that fail; note duration (a stretched or pitched take is too long for its phonemes).
+4. `ab_video.py` with the take in use first. Send the video and the key; wait for a number.
+5. `splice.py` if only part of a take is wanted. Cut inside a pause, a stop closure or just before a fricative.
+6. `place_line.py locate` then `replace`: change only the stretch that changed. Check the margin to the next line; if it is negative, trim a pause inside the take.
+7. `onsets.py` on the new track for every caption near the change; move a caption only if its word's onset moved, and by the same number of frames.
+8. `reelkit diff old.mp4 new.mp4 --allow <frames> --allow-audio <seconds>s-<seconds>s` to prove nothing else moved.

package/tools/voice/_audio.py

@@ -0,0 +1,118 @@
+"""Shared audio helpers for the voice tools. Needs ffmpeg and numpy; nothing here talks to a network."""
+import subprocess
+import sys
+try:
+    import numpy as np
+except ImportError:
+    print("These voice tools need numpy. In a virtual environment: pip install numpy", file=sys.stderr)
+    raise SystemExit(2)
+
+SR = 48000
+
+
+def load(path, sr=SR):
+    """Any audio file as mono float32 at `sr`."""
+    raw = subprocess.run(["ffmpeg", "-nostdin", "-v", "error", "-i", path, "-vn", "-ac", "1", "-ar", str(sr), "-f", "f32le", "-"], capture_output=True, check=True).stdout
+    return np.frombuffer(raw, np.float32).astype(np.float64)
+
+
+def save(path, x, sr=SR):
+    """Mono float samples to a 16-bit wav (or anything ffmpeg writes, by extension)."""
+    pcm = np.clip(np.round(np.asarray(x) * 32767), -32768, 32767).astype("<i2").tobytes()
+    extra = ["-c:a", "libmp3lame", "-b:a", "192k"] if path.lower().endswith(".mp3") else []
+    subprocess.run(["ffmpeg", "-nostdin", "-y", "-v", "error", "-f", "s16le", "-ar", str(sr), "-ac", "1", "-i", "-", *extra, path], input=pcm, check=True)
+
+
+def rms(x):
+    x = np.asarray(x, dtype=np.float64)
+    return float(np.sqrt(np.mean(x * x))) if len(x) else 0.0
+
+
+def db(v):
+    return 20 * np.log10(max(v, 1e-9))
+
+
+def envelope_db(x, sr=SR, win=0.01):
+    """Level of each `win`-second window in dB."""
+    n = max(1, int(sr * win))
+    k = len(x) // n
+    if k == 0:
+        return np.array([])
+    frames = np.asarray(x[: k * n]).reshape(k, n)
+    return 20 * np.log10(np.sqrt((frames ** 2).mean(axis=1)) + 1e-9)
+
+
+def nearest_zero_crossing(x, at, sr=SR, within=0.004):
+    """The sample nearest `at` (a sample index) where the signal crosses zero, searched `within` seconds either side."""
+    r = int(sr * within)
+    lo, hi = max(1, at - r), min(len(x) - 1, at + r)
+    seg = x[lo - 1: hi + 1]
+    idx = np.nonzero(np.signbit(seg[:-1]) != np.signbit(seg[1:]))[0]
+    if len(idx) == 0:
+        return at
+    cand = idx + lo
+    return int(cand[np.argmin(np.abs(cand - at))])
+
+
+def quietest(x, a, b, sr=SR, win=0.006):
+    """The centre (sample index) of the quietest `win` seconds between samples a and b: where a pause or a stop closure is deepest."""
+    n = max(1, int(sr * win))
+    best, where = None, (a + b) // 2
+    for s in range(a, max(a + 1, b - n), max(1, n // 4)):
+        v = rms(x[s: s + n])
+        if best is None or v < best:
+            best, where = v, s + n // 2
+    return where
+
+
+def speech_rms(x, sr=SR, win=0.01):
+    """RMS over the parts that are speech: the 10 ms windows within 20 dB of the loudest. Pauses inside the stretch do not pull the level down."""
+    n = max(1, int(sr * win))
+    k = len(x) // n
+    if k == 0:
+        return rms(x)
+    power = (np.asarray(x[: k * n]).reshape(k, n) ** 2).mean(axis=1)
+    keep = power >= power.max() * 0.01
+    return float(np.sqrt(power[keep].mean())) if keep.any() else 0.0
+
+
+def f0_track(x, sr=SR, lo=70.0, hi=400.0):
+    """Pitch every 10 ms: (times, Hz) of the voiced frames. Uses Praat (the `praat-parselmouth` package) when it is installed, which is the
+    reliable way. Without it, a plain autocorrelation over 40 ms frames is used: the peak is looked for only after the autocorrelation has first
+    fallen below zero, hissing frames (many zero crossings) are skipped, and single-frame outliers are removed by a median of five."""
+    try:
+        import parselmouth
+        p = parselmouth.Sound(np.asarray(x, dtype=np.float64), sr).to_pitch(time_step=0.01, pitch_floor=lo, pitch_ceiling=hi)
+        f = p.selected_array["frequency"]
+        m = f > 0
+        return p.xs()[m], f[m]
+    except ImportError:
+        pass
+    n, hop = int(sr * 0.04), int(sr * 0.01)
+    ts, fs = [], []
+    peak = float(np.abs(x).max()) if len(x) else 0.0
+    for s in range(0, len(x) - n, hop):
+        f = np.asarray(x[s: s + n])
+        if rms(f) < max(0.004, peak * 0.05):
+            continue
+        if np.count_nonzero(np.signbit(f[:-1]) != np.signbit(f[1:])) / 0.04 > 3000:
+            continue
+        f = (f - f.mean()) * np.hanning(n)
+        ac = np.correlate(f, f, "full")[n - 1:]
+        neg = np.nonzero(ac < 0)[0]
+        a, b = max(int(sr / hi), int(neg[0]) if len(neg) else n), min(int(sr / lo), n - 1)
+        if ac[0] <= 0 or b <= a + 2:
+            continue
+        k = a + int(np.argmax(ac[a:b]))
+        if k > a and ac[k] > 0.55 * ac[0]:
+            ts.append(s / sr); fs.append(sr / k)
+    fs = np.array(fs)
+    if len(fs) >= 5:
+        fs = np.array([np.median(fs[max(0, i - 2): i + 3]) for i in range(len(fs))])
+    return np.array(ts), fs
+
+
+def f0_median(x, sr=SR, lo=70.0, hi=400.0):
+    """Median pitch of a stretch; None when it is not voiced. Good enough to compare the two sides of a join."""
+    _, f = f0_track(x, sr, lo, hi)
+    return round(float(np.median(f)), 1) if len(f) >= 3 else None

package/tools/voice/ab_video.py

@@ -0,0 +1,93 @@
+#!/usr/bin/env python3
+"""Build one video that plays several takes of a line one after another, each under a big number, so that someone can listen on a phone and answer "3".
+
+  ab_video.py --title "the line as written" --out options.mp4 take1.mp3:"current take" take2.mp3:"slower" take3.mp3:"new spelling"
+
+Each take plays over a card with its number, the title and its description; 0.7 s of black and silence separates them. A key file is written beside the
+video (options.key.json) listing number, file, length and description, so the answer "3" maps back to a file without guessing. Put the take in use
+FIRST and say so in its description: the listener needs the reference.
+The video is encoded so that a phone plays it: 1280x720 H.264 High level 4.0, TV-range yuv420p tagged BT.709, AAC 48 kHz, index at the front.
+Needs ffmpeg and Pillow. A title in a right-to-left script needs Pillow built with libraqm; without it the title is drawn as given.
+"""
+import argparse, json, os, subprocess, sys, tempfile
+try:
+    from PIL import Image, ImageDraw, ImageFont, features
+except ImportError:
+    Image = ImageDraw = ImageFont = features = None
+
+FONTS = ["/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf", "/System/Library/Fonts/Supplemental/Arial Bold.ttf", "/Library/Fonts/Arial Bold.ttf", "C:/Windows/Fonts/arialbd.ttf"]
+ENC = ["-c:v", "libx264", "-preset", "medium", "-profile:v", "high", "-level:v", "4.0", "-pix_fmt", "yuv420p", "-r", "30", "-color_range", "tv", "-colorspace", "bt709", "-color_trc", "bt709", "-color_primaries", "bt709",
+       "-x264-params", "colorprim=bt709:transfer=bt709:colormatrix=bt709:range=tv", "-c:a", "aac", "-b:a", "160k", "-ar", "48000", "-ac", "2"]
+TAG = "format=yuv420p,setparams=range=tv:color_primaries=bt709:color_trc=bt709:colorspace=bt709"
+
+
+def ff(*args):
+    subprocess.run(["ffmpeg", "-nostdin", "-y", "-v", "error", *args], check=True)
+
+
+def duration(path):
+    return float(subprocess.check_output(["ffprobe", "-v", "error", "-show_entries", "format=duration", "-of", "csv=p=0", path]))
+
+
+def font(path, size):
+    for f in ([path] if path else []) + FONTS:
+        if f and os.path.exists(f):
+            return ImageFont.truetype(f, size)
+    return ImageFont.load_default()
+
+
+def card(path, number, title, sub, font_path, rtl):
+    im = Image.new("RGB", (1280, 720), "#101010")
+    d = ImageDraw.Draw(im)
+    if number is not None:
+        d.text((640, 290), str(number), font=font(font_path, 330), fill="#F2E9D8", anchor="mm")
+        kw = {"direction": "rtl"} if rtl and features.check("raqm") else {}
+        d.text((640, 540), title, font=font(font_path, 46), fill="#E23B2E", anchor="mm", **kw)
+        d.text((640, 620), sub[:90], font=font(font_path, 28), fill="#9a9a9a", anchor="mm")
+    else:
+        d.rectangle([0, 0, 1280, 720], fill="#000000")
+    im.save(path)
+
+
+def build(title, items, out, gap=0.7, font_path=None, rtl=False):
+    tmp = tempfile.mkdtemp(prefix="ab_")
+    card(f"{tmp}/gap.png", None, "", "", font_path, rtl)
+    gap_file = f"{tmp}/gap.mp4"
+    ff("-loop", "1", "-framerate", "30", "-i", f"{tmp}/gap.png", "-f", "lavfi", "-i", "anullsrc=r=48000:cl=stereo", "-t", str(gap), "-vf", TAG, *ENC, gap_file)
+    parts, key = [], []
+    for n, (path, desc) in enumerate(items, 1):
+        d = duration(path)
+        card(f"{tmp}/c{n}.png", n, title, desc, font_path, rtl)
+        seg = f"{tmp}/s{n}.mp4"
+        ff("-loop", "1", "-framerate", "30", "-i", f"{tmp}/c{n}.png", "-i", path, "-af", "apad=pad_dur=0.15", "-t", f"{d + 0.15:.3f}", "-vf", TAG, *ENC, seg)
+        parts += [gap_file, seg]
+        key.append({"n": n, "file": os.path.abspath(path), "dur": round(d, 2), "desc": desc})
+    parts.append(gap_file)
+    with open(f"{tmp}/list.txt", "w") as f:
+        f.write("".join(f"file '{p}'\n" for p in parts))
+    # The pieces share one encoder setting, so they are joined once more through the encoder only to get one clean timeline; the range is not converted again.
+    ff("-f", "concat", "-safe", "0", "-i", f"{tmp}/list.txt", "-vf", TAG, *ENC, "-crf", "22", "-movflags", "+faststart", out)
+    key_path = os.path.splitext(out)[0] + ".key.json"
+    with open(key_path, "w", encoding="utf8") as f:
+        json.dump(key, f, ensure_ascii=False, indent=1)
+    os.sync()
+    errors = subprocess.run(["ffmpeg", "-nostdin", "-v", "error", "-i", out, "-f", "null", "-"], capture_output=True, text=True).stderr.strip()
+    return {"video": out, "key": key_path, "seconds": round(duration(out), 2), "takes": len(items), "decodes_cleanly": errors == ""}
+
+
+if __name__ == "__main__":
+    p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    p.add_argument("takes", nargs="+", help='file:"description", in the order they should play; the take in use first')
+    p.add_argument("--title", required=True); p.add_argument("--out", required=True); p.add_argument("--gap", type=float, default=0.7)
+    p.add_argument("--font", help="a TrueType font that has the title's script"); p.add_argument("--rtl", action="store_true", help="the title is right to left")
+    a = p.parse_args()
+    if Image is None:
+        print("options video needs Pillow. In a virtual environment: pip install pillow", file=sys.stderr)
+        sys.exit(2)
+    items = []
+    for t in a.takes:
+        path, _, desc = t.partition(":")
+        if not os.path.exists(path):
+            sys.exit(f"{path} does not exist")
+        items.append((path, desc or os.path.basename(path)))
+    json.dump(build(a.title, items, a.out, a.gap, a.font, a.rtl), sys.stdout, indent=1); print()

package/tools/voice/credits.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python3
+"""Before generating speech: what it will cost, and whether the account can pay for it.
+
+  credits.py --takes 6 --chars 28            # an estimate only; nothing is sent anywhere
+  credits.py --takes 6 --chars 28 --check    # also asks the provider how many credits are left
+
+The estimate is takes x characters x --rate (credits per character; 2 fits the short expressive takes of this project, about 57 credits for a
+28-character line). With --check the remaining credits are read from ElevenLabs (GET /v1/user/subscription) using the key in the environment variable
+ELEVENLABS_API_KEY. The key is never printed, logged or written. Exit code 1 when the estimate is more than what is left: stop and tell the user
+rather than generating half a set.
+"""
+import argparse, json, os, sys, urllib.request
+
+if __name__ == "__main__":
+    p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    p.add_argument("--takes", type=int, required=True); p.add_argument("--chars", type=int, required=True); p.add_argument("--rate", type=float, default=2.0)
+    p.add_argument("--check", action="store_true")
+    a = p.parse_args()
+    need = round(a.takes * a.chars * a.rate)
+    out = {"takes": a.takes, "chars_each": a.chars, "estimated_credits": need}
+    if a.check:
+        key = os.environ.get("ELEVENLABS_API_KEY")
+        if not key:
+            out["remaining"] = None; out["note"] = "ELEVENLABS_API_KEY is not set: the balance was not checked"
+        else:
+            req = urllib.request.Request("https://api.elevenlabs.io/v1/user/subscription", headers={"xi-api-key": key})
+            try:
+                with urllib.request.urlopen(req, timeout=20) as r:
+                    sub = json.load(r)
+                left = int(sub.get("character_limit", 0)) - int(sub.get("character_count", 0))
+                out["remaining"] = left; out["enough"] = left >= need
+            except Exception as e:  # the message of a failed request must never carry the key
+                out["remaining"] = None; out["note"] = f"the balance could not be read ({type(e).__name__})"
+    print(json.dumps(out))
+    sys.exit(1 if out.get("enough") is False else 0)

package/tools/voice/onsets.py

@@ -0,0 +1,70 @@
+#!/usr/bin/env python3
+"""Where a word really starts in a recording, to place its caption within a frame.
+
+  onsets.py voice.wav --from 8.4 --to 9.3 [--fps 30] [--hf 5500]
+
+Prints the level of each 10 ms window between --from and --to in two bands (everything, and above --hf Hz), then the moments that look like onsets:
+a rise out of a pause, the start of a fricative (s, sh, ts, f: high-band energy with little low-band energy), and the release after a stop closure
+(a dip of 30 ms or more). Speech-to-text word times are good to about 60 ms and are thrown off by pauses; these are good to about 10 ms.
+Place a caption one or two frames before the onset. Each time is given in seconds and as a frame at --fps.
+"""
+import argparse, json, sys
+import numpy as np
+from _audio import SR, envelope_db, load
+
+
+def bands(x, sr, hf):
+    spec = np.fft.rfft(x)
+    freqs = np.fft.rfftfreq(len(x), 1 / sr)
+    hi = np.fft.irfft(np.where(freqs >= hf, spec, 0), len(x))
+    lo = np.fft.irfft(np.where(freqs <= 1500, spec, 0), len(x))
+    return hi, lo
+
+
+def onsets(x, sr=SR, t0=0.0, hf=5500.0, win=0.01):
+    hi, lo = bands(x, sr, hf)
+    full, e_hi, e_lo = envelope_db(x, sr, win), envelope_db(hi, sr, win), envelope_db(lo, sr, win)
+    peak = full.max() if len(full) else 0
+    found = []
+    for i in range(1, len(full)):
+        t = t0 + i * win
+        # out of a pause: at least 30 ms more than 30 dB under the peak, then a rise
+        if i >= 3 and (full[i - 3:i] < peak - 30).all() and full[i] > peak - 24:
+            found.append({"t": round(t, 3), "kind": "rise out of a pause"})
+        # a fricative: the high band comes up while the low band stays down
+        if e_hi[i] > e_hi.max() - 12 and e_hi[i - 1] < e_hi.max() - 18 and e_lo[i] < e_lo.max() - 12:
+            found.append({"t": round(t, 3), "kind": "fricative starts"})
+    # release after a stop closure: a dip of 30 ms or more inside speech
+    i = 1
+    while i < len(full) - 1:
+        if full[i] < peak - 26 and full[i - 1] >= peak - 26:
+            j = i
+            while j < len(full) and full[j] < peak - 26:
+                j += 1
+            if 3 <= j - i <= 12 and j < len(full):
+                found.append({"t": round(t0 + j * win, 3), "kind": f"release after a {int((j - i) * win * 1000)} ms closure"})
+            i = j
+        else:
+            i += 1
+    return full, e_hi, e_lo, sorted(found, key=lambda f: f["t"])
+
+
+if __name__ == "__main__":
+    p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    p.add_argument("file"); p.add_argument("--from", dest="t0", type=float, default=0.0); p.add_argument("--to", dest="t1", type=float)
+    p.add_argument("--fps", type=float, default=30.0); p.add_argument("--hf", type=float, default=5500.0); p.add_argument("--json", action="store_true")
+    a = p.parse_args()
+    x = load(a.file)
+    x = x[int(a.t0 * SR): int(a.t1 * SR) if a.t1 else None]
+    full, hi, lo, found = onsets(x, SR, a.t0, a.hf)
+    for f in found:
+        f["frame"] = round(f["t"] * a.fps, 1)
+    if a.json:
+        json.dump({"onsets": found}, sys.stdout, indent=1); print()
+    else:
+        print("time   all  high  low   (dB per 10 ms)")
+        for i in range(len(full)):
+            print(f"{a.t0 + i * 0.01:6.2f} {full[i]:5.0f} {hi[i]:5.0f} {lo[i]:5.0f}")
+        print("\nonsets:")
+        for f in found:
+            print(f"  {f['t']:.3f} s  frame {f['frame']:.1f}  {f['kind']}")