reelkit-cli 0.10.5 → 0.11.0

package/src/render/chunked.ts

@@ -0,0 +1,139 @@
+// Rendering a long film in pieces that survive a crash. Each piece is a silent H.264 file of a range of frames, written whole or not at all; a piece
+// that is already on disk, reads back with the right number of frames and has its audio sidecar is not rendered again. The pieces are joined
+// without re-encoding. The sound is mixed once from those sidecars and laid under them. The planning here is pure; the rendering is handed in.
+import { createHash } from "node:crypto";
+import { existsSync, mkdirSync, readFileSync, readdirSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { syncFile } from "../media/ffmpeg";
+import { tool } from "../project/refmeasure";
+
+export const DEFAULT_CHUNK_FRAMES = 300;
+export const MAX_ATTEMPTS = 3;
+
+export type Chunk = { index: number; from: number; to: number; file: string };
+
+// Frame ranges (inclusive) that cover the film.
+export function planChunks(totalFrames: number, chunkFrames = DEFAULT_CHUNK_FRAMES): Chunk[] {
+  const size = Math.max(1, Math.floor(chunkFrames));
+  const out: Chunk[] = [];
+  for (let from = 0, index = 0; from < totalFrames; from += size, index++) {
+    const to = Math.min(totalFrames, from + size) - 1;
+    out.push({ index, from, to, file: `chunk-${String(from).padStart(6, "0")}-${String(to).padStart(6, "0")}.mp4` });
+  }
+  return out;
+}
+
+// What the pieces were rendered from. A piece made from other source, other media or another size is not reused.
+export function renderKey(parts: { sources: Record<string, string>; manifest: unknown; media: Record<string, number>; chunkFrames: number }): string {
+  const h = createHash("sha256");
+  for (const name of Object.keys(parts.sources).sort()) h.update(name).update("\0").update(parts.sources[name]!).update("\0");
+  h.update(JSON.stringify(parts.manifest)).update("\0");
+  for (const name of Object.keys(parts.media).sort()) h.update(`${name}:${parts.media[name]}\0`);
+  h.update(String(parts.chunkFrames));
+  return h.digest("hex").slice(0, 16);
+}
+
+// A failure of the browser rather than of the composition: worth another attempt.
+export const isBrowserFailure = (e: unknown): boolean => /browser|Target closed|ProtocolError|Protocol error|Navigation|WebSocket|ECONNRESET|ECONNREFUSED|Timed out|timeout|crash|disconnected|Session closed/i.test(e instanceof Error ? e.message : String(e));
+
+// Runs fn, and again after a browser failure, up to `attempts` times in all. An error of the composition itself is thrown at once.
+export async function withRetry<T>(fn: (attempt: number) => Promise<T>, opts: { attempts?: number; onRetry?: (attempt: number, error: unknown) => void; wait?: (ms: number) => Promise<void> } = {}): Promise<T> {
+  const attempts = opts.attempts ?? MAX_ATTEMPTS;
+  for (let attempt = 1; ; attempt++) {
+    try { return await fn(attempt); }
+    catch (e) {
+      if (attempt >= attempts || !isBrowserFailure(e)) throw e;
+      opts.onRetry?.(attempt, e);
+      await (opts.wait ?? ((ms) => new Promise((r) => setTimeout(r, ms))))(1500 * attempt);
+    }
+  }
+}
+
+export type ChunkDeps = {
+  // Renders frames from..to (inclusive), silent, to `out`. May also write `${out}.audio.json`.
+  renderRange: (from: number, to: number, out: string) => Promise<void>;
+  // Renders the whole film's sound to `out` (a wav or m4a); resolves false when the film has none.
+  renderSound: (out: string) => Promise<boolean>;
+  // How many frames a finished piece holds; undefined when it cannot be read.
+  countFrames: (file: string) => Promise<number | undefined>;
+  log: (line: string) => void;
+  concat?: (list: string, sound: string | undefined, out: string) => Promise<void>;
+  // Extra reason to render a piece again (for example its audio sidecar is missing).
+  reuseExtra?: (file: string) => boolean;
+};
+
+export type CapturedAudio = {
+  id: string; src: string; start: number; end: number; mediaStart: number; layer: number; volume: number;
+  volumeKeyframes: { time: number; volume: number }[]; type: "audio" | "video"; loop: boolean;
+};
+
+// Pieces of one film are captured in separate browser sessions, so the same clip can come back under different ids.
+// Merge by where it sits, and keep every volume sample.
+export function mergeAudioRecords(lists: CapturedAudio[][]): CapturedAudio[] {
+  const byKey = new Map<string, CapturedAudio>();
+  for (const list of lists) for (const record of list) {
+    const key = `${record.type}|${record.src}|${record.start}|${record.mediaStart}|${record.loop}`;
+    const prev = byKey.get(key);
+    if (!prev) { byKey.set(key, { ...record, volumeKeyframes: [...record.volumeKeyframes] }); continue; }
+    prev.end = Math.max(prev.end, record.end);
+    prev.volumeKeyframes.push(...record.volumeKeyframes);
+  }
+  for (const record of byKey.values()) {
+    const seen = new Map<number, { time: number; volume: number }>();
+    for (const key of record.volumeKeyframes) seen.set(key.time, key);
+    record.volumeKeyframes = [...seen.values()].sort((a, b) => a.time - b.time);
+  }
+  return [...byKey.values()];
+}
+
+export async function countFrames(file: string): Promise<number | undefined> {
+  try {
+    const { stdout } = await tool("ffprobe", ["-v", "error", "-select_streams", "v:0", "-count_packets", "-show_entries", "stream=nb_read_packets", "-of", "csv=p=0", file]);
+    const n = Number(stdout.toString("utf8").trim().split(/[,\n]/)[0]);
+    return Number.isFinite(n) ? n : undefined;
+  } catch { return undefined; }
+}
+
+// Joins the pieces without touching their pictures and lays the sound under them.
+export async function concatChunks(list: string, sound: string | undefined, out: string): Promise<void> {
+  await tool("ffmpeg", ["-nostdin", "-y", "-v", "error", "-f", "concat", "-safe", "0", "-i", list, ...(sound ? ["-i", sound] : []), "-map", "0:v:0", ...(sound ? ["-map", "1:a:0", "-c:a", "aac", "-b:a", "320k", "-ar", "48000"] : []), "-c:v", "copy", "-movflags", "+faststart", out]);
+}
+
+export type ChunkedResult = { chunks: number; reused: number; rendered: number; retries: number };
+
+// Renders whatever pieces are missing into `dir`, then assembles `outPath`. Safe to run again after any interruption: finished pieces are kept.
+export async function renderChunked(dir: string, key: string, totalFrames: number, chunkFrames: number, outPath: string, deps: ChunkDeps): Promise<ChunkedResult> {
+  mkdirSync(dir, { recursive: true });
+  const keyFile = join(dir, "key");
+  // Pieces of another version of the film are of no use: start clean.
+  if (existsSync(keyFile) && readFileSync(keyFile, "utf8").trim() !== key) { for (const f of readdirSync(dir)) rmSync(join(dir, f), { force: true, recursive: true }); }
+  writeFileSync(keyFile, key);
+  const chunks = planChunks(totalFrames, chunkFrames);
+  let reused = 0, rendered = 0, retries = 0;
+  for (const c of chunks) {
+    const file = join(dir, c.file);
+    const want = c.to - c.from + 1;
+    if (existsSync(file) && statSync(file).size > 0 && (await deps.countFrames(file)) === want && (deps.reuseExtra?.(file) ?? true)) { reused++; continue; }
+    const part = `${file}.part.mp4`;
+    const partAudio = `${part}.audio.json`;
+    await withRetry(async () => { rmSync(part, { force: true }); rmSync(partAudio, { force: true }); await deps.renderRange(c.from, c.to, part); }, { onRetry: (n, e) => { retries++; deps.log(`The browser failed on frames ${c.from}-${c.to} (${(e instanceof Error ? e.message : String(e)).split("\n")[0]}); trying again (${n + 1} of ${MAX_ATTEMPTS}).`); } });
+    const got = await deps.countFrames(part);
+    if (got !== want) throw new Error(`frames ${c.from}-${c.to} rendered as ${got ?? "an unreadable file"} instead of ${want} frames`);
+    syncFile(part);
+    renameSync(part, file);
+    if (existsSync(partAudio)) { syncFile(partAudio); renameSync(partAudio, `${file}.audio.json`); }
+    rendered++;
+    deps.log(`Frames ${c.from}-${c.to} done (${c.index + 1} of ${chunks.length}).`);
+  }
+  const soundFile = join(dir, "sound.m4a");
+  let hasSound = existsSync(soundFile) && statSync(soundFile).size > 44;
+  if (!hasSound) {
+    const part = join(dir, "sound.part.m4a");
+    hasSound = await withRetry(() => deps.renderSound(part), { onRetry: () => { retries++; } });
+    if (hasSound) { syncFile(part); renameSync(part, soundFile); } else rmSync(part, { force: true });
+  }
+  const list = join(dir, "list.txt");
+  writeFileSync(list, chunks.map((c) => `file '${join(dir, c.file).replace(/'/g, "'\\''")}'`).join("\n") + "\n");
+  await (deps.concat ?? concatChunks)(list, hasSound ? soundFile : undefined, outPath);
+  return { chunks: chunks.length, reused, rendered, retries };
+}

package/src/render/render.ts

@@ -45,6 +45,7 @@ export async function disposeBundle(dir: string): Promise<void> {
 export type EnsureBrowser = (onDownload: () => void) => Promise<void>;
 export const GL_BACKEND = "software" as const;
 export type RenderOptions = { uses3D?: boolean; silent?: boolean; crf?: number };
+export type FrameRange = { from: number; to: number };
 export function browserPath(): string | undefined {
   if (process.env.REELKIT_CHROME_PATH) return process.env.REELKIT_CHROME_PATH;
   const paths = process.platform === "darwin" ? ["/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", "/Applications/Chromium.app/Contents/MacOS/Chromium"] : process.platform === "linux" ? ["/usr/bin/chromium", "/usr/bin/chromium-browser", "/usr/bin/google-chrome"] : [];
@@ -63,12 +64,12 @@ export const ensureRenderBrowser: EnsureBrowser = async (onDownload) => {
   const browser = await install({ browser: Browser.CHROMEHEADLESSSHELL, buildId: await resolveBuildId(Browser.CHROMEHEADLESSSHELL, platform, "stable"), cacheDir: join(PKG_ROOT, ".browser"), platform });
   process.env.REELKIT_CHROME_PATH = browser.executablePath;
 };
-async function render(dir: string, props: VideoProps, mode: "stills" | "video", output: string, frames: number[], options?: RenderOptions) {
+async function render(dir: string, props: VideoProps, mode: "stills" | "video", output: string, frames: number[], options?: RenderOptions, extra?: { range?: FrameRange; audioSidecar?: string }) {
   await ensureRenderBrowser(() => {});
   await writeFile(join(dir, "props.js"), `window.__reelkitProps=${JSON.stringify(props).replace(/</g, "\\u003c")};`);
   // Isolate engine diagnostics from the CLI's JSON stdout.
   const request = join(dir, "render-request.json");
-  await writeFile(request, JSON.stringify({ dir, props, mode, output, frames, options, chromePath: browserPath() }));
+  await writeFile(request, JSON.stringify({ dir, props, mode, output, frames, options, chromePath: browserPath(), ...extra }));
   await run(process.execPath, ["--import", join(packageDir("tsx"), "dist/loader.mjs"), join(PKG_ROOT, "src/render/worker.ts"), request], { maxBuffer: 16 * 1024 * 1024 });
 }
 export async function renderStills(dir: string, props: VideoProps, frames: number[], outDir: string, options?: RenderOptions) {
@@ -77,3 +78,19 @@ export async function renderStills(dir: string, props: VideoProps, frames: numbe
 export async function renderVideo(dir: string, props: VideoProps, outPath: string, options?: RenderOptions) {
   await mkdir(dirname(outPath), { recursive: true }); await render(dir, props, "video", outPath, [], options);
 }
+// One inclusive range of frames as a silent file, plus the audio records those frames registered (`${outPath}.audio.json`).
+export async function renderVideoRange(dir: string, props: VideoProps, outPath: string, from: number, to: number, options?: RenderOptions) {
+  await mkdir(dirname(outPath), { recursive: true });
+  await render(dir, props, "video", outPath, [], { ...options, silent: true }, { range: { from, to }, audioSidecar: `${outPath}.audio.json` });
+}
+// Mix audio records already collected from the pieces. No picture is drawn.
+export async function mixCapturedAudio(records: unknown[], outPath: string, durationSec: number) {
+  const dir = await mkdtemp(join(tmpdir(), "reelkit-mix-"));
+  try {
+    const request = join(dir, "render-request.json");
+    await writeFile(request, JSON.stringify({ mode: "mix", output: outPath, records, durationSec, frames: [] }));
+    await run(process.execPath, ["--import", join(packageDir("tsx"), "dist/loader.mjs"), join(PKG_ROOT, "src/render/worker.ts"), request], { maxBuffer: 16 * 1024 * 1024 });
+  } finally {
+    await rm(dir, { recursive: true, force: true });
+  }
+}

package/src/render/worker.ts

@@ -2,70 +2,114 @@
 import { execFile } from "node:child_process";
 import { cp, mkdir, mkdtemp, rm, writeFile, readFile } from "node:fs/promises";
 import { tmpdir } from "node:os";
-import { join, extname } from "node:path";
+import { dirname, join, extname } from "node:path";
 import { promisify } from "node:util";
 import { createCaptureSession, initializeSession, captureFrame, closeCaptureSession, encodeFramesFromDir, processCompositionAudio, muxVideoWithAudio, applyFaststart, type AudioElement } from "@hyperframes/engine";
 import { serveComposition } from "./serve";
 import type { VideoProps } from "../hyperframes/types";
 import type { AudioRecord } from "../hyperframes/frame";
-import type { RenderOptions } from "./render";
+import type { FrameRange, RenderOptions } from "./render";
 
 const run = promisify(execFile);
-const input = JSON.parse(await readFile(process.argv[2]!, "utf8")) as { dir: string; props: VideoProps; mode: "stills" | "video"; output: string; frames: number[]; options?: RenderOptions; chromePath?: string };
-const m = input.props.manifest, fps = { num: m.fps, den: 1 };
-const work = await mkdtemp(join(tmpdir(), "reelkit-capture-"));
-const server = await serveComposition(input.dir);
-let session: Awaited<ReturnType<typeof createCaptureSession>> | undefined;
-try {
-  session = await createCaptureSession(server.url.replace(/\/$/, ""), work, { width: m.width, height: m.height, fps, format: "png", compositionDurationSeconds: m.totalFrames / m.fps }, async (page) => {
-    await page.evaluate(async () => { await (window as unknown as { __reelkitReady: Promise<void> }).__reelkitReady; });
-  }, { chromePath: input.chromePath, forceScreenshot: true, useDrawElement: false, staticFrameDedup: false, enableBrowserPool: false, browserGpuMode: input.options?.uses3D ? "software" : "auto" });
-  await initializeSession(session);
-  await session.page.evaluate(async () => { await (window as unknown as { __reelkitReady: Promise<void> }).__reelkitReady; });
-  const frames = input.mode === "stills" ? input.frames : Array.from({ length: m.totalFrames }, (_, i) => i);
-  for (const frame of frames) {
-    if (!Number.isInteger(frame) || frame < 0 || frame >= m.totalFrames) throw new Error(`Frame ${frame} is outside the composition.`);
-    const captured = await captureFrame(session, frame, frame / m.fps);
-    if (input.mode === "stills") await cp(captured.path, join(input.output, `still-${frame}.png`));
+const input = JSON.parse(await readFile(process.argv[2]!, "utf8")) as {
+  dir?: string; props?: VideoProps; mode: "stills" | "video" | "mix"; output: string; frames: number[];
+  options?: RenderOptions; chromePath?: string; range?: FrameRange; audioSidecar?: string; records?: AudioRecord[]; durationSec?: number;
+};
+
+async function mixRecords(records: AudioRecord[], output: string, durationSec: number, work: string): Promise<boolean> {
+  if (!records.length) return false;
+  const audio: AudioElement[] = [];
+  const localSources = new Map<string, string>();
+  for (const record of records) {
+    let src = localSources.get(record.src);
+    if (!src) {
+      const response = await fetch(record.src); if (!response.ok) throw new Error(`Audio source returned ${response.status}.`);
+      src = join(work, `source-${localSources.size}${extname(new URL(record.src).pathname) || ".media"}`);
+      await writeFile(src, new Uint8Array(await response.arrayBuffer())); localSources.set(record.src, src);
+    }
+    const info = JSON.parse((await run("ffprobe", ["-v", "error", "-show_format", "-of", "json", src])).stdout) as { format: { duration: string } };
+    let end = Math.min(record.end, record.start + Number(info.format.duration) - record.mediaStart);
+    if (record.loop) {
+      const looped = join(work, `loop-${audio.length}.wav`);
+      await run("ffmpeg", ["-nostdin", "-y", "-v", "error", "-stream_loop", "-1", "-i", src, "-t", String(record.end - record.start + record.mediaStart), "-vn", looped]);
+      src = looped; end = record.end;
+    }
+    if (end <= record.start) continue;
+    const keys = [...new Map(record.volumeKeyframes.map((k) => [k.time, k])).values()].sort((a, b) => a.time - b.time);
+    audio.push({ ...record, src, end, volumeKeyframes: keys.length ? keys : undefined });
+  }
+  if (!audio.length) return false;
+  const mixed = output.endsWith(".m4a") ? output : join(work, "audio.m4a");
+  const result = await processCompositionAudio(audio, work, join(work, "audio-work"), mixed, durationSec);
+  if (!result.success) throw new Error(result.error ?? "HyperFrames audio mixing failed.");
+  if (mixed !== output) await cp(mixed, output);
+  return true;
+}
+
+if (input.mode === "mix") {
+  const work = await mkdtemp(join(tmpdir(), "reelkit-mix-"));
+  try {
+    await mkdir(dirname(input.output), { recursive: true });
+    const wrote = await mixRecords(input.records ?? [], input.output, input.durationSec ?? 0, work);
+    if (!wrote) throw new Error("This film has no sound to mix.");
+  } finally {
+    await rm(work, { recursive: true, force: true });
   }
-  if (input.mode === "video") {
-    const silent = join(work, "silent.mp4");
-    const encoded = await encodeFramesFromDir(work, "frame_%06d.png", silent, { fps, width: m.width, height: m.height, codec: "h264", pixelFormat: "yuv420p", quality: input.options?.crf ?? 18, preset: "fast" });
-    if (!encoded.success) throw new Error(encoded.error ?? "HyperFrames encoding failed.");
-    const records = input.options?.silent ? [] : await session.page.evaluate(() => [...(window as unknown as { __reelkitAudio: Map<string, AudioRecord> }).__reelkitAudio.values()]);
-    const audio: AudioElement[] = [];
-    const localSources = new Map<string, string>();
-    for (const record of records) {
-      let src = localSources.get(record.src);
-      if (!src) {
-        const response = await fetch(record.src); if (!response.ok) throw new Error(`Audio source returned ${response.status}.`);
-        src = join(work, `source-${localSources.size}${extname(new URL(record.src).pathname) || ".media"}`);
-        await writeFile(src, new Uint8Array(await response.arrayBuffer())); localSources.set(record.src, src);
+} else {
+  const props = input.props!;
+  const m = props.manifest, fps = { num: m.fps, den: 1 };
+  const work = await mkdtemp(join(tmpdir(), "reelkit-capture-"));
+  const server = await serveComposition(input.dir!);
+  let session: Awaited<ReturnType<typeof createCaptureSession>> | undefined;
+  try {
+    session = await createCaptureSession(server.url.replace(/\/$/, ""), work, { width: m.width, height: m.height, fps, format: "png", compositionDurationSeconds: m.totalFrames / m.fps }, async (page) => {
+      await page.evaluate(async () => { await (window as unknown as { __reelkitReady: Promise<void> }).__reelkitReady; });
+    }, { chromePath: input.chromePath, forceScreenshot: true, useDrawElement: false, staticFrameDedup: false, enableBrowserPool: false, browserGpuMode: input.options?.uses3D ? "software" : "auto" });
+    await initializeSession(session);
+    await session.page.evaluate(async () => { await (window as unknown as { __reelkitReady: Promise<void> }).__reelkitReady; });
+    const from = input.range?.from ?? 0, to = input.range?.to ?? m.totalFrames - 1;
+    const frames = input.mode === "stills" ? input.frames : Array.from({ length: to - from + 1 }, (_, i) => from + i);
+    for (const frame of frames) {
+      if (!Number.isInteger(frame) || frame < 0 || frame >= m.totalFrames) throw new Error(`Frame ${frame} is outside the composition.`);
+      const captured = await captureFrame(session, frame, frame / m.fps);
+      if (input.mode === "stills") await cp(captured.path, join(input.output, `still-${frame}.png`));
+    }
+    if (input.mode === "video") {
+      // The encoder reads frame_000000.png onward. A piece that starts later is renamed into its own folder.
+      let framesDir = work, pattern = "frame_%06d.png";
+      if (from !== 0) {
+        framesDir = join(work, "seq"); await mkdir(framesDir);
+        for (let i = 0; i < frames.length; i++) await cp(join(work, `frame_${String(from + i).padStart(6, "0")}.png`), join(framesDir, `frame_${String(i).padStart(6, "0")}.png`));
       }
-      const info = JSON.parse((await run("ffprobe", ["-v", "error", "-show_format", "-of", "json", src])).stdout) as { format: { duration: string } };
-      let end = Math.min(record.end, record.start + Number(info.format.duration) - record.mediaStart);
-      if (record.loop) {
-        const looped = join(work, `loop-${audio.length}.wav`);
-        await run("ffmpeg", ["-y", "-v", "error", "-stream_loop", "-1", "-i", src, "-t", String(record.end - record.start + record.mediaStart), "-vn", looped]);
-        src = looped; end = record.end;
+      const silent = join(work, "silent.mp4");
+      const encoded = await encodeFramesFromDir(framesDir, pattern, silent, { fps, width: m.width, height: m.height, codec: "h264", pixelFormat: "yuv420p", quality: input.options?.crf ?? 18, preset: "fast" });
+      if (!encoded.success) throw new Error(encoded.error ?? "HyperFrames encoding failed.");
+      const collect = Boolean(input.audioSidecar) || !input.options?.silent;
+      const records: AudioRecord[] = collect ? await session.page.evaluate(() => [...(window as unknown as { __reelkitAudio: Map<string, AudioRecord> }).__reelkitAudio.values()]) : [];
+      if (input.audioSidecar) await writeFile(input.audioSidecar, JSON.stringify(records));
+      if (input.options?.silent || input.range) {
+        await mkdir(join(work, "faststart"));
+        const result = await applyFaststart(silent, input.output);
+        if (!result.success) throw new Error(result.error ?? "Faststart failed.");
+      } else if (records.length) {
+        const mixed = join(work, "audio.m4a");
+        const wrote = await mixRecords(records, mixed, m.totalFrames / m.fps, work);
+        if (!wrote) {
+          await mkdir(join(work, "faststart"));
+          const result = await applyFaststart(silent, input.output);
+          if (!result.success) throw new Error(result.error ?? "Faststart failed.");
+        } else {
+          const muxed = await muxVideoWithAudio(silent, mixed, input.output, undefined, { audioCodec: "aac" });
+          if (!muxed.success) throw new Error(muxed.error ?? "HyperFrames audio mux failed.");
+        }
+      } else {
+        await mkdir(join(work, "faststart"));
+        const result = await applyFaststart(silent, input.output);
+        if (!result.success) throw new Error(result.error ?? "Faststart failed.");
       }
-      if (end <= record.start) continue;
-      const keys = [...new Map(record.volumeKeyframes.map((k) => [k.time, k])).values()].sort((a, b) => a.time - b.time);
-      audio.push({ ...record, src, end, volumeKeyframes: keys.length ? keys : undefined });
-    }
-    if (audio.length) {
-      const mixed = join(work, "audio.m4a");
-      const result = await processCompositionAudio(audio, work, join(work, "audio-work"), mixed, m.totalFrames / m.fps);
-      if (!result.success) throw new Error(result.error ?? "HyperFrames audio mixing failed.");
-      const muxed = await muxVideoWithAudio(silent, mixed, input.output, undefined, { audioCodec: "aac" });
-      if (!muxed.success) throw new Error(muxed.error ?? "HyperFrames audio mux failed.");
-    } else {
-      await mkdir(join(work, "faststart"));
-      const result = await applyFaststart(silent, input.output);
-      if (!result.success) throw new Error(result.error ?? "Faststart failed.");
     }
+  } finally {
+    if (session) await closeCaptureSession(session);
+    await server.close(); await rm(work, { recursive: true, force: true });
   }
-} finally {
-  if (session) await closeCaptureSession(session);
-  await server.close(); await rm(work, { recursive: true, force: true });
 }

package/tools/voice/README.md

@@ -0,0 +1,26 @@
+# Voice tools
+
+Scripts for getting one narration line right and into a film without disturbing anything else. They are plain Python (numpy, ffmpeg; Pillow for the
+options video; soundfile for `place_line.py replace`). None of them generates speech; only `credits.py --check` talks to a network, and only to read a balance.
+
+| script | what it answers |
+|---|---|
+| `credits.py --takes N --chars C [--check]` | What will this batch cost, and can the account pay for it? Run before generating. |
+| `phonemes.py take.wav --want "m e a f j e n i m"` | What does the take actually say? Speech-to-text spells the expected word even when it was mispronounced; phonemes do not. Exit 2 (not a failure) when the model's packages are not installed. |
+| `pitch_check.py take.wav` | Pitch range, largest step, and the pitch over the last 0.6 s. Verdicts on octave jumps and question-like endings need `praat-parselmouth`. |
+| `ab_video.py --title "..." --out options.mp4 a.mp3:"current" b.mp3:"slower"` | One phone-safe video that plays every take under a big number, with a key file, so the user can answer "3". |
+| `splice.py head.wav tail.wav --head-end 2.68 --tail-start 0.05 --out j.wav` | Join two takes at the quietest point near the cut, at a zero crossing, with a 10 ms crossfade; reports levels, pitch either side and the exact length. |
+| `place_line.py locate track.wav line.mp3 --near 5.9` | Where is this line inside the assembled voice track? Often in several pieces at different offsets. |
+| `place_line.py replace track.wav new.wav --at 8.485 --until 10.45 --from 2.705 --match-old --out t.wav` | Overwrite only that stretch; every other sample stays identical. Reports the margin to the next line. |
+| `onsets.py voice.wav --from 8.4 --to 9.3` | Where does the word really start (rise out of a pause, fricative onset, release after a stop)? For placing a caption within a frame. |
+
+## The order that worked
+
+1. `credits.py` before any generation. Stop and say so if the balance is short.
+2. Generate two or three spellings, two takes each (pointed, unpointed, hyphenated: pointing a whole sentence can change the neighbouring words).
+3. `phonemes.py` and `pitch_check.py` on every take, on the clean voice. Drop the ones that fail; note duration (a stretched or pitched take is too long for its phonemes).
+4. `ab_video.py` with the take in use first. Send the video and the key; wait for a number.
+5. `splice.py` if only part of a take is wanted. Cut inside a pause, a stop closure or just before a fricative.
+6. `place_line.py locate` then `replace`: change only the stretch that changed. Check the margin to the next line; if it is negative, trim a pause inside the take.
+7. `onsets.py` on the new track for every caption near the change; move a caption only if its word's onset moved, and by the same number of frames.
+8. `reelkit diff old.mp4 new.mp4 --allow <frames> --allow-audio <seconds>s-<seconds>s` to prove nothing else moved.

package/tools/voice/_audio.py

@@ -0,0 +1,118 @@
+"""Shared audio helpers for the voice tools. Needs ffmpeg and numpy; nothing here talks to a network."""
+import subprocess
+import sys
+try:
+    import numpy as np
+except ImportError:
+    print("These voice tools need numpy. In a virtual environment: pip install numpy", file=sys.stderr)
+    raise SystemExit(2)
+
+SR = 48000
+
+
+def load(path, sr=SR):
+    """Any audio file as mono float32 at `sr`."""
+    raw = subprocess.run(["ffmpeg", "-nostdin", "-v", "error", "-i", path, "-vn", "-ac", "1", "-ar", str(sr), "-f", "f32le", "-"], capture_output=True, check=True).stdout
+    return np.frombuffer(raw, np.float32).astype(np.float64)
+
+
+def save(path, x, sr=SR):
+    """Mono float samples to a 16-bit wav (or anything ffmpeg writes, by extension)."""
+    pcm = np.clip(np.round(np.asarray(x) * 32767), -32768, 32767).astype("<i2").tobytes()
+    extra = ["-c:a", "libmp3lame", "-b:a", "192k"] if path.lower().endswith(".mp3") else []
+    subprocess.run(["ffmpeg", "-nostdin", "-y", "-v", "error", "-f", "s16le", "-ar", str(sr), "-ac", "1", "-i", "-", *extra, path], input=pcm, check=True)
+
+
+def rms(x):
+    x = np.asarray(x, dtype=np.float64)
+    return float(np.sqrt(np.mean(x * x))) if len(x) else 0.0
+
+
+def db(v):
+    return 20 * np.log10(max(v, 1e-9))
+
+
+def envelope_db(x, sr=SR, win=0.01):
+    """Level of each `win`-second window in dB."""
+    n = max(1, int(sr * win))
+    k = len(x) // n
+    if k == 0:
+        return np.array([])
+    frames = np.asarray(x[: k * n]).reshape(k, n)
+    return 20 * np.log10(np.sqrt((frames ** 2).mean(axis=1)) + 1e-9)
+
+
+def nearest_zero_crossing(x, at, sr=SR, within=0.004):
+    """The sample nearest `at` (a sample index) where the signal crosses zero, searched `within` seconds either side."""
+    r = int(sr * within)
+    lo, hi = max(1, at - r), min(len(x) - 1, at + r)
+    seg = x[lo - 1: hi + 1]
+    idx = np.nonzero(np.signbit(seg[:-1]) != np.signbit(seg[1:]))[0]
+    if len(idx) == 0:
+        return at
+    cand = idx + lo
+    return int(cand[np.argmin(np.abs(cand - at))])
+
+
+def quietest(x, a, b, sr=SR, win=0.006):
+    """The centre (sample index) of the quietest `win` seconds between samples a and b: where a pause or a stop closure is deepest."""
+    n = max(1, int(sr * win))
+    best, where = None, (a + b) // 2
+    for s in range(a, max(a + 1, b - n), max(1, n // 4)):
+        v = rms(x[s: s + n])
+        if best is None or v < best:
+            best, where = v, s + n // 2
+    return where
+
+
+def speech_rms(x, sr=SR, win=0.01):
+    """RMS over the parts that are speech: the 10 ms windows within 20 dB of the loudest. Pauses inside the stretch do not pull the level down."""
+    n = max(1, int(sr * win))
+    k = len(x) // n
+    if k == 0:
+        return rms(x)
+    power = (np.asarray(x[: k * n]).reshape(k, n) ** 2).mean(axis=1)
+    keep = power >= power.max() * 0.01
+    return float(np.sqrt(power[keep].mean())) if keep.any() else 0.0
+
+
+def f0_track(x, sr=SR, lo=70.0, hi=400.0):
+    """Pitch every 10 ms: (times, Hz) of the voiced frames. Uses Praat (the `praat-parselmouth` package) when it is installed, which is the
+    reliable way. Without it, a plain autocorrelation over 40 ms frames is used: the peak is looked for only after the autocorrelation has first
+    fallen below zero, hissing frames (many zero crossings) are skipped, and single-frame outliers are removed by a median of five."""
+    try:
+        import parselmouth
+        p = parselmouth.Sound(np.asarray(x, dtype=np.float64), sr).to_pitch(time_step=0.01, pitch_floor=lo, pitch_ceiling=hi)
+        f = p.selected_array["frequency"]
+        m = f > 0
+        return p.xs()[m], f[m]
+    except ImportError:
+        pass
+    n, hop = int(sr * 0.04), int(sr * 0.01)
+    ts, fs = [], []
+    peak = float(np.abs(x).max()) if len(x) else 0.0
+    for s in range(0, len(x) - n, hop):
+        f = np.asarray(x[s: s + n])
+        if rms(f) < max(0.004, peak * 0.05):
+            continue
+        if np.count_nonzero(np.signbit(f[:-1]) != np.signbit(f[1:])) / 0.04 > 3000:
+            continue
+        f = (f - f.mean()) * np.hanning(n)
+        ac = np.correlate(f, f, "full")[n - 1:]
+        neg = np.nonzero(ac < 0)[0]
+        a, b = max(int(sr / hi), int(neg[0]) if len(neg) else n), min(int(sr / lo), n - 1)
+        if ac[0] <= 0 or b <= a + 2:
+            continue
+        k = a + int(np.argmax(ac[a:b]))
+        if k > a and ac[k] > 0.55 * ac[0]:
+            ts.append(s / sr); fs.append(sr / k)
+    fs = np.array(fs)
+    if len(fs) >= 5:
+        fs = np.array([np.median(fs[max(0, i - 2): i + 3]) for i in range(len(fs))])
+    return np.array(ts), fs
+
+
+def f0_median(x, sr=SR, lo=70.0, hi=400.0):
+    """Median pitch of a stretch; None when it is not voiced. Good enough to compare the two sides of a join."""
+    _, f = f0_track(x, sr, lo, hi)
+    return round(float(np.median(f)), 1) if len(f) >= 3 else None

package/tools/voice/ab_video.py

@@ -0,0 +1,93 @@
+#!/usr/bin/env python3
+"""Build one video that plays several takes of a line one after another, each under a big number, so that someone can listen on a phone and answer "3".
+
+  ab_video.py --title "the line as written" --out options.mp4 take1.mp3:"current take" take2.mp3:"slower" take3.mp3:"new spelling"
+
+Each take plays over a card with its number, the title and its description; 0.7 s of black and silence separates them. A key file is written beside the
+video (options.key.json) listing number, file, length and description, so the answer "3" maps back to a file without guessing. Put the take in use
+FIRST and say so in its description: the listener needs the reference.
+The video is encoded so that a phone plays it: 1280x720 H.264 High level 4.0, TV-range yuv420p tagged BT.709, AAC 48 kHz, index at the front.
+Needs ffmpeg and Pillow. A title in a right-to-left script needs Pillow built with libraqm; without it the title is drawn as given.
+"""
+import argparse, json, os, subprocess, sys, tempfile
+try:
+    from PIL import Image, ImageDraw, ImageFont, features
+except ImportError:
+    Image = ImageDraw = ImageFont = features = None
+
+FONTS = ["/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf", "/System/Library/Fonts/Supplemental/Arial Bold.ttf", "/Library/Fonts/Arial Bold.ttf", "C:/Windows/Fonts/arialbd.ttf"]
+ENC = ["-c:v", "libx264", "-preset", "medium", "-profile:v", "high", "-level:v", "4.0", "-pix_fmt", "yuv420p", "-r", "30", "-color_range", "tv", "-colorspace", "bt709", "-color_trc", "bt709", "-color_primaries", "bt709",
+       "-x264-params", "colorprim=bt709:transfer=bt709:colormatrix=bt709:range=tv", "-c:a", "aac", "-b:a", "160k", "-ar", "48000", "-ac", "2"]
+TAG = "format=yuv420p,setparams=range=tv:color_primaries=bt709:color_trc=bt709:colorspace=bt709"
+
+
+def ff(*args):
+    subprocess.run(["ffmpeg", "-nostdin", "-y", "-v", "error", *args], check=True)
+
+
+def duration(path):
+    return float(subprocess.check_output(["ffprobe", "-v", "error", "-show_entries", "format=duration", "-of", "csv=p=0", path]))
+
+
+def font(path, size):
+    for f in ([path] if path else []) + FONTS:
+        if f and os.path.exists(f):
+            return ImageFont.truetype(f, size)
+    return ImageFont.load_default()
+
+
+def card(path, number, title, sub, font_path, rtl):
+    im = Image.new("RGB", (1280, 720), "#101010")
+    d = ImageDraw.Draw(im)
+    if number is not None:
+        d.text((640, 290), str(number), font=font(font_path, 330), fill="#F2E9D8", anchor="mm")
+        kw = {"direction": "rtl"} if rtl and features.check("raqm") else {}
+        d.text((640, 540), title, font=font(font_path, 46), fill="#E23B2E", anchor="mm", **kw)
+        d.text((640, 620), sub[:90], font=font(font_path, 28), fill="#9a9a9a", anchor="mm")
+    else:
+        d.rectangle([0, 0, 1280, 720], fill="#000000")
+    im.save(path)
+
+
+def build(title, items, out, gap=0.7, font_path=None, rtl=False):
+    tmp = tempfile.mkdtemp(prefix="ab_")
+    card(f"{tmp}/gap.png", None, "", "", font_path, rtl)
+    gap_file = f"{tmp}/gap.mp4"
+    ff("-loop", "1", "-framerate", "30", "-i", f"{tmp}/gap.png", "-f", "lavfi", "-i", "anullsrc=r=48000:cl=stereo", "-t", str(gap), "-vf", TAG, *ENC, gap_file)
+    parts, key = [], []
+    for n, (path, desc) in enumerate(items, 1):
+        d = duration(path)
+        card(f"{tmp}/c{n}.png", n, title, desc, font_path, rtl)
+        seg = f"{tmp}/s{n}.mp4"
+        ff("-loop", "1", "-framerate", "30", "-i", f"{tmp}/c{n}.png", "-i", path, "-af", "apad=pad_dur=0.15", "-t", f"{d + 0.15:.3f}", "-vf", TAG, *ENC, seg)
+        parts += [gap_file, seg]
+        key.append({"n": n, "file": os.path.abspath(path), "dur": round(d, 2), "desc": desc})
+    parts.append(gap_file)
+    with open(f"{tmp}/list.txt", "w") as f:
+        f.write("".join(f"file '{p}'\n" for p in parts))
+    # The pieces share one encoder setting, so they are joined once more through the encoder only to get one clean timeline; the range is not converted again.
+    ff("-f", "concat", "-safe", "0", "-i", f"{tmp}/list.txt", "-vf", TAG, *ENC, "-crf", "22", "-movflags", "+faststart", out)
+    key_path = os.path.splitext(out)[0] + ".key.json"
+    with open(key_path, "w", encoding="utf8") as f:
+        json.dump(key, f, ensure_ascii=False, indent=1)
+    os.sync()
+    errors = subprocess.run(["ffmpeg", "-nostdin", "-v", "error", "-i", out, "-f", "null", "-"], capture_output=True, text=True).stderr.strip()
+    return {"video": out, "key": key_path, "seconds": round(duration(out), 2), "takes": len(items), "decodes_cleanly": errors == ""}
+
+
+if __name__ == "__main__":
+    p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    p.add_argument("takes", nargs="+", help='file:"description", in the order they should play; the take in use first')
+    p.add_argument("--title", required=True); p.add_argument("--out", required=True); p.add_argument("--gap", type=float, default=0.7)
+    p.add_argument("--font", help="a TrueType font that has the title's script"); p.add_argument("--rtl", action="store_true", help="the title is right to left")
+    a = p.parse_args()
+    if Image is None:
+        print("options video needs Pillow. In a virtual environment: pip install pillow", file=sys.stderr)
+        sys.exit(2)
+    items = []
+    for t in a.takes:
+        path, _, desc = t.partition(":")
+        if not os.path.exists(path):
+            sys.exit(f"{path} does not exist")
+        items.append((path, desc or os.path.basename(path)))
+    json.dump(build(a.title, items, a.out, a.gap, a.font, a.rtl), sys.stdout, indent=1); print()

package/tools/voice/credits.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python3
+"""Before generating speech: what it will cost, and whether the account can pay for it.
+
+  credits.py --takes 6 --chars 28            # an estimate only; nothing is sent anywhere
+  credits.py --takes 6 --chars 28 --check    # also asks the provider how many credits are left
+
+The estimate is takes x characters x --rate (credits per character; 2 fits the short expressive takes of this project, about 57 credits for a
+28-character line). With --check the remaining credits are read from ElevenLabs (GET /v1/user/subscription) using the key in the environment variable
+ELEVENLABS_API_KEY. The key is never printed, logged or written. Exit code 1 when the estimate is more than what is left: stop and tell the user
+rather than generating half a set.
+"""
+import argparse, json, os, sys, urllib.request
+
+if __name__ == "__main__":
+    p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    p.add_argument("--takes", type=int, required=True); p.add_argument("--chars", type=int, required=True); p.add_argument("--rate", type=float, default=2.0)
+    p.add_argument("--check", action="store_true")
+    a = p.parse_args()
+    need = round(a.takes * a.chars * a.rate)
+    out = {"takes": a.takes, "chars_each": a.chars, "estimated_credits": need}
+    if a.check:
+        key = os.environ.get("ELEVENLABS_API_KEY")
+        if not key:
+            out["remaining"] = None; out["note"] = "ELEVENLABS_API_KEY is not set: the balance was not checked"
+        else:
+            req = urllib.request.Request("https://api.elevenlabs.io/v1/user/subscription", headers={"xi-api-key": key})
+            try:
+                with urllib.request.urlopen(req, timeout=20) as r:
+                    sub = json.load(r)
+                left = int(sub.get("character_limit", 0)) - int(sub.get("character_count", 0))
+                out["remaining"] = left; out["enough"] = left >= need
+            except Exception as e:  # the message of a failed request must never carry the key
+                out["remaining"] = None; out["note"] = f"the balance could not be read ({type(e).__name__})"
+    print(json.dumps(out))
+    sys.exit(1 if out.get("enough") is False else 0)