reelkit-cli 0.10.6 → 0.12.0

package/src/lint/pixel-rules.ts

@@ -0,0 +1,149 @@
+// What can be read off the rendered pictures alone: letters cut by an edge of the frame, and flashes that cover a held shot. Pure: grey frames in, findings out.
+// These are measurements with known limits, so they are reported as advice with frame numbers to look at, never as errors.
+
+export type Edge = "top" | "bottom" | "left" | "right";
+export const EDGES: Edge[] = ["top", "bottom", "left", "right"];
+
+export type EdgeOptions = {
+  // How deep the strip along each edge is, as a share of the frame's short side.
+  depth?: number;
+  // How far a stroke must be from the ground behind it, in grey levels.
+  contrast?: number;
+  // The widest a stroke may be, as a share of the frame's long side.
+  maxStroke?: number;
+};
+
+export type EdgeHit = { edge: Edge; // Where along the edge the cut strokes are, 0 to 1 (left to right, or top to bottom).
+  at: number; strokes: number };
+
+const median = (values: number[]): number => { const s = [...values].sort((a, b) => a - b); return s[Math.floor(s.length / 2)] ?? 0; };
+
+// The strip along one edge as rows of `depth` lines, line 0 being the outermost. Each line runs along the edge.
+function strip(frame: Uint8Array, w: number, h: number, edge: Edge, depth: number): Uint8Array[] {
+  const lines: Uint8Array[] = [];
+  const along = edge === "top" || edge === "bottom" ? w : h;
+  for (let d = 0; d < depth; d++) {
+    const line = new Uint8Array(along);
+    for (let i = 0; i < along; i++) {
+      const x = edge === "left" ? d : edge === "right" ? w - 1 - d : i;
+      const y = edge === "top" ? d : edge === "bottom" ? h - 1 - d : i;
+      line[i] = frame[y * w + x]!;
+    }
+    lines.push(line);
+  }
+  return lines;
+}
+
+const shareNear = (values: number[], tol: number) => { const g = median(values); return values.filter((v) => Math.abs(v - g) <= tol).length / values.length; };
+
+// Letters cut by an edge look like this: several short strokes of one ink on the last line of pixels, still there a couple of lines further in,
+// on a ground that is calm beside them. A photograph that runs off the frame has no calm ground. A plate or a bar is one wide run, which is ignored
+// so it does not hide the letters sitting on it. A wide run used to throw the whole window away, which missed titles and pills cut beside a plate.
+export function edgeHits(frame: Uint8Array, w: number, h: number, opts: EdgeOptions = {}): EdgeHit[] {
+  const depth = Math.max(4, Math.round(Math.min(w, h) * (opts.depth ?? 0.035)));
+  const contrast = opts.contrast ?? 64;
+  const hits: EdgeHit[] = [];
+  for (const edge of EDGES) {
+    const lines = strip(frame, w, h, edge, depth);
+    const along = lines[0]!.length;
+    const win = Math.max(24, Math.round(along * 0.16));
+    // A stem of a letter is thin: at most about 1.6% of the frame's long side. Anything wider is a shape, a plate or a picture.
+    const maxStroke = Math.max(2, Math.round(Math.max(w, h) * (opts.maxStroke ?? 0.016)));
+    for (let start = 0; start < along; start += Math.round(win / 2)) {
+      const end = Math.min(along, start + win);
+      if (end - start < win / 2) break;
+      const all: number[] = [];
+      for (const line of lines) for (let i = start; i < end; i++) all.push(line[i]!);
+      const ground = median(all);
+      const flat = shareNear(all, 14);
+      // Strokes on the outermost line that are still there two lines further in. A wide run is a plate or a bar: skip that run, keep any letters beside it.
+      let strokes = 0, covered = 0, run = 0, first = -1, last = -1;
+      const ink: number[] = [], cols: number[] = [];
+      const close = (i: number) => {
+        if (run > 0 && run <= maxStroke) { strokes++; covered += run; if (first < 0) first = i - run; last = i; for (let k = i - run; k < i; k++) { ink.push(lines[0]![k]!); cols.push(k); } }
+        run = 0;
+      };
+      for (let i = start; i < end; i++) {
+        const on = Math.abs(lines[0]![i]! - ground) > contrast && Math.abs(lines[2]![i]! - ground) > contrast;
+        if (on) run++; else close(i);
+      }
+      close(end);
+      if (strokes < 3 || covered / (end - start) > 0.4) continue;
+      // Lettering is one ink: the strokes share a level. The edge of a cut-out photograph does not.
+      const inkLevel = median(ink);
+      if (ink.filter((v) => Math.abs(v - inkLevel) <= 28).length / ink.length < 0.8) continue;
+      if (flat < 0.6) {
+        // The strokes themselves make a textured window look uneven. Judge the ground on the pixels that are not strokes.
+        const stroke = new Set(cols);
+        const quiet: number[] = [];
+        for (const line of lines.slice(2)) for (let i = start; i < end; i++) if (!stroke.has(i)) quiet.push(line[i]!);
+        if (quiet.length < 8 || shareNear(quiet, 22) < 0.72) continue;
+      }
+      hits.push({ edge, at: Math.round(((first + last) / 2 / along) * 100) / 100, strokes });
+    }
+  }
+  return hits;
+}
+
+export type EdgeFinding = { edge: Edge; at: number; from: number; to: number };
+
+// Dust and scratches of a film look touch an edge for a frame or two; a caption that is cut stays cut. Only a hit that lasts is kept:
+// the same edge, at about the same place, for at least `minFrames` frames in a row.
+export function persistentEdgeHits(perFrame: EdgeHit[][], minFrames = 4, firstFrame = 0): EdgeFinding[] {
+  const out: EdgeFinding[] = [];
+  const open: { edge: Edge; at: number; from: number; last: number }[] = [];
+  perFrame.forEach((hits, i) => {
+    for (const h of hits) {
+      const o = open.find((x) => x.edge === h.edge && Math.abs(x.at - h.at) <= 0.12 && x.last >= i - 1);
+      if (o) { o.last = i; o.at = (o.at + h.at) / 2; }
+      else open.push({ edge: h.edge, at: h.at, from: i, last: i });
+    }
+    for (let k = open.length - 1; k >= 0; k--) {
+      const o = open[k]!;
+      if (o.last < i - 1) { if (o.last - o.from + 1 >= minFrames) out.push({ edge: o.edge, at: Math.round(o.at * 100) / 100, from: o.from + firstFrame, to: o.last + firstFrame }); open.splice(k, 1); }
+    }
+  });
+  for (const o of open) if (o.last - o.from + 1 >= minFrames) out.push({ edge: o.edge, at: Math.round(o.at * 100) / 100, from: o.from + firstFrame, to: o.last + firstFrame });
+  return out.sort((a, b) => a.from - b.from);
+}
+
+export type FrameStat = { mean: number; std: number };
+
+export function frameStat(frame: Uint8Array): FrameStat {
+  let sum = 0, sq = 0;
+  for (let i = 0; i < frame.length; i++) { sum += frame[i]!; sq += frame[i]! * frame[i]!; }
+  const mean = sum / frame.length;
+  return { mean, std: Math.sqrt(Math.max(0, sq / frame.length - mean * mean)) };
+}
+
+export type Flash = { from: number; to: number; // "fill": a near-uniform frame (a white, black or colour flash); "picture": another picture cut in.
+  kind: "fill" | "picture" };
+
+export type FlashInput = {
+  // Mean absolute difference of each frame to the one before it (index 0 is 0).
+  toPrev: number[];
+  // Difference between the frame before a candidate flash and the frame after it: asked for only where needed.
+  across: (before: number, after: number) => number;
+  stats: FrameStat[];
+};
+
+// A flash covers a held shot when the picture jumps away and, one to `maxFrames` frames later, jumps back to nearly what it was. Whatever was on screen
+// (a caption, a word being said) is hidden for those frames.
+export function flashes(input: FlashInput, opts: { jump?: number; back?: number; maxFrames?: number } = {}): Flash[] {
+  const jump = opts.jump ?? 22, back = opts.back ?? 9, maxFrames = opts.maxFrames ?? 3;
+  const out: Flash[] = [];
+  const n = input.toPrev.length;
+  for (let i = 1; i < n - 1; i++) {
+    if (input.toPrev[i]! < jump) continue;
+    for (let len = 1; len <= maxFrames && i + len < n; len++) {
+      if (input.toPrev[i + len]! < jump) continue;
+      if (input.across(i - 1, i + len) <= back) {
+        const fill = input.stats.slice(i, i + len).every((s) => s.std < 12);
+        out.push({ from: i, to: i + len - 1, kind: fill ? "fill" : "picture" });
+        i += len;
+      }
+      break;
+    }
+  }
+  return out;
+}

package/src/lint/source-rules.ts

@@ -0,0 +1,186 @@
+// What can be read off the composition's source: right-to-left text that is never told to run right to left, vowel points in captions, a Sequence
+// that falls outside the one around it, and shots on a reference clock that the film never shows. Pure: source text in, findings out.
+import ts from "typescript";
+import { filmRanges, type Clock } from "../clock/clock";
+
+export type Finding = { rule: string; file: string; line: number; message: string };
+
+const RTL = /[֐-׿؀-ۿיִ-ﭏ]/;
+// Hebrew cantillation marks and vowel points (not the maqaf U+05BE, a hyphen that plain text does use).
+const NIQQUD = /[֑-ׇֽֿׁׂׅׄ]/;
+
+const parse = (source: string, file: string) => ts.createSourceFile(file, source, ts.ScriptTarget.Latest, true, ts.ScriptKind.TSX);
+const lineOf = (sf: ts.SourceFile, node: ts.Node) => sf.getLineAndCharacterOfPosition(node.getStart(sf)).line + 1;
+const short = (s: string) => (s.length > 24 ? `${s.slice(0, 24)}...` : s);
+
+const opening = (node: ts.Node): ts.JsxOpeningLikeElement | undefined => (ts.isJsxSelfClosingElement(node) ? node : ts.isJsxElement(node) ? node.openingElement : undefined);
+const attr = (el: ts.JsxOpeningLikeElement, name: string): ts.JsxAttribute | undefined => el.attributes.properties.find((p): p is ts.JsxAttribute => ts.isJsxAttribute(p) && p.name.getText() === name);
+
+// A literal number in a JSX attribute: from={12}, from={-3}, from={1541 - 1534}. Anything else is unknown.
+function numberOf(e: ts.Expression | undefined): number | undefined {
+  if (!e) return undefined;
+  if (ts.isParenthesizedExpression(e)) return numberOf(e.expression);
+  if (ts.isNumericLiteral(e)) return Number(e.text);
+  if (ts.isPrefixUnaryExpression(e) && e.operator === ts.SyntaxKind.MinusToken) { const v = numberOf(e.operand); return v === undefined ? undefined : -v; }
+  if (ts.isBinaryExpression(e)) {
+    const a = numberOf(e.left), b = numberOf(e.right);
+    if (a === undefined || b === undefined) return undefined;
+    if (e.operatorToken.kind === ts.SyntaxKind.PlusToken) return a + b;
+    if (e.operatorToken.kind === ts.SyntaxKind.MinusToken) return a - b;
+    if (e.operatorToken.kind === ts.SyntaxKind.AsteriskToken) return a * b;
+  }
+  return undefined;
+}
+const numAttr = (el: ts.JsxOpeningLikeElement, name: string): number | undefined => { const a = attr(el, name); return a?.initializer && ts.isJsxExpression(a.initializer) ? numberOf(a.initializer.expression) : undefined; };
+
+// Whether this element says which way its text runs: a `direction` in its style, a dir attribute, or an rtl / ltr prop.
+function setsDirection(el: ts.JsxOpeningLikeElement, sf: ts.SourceFile): boolean {
+  if (attr(el, "dir") || attr(el, "rtl") || attr(el, "ltr") || attr(el, "direction")) return true;
+  const style = attr(el, "style");
+  return Boolean(style && /\bdirection\b/.test(style.getText(sf)));
+}
+
+// The source of each component declared in the file, by name: `const Word: React.FC = ...`, `function Word(...)`.
+function declaredComponents(sf: ts.SourceFile): Map<string, ts.Node> {
+  const out = new Map<string, ts.Node>();
+  sf.forEachChild((node) => {
+    if (ts.isFunctionDeclaration(node) && node.name) out.set(node.name.text, node);
+    if (ts.isVariableStatement(node)) for (const d of node.declarationList.declarations) if (ts.isIdentifier(d.name) && d.initializer) out.set(d.name.text, d.initializer);
+  });
+  return out;
+}
+
+// Right-to-left text needs `direction: "rtl"` (or dir="rtl") on the element or on something around it. Without it, punctuation and numbers land on the
+// wrong side and a mixed line reorders. Text handed to a component as a prop is that component's job: it is flagged only when the component is declared
+// in this file and never mentions a direction.
+export function rtlFindings(source: string, file: string): Finding[] {
+  const sf = parse(source, file);
+  const comps = declaredComponents(sf);
+  const out: Finding[] = [];
+  const visit = (node: ts.Node, directed: boolean) => {
+    const el = opening(node);
+    const here = directed || Boolean(el && setsDirection(el, sf));
+    if (ts.isJsxText(node) && RTL.test(node.text) && !directed) {
+      out.push({ rule: "rtl-direction", file, line: lineOf(sf, node), message: `Right-to-left text "${short(node.text.trim())}" is drawn with no direction set. Put direction: "rtl" in the style of this element or of one around it.` });
+    }
+    if (el) {
+      const tag = el.tagName.getText(sf);
+      const body = comps.get(tag);
+      if (body && !/\bdirection\b|\bdir=/.test(body.getText(sf))) {
+        for (const p of el.attributes.properties) {
+          if (!ts.isJsxAttribute(p) || !p.initializer) continue;
+          const text = ts.isStringLiteral(p.initializer) ? p.initializer.text : ts.isJsxExpression(p.initializer) && p.initializer.expression && ts.isStringLiteralLike(p.initializer.expression) ? p.initializer.expression.text : undefined;
+          if (text && RTL.test(text) && !here) out.push({ rule: "rtl-direction", file, line: lineOf(sf, p), message: `<${tag}> is given right-to-left text "${short(text)}" but never sets a direction. Add direction: "rtl" where ${tag} draws its text.` });
+        }
+      }
+    }
+    ts.forEachChild(node, (c) => visit(c, here));
+  };
+  visit(sf, false);
+  return out;
+}
+
+// Vowel points belong in the text sent to a voice, not in what is drawn. Every string in the composition that carries them is listed.
+export function niqqudFindings(source: string, file: string): Finding[] {
+  const sf = parse(source, file);
+  const out: Finding[] = [];
+  const visit = (node: ts.Node) => {
+    const text = ts.isStringLiteralLike(node) ? node.text : ts.isJsxText(node) ? node.text : undefined;
+    if (text !== undefined && NIQQUD.test(text)) out.push({ rule: "niqqud", file, line: lineOf(sf, node), message: `"${short(text.trim())}" carries vowel points (niqqud). On-screen text is plain: keep the pointed spelling for the voice only.` });
+    ts.forEachChild(node, visit);
+  };
+  visit(sf);
+  return out;
+}
+
+// A <Sequence> (or anything with from and durationInFrames) written inside another one runs on the outer one's clock and is clipped to its length.
+// A child that starts at or after the parent's end, or ends at or before its start, is never drawn; one that sticks out is cut short.
+export function nestedSequenceFindings(source: string, file: string): Finding[] {
+  const sf = parse(source, file);
+  const out: Finding[] = [];
+  const visit = (node: ts.Node, parent?: { tag: string; duration: number }) => {
+    const el = opening(node);
+    let next = parent;
+    if (el) {
+      const tag = el.tagName.getText(sf);
+      const from = numAttr(el, "from"), duration = numAttr(el, "durationInFrames");
+      if (parent && from !== undefined) {
+        const end = duration === undefined ? undefined : from + duration;
+        if (from >= parent.duration || (end !== undefined && end <= 0)) out.push({ rule: "sequence-outside-parent", file, line: lineOf(sf, node), message: `<${tag} from={${from}}${duration === undefined ? "" : ` durationInFrames={${duration}}`}> lies outside the <${parent.tag}> around it (${parent.duration} frames long): it is never drawn.` });
+        else if (end !== undefined && end > parent.duration) out.push({ rule: "sequence-outside-parent", file, line: lineOf(sf, node), message: `<${tag}> runs to frame ${end} of the <${parent.tag}> around it, which is only ${parent.duration} frames long: its last ${end - parent.duration} frame(s) are cut off.` });
+      }
+      if (duration !== undefined && attr(el, "durationInFrames")) next = { tag, duration };
+    }
+    ts.forEachChild(node, (c) => visit(c, next));
+  };
+  visit(sf);
+  return out;
+}
+
+export type ShotOptions = { tags?: string[]; scenes?: { id: string; startFrame: number; durationFrames: number }[]; sectionsConst?: string };
+
+type Shot = { a: number; b: number; line: number; tag: string };
+
+// Shots written on a reference clock, such as <S a={802} b={805}>: one that lies wholly inside a range the film cuts out is silently never shown, and
+// one placed in the wrong scene section is clipped away by that scene. The first needs only the clock. The second needs to know which section a shot is
+// in: the composition's `SECTIONS = { sceneId: Component }` map, followed through the components each section draws.
+export function shotFindings(source: string, file: string, clock: Clock, opts: ShotOptions = {}): Finding[] {
+  const sf = parse(source, file);
+  const tags = new Set(opts.tags ?? ["S", "Shot"]);
+  const comps = declaredComponents(sf);
+  const shotsIn = new Map<string, Shot[]>();
+  const usesIn = new Map<string, Set<string>>();
+  const all: Shot[] = [];
+  for (const [name, body] of comps) {
+    const shots: Shot[] = [], uses = new Set<string>();
+    const visit = (node: ts.Node) => {
+      const el = opening(node);
+      if (el) {
+        const tag = el.tagName.getText(sf);
+        if (tags.has(tag)) {
+          const a = numAttr(el, "a") ?? numAttr(el, "from"), b = numAttr(el, "b") ?? numAttr(el, "to");
+          if (a !== undefined && b !== undefined) shots.push({ a, b, line: lineOf(sf, node), tag });
+        } else if (comps.has(tag) && tag !== name) uses.add(tag);
+      }
+      ts.forEachChild(node, visit);
+    };
+    visit(body);
+    shotsIn.set(name, shots); usesIn.set(name, uses); all.push(...shots);
+  }
+  const out: Finding[] = [];
+  for (const s of all) {
+    if (s.b <= s.a) { out.push({ rule: "shot-range", file, line: s.line, message: `<${s.tag} a={${s.a}} b={${s.b}}> ends at or before its start: it is never drawn.` }); continue; }
+    // The opening runs on its own clock from 0, so a range below the first kept range belongs to it.
+    if (clock.opening > 0 && s.b <= clock.opening) continue;
+    if (!filmRanges(clock, s.a, s.b).length) out.push({ rule: "shot-in-cut", file, line: s.line, message: `<${s.tag} a={${s.a}} b={${s.b}}> lies wholly inside a range the film cuts out: it is never drawn. Remove it, or move it into a kept range.` });
+  }
+  // Which scene section each shot is drawn in, and whether that scene shows those frames.
+  const sectionsName = opts.sectionsConst ?? "SECTIONS";
+  const sections = comps.get(sectionsName);
+  if (opts.scenes?.length && sections && ts.isObjectLiteralExpression(sections)) {
+    const collect = (comp: string, seen = new Set<string>()): Shot[] => {
+      if (seen.has(comp)) return [];
+      seen.add(comp);
+      return [...(shotsIn.get(comp) ?? []), ...[...(usesIn.get(comp) ?? [])].flatMap((u) => collect(u, seen))];
+    };
+    for (const prop of sections.properties) {
+      if (!ts.isPropertyAssignment(prop) && !ts.isShorthandPropertyAssignment(prop)) continue;
+      const id = prop.name.getText(sf).replace(/^["']|["']$/g, "");
+      const comp = ts.isPropertyAssignment(prop) ? prop.initializer.getText(sf) : id;
+      const scene = opts.scenes.find((sc) => sc.id === id);
+      if (!scene || !comps.has(comp)) continue;
+      const lo = scene.startFrame, hi = scene.startFrame + scene.durationFrames;
+      for (const s of collect(comp)) {
+        // A section that starts inside the opening is written on the opening's own clock.
+        const inOpening = clock.opening > 0 && lo < clock.opening;
+        const ranges = inOpening ? [{ from: s.a, to: s.b }] : filmRanges(clock, s.a, s.b);
+        if (!ranges.length) continue;
+        const shown = ranges.some((r) => r.from < hi && r.to > lo);
+        const clipped = ranges.some((r) => r.from < lo || r.to > hi);
+        if (!shown) out.push({ rule: "shot-outside-section", file, line: s.line, message: `<${s.tag} a={${s.a}} b={${s.b}}> is film frames ${ranges.map((r) => `${r.from}-${r.to}`).join(", ")} but sits in the "${id}" section, which is film frames ${lo}-${hi}: the scene clips it away and it is never drawn. Move it into the section that holds those frames.` });
+        else if (clipped) out.push({ rule: "shot-outside-section", file, line: s.line, message: `<${s.tag} a={${s.a}} b={${s.b}}> (film frames ${ranges.map((r) => `${r.from}-${r.to}`).join(", ")}) sticks out of the "${id}" section (film frames ${lo}-${hi}): the part outside is not drawn.` });
+      }
+    }
+  }
+  return out.sort((x, y) => x.line - y.line);
+}

package/src/media/ffmpeg.ts

@@ -0,0 +1,84 @@
+// Small ffmpeg and ffprobe helpers shared by export, diff, lint and the chunked render. Each failure is a one-line Error for the person at the terminal.
+import { spawn } from "node:child_process";
+import { closeSync, fsyncSync, openSync, readSync, statSync } from "node:fs";
+import { tool } from "../project/refmeasure";
+import type { ProbedStream } from "../export/presets";
+
+export async function probeStreams(path: string): Promise<{ streams: ProbedStream[]; durationSec: number }> {
+  const { stdout } = await tool("ffprobe", ["-v", "error", "-print_format", "json", "-show_format", "-show_streams", path]);
+  const info = JSON.parse(stdout.toString("utf8")) as { streams?: ProbedStream[]; format?: { duration?: string } };
+  return { streams: info.streams ?? [], durationSec: Number(info.format?.duration ?? 0) };
+}
+
+export type VideoFacts = { width: number; height: number; fps: number; pixFmt?: string; colorRange?: string; hasAudio: boolean; durationSec: number };
+
+export async function videoFacts(path: string): Promise<VideoFacts> {
+  const { streams, durationSec } = await probeStreams(path);
+  const v = streams.find((s) => s.codec_type === "video") as (ProbedStream & { avg_frame_rate?: string; r_frame_rate?: string }) | undefined;
+  if (!v || !v.width || !v.height) throw new Error(`${path} has no video stream that can be read.`);
+  const rate = (r?: string) => { const [a, b] = (r ?? "").split("/").map(Number); return a && b ? a / b : a || 0; };
+  return { width: v.width, height: v.height, fps: rate(v.avg_frame_rate) || rate(v.r_frame_rate), pixFmt: v.pix_fmt, colorRange: v.color_range, hasAudio: streams.some((s) => s.codec_type === "audio"), durationSec };
+}
+
+// Decodes the whole file and throws nothing away but the pictures: the number of frames decoded and every line ffmpeg printed at -v error.
+export async function fullDecode(path: string): Promise<{ frames: number; errorLines: string[] }> {
+  return new Promise((resolve, reject) => {
+    const p = spawn("ffmpeg", ["-nostdin", "-v", "error", "-stats", "-i", path, "-f", "null", "-"], { stdio: ["ignore", "ignore", "pipe"] });
+    let err = "";
+    p.stderr.on("data", (d: Buffer) => { err += d.toString("utf8"); });
+    p.on("error", (e) => reject((e as NodeJS.ErrnoException).code === "ENOENT" ? new Error("ffmpeg was not found. Install ffmpeg and try again.") : e));
+    p.on("close", () => {
+      // -stats writes "frame= 2000 fps=..." progress on carriage returns; everything else is an error line.
+      const pieces = err.split(/[\r\n]+/).map((l) => l.trim()).filter(Boolean);
+      const stats = pieces.filter((l) => /^frame=/.test(l));
+      const frames = Number(/^frame=\s*(\d+)/.exec(stats.at(-1) ?? "")?.[1] ?? 0);
+      resolve({ frames, errorLines: pieces.filter((l) => !/^frame=/.test(l) && !/^size=/.test(l)) });
+    });
+  });
+}
+
+// Streams a video as small grey frames, calling onFrame for each. Frames come in order, one at a time, so a long film never sits in memory.
+export async function grayFrames(path: string, width: number, height: number, onFrame: (frame: Uint8Array, index: number) => void, opts: { from?: number; to?: number } = {}): Promise<number> {
+  const select = opts.from !== undefined || opts.to !== undefined ? [`select='between(n\\,${opts.from ?? 0}\\,${opts.to ?? 1e9})'`] : [];
+  const vf = [...select, `scale=${width}:${height}:flags=area`, "format=gray"].join(",");
+  return new Promise((resolve, reject) => {
+    const p = spawn("ffmpeg", ["-nostdin", "-v", "error", "-i", path, "-an", "-vf", vf, "-fps_mode", "passthrough", "-f", "rawvideo", "-pix_fmt", "gray", "-"], { stdio: ["ignore", "pipe", "pipe"] });
+    const size = width * height;
+    let held: Buffer = Buffer.alloc(0);
+    let index = opts.from ?? 0, err = "";
+    p.stdout.on("data", (d: Buffer) => {
+      held = held.length ? Buffer.concat([held, d]) : d;
+      while (held.length >= size) {
+        onFrame(new Uint8Array(held.buffer, held.byteOffset, size).slice(), index++);
+        held = held.subarray(size);
+      }
+    });
+    p.stderr.on("data", (d: Buffer) => { err += d.toString("utf8"); });
+    p.on("error", (e) => reject((e as NodeJS.ErrnoException).code === "ENOENT" ? new Error("ffmpeg was not found. Install ffmpeg and try again.") : e));
+    p.on("close", (code) => (code === 0 ? resolve(index - (opts.from ?? 0)) : reject(new Error(`ffmpeg could not decode ${path}: ${err.trim().split("\n")[0] ?? `exit ${code}`}`))));
+  });
+}
+
+// The sound as mono 16-bit samples at `rate`, or undefined for a file with no audio.
+export async function monoSamples(path: string, rate = 8000): Promise<Int16Array | undefined> {
+  const { streams } = await probeStreams(path);
+  if (!streams.some((s) => s.codec_type === "audio")) return undefined;
+  const { stdout } = await tool("ffmpeg", ["-nostdin", "-v", "error", "-i", path, "-vn", "-ac", "1", "-ar", String(rate), "-f", "s16le", "-"]);
+  return new Int16Array(stdout.buffer, stdout.byteOffset, Math.floor(stdout.byteLength / 2));
+}
+
+export function readBytes(path: string): { read: (offset: number, length: number) => Uint8Array; size: number; close: () => void } {
+  const fd = openSync(path, "r");
+  const size = statSync(path).size;
+  return {
+    size,
+    read: (offset, length) => { const b = Buffer.alloc(Math.min(length, Math.max(0, size - offset))); readSync(fd, b, 0, b.length, offset); return b; },
+    close: () => closeSync(fd),
+  };
+}
+
+// Asks the system to put the file on the disk now. After a crash or a reboot an unsynced file can be empty although the program reported success.
+export function syncFile(path: string): void {
+  const fd = openSync(path, "r+");
+  try { fsyncSync(fd); } finally { closeSync(fd); }
+}

package/src/render/chunked.ts

@@ -0,0 +1,139 @@
+// Rendering a long film in pieces that survive a crash. Each piece is a silent H.264 file of a range of frames, written whole or not at all; a piece
+// that is already on disk, reads back with the right number of frames and has its audio sidecar is not rendered again. The pieces are joined
+// without re-encoding. The sound is mixed once from those sidecars and laid under them. The planning here is pure; the rendering is handed in.
+import { createHash } from "node:crypto";
+import { existsSync, mkdirSync, readFileSync, readdirSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { syncFile } from "../media/ffmpeg";
+import { tool } from "../project/refmeasure";
+
+export const DEFAULT_CHUNK_FRAMES = 300;
+export const MAX_ATTEMPTS = 3;
+
+export type Chunk = { index: number; from: number; to: number; file: string };
+
+// Frame ranges (inclusive) that cover the film.
+export function planChunks(totalFrames: number, chunkFrames = DEFAULT_CHUNK_FRAMES): Chunk[] {
+  const size = Math.max(1, Math.floor(chunkFrames));
+  const out: Chunk[] = [];
+  for (let from = 0, index = 0; from < totalFrames; from += size, index++) {
+    const to = Math.min(totalFrames, from + size) - 1;
+    out.push({ index, from, to, file: `chunk-${String(from).padStart(6, "0")}-${String(to).padStart(6, "0")}.mp4` });
+  }
+  return out;
+}
+
+// What the pieces were rendered from. A piece made from other source, other media or another size is not reused.
+export function renderKey(parts: { sources: Record<string, string>; manifest: unknown; media: Record<string, number>; chunkFrames: number }): string {
+  const h = createHash("sha256");
+  for (const name of Object.keys(parts.sources).sort()) h.update(name).update("\0").update(parts.sources[name]!).update("\0");
+  h.update(JSON.stringify(parts.manifest)).update("\0");
+  for (const name of Object.keys(parts.media).sort()) h.update(`${name}:${parts.media[name]}\0`);
+  h.update(String(parts.chunkFrames));
+  return h.digest("hex").slice(0, 16);
+}
+
+// A failure of the browser rather than of the composition: worth another attempt.
+export const isBrowserFailure = (e: unknown): boolean => /browser|Target closed|ProtocolError|Protocol error|Navigation|WebSocket|ECONNRESET|ECONNREFUSED|Timed out|timeout|crash|disconnected|Session closed/i.test(e instanceof Error ? e.message : String(e));
+
+// Runs fn, and again after a browser failure, up to `attempts` times in all. An error of the composition itself is thrown at once.
+export async function withRetry<T>(fn: (attempt: number) => Promise<T>, opts: { attempts?: number; onRetry?: (attempt: number, error: unknown) => void; wait?: (ms: number) => Promise<void> } = {}): Promise<T> {
+  const attempts = opts.attempts ?? MAX_ATTEMPTS;
+  for (let attempt = 1; ; attempt++) {
+    try { return await fn(attempt); }
+    catch (e) {
+      if (attempt >= attempts || !isBrowserFailure(e)) throw e;
+      opts.onRetry?.(attempt, e);
+      await (opts.wait ?? ((ms) => new Promise((r) => setTimeout(r, ms))))(1500 * attempt);
+    }
+  }
+}
+
+export type ChunkDeps = {
+  // Renders frames from..to (inclusive), silent, to `out`. May also write `${out}.audio.json`.
+  renderRange: (from: number, to: number, out: string) => Promise<void>;
+  // Renders the whole film's sound to `out` (a wav or m4a); resolves false when the film has none.
+  renderSound: (out: string) => Promise<boolean>;
+  // How many frames a finished piece holds; undefined when it cannot be read.
+  countFrames: (file: string) => Promise<number | undefined>;
+  log: (line: string) => void;
+  concat?: (list: string, sound: string | undefined, out: string) => Promise<void>;
+  // Extra reason to render a piece again (for example its audio sidecar is missing).
+  reuseExtra?: (file: string) => boolean;
+};
+
+export type CapturedAudio = {
+  id: string; src: string; start: number; end: number; mediaStart: number; layer: number; volume: number;
+  volumeKeyframes: { time: number; volume: number }[]; type: "audio" | "video"; loop: boolean;
+};
+
+// Pieces of one film are captured in separate browser sessions, so the same clip can come back under different ids.
+// Merge by where it sits, and keep every volume sample.
+export function mergeAudioRecords(lists: CapturedAudio[][]): CapturedAudio[] {
+  const byKey = new Map<string, CapturedAudio>();
+  for (const list of lists) for (const record of list) {
+    const key = `${record.type}|${record.src}|${record.start}|${record.mediaStart}|${record.loop}`;
+    const prev = byKey.get(key);
+    if (!prev) { byKey.set(key, { ...record, volumeKeyframes: [...record.volumeKeyframes] }); continue; }
+    prev.end = Math.max(prev.end, record.end);
+    prev.volumeKeyframes.push(...record.volumeKeyframes);
+  }
+  for (const record of byKey.values()) {
+    const seen = new Map<number, { time: number; volume: number }>();
+    for (const key of record.volumeKeyframes) seen.set(key.time, key);
+    record.volumeKeyframes = [...seen.values()].sort((a, b) => a.time - b.time);
+  }
+  return [...byKey.values()];
+}
+
+export async function countFrames(file: string): Promise<number | undefined> {
+  try {
+    const { stdout } = await tool("ffprobe", ["-v", "error", "-select_streams", "v:0", "-count_packets", "-show_entries", "stream=nb_read_packets", "-of", "csv=p=0", file]);
+    const n = Number(stdout.toString("utf8").trim().split(/[,\n]/)[0]);
+    return Number.isFinite(n) ? n : undefined;
+  } catch { return undefined; }
+}
+
+// Joins the pieces without touching their pictures and lays the sound under them.
+export async function concatChunks(list: string, sound: string | undefined, out: string): Promise<void> {
+  await tool("ffmpeg", ["-nostdin", "-y", "-v", "error", "-f", "concat", "-safe", "0", "-i", list, ...(sound ? ["-i", sound] : []), "-map", "0:v:0", ...(sound ? ["-map", "1:a:0", "-c:a", "aac", "-b:a", "320k", "-ar", "48000"] : []), "-c:v", "copy", "-movflags", "+faststart", out]);
+}
+
+export type ChunkedResult = { chunks: number; reused: number; rendered: number; retries: number };
+
+// Renders whatever pieces are missing into `dir`, then assembles `outPath`. Safe to run again after any interruption: finished pieces are kept.
+export async function renderChunked(dir: string, key: string, totalFrames: number, chunkFrames: number, outPath: string, deps: ChunkDeps): Promise<ChunkedResult> {
+  mkdirSync(dir, { recursive: true });
+  const keyFile = join(dir, "key");
+  // Pieces of another version of the film are of no use: start clean.
+  if (existsSync(keyFile) && readFileSync(keyFile, "utf8").trim() !== key) { for (const f of readdirSync(dir)) rmSync(join(dir, f), { force: true, recursive: true }); }
+  writeFileSync(keyFile, key);
+  const chunks = planChunks(totalFrames, chunkFrames);
+  let reused = 0, rendered = 0, retries = 0;
+  for (const c of chunks) {
+    const file = join(dir, c.file);
+    const want = c.to - c.from + 1;
+    if (existsSync(file) && statSync(file).size > 0 && (await deps.countFrames(file)) === want && (deps.reuseExtra?.(file) ?? true)) { reused++; continue; }
+    const part = `${file}.part.mp4`;
+    const partAudio = `${part}.audio.json`;
+    await withRetry(async () => { rmSync(part, { force: true }); rmSync(partAudio, { force: true }); await deps.renderRange(c.from, c.to, part); }, { onRetry: (n, e) => { retries++; deps.log(`The browser failed on frames ${c.from}-${c.to} (${(e instanceof Error ? e.message : String(e)).split("\n")[0]}); trying again (${n + 1} of ${MAX_ATTEMPTS}).`); } });
+    const got = await deps.countFrames(part);
+    if (got !== want) throw new Error(`frames ${c.from}-${c.to} rendered as ${got ?? "an unreadable file"} instead of ${want} frames`);
+    syncFile(part);
+    renameSync(part, file);
+    if (existsSync(partAudio)) { syncFile(partAudio); renameSync(partAudio, `${file}.audio.json`); }
+    rendered++;
+    deps.log(`Frames ${c.from}-${c.to} done (${c.index + 1} of ${chunks.length}).`);
+  }
+  const soundFile = join(dir, "sound.m4a");
+  let hasSound = existsSync(soundFile) && statSync(soundFile).size > 44;
+  if (!hasSound) {
+    const part = join(dir, "sound.part.m4a");
+    hasSound = await withRetry(() => deps.renderSound(part), { onRetry: () => { retries++; } });
+    if (hasSound) { syncFile(part); renameSync(part, soundFile); } else rmSync(part, { force: true });
+  }
+  const list = join(dir, "list.txt");
+  writeFileSync(list, chunks.map((c) => `file '${join(dir, c.file).replace(/'/g, "'\\''")}'`).join("\n") + "\n");
+  await (deps.concat ?? concatChunks)(list, hasSound ? soundFile : undefined, outPath);
+  return { chunks: chunks.length, reused, rendered, retries };
+}

package/src/render/render.ts

@@ -45,6 +45,7 @@ export async function disposeBundle(dir: string): Promise<void> {
 export type EnsureBrowser = (onDownload: () => void) => Promise<void>;
 export const GL_BACKEND = "software" as const;
 export type RenderOptions = { uses3D?: boolean; silent?: boolean; crf?: number };
+export type FrameRange = { from: number; to: number };
 export function browserPath(): string | undefined {
   if (process.env.REELKIT_CHROME_PATH) return process.env.REELKIT_CHROME_PATH;
   const paths = process.platform === "darwin" ? ["/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", "/Applications/Chromium.app/Contents/MacOS/Chromium"] : process.platform === "linux" ? ["/usr/bin/chromium", "/usr/bin/chromium-browser", "/usr/bin/google-chrome"] : [];
@@ -63,17 +64,34 @@ export const ensureRenderBrowser: EnsureBrowser = async (onDownload) => {
   const browser = await install({ browser: Browser.CHROMEHEADLESSSHELL, buildId: await resolveBuildId(Browser.CHROMEHEADLESSSHELL, platform, "stable"), cacheDir: join(PKG_ROOT, ".browser"), platform });
   process.env.REELKIT_CHROME_PATH = browser.executablePath;
 };
-async function render(dir: string, props: VideoProps, mode: "stills" | "video", output: string, frames: number[], options?: RenderOptions) {
+async function render(dir: string, props: VideoProps, mode: "stills" | "video", output: string, frames: number[], options?: RenderOptions, extra?: { range?: FrameRange; audioSidecar?: string }) {
   await ensureRenderBrowser(() => {});
   await writeFile(join(dir, "props.js"), `window.__reelkitProps=${JSON.stringify(props).replace(/</g, "\\u003c")};`);
   // Isolate engine diagnostics from the CLI's JSON stdout.
   const request = join(dir, "render-request.json");
-  await writeFile(request, JSON.stringify({ dir, props, mode, output, frames, options, chromePath: browserPath() }));
+  await writeFile(request, JSON.stringify({ dir, props, mode, output, frames, options, chromePath: browserPath(), ...extra }));
   await run(process.execPath, ["--import", join(packageDir("tsx"), "dist/loader.mjs"), join(PKG_ROOT, "src/render/worker.ts"), request], { maxBuffer: 16 * 1024 * 1024 });
 }
 export async function renderStills(dir: string, props: VideoProps, frames: number[], outDir: string, options?: RenderOptions) {
   await mkdir(outDir, { recursive: true }); await render(dir, props, "stills", outDir, frames, options);
 }
-export async function renderVideo(dir: string, props: VideoProps, outPath: string, options?: RenderOptions) {
-  await mkdir(dirname(outPath), { recursive: true }); await render(dir, props, "video", outPath, [], options);
+// `audioFile`, when given, receives the audio records the film registered (which sound plays from when to when), as JSON.
+export async function renderVideo(dir: string, props: VideoProps, outPath: string, options?: RenderOptions, audioFile?: string) {
+  await mkdir(dirname(outPath), { recursive: true }); await render(dir, props, "video", outPath, [], options, audioFile ? { audioSidecar: audioFile } : undefined);
+}
+// One inclusive range of frames as a silent file, plus the audio records those frames registered (`${outPath}.audio.json`).
+export async function renderVideoRange(dir: string, props: VideoProps, outPath: string, from: number, to: number, options?: RenderOptions) {
+  await mkdir(dirname(outPath), { recursive: true });
+  await render(dir, props, "video", outPath, [], { ...options, silent: true }, { range: { from, to }, audioSidecar: `${outPath}.audio.json` });
+}
+// Mix audio records already collected from the pieces. No picture is drawn.
+export async function mixCapturedAudio(records: unknown[], outPath: string, durationSec: number) {
+  const dir = await mkdtemp(join(tmpdir(), "reelkit-mix-"));
+  try {
+    const request = join(dir, "render-request.json");
+    await writeFile(request, JSON.stringify({ mode: "mix", output: outPath, records, durationSec, frames: [] }));
+    await run(process.execPath, ["--import", join(packageDir("tsx"), "dist/loader.mjs"), join(PKG_ROOT, "src/render/worker.ts"), request], { maxBuffer: 16 * 1024 * 1024 });
+  } finally {
+    await rm(dir, { recursive: true, force: true });
+  }
 }