reelkit-cli 0.6.0 → 0.10.0

package/src/project/soundreport.ts

@@ -0,0 +1,347 @@
+// Measures how a finished film sounds against what it shows: how often something hits, how often a swell rises, and how many of the big
+// changes in the picture have a hit beside them. Everything but the two decoders is a pure function over arrays.
+import { loudnessLufs, meanAbsDiffs, probeVideo, tool } from "./refmeasure";
+
+export const AUDIO_RATE = 22050;
+export const PICTURE_FPS = 30;
+export const PICTURE_WIDTH = 64;
+const WINDOW_SEC = 0.046, HOP_SEC = 0.012;
+// A hit this close to a picture change belongs to it.
+export const HIT_NEAR_SEC = 0.1;
+// Nothing quieter than about -50 dBFS counts as sound, so a track that is only dither does not start the film.
+const AUDIBLE_RMS = 0.003;
+// A swell lives in the 0.5 to 10 kHz band, the air of a whoosh or a riser. Measured on the library's own sounds, whooshes keep their energy between
+// 0.2 and 5 kHz (the Cinematic and Quantum Motion ones put 40 to 90 percent of it between 0.5 and 5 kHz, risers 40 to 95 percent between 2 and 10 kHz)
+// and only a riser has a real share above 5 kHz, so a 5 to 10 kHz band could not hear a whoosh at all. Bass and kicks stay below it.
+export const SWELL_BAND: [number, number] = [500, 10000];
+// The band must be at least this loud (about -50 dBFS) to count as a swell, so that silence and dither are not counted.
+const BAND_FLOOR = 0.003;
+// A swell is a rise of the band above its own local level: the level is the median of the band over about 1.5 s around it, the rise must reach
+// SWELL_RISE_DB above it, stay above SWELL_EDGE_DB for at least SWELL_MIN_SEC, and take at least SWELL_RAMP_SEC to build (a click or a hit does not).
+const FLOOR_SEC = 1.5, SWELL_EDGE_DB = 2, SWELL_RISE_DB = 4, SWELL_MIN_SEC = 0.12, SWELL_RAMP_SEC = 0.08;
+
+const mean = (a: ArrayLike<number>) => { let s = 0; for (let i = 0; i < a.length; i++) s += a[i]!; return a.length ? s / a.length : 0; };
+const sd = (a: ArrayLike<number>, m = mean(a)) => { let s = 0; for (let i = 0; i < a.length; i++) s += (a[i]! - m) ** 2; return a.length ? Math.sqrt(s / a.length) : 0; };
+
+// The root-mean-square level of each ~46 ms window, every ~12 ms.
+export function envelope(samples: ArrayLike<number>, rate: number): { values: Float64Array; hopSec: number } {
+  const win = Math.max(1, Math.round(WINDOW_SEC * rate)), hop = Math.max(1, Math.round(HOP_SEC * rate));
+  const n = samples.length >= win ? Math.floor((samples.length - win) / hop) + 1 : 0;
+  const values = new Float64Array(n);
+  for (let f = 0; f < n; f++) {
+    let s = 0;
+    for (let i = f * hop, end = i + win; i < end; i++) s += samples[i]! * samples[i]!;
+    values[f] = Math.sqrt(s / win);
+  }
+  return { values, hopSec: hop / rate };
+}
+
+// Peaks of a series that rise above `threshold`, at least `gap` entries apart (the higher of two close ones is kept).
+function pickPeaks(series: ArrayLike<number>, threshold: number, gap: number): number[] {
+  const peaks: number[] = [];
+  for (let i = 0; i < series.length; i++) {
+    const v = series[i]!;
+    if (!(v > threshold) || v < (series[i - 1] ?? -Infinity) || v < (series[i + 1] ?? -Infinity)) continue;
+    const last = peaks[peaks.length - 1];
+    if (last !== undefined && i - last < gap) { if (v > series[last]!) peaks[peaks.length - 1] = i; } else peaks.push(i);
+  }
+  return peaks;
+}
+
+// The times in seconds of the hits: sudden rises of the full-band level. A hit is where the level of an ~46 ms window jumps by at least HIT_JUMP_DB
+// over what it was 48 ms before and ends up audible, at least 90 ms apart. The jump is relative to the sound just before it, so a quiet tap is a hit
+// whatever else the film holds: a loud drop does not raise the bar for the rest (a threshold taken from the whole film's statistics did).
+const HIT_JUMP_DB = 8, HIT_LOOKBACK = 4, HIT_FLOOR = 0.001;
+export function hitTimes(samples: ArrayLike<number>, rate: number): number[] {
+  const { values, hopSec } = envelope(samples, rate);
+  const jump = new Float64Array(values.length);
+  for (let i = HIT_LOOKBACK; i < values.length; i++) {
+    if (values[i]! < HIT_FLOOR) continue;
+    jump[i] = 20 * Math.log10(values[i]! / Math.max(values[i - HIT_LOOKBACK]!, 1e-5));
+  }
+  return pickPeaks(jump, HIT_JUMP_DB, Math.max(1, Math.round(0.09 / hopSec))).map((i) => (i * hopSec) + WINDOW_SEC / 2);
+}
+
+// One second-order section (RBJ cookbook). Two of them in a row make each edge of the band steeper.
+function biquad(x: ArrayLike<number>, type: "high" | "low", freq: number, rate: number): Float64Array {
+  const w0 = (2 * Math.PI * freq) / rate, cos = Math.cos(w0), alpha = Math.sin(w0) / (2 * Math.SQRT1_2);
+  const [b0, b1, b2] = type === "high" ? [(1 + cos) / 2, -(1 + cos), (1 + cos) / 2] : [(1 - cos) / 2, 1 - cos, (1 - cos) / 2];
+  const a0 = 1 + alpha, a1 = -2 * cos, a2 = 1 - alpha;
+  const y = new Float64Array(x.length);
+  let x1 = 0, x2 = 0, y1 = 0, y2 = 0;
+  for (let i = 0; i < x.length; i++) {
+    const v = (b0 * x[i]! + b1 * x1 + b2 * x2 - a1 * y1 - a2 * y2) / a0;
+    x2 = x1; x1 = x[i]!; y2 = y1; y1 = v; y[i] = v;
+  }
+  return y;
+}
+
+export type Swell = { start: number; peak: number; length: number };
+
+// The swells of a sound: where the 0.5 to 10 kHz band rises ramping up above its own local level (see the constants above). Each is a start (where it
+// leaves the local level), a peak (its loudest moment) and a length, all in seconds. The level is local, so one loud riser does not hide the quiet
+// whooshes around it, and a steady bed of music, which is its own level everywhere, gives none.
+export function swells(samples: ArrayLike<number>, rate: number): Swell[] {
+  if (samples.length < rate / 10) return [];
+  const [lo, hi] = [SWELL_BAND[0], Math.min(SWELL_BAND[1], rate * 0.45)];
+  const band = biquad(biquad(biquad(biquad(samples, "high", lo, rate), "high", lo, rate), "low", hi, rate), "low", hi, rate);
+  const fine = envelope(band, rate);
+  // One value every four hops (about 48 ms) is enough resolution and keeps the median cheap.
+  const step = 4, hopSec = fine.hopSec * step, n = Math.floor(fine.values.length / step);
+  const level = new Float64Array(n), db = new Float64Array(n);
+  for (let i = 0; i < n; i++) { level[i] = fine.values[i * step]!; db[i] = 20 * Math.log10(Math.max(level[i]!, 1e-6)); }
+  const half = Math.max(1, Math.round(FLOOR_SEC / 2 / hopSec));
+  const excess = new Float64Array(n);
+  for (let i = 0; i < n; i++) {
+    const win = Array.from(db.subarray(Math.max(0, i - half), Math.min(n, i + half + 1))).sort((x, y) => x - y);
+    excess[i] = db[i]! - win[win.length >> 1]!;
+  }
+  const out: Swell[] = [];
+  for (let i = 0; i < n;) {
+    if (!(excess[i]! > SWELL_EDGE_DB)) { i++; continue; }
+    let j = i;
+    while (j < n && excess[j]! > SWELL_EDGE_DB) j++;
+    let top = i, loudest = i;
+    for (let k = i; k < j; k++) { if (excess[k]! > excess[top]!) top = k; if (level[k]! > level[loudest]!) loudest = k; }
+    // How long the rise took: from leaving the local level to coming within 3 dB of the run's own top.
+    let near = i;
+    while (near < top && excess[near]! < excess[top]! - 3) near++;
+    const loud = level[loudest]! >= BAND_FLOOR;
+    if (loud && excess[top]! >= SWELL_RISE_DB && (j - i) * hopSec >= SWELL_MIN_SEC && (near - i + 1) * hopSec >= SWELL_RAMP_SEC) {
+      out.push({ start: i * hopSec, peak: loudest * hopSec, length: (j - i) * hopSec });
+    }
+    i = j;
+  }
+  return out;
+}
+
+// The start times in seconds of the swells.
+export const swellTimes = (samples: ArrayLike<number>, rate: number): number[] => swells(samples, rate).map((w) => w.start);
+
+// A big change of the picture. A transition is a span of frames with a high difference, not a single frame: `start` and `end` (seconds) bound the
+// run of frames above the change threshold around the peak, `time` and `frame` are the peak itself, and `strength` is the peak's mean difference
+// between neighbouring frames (0 to 255, on a small grey picture).
+export type PictureChange = { time: number; frame: number; strength: number; start: number; end: number };
+
+// The big picture changes: peaks of the mean absolute difference between neighbouring frames above its mean plus 1.5 standard deviations, at least
+// 0.2 s apart, each with the contiguous run of frames above that same threshold around it. `diffs[i]` is the change from frame i to frame i + 1.
+export function pictureChanges(diffs: ArrayLike<number>, fps: number): PictureChange[] {
+  const m = mean(diffs);
+  const threshold = Math.max(0.5, m + 1.5 * sd(diffs, m));
+  return pickPeaks(diffs, threshold, Math.max(1, Math.round(0.2 * fps))).map((i) => {
+    let a = i, b = i;
+    while (a > 0 && diffs[a - 1]! > threshold) a--;
+    while (b < diffs.length - 1 && diffs[b + 1]! > threshold) b++;
+    return { time: (i + 1) / fps, frame: i + 1, strength: Math.round(diffs[i]! * 10) / 10, start: (a + 1) / fps, end: (b + 1) / fps };
+  });
+}
+
+export const pictureChangeTimes = (diffs: ArrayLike<number>, fps: number): number[] => pictureChanges(diffs, fps).map((c) => c.time);
+
+// The hit nearest to a change, measured to the nearest point of the change's own span: 0 when the hit lies inside it. `side` says which way it is.
+export type HitMatch = { hit: number; distance: number; side: "inside" | "before" | "after" };
+export function nearestHit(change: PictureChange | number, hits: readonly number[]): HitMatch | undefined {
+  const [a, b] = typeof change === "number" ? [change, change] : [change.start, change.end];
+  let best: HitMatch | undefined;
+  for (const h of hits) {
+    const d = h < a ? a - h : h > b ? h - b : 0;
+    if (!best || d < best.distance) best = { hit: h, distance: d, side: h < a ? "before" : h > b ? "after" : "inside" };
+  }
+  return best;
+}
+
+// How many changes have a hit within `near` seconds of their span.
+export const countWithHit = (changes: readonly (PictureChange | number)[], hits: readonly number[], near = HIT_NEAR_SEC): number =>
+  changes.filter((c) => (nearestHit(c, hits)?.distance ?? Infinity) <= near + 1e-9).length;
+
+// The first moment the full-band level reaches audible, or null for a film that is silent.
+export function firstSoundSec(samples: ArrayLike<number>, rate: number): number | null {
+  const { values, hopSec } = envelope(samples, rate);
+  const i = values.findIndex((v) => v > AUDIBLE_RMS);
+  return i < 0 ? null : Math.round(i * hopSec * 100) / 100;
+}
+
+export type SoundDetail = {
+  changes: (PictureChange & { hit: number | null; distance: number | null; side: HitMatch["side"] | null; matched: boolean })[];
+  swells: Swell[];
+  hits: number[];
+  firstSoundSec: number | null;
+};
+export type SoundReport = { hitsPerSecond: number; swellsPerSecond: number; pictureChanges: number; changesWithHit: number; firstSoundSec: number | null; loudnessLufs?: number; detail?: SoundDetail; voice?: VoiceMusic };
+const round = (n: number, d = 2) => Math.round(n * 10 ** d) / 10 ** d;
+
+// The whole measurement from decoded audio and the picture's frame differences. `detail` lists every event the counts were made from.
+export function soundReport(samples: ArrayLike<number>, rate: number, pictureDiffs: ArrayLike<number>, fps: number): SoundReport {
+  const seconds = samples.length / rate;
+  const hits = hitTimes(samples, rate), found = swells(samples, rate), changes = pictureChanges(pictureDiffs, fps), first = firstSoundSec(samples, rate);
+  return {
+    hitsPerSecond: seconds > 0 ? round(hits.length / seconds) : 0, swellsPerSecond: seconds > 0 ? round(found.length / seconds, 3) : 0,
+    pictureChanges: changes.length, changesWithHit: countWithHit(changes, hits), firstSoundSec: first,
+    detail: {
+      changes: changes.map((c) => { const m = nearestHit(c, hits); return { ...c, hit: m ? round(m.hit, 3) : null, distance: m ? round(m.distance, 3) : null, side: m?.side ?? null, matched: (m?.distance ?? Infinity) <= HIT_NEAR_SEC + 1e-9 }; }),
+      swells: found.map((w) => ({ start: round(w.start, 2), peak: round(w.peak, 2), length: round(w.length, 2) })),
+      hits: hits.map((h) => round(h, 3)),
+      firstSoundSec: first,
+    },
+  };
+}
+
+// The same report as lines of text, one per event, for `reelkit sound --detail`.
+export function detailLines(d: SoundDetail): string[] {
+  const t = (n: number) => `${n.toFixed(2)}s`;
+  const lines = ["Picture changes (the time and frame of the peak, how strong, and the nearest hit measured to the changing span):"];
+  for (const c of d.changes) {
+    const span = c.end > c.start ? ` (changing ${t(c.start)} to ${t(c.end)})` : "";
+    const hit = c.hit === null ? "no hit at all" : c.side === "inside" ? `hit at ${t(c.hit)}, inside the change` : `nearest hit at ${t(c.hit)}, ${t(c.distance!)} ${c.side} it`;
+    lines.push(`  ${t(c.time)}  frame ${c.frame}  strength ${c.strength}${span}: ${hit}${c.matched ? "" : "  <- no sound within 0.1s"}`);
+  }
+  if (!d.changes.length) lines.push("  none found");
+  lines.push("Swells (start, peak, length):");
+  for (const w of d.swells) lines.push(`  ${t(w.start)}  peak ${t(w.peak)}  length ${t(w.length)}`);
+  if (!d.swells.length) lines.push("  none found");
+  lines.push(`Hits: ${d.hits.length}${d.hits.length ? ` (${d.hits.slice(0, 40).map((h) => h.toFixed(2)).join(", ")}${d.hits.length > 40 ? ", ..." : ""})` : ""}`);
+  lines.push(`First sound: ${d.firstSoundSec === null ? "never" : t(d.firstSoundSec)}`);
+  return lines;
+}
+
+// Plain-word advice for a film with no voice, against the targets in reference/launch-film.md. It is a measurement and never a verdict.
+export function soundAdvice(r: SoundReport): string {
+  const hits = `${r.hitsPerSecond.toFixed(1)} hits a second`;
+  const every = r.swellsPerSecond > 0 ? 1 / r.swellsPerSecond : undefined;
+  const swell = every === undefined ? "no swell at all" : `a swell every ${every < 10 ? every.toFixed(1) : Math.round(every)} seconds`;
+  const swellFix = r.swellsPerSecond < 0.5 ? " (films like this have one every 1 to 2 seconds: add a whoosh to each move)" : r.swellsPerSecond > 1.3 ? " (films like this have one every 1 to 2 seconds: that is busy, so leave some moves silent)" : "";
+  const aim = Math.max(1, Math.ceil(r.pictureChanges * 0.6));
+  const changes = r.pictureChanges ? `${r.changesWithHit} of ${r.pictureChanges} picture changes have a sound within a tenth of a second (aim for ${aim} or more)` : "no clear picture change was found to compare with";
+  const start = r.firstSoundSec === null ? " The film is silent." : r.firstSoundSec > 0.1 ? ` The first sound is at ${r.firstSoundSec.toFixed(2)}s; the films started within 0.05s.` : "";
+  return `Sound: ${hits}; ${swell}${swellFix}; ${changes}.${start} This is a measurement of the file, not a judgement of how it sounds: listen to it too.`;
+}
+
+// ---- The voice against the music, for a narrated film ----------------------------------------------------------------------------------------
+// The mix is one channel, so the music cannot be measured alone. What can be measured is the mix while the voice speaks, the mix in the pauses
+// (where only the music and effects are heard), and the mix in the short pauses between words, where the music is still held down: that last level
+// is the estimate of "the music under the voice". The numbers say "estimated" for that reason.
+
+// A word's time in the film, in seconds. `sentenceEnd` is true for the last word of a sentence (it ends with . ? ! or ends its scene).
+export type SpokenWord = { start: number; end: number; sentenceEnd?: boolean };
+export type VoiceMusic = {
+  // The mix's level while the voice speaks, dBFS (RMS), and in pauses of at least 0.5 s where it does not.
+  speechDb: number;
+  pauseDb?: number;
+  // speechDb minus pauseDb: how far the voice stands above what is heard in a real pause.
+  voiceOverPauseDb?: number;
+  // The music's level under the voice, from the short pauses between words, and how far below the voice it is (estimated).
+  underVoiceDb?: number;
+  musicBelowVoiceDb?: number;
+  // How many times the level between two sentences rose more than 6 dB above the level under the voice, and how many such gaps were looked at.
+  pumps?: number;
+  pumpsPer30s?: number;
+  sentenceGaps?: number;
+  estimated: true;
+};
+
+export const PUMP_RISE_DB = 6;
+// The music under the voice should sit this far below it (dB): closer masks the voice, farther is not heard.
+export const MUSIC_BELOW_VOICE_DB: [number, number] = [10, 22];
+export const MAX_PUMPS_PER_30S = 2;
+const SPEECH_EDGE = 0.02, PAUSE_MIN = 0.5, PAUSE_EDGE = 0.1, SHORT_PAUSE_MIN = 0.12, SHORT_PAUSE_START = 0.05, SHORT_PAUSE_END = 0.03;
+const SENTENCE_GAP_MIN = 0.2, SENTENCE_GAP_START = 0.1, SENTENCE_GAP_END = 0.05, PUMP_WINDOW = 0.1, PUMP_STEP = 0.05;
+
+const toDb = (power: number) => (power > 1e-12 ? 10 * Math.log10(power) : -120);
+const r1 = (n: number) => Math.round(n * 10) / 10;
+
+// The measurement itself, from the decoded mix and the words' times. Returns undefined for a film with no words.
+export function voiceAgainstMusic(samples: ArrayLike<number>, rate: number, words: SpokenWord[]): VoiceMusic | undefined {
+  const ws = words.filter((w) => w.end > w.start).sort((a, b) => a.start - b.start);
+  if (!ws.length || samples.length < rate) return undefined;
+  const seconds = samples.length / rate;
+  // Prefix sums of the squares, so the power of any stretch is two lookups.
+  const acc = new Float64Array(samples.length + 1);
+  for (let i = 0; i < samples.length; i++) acc[i + 1] = acc[i]! + samples[i]! * samples[i]!;
+  const sumOf = (a: number, b: number): { sum: number; n: number } => {
+    const i = Math.max(0, Math.min(samples.length, Math.round(a * rate))), j = Math.max(0, Math.min(samples.length, Math.round(b * rate)));
+    return j > i ? { sum: acc[j]! - acc[i]!, n: j - i } : { sum: 0, n: 0 };
+  };
+  const powerOver = (spans: [number, number][]): number | undefined => {
+    let sum = 0, n = 0;
+    for (const [a, b] of spans) { const r = sumOf(a, b); sum += r.sum; n += r.n; }
+    return n > 0 ? sum / n : undefined;
+  };
+
+  const speech = powerOver(ws.map((w): [number, number] => [w.start + SPEECH_EDGE, w.end - SPEECH_EDGE]));
+  if (speech === undefined) return undefined;
+
+  // The gaps between consecutive words, with the start and the end of the film counted as gaps too (the last second is a fade-out, so it is left out).
+  type Gap = { from: number; to: number; after?: SpokenWord };
+  const gaps: Gap[] = [{ from: 0, to: ws[0]!.start }];
+  for (let i = 0; i + 1 < ws.length; i++) gaps.push({ from: ws[i]!.end, to: ws[i + 1]!.start, after: ws[i] });
+  gaps.push({ from: ws[ws.length - 1]!.end, to: Math.max(ws[ws.length - 1]!.end, seconds - 1) });
+
+  const pauses = gaps.filter((g) => g.to - g.from >= PAUSE_MIN).map((g): [number, number] => [g.from + PAUSE_EDGE, g.to - PAUSE_EDGE]).filter(([a, b]) => b - a >= 0.2);
+  const pause = powerOver(pauses);
+  // The music held down under the voice: the quiet moments between words (and, with tight gaps, between sentences), the voice's own tail trimmed off.
+  // Word times are rounded, so a short pause may still hold the end of a word: take the quietest quarter of its 50 ms blocks, which is the music (its median is what the gaps between sentences are compared with).
+  const blocks: number[] = [];
+  for (const g of gaps) {
+    if (!g.after || g.to - g.from < SHORT_PAUSE_MIN || g.to - g.from >= PAUSE_MIN) continue;
+    for (let t = g.from + SHORT_PAUSE_START; t + 0.05 <= g.to - SHORT_PAUSE_END + 1e-9; t += 0.05) { const r = sumOf(t, t + 0.05); if (r.n) blocks.push(r.sum / r.n); }
+  }
+  blocks.sort((a, b) => a - b);
+  const short = blocks.length ? blocks[Math.floor((blocks.length - 1) * 0.25)] : undefined;
+  // Between two sentences the music is compared with the typical level of those pauses.
+  const typical = blocks.length ? blocks[Math.floor((blocks.length - 1) * 0.5)] : undefined;
+
+  const out: VoiceMusic = { speechDb: r1(toDb(speech)), estimated: true };
+  if (pause !== undefined) { out.pauseDb = r1(toDb(pause)); out.voiceOverPauseDb = r1(toDb(speech) - toDb(pause)); }
+  if (short !== undefined) {
+    out.underVoiceDb = r1(toDb(short));
+    // The voice alone is the speech power less the music's: a level only a few dB above the music leaves little voice, so the floor is a tenth of it.
+    out.musicBelowVoiceDb = r1(toDb(Math.max(speech - short, speech * 0.1)) - toDb(short));
+    // Between two sentences: the loudest 0.1 s of the gap, apart from the voice's tail at its start and the next word's breath at its end.
+    let pumps = 0, looked = 0;
+    for (const g of gaps) {
+      if (!g.after?.sentenceEnd || g.to - g.from < SENTENCE_GAP_MIN) continue;
+      const a = g.from + SENTENCE_GAP_START, b = g.to - SENTENCE_GAP_END;
+      if (b - a < PUMP_WINDOW) continue;
+      looked++;
+      let loudest = -Infinity;
+      for (let t = a; t + PUMP_WINDOW <= b + 1e-9; t += PUMP_STEP) { const r = sumOf(t, t + PUMP_WINDOW); if (r.n) loudest = Math.max(loudest, toDb(r.sum / r.n)); }
+      if (loudest - toDb(typical ?? short) > PUMP_RISE_DB) pumps++;
+    }
+    out.pumps = pumps; out.sentenceGaps = looked; out.pumpsPer30s = r1((pumps * 30) / seconds);
+  }
+  return out;
+}
+
+// The words of a narrated film as times in the film, from its manifest: a word ends a sentence when it ends with . ? ! … or is a scene's last.
+export function spokenWords(manifest: { fps: number; scenes: { startFrame: number; words: { word: string; startSec: number; endSec: number }[] }[] }): SpokenWord[] {
+  return manifest.scenes.flatMap((s) => s.words.map((w, i) => ({ start: s.startFrame / manifest.fps + w.startSec, end: s.startFrame / manifest.fps + w.endSec, sentenceEnd: i === s.words.length - 1 || /[.!?…؟׃。！？]["'”’»)\]]*$/.test(w.word.trim()) })));
+}
+
+// Plain-word lines about the voice and the music, ending with advice when the music is too near the voice or too far from it, or pumps.
+export function voiceAdvice(v: VoiceMusic): string {
+  const parts: string[] = [];
+  if (v.voiceOverPauseDb !== undefined) parts.push(`the voice speaks ${v.voiceOverPauseDb.toFixed(1)} dB above the level of the pauses (${v.speechDb.toFixed(1)} against ${v.pauseDb!.toFixed(1)} dBFS)`);
+  else parts.push(`the voice speaks at ${v.speechDb.toFixed(1)} dBFS and there is no pause of 0.5 s to compare it with`);
+  if (v.musicBelowVoiceDb !== undefined) parts.push(`the music under the voice is about ${v.musicBelowVoiceDb.toFixed(0)} dB below it`);
+  if (v.pumps !== undefined) parts.push(`${v.pumps} pump${v.pumps === 1 ? "" : "s"} (${v.pumpsPer30s!.toFixed(1)} per 30 s) in ${v.sentenceGaps} gap${v.sentenceGaps === 1 ? "" : "s"} between sentences`);
+  const advice: string[] = [];
+  if (v.musicBelowVoiceDb !== undefined && v.musicBelowVoiceDb < MUSIC_BELOW_VOICE_DB[0]) advice.push(`The music is too close to the voice (under ${MUSIC_BELOW_VOICE_DB[0]} dB below it) and will compete with it: lower duckTo on <Music>.`);
+  if (v.musicBelowVoiceDb !== undefined && v.musicBelowVoiceDb > MUSIC_BELOW_VOICE_DB[1]) advice.push(`The music is hardly there under the voice (over ${MUSIC_BELOW_VOICE_DB[1]} dB below it): raise duckTo on <Music>.`);
+  if (v.pumpsPer30s !== undefined && v.pumpsPer30s > MAX_PUMPS_PER_30S) advice.push(`The music pumps: its level rises between sentences more than ${MAX_PUMPS_PER_30S} times in 30 s. Keep the gaps tight (plan gap), leave Music's defaults, and give no volume per scene.`);
+  return `Voice against music (estimated from the mix, which has the music and the voice in one signal): ${parts.join("; ")}. ${advice.length ? advice.join(" ") : "Both are inside the targets (the music 10 to 22 dB below the voice, at most 2 pumps in 30 s)."}`;
+}
+
+// Decodes the audio and the picture of a finished film and measures them. The loudness is passed in when mastering already measured it.
+export async function measureSound(path: string, loudness?: number, words?: SpokenWord[]): Promise<SoundReport> {
+  const audio = await tool("ffmpeg", ["-v", "error", "-i", path, "-vn", "-ac", "1", "-ar", String(AUDIO_RATE), "-f", "f32le", "-"]);
+  const samples = new Float32Array(audio.stdout.buffer.slice(audio.stdout.byteOffset, audio.stdout.byteOffset + Math.floor(audio.stdout.length / 4) * 4));
+  const info = await probeVideo(path, path);
+  const w = PICTURE_WIDTH, h = Math.max(2, Math.round((w * info.height) / info.width / 2) * 2);
+  const frames = await tool("ffmpeg", ["-v", "error", "-i", path, "-an", "-vf", `fps=${PICTURE_FPS},scale=${w}:${h}:flags=area,format=gray`, "-f", "rawvideo", "-pix_fmt", "gray", "-"]);
+  const diffs = meanAbsDiffs(new Uint8Array(frames.stdout.buffer, frames.stdout.byteOffset, frames.stdout.length), w * h);
+  const base = soundReport(samples, AUDIO_RATE, diffs, PICTURE_FPS);
+  const voice = words?.length ? voiceAgainstMusic(samples, AUDIO_RATE, words) : undefined;
+  const report: SoundReport = voice ? { ...base, voice } : base;
+  const lufs = loudness ?? (await loudnessLufs(path));
+  return lufs === undefined ? report : { ...report, loudnessLufs: lufs };
+}

package/src/project/svgcheck.ts

@@ -0,0 +1,21 @@
+// What a downloaded vector graphic must be before it is saved into a project. The server returns a cleaned SVG, but a file that will be drawn in the
+// renderer's browser is checked here as well, so a mistake or a changed server cannot put a script or an outside link into a film.
+// Returns one plain line saying what is wrong, or undefined when the file is fine.
+export function svgProblem(text: string): string | undefined {
+  const body = text.replace(/^/, "").replace(/<\?xml[\s\S]*?\?>/g, "").replace(/<!--[\s\S]*?-->/g, "").replace(/<!DOCTYPE[^>]*>/gi, "").trim();
+  if (!/^<svg[\s>]/i.test(body) || !/<\/svg\s*>\s*$/i.test(body) && !/^<svg[^>]*\/>\s*$/i.test(body)) return "The file is not an SVG: it does not start with an <svg> element. Nothing was saved; run the command again.";
+  if (/<script/i.test(body)) return "The SVG contains a <script>. Nothing was saved; run the command again.";
+  if (/<foreignObject/i.test(body)) return "The SVG contains a <foreignObject>. Nothing was saved; run the command again.";
+  if (/\son[a-z]+\s*=/i.test(body)) return "The SVG contains an event attribute (such as onclick). Nothing was saved; run the command again.";
+  for (const m of body.matchAll(/\s(?:xlink:)?(?:href|src)\s*=\s*("([^"]*)"|'([^']*)')/gi)) {
+    const value = (m[2] ?? m[3] ?? "").trim();
+    if (!(value.startsWith("#") || /^data:image\//i.test(value))) return `The SVG points outside the file (${value.slice(0, 40)}). Nothing was saved; run the command again.`;
+  }
+  // A style may also reach out: url(...) other than a #fragment or a data: image, and @import.
+  for (const m of body.matchAll(/url\(\s*(['"]?)([^'")]*)\1\s*\)/gi)) {
+    const value = m[2]!.trim();
+    if (!(value.startsWith("#") || /^data:image\//i.test(value))) return `The SVG points outside the file (${value.slice(0, 40)}). Nothing was saved; run the command again.`;
+  }
+  if (/@import/i.test(body)) return "The SVG imports a style from outside the file. Nothing was saved; run the command again.";
+  return undefined;
+}

package/src/render/component-preview.ts

@@ -1,65 +1,21 @@
-import { execFile } from "node:child_process";
-import { mkdir, mkdtemp, rename, rm, writeFile } from "node:fs/promises";
+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { dirname, join } from "node:path";
-import { promisify } from "node:util";
-import { renderMedia, selectComposition } from "@remotion/renderer";
-import { bundleProject, disposeBundle } from "./render";
-
-const run = promisify(execFile);
-const NAME = /^[A-Z][A-Za-z0-9]*$/;
-
+import { bundleProject, disposeBundle, renderVideo } from "./render";
+import { migrateFrameImports } from "./static-check";
 export const PREVIEW = { width: 960, height: 600, fps: 30, frames: 105 } as const;
-
-// The generated entry: the kit's default dark background with the example filling the frame. Importing the kit also loads
-// its fonts, which is what makes text render in the kit fonts. `name` is checked against NAME and `example` starts with `<name`,
-// because both are interpolated into this code; the example is trusted seed text from the owner's own repository.
-const entrySource = (name: string, example: string) => `import React from "react";
-import { AbsoluteFill, Composition, registerRoot } from "remotion";
-import { palettes } from "reelkit/kit";
-import { ${name} } from "./${name}";
-
-void ${name};
-const Preview: React.FC = () => (
-  <AbsoluteFill style={{ background: palettes.darkTech.bg }}>
-    ${example}
-  </AbsoluteFill>
-);
-
-registerRoot(() => (
-  <Composition id="Preview" component={Preview} durationInFrames={${PREVIEW.frames}} fps={${PREVIEW.fps}} width={${PREVIEW.width}} height={${PREVIEW.height}} />
-));
-`;
-
-// Renders one component with its example to a short silent MP4 (960x600, 30 fps, 3.5 s, H.264, yuv420p, CRF 28).
-// Everything it creates besides outPath (the temp folder and the bundle) is removed again, also when it fails.
+const NAME = /^[A-Z][A-Za-z0-9]*$/;
 export async function renderComponentPreview(input: { name: string; source: string; example: string }, outPath: string): Promise<void> {
   const { name, source, example } = input;
   if (!NAME.test(name)) throw new Error(`Component name "${name}" must be PascalCase letters and digits.`);
   if (!example.trimStart().startsWith(`<${name}`)) throw new Error(`The example for ${name} must start with <${name}.`);
-
-  const work = await mkdtemp(join(tmpdir(), "rk-cmp-preview-"));
-  let serveUrl: string | undefined;
+  const work = await mkdtemp(join(tmpdir(), "rk-cmp-preview-")); let bundle: string | undefined;
+  const props = { manifest: { width: PREVIEW.width, height: PREVIEW.height, fps: PREVIEW.fps, totalFrames: PREVIEW.frames, scenes: [] }, urls: {} };
   try {
-    await writeFile(join(work, `${name}.tsx`), source);
+    await writeFile(join(work, `${name}.tsx`), migrateFrameImports(source));
     const entry = join(work, "index.tsx");
-    await writeFile(entry, entrySource(name, example.trim()));
-    serveUrl = await bundleProject(entry);
-    const composition = await selectComposition({ serveUrl, id: "Preview" });
-    const rendered = join(work, "rendered.mp4");
-    await renderMedia({
-      composition, serveUrl, codec: "h264", outputLocation: rendered, crf: 28, pixelFormat: "yuv420p", colorSpace: "bt709", muted: true, logLevel: "error",
-    });
-    // The last pass guarantees no audio stream and the index at the front, whatever Remotion wrote; the video is copied, not re-encoded.
-    const finished = join(work, "final.mp4");
-    await run("ffmpeg", ["-y", "-v", "error", "-i", rendered, "-an", "-movflags", "+faststart", "-c:v", "copy", finished]);
-    await mkdir(dirname(outPath), { recursive: true });
-    await rename(finished, outPath).catch(async () => {
-      // A different file system than the temp folder: copy instead.
-      await run("ffmpeg", ["-y", "-v", "error", "-i", finished, "-c", "copy", "-movflags", "+faststart", outPath]);
-    });
-  } finally {
-    if (serveUrl) await disposeBundle(serveUrl).catch(() => {});
-    await rm(work, { recursive: true, force: true });
-  }
+    await writeFile(entry, `import React from "react";\nimport { AbsoluteFill } from "reelkit/frame";\nimport { mountVideo } from "reelkit/host";\nimport { palettes } from "reelkit/kit";\nimport { ${name} } from "./${name}";\nmountVideo(() => <AbsoluteFill style={{background: palettes.darkTech.bg}}>${example.trim()}</AbsoluteFill>);\n`);
+    bundle = await bundleProject(entry); await mkdir(dirname(outPath), { recursive: true });
+    await renderVideo(bundle, props, outPath, { silent: true, crf: 28 });
+  } finally { if (bundle) await disposeBundle(bundle).catch(() => {}); await rm(work, { recursive: true, force: true }); }
 }

package/src/render/contact-sheet.ts

@@ -0,0 +1,39 @@
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+
+const exec = promisify(execFile);
+
+// The grid of a sheet of `count` tiles for a frame of `aspect` (width over height): 6 across for a tall frame, 4 for a wide one, so that the whole sheet
+// is at most about `maxWidth` pixels across and a phone-shaped frame is still readable.
+export function sheetGrid(count: number, aspect: number, maxWidth = 2400): { cols: number; rows: number; tileW: number; tileH: number } {
+  const cols = Math.max(1, Math.min(count, aspect < 1 ? 6 : aspect > 1.2 ? 4 : 5));
+  const tileW = Math.min(540, Math.floor(maxWidth / cols / 2) * 2);
+  const tileH = Math.max(2, Math.round(tileW / aspect / 2) * 2);
+  return { cols, rows: Math.ceil(count / cols), tileW, tileH };
+}
+
+// The drawn text is plain ASCII (the default font has no arrows): the arrow of a word label is drawn as >.
+const clean = (s: string) => s.replace(/\u25B8/g, ">").replace(/[^A-Za-z0-9 _.>#-]/g, "");
+
+// One picture of many: the files in order, each with its label in a box on its top left corner. Falls back to the same sheet without labels if this ffmpeg
+// has no text drawing; the caller still lists the order. `newRowAt` starts a new row at that tile (the tiles before it are padded with empty ones), so a
+// second kind of frame sits in rows of its own. Returns the grid, and whether the labels were drawn.
+export async function contactSheet(files: string[], labels: string[], out: string, aspect: number, opts: { newRowAt?: number } = {}): Promise<{ cols: number; rows: number; width: number; height: number; labelled: boolean }> {
+  const base = sheetGrid(files.length, aspect);
+  const at = opts.newRowAt !== undefined && opts.newRowAt > 0 && opts.newRowAt < files.length ? opts.newRowAt : undefined;
+  const blanks = at === undefined ? 0 : (base.cols - (at % base.cols)) % base.cols;
+  const g = { ...base, rows: Math.ceil((files.length + blanks) / base.cols) };
+  const build = (labelled: boolean) => {
+    const tile = (i: number) => `[${i}:v]scale=${g.tileW}:${g.tileH},setsar=1${labelled ? `,drawtext=text='${clean(labels[i] ?? "")}':x=6:y=6:fontsize=${Math.max(12, Math.round(g.tileW / 22))}:fontcolor=white:box=1:boxcolor=black@0.65:boxborderw=4` : ""}[t${i}]`;
+    const parts = files.map((_, i) => tile(i));
+    // The empty tiles are extra inputs of plain colour, one frame each.
+    const blankInputs = Array.from({ length: blanks }, () => ["-f", "lavfi", "-i", `color=c=0x202020:s=${g.tileW}x${g.tileH}:r=25`]).flat();
+    const blankParts = Array.from({ length: blanks }, (_, k) => `[${files.length + k}:v]trim=end_frame=1,setpts=PTS-STARTPTS,format=yuvj420p,setsar=1[b${k}]`);
+    const order = [...files.map((_, i) => (i === at ? [...Array.from({ length: blanks }, (_, k) => `[b${k}]`), `[t${i}]`] : [`[t${i}]`])).flat()];
+    const filter = `${[...parts, ...blankParts].join(";")};${order.join("")}concat=n=${files.length + blanks}:v=1:a=0,tile=${g.cols}x${g.rows}:padding=2:color=0x202020[out]`;
+    return ["-y", "-loglevel", "error", ...files.flatMap((f) => ["-i", f]), ...blankInputs, "-filter_complex", filter, "-map", "[out]", "-frames:v", "1", "-q:v", "4", out];
+  };
+  let labelled = true;
+  try { await exec("ffmpeg", build(true)); } catch { labelled = false; await exec("ffmpeg", build(false)); }
+  return { cols: g.cols, rows: g.rows, width: g.cols * g.tileW + (g.cols - 1) * 2, height: g.rows * g.tileH + (g.rows - 1) * 2, labelled };
+}

package/src/render/continuity.ts

@@ -10,7 +10,8 @@ export const CARRIED_AT = 0.25;
 const MIN_EDGE_SHARE = 0.002;
 
 export type Gray = ArrayLike<number>;
-export type BoundaryKind = "carried" | "cut" | "empty";
+// "cut (intended)" is a change the plan marked with cutIn: it is reported but never counts against the score.
+export type BoundaryKind = "carried" | "cut" | "empty" | "cut (intended)";
 
 // A pixel is an edge where the brightness steps by more than the threshold to the pixel on its right or below it.
 export function edgeMap(gray: Gray, w: number, h: number, threshold = EDGE_THRESHOLD): Uint8Array {
@@ -57,6 +58,13 @@ export function dilate(map: Uint8Array, w: number, h: number, radius: number): U
   return out;
 }
 
+// The later scene is compared at two moments, its first fully visible frame and one a few frames on while a move is still settling (a plate
+// that becomes a card has not yet reached its edges at the first). The better of the two says whether something was carried.
+export function compareAcross(earlier: Gray, laters: Gray[], w: number, h: number): ReturnType<typeof compareFrames> {
+  const all = laters.map((l) => compareFrames(earlier, l, w, h));
+  return all.reduce((best, r) => (r.survived > best.survived ? r : best));
+}
+
 export function compareFrames(earlier: Gray, later: Gray, w: number, h: number): { survived: number; edges: number; kind: BoundaryKind } {
   const band = captionBand(w, h);
   const before = clearBottom(edgeMap(earlier, w, h), w, h, band);
@@ -72,15 +80,17 @@ export type Boundary = { from: string; to: string; survived: number; kind: Bound
 
 // The score is carried over carried plus cut: a change with nothing on screen to compare says nothing either way.
 export function summariseContinuity(boundaries: Boundary[]): { score: number | null; lines: string[] } {
+  const intended = boundaries.filter((b) => b.kind === "cut (intended)").length;
   const carried = boundaries.filter((b) => b.kind === "carried").length;
   const cut = boundaries.filter((b) => b.kind === "cut");
   const counted = carried + cut.length;
-  const empty = boundaries.length - counted;
-  if (!counted) return { score: null, lines: ["No scene change could be measured: the picture is nearly empty at the end of every scene."] };
+  const empty = boundaries.length - counted - intended;
+  const hard = intended ? ` ${intended} ${intended === 1 ? "is" : "are"} intended hard cut${intended === 1 ? "" : "s"}, not counted.` : "";
+  if (!counted) return { score: null, lines: [intended && !empty ? `Every scene change that could be measured is an intended hard cut.` : "No scene change could be measured: the picture is nearly empty at the end of every scene."] };
   return {
     score: carried / counted,
     lines: [
-      `${carried} of ${counted} scene changes carry something across${empty ? ` (${empty} more had nothing on screen to compare)` : ""}.`,
+      `${carried} of ${counted} scene changes carry something across${empty ? ` (${empty} more had nothing on screen to compare)` : ""}.${hard}`,
       ...cut.map((b) => `${b.from} → ${b.to}: nothing carries over. Keep one element on screen through the change and move it into the next scene (see reference/continuity.md).`),
     ],
   };

package/src/render/deps.ts

@@ -1,8 +1,20 @@
+import { existsSync, readFileSync } from "node:fs";
 import { createRequire } from "node:module";
-import { dirname } from "node:path";
+import { dirname, join } from "node:path";
 
 const require = createRequire(import.meta.url);
 
 // The directory of an installed dependency of this package, found through its package.json so it does not depend on
-// where the package keeps its main file or on how the package manager lays out node_modules.
-export const packageDir = (name: string): string => dirname(require.resolve(`${name}/package.json`));
+// where the package keeps its main file or on how the package manager lays out node_modules. A package that does not export its
+// package.json (three does not) is found from its main file instead, by walking up to the folder that holds its own package.json.
+export const packageDir = (name: string): string => {
+  try {
+    return dirname(require.resolve(`${name}/package.json`));
+  } catch {
+    for (let dir = dirname(require.resolve(name)); dir !== dirname(dir); dir = dirname(dir)) {
+      const file = join(dir, "package.json");
+      if (existsSync(file) && (JSON.parse(readFileSync(file, "utf8")) as { name?: string }).name === name) return dir;
+    }
+    throw new Error(`Could not find the installed package ${name}.`);
+  }
+};