reelkit-cli 0.6.0 → 0.8.0

package/src/project/soundreport.ts

@@ -0,0 +1,347 @@
+// Measures how a finished film sounds against what it shows: how often something hits, how often a swell rises, and how many of the big
+// changes in the picture have a hit beside them. Everything but the two decoders is a pure function over arrays.
+import { loudnessLufs, meanAbsDiffs, probeVideo, tool } from "./refmeasure";
+
+export const AUDIO_RATE = 22050;
+export const PICTURE_FPS = 30;
+export const PICTURE_WIDTH = 64;
+const WINDOW_SEC = 0.046, HOP_SEC = 0.012;
+// A hit this close to a picture change belongs to it.
+export const HIT_NEAR_SEC = 0.1;
+// Nothing quieter than about -50 dBFS counts as sound, so a track that is only dither does not start the film.
+const AUDIBLE_RMS = 0.003;
+// A swell lives in the 0.5 to 10 kHz band, the air of a whoosh or a riser. Measured on the library's own sounds, whooshes keep their energy between
+// 0.2 and 5 kHz (the Cinematic and Quantum Motion ones put 40 to 90 percent of it between 0.5 and 5 kHz, risers 40 to 95 percent between 2 and 10 kHz)
+// and only a riser has a real share above 5 kHz, so a 5 to 10 kHz band could not hear a whoosh at all. Bass and kicks stay below it.
+export const SWELL_BAND: [number, number] = [500, 10000];
+// The band must be at least this loud (about -50 dBFS) to count as a swell, so that silence and dither are not counted.
+const BAND_FLOOR = 0.003;
+// A swell is a rise of the band above its own local level: the level is the median of the band over about 1.5 s around it, the rise must reach
+// SWELL_RISE_DB above it, stay above SWELL_EDGE_DB for at least SWELL_MIN_SEC, and take at least SWELL_RAMP_SEC to build (a click or a hit does not).
+const FLOOR_SEC = 1.5, SWELL_EDGE_DB = 2, SWELL_RISE_DB = 4, SWELL_MIN_SEC = 0.12, SWELL_RAMP_SEC = 0.08;
+
+const mean = (a: ArrayLike<number>) => { let s = 0; for (let i = 0; i < a.length; i++) s += a[i]!; return a.length ? s / a.length : 0; };
+const sd = (a: ArrayLike<number>, m = mean(a)) => { let s = 0; for (let i = 0; i < a.length; i++) s += (a[i]! - m) ** 2; return a.length ? Math.sqrt(s / a.length) : 0; };
+
+// The root-mean-square level of each ~46 ms window, every ~12 ms.
+export function envelope(samples: ArrayLike<number>, rate: number): { values: Float64Array; hopSec: number } {
+  const win = Math.max(1, Math.round(WINDOW_SEC * rate)), hop = Math.max(1, Math.round(HOP_SEC * rate));
+  const n = samples.length >= win ? Math.floor((samples.length - win) / hop) + 1 : 0;
+  const values = new Float64Array(n);
+  for (let f = 0; f < n; f++) {
+    let s = 0;
+    for (let i = f * hop, end = i + win; i < end; i++) s += samples[i]! * samples[i]!;
+    values[f] = Math.sqrt(s / win);
+  }
+  return { values, hopSec: hop / rate };
+}
+
+// Peaks of a series that rise above `threshold`, at least `gap` entries apart (the higher of two close ones is kept).
+function pickPeaks(series: ArrayLike<number>, threshold: number, gap: number): number[] {
+  const peaks: number[] = [];
+  for (let i = 0; i < series.length; i++) {
+    const v = series[i]!;
+    if (!(v > threshold) || v < (series[i - 1] ?? -Infinity) || v < (series[i + 1] ?? -Infinity)) continue;
+    const last = peaks[peaks.length - 1];
+    if (last !== undefined && i - last < gap) { if (v > series[last]!) peaks[peaks.length - 1] = i; } else peaks.push(i);
+  }
+  return peaks;
+}
+
+// The times in seconds of the hits: sudden rises of the full-band level. A hit is where the level of an ~46 ms window jumps by at least HIT_JUMP_DB
+// over what it was 48 ms before and ends up audible, at least 90 ms apart. The jump is relative to the sound just before it, so a quiet tap is a hit
+// whatever else the film holds: a loud drop does not raise the bar for the rest (a threshold taken from the whole film's statistics did).
+const HIT_JUMP_DB = 8, HIT_LOOKBACK = 4, HIT_FLOOR = 0.001;
+export function hitTimes(samples: ArrayLike<number>, rate: number): number[] {
+  const { values, hopSec } = envelope(samples, rate);
+  const jump = new Float64Array(values.length);
+  for (let i = HIT_LOOKBACK; i < values.length; i++) {
+    if (values[i]! < HIT_FLOOR) continue;
+    jump[i] = 20 * Math.log10(values[i]! / Math.max(values[i - HIT_LOOKBACK]!, 1e-5));
+  }
+  return pickPeaks(jump, HIT_JUMP_DB, Math.max(1, Math.round(0.09 / hopSec))).map((i) => (i * hopSec) + WINDOW_SEC / 2);
+}
+
+// One second-order section (RBJ cookbook). Two of them in a row make each edge of the band steeper.
+function biquad(x: ArrayLike<number>, type: "high" | "low", freq: number, rate: number): Float64Array {
+  const w0 = (2 * Math.PI * freq) / rate, cos = Math.cos(w0), alpha = Math.sin(w0) / (2 * Math.SQRT1_2);
+  const [b0, b1, b2] = type === "high" ? [(1 + cos) / 2, -(1 + cos), (1 + cos) / 2] : [(1 - cos) / 2, 1 - cos, (1 - cos) / 2];
+  const a0 = 1 + alpha, a1 = -2 * cos, a2 = 1 - alpha;
+  const y = new Float64Array(x.length);
+  let x1 = 0, x2 = 0, y1 = 0, y2 = 0;
+  for (let i = 0; i < x.length; i++) {
+    const v = (b0 * x[i]! + b1 * x1 + b2 * x2 - a1 * y1 - a2 * y2) / a0;
+    x2 = x1; x1 = x[i]!; y2 = y1; y1 = v; y[i] = v;
+  }
+  return y;
+}
+
+export type Swell = { start: number; peak: number; length: number };
+
+// The swells of a sound: where the 0.5 to 10 kHz band rises ramping up above its own local level (see the constants above). Each is a start (where it
+// leaves the local level), a peak (its loudest moment) and a length, all in seconds. The level is local, so one loud riser does not hide the quiet
+// whooshes around it, and a steady bed of music, which is its own level everywhere, gives none.
+export function swells(samples: ArrayLike<number>, rate: number): Swell[] {
+  if (samples.length < rate / 10) return [];
+  const [lo, hi] = [SWELL_BAND[0], Math.min(SWELL_BAND[1], rate * 0.45)];
+  const band = biquad(biquad(biquad(biquad(samples, "high", lo, rate), "high", lo, rate), "low", hi, rate), "low", hi, rate);
+  const fine = envelope(band, rate);
+  // One value every four hops (about 48 ms) is enough resolution and keeps the median cheap.
+  const step = 4, hopSec = fine.hopSec * step, n = Math.floor(fine.values.length / step);
+  const level = new Float64Array(n), db = new Float64Array(n);
+  for (let i = 0; i < n; i++) { level[i] = fine.values[i * step]!; db[i] = 20 * Math.log10(Math.max(level[i]!, 1e-6)); }
+  const half = Math.max(1, Math.round(FLOOR_SEC / 2 / hopSec));
+  const excess = new Float64Array(n);
+  for (let i = 0; i < n; i++) {
+    const win = Array.from(db.subarray(Math.max(0, i - half), Math.min(n, i + half + 1))).sort((x, y) => x - y);
+    excess[i] = db[i]! - win[win.length >> 1]!;
+  }
+  const out: Swell[] = [];
+  for (let i = 0; i < n;) {
+    if (!(excess[i]! > SWELL_EDGE_DB)) { i++; continue; }
+    let j = i;
+    while (j < n && excess[j]! > SWELL_EDGE_DB) j++;
+    let top = i, loudest = i;
+    for (let k = i; k < j; k++) { if (excess[k]! > excess[top]!) top = k; if (level[k]! > level[loudest]!) loudest = k; }
+    // How long the rise took: from leaving the local level to coming within 3 dB of the run's own top.
+    let near = i;
+    while (near < top && excess[near]! < excess[top]! - 3) near++;
+    const loud = level[loudest]! >= BAND_FLOOR;
+    if (loud && excess[top]! >= SWELL_RISE_DB && (j - i) * hopSec >= SWELL_MIN_SEC && (near - i + 1) * hopSec >= SWELL_RAMP_SEC) {
+      out.push({ start: i * hopSec, peak: loudest * hopSec, length: (j - i) * hopSec });
+    }
+    i = j;
+  }
+  return out;
+}
+
+// The start times in seconds of the swells.
+export const swellTimes = (samples: ArrayLike<number>, rate: number): number[] => swells(samples, rate).map((w) => w.start);
+
+// A big change of the picture. A transition is a span of frames with a high difference, not a single frame: `start` and `end` (seconds) bound the
+// run of frames above the change threshold around the peak, `time` and `frame` are the peak itself, and `strength` is the peak's mean difference
+// between neighbouring frames (0 to 255, on a small grey picture).
+export type PictureChange = { time: number; frame: number; strength: number; start: number; end: number };
+
+// The big picture changes: peaks of the mean absolute difference between neighbouring frames above its mean plus 1.5 standard deviations, at least
+// 0.2 s apart, each with the contiguous run of frames above that same threshold around it. `diffs[i]` is the change from frame i to frame i + 1.
+export function pictureChanges(diffs: ArrayLike<number>, fps: number): PictureChange[] {
+  const m = mean(diffs);
+  const threshold = Math.max(0.5, m + 1.5 * sd(diffs, m));
+  return pickPeaks(diffs, threshold, Math.max(1, Math.round(0.2 * fps))).map((i) => {
+    let a = i, b = i;
+    while (a > 0 && diffs[a - 1]! > threshold) a--;
+    while (b < diffs.length - 1 && diffs[b + 1]! > threshold) b++;
+    return { time: (i + 1) / fps, frame: i + 1, strength: Math.round(diffs[i]! * 10) / 10, start: (a + 1) / fps, end: (b + 1) / fps };
+  });
+}
+
+export const pictureChangeTimes = (diffs: ArrayLike<number>, fps: number): number[] => pictureChanges(diffs, fps).map((c) => c.time);
+
+// The hit nearest to a change, measured to the nearest point of the change's own span: 0 when the hit lies inside it. `side` says which way it is.
+export type HitMatch = { hit: number; distance: number; side: "inside" | "before" | "after" };
+export function nearestHit(change: PictureChange | number, hits: readonly number[]): HitMatch | undefined {
+  const [a, b] = typeof change === "number" ? [change, change] : [change.start, change.end];
+  let best: HitMatch | undefined;
+  for (const h of hits) {
+    const d = h < a ? a - h : h > b ? h - b : 0;
+    if (!best || d < best.distance) best = { hit: h, distance: d, side: h < a ? "before" : h > b ? "after" : "inside" };
+  }
+  return best;
+}
+
+// How many changes have a hit within `near` seconds of their span.
+export const countWithHit = (changes: readonly (PictureChange | number)[], hits: readonly number[], near = HIT_NEAR_SEC): number =>
+  changes.filter((c) => (nearestHit(c, hits)?.distance ?? Infinity) <= near + 1e-9).length;
+
+// The first moment the full-band level reaches audible, or null for a film that is silent.
+export function firstSoundSec(samples: ArrayLike<number>, rate: number): number | null {
+  const { values, hopSec } = envelope(samples, rate);
+  const i = values.findIndex((v) => v > AUDIBLE_RMS);
+  return i < 0 ? null : Math.round(i * hopSec * 100) / 100;
+}
+
+export type SoundDetail = {
+  changes: (PictureChange & { hit: number | null; distance: number | null; side: HitMatch["side"] | null; matched: boolean })[];
+  swells: Swell[];
+  hits: number[];
+  firstSoundSec: number | null;
+};
+export type SoundReport = { hitsPerSecond: number; swellsPerSecond: number; pictureChanges: number; changesWithHit: number; firstSoundSec: number | null; loudnessLufs?: number; detail?: SoundDetail; voice?: VoiceMusic };
+const round = (n: number, d = 2) => Math.round(n * 10 ** d) / 10 ** d;
+
+// The whole measurement from decoded audio and the picture's frame differences. `detail` lists every event the counts were made from.
+export function soundReport(samples: ArrayLike<number>, rate: number, pictureDiffs: ArrayLike<number>, fps: number): SoundReport {
+  const seconds = samples.length / rate;
+  const hits = hitTimes(samples, rate), found = swells(samples, rate), changes = pictureChanges(pictureDiffs, fps), first = firstSoundSec(samples, rate);
+  return {
+    hitsPerSecond: seconds > 0 ? round(hits.length / seconds) : 0, swellsPerSecond: seconds > 0 ? round(found.length / seconds, 3) : 0,
+    pictureChanges: changes.length, changesWithHit: countWithHit(changes, hits), firstSoundSec: first,
+    detail: {
+      changes: changes.map((c) => { const m = nearestHit(c, hits); return { ...c, hit: m ? round(m.hit, 3) : null, distance: m ? round(m.distance, 3) : null, side: m?.side ?? null, matched: (m?.distance ?? Infinity) <= HIT_NEAR_SEC + 1e-9 }; }),
+      swells: found.map((w) => ({ start: round(w.start, 2), peak: round(w.peak, 2), length: round(w.length, 2) })),
+      hits: hits.map((h) => round(h, 3)),
+      firstSoundSec: first,
+    },
+  };
+}
+
+// The same report as lines of text, one per event, for `reelkit sound --detail`.
+export function detailLines(d: SoundDetail): string[] {
+  const t = (n: number) => `${n.toFixed(2)}s`;
+  const lines = ["Picture changes (the time and frame of the peak, how strong, and the nearest hit measured to the changing span):"];
+  for (const c of d.changes) {
+    const span = c.end > c.start ? ` (changing ${t(c.start)} to ${t(c.end)})` : "";
+    const hit = c.hit === null ? "no hit at all" : c.side === "inside" ? `hit at ${t(c.hit)}, inside the change` : `nearest hit at ${t(c.hit)}, ${t(c.distance!)} ${c.side} it`;
+    lines.push(`  ${t(c.time)}  frame ${c.frame}  strength ${c.strength}${span}: ${hit}${c.matched ? "" : "  <- no sound within 0.1s"}`);
+  }
+  if (!d.changes.length) lines.push("  none found");
+  lines.push("Swells (start, peak, length):");
+  for (const w of d.swells) lines.push(`  ${t(w.start)}  peak ${t(w.peak)}  length ${t(w.length)}`);
+  if (!d.swells.length) lines.push("  none found");
+  lines.push(`Hits: ${d.hits.length}${d.hits.length ? ` (${d.hits.slice(0, 40).map((h) => h.toFixed(2)).join(", ")}${d.hits.length > 40 ? ", ..." : ""})` : ""}`);
+  lines.push(`First sound: ${d.firstSoundSec === null ? "never" : t(d.firstSoundSec)}`);
+  return lines;
+}
+
+// Plain-word advice for a film with no voice, against the targets in reference/launch-film.md. It is a measurement and never a verdict.
+export function soundAdvice(r: SoundReport): string {
+  const hits = `${r.hitsPerSecond.toFixed(1)} hits a second`;
+  const every = r.swellsPerSecond > 0 ? 1 / r.swellsPerSecond : undefined;
+  const swell = every === undefined ? "no swell at all" : `a swell every ${every < 10 ? every.toFixed(1) : Math.round(every)} seconds`;
+  const swellFix = r.swellsPerSecond < 0.5 ? " (films like this have one every 1 to 2 seconds: add a whoosh to each move)" : r.swellsPerSecond > 1.3 ? " (films like this have one every 1 to 2 seconds: that is busy, so leave some moves silent)" : "";
+  const aim = Math.max(1, Math.ceil(r.pictureChanges * 0.6));
+  const changes = r.pictureChanges ? `${r.changesWithHit} of ${r.pictureChanges} picture changes have a sound within a tenth of a second (aim for ${aim} or more)` : "no clear picture change was found to compare with";
+  const start = r.firstSoundSec === null ? " The film is silent." : r.firstSoundSec > 0.1 ? ` The first sound is at ${r.firstSoundSec.toFixed(2)}s; the films started within 0.05s.` : "";
+  return `Sound: ${hits}; ${swell}${swellFix}; ${changes}.${start} This is a measurement of the file, not a judgement of how it sounds: listen to it too.`;
+}
+
+// ---- The voice against the music, for a narrated film ----------------------------------------------------------------------------------------
+// The mix is one channel, so the music cannot be measured alone. What can be measured is the mix while the voice speaks, the mix in the pauses
+// (where only the music and effects are heard), and the mix in the short pauses between words, where the music is still held down: that last level
+// is the estimate of "the music under the voice". The numbers say "estimated" for that reason.
+
+// A word's time in the film, in seconds. `sentenceEnd` is true for the last word of a sentence (it ends with . ? ! or ends its scene).
+export type SpokenWord = { start: number; end: number; sentenceEnd?: boolean };
+export type VoiceMusic = {
+  // The mix's level while the voice speaks, dBFS (RMS), and in pauses of at least 0.5 s where it does not.
+  speechDb: number;
+  pauseDb?: number;
+  // speechDb minus pauseDb: how far the voice stands above what is heard in a real pause.
+  voiceOverPauseDb?: number;
+  // The music's level under the voice, from the short pauses between words, and how far below the voice it is (estimated).
+  underVoiceDb?: number;
+  musicBelowVoiceDb?: number;
+  // How many times the level between two sentences rose more than 6 dB above the level under the voice, and how many such gaps were looked at.
+  pumps?: number;
+  pumpsPer30s?: number;
+  sentenceGaps?: number;
+  estimated: true;
+};
+
+export const PUMP_RISE_DB = 6;
+// The music under the voice should sit this far below it (dB): closer masks the voice, farther is not heard.
+export const MUSIC_BELOW_VOICE_DB: [number, number] = [10, 22];
+export const MAX_PUMPS_PER_30S = 2;
+const SPEECH_EDGE = 0.02, PAUSE_MIN = 0.5, PAUSE_EDGE = 0.1, SHORT_PAUSE_MIN = 0.12, SHORT_PAUSE_START = 0.05, SHORT_PAUSE_END = 0.03;
+const SENTENCE_GAP_MIN = 0.2, SENTENCE_GAP_START = 0.1, SENTENCE_GAP_END = 0.05, PUMP_WINDOW = 0.1, PUMP_STEP = 0.05;
+
+const toDb = (power: number) => (power > 1e-12 ? 10 * Math.log10(power) : -120);
+const r1 = (n: number) => Math.round(n * 10) / 10;
+
+// The measurement itself, from the decoded mix and the words' times. Returns undefined for a film with no words.
+export function voiceAgainstMusic(samples: ArrayLike<number>, rate: number, words: SpokenWord[]): VoiceMusic | undefined {
+  const ws = words.filter((w) => w.end > w.start).sort((a, b) => a.start - b.start);
+  if (!ws.length || samples.length < rate) return undefined;
+  const seconds = samples.length / rate;
+  // Prefix sums of the squares, so the power of any stretch is two lookups.
+  const acc = new Float64Array(samples.length + 1);
+  for (let i = 0; i < samples.length; i++) acc[i + 1] = acc[i]! + samples[i]! * samples[i]!;
+  const sumOf = (a: number, b: number): { sum: number; n: number } => {
+    const i = Math.max(0, Math.min(samples.length, Math.round(a * rate))), j = Math.max(0, Math.min(samples.length, Math.round(b * rate)));
+    return j > i ? { sum: acc[j]! - acc[i]!, n: j - i } : { sum: 0, n: 0 };
+  };
+  const powerOver = (spans: [number, number][]): number | undefined => {
+    let sum = 0, n = 0;
+    for (const [a, b] of spans) { const r = sumOf(a, b); sum += r.sum; n += r.n; }
+    return n > 0 ? sum / n : undefined;
+  };
+
+  const speech = powerOver(ws.map((w): [number, number] => [w.start + SPEECH_EDGE, w.end - SPEECH_EDGE]));
+  if (speech === undefined) return undefined;
+
+  // The gaps between consecutive words, with the start and the end of the film counted as gaps too (the last second is a fade-out, so it is left out).
+  type Gap = { from: number; to: number; after?: SpokenWord };
+  const gaps: Gap[] = [{ from: 0, to: ws[0]!.start }];
+  for (let i = 0; i + 1 < ws.length; i++) gaps.push({ from: ws[i]!.end, to: ws[i + 1]!.start, after: ws[i] });
+  gaps.push({ from: ws[ws.length - 1]!.end, to: Math.max(ws[ws.length - 1]!.end, seconds - 1) });
+
+  const pauses = gaps.filter((g) => g.to - g.from >= PAUSE_MIN).map((g): [number, number] => [g.from + PAUSE_EDGE, g.to - PAUSE_EDGE]).filter(([a, b]) => b - a >= 0.2);
+  const pause = powerOver(pauses);
+  // The music held down under the voice: the quiet moments between words (and, with tight gaps, between sentences), the voice's own tail trimmed off.
+  // Word times are rounded, so a short pause may still hold the end of a word: take the quietest quarter of its 50 ms blocks, which is the music (its median is what the gaps between sentences are compared with).
+  const blocks: number[] = [];
+  for (const g of gaps) {
+    if (!g.after || g.to - g.from < SHORT_PAUSE_MIN || g.to - g.from >= PAUSE_MIN) continue;
+    for (let t = g.from + SHORT_PAUSE_START; t + 0.05 <= g.to - SHORT_PAUSE_END + 1e-9; t += 0.05) { const r = sumOf(t, t + 0.05); if (r.n) blocks.push(r.sum / r.n); }
+  }
+  blocks.sort((a, b) => a - b);
+  const short = blocks.length ? blocks[Math.floor((blocks.length - 1) * 0.25)] : undefined;
+  // Between two sentences the music is compared with the typical level of those pauses.
+  const typical = blocks.length ? blocks[Math.floor((blocks.length - 1) * 0.5)] : undefined;
+
+  const out: VoiceMusic = { speechDb: r1(toDb(speech)), estimated: true };
+  if (pause !== undefined) { out.pauseDb = r1(toDb(pause)); out.voiceOverPauseDb = r1(toDb(speech) - toDb(pause)); }
+  if (short !== undefined) {
+    out.underVoiceDb = r1(toDb(short));
+    // The voice alone is the speech power less the music's: a level only a few dB above the music leaves little voice, so the floor is a tenth of it.
+    out.musicBelowVoiceDb = r1(toDb(Math.max(speech - short, speech * 0.1)) - toDb(short));
+    // Between two sentences: the loudest 0.1 s of the gap, apart from the voice's tail at its start and the next word's breath at its end.
+    let pumps = 0, looked = 0;
+    for (const g of gaps) {
+      if (!g.after?.sentenceEnd || g.to - g.from < SENTENCE_GAP_MIN) continue;
+      const a = g.from + SENTENCE_GAP_START, b = g.to - SENTENCE_GAP_END;
+      if (b - a < PUMP_WINDOW) continue;
+      looked++;
+      let loudest = -Infinity;
+      for (let t = a; t + PUMP_WINDOW <= b + 1e-9; t += PUMP_STEP) { const r = sumOf(t, t + PUMP_WINDOW); if (r.n) loudest = Math.max(loudest, toDb(r.sum / r.n)); }
+      if (loudest - toDb(typical ?? short) > PUMP_RISE_DB) pumps++;
+    }
+    out.pumps = pumps; out.sentenceGaps = looked; out.pumpsPer30s = r1((pumps * 30) / seconds);
+  }
+  return out;
+}
+
+// The words of a narrated film as times in the film, from its manifest: a word ends a sentence when it ends with . ? ! … or is a scene's last.
+export function spokenWords(manifest: { fps: number; scenes: { startFrame: number; words: { word: string; startSec: number; endSec: number }[] }[] }): SpokenWord[] {
+  return manifest.scenes.flatMap((s) => s.words.map((w, i) => ({ start: s.startFrame / manifest.fps + w.startSec, end: s.startFrame / manifest.fps + w.endSec, sentenceEnd: i === s.words.length - 1 || /[.!?…؟׃。！？]["'”’»)\]]*$/.test(w.word.trim()) })));
+}
+
+// Plain-word lines about the voice and the music, ending with advice when the music is too near the voice or too far from it, or pumps.
+export function voiceAdvice(v: VoiceMusic): string {
+  const parts: string[] = [];
+  if (v.voiceOverPauseDb !== undefined) parts.push(`the voice speaks ${v.voiceOverPauseDb.toFixed(1)} dB above the level of the pauses (${v.speechDb.toFixed(1)} against ${v.pauseDb!.toFixed(1)} dBFS)`);
+  else parts.push(`the voice speaks at ${v.speechDb.toFixed(1)} dBFS and there is no pause of 0.5 s to compare it with`);
+  if (v.musicBelowVoiceDb !== undefined) parts.push(`the music under the voice is about ${v.musicBelowVoiceDb.toFixed(0)} dB below it`);
+  if (v.pumps !== undefined) parts.push(`${v.pumps} pump${v.pumps === 1 ? "" : "s"} (${v.pumpsPer30s!.toFixed(1)} per 30 s) in ${v.sentenceGaps} gap${v.sentenceGaps === 1 ? "" : "s"} between sentences`);
+  const advice: string[] = [];
+  if (v.musicBelowVoiceDb !== undefined && v.musicBelowVoiceDb < MUSIC_BELOW_VOICE_DB[0]) advice.push(`The music is too close to the voice (under ${MUSIC_BELOW_VOICE_DB[0]} dB below it) and will compete with it: lower duckTo on <Music>.`);
+  if (v.musicBelowVoiceDb !== undefined && v.musicBelowVoiceDb > MUSIC_BELOW_VOICE_DB[1]) advice.push(`The music is hardly there under the voice (over ${MUSIC_BELOW_VOICE_DB[1]} dB below it): raise duckTo on <Music>.`);
+  if (v.pumpsPer30s !== undefined && v.pumpsPer30s > MAX_PUMPS_PER_30S) advice.push(`The music pumps: its level rises between sentences more than ${MAX_PUMPS_PER_30S} times in 30 s. Keep the gaps tight (plan gap), leave Music's defaults, and give no volume per scene.`);
+  return `Voice against music (estimated from the mix, which has the music and the voice in one signal): ${parts.join("; ")}. ${advice.length ? advice.join(" ") : "Both are inside the targets (the music 10 to 22 dB below the voice, at most 2 pumps in 30 s)."}`;
+}
+
+// Decodes the audio and the picture of a finished film and measures them. The loudness is passed in when mastering already measured it.
+export async function measureSound(path: string, loudness?: number, words?: SpokenWord[]): Promise<SoundReport> {
+  const audio = await tool("ffmpeg", ["-v", "error", "-i", path, "-vn", "-ac", "1", "-ar", String(AUDIO_RATE), "-f", "f32le", "-"]);
+  const samples = new Float32Array(audio.stdout.buffer.slice(audio.stdout.byteOffset, audio.stdout.byteOffset + Math.floor(audio.stdout.length / 4) * 4));
+  const info = await probeVideo(path, path);
+  const w = PICTURE_WIDTH, h = Math.max(2, Math.round((w * info.height) / info.width / 2) * 2);
+  const frames = await tool("ffmpeg", ["-v", "error", "-i", path, "-an", "-vf", `fps=${PICTURE_FPS},scale=${w}:${h}:flags=area,format=gray`, "-f", "rawvideo", "-pix_fmt", "gray", "-"]);
+  const diffs = meanAbsDiffs(new Uint8Array(frames.stdout.buffer, frames.stdout.byteOffset, frames.stdout.length), w * h);
+  const base = soundReport(samples, AUDIO_RATE, diffs, PICTURE_FPS);
+  const voice = words?.length ? voiceAgainstMusic(samples, AUDIO_RATE, words) : undefined;
+  const report: SoundReport = voice ? { ...base, voice } : base;
+  const lufs = loudness ?? (await loudnessLufs(path));
+  return lufs === undefined ? report : { ...report, loudnessLufs: lufs };
+}

package/src/project/svgcheck.ts

@@ -0,0 +1,21 @@
+// What a downloaded vector graphic must be before it is saved into a project. The server returns a cleaned SVG, but a file that will be drawn in the
+// renderer's browser is checked here as well, so a mistake or a changed server cannot put a script or an outside link into a film.
+// Returns one plain line saying what is wrong, or undefined when the file is fine.
+export function svgProblem(text: string): string | undefined {
+  const body = text.replace(/^/, "").replace(/<\?xml[\s\S]*?\?>/g, "").replace(/<!--[\s\S]*?-->/g, "").replace(/<!DOCTYPE[^>]*>/gi, "").trim();
+  if (!/^<svg[\s>]/i.test(body) || !/<\/svg\s*>\s*$/i.test(body) && !/^<svg[^>]*\/>\s*$/i.test(body)) return "The file is not an SVG: it does not start with an <svg> element. Nothing was saved; run the command again.";
+  if (/<script/i.test(body)) return "The SVG contains a <script>. Nothing was saved; run the command again.";
+  if (/<foreignObject/i.test(body)) return "The SVG contains a <foreignObject>. Nothing was saved; run the command again.";
+  if (/\son[a-z]+\s*=/i.test(body)) return "The SVG contains an event attribute (such as onclick). Nothing was saved; run the command again.";
+  for (const m of body.matchAll(/\s(?:xlink:)?(?:href|src)\s*=\s*("([^"]*)"|'([^']*)')/gi)) {
+    const value = (m[2] ?? m[3] ?? "").trim();
+    if (!(value.startsWith("#") || /^data:image\//i.test(value))) return `The SVG points outside the file (${value.slice(0, 40)}). Nothing was saved; run the command again.`;
+  }
+  // A style may also reach out: url(...) other than a #fragment or a data: image, and @import.
+  for (const m of body.matchAll(/url\(\s*(['"]?)([^'")]*)\1\s*\)/gi)) {
+    const value = m[2]!.trim();
+    if (!(value.startsWith("#") || /^data:image\//i.test(value))) return `The SVG points outside the file (${value.slice(0, 40)}). Nothing was saved; run the command again.`;
+  }
+  if (/@import/i.test(body)) return "The SVG imports a style from outside the file. Nothing was saved; run the command again.";
+  return undefined;
+}

package/src/remotion/kit/Assemble3D.tsx

@@ -0,0 +1,92 @@
+import React from "react";
+import { useCurrentFrame, useVideoConfig } from "remotion";
+import { Color, Object3D, type InstancedMesh } from "three";
+import { CameraKeysContext } from "./scene3d-context";
+import { hash01 } from "./seeded";
+import { textSlots } from "./sample-text";
+import { assembleRanks, centred, MAX_PIECES, pieceAt, shapeSlots, type AssembleOrder, type AssembleShape } from "./three-fx-math";
+import { nearestVisible, type Vec3 } from "./three-math";
+
+// Instanced blocks that fly from a seeded scattered cloud into an arrangement, one after another, each on an arc with a turn that unwinds, then rest: the build-up
+// beat, up to 4000 pieces in one draw. `shape` is "grid" (a slab), "ring" (a band), "sphere", "wall" (bricks) or "text" (the blocks fall inside the letters of `text`, a
+// short Latin word in the kit's typeface), or give `targets` (any [x, y, z] list; it is centred and fitted). `count` (400 unless set, at most 4000) is how many pieces;
+// `order` is who goes first: "y" bottom to top (default), "x", "radial" from the middle out or "random"; `from` is the frame it starts (default 0) and `frames` how long
+// the whole build takes (default 60: every piece is home at from + frames); `colors` are two or three colours, jittered a little per piece; `fit` is how much of the
+// frame's width the arrangement fills at the closest the Scene3D camera comes (0.8), or give `size` in scene units. `spread` is how far the cloud reaches, in arrangements
+// (1.6). Everything is seeded (`seed`), so a frame is the same every time. assembleProgress, assembledCount and assembleEnd (exported) give a piece counter and the frame of the lock sound.
+export const Assemble3D: React.FC<{
+  shape?: AssembleShape;
+  text?: string;
+  targets?: Vec3[];
+  count?: number;
+  order?: AssembleOrder;
+  from?: number;
+  frames?: number;
+  colors?: string[];
+  fit?: number;
+  size?: number;
+  spread?: number;
+  seed?: number;
+  position?: Vec3;
+}> = ({ shape = "grid", text = "Relay", targets, count = 400, order = "y", from = 0, frames = 60, colors = ["#7C3AED", "#22D3EE"], fit = 0.8, size, spread = 1.6, seed = 0, position = [0, 0, 0] }) => {
+  const frame = useCurrentFrame();
+  const { width, height } = useVideoConfig();
+  const keys = React.useContext(CameraKeysContext);
+  const n0 = Math.max(1, Math.min(MAX_PIECES, Math.round(count)));
+  const slots = React.useMemo(() => (targets?.length ? centred(targets.slice(0, MAX_PIECES)) : shape === "text" ? textSlots(text, n0) : shapeSlots(shape, n0)), [shape, text, targets, n0]);
+  const n = slots.length;
+  const ranks = React.useMemo(() => assembleRanks(slots, order, seed), [slots, order, seed]);
+  // The arrangement's extent, so that it fits the frame, and the size of a piece from how close its neighbours are.
+  const geo = React.useMemo(() => {
+    let x0 = Infinity, x1 = -Infinity, y0 = Infinity, y1 = -Infinity;
+    for (const p of slots) { x0 = Math.min(x0, p[0]); x1 = Math.max(x1, p[0]); y0 = Math.min(y0, p[1]); y1 = Math.max(y1, p[1]); }
+    const probe = Math.min(n, 48), gaps: number[] = [];
+    for (let a = 0; a < probe; a++) {
+      const i = Math.floor((a / probe) * n);
+      let best = Infinity;
+      for (let j = 0; j < n; j++) if (j !== i) best = Math.min(best, Math.hypot(slots[i]![0] - slots[j]![0], slots[i]![1] - slots[j]![1], slots[i]![2] - slots[j]![2]));
+      if (Number.isFinite(best)) gaps.push(best);
+    }
+    gaps.sort((a, b) => a - b);
+    return { w: Math.max(1e-3, x1 - x0), h: Math.max(1e-3, y1 - y0), gap: gaps[Math.floor(gaps.length / 2)] ?? 0.1 };
+  }, [slots, n]);
+  const view = nearestVisible(keys, width / height, position[2]);
+  const scale = size ? size / geo.w : Math.min((fit * view.width) / geo.w, (fit * view.height) / geo.h);
+  const piece = geo.gap * 0.84;
+  const mesh = React.useRef<InstancedMesh>(null);
+  const dummy = React.useMemo(() => new Object3D(), []);
+  const palette = React.useMemo(() => colors.map((c) => new Color(c)), [colors]);
+  React.useLayoutEffect(() => {
+    const m = mesh.current;
+    if (!m) return;
+    const c = new Color();
+    for (let i = 0; i < n; i++) {
+      const t = hash01(i, seed + 61) * (palette.length - 1), k = Math.min(palette.length - 2, Math.floor(t));
+      c.copy(palette[Math.max(0, k)]!).lerp(palette[Math.min(palette.length - 1, k + 1)]!, palette.length > 1 ? t - k : 0);
+      c.offsetHSL(0, 0, (hash01(i, seed + 62) - 0.5) * 0.16);
+      m.setColorAt(i, c);
+    }
+    if (m.instanceColor) m.instanceColor.needsUpdate = true;
+  }, [palette, n, seed]);
+  React.useLayoutEffect(() => {
+    const m = mesh.current;
+    if (!m) return;
+    for (let i = 0; i < n; i++) {
+      const s = pieceAt(frame, i, slots[i]!, { ranks, from, frames, seed, spread });
+      dummy.position.set(s.position[0] * scale, s.position[1] * scale, s.position[2] * scale);
+      dummy.rotation.set(s.rotation[0], s.rotation[1], s.rotation[2]);
+      dummy.scale.setScalar(piece * scale * s.scale);
+      dummy.updateMatrix();
+      m.setMatrixAt(i, dummy.matrix);
+    }
+    m.instanceMatrix.needsUpdate = true;
+  }, [frame, n, slots, ranks, from, frames, seed, spread, scale, piece, dummy]);
+  return (
+    <group position={position}>
+      <instancedMesh ref={mesh} args={[undefined, undefined, n]} frustumCulled={false}>
+        <boxGeometry args={[1, 1, 1]} />
+        <meshStandardMaterial roughness={0.42} metalness={0.12} />
+      </instancedMesh>
+    </group>
+  );
+};

package/src/remotion/kit/BrowserFrame.tsx

@@ -0,0 +1,83 @@
+import React from "react";
+import { continueRender, delayRender, interpolate, spring, useCurrentFrame, useVideoConfig } from "remotion";
+import { Place } from "./Place";
+import { useMedia } from "./media";
+import { fonts } from "./theme";
+import { DEFAULT_HERO, rgba, uiColors, type UiTheme } from "./ui-theme";
+import { PANEL_SPRING, inOutCubic, browserFrames } from "./ui-math";
+
+// The natural size of an image, read once. The frame waits for it, so every frame knows how far the picture can scroll.
+function useImageSize(url: string): { w: number; h: number } | null {
+  const [size, setSize] = React.useState<{ w: number; h: number } | null>(null);
+  React.useEffect(() => {
+    const handle = delayRender(`size of ${url.slice(0, 40)}`);
+    const img = new Image();
+    img.onload = () => { setSize({ w: img.naturalWidth, h: img.naturalHeight }); continueRender(handle); };
+    img.onerror = () => continueRender(handle);
+    img.src = url;
+    return () => { continueRender(handle); };
+  }, [url]);
+  return size;
+}
+
+// A browser window (or, with device="phone", a phone) in perspective, carrying your own screenshot `src` (a project path or urls[...]). The picture starts at
+// its top and scrolls down by `scroll`, a fraction of the window's height (default 0.7), over `frames` (default 90) frames from `delay`; it never scrolls past
+// the end of the picture. A soft shadow lies under it and a sheen sweeps across once as it lands. `tilt` is the turn about the vertical axis in degrees
+// (default -12 for a browser, -16 for a phone; 0 faces the camera), `url` the text of the address bar. CSS 3D, no three.js. Times are frames from the
+// component's own mount; browserFrames({ delay }) gives the frame it settles and the sheen's span. `width` is a fraction of the frame's width (0.86 for a
+// browser, 0.5 for a phone); `theme` "dark" (default) or "light" colours the frame.
+export const BrowserFrame: React.FC<{
+  src: string;
+  device?: "browser" | "phone";
+  scroll?: number;
+  frames?: number;
+  delay?: number;
+  tilt?: number;
+  url?: string;
+  theme?: UiTheme;
+  hero?: string;
+  font?: string;
+  width?: number;
+  viewAspect?: number;
+  y?: number;
+}> = ({ src, device = "browser", scroll = 0.7, frames = 90, delay = 0, tilt, url = "relay.app", theme = "dark", hero = DEFAULT_HERO, font = fonts.body, width: given, viewAspect, y = 0.5 }) => {
+  const frame = useCurrentFrame();
+  const { fps, width: W, height: H } = useVideoConfig();
+  const link = useMedia(src);
+  const size = useImageSize(link);
+  const c = uiColors(theme, hero);
+  const phone = device === "phone";
+  const P = W * (given ?? (phone ? 0.5 : 0.86));
+  const turn = tilt ?? (phone ? -16 : -12);
+  const p = spring({ frame: frame - delay, fps, config: PANEL_SPRING });
+  const bar = phone ? 0 : P * 0.085, bezel = phone ? P * 0.035 : 0;
+  const viewW = P - bezel * 2, viewH = viewW * (viewAspect ?? (phone ? 2.05 : 0.78));
+  const imgH = size ? (viewW * size.h) / size.w : viewH;
+  const travel = Math.max(0, Math.min(scroll * viewH, imgH - viewH));
+  const ev = browserFrames({ delay, fps });
+  const sc = inOutCubic((frame - delay - 10) / Math.max(1, frames)) * travel;
+  const sheen = interpolate(frame, [ev.sheenStart, ev.sheenEnd], [-0.4, 1.4], { extrapolateLeft: "clamp", extrapolateRight: "clamp" });
+  const totalH = bar + viewH + bezel * 2;
+  return (
+    <Place y={y}>
+      <div style={{ position: "relative", width: P, height: totalH, perspective: P * 2.4, opacity: Math.min(1, p * 1.5) }}>
+        <div style={{ position: "absolute", left: "6%", right: "6%", bottom: -P * 0.07, height: P * 0.1, background: "radial-gradient(ellipse at 50% 50%, rgba(0,0,0,0.55), rgba(0,0,0,0) 70%)", filter: `blur(${P * 0.03}px)`, transform: `translateY(${(1 - p) * H * 0.05}px)` }} />
+        <div style={{ position: "absolute", inset: 0, transform: `translateY(${(1 - p) * H * 0.06}px) rotateY(${turn * p}deg) rotateX(${Math.abs(turn) * 0.38 * p}deg)`, transformStyle: "preserve-3d" }}>
+          <div style={{ position: "absolute", inset: 0, borderRadius: phone ? P * 0.12 : P * 0.035, background: phone ? "#0b0b10" : c.panel, border: `${Math.max(2, P * 0.004)}px solid ${phone ? "rgba(255,255,255,0.28)" : c.stroke}`, boxShadow: `0 ${P * 0.06}px ${P * 0.14}px ${c.shadow}, 0 0 ${P * 0.18}px ${rgba(hero, 0.2)}`, overflow: "hidden" }}>
+            {!phone ? (
+              <div style={{ height: bar, display: "flex", alignItems: "center", gap: P * 0.015, padding: `0 ${P * 0.035}px`, background: c.field, borderBottom: `1px solid ${c.stroke}` }}>
+                {["#ff5f57", "#febc2e", "#28c840"].map((d) => <div key={d} style={{ width: P * 0.022, height: P * 0.022, borderRadius: 999, background: d }} />)}
+                <div style={{ flex: 1, margin: `0 ${P * 0.03}px 0 ${P * 0.03}px`, background: c.chip, borderRadius: 999, padding: `${P * 0.008}px ${P * 0.03}px`, fontFamily: font, fontSize: P * 0.032, color: c.dim, textAlign: "center", fontWeight: 500 }}>{url}</div>
+              </div>
+            ) : null}
+            <div style={{ position: "absolute", left: bezel, right: bezel, top: bar + bezel, height: viewH, overflow: "hidden", borderRadius: phone ? P * 0.09 : 0, background: "#ffffff" }}>
+              <img src={link} style={{ display: "block", width: "100%", height: "auto", transform: `translateY(${-sc}px)` }} />
+              {phone ? <div style={{ position: "absolute", left: "50%", top: viewW * 0.03, width: viewW * 0.3, height: viewW * 0.085, borderRadius: 999, background: "#000", transform: "translateX(-50%)" }} /> : null}
+            </div>
+            <div style={{ position: "absolute", inset: 0, pointerEvents: "none", background: `linear-gradient(105deg, rgba(255,255,255,0) ${sheen * 100 - 14}%, rgba(255,255,255,0.22) ${sheen * 100}%, rgba(255,255,255,0) ${sheen * 100 + 14}%)` }} />
+          </div>
+        </div>
+      </div>
+    </Place>
+  );
+};

package/src/remotion/kit/Camera.tsx

@@ -1,22 +1,24 @@
 import React from "react";
 import { AbsoluteFill, useCurrentFrame, useVideoConfig } from "remotion";
-import { cameraAt, cameraTransform, type CameraKey } from "./motion-math";
+import { applyPunches, cameraAt, cameraTransform, type CameraKey, type Punch } from "./motion-math";
 
 // Moves the whole picture. Wrap all the scenes in it for one camera that never cuts, or wrap one scene's content.
 // Each key gives the point of the content, as fractions of the frame, that sits at the middle of the screen (x, y), the zoom (1 shows
 // the content as it fits) and a turn in degrees; what a key leaves out stays as it was. The move toward a key starts `lead` frames
 // before its frame (12 unless set) on a spring and lands on the key's frame. drift is a very slow push on top, as a fraction of the
-// zoom per second (0.02 is two percent a second), so a held shot is never perfectly still.
+// zoom per second (0.02 is two percent a second), so a held shot is never perfectly still. punches are quick emphasis pushes of 8 to 15 percent onto a
+// point, { frame, x, y, amount?: 0.1, frames?: 5, hold?: 6 }, on top of the keys. To push into an element until it fills the frame, put zoomTo(frame, box) in keys.
 export const Camera: React.FC<{
   keys: CameraKey[];
   drift?: number;
+  punches?: Punch[];
   lead?: number;
   stiffness?: number;
   damping?: number;
   children: React.ReactNode;
-}> = ({ keys, drift, lead, stiffness, damping, children }) => {
+}> = ({ keys, drift, punches, lead, stiffness, damping, children }) => {
   const frame = useCurrentFrame();
   const { fps, width, height } = useVideoConfig();
-  const { origin, transform } = cameraTransform(cameraAt(keys, frame, fps, { lead, stiffness, damping, drift }), width, height);
+  const { origin, transform } = cameraTransform(applyPunches(cameraAt(keys, frame, fps, { lead, stiffness, damping, drift }), punches ?? [], frame), width, height);
   return <AbsoluteFill style={{ transformOrigin: origin, transform }}>{children}</AbsoluteFill>;
 };