reelson 0.7.0

package/skills/reelson-compose/scripts/voice.ts

@@ -0,0 +1,224 @@
+/**
+ * Voice-over: each callout spoken as it appears (video.json "voice": true).
+ *
+ * Lines are spoken by a text-to-speech provider (demo.config.json `voice.provider`: OpenAI,
+ * ElevenLabs, a local Piper voice or any local command — see tts.ts), trimmed of the silence
+ * around them and cached in <demo>/voice/<hash>.mp3 — the hash covers the words and every
+ * setting that changes the sound, so a line is only made once and a re-render never calls the
+ * provider. `reelson voice <slug>` (and `render`) make the missing lines; the build (sync)
+ * mixes whatever is cached into one track per version (landscape, portrait, square: each has
+ * its own timing), and says what is missing.
+ *
+ * What is spoken: a callout's `say` (false: nothing), else its text; plus video.json
+ * `voice.intro` over the intro, when given.
+ */
+import { spawnSync } from 'node:child_process'
+import { createHash } from 'node:crypto'
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join, resolve } from 'node:path'
+import type { LoadedConfig } from '../../reelson-record/scripts/config.ts'
+import { ReelsonError } from './project.ts'
+import { round, type Timeline, type VideoSpec } from './timeline.ts'
+import { missingSetup, providerDefaults, speak, type SpeechSettings } from './tts.ts'
+
+export interface VoiceSettings extends SpeechSettings {
+    /** Loudness of the voice track. */
+    lufs: number
+}
+
+export interface VoiceLine {
+    /** Composition seconds the line starts. */
+    at: number
+    text: string
+    /** Cache file name: <hash>.mp3 in <demo>/voice/. */
+    file: string
+    /** When the next line (or the end of the recording) comes — the line should be done by then. */
+    until: number
+    /** The line over the intro (the steps' lines are checked by the timeline). */
+    intro?: true
+}
+
+/** A line starts this long after its callout appears (the pill is read, then heard). */
+const LEAD_IN = 0.25
+
+/** The voice settings for a video, or null when it has no voice-over. */
+export function voiceSettings(spec: VideoSpec, config: LoadedConfig): VoiceSettings | null {
+    if (!spec.voice) {
+        return null
+    }
+    const own = typeof spec.voice === 'object' ? spec.voice : {}
+    const project = config.voice
+    const provider = own.provider ?? project.provider
+    // A video switching provider starts from that provider's defaults, not the project's voice.
+    const same = provider === project.provider
+    const defaults = providerDefaults(provider, config.language)
+    return {
+        provider,
+        model: own.model ?? (same ? project.model : undefined) ?? defaults.model,
+        voice: own.voice ?? (same ? project.voice : undefined) ?? defaults.voice,
+        instructions: own.instructions ?? project.instructions,
+        ...((own.speed ?? project.speed) !== undefined ? { speed: own.speed ?? project.speed } : {}),
+        language: config.language,
+        ...(same && project.baseURL ? { baseURL: project.baseURL } : {}),
+        ...(same && project.command ? { command: project.command } : {}),
+        ...(same && project.options ? { options: project.options } : {}),
+        lufs: project.lufs,
+    }
+}
+
+/** Everything spoken in this video, in order, timed to `t`. Pure. */
+export function voiceLines(t: Timeline, spec: VideoSpec, settings: VoiceSettings): VoiceLine[] {
+    const intro = typeof spec.voice === 'object' ? spec.voice.intro : undefined
+    const spoken: { at: number; text: string; intro?: true }[] = [
+        ...(intro ? [{ at: round(t.intro.start + 0.4), text: intro, intro: true as const }] : []),
+        ...t.callouts.map((c) => ({ at: round(c.at + LEAD_IN), text: lineOf(c) })),
+    ].filter((l) => l.text)
+    return spoken.map((line, i) => ({
+        ...line,
+        file: `${lineHash(line.text, settings)}.mp3`,
+        // The intro line runs until the recording comes in; a step's, until the next line.
+        until: round(i === 0 && intro ? t.clipStart + 0.6 : (spoken[i + 1]?.at ?? t.clipEnd)),
+    }))
+}
+
+/** Bumped when the post-processing (the trim) changes, so cached lines are made again. */
+const CACHE_VERSION = 2
+
+/** The cache name of a line: everything that changes how it sounds (not the loudness: the mix sets it). */
+export function lineHash(text: string, s: VoiceSettings): string {
+    const { lufs: _, ...sound } = s
+    return createHash('sha1')
+        .update(JSON.stringify([CACHE_VERSION, sound, text]))
+        .digest('hex')
+        .slice(0, 16)
+}
+
+/** What a callout says (its `say`, else its text); '' when it is silent. */
+function lineOf(c: { text: string; say?: string | false }): string {
+    return c.say === false ? '' : (c.say || c.text).trim()
+}
+
+/**
+ * For the timeline: seconds from a callout appearing until its cached line is done (the lead-in
+ * plus the line); 0 when it is silent or not spoken yet.
+ */
+export function spokenLength(settings: VoiceSettings, cacheDir: string): (c: { text: string; say?: string | false }) => number {
+    const lengths = new Map<string, number>()
+    return (c) => {
+        const text = lineOf(c)
+        const file = resolve(cacheDir, `${lineHash(text, settings)}.mp3`)
+        if (!text || !existsSync(file)) {
+            return 0
+        }
+        if (!lengths.has(file)) {
+            lengths.set(file, lengthOf(file))
+        }
+        return round(LEAD_IN + (lengths.get(file) as number))
+    }
+}
+
+/** Every line the video would speak, without a timeline (for fetching). */
+export function spokenTexts(spec: VideoSpec, callouts: { text: string; say?: string | false }[]): string[] {
+    const intro = typeof spec.voice === 'object' ? spec.voice.intro : undefined
+    return [intro, ...callouts.map(lineOf)].filter((t): t is string => !!t)
+}
+
+/**
+ * Speaks the lines missing from `cacheDir` (see tts.ts). Returns how many it made.
+ * Throws a ReelsonError when the provider cannot speak here (only when something is missing).
+ */
+export async function fetchLines(texts: string[], settings: VoiceSettings, cacheDir: string, log: (l: string) => void): Promise<number> {
+    const missing = [...new Set(texts)].filter((text) => !existsSync(resolve(cacheDir, `${lineHash(text, settings)}.mp3`)))
+    if (!missing.length) {
+        return 0
+    }
+    const problem = missingSetup(settings)
+    if (problem) {
+        throw new ReelsonError(`voice-over: ${missing.length} line(s) to speak with ${settings.provider} and ${problem} (or turn "voice" off in video.json)`)
+    }
+    mkdirSync(cacheDir, { recursive: true })
+    const scratch = mkdtempSync(join(tmpdir(), 'reelson-voice-'))
+    try {
+        for (const text of missing) {
+            const raw = resolve(scratch, settings.provider === 'openai' || settings.provider === 'elevenlabs' ? 'line.mp3' : 'line.wav')
+            rmSync(raw, { force: true })
+            await speak(text, settings, raw)
+            trimInto(raw, resolve(cacheDir, `${lineHash(text, settings)}.mp3`))
+            log(`  voice: spoke "${text}" (${settings.provider})`)
+        }
+    } finally {
+        rmSync(scratch, { recursive: true, force: true })
+    }
+    return missing.length
+}
+
+/** The silence around a line, cut (a breath of it kept) and saved as mp3. */
+const TRIM = 'silenceremove=start_periods=1:start_threshold=-50dB:start_silence=0.05'
+
+function trimInto(raw: string, target: string): void {
+    const run = spawnSync(
+        'ffmpeg',
+        ['-y', '-loglevel', 'error', '-i', raw, '-af', `${TRIM},areverse,${TRIM},areverse`, '-ar', '48000', '-c:a', 'libmp3lame', '-q:a', '2', target],
+        { stdio: 'inherit' },
+    )
+    if (run.status !== 0 || !existsSync(target)) {
+        throw new ReelsonError(`ffmpeg could not read the spoken line ${raw}`)
+    }
+}
+
+/** Seconds of audio in `file` (ffprobe). */
+function lengthOf(file: string): number {
+    const probe = spawnSync('ffprobe', ['-v', 'error', '-show_entries', 'format=duration', '-of', 'csv=p=0', file], { encoding: 'utf8' })
+    return Number.parseFloat(probe.stdout.trim()) || 0
+}
+
+/**
+ * Mixes the cached lines into one track as long as the video (`target`, m4a). Returns false
+ * when none is cached. Warns about missing lines and an intro line that outlasts the intro.
+ */
+export function renderVoiceTrack(
+    lines: VoiceLine[],
+    settings: VoiceSettings,
+    cacheDir: string,
+    total: number,
+    target: string,
+    log: (l: string) => void,
+): boolean {
+    const cached = lines.filter((l) => existsSync(resolve(cacheDir, l.file)))
+    const missing = lines.length - cached.length
+    if (missing) {
+        log(`  warning: voice-over: ${missing} line(s) not spoken yet — run \`reelson voice <slug>\` (render does it for you)`)
+    }
+    if (!cached.length) {
+        return false
+    }
+    for (const line of cached.filter((l) => l.intro)) {
+        const length = lengthOf(resolve(cacheDir, line.file))
+        if (line.at + length > line.until + 0.05) {
+            log(`  warning: voice-over: "${line.text}" (${length.toFixed(1)}s) runs ${(line.at + length - line.until).toFixed(1)}s into the recording — shorten \`voice.intro\``)
+        }
+    }
+    const key = JSON.stringify({ lines: cached.map((l) => [l.at, l.file]), total, lufs: settings.lufs, v: 2 })
+    const stamp = `${target}.key`
+    if (existsSync(target) && existsSync(stamp) && readFileSync(stamp, 'utf8') === key) {
+        return true
+    }
+    const inputs = cached.flatMap((l) => ['-i', resolve(cacheDir, l.file)])
+    const delayed = cached.map((l, i) => `[${i}:a]aresample=48000,adelay=${Math.round(l.at * 1000)}:all=1[v${i}]`).join(';')
+    // 0.1 s past the video: AAC's encoder padding would otherwise leave the track a frame or
+    // two short of its slot. loudnorm works at 192 kHz, hence the resample back.
+    const mix =
+        `${delayed};${cached.map((_, i) => `[v${i}]`).join('')}amix=inputs=${cached.length}:normalize=0,` +
+        `loudnorm=I=${settings.lufs}:TP=-1.5:LRA=11,aresample=48000,apad,atrim=0:${round(total + 0.1)}[out]`
+    const render = spawnSync(
+        'ffmpeg',
+        ['-y', '-loglevel', 'error', ...inputs, '-filter_complex', mix, '-map', '[out]', '-c:a', 'aac', '-b:a', '160k', target],
+        { stdio: 'inherit' },
+    )
+    if (render.status !== 0) {
+        throw new ReelsonError('ffmpeg could not mix the voice-over')
+    }
+    writeFileSync(stamp, key)
+    return true
+}

package/skills/reelson-compose/scripts/zooms.js

@@ -0,0 +1,259 @@
+import { round } from "./timeline.js";
+export const ZOOM_RULES = {
+    defaultEase: 0.8, // the template's default zoom in/out duration
+    minEase: 0.4, // shorter reads as a jump
+    settle: 0.1, // the zoom is done this long before the click
+    early: 0.35, // may start this long before the glide (not more: no waiting)
+    linger: 1.2, // with no next target, hold this long after the last action
+    absorb: 1.0, // a visible later click whose glide starts within this joins the hold
+    maxIdle: 3.0, // a longer wait for the next glide: linger and zoom out, don't sit zoomed
+    eps: 0.005, // float slack (13.03 + 4.36 must not fail against 17.39)
+    margin: 0.03, // keep clicks this far (fraction of the frame) inside the view
+};
+const R = ZOOM_RULES;
+/**
+ * How a following zoom pans: sampled every `step` s, easing towards the cursor with `lag` s
+ * of smoothing, aimed `lead` s ahead — where the cursor is going, not where it was (which
+ * also cancels the smoothing's delay, and keeps the zoom-in on the click it glides to).
+ */
+export const FOLLOW = { step: 1 / 15, lag: 0.45, lead: 0.45 };
+/**
+ * The focus path of a zoom that follows the cursor: at each sample, the transform-origin
+ * that puts the cursor in the middle of the zoomed view (clamped so the view never leaves
+ * the frame), smoothed so the camera glides instead of jittering. Starts from the planned
+ * focus. Pure; `cursor` is the timeline's cursor layer (composition s, recording px).
+ */
+export function followPath(z, cursor, viewport) {
+    const at = (t) => {
+        const path = cursor.path;
+        let i = path.findIndex((p) => p[0] > t);
+        if (i === -1)
+            i = path.length;
+        const a = path[Math.max(0, i - 1)];
+        const b = path[i];
+        if (!b || i === 0 || b[0] - a[0] > 0.1) {
+            return [a[1], a[2]];
+        }
+        const u = (t - a[0]) / (b[0] - a[0]);
+        return [a[1] + (b[1] - a[1]) * u, a[2] + (b[2] - a[2]) * u];
+    };
+    const originFor = (p) => Math.min(1, Math.max(0, (p - 0.5 / z.scale) / (1 - 1 / z.scale)));
+    const alpha = 1 - Math.exp(-FOLLOW.step / FOLLOW.lag);
+    const end = z.at + z.duration;
+    let ox = z.x;
+    let oy = z.y;
+    const out = [[round(z.at), z.x, z.y]];
+    for (let t = z.at + FOLLOW.step; t < end + 1e-9; t += FOLLOW.step) {
+        const [cx, cy] = at(Math.min(t + FOLLOW.lead, end));
+        ox += (originFor(cx / viewport.width) - ox) * alpha;
+        oy += (originFor(cy / viewport.height) - oy) * alpha;
+        out.push([Math.round(t * 1000) / 1000, Math.round(ox * 10000) / 10000, Math.round(oy * 10000) / 10000]);
+    }
+    return out;
+}
+export class ZoomError extends Error {
+}
+export function compositionClicks(markers, timeline) {
+    if (!markers.clicks || markers.clicks.some((c) => c.move === undefined)) {
+        return [];
+    }
+    return markers.clicks
+        .map((c, i) => ({ c, index: i + 1 }))
+        .filter(({ c }) => c.at >= timeline.mediaStart && c.at <= timeline.mediaEnd)
+        .map(({ c, index }) => ({
+        index,
+        kind: c.kind,
+        comp: timeline.toComposition(c.at),
+        glide: timeline.toComposition(c.move ?? c.at),
+        until: timeline.toComposition(c.until ?? c.at),
+        fx: c.x / timeline.viewport.width,
+        fy: c.y / timeline.viewport.height,
+    }));
+}
+/** Resolves a video.json zoom (click-anchored or manual) to composition time. */
+export function planZoom(spec, clicks, timeline) {
+    const scale = spec.scale;
+    if (!spec.clicks) {
+        if (spec.at === undefined || spec.duration === undefined || spec.x === undefined || spec.y === undefined) {
+            throw new ZoomError('a zoom needs `clicks`, or all of `at`, `duration`, `x`, `y` (manual)');
+        }
+        return {
+            at: timeline.toComposition(spec.at),
+            duration: spec.duration,
+            x: spec.x,
+            y: spec.y,
+            scale,
+            in: spec.in ?? R.defaultEase,
+            out: spec.out ?? R.defaultEase,
+        };
+    }
+    if (spec.at !== undefined || spec.duration !== undefined) {
+        throw new ZoomError('a zoom with `clicks` is timed automatically; drop `at`/`duration` (or drop `clicks` for a manual zoom)');
+    }
+    const [a, b = a] = spec.clicks;
+    if (b < a) {
+        throw new ZoomError(`clicks [${a}, ${b}]: the first number must not be after the second`);
+    }
+    const byIndex = (n) => {
+        const found = clicks.find((c) => c.index === n);
+        if (!found) {
+            const available = clicks.map((c) => c.index);
+            throw new ZoomError(`click ${n} is not in the trimmed recording (available: ${available.length ? `${available[0]}–${available.at(-1)}` : 'none — re-record to log clicks'})`);
+        }
+        return found;
+    };
+    const first = byIndex(a);
+    byIndex(b);
+    const selected = clicks.filter((c) => c.index >= a && c.index <= b);
+    // A following zoom pans to each click, so they need not fit in one view: it starts with
+    // the first click in the middle of the view.
+    const centred = (p) => Math.round(Math.min(1, Math.max(0, (p - 0.5 / scale) / (1 - 1 / scale))) * 10000) / 10000;
+    const x = spec.x ?? (spec.follow ? centred(first.fx) : fitFocus(selected.map((c) => c.fx), scale, `clicks ${a}–${b}`, 'wide'));
+    const y = spec.y ?? (spec.follow ? centred(first.fy) : fitFocus(selected.map((c) => c.fy), scale, `clicks ${a}–${b}`, 'tall'));
+    const visible = spec.follow ? () => true : isVisibleIn(viewOf(x, y, scale));
+    const zoomIn = spec.in ?? R.defaultEase;
+    const at = spec.in !== undefined
+        ? round(first.comp - R.settle - zoomIn)
+        : round(Math.max(first.glide - R.early, first.comp - R.settle - R.defaultEase));
+    const inEase = spec.in ?? round(Math.max(R.minEase, Math.min(R.defaultEase, first.comp - R.settle - at)));
+    // The zoom holds its clicks, plus later ones it still shows that follow right away
+    // (zooming out and straight back in would jump); the first other click ends it.
+    let holdEnd = Math.max(...selected.map((c) => c.until));
+    let next;
+    for (const c of clicks.filter((c) => c.index > b)) {
+        if (visible(c) && c.glide - holdEnd <= R.absorb) {
+            holdEnd = Math.max(holdEnd, c.until);
+            continue;
+        }
+        next = c;
+        break;
+    }
+    // The recording belts out before a hand-off card and fades out at the end.
+    const nextCard = timeline.transitions.find((t) => t.at > first.comp);
+    const limit = Math.min(timeline.clipEnd - 0.1, nextCard ? nextCard.at - timeline.belt : Infinity);
+    let outEnd;
+    let outEase;
+    if (next && next.comp - R.settle <= limit && next.glide - holdEnd <= R.maxIdle) {
+        outEnd = next.comp - R.settle;
+        const outStart = Math.max(next.glide - R.early, outEnd - R.defaultEase, holdEnd + 0.2);
+        outEase = spec.out ?? Math.max(R.minEase, outEnd - outStart);
+    }
+    else {
+        outEase = spec.out ?? R.defaultEase;
+        outEnd = Math.min(holdEnd + R.linger + outEase, limit);
+    }
+    return {
+        at,
+        duration: round(outEnd - at),
+        x,
+        y,
+        scale,
+        in: round(inEase),
+        out: round(outEase),
+    };
+}
+/**
+ * Zooms on screen at the same time (they would fight over the frame): one message per
+ * overlapping pair, keyed by the 0-based index of the later zoom in video.json order.
+ */
+export function zoomOverlaps(zooms) {
+    const found = [];
+    zooms.forEach((z, j) => {
+        zooms.forEach((other, i) => {
+            if (i < j && z.at < round(other.at + other.duration) && other.at < round(z.at + z.duration)) {
+                found.push({
+                    index: j,
+                    message: `overlaps zoom ${i + 1} (${other.at}–${round(other.at + other.duration)}s) — one zoom over both click ranges, or drop one`,
+                });
+            }
+        });
+    });
+    return found;
+}
+/** Verifies a zoom against the clicks; problems carry a copy-paste fix. */
+export function checkZoom(z, clicks, timeline) {
+    const end = round(z.at + z.duration);
+    const visible = isVisibleIn(viewOf(z.x, z.y, z.scale));
+    const describe = (c) => `${c.kind} #${c.index} at ${c.comp}s (glide from ${c.glide}s, ${Math.round(c.fx * 100)}%, ${Math.round(c.fy * 100)}%)`;
+    const problems = [];
+    const inside = clicks.filter((c) => c.comp >= z.at && c.comp <= end);
+    // A following zoom keeps the cursor in view: every click in it is framed.
+    const framed = z.path ? inside : inside.filter(visible);
+    const first = framed[0];
+    const last = framed.at(-1);
+    const next = clicks.find((c) => c.comp > (last?.comp ?? z.at) && !(c.comp <= end && (z.path || visible(c))));
+    if (!first) {
+        problems.push('frames no click — anchor it with `clicks`, or drop the zoom');
+    }
+    else {
+        const fix = ` → use { "clicks": [${first.index}, ${(last ?? first).index}], "scale": ${z.scale} }`;
+        if (z.at + z.in > first.comp - R.settle + R.eps) {
+            problems.push(`zoom-in not done before the ${describe(first)}${fix}`);
+        }
+        else if (z.at < first.glide - R.early - R.eps) {
+            problems.push(`zoom-in starts before the cursor moves (the viewer waits)${fix}`);
+        }
+        for (const c of inside) {
+            if (!visible(c) && !z.path) {
+                problems.push(`${describe(c)} is outside the zoomed view${fix}`);
+            }
+            else if (c.comp > end - z.out + R.eps) {
+                problems.push(`${describe(c)} happens while the zoom eases out${fix}`);
+            }
+            else if (c.until > end - z.out + R.eps) {
+                problems.push(`typing (${describe(c)}) is still going when the zoom eases out${fix}`);
+            }
+        }
+        if (next && end > next.comp - R.settle + R.eps && !inside.includes(next)) {
+            problems.push(`zoom-out not done before the ${describe(next)}${fix}`);
+        }
+        else if (next && end - z.out < next.glide - R.early - R.eps && end > next.glide - R.early + R.eps) {
+            // Easing out across the start of the glide without riding it (a zoom that is fully
+            // out before the cursor even moves is fine: it lingered, then left).
+            problems.push(`zoom-out starts before the cursor heads for the next target${fix}`);
+        }
+    }
+    if (z.in < R.minEase || z.out < R.minEase) {
+        problems.push(`ease shorter than ${R.minEase}s reads as a jump — add a beat in the scenario instead`);
+    }
+    if (end > timeline.clipEnd + R.eps) {
+        problems.push(`runs past the end of the recording (${timeline.clipEnd}s)`);
+    }
+    for (const t of timeline.transitions) {
+        if (z.at < t.at && end > t.at - timeline.belt + R.eps) {
+            problems.push(`still zoomed when the recording leaves for the hand-off card at ${t.at}s`);
+        }
+    }
+    return { problems, framed };
+}
+/** The part of the frame visible at full zoom (transform-origin = focus point). */
+function viewOf(x, y, scale) {
+    return {
+        left: x - x / scale,
+        right: x + (1 - x) / scale,
+        top: y - y / scale,
+        bottom: y + (1 - y) / scale,
+    };
+}
+function isVisibleIn(view) {
+    return (c) => c.fx >= view.left + R.margin - R.eps &&
+        c.fx <= view.right - R.margin + R.eps &&
+        c.fy >= view.top + R.margin - R.eps &&
+        c.fy <= view.bottom - R.margin + R.eps;
+}
+/**
+ * A focus coordinate (0..1) that keeps every point visible at `scale`, as close
+ * to their centre as possible. With origin o, point p shows iff
+ * o - o/s ≤ p ≤ o + (1 - o)/s.
+ */
+function fitFocus(points, scale, what, axis) {
+    const p1 = Math.min(...points) - R.margin;
+    const p2 = Math.max(...points) + R.margin;
+    const lo = Math.max(0, (p2 * scale - 1) / (scale - 1));
+    const hi = Math.min(1, (p1 * scale) / (scale - 1));
+    if (lo > hi + 1e-9) {
+        throw new ZoomError(`${what} are too far apart (${axis}) to show at ${scale}x — lower the scale or split the zoom`);
+    }
+    const mid = (Math.min(...points) + Math.max(...points)) / 2;
+    return Math.round(Math.min(hi, Math.max(lo, mid)) * 1000) / 1000;
+}