videobook 0.0.0-stage → 0.2.0

package/dist/jobs/export-sequence.js

@@ -0,0 +1,365 @@
+import { mkdtemp, rm, writeFile } from "node:fs/promises";
+import { createHash } from "node:crypto";
+import { extname, join } from "node:path";
+import { tmpdir } from "node:os";
+import { runFfmpeg, probeVideo, AUDIO_OPTS, H264_MEDIUM } from "../ffmpeg.js";
+import { registerJobHandler } from "../queue.js";
+import { replaceArtifactFileFromPath } from "./_persist-output.js";
+import { sequenceExportDimensions } from "../tools/export-sequence.js";
+function rational(value) {
+    return value.numerator / value.denominator;
+}
+function secondsForFrames(sequence, frames) {
+    return frames / rational(sequence.frameRate);
+}
+function sourceSeconds(clip) {
+    if (clip.source.kind === "still")
+        return { start: 0, duration: 0 };
+    const timeBase = rational(clip.source.range.timeBase);
+    return {
+        start: clip.source.range.startTick * timeBase,
+        duration: clip.source.range.durationTicks * timeBase,
+    };
+}
+function escapeFilterPath(path) {
+    return path.replace(/\\/gu, "\\\\").replace(/:/gu, "\\:").replace(/'/gu, "\\'");
+}
+function captionTimestamp(seconds) {
+    const totalMilliseconds = Math.max(0, Math.round(seconds * 1000));
+    const hours = Math.floor(totalMilliseconds / 3_600_000);
+    const minutes = Math.floor(totalMilliseconds % 3_600_000 / 60_000);
+    const secs = Math.floor(totalMilliseconds % 60_000 / 1000);
+    const milliseconds = totalMilliseconds % 1000;
+    return `${String(hours).padStart(2, "0")}:${String(minutes).padStart(2, "0")}:${String(secs).padStart(2, "0")}.${String(milliseconds).padStart(3, "0")}`;
+}
+function webVtt(sequence) {
+    const cues = [...sequence.captions].sort((left, right) => left.timelineStartFrame - right.timelineStartFrame);
+    const body = cues.map((cue) => {
+        const start = secondsForFrames(sequence, cue.timelineStartFrame);
+        const end = secondsForFrames(sequence, cue.timelineStartFrame + cue.durationFrames);
+        return `${captionTimestamp(start)} --> ${captionTimestamp(end)}\n${cue.text}`;
+    }).join("\n\n");
+    return `WEBVTT\n\n${body}${body ? "\n" : ""}`;
+}
+async function resolveClips(engine, sequence, workspace, signal) {
+    const tracks = new Map(sequence.tracks.map((track) => [track.trackId, track]));
+    const resolved = [];
+    for (const clip of sequence.clips.filter((candidate) => candidate.enabled)) {
+        const track = tracks.get(clip.trackId);
+        if (!track || !track.enabled || track.kind === "caption")
+            continue;
+        signal.throwIfAborted();
+        const expectedHash = clip.source.kind === "timed"
+            ? clip.source.range.objectHash
+            : clip.source.objectHash;
+        let sourcePath;
+        if (clip.source.kind === "timed") {
+            const stream = engine.streams.getAtRevision(clip.source.range.streamId, sequence.revision);
+            if (!stream.ok)
+                throw new Error(stream.error.message);
+            if (stream.value.artifactId !== clip.source.artifactId || stream.value.objectHash !== expectedHash) {
+                throw new Error(`Source stream does not match the confirmed clip ${clip.clipId}`);
+            }
+            sourcePath = stream.value.sourcePath;
+        }
+        else {
+            sourcePath = clip.source.sourcePath;
+        }
+        const snapshot = await engine.files.readAtRevision(clip.source.artifactId, sourcePath, sequence.revision);
+        if (!snapshot.ok)
+            throw new Error(snapshot.error.message);
+        if (createHash("sha256").update(snapshot.value).digest("hex") !== expectedHash) {
+            throw new Error(`Source object does not match the confirmed clip ${clip.clipId}`);
+        }
+        const path = join(workspace, `input-${resolved.length}${extname(sourcePath)}`);
+        await writeFile(path, snapshot.value, { signal });
+        const probe = await probeVideo(path, { signal });
+        resolved.push({
+            clip,
+            track,
+            path,
+            inputIndex: resolved.length + 1,
+            hasVideo: probe.videoStream !== null,
+            hasAudio: probe.hasAudio,
+        });
+    }
+    return resolved;
+}
+function inputArgs(sequence, resolved, width, height) {
+    const duration = secondsForFrames(sequence, Math.max(...sequence.clips.map((clip) => clip.timelineStartFrame + clip.durationFrames)));
+    const fps = rational(sequence.frameRate);
+    const args = [
+        "-f", "lavfi",
+        "-i", `color=c=black:s=${width}x${height}:r=${fps}:d=${duration}`,
+    ];
+    for (const item of resolved) {
+        const outputDuration = secondsForFrames(sequence, item.clip.durationFrames);
+        const source = sourceSeconds(item.clip);
+        if (item.clip.source.kind === "still") {
+            args.push("-loop", "1", "-t", String(outputDuration), "-i", item.path);
+        }
+        else {
+            args.push("-ss", String(source.start), "-t", String(source.duration), "-i", item.path);
+        }
+    }
+    return args;
+}
+function videoFilter(sequence, resolved, width, height) {
+    const fps = rational(sequence.frameRate);
+    const filters = [
+        `[0:v]scale=${width}:${height},fps=${fps},format=yuv420p[vbase]`,
+    ];
+    let previous = "vbase";
+    const visual = resolved
+        .filter((item) => item.track.kind === "video" && item.hasVideo)
+        .sort((left, right) => left.track.ordinal - right.track.ordinal
+        || left.clip.timelineStartFrame - right.clip.timelineStartFrame);
+    visual.forEach((item, index) => {
+        const transform = item.clip.transform;
+        const fit = transform?.fit ?? "fit";
+        const crop = transform
+            ? `crop=iw*${1 - transform.cropLeft - transform.cropRight}:ih*${1 - transform.cropTop - transform.cropBottom}:iw*${transform.cropLeft}:ih*${transform.cropTop},`
+            : "";
+        const scale = fit === "fit"
+            ? `scale=${width}:${height}:force_original_aspect_ratio=decrease,pad=${width}:${height}:(ow-iw)/2:(oh-ih)/2:color=black@0`
+            : `scale=${width}:${height}:force_original_aspect_ratio=increase,crop=${width}:${height}`;
+        const duration = secondsForFrames(sequence, item.clip.durationFrames);
+        const start = secondsForFrames(sequence, item.clip.timelineStartFrame);
+        const source = sourceSeconds(item.clip);
+        const sourceDuration = item.clip.source.kind === "timed"
+            ? source.duration
+            : duration;
+        const playbackRate = item.clip.source.kind === "timed"
+            ? item.clip.speed.numerator
+                / item.clip.speed.denominator
+            : 1;
+        const clipLabel = `vclip${index}`;
+        const outputLabel = `vstage${index}`;
+        const opacity = transform?.opacity ?? 1;
+        const reverse = item.clip.source.kind === "timed"
+            && item.clip.reverse
+            ? "reverse,"
+            : "";
+        const transformFilters = transform
+            ? `,scale=iw*${transform.scaleX}:ih*${transform.scaleY},rotate=${transform.rotationDegrees}*PI/180:ow=rotw(iw):oh=roth(ih):c=none`
+            : "";
+        const transition = clipTransitions(sequence, item.clip);
+        const fadeIn = transition.incoming
+            ? `,fade=t=in:st=0:d=${secondsForFrames(sequence, transition.incoming.durationFrames)}:alpha=1`
+            : "";
+        const fadeOut = transition.outgoing
+            ? `,fade=t=out:st=${Math.max(0, duration - secondsForFrames(sequence, transition.outgoing.durationFrames))}:d=${secondsForFrames(sequence, transition.outgoing.durationFrames)}:alpha=1`
+            : "";
+        filters.push(`[${item.inputIndex}:v]${crop}${reverse}${scale}${transformFilters},fps=${fps},trim=duration=${sourceDuration},setpts=(PTS-STARTPTS)/${playbackRate},tpad=stop_mode=clone:stop_duration=${duration},trim=duration=${duration},setpts=PTS+${start}/TB,format=rgba,colorchannelmixer=aa=${opacity}${fadeIn}${fadeOut}[${clipLabel}]`);
+        const x = `(W-w)/2+${transform?.positionX ?? 0}`;
+        const y = `(H-h)/2+${transform?.positionY ?? 0}`;
+        filters.push(`[${previous}][${clipLabel}]overlay=${x}:${y}:enable='between(t,${start},${start + duration})'[${outputLabel}]`);
+        previous = outputLabel;
+    });
+    return { filters, output: previous };
+}
+function clipTransitions(sequence, clip) {
+    return {
+        incoming: sequence.transitions.find((transition) => transition.kind === "dissolve" && transition.incomingClipId === clip.clipId),
+        outgoing: sequence.transitions.find((transition) => transition.kind === "dissolve" && transition.outgoingClipId === clip.clipId),
+    };
+}
+function atempoFilters(speed) {
+    const filters = [];
+    let remaining = speed;
+    while (remaining > 2) {
+        filters.push("atempo=2");
+        remaining /= 2;
+    }
+    while (remaining < 0.5) {
+        filters.push("atempo=0.5");
+        remaining /= 0.5;
+    }
+    if (Math.abs(remaining - 1) > 0.000001) {
+        filters.push(`atempo=${remaining}`);
+    }
+    return filters;
+}
+function audioFilter(sequence, resolved, duration) {
+    const filters = [
+        `anullsrc=r=${sequence.audioSampleRateHz}:cl=stereo,atrim=duration=${duration}[asilence]`,
+    ];
+    const audio = resolved.filter((item) => {
+        if (!item.hasAudio)
+            return false;
+        if (item.track.kind === "audio")
+            return !item.track.muted;
+        const clip = item.clip;
+        if (item.track.kind !== "video" || clip.source.kind !== "timed")
+            return false;
+        return clip.audioPolicy !== "mute";
+    });
+    const labels = ["[asilence]"];
+    audio.forEach((item, index) => {
+        const clipDuration = secondsForFrames(sequence, item.clip.durationFrames);
+        const start = secondsForFrames(sequence, item.clip.timelineStartFrame);
+        const audioSettings = item.clip.audio;
+        const timed = item.clip;
+        const speed = timed.speed.numerator / timed.speed.denominator;
+        const gain = audioSettings?.muted
+            ? 0
+            : 10 ** ((audioSettings?.gainDb ?? 0) / 20);
+        const transitions = clipTransitions(sequence, item.clip);
+        const antiClickFade = Math.min(0.005, clipDuration / 2);
+        const fadeIn = Math.max(antiClickFade, secondsForFrames(sequence, audioSettings?.fadeInFrames ?? 0), transitions.incoming
+            ? secondsForFrames(sequence, transitions.incoming.durationFrames)
+            : 0);
+        const fadeOut = Math.max(antiClickFade, secondsForFrames(sequence, audioSettings?.fadeOutFrames ?? 0), transitions.outgoing
+            ? secondsForFrames(sequence, transitions.outgoing.durationFrames)
+            : 0);
+        const chain = [
+            ...(timed.reverse ? ["areverse"] : []),
+            ...atempoFilters(speed),
+            "apad",
+            `atrim=duration=${clipDuration}`,
+            "asetpts=PTS-STARTPTS",
+            `volume=${gain}`,
+            ...(fadeIn > 0 ? [`afade=t=in:d=${fadeIn}`] : []),
+            ...(fadeOut > 0
+                ? [`afade=t=out:st=${Math.max(0, clipDuration - fadeOut)}:d=${fadeOut}`]
+                : []),
+            `adelay=${Math.round(start * 1000)}|${Math.round(start * 1000)}`,
+        ];
+        const label = `aclip${index}`;
+        filters.push(`[${item.inputIndex}:a]${chain.join(",")}[${label}]`);
+        labels.push(`[${label}]`);
+    });
+    filters.push(`${labels.join("")}amix=inputs=${labels.length}:duration=longest:dropout_transition=0:normalize=0[aout]`);
+    return { filters, output: "aout" };
+}
+function buildSequenceFfmpegArgs(sequence, resolved, width, height, outputPath, captionPath) {
+    const durationFrames = Math.max(...sequence.clips.map((clip) => clip.timelineStartFrame + clip.durationFrames));
+    const duration = secondsForFrames(sequence, durationFrames);
+    const video = videoFilter(sequence, resolved, width, height);
+    const audio = audioFilter(sequence, resolved, duration);
+    const filters = [...video.filters, ...audio.filters];
+    let videoOutput = video.output;
+    if (captionPath) {
+        filters.push(`[${videoOutput}]subtitles='${escapeFilterPath(captionPath)}'[vcaptioned]`);
+        videoOutput = "vcaptioned";
+    }
+    return [
+        ...inputArgs(sequence, resolved, width, height),
+        "-filter_complex", filters.join(";"),
+        "-map", `[${videoOutput}]`,
+        ...H264_MEDIUM,
+        "-pix_fmt", "yuv420p",
+        "-map", `[${audio.output}]`,
+        ...AUDIO_OPTS,
+        "-t", String(duration),
+        "-movflags", "+faststart",
+        "-y", outputPath,
+    ];
+}
+export function registerExportSequenceJob(engine) {
+    registerJobHandler("export_sequence", async (payload, context) => {
+        const request = payload;
+        const snapshot = engine.sequences.getAtRevision(engine.sequences.getPrimary().sequenceId, request.sequenceRevision);
+        if (!snapshot.ok)
+            throw new Error(snapshot.error.message);
+        const sequence = snapshot.value;
+        const [width, height] = sequenceExportDimensions(request.orientation);
+        const workspace = await mkdtemp(join(tmpdir(), "videobook-export-"));
+        const outputFilename = "original.mp4";
+        const outputPath = join(workspace, outputFilename);
+        const captionPath = join(workspace, "captions.vtt");
+        try {
+            const resolved = await resolveClips(engine, sequence, workspace, context.signal);
+            if (request.captions !== "omit") {
+                await writeFile(captionPath, webVtt(sequence), "utf8");
+            }
+            await runFfmpeg(buildSequenceFfmpegArgs(sequence, resolved, width, height, outputPath, request.captions === "burn-in" ? captionPath : undefined), { signal: context.signal });
+            const label = `final-${request.orientation}-${request.captions}-${request.sequenceRevision.slice(0, 8)}`;
+            const existing = engine.artifacts.list().find((artifact) => artifact.kind === "final" && artifact.label === label);
+            const final = existing !== undefined
+                ? { ok: true, value: existing }
+                : await engine.artifacts.create({ kind: "final", label });
+            if (!final.ok)
+                throw new Error(final.error.message);
+            await replaceArtifactFileFromPath(engine, final.value.artifactId, outputFilename, outputPath);
+            if (request.captions === "sidecar") {
+                await replaceArtifactFileFromPath(engine, final.value.artifactId, "captions.vtt", captionPath);
+            }
+            const manifest = await engine.files.manifest(final.value.artifactId);
+            if (!manifest.ok)
+                throw new Error(manifest.error.message);
+            const output = manifest.value.files.find((file) => file.name === outputFilename);
+            if (!output)
+                throw new Error("Rendered MP4 was not persisted");
+            const persistedPath = join(manifest.value.path, outputFilename);
+            const probe = await probeVideo(persistedPath, { signal: context.signal });
+            const registered = engine.streams.list(final.value.artifactId);
+            if (!registered.ok)
+                throw new Error(registered.error.message);
+            if (probe.videoStream
+                && !registered.value.some((stream) => stream.kind === "video" && stream.objectHash === output.objectHash)) {
+                const stream = await engine.streams.register({
+                    artifactId: final.value.artifactId,
+                    sourcePath: outputFilename,
+                    objectHash: output.objectHash,
+                    streamIndex: probe.videoStream.index,
+                    kind: "video",
+                    timeBase: probe.videoStream.timeBase,
+                    durationTicks: probe.videoStream.durationTicks,
+                    codec: probe.videoStream.codec,
+                    video: probe.videoStream.video,
+                });
+                if (!stream.ok)
+                    throw new Error(stream.error.message);
+            }
+            if (probe.audioStream
+                && !registered.value.some((stream) => stream.kind === "audio" && stream.objectHash === output.objectHash)) {
+                const stream = await engine.streams.register({
+                    artifactId: final.value.artifactId,
+                    sourcePath: outputFilename,
+                    objectHash: output.objectHash,
+                    streamIndex: probe.audioStream.index,
+                    kind: "audio",
+                    timeBase: probe.audioStream.timeBase,
+                    durationTicks: probe.audioStream.durationTicks,
+                    codec: probe.audioStream.codec,
+                    audio: probe.audioStream.audio,
+                });
+                if (!stream.ok)
+                    throw new Error(stream.error.message);
+            }
+            const provenance = {
+                sequenceId: sequence.sequenceId,
+                sequenceRevision: request.sequenceRevision,
+                renderer: "ffmpeg",
+                rendererVersion: "videobook-canonical-v1",
+                settings: {
+                    orientation: request.orientation,
+                    captions: request.captions,
+                    width,
+                    height,
+                    frameRate: sequence.frameRate,
+                    videoCodec: "h264",
+                    audioCodec: "aac",
+                },
+                inputObjectHashes: [...new Set(resolved.map(({ clip }) => clip.source.kind === "timed" ? clip.source.range.objectHash : clip.source.objectHash))].sort(),
+                outputObjectHash: output.objectHash,
+                renderedAt: new Date().toISOString(),
+            };
+            const metadata = await engine.metadata.artifacts.write(final.value.artifactId, "export", provenance);
+            if (!metadata.ok)
+                throw new Error(metadata.error.message);
+            return {
+                success: true,
+                artifactId: final.value.artifactId,
+                label: final.value.label,
+                output: outputFilename,
+                outputObjectHash: output.objectHash,
+                provenance,
+            };
+        }
+        finally {
+            await rm(workspace, { recursive: true, force: true });
+        }
+    });
+}

package/dist/jobs/extend-audio.js

@@ -0,0 +1,229 @@
+import { createElevenLabs } from "../usage-providers.js";
+import { extname, join } from "node:path";
+import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { audioCrossfadeMs, buildSoundContinuationPrompt, describeSourceAudio, ELEVENLABS_SCRIBE_MODEL, ELEVENLABS_SOUND_EFFECT_MODEL, ELEVENLABS_SOUND_EFFECT_OUTPUT_FORMAT, MAX_SOUND_GENERATION_PROMPT_CHARS, } from "../elevenlabs-audio-extension.js";
+import { extractWaveform, probeVideo, runFfmpeg } from "../ffmpeg.js";
+import { requireApiKey } from "../api-key.js";
+import { registerJobHandler } from "../queue.js";
+const ELEVENLABS_TIMEOUT_MS = 10 * 60_000;
+const DURATION_TOLERANCE_SECONDS = 0.25;
+const TAIL_ACTIVITY_THRESHOLD = 0.005;
+function audioMime(filename) {
+    switch (extname(filename).toLowerCase()) {
+        case ".mp3": return "audio/mpeg";
+        case ".wav": return "audio/wav";
+        case ".m4a": return "audio/mp4";
+        case ".ogg": return "audio/ogg";
+        case ".flac": return "audio/flac";
+        default: return "application/octet-stream";
+    }
+}
+async function renderContinuation(source, sourceFilename, generated, sourceDurationMs, targetDurationMs, crossfadeMs, signal) {
+    const tempDir = await mkdtemp(join(tmpdir(), "videocity-elevenlabs-extend-"));
+    const sourcePath = join(tempDir, `source${extname(sourceFilename) || ".audio"}`);
+    const generatedPath = join(tempDir, "continuation.mp3");
+    const outputPath = join(tempDir, "extended.mp3");
+    const sourceSeconds = sourceDurationMs / 1000;
+    const targetSeconds = targetDurationMs / 1000;
+    const continuationSeconds = (targetDurationMs - sourceDurationMs) / 1000;
+    const crossfadeSeconds = crossfadeMs / 1000;
+    const generatedSeconds = continuationSeconds + crossfadeSeconds;
+    const filter = [
+        `[0:a:0]aresample=48000,aformat=sample_fmts=fltp:channel_layouts=stereo,atrim=duration=${sourceSeconds},asetpts=PTS-STARTPTS[source]`,
+        `[1:a:0]aresample=48000,aformat=sample_fmts=fltp:channel_layouts=stereo,atrim=duration=${generatedSeconds},asetpts=PTS-STARTPTS[continuation]`,
+        `[source][continuation]acrossfade=d=${crossfadeSeconds}:c1=tri:c2=tri,atrim=duration=${targetSeconds},asetpts=PTS-STARTPTS[out]`,
+    ].join(";");
+    try {
+        await writeFile(sourcePath, source);
+        await writeFile(generatedPath, new Uint8Array(generated));
+        await runFfmpeg([
+            "-y",
+            "-i", sourcePath,
+            "-i", generatedPath,
+            "-filter_complex", filter,
+            "-map", "[out]",
+            "-c:a", "libmp3lame",
+            "-b:a", "192k",
+            "-ar", "48000",
+            "-ac", "2",
+            "-t", String(targetSeconds),
+            outputPath,
+        ], { signal });
+        return await readFile(outputPath);
+    }
+    finally {
+        await rm(tempDir, { recursive: true, force: true, maxRetries: 3 });
+    }
+}
+export async function hasAudibleContinuationTail(audioPath) {
+    const waveform = await extractWaveform(audioPath, 100);
+    if (waveform.length === 0)
+        return false;
+    const tail = waveform.slice(Math.floor(waveform.length * 0.85));
+    const active = tail.filter((level) => level >= TAIL_ACTIVITY_THRESHOLD).length;
+    const mean = tail.reduce((sum, level) => sum + level, 0) / tail.length;
+    return active / tail.length >= 0.2 && mean >= TAIL_ACTIVITY_THRESHOLD / 2;
+}
+export async function fillLoopableContinuation(audio, durationSeconds, signal) {
+    const tempDir = await mkdtemp(join(tmpdir(), "videocity-elevenlabs-loop-fill-"));
+    const inputPath = join(tempDir, "input.mp3");
+    const trimmedPath = join(tempDir, "trimmed.wav");
+    const outputPath = join(tempDir, "filled.mp3");
+    try {
+        await writeFile(inputPath, new Uint8Array(audio));
+        await runFfmpeg([
+            "-y",
+            "-i", inputPath,
+            "-af", "areverse,silenceremove=start_periods=1:start_duration=0.15:start_threshold=-45dB,areverse",
+            "-c:a", "pcm_s16le",
+            "-ar", "48000",
+            "-ac", "2",
+            trimmedPath,
+        ], { signal });
+        const trimmedProbe = await probeVideo(trimmedPath, { signal });
+        const trimmedDuration = trimmedProbe.audioDuration ?? trimmedProbe.duration;
+        if (!Number.isFinite(trimmedDuration) || trimmedDuration < 0.5) {
+            throw new Error("ElevenLabs loopable continuation has too little audible content");
+        }
+        const crossfadeSeconds = Math.min(0.15, trimmedDuration / 4);
+        const repeatCount = Math.max(1, Math.ceil((durationSeconds - crossfadeSeconds) / (trimmedDuration - crossfadeSeconds)));
+        const args = ["-y"];
+        for (let index = 0; index < repeatCount; index++) {
+            args.push("-i", trimmedPath);
+        }
+        const filters = [];
+        if (repeatCount === 1) {
+            filters.push(`[0:a:0]atrim=duration=${durationSeconds},asetpts=PTS-STARTPTS[out]`);
+        }
+        else {
+            let previous = "[0:a:0]";
+            for (let index = 1; index < repeatCount; index++) {
+                const output = index === repeatCount - 1 ? "[joined]" : `[join${index}]`;
+                filters.push(`${previous}[${index}:a:0]acrossfade=d=${crossfadeSeconds}:c1=tri:c2=tri${output}`);
+                previous = output;
+            }
+            filters.push(`[joined]atrim=duration=${durationSeconds},asetpts=PTS-STARTPTS[out]`);
+        }
+        args.push("-filter_complex", filters.join(";"), "-map", "[out]", "-c:a", "libmp3lame", "-b:a", "192k", "-ar", "48000", "-ac", "2", "-t", String(durationSeconds), outputPath);
+        await runFfmpeg(args, { signal });
+        if (!(await hasAudibleContinuationTail(outputPath))) {
+            throw new Error("ElevenLabs loop-filled continuation still has a silent tail");
+        }
+        return Uint8Array.from(await readFile(outputPath)).buffer;
+    }
+    finally {
+        await rm(tempDir, { recursive: true, force: true, maxRetries: 3 });
+    }
+}
+async function generateAudibleContinuation(provider, generationPrompt, generationDurationSeconds, signal) {
+    const retrySuffix = " Audible every moment; seamless loop; no quiet tail.";
+    const attempts = [
+        { text: generationPrompt, loop: false },
+        {
+            text: `${generationPrompt.slice(0, MAX_SOUND_GENERATION_PROMPT_CHARS - retrySuffix.length)}${retrySuffix}`,
+            loop: true,
+        },
+    ];
+    for (let index = 0; index < attempts.length; index++) {
+        const attempt = attempts[index];
+        const audio = await provider.v1.soundGeneration({
+            text: attempt.text,
+            model_id: ELEVENLABS_SOUND_EFFECT_MODEL,
+            duration_seconds: generationDurationSeconds,
+            prompt_influence: 0.9,
+            loop: attempt.loop,
+            output_format: ELEVENLABS_SOUND_EFFECT_OUTPUT_FORMAT,
+        }, signal);
+        const tempDir = await mkdtemp(join(tmpdir(), "videocity-elevenlabs-tail-check-"));
+        const audioPath = join(tempDir, "continuation.mp3");
+        try {
+            await writeFile(audioPath, new Uint8Array(audio));
+            if (await hasAudibleContinuationTail(audioPath)) {
+                return { audio, attempts: index + 1, loopFilled: false };
+            }
+        }
+        finally {
+            await rm(tempDir, { recursive: true, force: true, maxRetries: 3 });
+        }
+        if (attempt.loop) {
+            const filled = await fillLoopableContinuation(audio, generationDurationSeconds, signal);
+            return {
+                audio: filled,
+                attempts: index + 1,
+                loopFilled: true,
+            };
+        }
+    }
+    throw new Error("ElevenLabs generated an unusable continuation twice");
+}
+export async function extendAudioWithElevenLabs(engine, payload, provider, signal) {
+    const source = await engine.files.read(payload.sourceAssetId, payload.sourceFilename);
+    if (!source.ok)
+        throw new Error(`Failed to read source audio: ${source.error.message}`);
+    const sourceBytes = new Uint8Array(source.value);
+    const transcript = await provider.v1.speechToText({
+        model_id: ELEVENLABS_SCRIBE_MODEL,
+        file: new Blob([sourceBytes], { type: audioMime(payload.sourceFilename) }),
+        tag_audio_events: true,
+        diarize: false,
+    }, signal);
+    const sourceDescription = describeSourceAudio(transcript);
+    const continuationDurationMs = payload.targetDurationMs - payload.sourceDurationMs;
+    const crossfadeMs = audioCrossfadeMs(payload.sourceDurationMs, continuationDurationMs);
+    const generationDurationSeconds = (continuationDurationMs + crossfadeMs) / 1000;
+    const generationPrompt = buildSoundContinuationPrompt(sourceDescription, generationDurationSeconds, payload.continuationPrompt);
+    const generated = await generateAudibleContinuation(provider, generationPrompt, generationDurationSeconds, signal);
+    const output = await renderContinuation(source.value, payload.sourceFilename, generated.audio, payload.sourceDurationMs, payload.targetDurationMs, crossfadeMs, signal);
+    const write = await engine.files.write(payload.audAssetId, "original.mp3", output);
+    if (!write.ok)
+        throw new Error(`Failed to write extended audio: ${write.error.message}`);
+    const assetDir = await engine.workspaces.resolveArtifact(payload.audAssetId);
+    if (!assetDir.ok)
+        throw new Error(`Failed to resolve extended audio: ${assetDir.error.message}`);
+    const probe = await probeVideo(join(assetDir.value, "original.mp3"));
+    const duration = probe.audioDuration ?? probe.duration;
+    const targetSeconds = payload.targetDurationMs / 1000;
+    if (!Number.isFinite(duration) || Math.abs(duration - targetSeconds) > DURATION_TOLERANCE_SECONDS) {
+        throw new Error(`Extended audio is ${duration.toFixed(3)}s, expected ${targetSeconds.toFixed(3)}s`);
+    }
+    const metadata = await engine.metadata.artifacts.write(payload.audAssetId, "original", {
+        origin: "elevenlabs_sound_generation_extend",
+        provider: "elevenlabs",
+        transcription_model: ELEVENLABS_SCRIBE_MODEL,
+        generation_model: ELEVENLABS_SOUND_EFFECT_MODEL,
+        output_format: ELEVENLABS_SOUND_EFFECT_OUTPUT_FORMAT,
+        source_asset: payload.sourceAssetId,
+        source_description: sourceDescription,
+        source_duration: payload.sourceDurationMs / 1000,
+        target_duration: targetSeconds,
+        continuation_duration: continuationDurationMs / 1000,
+        generation_duration: generationDurationSeconds,
+        generation_attempts: generated.attempts,
+        generation_loop_filled: generated.loopFilled,
+        crossfade_duration: crossfadeMs / 1000,
+        continuation_prompt: payload.continuationPrompt ?? null,
+        duration,
+        generated_at: new Date().toISOString(),
+    });
+    if (!metadata.ok)
+        throw new Error(`Failed to write extended audio metadata: ${metadata.error.message}`);
+    return {
+        duration,
+        sourceDescription,
+        generationAttempts: generated.attempts,
+        loopFilled: generated.loopFilled,
+    };
+}
+export function registerExtendAudioJob(engine) {
+    registerJobHandler("extend_audio", async (rawPayload, ctx) => {
+        const payload = rawPayload;
+        ctx.getOwnerId();
+        const provider = createElevenLabs({
+            apiKey: requireApiKey("ELEVENLABS_API_KEY"),
+            timeout: ELEVENLABS_TIMEOUT_MS,
+        });
+        const result = await extendAudioWithElevenLabs(engine, payload, provider, ctx.signal);
+        return { success: true, provider: "elevenlabs", ...result };
+    });
+}

package/dist/jobs/extract-audio.js

@@ -0,0 +1,72 @@
+import { mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { registerJobHandler } from "../queue.js";
+import { withLockedLocalLease } from "./_lease.js";
+import { extractWaveform, probeVideo, runFfmpeg } from "../ffmpeg.js";
+import { validateMediaOutputPaths, withMediaOutputLock } from "../media-output-path.js";
+import { registerMediaStreams } from "./process-video.js";
+const LOCK_DURATION_MS = 120_000;
+export function registerExtractAudioJob(engine) {
+    registerJobHandler("extract_audio", async (payload, ctx) => {
+        const { inputPath, audioAssetId, audioAssetDir, sourceVideoId, destination, range, } = payload;
+        const destinationMode = destination !== undefined;
+        const extract = async () => {
+            const outputFilename = destination
+                ? (await validateMediaOutputPaths(engine, audioAssetId, [destination.filename], ".mp3"))[0]
+                : "original.mp3";
+            const workspace = await mkdtemp(join(tmpdir(), "videobook-extract-audio-"));
+            const outputPath = join(workspace, "output.mp3");
+            try {
+                const ffmpegArgs = [
+                    "-y",
+                    "-i", inputPath,
+                    "-vn",
+                    "-acodec", "libmp3lame",
+                    "-q:a", "2",
+                    ...(range ? ["-ss", String(range.start_s), "-t", String(range.duration_s)] : []),
+                    outputPath,
+                ];
+                await runFfmpeg(ffmpegArgs, { signal: ctx.signal });
+                const probe = await probeVideo(outputPath, { signal: ctx.signal });
+                const duration = probe.audioDuration ?? probe.duration;
+                if (!probe.hasAudio || !Number.isFinite(duration) || duration <= 0) {
+                    throw new Error("Audio extraction produced no playable audio");
+                }
+                const written = await engine.files.writeFromPath(audioAssetId, outputFilename, outputPath);
+                if (!written.ok)
+                    throw new Error(written.error.message);
+                const recorded = await engine.history.recordOperation("extract_audio", audioAssetId, {
+                    source_asset: sourceVideoId,
+                    filename: outputFilename,
+                    ...(range ? { range } : {}),
+                    duration,
+                });
+                if (!recorded.ok)
+                    throw new Error(recorded.error.message);
+                if (!destinationMode) {
+                    const origin = {
+                        origin: "extract_audio",
+                        source_asset: sourceVideoId,
+                        duration,
+                    };
+                    await engine.metadata.artifacts.write(audioAssetId, "original", origin);
+                    await registerMediaStreams(engine, audioAssetId, outputFilename, probe);
+                    const peaks = await extractWaveform(outputPath);
+                    if (peaks.length > 0) {
+                        const waveform = await engine.metadata.waveforms.write(audioAssetId, peaks);
+                        if (!waveform.ok)
+                            throw new Error(waveform.error.message);
+                    }
+                }
+                return { success: true };
+            }
+            finally {
+                await rm(workspace, { recursive: true, force: true });
+            }
+        };
+        if (destinationMode)
+            return withMediaOutputLock(engine, audioAssetDir, ctx.signal, extract);
+        return withLockedLocalLease(engine, audioAssetId, audioAssetDir, ctx, "extract", { durationMs: LOCK_DURATION_MS }, extract);
+    });
+}