videobook 0.0.0-stage → 0.2.0

package/dist/jobs/assemble-goal.js

@@ -0,0 +1,98 @@
+import { findVideoFile, MVP_CONTRACT_VERSION, sourceTicksToSequenceFrames } from "videobook-engine";
+import { v7 as uuidv7 } from "uuid";
+import { registerJobHandler } from "../queue.js";
+import { readGoal, saveGoal } from "../goal-store.js";
+import { withNotebookMutation } from "../notebook-mutations.js";
+import { fittedTransform } from "../rhythm/visual-fit.js";
+export async function appendGoalClips(engine, goal, artifactIds) {
+    if (artifactIds.length === 0 || goal.clipIds.length !== artifactIds.length) {
+        throw new Error("The finished clips do not match the reviewed goal");
+    }
+    const sequence = engine.sequences.getPrimary();
+    const existing = sequence.clips.filter((clip) => goal.clipIds.includes(clip.clipId));
+    const snapshot = JSON.stringify(existing);
+    const unchangedSources = existing.length === artifactIds.length && goal.clipIds.every((id, index) => existing.find(clip => clip.clipId === id)?.source.artifactId === artifactIds[index]);
+    if (existing.length) {
+        if (existing.length !== artifactIds.length)
+            throw new Error("Some goal clips were already edited or removed. Review the sequence before assembling again.");
+        if (unchangedSources)
+            return {
+                sequenceId: sequence.sequenceId,
+                startFrame: Math.min(...existing.map((clip) => clip.timelineStartFrame)),
+                durationFrames: existing.reduce((sum, clip) => sum + clip.durationFrames, 0),
+                clipSnapshot: goal.assembly?.clipSnapshot ?? snapshot, artifactIds,
+            };
+        if (!goal.assembly?.clipSnapshot || goal.assembly.clipSnapshot !== snapshot)
+            throw new Error("The goal's timeline clips were edited. Keep those edits and place the revised videos manually.");
+    }
+    if (goal.assembly && !existing.length)
+        throw new Error("This goal was already assembled; its clips have since been removed");
+    const track = sequence.tracks.find((candidate) => candidate.trackId === goal.trackId);
+    if (!track || track.kind !== "video" || track.locked || !track.enabled) {
+        throw new Error("The goal's video track is no longer editable. Unlock or enable it, then assemble again.");
+    }
+    const startFrame = existing.length ? Math.min(...existing.map(clip => clip.timelineStartFrame)) : Math.max(0, ...sequence.clips.map((clip) => clip.timelineStartFrame + clip.durationFrames), ...sequence.captions.map((caption) => caption.timelineStartFrame + caption.durationFrames));
+    let cursor = startFrame;
+    const operations = existing.map(clip => ({ kind: "remove-clip", clipId: clip.clipId, ripple: false }));
+    for (const [index, artifactId] of artifactIds.entries()) {
+        const status = await engine.status.get(artifactId);
+        if (!status.ok || status.value !== "ready")
+            throw new Error(`Shot ${index + 1} is still processing`);
+        const manifest = await engine.files.manifest(artifactId);
+        if (!manifest.ok)
+            throw new Error(manifest.error.message);
+        const file = findVideoFile(manifest.value.files);
+        const streams = engine.streams.list(artifactId);
+        if (!streams.ok)
+            throw new Error(streams.error.message);
+        const stream = streams.value.find((candidate) => candidate.kind === "video"
+            && candidate.objectHash === file?.objectHash && candidate.sourcePath === file?.name);
+        if (!stream)
+            throw new Error(`Shot ${index + 1} has no playable video stream`);
+        const durationFrames = sourceTicksToSequenceFrames(stream.durationTicks, stream.timeBase, sequence.frameRate, "floor");
+        if (durationFrames < 1)
+            throw new Error(`Shot ${index + 1} has no playable frames`);
+        operations.push({
+            kind: "insert-clip", mode: "overwrite", clipId: goal.clipIds[index],
+            placement: {
+                trackId: track.trackId, timelineStartFrame: cursor, durationFrames,
+                source: { kind: "timed", artifactId, range: {
+                        streamId: stream.streamId, objectHash: stream.objectHash,
+                        startTick: 0, durationTicks: stream.durationTicks, timeBase: stream.timeBase,
+                    } },
+                speed: { numerator: 1, denominator: 1 }, reverse: false, audioPolicy: "preserve-pitch",
+                audio: { gainDb: 0, muted: false, fadeInFrames: 0, fadeOutFrames: 0 },
+                transform: fittedTransform("fit"),
+            },
+        });
+        cursor += durationFrames;
+    }
+    if (sequence.clips.some(clip => !goal.clipIds.includes(clip.clipId) && clip.trackId === track.trackId && clip.timelineStartFrame < cursor && clip.timelineStartFrame + clip.durationFrames > startFrame)) {
+        throw new Error("The revised clips would overlap other timeline edits. Place the revised videos manually.");
+    }
+    const intent = {
+        intentVersion: MVP_CONTRACT_VERSION, commandId: uuidv7(), sequenceId: sequence.sequenceId,
+        baseRevision: sequence.revision, actor: "goal", sourceSurface: "system",
+        confirmationPolicy: "reversible-single-step", operations,
+    };
+    const preview = engine.edits.preview(intent);
+    if (!preview.ok)
+        throw new Error(preview.error.message);
+    const committed = await engine.edits.commit(intent, preview.value.previewHash);
+    if (!committed.ok)
+        throw new Error(committed.error.message);
+    return { sequenceId: sequence.sequenceId, startFrame, durationFrames: cursor - startFrame, artifactIds, clipSnapshot: JSON.stringify(engine.sequences.getPrimary().clips.filter(clip => goal.clipIds.includes(clip.clipId))) };
+}
+export function registerAssembleGoalJob(engine) {
+    registerJobHandler("assemble_goal", async (payload, context) => {
+        const goalId = String(payload.goalId);
+        return withNotebookMutation(`goal:${goalId}`, async () => {
+            const goal = readGoal(engine, goalId);
+            if (context.signal.aborted || goal.status === "cancelled")
+                throw new Error("Goal cancelled");
+            const assembly = await appendGoalClips(engine, goal, payload.artifactIds);
+            saveGoal(engine, { ...readGoal(engine, goalId), assembly });
+            return { sequence_id: assembly.sequenceId, status: "completed" };
+        });
+    });
+}

package/dist/jobs/audio-boost.js

@@ -0,0 +1,71 @@
+import { mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { basename, extname, join } from "node:path";
+import { registerJobHandler, queue } from "../queue.js";
+import { extractWaveform, probeVideo, runFfmpeg } from "../ffmpeg.js";
+import { audioOnlyOutputOptions, videoAudioOutputOptions } from "../audio-encoding.js";
+import { registerMediaStreams } from "./process-video.js";
+import { withLockedLocalLease } from "./_lease.js";
+import { replaceArtifactFileFromPath } from "./_persist-output.js";
+const LOCK_DURATION_MS = 300_000;
+function buildBoostOutputOptions(boost, isVideo, inputPath) {
+    const audioFilter = `volume=${boost}`;
+    if (isVideo) {
+        return ["-c:v", "copy", "-af", audioFilter, ...videoAudioOutputOptions(inputPath)];
+    }
+    return ["-af", audioFilter, ...audioOnlyOutputOptions(inputPath)];
+}
+export function registerAudioBoostJob(engine) {
+    registerJobHandler("audio_boost", async (payload, ctx) => {
+        const { inputPath, boost, isVideo, assetId, assetDir, bookId, } = payload;
+        const result = await withLockedLocalLease(engine, assetId, assetDir, ctx, "apply_sfx", { durationMs: LOCK_DURATION_MS, data: { state: "boosting-audio" } }, async () => {
+            const inputFilename = basename(inputPath);
+            const mediaPath = join(assetDir, inputFilename);
+            const workspace = await mkdtemp(join(tmpdir(), "videobook-audio-boost-"));
+            const outputPath = join(workspace, `output${extname(inputFilename)}`);
+            try {
+                const source = await probeVideo(inputPath, { signal: ctx.signal });
+                if (!source.hasAudio)
+                    throw new Error("Source has no playable audio stream");
+                await runFfmpeg(["-y", "-i", inputPath, ...buildBoostOutputOptions(boost, isVideo, inputPath), outputPath], { signal: ctx.signal });
+                const output = await probeVideo(outputPath, { signal: ctx.signal });
+                if (!output.hasAudio || Math.abs(output.duration - source.duration) > 0.15) {
+                    throw new Error("Audio boost produced an invalid duration");
+                }
+                await replaceArtifactFileFromPath(engine, assetId, inputFilename, outputPath);
+                if (!isVideo) {
+                    await registerMediaStreams(engine, assetId, inputFilename, output);
+                    const previous = await engine.metadata.artifacts.read(assetId, "original");
+                    const metadata = await engine.metadata.artifacts.write(assetId, "original", {
+                        ...(previous.ok ? previous.value : {}),
+                        origin: "audio_boost",
+                        boost,
+                        duration: output.audioDuration ?? output.duration,
+                    });
+                    if (!metadata.ok)
+                        throw new Error(metadata.error.message);
+                    const waveform = await engine.metadata.waveforms.write(assetId, await extractWaveform(mediaPath));
+                    if (!waveform.ok)
+                        throw new Error(waveform.error.message);
+                }
+            }
+            finally {
+                await rm(workspace, { recursive: true, force: true });
+            }
+            return { mediaPath };
+        }, { durationMs: LOCK_DURATION_MS, waitForActiveMs: 30_000 });
+        if (isVideo) {
+            await queue.enqueue(`process-video-${assetId}-${Date.now()}`, {
+                type: "process_video",
+                videoPath: result.mediaPath,
+                assetId,
+                assetDir,
+                bookId,
+                provenance: { origin: "audio_boost", boost, boosted_at: new Date().toISOString() },
+                force: true,
+                skipDownstreamJobs: true,
+            });
+        }
+        return { success: true };
+    });
+}

package/dist/jobs/audio-mix.js

@@ -0,0 +1,161 @@
+import { mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { extname, join } from "node:path";
+import { extractWaveform, probeVideo, runFfmpeg } from "../ffmpeg.js";
+import { queue, registerJobHandler } from "../queue.js";
+import { withLockedLocalLease } from "./_lease.js";
+import { audioOnlyOutputOptions, videoAudioOutputOptions } from "../audio-encoding.js";
+import { replaceArtifactFileFromPath } from "./_persist-output.js";
+import { registerMediaStreams } from "./process-video.js";
+const LOCK_DURATION_MS = 30 * 60_000;
+const STALE_DERIVATIVES = [
+    ".original.analysis.json",
+    ".analysis.gpt4otrans.json",
+    "elevenlabs.json",
+    ".suggest-sfx.json",
+];
+function decimal(value) {
+    return value.toFixed(6).replace(/0+$/, "").replace(/\.$/, "");
+}
+function outputContainerOptions(outputPath) {
+    const extension = extname(outputPath).toLowerCase();
+    return extension === ".mp4" || extension === ".mov" || extension === ".m4a"
+        ? ["-movflags", "+faststart"]
+        : [];
+}
+export function buildAudioMixFilter(targetHasAudio, targetDurationSeconds, layerVolumePercent) {
+    if (!Number.isFinite(targetDurationSeconds) || targetDurationSeconds <= 0) {
+        throw new Error(`Invalid target duration: ${targetDurationSeconds}`);
+    }
+    if (!Number.isFinite(layerVolumePercent) || layerVolumePercent < 0 || layerVolumePercent > 100) {
+        throw new Error(`Layer volume must be between 0 and 100, got: ${layerVolumePercent}`);
+    }
+    const duration = decimal(targetDurationSeconds);
+    const volume = decimal(layerVolumePercent / 100);
+    const base = targetHasAudio
+        ? `[0:a:0]aresample=48000,aformat=sample_fmts=fltp:channel_layouts=stereo,apad,atrim=duration=${duration},asetpts=PTS-STARTPTS[base]`
+        : `anullsrc=channel_layout=stereo:sample_rate=48000,atrim=duration=${duration},asetpts=PTS-STARTPTS[base]`;
+    const layer = `[1:a:0]aresample=48000,aformat=sample_fmts=fltp:channel_layouts=stereo,volume=${volume},apad,atrim=duration=${duration},asetpts=PTS-STARTPTS[layer]`;
+    const mix = "[base][layer]amix=inputs=2:duration=first:dropout_transition=0:normalize=0,alimiter=limit=0.95:level=false[outa]";
+    return `${base};${layer};${mix}`;
+}
+export async function mixAudioMedia(targetPath, layerPath, outputPath, targetKind, layerVolumePercent, signal) {
+    const [targetProbe, layerProbe] = await Promise.all([
+        probeVideo(targetPath, { signal }),
+        probeVideo(layerPath, { signal }),
+    ]);
+    if (!layerProbe.hasAudio)
+        throw new Error("Layer asset has no audio stream");
+    if (targetKind === "video" && targetProbe.videoCodec === null) {
+        throw new Error("Video target has no video stream");
+    }
+    if (targetKind === "audio" && !targetProbe.hasAudio) {
+        throw new Error("Audio target has no audio stream");
+    }
+    const targetDuration = targetKind === "video"
+        ? (targetProbe.videoDuration > 0 ? targetProbe.videoDuration : targetProbe.duration)
+        : (targetProbe.audioDuration ?? targetProbe.duration);
+    const filter = buildAudioMixFilter(targetProbe.hasAudio, targetDuration, layerVolumePercent);
+    const args = [
+        "-y",
+        "-i", targetPath,
+        "-i", layerPath,
+        "-filter_complex", filter,
+    ];
+    if (targetKind === "video") {
+        args.push("-map", "0:v:0", "-map", "[outa]", "-map_metadata", "0", "-map_chapters", "0", "-c:v", "copy", ...videoAudioOutputOptions(outputPath), ...outputContainerOptions(outputPath), outputPath);
+    }
+    else {
+        args.push("-map", "[outa]", ...audioOnlyOutputOptions(outputPath), ...outputContainerOptions(outputPath), outputPath);
+    }
+    await runFfmpeg(args, { signal });
+    const outputProbe = await probeVideo(outputPath, { signal });
+    if (!outputProbe.hasAudio)
+        throw new Error("Audio mix output has no audio stream");
+    if (targetKind === "video") {
+        if (outputProbe.videoCodec === null)
+            throw new Error("Audio mix output has no video stream");
+        if (outputProbe.videoCodec !== targetProbe.videoCodec) {
+            throw new Error(`Audio mix changed the video codec from ${targetProbe.videoCodec} to ${outputProbe.videoCodec}`);
+        }
+        if (Math.abs(outputProbe.videoDuration - targetDuration) > 0.5) {
+            throw new Error(`Audio mix changed video duration from ${targetDuration.toFixed(3)}s to ${outputProbe.videoDuration.toFixed(3)}s`);
+        }
+        return outputProbe;
+    }
+    const outputDuration = outputProbe.audioDuration ?? outputProbe.duration;
+    if (Math.abs(outputDuration - targetDuration) > 0.5) {
+        throw new Error(`Audio mix changed audio duration from ${targetDuration.toFixed(3)}s to ${outputDuration.toFixed(3)}s`);
+    }
+    return outputProbe;
+}
+async function deleteIfPresent(engine, assetId, filename) {
+    const result = await engine.files.delete(assetId, filename);
+    if (!result.ok && result.error.code !== "NOT_FOUND") {
+        throw new Error(`Failed to delete ${filename}: ${result.error.message}`);
+    }
+}
+export function registerAudioMixJob(engine) {
+    registerJobHandler("audio_mix", async (payload, ctx) => {
+        const { targetPath, targetFilename, targetKind, layerPath, layerAssetId, volume, assetId, assetDir, bookId, } = payload;
+        const result = await withLockedLocalLease(engine, assetId, assetDir, ctx, "replace_audio", { durationMs: LOCK_DURATION_MS, data: { state: "mixing-audio" } }, async () => {
+            const tempDir = await mkdtemp(join(tmpdir(), "videocity-audio-mix-"));
+            const extension = extname(targetFilename) || (targetKind === "video" ? ".mp4" : ".mp3");
+            const outputPath = join(tempDir, `mixed${extension}`);
+            try {
+                const outputProbe = await mixAudioMedia(targetPath, layerPath, outputPath, targetKind, volume, ctx.signal);
+                await replaceArtifactFileFromPath(engine, assetId, targetFilename, outputPath);
+                if (targetKind === "audio") {
+                    await registerMediaStreams(engine, assetId, targetFilename, outputProbe);
+                    const previous = await engine.metadata.artifacts.read(assetId, "original");
+                    const metadata = await engine.metadata.artifacts.write(assetId, "original", {
+                        ...(previous.ok ? previous.value : {}),
+                        origin: "audio_mix",
+                        source_audio_asset: layerAssetId,
+                        layer_volume: volume,
+                        duration: outputProbe.audioDuration ?? outputProbe.duration,
+                    });
+                    if (!metadata.ok)
+                        throw new Error(metadata.error.message);
+                }
+            }
+            finally {
+                await rm(tempDir, { recursive: true, force: true });
+            }
+            for (const filename of STALE_DERIVATIVES) {
+                await deleteIfPresent(engine, assetId, filename);
+            }
+            const mediaPath = join(assetDir, targetFilename);
+            if (targetKind === "audio") {
+                const peaks = await extractWaveform(mediaPath);
+                const waveformResult = await engine.metadata.waveforms.write(assetId, peaks);
+                if (!waveformResult.ok) {
+                    throw new Error(`Failed to update audio waveform: ${waveformResult.error.message}`);
+                }
+            }
+            return { mediaPath };
+        }, { durationMs: LOCK_DURATION_MS, waitForActiveMs: 30_000 });
+        if (targetKind === "video") {
+            await queue.enqueue(`process-video-${assetId}-${Date.now()}`, {
+                type: "process_video",
+                videoPath: result.mediaPath,
+                assetId,
+                assetDir,
+                bookId,
+                provenance: {
+                    origin: "audio_mix",
+                    source_audio_asset: layerAssetId,
+                    layer_volume: volume,
+                    mixed_at: new Date().toISOString(),
+                },
+                force: true,
+            });
+        }
+        return {
+            success: true,
+            target_asset_id: assetId,
+            layer_asset_id: layerAssetId,
+            volume,
+        };
+    });
+}

package/dist/jobs/backfill-asset-tags.js

@@ -0,0 +1,144 @@
+/**
+ * Tagging a library that already exists.
+ *
+ * The pass walks the catalog in bounded pages and runs the same coordinator
+ * new assets use, so a backfilled tag and a freshly tagged one are produced by
+ * one code path and carry the same fence and provenance.
+ *
+ * Everything here is shaped by the pass being interruptible. It checks the
+ * persisted status between pages, so pause and cancel land within one page
+ * rather than at the end of a library; it writes its cursor and counts after
+ * every page, so a restart resumes instead of re-running; and it holds no
+ * in-memory state that a crash would lose. Adopting cached analysis costs
+ * nothing, so the default pass reads caches only and reports how many assets
+ * would still need inference — the spend is a separate, explicit choice.
+ */
+import { v7 as uuidv7 } from "uuid";
+import { logger } from "../logger.js";
+import { registerJobHandler } from "../queue.js";
+import { readBackfillState, recordOutcome, writeBackfillState, EMPTY_COUNTS, } from "../tag-backfill-state.js";
+import { runAssetTagging } from "./tag-asset.js";
+/** Assets per page. Small enough that pause lands promptly, large enough that
+ *  a big library is not dominated by state writes. */
+const PAGE_SIZE = 25;
+/** Assets tagged at once within a page. Tagging is mostly file reads; the
+ *  provider work it may schedule is bounded by the queue runner, not here. */
+const PAGE_CONCURRENCY = 4;
+async function mapWithLimit(items, limit, run, canRun) {
+    const results = [];
+    let next = 0;
+    async function worker() {
+        while (canRun()) {
+            const index = next;
+            next += 1;
+            const item = items[index];
+            if (item === undefined)
+                return;
+            results[index] = await run(item);
+        }
+    }
+    await Promise.all(Array.from({ length: Math.min(limit, items.length) }, () => worker()));
+    return results;
+}
+/**
+ * Ids after the cursor, in catalog order. The cursor is an artifact id rather
+ * than an offset because the catalog changes while the pass runs: deleting one
+ * asset behind an offset shifts every later page, silently skipping rows.
+ */
+function pageAfter(engine, cursor) {
+    const ids = engine.artifacts
+        .list({ sort: "oldest" })
+        .map((artifact) => artifact.artifactId);
+    const start = cursor ? ids.findIndex((id) => id > cursor) : 0;
+    if (start < 0)
+        return [];
+    return ids.slice(start, start + PAGE_SIZE);
+}
+function totalArtifacts(engine) {
+    return engine.artifacts.list().length;
+}
+export async function runAssetTagBackfill(engine, payload, context) {
+    let state = readBackfillState(engine);
+    const runId = payload.runId ?? null;
+    if (state.status !== "running" || state.runId !== runId) {
+        return state;
+    }
+    const canRun = () => {
+        const latest = readBackfillState(engine);
+        return latest.runId === runId && latest.status === "running" && !context?.signal.aborted;
+    };
+    for (;;) {
+        state = readBackfillState(engine);
+        if (state.runId !== runId || state.status !== "running")
+            return state;
+        if (context?.signal.aborted) {
+            return writeBackfillState(engine, { ...state, status: "paused" });
+        }
+        const page = pageAfter(engine, state.cursor);
+        if (page.length === 0) {
+            return writeBackfillState(engine, {
+                ...state,
+                status: "completed",
+                finishedAt: Date.now(),
+            });
+        }
+        const outcomes = await mapWithLimit(page, PAGE_CONCURRENCY, async (assetId) => {
+            try {
+                const result = await runAssetTagging(engine, {
+                    type: "tag_asset",
+                    bookId: payload.bookId,
+                    assetId,
+                    ...(state.force ? { force: true } : {}),
+                    ...(state.allowAnalysis ? {} : { cacheOnly: true }),
+                }, { canRequestAnalysis: canRun });
+                return result.outcome;
+            }
+            catch (cause) {
+                logger.warn("tag-asset-failed", { assetId, error: cause });
+                return "failed";
+            }
+        }, canRun);
+        const latest = readBackfillState(engine);
+        if (latest.runId !== runId)
+            return latest;
+        let counts = latest.counts;
+        for (const outcome of outcomes)
+            counts = recordOutcome(counts, outcome);
+        state = writeBackfillState(engine, {
+            ...latest,
+            cursor: page[outcomes.length - 1] ?? latest.cursor,
+            scanned: latest.scanned + outcomes.length,
+            counts,
+        });
+        if (state.status !== "running")
+            return state;
+    }
+}
+/** Starting state for a fresh pass; resume keeps the cursor and counts. */
+export function startedState(previous, options) {
+    const resuming = options.resume && previous.status === "paused";
+    return {
+        runId: uuidv7(),
+        status: "running",
+        cursor: resuming ? previous.cursor : null,
+        scanned: resuming ? previous.scanned : 0,
+        counts: resuming ? previous.counts : EMPTY_COUNTS,
+        allowAnalysis: options.allowAnalysis,
+        force: options.force,
+        startedAt: resuming ? previous.startedAt ?? Date.now() : Date.now(),
+        updatedAt: Date.now(),
+        finishedAt: null,
+        lastError: null,
+    };
+}
+export function backfillProgress(engine, state) {
+    const total = totalArtifacts(engine);
+    return {
+        state,
+        total,
+        remaining: Math.max(0, total - state.scanned),
+    };
+}
+export function registerBackfillAssetTagsJob(engine) {
+    registerJobHandler("backfill_asset_tags", async (payload, ctx) => await runAssetTagBackfill(engine, payload, ctx));
+}