videobook 0.0.0-stage → 0.2.0

package/dist/tools/recover-asset.js

@@ -0,0 +1,28 @@
+import { z } from "zod";
+function error(text) {
+    return { isError: true, content: [{ type: "text", text }] };
+}
+export function recoverAssetDef(engine) {
+    return {
+        name: "recover_asset",
+        description: "Clear a failed asset's generation error and keep its media, returning it to ready. Use clean_failed_assets instead to delete failed generations.",
+        schema: {
+            asset_id: z.string().describe("Asset ID to recover (vid-*, img-*, etc., or @-prefixed)"),
+            book_id: z.string().describe("Book id"),
+        },
+        handler: async (args) => {
+            const { asset_id, book_id: _book_id } = args;
+            const assetId = asset_id.replace(/^@/, "");
+            const cleared = await engine.jobs.failures.clear(assetId);
+            if (!cleared.ok) {
+                return error(`Failed to clear error for ${assetId}: ${cleared.error.message}`);
+            }
+            return {
+                content: [{
+                        type: "text",
+                        text: JSON.stringify({ asset_id: assetId, recovered: cleared.value }),
+                    }],
+            };
+        },
+    };
+}

package/dist/tools/remove-sound.js

@@ -0,0 +1,72 @@
+import { findAudioFile, findVideoFile, } from "videobook-engine";
+import { z } from "zod";
+import { queue } from "../queue.js";
+import { toolError } from "./_shared.js";
+const JOB_TIMEOUT_MS = 30 * 60_000;
+export function removeSoundDef(engine) {
+    return {
+        name: "remove_sound",
+        description: "Silence a described sound (e.g. a horse neigh) in a video or audio asset in place, using ElevenLabs Scribe audio-event timestamps; the picture is unchanged.",
+        schema: {
+            asset_id: z.string().describe("Artifact id to edit in-place (video or audio)"),
+            sound: z.string().min(1).max(200).describe('Description of the sound to remove, e.g. "horse neighing"'),
+            book_id: z.string().describe("Book UUID"),
+        },
+        handler: async (args) => {
+            const { asset_id, sound, book_id } = args;
+            if (!process.env.ELEVENLABS_API_KEY) {
+                return toolError("ELEVENLABS_API_KEY environment variable is not set");
+            }
+            const assetId = asset_id.replace(/^@/, "");
+            const artifact = engine.artifacts.get(assetId);
+            if (!artifact.ok)
+                return toolError(artifact.error.message);
+            const kind = artifact.value.kind === "video"
+                ? "video"
+                : artifact.value.kind === "audio" ? "audio" : null;
+            if (!kind) {
+                return toolError(`remove_sound only supports video or audio artifacts, got ${artifact.value.kind} artifact ${assetId}`);
+            }
+            const manifest = await engine.files.manifest(assetId);
+            if (!manifest.ok)
+                return toolError(manifest.error.message);
+            const file = kind === "video"
+                ? findVideoFile(manifest.value.files)
+                : findAudioFile(manifest.value.files, artifact.value.kind);
+            if (!file) {
+                return toolError(`No ${kind} file found in artifact ${assetId}`);
+            }
+            const jobId = `remove-sound-${assetId}-${Date.now()}`;
+            let result;
+            try {
+                result = await queue.enqueueAndWait(jobId, {
+                    type: "remove_sound",
+                    assetId,
+                    assetKind: kind,
+                    assetDir: manifest.value.path,
+                    filename: file.name,
+                    sound: sound.trim(),
+                    bookId: book_id,
+                }, JOB_TIMEOUT_MS);
+            }
+            catch (error) {
+                return toolError(`Sound removal failed: ${error instanceof Error ? error.message : String(error)}`);
+            }
+            return {
+                content: [{
+                        type: "text",
+                        text: JSON.stringify({
+                            asset_id: result.asset_id,
+                            sound: result.sound,
+                            matched_events: result.matched_events,
+                            removed_segments: result.removed_segments,
+                            removed_seconds: result.removed_seconds,
+                            detected_events: result.detected_events,
+                            job_id: jobId,
+                            status: "completed",
+                        }),
+                    }],
+            };
+        },
+    };
+}

package/dist/tools/remove-timeline-audio.js

@@ -0,0 +1,34 @@
+import { z } from "zod";
+import { readTimeline, timelineFailure, timelineInput, timelinePayload, writeTimeline, } from "../timeline-helpers.js";
+export function removeTimelineAudioDef(engine) {
+    return {
+        name: "remove_timeline_audio",
+        description: "Remove an audio overlay by its UUIDv7 id",
+        schema: {
+            book_id: z.string().uuid(),
+            audio_id: z.string().uuid(),
+        },
+        handler: async (args) => {
+            const timeline = readTimeline(engine);
+            const audioId = String(args.audio_id);
+            if (!timeline.audio.some((clip) => clip.id === audioId)) {
+                return timelineFailure(`Timeline audio clip not found: ${audioId}`);
+            }
+            const result = await writeTimeline(engine, {
+                ...timelineInput(timeline),
+                audio: timeline.audio.filter((clip) => clip.id !== audioId).map((clip) => ({
+                    id: clip.id,
+                    artifactId: clip.artifactId,
+                    startFrame: clip.startFrame,
+                    durationFrames: clip.durationFrames,
+                    ...(clip.volume === undefined ? {} : { volume: clip.volume }),
+                    ...(clip.fadeIn === undefined ? {} : { fadeIn: clip.fadeIn }),
+                    ...(clip.fadeOut === undefined ? {} : { fadeOut: clip.fadeOut }),
+                })),
+            });
+            return result.ok ? {
+                content: [{ type: "text", text: JSON.stringify(timelinePayload(result.value)) }],
+            } : timelineFailure(result.error.message);
+        },
+    };
+}

package/dist/tools/remove-timeline-slot.js

@@ -0,0 +1,40 @@
+import { z } from "zod";
+import { slotIndex, readTimeline, timelineFailure, timelineInput, timelineSuccess, writeTimeline, } from "../timeline-helpers.js";
+export function removeTimelineSlotDef(engine) {
+    return {
+        name: "remove_timeline_slot",
+        description: "Remove a typed timeline slot and shift later slots left",
+        schema: {
+            book_id: z.string().uuid(),
+            slot: z.string(),
+        },
+        handler: async (args) => {
+            const index = slotIndex(String(args.slot));
+            if (index === null)
+                return timelineFailure("Slot must use the form s01, s02, and so on");
+            const timeline = readTimeline(engine);
+            if (index >= timeline.slots.length) {
+                return timelineFailure(`Slot ${String(args.slot)} is outside the current timeline`);
+            }
+            const removed = timeline.slots[index];
+            const result = await writeTimeline(engine, {
+                ...timelineInput(timeline),
+                slots: timeline.slots.filter((_, position) => position !== index).map((slot) => ({
+                    id: slot.id,
+                    artifactId: slot.artifactId,
+                    ...(slot.volume === undefined ? {} : { volume: slot.volume }),
+                    ...(slot.audioFadeIn === undefined ? {} : { audioFadeIn: slot.audioFadeIn }),
+                    ...(slot.audioFadeOut === undefined ? {} : { audioFadeOut: slot.audioFadeOut }),
+                })),
+            });
+            if (!result.ok)
+                return timelineFailure(result.error.message);
+            return {
+                content: [{
+                        type: "text",
+                        text: JSON.stringify({ ...JSON.parse(timelineSuccess(result.value).content[0].text), removed_artifact_id: removed.artifactId }),
+                    }],
+            };
+        },
+    };
+}

package/dist/tools/rename-all.js

@@ -0,0 +1,434 @@
+import { z } from "zod";
+import { readFile } from "node:fs/promises";
+import { join } from "node:path";
+import { isRenamableKind, buildSuggestionPrompt, parseSuggestionResponse, buildRenameRows, validateEditedRenames, applyableRows, isProcessOrigin, } from "../rename-helpers.js";
+import { INTELLIGENCE_MODEL, buildIntelligenceResponses, firstResponseText, intelligenceRequest, } from "../openai-intelligence.js";
+import { scanNotebookMentions, stripNotebookMentions, } from "videobook-engine/notebook-mentions";
+const schema = {
+    book_id: z.string().describe("Book id"),
+    dry_run: z
+        .boolean()
+        .describe("true = preview only (no renames). false = apply the confirmed renames."),
+    renames: z
+        .array(z.object({
+        asset_id: z.string(),
+        suggested_name: z.string(),
+    }))
+        .optional()
+        .describe("Edited renames for revalidation (dry_run:true) or confirmed renames to apply (dry_run:false). " +
+        "Omit on the first dry run to let GPT-5.6 Sol propose labels."),
+};
+function renameCandidates(assets) {
+    return assets.map((a) => ({
+        artifactId: a.artifactId,
+        ...(a.label ? { label: a.label } : {}),
+        kind: a.kind,
+    }));
+}
+export function renameAllDef(engine) {
+    return {
+        name: "rename_all",
+        description: "Relabel video and image assets in bulk. dry_run proposes descriptive labels from their analysis, transcript and prompts; pass edited renames with dry_run false to apply. Labels are display-only.",
+        schema,
+        handler: async (args) => {
+            const { dry_run, renames } = args;
+            const assets = await engine.artifacts.list();
+            if (dry_run) {
+                return renames && renames.length > 0
+                    ? handleDryRunRevalidation(renames, renameCandidates(assets))
+                    : handleDryRunSuggest(engine, assets);
+            }
+            if (!renames || renames.length === 0) {
+                return {
+                    isError: true,
+                    content: [{
+                            type: "text",
+                            text: "Applying requires a non-empty `renames` array. Run a dry run first.",
+                        }],
+                };
+            }
+            return handleApply(engine, renames, renameCandidates(assets));
+        },
+    };
+}
+async function handleDryRunSuggest(engine, assets) {
+    const apiKey = process.env.OPENAI_API_KEY;
+    if (!apiKey) {
+        return {
+            isError: true,
+            content: [{
+                    type: "text",
+                    text: "OPENAI_API_KEY environment variable is not set",
+                }],
+        };
+    }
+    const candidates = assets.filter((a) => isRenamableKind(a.kind));
+    if (candidates.length === 0) {
+        return {
+            content: [{
+                    type: "text",
+                    text: JSON.stringify({
+                        dry_run: true,
+                        rows: [],
+                        note: "No video or image assets found.",
+                    }),
+                }],
+        };
+    }
+    const contexts = [];
+    for (const a of candidates) {
+        const ctx = await loadAssetContext(engine, a);
+        contexts.push({
+            ...(ctx ?? {
+                asset_id: a.artifactId,
+                type: a.kind,
+                context_text: "",
+            }),
+            ...(a.label ? { label: a.label } : {}),
+        });
+    }
+    const previews = await loadPreviewStills(engine, contexts);
+    const hasImages = previews.some((p) => p.parts.length > 0);
+    const contentParts = buildMultimodalContent(contexts, previews, hasImages);
+    const responses = buildIntelligenceResponses(apiKey, { timeout: 120_000 });
+    const result = await responses(intelligenceRequest({
+        input: [{
+                role: "user",
+                content: responseContentFromChat(contentParts),
+            }],
+        text: { format: { type: "json_object" } },
+        max_output_tokens: 2048,
+    }));
+    const suggestions = parseSuggestionResponse(firstResponseText(result));
+    const rows = buildRenameRows(contexts, suggestions);
+    return {
+        content: [{
+                type: "text",
+                text: JSON.stringify({ dry_run: true, rows, model: INTELLIGENCE_MODEL }),
+            }],
+    };
+}
+async function handleDryRunRevalidation(renames, candidates) {
+    const edits = renames.map((r) => ({
+        asset_id: r.asset_id,
+        suggested_name: r.suggested_name,
+    }));
+    const rows = validateEditedRenames(edits, candidates);
+    return {
+        content: [{
+                type: "text",
+                text: JSON.stringify({ dry_run: true, rows }),
+            }],
+    };
+}
+async function handleApply(engine, renames, candidates) {
+    const edits = renames.map((r) => ({
+        asset_id: r.asset_id,
+        suggested_name: r.suggested_name,
+    }));
+    const rows = validateEditedRenames(edits, candidates);
+    const toApply = applyableRows(rows);
+    if (toApply.length === 0) {
+        return {
+            isError: true,
+            content: [{
+                    type: "text",
+                    text: "No renamable rows after validation. Resolve conflicts/skips and re-run the dry run.",
+                }],
+        };
+    }
+    const applied = [];
+    // Relabeling is display-only: artifact ids are the sole reference currency,
+    // so no prompts, timeline slots, or notebook cells need rewriting.
+    for (const row of toApply) {
+        const result = await engine.artifacts.rename(row.asset_id, row.suggested_name);
+        if (!result.ok) {
+            applied.push({
+                asset_id: row.asset_id,
+                label: row.current_name,
+                renamed: false,
+                error: result.error.message,
+            });
+            continue;
+        }
+        applied.push({
+            asset_id: result.value.artifactId,
+            label: result.value.label ?? row.suggested_name,
+            renamed: true,
+        });
+    }
+    return {
+        content: [{
+                type: "text",
+                text: JSON.stringify({ dry_run: false, applied }),
+            }],
+    };
+}
+/** Load the stored context text used to drive GPT's name suggestion.
+ *  Videos: visual analysis + generation prompt + short transcript.
+ *  Images: analysis, generation prompt, and parent-video context for
+ *  extracted frames. Returns null when no useful text is available. */
+async function loadAssetContext(engine, artifact) {
+    if (artifact.kind === "video") {
+        return loadVideoContext(engine, artifact.artifactId);
+    }
+    if (artifact.kind === "image") {
+        return loadImageContext(engine, artifact.artifactId);
+    }
+    return null;
+}
+function decodeJsonBuffer(value) {
+    return typeof value === "string" ? value : new TextDecoder().decode(value);
+}
+/** Pull summary / subjects / setting / tags from `.original.analysis.json`. */
+async function readAnalysisParts(engine, assetId) {
+    const parts = [];
+    const analysisRead = await engine.files.read(assetId, ".original.analysis.json");
+    if (!analysisRead.ok)
+        return parts;
+    try {
+        const analysis = JSON.parse(decodeJsonBuffer(analysisRead.value));
+        const summary = typeof analysis.summary === "string" ? analysis.summary : "";
+        const subjects = Array.isArray(analysis.subjects) ? analysis.subjects.join(", ") : "";
+        const setting = typeof analysis.setting === "string" ? analysis.setting : "";
+        const tags = Array.isArray(analysis.tags) ? analysis.tags.join(", ") : "";
+        const prompt = typeof analysis.prompt === "string" ? analysis.prompt : "";
+        if (summary)
+            parts.push(summary);
+        if (subjects)
+            parts.push(`subjects: ${subjects}`);
+        if (setting)
+            parts.push(`setting: ${setting}`);
+        if (tags)
+            parts.push(`tags: ${tags}`);
+        // Image describe_image caches a reproduction prompt in analysis.
+        if (prompt && !summary)
+            parts.push(prompt.slice(0, 400));
+    }
+    catch {
+        // ignore parse errors
+    }
+    return parts;
+}
+async function loadVideoContext(engine, assetId) {
+    const parts = [];
+    parts.push(...(await readAnalysisParts(engine, assetId)));
+    // Generation prompt is often the richest description when analysis is missing.
+    const meta = await engine.metadata.artifacts.read(assetId, "original");
+    if (meta.ok) {
+        const prompt = typeof meta.value.prompt === "string" ? meta.value.prompt.trim() : "";
+        if (prompt) {
+            // Strip long @refs so the model focuses on the scene description.
+            const cleaned = stripNotebookMentions(prompt, scanNotebookMentions(prompt).map((mention) => mention.reference))
+                .replace(/\s+/g, " ")
+                .trim()
+                .slice(0, 500);
+            if (cleaned)
+                parts.push(`generation prompt: ${cleaned}`);
+        }
+    }
+    // Transcript is secondary — cap length so it doesn't drown out visuals.
+    for (const transFile of [".analysis.gpt4otrans.json", "elevenlabs.json"]) {
+        const transRead = await engine.files.read(assetId, transFile);
+        if (!transRead.ok)
+            continue;
+        try {
+            const parsed = JSON.parse(decodeJsonBuffer(transRead.value));
+            if (parsed.text && parsed.text.trim()) {
+                const snippet = parsed.text.trim().replace(/\s+/g, " ").slice(0, 220);
+                parts.push(`spoken: ${snippet}`);
+                break;
+            }
+        }
+        catch {
+            // ignore
+        }
+    }
+    if (parts.length === 0)
+        return null;
+    return { asset_id: assetId, type: "video", context_text: parts.join("\n") };
+}
+async function loadImageContext(engine, assetId) {
+    const parts = [];
+    parts.push(...(await readAnalysisParts(engine, assetId)));
+    const meta = await engine.metadata.artifacts.read(assetId, "original");
+    if (meta.ok) {
+        const origin = typeof meta.value.origin === "string" ? meta.value.origin : "";
+        const prompt = typeof meta.value.prompt === "string" ? meta.value.prompt.trim() : "";
+        const sourceAsset = typeof meta.value.source_asset === "string" ? meta.value.source_asset : "";
+        const sourceAssets = Array.isArray(meta.value.source_assets)
+            ? meta.value.source_assets.filter((s) => typeof s === "string")
+            : [];
+        // Extracted frames: inherit the parent video's visual analysis, not "last_frame".
+        if (sourceAsset && origin && /_frame$/i.test(origin)) {
+            const parent = await loadVideoContext(engine, sourceAsset);
+            if (parent?.context_text) {
+                parts.push(`still from ${sourceAsset}: ${parent.context_text.slice(0, 400)}`);
+            }
+        }
+        else if (sourceAssets.length > 0) {
+            // Composite / edit images: briefly describe source materials, then edit intent.
+            for (const src of sourceAssets.slice(0, 3)) {
+                const srcArtifact = engine.artifacts.get(src);
+                if (!srcArtifact.ok)
+                    continue;
+                if (srcArtifact.value.kind === "video") {
+                    const parent = await loadVideoContext(engine, src);
+                    if (parent?.context_text) {
+                        parts.push(`source ${src}: ${parent.context_text.slice(0, 200)}`);
+                    }
+                }
+                else if (srcArtifact.value.kind === "image") {
+                    const srcAnalysis = await readAnalysisParts(engine, src);
+                    if (srcAnalysis.length > 0) {
+                        parts.push(`source ${src}: ${srcAnalysis.join(" ").slice(0, 200)}`);
+                    }
+                    else {
+                        const srcMeta = await engine.metadata.artifacts.read(src, "original");
+                        if (srcMeta.ok) {
+                            const srcOrigin = typeof srcMeta.value.origin === "string" ? srcMeta.value.origin : "";
+                            const srcSource = typeof srcMeta.value.source_asset === "string"
+                                ? srcMeta.value.source_asset
+                                : "";
+                            if (srcSource && srcOrigin && /_frame$/i.test(srcOrigin)) {
+                                const parent = await loadVideoContext(engine, srcSource);
+                                if (parent?.context_text) {
+                                    parts.push(`source ${src}: ${parent.context_text.slice(0, 200)}`);
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+            if (prompt) {
+                parts.push(`edit intent: ${prompt.slice(0, 300)}`);
+            }
+        }
+        else if (prompt) {
+            parts.push(prompt.slice(0, 500));
+        }
+        // Process origins (last_frame, external, …) are not naming signals.
+        if (origin && !isProcessOrigin(origin)) {
+            parts.push(`origin: ${origin}`);
+        }
+    }
+    if (parts.length === 0)
+        return null;
+    return { asset_id: assetId, type: "image", context_text: parts.join("\n") };
+}
+const PREVIEW_CANDIDATES = [
+    "square.jpg",
+    "landscape.jpg",
+    "original.jpg",
+    "portrait.jpg",
+    "original.png",
+];
+/** Prefer first/last original_frames for videos; fall back to poster jpg. */
+async function loadPreviewStills(engine, contexts) {
+    const out = [];
+    for (const ctx of contexts) {
+        const parts = ctx.type === "video"
+            ? await loadVideoPreviewParts(engine, ctx.asset_id)
+            : await loadImagePreviewParts(engine, ctx.asset_id);
+        out.push({ asset_id: ctx.asset_id, parts });
+    }
+    return out;
+}
+async function loadVideoPreviewParts(engine, assetId) {
+    const frameParts = await loadEndFrameParts(engine, assetId);
+    if (frameParts.length > 0)
+        return frameParts;
+    return loadImagePreviewParts(engine, assetId);
+}
+/** Read first + last jpg from original_frames via the asset dir on disk.
+ *  Nested frame paths are not readable through engine.files.read(filename safety). */
+async function loadEndFrameParts(engine, assetId) {
+    const manifest = await engine.files.manifest(assetId);
+    if (!manifest.ok)
+        return [];
+    const frames = manifest.value.directories?.original_frames
+        ?? manifest.value.directories?.landscape_frames
+        ?? null;
+    if (!frames || frames.length === 0)
+        return [];
+    const jpgs = frames.filter((f) => f.endsWith(".jpg")).sort((a, b) => {
+        const ta = parseFloat(a.replace(/\.jpg$/i, "")) || 0;
+        const tb = parseFloat(b.replace(/\.jpg$/i, "")) || 0;
+        return ta - tb;
+    });
+    if (jpgs.length === 0)
+        return [];
+    const first = jpgs[0];
+    const last = jpgs[jpgs.length - 1];
+    const dirKey = manifest.value.directories?.original_frames
+        ? "original_frames"
+        : "landscape_frames";
+    const picks = first === last ? [first] : [first, last];
+    const parts = [];
+    for (const name of picks) {
+        try {
+            const buf = await readFile(join(manifest.value.path, dirKey, name));
+            parts.push(imagePartFromBuffer(buf, "image/jpeg"));
+        }
+        catch {
+            // skip missing frame
+        }
+    }
+    return parts;
+}
+async function loadImagePreviewParts(engine, assetId) {
+    for (const filename of PREVIEW_CANDIDATES) {
+        const read = await engine.files.read(assetId, filename);
+        if (!read.ok)
+            continue;
+        const buf = Buffer.isBuffer(read.value)
+            ? read.value
+            : Buffer.from(read.value);
+        // Cap very large originals (~1.5MB decoded) so the request stays reasonable.
+        if (buf.length > 1_500_000)
+            continue;
+        const mime = filename.endsWith(".png") ? "image/png" : "image/jpeg";
+        return [imagePartFromBuffer(buf, mime)];
+    }
+    return [];
+}
+function imagePartFromBuffer(buf, mime) {
+    return {
+        type: "image_url",
+        image_url: {
+            url: `data:${mime};base64,${buf.toString("base64")}`,
+            detail: "low",
+        },
+    };
+}
+/** Build multimodal chat content: instruction text + per-asset stills. */
+function buildMultimodalContent(contexts, previews, hasImages) {
+    const text = buildSuggestionPrompt(contexts, { hasImages });
+    if (!hasImages)
+        return text;
+    const byId = new Map(previews.map((p) => [p.asset_id, p.parts]));
+    const parts = [{ type: "text", text }];
+    for (const ctx of contexts) {
+        const images = byId.get(ctx.asset_id) ?? [];
+        if (images.length === 0)
+            continue;
+        parts.push({
+            type: "text",
+            text: `Preview for ${ctx.asset_id}:`,
+        });
+        parts.push(...images);
+    }
+    return parts;
+}
+function responseContentFromChat(content) {
+    if (typeof content === "string") {
+        return [{ type: "input_text", text: content }];
+    }
+    return content.map((part) => part.type === "text"
+        ? { type: "input_text", text: part.text }
+        : {
+            type: "input_image",
+            image_url: part.image_url.url,
+            detail: part.image_url.detail ?? "auto",
+        });
+}

package/dist/tools/rename-asset.js

@@ -0,0 +1,34 @@
+import { z } from "zod";
+export function renameAssetDef(engine) {
+    return {
+        name: "rename_asset",
+        description: "Set an asset's display label. Labels are display-only — the artifact id and all references stay unchanged.",
+        schema: {
+            asset_id: z.string().describe("Artifact id of the asset to relabel"),
+            new_name: z.string().describe("New display label (free text, e.g. sunset beach)"),
+            book_id: z.string().describe("Book id"),
+        },
+        handler: async (args) => {
+            const { asset_id, new_name } = args;
+            const result = await engine.artifacts.rename({
+                artifact: asset_id,
+                label: new_name,
+            });
+            if (!result.ok) {
+                return {
+                    isError: true,
+                    content: [{ type: "text", text: result.error.message }],
+                };
+            }
+            return {
+                content: [{
+                        type: "text",
+                        text: JSON.stringify({
+                            artifact_id: result.value.artifactId,
+                            label: result.value.label ?? new_name,
+                        }),
+                    }],
+            };
+        },
+    };
+}