videobook 0.0.0-stage → 0.2.0

package/dist/tools/extract-frame.js

@@ -0,0 +1,108 @@
+import { z } from "zod";
+import { join } from "node:path";
+import { queue } from "../queue.js";
+import { findVideoFile } from "videobook-engine";
+import { probeVideo } from "../ffmpeg.js";
+async function extractFrame(videoAssetId, bookId, mode, engine, customTimestamp) {
+    const sourceArtifact = engine.artifacts.get(videoAssetId);
+    if (!sourceArtifact.ok) {
+        return {
+            isError: true,
+            content: [{ type: "text", text: sourceArtifact.error.message }],
+        };
+    }
+    const manifestResult = await engine.files.manifest(videoAssetId);
+    if (!manifestResult.ok) {
+        return {
+            isError: true,
+            content: [{ type: "text", text: manifestResult.error.message }],
+        };
+    }
+    const manifest = manifestResult.value;
+    const videoFile = findVideoFile(manifest.files);
+    if (!videoFile) {
+        return {
+            isError: true,
+            content: [{ type: "text", text: `No video file found in asset ${videoAssetId}` }],
+        };
+    }
+    const videoPath = join(manifest.path, videoFile.name);
+    let probe;
+    try {
+        probe = await probeVideo(videoPath);
+    }
+    catch (error) {
+        const msg = error instanceof Error ? error.message : String(error);
+        return {
+            isError: true,
+            content: [{ type: "text", text: `Failed to probe video: ${msg}` }],
+        };
+    }
+    if (!Number.isFinite(probe.videoDuration) || probe.videoDuration <= 0 || probe.width <= 0 || probe.height <= 0) {
+        return {
+            isError: true,
+            content: [{ type: "text", text: "Source has no decodable video duration" }],
+        };
+    }
+    const frameSeconds = probe.fps > 0 ? 1 / probe.fps : 0.1;
+    const lastTimestamp = Math.max(0, probe.videoDuration - frameSeconds - 0.000001);
+    const requestedTimestamp = mode === "custom"
+        ? customTimestamp ?? 0
+        : mode === "first" ? 0.1 : probe.videoDuration - 0.1;
+    const timestamp = Math.max(0, Math.min(requestedTimestamp, lastTimestamp));
+    const baseName = sourceArtifact.value.label ?? sourceArtifact.value.artifactId;
+    const imageName = mode === "custom"
+        ? `${baseName}-frame-${String(Math.round(timestamp * 100) / 100).replace(".", "-")}s`
+        : `${baseName}-${mode}-frame`;
+    const assetResult = await engine.artifacts.create("image", imageName);
+    if (!assetResult.ok) {
+        return {
+            isError: true,
+            content: [{ type: "text", text: `Failed to create image asset: ${assetResult.error.message}` }],
+        };
+    }
+    const { artifactId: imageAssetId, path: imageAssetDir } = assetResult.value;
+    const jobId = `extract-frame-${imageAssetId}-${Date.now()}`;
+    await queue.enqueue(jobId, {
+        type: "extract_frame",
+        inputPath: videoPath,
+        timestamp,
+        imageAssetId,
+        imageAssetDir,
+        sourceVideoId: videoAssetId,
+        bookId,
+        width: probe.width,
+        height: probe.height,
+        mode,
+    });
+    return {
+        content: [{
+                type: "text",
+                text: JSON.stringify({ image_asset_id: imageAssetId, source_video: videoAssetId, job_id: jobId, status: "enqueued" }),
+            }],
+    };
+}
+export function extractFrameDef(engine) {
+    return {
+        name: "extract_frame",
+        description: "Save one video frame as a new image asset (JPEG): the first frame, the last frame, or the frame at timestamp seconds.",
+        schema: {
+            book_id: z.string().describe("Book UUID"),
+            asset_id: z.string().describe("Video artifact id"),
+            frame: z.enum(["first", "last"]).optional().describe("Take the first or last frame instead of the frame at timestamp"),
+            timestamp: z.number().min(0).optional().describe("Seconds into the video; required unless frame is set"),
+        },
+        handler: async (args) => {
+            const { asset_id, book_id, frame, timestamp } = args;
+            if (frame)
+                return extractFrame(asset_id, book_id, frame, engine);
+            if (typeof timestamp !== "number") {
+                return {
+                    isError: true,
+                    content: [{ type: "text", text: "timestamp is required unless frame is first or last" }],
+                };
+            }
+            return extractFrame(asset_id, book_id, "custom", engine, timestamp);
+        },
+    };
+}

package/dist/tools/extract-notebook-audio.js

@@ -0,0 +1,173 @@
+import { notebookOccupiedSlots } from "videobook-engine/notebook-grid";
+import { findVideoFile, nextHorizontalSlotFrom, notebookGridTag, resolveNotebookCellArtifactId, } from "videobook-engine";
+import { join } from "node:path";
+import { z } from "zod";
+import { v7 as uuidv7 } from "uuid";
+import { mutateNotebook } from "../notebook-mutations.js";
+import { resolveAudioExtractionRange } from "../notebook-media-ops.js";
+import { probeVideo } from "../ffmpeg.js";
+import { EXTRACT_AUDIO_OUTPUT_ROLE, findGenerationMediaOutput, normalizeLegacyExtractCells, } from "../notebook-cell-outputs.js";
+import { slotRight } from "../notebook-placement.js";
+import { resolvePublishedPostSourceCell } from "../notebook-published-post.js";
+import { notebookCellFingerprint, notebookCellMediaType, } from "../jobs/run-notebook.js";
+import { queue } from "../queue.js";
+import { toolError } from "./_shared.js";
+function effectiveVideoSource(document, cell) {
+    const holder = resolvePublishedPostSourceCell(document, cell);
+    const output = findGenerationMediaOutput(document, holder);
+    return output?.outputCell ?? holder;
+}
+export function extractNotebookAudioDef(engine) {
+    return {
+        name: "extract_notebook_audio",
+        description: "Extract audio from a video cell into a new audio cell wired to its source, recording the source cell and optional time range",
+        scope: "book",
+        schema: {
+            book_id: z.string().uuid().describe("Book UUID"),
+            notebook_id: z.string().uuid().describe("Notebook holding the source cell"),
+            source_cell_id: z.string().uuid()
+                .describe("Video-producing cell to extract from"),
+            start_seconds: z.number().min(0).optional()
+                .describe("Optional range start in source seconds"),
+            duration_seconds: z.number().positive().optional()
+                .describe("Optional range duration in seconds"),
+            label: z.string().optional().describe("Optional label for the audio cell"),
+            expected_revision: z.string().min(1).optional()
+                .describe("Reject the extraction when the notebook has moved past this revision"),
+        },
+        handler: async (args) => {
+            const notebookId = String(args.notebook_id);
+            const sourceCellId = String(args.source_cell_id);
+            const startSeconds = typeof args.start_seconds === "number"
+                ? args.start_seconds
+                : undefined;
+            const durationSeconds = typeof args.duration_seconds === "number"
+                ? args.duration_seconds
+                : undefined;
+            if (typeof args.expected_revision === "string"
+                && args.expected_revision !== engine.head) {
+                return toolError(`Stale notebook revision: expected ${args.expected_revision}`);
+            }
+            const read = engine.notebooks.read(notebookId);
+            if (!read.ok)
+                return toolError(read.error.message);
+            const document = normalizeLegacyExtractCells(read.value);
+            const requested = document.cells.find((cell) => cell.id === sourceCellId);
+            if (!requested) {
+                return toolError(`Source cell not found: ${sourceCellId}`);
+            }
+            const source = effectiveVideoSource(document, requested);
+            if (notebookCellMediaType(source) !== "video"
+                || source.type === "split_video") {
+                return toolError(`${notebookGridTag(source.slot)} is not a video-producing cell`);
+            }
+            const sourceArtifactId = resolveNotebookCellArtifactId(document, source);
+            if (!sourceArtifactId) {
+                return toolError(`${notebookGridTag(source.slot)} has no video to extract from yet`);
+            }
+            const audioRange = startSeconds === undefined && durationSeconds === undefined
+                ? undefined
+                : { startSeconds: startSeconds ?? 0, ...(durationSeconds === undefined ? {} : { durationSeconds }) };
+            try {
+                const manifest = await engine.files.manifest(sourceArtifactId);
+                if (!manifest.ok)
+                    throw new Error(manifest.error.message);
+                const file = findVideoFile(manifest.value.files);
+                if (!file)
+                    throw new Error("Source has no video file");
+                resolveAudioExtractionRange(await probeVideo(join(manifest.value.path, file.name)), audioRange);
+            }
+            catch (error) {
+                return toolError(error instanceof Error ? error.message : String(error));
+            }
+            const runId = uuidv7();
+            const label = typeof args.label === "string" && args.label.length > 0
+                ? args.label
+                : startSeconds === undefined
+                    ? "Extracted audio"
+                    : `Audio ${startSeconds}s`;
+            const placement = { cell: null };
+            const result = await mutateNotebook(engine, notebookId, "extract_notebook_audio", (notebook) => {
+                if (typeof args.expected_revision === "string" && args.expected_revision !== engine.head) {
+                    throw new Error(`Stale notebook revision: expected ${args.expected_revision}`);
+                }
+                const anchor = notebook.cells.find((cell) => cell.id === source.id);
+                if (!anchor)
+                    throw new Error(`Source cell not found: ${source.id}`);
+                const occupied = notebookOccupiedSlots(notebook.cells);
+                const slot = nextHorizontalSlotFrom(slotRight(anchor.slot), occupied);
+                const cell = engine.notebooks.createCell({
+                    type: "audio",
+                    label,
+                    slot,
+                    inputs: {
+                        mediaRole: EXTRACT_AUDIO_OUTPUT_ROLE,
+                        sourceCellId: anchor.id,
+                        ...(startSeconds === undefined ? {} : { startSeconds }),
+                        ...(durationSeconds === undefined ? {} : { durationSeconds }),
+                        ...(audioRange === undefined ? {} : { audioRange }),
+                        state: "queued",
+                    },
+                });
+                const edge = engine.notebooks.createEdge({
+                    source: anchor.id,
+                    target: cell.id,
+                    targetInput: "segment",
+                });
+                placement.cell = cell;
+                const queuedAt = new Date().toISOString();
+                return {
+                    ...notebook,
+                    cells: [...notebook.cells, cell],
+                    edges: [...notebook.edges, edge],
+                    execution: {
+                        ...(notebook.execution ?? {}),
+                        [cell.id]: {
+                            fingerprint: notebookCellFingerprint(notebook, cell),
+                            status: "queued",
+                            runId,
+                            startedAt: queuedAt,
+                            updatedAt: queuedAt,
+                            tool: "ffmpeg",
+                        },
+                    },
+                };
+            }, { sourceCellId: source.id, runId });
+            if (!result.ok)
+                return toolError(result.error.message);
+            const cell = placement.cell;
+            if (!cell)
+                return toolError("Audio cell was not created");
+            const address = notebookGridTag(cell.slot);
+            const jobId = `extract-notebook-audio-${cell.id}`;
+            const payload = {
+                type: "extract_notebook_audio",
+                bookId: String(args.book_id),
+                notebookId,
+                cellId: cell.id,
+                runId,
+                sourceCellId: source.id,
+                sourceArtifactId,
+                title: `${address} · extracted audio`,
+                ...(startSeconds === undefined ? {} : { startSeconds }),
+                ...(durationSeconds === undefined ? {} : { durationSeconds }),
+            };
+            await queue.enqueue(jobId, payload);
+            return {
+                content: [{
+                        type: "text",
+                        text: JSON.stringify({
+                            cell_id: cell.id,
+                            address,
+                            job_id: jobId,
+                            run_id: runId,
+                            notebook_id: notebookId,
+                            source_cell_id: source.id,
+                            source_address: notebookGridTag(source.slot),
+                            message: `Extracting audio into ${address}`,
+                        }),
+                    }],
+            };
+        },
+    };
+}

package/dist/tools/extract-notebook-frame.js

@@ -0,0 +1,160 @@
+import { notebookOccupiedSlots } from "videobook-engine/notebook-grid";
+import { nextHorizontalSlotFrom, notebookGridTag, resolveNotebookCellArtifactId, } from "videobook-engine";
+import { z } from "zod";
+import { v7 as uuidv7 } from "uuid";
+import { mutateNotebook } from "../notebook-mutations.js";
+import { EXTRACT_FRAME_OUTPUT_ROLE, findGenerationMediaOutput, normalizeLegacyExtractCells, } from "../notebook-cell-outputs.js";
+import { slotRight } from "../notebook-placement.js";
+import { resolvePublishedPostSourceCell } from "../notebook-published-post.js";
+import { notebookCellFingerprint, notebookCellMediaType, } from "../jobs/run-notebook.js";
+import { queue } from "../queue.js";
+import { toolError } from "./_shared.js";
+function frameLabel(mode, timestampSeconds) {
+    if (mode === "first")
+        return "First frame";
+    if (mode === "last")
+        return "Last frame";
+    return `Frame at ${timestampSeconds}s`;
+}
+function effectiveVideoSource(document, cell) {
+    const holder = resolvePublishedPostSourceCell(document, cell);
+    const output = findGenerationMediaOutput(document, holder);
+    return output?.outputCell ?? holder;
+}
+export function extractNotebookFrameDef(engine) {
+    return {
+        name: "extract_notebook_frame",
+        description: "Extract a frame from a video cell into a new image cell wired to its source, recording the source cell and timestamp",
+        scope: "book",
+        schema: {
+            book_id: z.string().uuid().describe("Book UUID"),
+            notebook_id: z.string().uuid().describe("Notebook holding the source cell"),
+            source_cell_id: z.string().uuid()
+                .describe("Video-producing cell to extract from"),
+            frame_mode: z.enum(["first", "time", "last"])
+                .describe("Capture the first frame, an exact timestamp, or the last frame"),
+            timestamp_seconds: z.number().min(0).optional()
+                .describe("Source timestamp in seconds, required when frame_mode is time"),
+            label: z.string().optional().describe("Optional label for the image cell"),
+            expected_revision: z.string().min(1).optional()
+                .describe("Reject the extraction when the notebook has moved past this revision"),
+        },
+        handler: async (args) => {
+            const notebookId = String(args.notebook_id);
+            const sourceCellId = String(args.source_cell_id);
+            const frameMode = args.frame_mode;
+            const timestampSeconds = typeof args.timestamp_seconds === "number"
+                ? args.timestamp_seconds
+                : undefined;
+            if (frameMode === "time" && timestampSeconds === undefined) {
+                return toolError("frame_mode \"time\" requires timestamp_seconds");
+            }
+            if (typeof args.expected_revision === "string"
+                && args.expected_revision !== engine.head) {
+                return toolError(`Stale notebook revision: expected ${args.expected_revision}`);
+            }
+            const read = engine.notebooks.read(notebookId);
+            if (!read.ok)
+                return toolError(read.error.message);
+            const document = normalizeLegacyExtractCells(read.value);
+            const requested = document.cells.find((cell) => cell.id === sourceCellId);
+            if (!requested) {
+                return toolError(`Source cell not found: ${sourceCellId}`);
+            }
+            const source = effectiveVideoSource(document, requested);
+            if (notebookCellMediaType(source) !== "video"
+                || source.type === "split_video") {
+                return toolError(`${notebookGridTag(source.slot)} is not a video-producing cell`);
+            }
+            const sourceArtifactId = resolveNotebookCellArtifactId(document, source);
+            if (!sourceArtifactId) {
+                return toolError(`${notebookGridTag(source.slot)} has no video to extract from yet`);
+            }
+            const runId = uuidv7();
+            const label = typeof args.label === "string" && args.label.length > 0
+                ? args.label
+                : frameLabel(frameMode, timestampSeconds);
+            const placement = { cell: null };
+            const result = await mutateNotebook(engine, notebookId, "extract_notebook_frame", (notebook) => {
+                if (typeof args.expected_revision === "string" && args.expected_revision !== engine.head) {
+                    throw new Error(`Stale notebook revision: expected ${args.expected_revision}`);
+                }
+                const anchor = notebook.cells.find((cell) => cell.id === source.id);
+                if (!anchor)
+                    throw new Error(`Source cell not found: ${source.id}`);
+                const occupied = notebookOccupiedSlots(notebook.cells);
+                const slot = nextHorizontalSlotFrom(slotRight(anchor.slot), occupied);
+                const cell = engine.notebooks.createCell({
+                    type: "image",
+                    label,
+                    slot,
+                    inputs: {
+                        mediaRole: EXTRACT_FRAME_OUTPUT_ROLE,
+                        frameMode,
+                        ...(timestampSeconds === undefined ? {} : { timestampSeconds }),
+                        sourceCellId: anchor.id,
+                        state: "queued",
+                    },
+                });
+                const edge = engine.notebooks.createEdge({
+                    source: anchor.id,
+                    target: cell.id,
+                    targetInput: "segment",
+                });
+                placement.cell = cell;
+                const queuedAt = new Date().toISOString();
+                return {
+                    ...notebook,
+                    cells: [...notebook.cells, cell],
+                    edges: [...notebook.edges, edge],
+                    execution: {
+                        ...(notebook.execution ?? {}),
+                        [cell.id]: {
+                            fingerprint: notebookCellFingerprint(notebook, cell),
+                            status: "queued",
+                            runId,
+                            startedAt: queuedAt,
+                            updatedAt: queuedAt,
+                            tool: "ffmpeg",
+                        },
+                    },
+                };
+            }, { sourceCellId: source.id, frameMode, runId });
+            if (!result.ok)
+                return toolError(result.error.message);
+            const cell = placement.cell;
+            if (!cell)
+                return toolError("Frame cell was not created");
+            const address = notebookGridTag(cell.slot);
+            const jobId = `extract-notebook-frame-${cell.id}`;
+            const payload = {
+                type: "extract_notebook_frame",
+                bookId: String(args.book_id),
+                notebookId,
+                cellId: cell.id,
+                runId,
+                sourceCellId: source.id,
+                sourceArtifactId,
+                frameMode,
+                ...(timestampSeconds === undefined ? {} : { timestampSeconds }),
+                title: `${address} · ${frameMode} frame`,
+            };
+            await queue.enqueue(jobId, payload);
+            return {
+                content: [{
+                        type: "text",
+                        text: JSON.stringify({
+                            cell_id: cell.id,
+                            address,
+                            job_id: jobId,
+                            run_id: runId,
+                            notebook_id: notebookId,
+                            source_cell_id: source.id,
+                            source_address: notebookGridTag(source.slot),
+                            message: `Extracting ${frameMode} frame into ${address}`,
+                        }),
+                    }],
+            };
+        },
+    };
+}

package/dist/tools/extract-video-clip.js

@@ -0,0 +1,101 @@
+import { z } from "zod";
+import { queue } from "../queue.js";
+import { resolveVideoAssetReference } from "./video-source-reference.js";
+import { parseVideoTimestamp, videoTimestampSchema } from "./video-time-input.js";
+const MIN_CLIP_SECONDS = 0.5;
+function derivedClipName(base, startTime, endTime) {
+    return `${base}-${startTime}s-${endTime}s`;
+}
+export function extractVideoClipDef(engine) {
+    return {
+        name: "extract_video_clip",
+        description: "Cut a new video clip from a source time range, keeping the original (for example 'a new clip from @a1 from 15s to 44s'). Times accept seconds, 15s, 1:30 or 2m20s.",
+        annotations: {
+            title: "Extract new video clip",
+            readOnlyHint: false,
+            destructiveHint: false,
+            idempotentHint: false,
+            openWorldHint: false,
+        },
+        schema: {
+            asset_id: z.string().describe("Source video artifact id or notebook reference such as @a1"),
+            book_id: z.string().describe("Book UUID"),
+            notebook_id: z.string().uuid().optional().describe("Notebook UUID when asset_id is a grid reference and the book has multiple matching notebooks"),
+            start_time: videoTimestampSchema.optional().describe("Start of the new clip; defaults to 0 seconds"),
+            end_time: videoTimestampSchema.describe("End of the new clip"),
+            name: z.string().min(1).optional().describe("Optional name for the new video asset"),
+        },
+        handler: async (args) => {
+            const { asset_id, book_id, notebook_id, start_time, end_time, name } = args;
+            let assetId;
+            let startTime;
+            let endTime;
+            try {
+                assetId = resolveVideoAssetReference(engine, asset_id, notebook_id);
+                startTime = parseVideoTimestamp(start_time, "start_time") ?? 0;
+                endTime = parseVideoTimestamp(end_time, "end_time");
+            }
+            catch (err) {
+                return {
+                    isError: true,
+                    content: [{ type: "text", text: err instanceof Error ? err.message : String(err) }],
+                };
+            }
+            if (startTime < 0 || endTime <= startTime) {
+                return {
+                    isError: true,
+                    content: [{ type: "text", text: "The clip range must have a non-negative start_time before end_time." }],
+                };
+            }
+            if (endTime - startTime < MIN_CLIP_SECONDS) {
+                return {
+                    isError: true,
+                    content: [{ type: "text", text: `The new clip must be at least ${MIN_CLIP_SECONDS}s long.` }],
+                };
+            }
+            const sourceArtifact = engine.artifacts.get(assetId);
+            if (!sourceArtifact.ok) {
+                return {
+                    isError: true,
+                    content: [{ type: "text", text: sourceArtifact.error.message }],
+                };
+            }
+            const clipBase = sourceArtifact.value.label ?? sourceArtifact.value.artifactId;
+            const jobId = `extract-clip-${assetId}-${Date.now()}`;
+            let extracted;
+            try {
+                extracted = await queue.enqueueAndWait(jobId, {
+                    type: "extract_video_clip",
+                    sourceAssetId: assetId,
+                    startTime,
+                    endTime,
+                    title: name ?? derivedClipName(clipBase, startTime, endTime),
+                    bookId: book_id,
+                });
+            }
+            catch (err) {
+                return {
+                    isError: true,
+                    content: [{ type: "text", text: `Clip extraction failed: ${err instanceof Error ? err.message : String(err)}` }],
+                };
+            }
+            const outputAssetId = extracted.artifactId;
+            const actualEndTime = startTime + extracted.durationSeconds;
+            return {
+                content: [{
+                        type: "text",
+                        text: JSON.stringify({
+                            source_asset_id: assetId,
+                            output_asset_id: outputAssetId,
+                            output_artifact_id: extracted.artifactId,
+                            start_time: startTime,
+                            end_time: actualEndTime,
+                            duration: extracted.durationSeconds,
+                            job_id: jobId,
+                            status: "completed",
+                        }),
+                    }],
+            };
+        },
+    };
+}