videobook 0.0.0-stage → 0.2.0

package/dist/tools/analyze-jokes.js

@@ -0,0 +1,56 @@
+import { z } from "zod";
+import { queue } from "../queue.js";
+function error(text) {
+    return { isError: true, content: [{ type: "text", text }] };
+}
+function hasSpokenWords(eleven) {
+    return Array.isArray(eleven.words) && eleven.words.some((w) => w.type === "word");
+}
+export function analyzeJokesDef(engine) {
+    return {
+        name: "analyze_jokes",
+        description: "Find and classify the jokes in a clip from its ElevenLabs transcript (verbatim text, start and end times, why each works) in a background job. Run transcribe first when there is no transcript.",
+        schema: {
+            asset_id: z.string().describe("Video asset ID (vid-* or @vid-slug)"),
+            book_id: z.string().describe("Book id"),
+        },
+        handler: async (args) => {
+            const { asset_id, book_id } = args;
+            const assetId = asset_id.replace(/^@/, "");
+            if (!process.env.OPENAI_API_KEY) {
+                return error("OPENAI_API_KEY environment variable is not set");
+            }
+            const manifestResult = await engine.files.manifest(assetId);
+            if (!manifestResult.ok) {
+                return error(manifestResult.error.message);
+            }
+            const read = await engine.files.read(assetId, "elevenlabs.json");
+            if (!read.ok) {
+                return error(`No ElevenLabs transcript found for ${assetId} — run /transcribe @${assetId} first.`);
+            }
+            let eleven;
+            try {
+                eleven = JSON.parse(read.value.toString());
+            }
+            catch {
+                return error(`elevenlabs.json for ${assetId} is not valid JSON`);
+            }
+            if (!hasSpokenWords(eleven)) {
+                return error(`No ElevenLabs transcript found for ${assetId} — run /transcribe @${assetId} first.`);
+            }
+            const jobId = `analyze-jokes-${assetId}-${Date.now()}`;
+            await queue.enqueue(jobId, {
+                type: "analyze_jokes",
+                assetId,
+                assetDir: manifestResult.value.path,
+                bookId: book_id,
+            });
+            return {
+                content: [{
+                        type: "text",
+                        text: JSON.stringify({ asset_id: assetId, job_id: jobId, status: "enqueued" }),
+                    }],
+            };
+        },
+    };
+}

package/dist/tools/analyze-video.js

@@ -0,0 +1,66 @@
+import { findVideoFile } from "videobook-engine";
+import { z } from "zod";
+import { queue } from "../queue.js";
+import { PLAIN_TRANSCRIPT_FILE, TIMED_TRANSCRIPT_FILE, } from "../transcript-projection.js";
+import { isAnalyzableVideo, VIDEO_ANALYSIS_FILE, } from "../video-analysis-projection.js";
+function error(text) {
+    return { isError: true, content: [{ type: "text", text }] };
+}
+function ok(payload) {
+    return { content: [{ type: "text", text: JSON.stringify(payload) }] };
+}
+export async function analyzeVideoJobPayload(engine, assetId, bookId, purpose) {
+    const manifest = await engine.files.manifest(assetId);
+    if (!manifest.ok)
+        throw new Error(manifest.error.message);
+    const [timed, plain] = await Promise.all([
+        engine.files.read(assetId, TIMED_TRANSCRIPT_FILE),
+        engine.files.read(assetId, PLAIN_TRANSCRIPT_FILE),
+    ]);
+    return {
+        type: "analyze_video",
+        assetId,
+        assetDir: manifest.value.path,
+        bookId,
+        hasTranscription: timed.ok || plain.ok,
+        ...(purpose ? { purpose } : {}),
+    };
+}
+export function analyzeVideoDef(engine) {
+    return {
+        name: "analyze_video",
+        description: "Analyze a video with vision and its transcript in a background job: summary, scenes, tags, subjects, and burned-in subtitles or watermarks. The analysis appears on the asset when the job finishes.",
+        schema: {
+            asset_id: z.string().describe("Video artifact id"),
+            book_id: z.string().describe("Book UUID"),
+            force: z.boolean().optional().default(false)
+                .describe("Re-analyze even when the asset already has an analysis"),
+        },
+        handler: async (args) => {
+            const { asset_id, book_id, force } = args;
+            if (!process.env.OPENAI_API_KEY)
+                return error("OPENAI_API_KEY is not set");
+            const artifact = engine.artifacts.get(asset_id);
+            if (!artifact.ok)
+                return error(artifact.error.message);
+            if (!isAnalyzableVideo(artifact.value.kind)) {
+                return error(`analyze_video only accepts video artifacts, got ${artifact.value.kind} artifact ${asset_id}`);
+            }
+            const manifestResult = await engine.files.manifest(asset_id);
+            if (!manifestResult.ok)
+                return error(manifestResult.error.message);
+            if (!findVideoFile(manifestResult.value.files)) {
+                return error(`${asset_id} has no video file yet; wait for its generation or processing to finish before analyzing it`);
+            }
+            if (!force && (await engine.files.read(asset_id, VIDEO_ANALYSIS_FILE)).ok) {
+                return ok({ asset_id, status: "already_analyzed" });
+            }
+            // Stable dedupe key: repeated clicks collapse onto one queued job, and the
+            // partial unique index only covers active states so a later re-analysis
+            // still enqueues.
+            const jobId = `analyze-video-${asset_id}`;
+            await queue.enqueue(jobId, await analyzeVideoJobPayload(engine, asset_id, book_id));
+            return ok({ asset_id, job_id: jobId, status: "enqueued" });
+        },
+    };
+}

package/dist/tools/animate-keywords.js

@@ -0,0 +1,217 @@
+import { join } from "node:path";
+import { findVideoFile } from "videobook-engine";
+import { z } from "zod";
+import { getApiKey } from "../api-key.js";
+import { isSubtitleStyle } from "../remotion/subtitles/styles.js";
+import { isKeywordPlacementCacheValid, keywordPlacementFingerprint, parseKeywordPlacementAnalysis, readKeywordPlacementAnalysis, } from "../keyword-placement-cache.js";
+import { queue } from "../queue.js";
+import { artifactKindOf } from "./_shared.js";
+import { DEFAULT_KEYWORD_GRAPHICS_LINGER_SECONDS, DEFAULT_KEYWORD_GRAPHICS_MAX_COUNT, parseKeywordTranscript, selectKeywordGraphics, selectRequestedKeywordGraphics, } from "../remotion/keyword-graphics.js";
+import { KEYWORD_FRAME_CACHE_FILENAME, resolveKeywordPlacements, } from "../remotion/keyword-placement.js";
+import { RENDER_ORIENTATIONS } from "./reconcile-project-work.js";
+async function persistSettings(engine, assetId, enabled, maxCount, lingerSeconds, keywords) {
+    for (const orientation of RENDER_ORIENTATIONS) {
+        const existing = await engine.metadata.artifacts.read(assetId, `${orientation}.render`);
+        const write = await engine.metadata.artifacts.write(assetId, `${orientation}.render`, {
+            ...(existing.ok ? existing.value : {}),
+            keywordGraphicsEnabled: enabled,
+            keywordGraphicsMaxCount: maxCount,
+            keywordGraphicsLingerSeconds: lingerSeconds,
+            keywordGraphicsKeywords: keywords,
+        });
+        if (!write.ok)
+            throw new Error(`Failed to persist ${orientation} keyword settings: ${write.error.message}`);
+    }
+}
+function isSubtitlePosition(value) {
+    return value === "top" || value === "middle" || value === "bottom";
+}
+function isFontSize(value) {
+    return value === "small" || value === "medium" || value === "large";
+}
+async function placementSummary(engine, assetId, graphics, analysis) {
+    const summaries = {};
+    for (const orientation of RENDER_ORIENTATIONS) {
+        const storedResult = await engine.metadata.artifacts.read(assetId, `${orientation}.render`);
+        const stored = storedResult.ok ? storedResult.value : {};
+        const settings = {
+            subtitlesEnabled: stored.subtitlesEnabled === true,
+            subtitlePosition: isSubtitlePosition(stored.position) ? stored.position : "bottom",
+            subtitleStyle: isSubtitleStyle(stored.highlightStyle) ? stored.highlightStyle : "impact",
+            subtitleFontSize: isFontSize(stored.fontSize) ? stored.fontSize : "medium",
+        };
+        const resolved = resolveKeywordPlacements(graphics, analysis, orientation, settings);
+        summaries[orientation] = {
+            keywords: resolved.graphics.map((graphic) => ({
+                text: graphic.text,
+                anchor: graphic.anchor,
+                scale: graphic.scale,
+                frame_time: graphic.frameTime,
+            })),
+            skipped: resolved.skipped,
+        };
+    }
+    return summaries;
+}
+function errorResult(status, message, details = {}) {
+    return {
+        isError: true,
+        content: [{ type: "text", text: JSON.stringify({ status, message, ...details }) }],
+    };
+}
+export function animateKeywordsDef(engine) {
+    return {
+        name: "animate_keywords",
+        description: "Turn transcript-timed keyword titles on or off for a video's renders, placed by vision away from faces, subtitles and on-screen text. Pass keywords for exact transcript words, or omit them for automatic picks.",
+        schema: {
+            asset_id: z.string().describe("Video artifact id"),
+            book_id: z.string().describe("Book id"),
+            enabled: z.boolean().optional().describe("Enable keyword graphics (default true)"),
+            max_keywords: z
+                .number()
+                .int()
+                .min(1)
+                .max(20)
+                .optional()
+                .describe("Maximum keyword graphics selected from the transcript (default 8)"),
+            linger_seconds: z
+                .number()
+                .min(0.8)
+                .max(5)
+                .optional()
+                .describe("How long each keyword lingers after it is spoken (default 2.4 seconds)"),
+            keywords: z
+                .array(z.string().min(1))
+                .min(1)
+                .max(20)
+                .optional()
+                .describe("Exact transcript words to animate, matched case-insensitively"),
+        },
+        handler: async (args) => {
+            const { asset_id, book_id, enabled = true, max_keywords = DEFAULT_KEYWORD_GRAPHICS_MAX_COUNT, linger_seconds = DEFAULT_KEYWORD_GRAPHICS_LINGER_SECONDS, keywords, } = args;
+            // Kept on the structured errorResult contract instead of
+            // requireArtifactKind so callers keep parsing a status code. A missing
+            // artifact falls through to the manifest read below, which reports
+            // asset_not_found.
+            const kind = artifactKindOf(engine, asset_id);
+            if (kind !== null && kind !== "video") {
+                return errorResult("invalid_asset", `animate_keywords requires a video artifact; ${asset_id} is ${kind}`);
+            }
+            const manifestResult = await engine.files.manifest(asset_id);
+            if (!manifestResult.ok) {
+                return errorResult("asset_not_found", manifestResult.error.message);
+            }
+            if (!enabled) {
+                try {
+                    await persistSettings(engine, asset_id, false, max_keywords, linger_seconds, null);
+                }
+                catch (error) {
+                    return errorResult("settings_write_failed", error instanceof Error ? error.message : String(error));
+                }
+                return {
+                    content: [{
+                            type: "text",
+                            text: JSON.stringify({ asset_id, status: "configured", enabled: false }),
+                        }],
+                };
+            }
+            const transcriptResult = await engine.files.read(asset_id, "elevenlabs.json");
+            if (!transcriptResult.ok) {
+                return errorResult("missing_transcript", `No timed transcript found for ${asset_id}. Run /transcribe @${asset_id} first.`);
+            }
+            let raw;
+            try {
+                raw = JSON.parse(transcriptResult.value.toString());
+            }
+            catch {
+                return errorResult("invalid_transcript", `The timed transcript for ${asset_id} is not valid JSON.`);
+            }
+            const segments = parseKeywordTranscript(raw);
+            const requested = keywords?.length
+                ? selectRequestedKeywordGraphics(segments, keywords, { lingerSeconds: linger_seconds })
+                : null;
+            if (requested && requested.missing.length > 0) {
+                return errorResult("keywords_not_found", `Some requested keywords were not found in the timed transcript for ${asset_id}.`, { missing_keywords: requested.missing });
+            }
+            const graphics = requested?.graphics ?? selectKeywordGraphics(segments, {
+                maxCount: max_keywords,
+                lingerSeconds: linger_seconds,
+            });
+            if (graphics.length === 0) {
+                return errorResult("no_timed_words", `The transcript for ${asset_id} has no usable word-level timing. Run /transcribe @${asset_id} again.`);
+            }
+            const videoFile = findVideoFile(manifestResult.value.files);
+            if (!videoFile) {
+                return errorResult("missing_video", `No video file found in ${asset_id}.`);
+            }
+            const fingerprint = keywordPlacementFingerprint(videoFile, transcriptResult.value, graphics);
+            let analysis = await readKeywordPlacementAnalysis(engine, asset_id);
+            let cached = await isKeywordPlacementCacheValid(engine, asset_id, analysis, fingerprint, graphics);
+            let jobId = null;
+            if (!cached) {
+                if (!getApiKey("OPENAI_API_KEY")) {
+                    return errorResult("missing_vision_provider", "OPENAI_API_KEY is required to verify face-safe keyword placement.");
+                }
+                const clear = await engine.jobs.failures.clear(asset_id);
+                if (!clear.ok) {
+                    return errorResult("placement_analysis_failed", clear.error.message);
+                }
+                jobId = `keyword-placement-${asset_id}-${Date.now()}`;
+                try {
+                    const result = await queue.enqueueAndWait(jobId, {
+                        type: "analyze_keyword_placement",
+                        assetId: asset_id,
+                        assetDir: manifestResult.value.path,
+                        bookId: book_id,
+                        videoPath: join(manifestResult.value.path, videoFile.name),
+                        fingerprint,
+                        graphics,
+                    }, 300_000);
+                    analysis = parseKeywordPlacementAnalysis(result.analysis);
+                    cached = false;
+                }
+                catch (error) {
+                    return errorResult("placement_analysis_failed", error instanceof Error ? error.message : String(error));
+                }
+            }
+            if (!analysis) {
+                return errorResult("placement_analysis_failed", "Keyword placement analysis was not persisted.");
+            }
+            let placements;
+            try {
+                placements = await placementSummary(engine, asset_id, graphics, analysis);
+            }
+            catch (error) {
+                return errorResult("placement_analysis_failed", error instanceof Error ? error.message : String(error));
+            }
+            const resolvedKeywords = requested ? graphics.map((graphic) => graphic.text) : null;
+            try {
+                await persistSettings(engine, asset_id, true, max_keywords, linger_seconds, resolvedKeywords);
+            }
+            catch (error) {
+                return errorResult("settings_write_failed", error instanceof Error ? error.message : String(error));
+            }
+            return {
+                content: [{
+                        type: "text",
+                        text: JSON.stringify({
+                            asset_id,
+                            status: "configured",
+                            enabled: true,
+                            style: "cinematic-serif",
+                            placement_status: "completed",
+                            cached,
+                            frame_cache: KEYWORD_FRAME_CACHE_FILENAME,
+                            job_id: jobId,
+                            mode: requested ? "explicit" : "automatic",
+                            max_keywords,
+                            linger_seconds,
+                            requested_keywords: resolvedKeywords,
+                            keywords: graphics,
+                            placements,
+                        }),
+                    }],
+            };
+        },
+    };
+}

package/dist/tools/api-status.js

@@ -0,0 +1,42 @@
+import { createKimiCoding } from "@apicity/kimicoding";
+import { isPlaceholderApiKey } from "../api-key.js";
+export function apiStatusDef() {
+    return {
+        name: "check_kimi_coding_api",
+        description: "Check whether the Kimi Coding API is reachable and its configured API key is valid",
+        schema: {},
+        handler: async () => {
+            const apiKey = process.env.KIMI_CODING_API_KEY;
+            if (!apiKey) {
+                return {
+                    isError: true,
+                    content: [{ type: "text", text: "KIMI_CODING_API_KEY environment variable is not set" }],
+                };
+            }
+            if (isPlaceholderApiKey(apiKey)) {
+                return {
+                    isError: true,
+                    content: [{ type: "text", text: "KIMI_CODING_API_KEY is a placeholder; refusing to call the real API" }],
+                };
+            }
+            try {
+                const provider = createKimiCoding({ apiKey });
+                await provider.post.coding.v1.messages({
+                    model: "k2p5",
+                    max_tokens: 1,
+                    messages: [{ role: "user", content: "hi" }],
+                });
+                return {
+                    content: [{ type: "text", text: "200 OK" }],
+                };
+            }
+            catch (error) {
+                const message = error instanceof Error ? error.message : String(error);
+                return {
+                    isError: true,
+                    content: [{ type: "text", text: message }],
+                };
+            }
+        },
+    };
+}

package/dist/tools/application-commands.js

@@ -0,0 +1,34 @@
+const text = (name, description, required = true) => ({ name, description, required, type: "string" });
+const number = (name, description, required = true) => ({ name, description, required, type: "number" });
+const clip = text("clip_id", "Sequence clip; choose from the current sequence");
+const track = text("track_id", "Sequence track");
+const media = text("asset_id", "Source media or notebook address, such as @a1");
+const start = number("start_frame", "Start frame in the sequence, beginning at zero");
+const duration = number("duration_frames", "Duration in frames");
+function command(name, description, params = [], example) {
+    return {
+        command: name, description, params, example: example ?? name, category: "action",
+        icon: "Play", size: "default", variant: "default", chevron: "right", exposure: "composite",
+    };
+}
+export const APPLICATION_COMMANDS = [
+    command("/caption", "Preview a sequence caption with timing and style", [start, duration, text("style", "clean, bold or high-contrast"), text("text", "Caption text")], "/caption 0 90 clean Opening scene"),
+    command("/duck", "Lower background audio beneath a narration clip", [text("narration_clip_id", "Narration clip"), number("reduction_db", "Reduction from -36 to -1 dB; default -12", false)]),
+    command("/fit", "Fit, fill or crop a visual sequence clip", [clip, text("mode", "fit, fill or crop")]),
+    command("/gap", "Preview inserting a gap that shifts later clips forward", [start, duration, { ...track, required: false }]),
+    command("/generate-cell", "Review generation for a configured notebook cell", [text("cell_id", "Generation cell, such as @a1")], "/generate-cell @a1"),
+    command("/insert", "Preview inserting media into the sequence", [media, start, { ...duration, required: false }], "/insert @a1 0 90"),
+    command("/move", "Preview moving a sequence clip to a track and frame", [clip, track, start]),
+    command("/overwrite", "Preview overwriting sequence content with media", [media, start, { ...duration, required: false }], "/overwrite @a1 0 90"),
+    command("/reframe", "Preview a crop transform for a visual sequence clip", [clip]),
+    command("/rough-cut", "Review a rough cut built from notebook media"),
+    command("/sound-effects", "Review suggested sound effects for the sequence"),
+    command("/track-add", "Add a video, audio or caption track", [text("kind", "video, audio or caption"), text("name", "Display name", false)], "/track-add audio Music"),
+    command("/track-remove", "Remove an empty sequence track", [track]),
+    command("/volume", "Preview a sequence clip's audio gain", [clip, number("gain_db", "Audio gain from -96 to 24 dB")]),
+    command("/clip-audio", "Edit a sequence audio clip's gain, mute, or fades", [clip, text("field", "gain, mute, fade-in, or fade-out"), text("value", "Gain from -96 to 24 dB, on or off, or fade length in frames")], "/clip-audio @clip-id gain -6"),
+    command("/clip-delete", "Preview deleting a sequence clip while retaining its source media", [clip, text("ripple", "Use --ripple to close the gap", false)]),
+    command("/clip-reverse", "Preview reversing a sequence clip while retaining its source media", [clip]),
+    command("/clip-trim", "Preview trimming a sequence clip in frames", [clip, start, duration]),
+    command("/export-sequence", "Review and export the current sequence", [text("aspect", "16:9, 9:16 or 1:1", false), text("captions", "burned, sidecar or omit", false)], "/export-sequence --aspect 16:9 --captions burned"),
+];

package/dist/tools/apply-cuts.js

@@ -0,0 +1,112 @@
+import { z } from "zod";
+import { join } from "node:path";
+import { queue } from "../queue.js";
+import { findVideoFile } from "videobook-engine";
+import { probeVideo } from "../ffmpeg.js";
+export async function resolveHasAudioForCuts(metaHasAudio, inputPath) {
+    if (typeof metaHasAudio === "boolean")
+        return metaHasAudio;
+    return (await probeVideo(inputPath)).hasAudio;
+}
+function validateCuts(cuts, duration) {
+    if (cuts.length === 0)
+        return { ok: false, error: "cuts must contain at least one entry" };
+    const sorted = [...cuts].sort((a, b) => a.cut_start - b.cut_start);
+    let lastEnd = -Infinity;
+    for (const c of sorted) {
+        if (c.cut_start < 0 || c.cut_end < 0) {
+            return { ok: false, error: "cut_start and cut_end must be >= 0" };
+        }
+        if (c.cut_start >= c.cut_end) {
+            return { ok: false, error: `cut_start must be < cut_end (got ${c.cut_start} >= ${c.cut_end})` };
+        }
+        if (c.cut_start < lastEnd) {
+            return { ok: false, error: `cuts overlap: ${c.cut_start.toFixed(2)} < previous end ${lastEnd.toFixed(2)}` };
+        }
+        if (duration !== null && c.cut_end > duration + 0.01) {
+            return { ok: false, error: `cut_end ${c.cut_end.toFixed(2)} exceeds video duration ${duration.toFixed(2)}` };
+        }
+        lastEnd = c.cut_end;
+    }
+    return { ok: true, sorted };
+}
+export function applyCutsDef(engine) {
+    return {
+        name: "apply_cuts",
+        description: "Remove one or more time ranges from a video in place, then reprocess its frames, transcript and analysis. Take ranges from suggest_cuts or choose them.",
+        schema: {
+            asset_id: z.string().describe("Video asset ID (vid-*)"),
+            book_id: z.string().describe("Book id"),
+            cuts: z
+                .array(z.object({
+                cut_start: z.number().min(0).describe("Start of range to remove (seconds)"),
+                cut_end: z.number().min(0).describe("End of range to remove (seconds)"),
+            }))
+                .min(1)
+                .describe("Ranges to remove. Sorted, non-overlapping, within [0, duration]."),
+        },
+        handler: async (args) => {
+            const { asset_id, book_id, cuts } = args;
+            const assetId = asset_id.replace(/^@/, "");
+            const manifestResult = await engine.files.manifest(assetId);
+            if (!manifestResult.ok) {
+                return {
+                    isError: true,
+                    content: [{ type: "text", text: manifestResult.error.message }],
+                };
+            }
+            const manifest = manifestResult.value;
+            const videoFile = findVideoFile(manifest.files);
+            if (!videoFile) {
+                return {
+                    isError: true,
+                    content: [{ type: "text", text: `No video file found in asset ${assetId}` }],
+                };
+            }
+            const metaResult = await engine.metadata.artifacts.read(assetId, "original");
+            const duration = metaResult.ok && typeof metaResult.value.duration === "number"
+                ? metaResult.value.duration
+                : null;
+            const metaHasAudio = metaResult.ok && typeof metaResult.value.hasAudio === "boolean"
+                ? metaResult.value.hasAudio
+                : undefined;
+            const validated = validateCuts(cuts, duration);
+            if (!validated.ok) {
+                return {
+                    isError: true,
+                    content: [{ type: "text", text: validated.error }],
+                };
+            }
+            const assetDir = manifest.path;
+            const inputPath = join(assetDir, videoFile.name);
+            const hasAudio = await resolveHasAudioForCuts(metaHasAudio, inputPath);
+            const outputPath = join(assetDir, "original_cuts.mp4");
+            const jobId = `apply-cuts-${assetId}-${Date.now()}`;
+            try {
+                await queue.enqueueAndWait(jobId, {
+                    type: "apply_cuts",
+                    inputPath,
+                    outputPath,
+                    cuts: validated.sorted,
+                    duration: duration ?? 0,
+                    hasAudio,
+                    assetId,
+                    assetDir,
+                    bookId: book_id,
+                });
+            }
+            catch (err) {
+                return {
+                    isError: true,
+                    content: [{ type: "text", text: `Apply cuts failed: ${err instanceof Error ? err.message : String(err)}` }],
+                };
+            }
+            return {
+                content: [{
+                        type: "text",
+                        text: JSON.stringify({ asset_id: assetId, job_id: jobId, status: "completed" }),
+                    }],
+            };
+        },
+    };
+}