videobook 0.0.0-stage → 0.2.0

package/dist/tools/wait-for-asset-ready.js

@@ -0,0 +1,143 @@
+import { z } from "zod";
+import { reconcileProjectWorkDef } from "./reconcile-project-work.js";
+function delay(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+export function waitForAssetReadyDef(engine) {
+    return {
+        name: "wait_for_asset_ready",
+        description: "Wait until an asset's asynchronous generation, reconciliation, and file writes finish or the timeout expires.",
+        schema: {
+            asset_id: z.string().describe("Asset ID to wait on"),
+            book_id: z.string().describe("Book id"),
+            timeout_seconds: z
+                .number()
+                .default(300)
+                .describe("Max seconds to wait before returning a timeout error"),
+        },
+        handler: async (args) => {
+            const { asset_id, book_id, timeout_seconds, } = args;
+            // Verify asset exists
+            const manifestResult = await engine.files.manifest(asset_id);
+            if (!manifestResult.ok) {
+                return {
+                    isError: true,
+                    content: [
+                        {
+                            type: "text",
+                            text: JSON.stringify({
+                                asset_id,
+                                status: "not_found",
+                                error: "Asset not found",
+                            }),
+                        },
+                    ],
+                };
+            }
+            const assetDir = manifestResult.value.path;
+            const reconcileHandler = reconcileProjectWorkDef(engine).handler;
+            const deadline = Date.now() + timeout_seconds * 1000;
+            const startTime = Date.now();
+            let iterations = 0;
+            let lastReconciliation = null;
+            while (Date.now() < deadline) {
+                iterations++;
+                // Drive state machine forward
+                const reconcileResult = await reconcileHandler({ book_id });
+                const reconcileData = JSON.parse(reconcileResult.content[0].text);
+                lastReconciliation = reconcileData;
+                // Check for failure
+                const failure = reconcileData.failed.find((f) => f.assetId === asset_id);
+                if (failure) {
+                    return {
+                        isError: true,
+                        content: [
+                            {
+                                type: "text",
+                                text: JSON.stringify({
+                                    asset_id,
+                                    status: "failed",
+                                    elapsed_seconds: Math.round((Date.now() - startTime) / 1000),
+                                    iterations,
+                                    error: failure.failMsg,
+                                    last_sync: lastReconciliation,
+                                }),
+                            },
+                        ],
+                    };
+                }
+                const jobs = engine.jobs.queue.list({
+                    artifactId: asset_id,
+                    limit: 20,
+                });
+                const failedJob = jobs.find((job) => job.state === "failed");
+                if (failedJob) {
+                    return {
+                        isError: true,
+                        content: [
+                            {
+                                type: "text",
+                                text: JSON.stringify({
+                                    asset_id,
+                                    status: "failed",
+                                    elapsed_seconds: Math.round((Date.now() - startTime) / 1000),
+                                    iterations,
+                                    error: failedJob.error?.message ?? "Asset job failed",
+                                    last_sync: lastReconciliation,
+                                }),
+                            },
+                        ],
+                    };
+                }
+                const activeJob = jobs.some((job) => job.state === "queued" ||
+                    job.state === "running" ||
+                    job.state === "completing");
+                const ptRead = await engine.jobs.pending.read(asset_id);
+                const pendingTask = ptRead.ok ? ptRead.value : null;
+                const locked = await engine.jobs.locks.isLocked(assetDir);
+                const workRead = await engine.jobs.artifactWork.read(asset_id);
+                const activeWork = workRead.ok &&
+                    workRead.value !== null &&
+                    (workRead.value.status === "working" ||
+                        (workRead.value.status === "pending" &&
+                            typeof workRead.value.meta.kind === "string"));
+                if (pendingTask === null &&
+                    !locked &&
+                    !activeJob &&
+                    !activeWork) {
+                    return {
+                        content: [
+                            {
+                                type: "text",
+                                text: JSON.stringify({
+                                    asset_id,
+                                    status: "ready",
+                                    elapsed_seconds: Math.round((Date.now() - startTime) / 1000),
+                                    iterations,
+                                    last_sync: lastReconciliation,
+                                }),
+                            },
+                        ],
+                    };
+                }
+                await delay(Math.min(5000, Math.max(1, deadline - Date.now())));
+            }
+            // Timeout
+            return {
+                isError: true,
+                content: [
+                    {
+                        type: "text",
+                        text: JSON.stringify({
+                            asset_id,
+                            status: "timeout",
+                            elapsed_seconds: Math.round((Date.now() - startTime) / 1000),
+                            iterations,
+                            last_sync: lastReconciliation,
+                        }),
+                    },
+                ],
+            };
+        },
+    };
+}

package/dist/tools/workspace-actions.js

@@ -0,0 +1,54 @@
+import { z } from "zod";
+import { buildWorkspaceActionCatalog } from "../workspace-action-catalog.js";
+import { WORKSPACE_TOOL_GROUPS } from "../workspace-action-groups.js";
+import { getCanonicalCommands } from "./commands.js";
+const ACTION_GROUPS = Object.keys(WORKSPACE_TOOL_GROUPS);
+function handlerName(action) {
+    switch (action.handler.kind) {
+        case "tool": return `tool:${action.handler.tool}`;
+        case "ui": return `ui:${action.handler.action}`;
+        case "command": return `command:${action.handler.command}`;
+    }
+}
+function actionSummary(action) {
+    return {
+        id: action.id,
+        command: action.command,
+        title: action.title,
+        group: action.group,
+        runs: handlerName(action),
+    };
+}
+function matchesQuery(action, terms) {
+    const text = [action.title, action.command, action.description, action.group, ...action.aliases, ...action.keywords]
+        .join(" ").toLowerCase();
+    return terms.every((term) => text.includes(term));
+}
+function selectActions(actions, selection) {
+    const terms = (selection.query ?? "").trim().toLowerCase().replace(/^\//u, "").split(/\s+/u).filter(Boolean);
+    const ids = selection.ids ? new Set(selection.ids) : null;
+    return actions.filter((action) => (!ids || ids.has(action.id))
+        && (!selection.group || action.group === selection.group)
+        && matchesQuery(action, terms));
+}
+export function workspaceActionsDef(getTools) {
+    return {
+        name: "list_workspace_actions",
+        description: "List workspace actions (tools, app screens and composite commands) as id, command, title, group and what each runs. Filter with query, group or ids; detail full adds typed inputs, surfaces, confirmation policy and tool coverage.",
+        scope: "global",
+        schema: {
+            query: z.string().optional().describe("Words that must all appear in an action's title, command, description or keywords"),
+            group: z.enum(ACTION_GROUPS).optional(),
+            ids: z.array(z.string()).optional().describe("Action ids such as tool.trim_video or command.goal"),
+            detail: z.enum(["summary", "full"]).optional(),
+        },
+        handler: async (args) => {
+            const catalog = buildWorkspaceActionCatalog(getTools(), getCanonicalCommands());
+            const actions = selectActions(catalog.actions, args);
+            const payload = args.detail === "full"
+                ? { actions, tools: catalog.tools }
+                : { actions: actions.map(actionSummary) };
+            return { content: [{ type: "text", text: JSON.stringify(payload) }] };
+        },
+    };
+}

package/dist/transcript-projection.js

@@ -0,0 +1,202 @@
+import { assessDialogQuality } from "./dialog-quality.js";
+/** Artifact kinds whose media can carry speech worth transcribing. */
+export function hasSpokenAudio(kind) {
+    return kind === "video" || kind === "audio" || kind === "final";
+}
+/** Word-level ElevenLabs speech-to-text output written by the transcribe job. */
+export const TIMED_TRANSCRIPT_FILE = "elevenlabs.json";
+/** Plain-text transcription sidecar written during ingest. */
+export const PLAIN_TRANSCRIPT_FILE = ".analysis.gpt4otrans.json";
+function parseJson(value) {
+    try {
+        return JSON.parse(value.toString());
+    }
+    catch {
+        return null;
+    }
+}
+function firstChannelTranscript(raw) {
+    if (typeof raw !== "object" || raw === null)
+        return null;
+    const obj = raw;
+    if (typeof obj.words !== "undefined")
+        return obj;
+    const channels = obj.transcripts;
+    if (Array.isArray(channels) && channels.length > 0)
+        return channels[0];
+    return null;
+}
+function isTimedWord(value) {
+    if (typeof value !== "object" || value === null)
+        return false;
+    const word = value;
+    return typeof word.text === "string"
+        && typeof word.start === "number"
+        && typeof word.end === "number";
+}
+/** Audio events ("[laughter]") and spacing entries are not spoken words. */
+function isSpeech(word) {
+    return word.type !== "audio_event" && word.type !== "spacing";
+}
+export function timedTranscriptFrom(raw) {
+    const transcript = firstChannelTranscript(raw);
+    if (!transcript || !Array.isArray(transcript.words))
+        return null;
+    const speech = transcript.words.filter(isTimedWord).filter(isSpeech);
+    if (speech.length === 0)
+        return { kind: "silence" };
+    const textEntries = transcript.words.filter(isTimedWord).filter((word) => word.type !== "audio_event");
+    const reconstructedText = textEntries.some((word) => word.type === "spacing")
+        ? textEntries.map((word) => word.text).join("").replace(/\s+/gu, " ").trim()
+        : speech.map((word) => word.text.trim()).join(" ");
+    return {
+        kind: "timed",
+        languageCode: transcript.language_code ?? null,
+        text: transcript.text ?? reconstructedText,
+        words: speech.map((word) => ({
+            text: word.text,
+            start: word.start,
+            end: word.end,
+            speakerId: word.speaker_id ?? null,
+        })),
+    };
+}
+function plainTranscriptFrom(raw) {
+    if (typeof raw !== "object" || raw === null)
+        return null;
+    const quality = assessDialogQuality(raw);
+    if (!quality.hasDialog && (quality.reason === "empty" || quality.reason === "noise_only")) {
+        return { kind: "silence" };
+    }
+    return { kind: "plain", text: quality.text };
+}
+function sidecarFile(source) {
+    return source === "elevenlabs" ? TIMED_TRANSCRIPT_FILE : PLAIN_TRANSCRIPT_FILE;
+}
+async function readTimedTranscript(engine, assetId) {
+    const timedRead = await engine.files.read(assetId, TIMED_TRANSCRIPT_FILE);
+    if (!timedRead.ok)
+        return null;
+    return timedTranscriptFrom(parseJson(timedRead.value));
+}
+async function readPlainTranscript(engine, assetId) {
+    const plainRead = await engine.files.read(assetId, PLAIN_TRANSCRIPT_FILE);
+    if (!plainRead.ok)
+        return null;
+    return plainTranscriptFrom(parseJson(plainRead.value));
+}
+async function readAssetTranscriptWithSource(engine, assetId) {
+    const timed = await readTimedTranscript(engine, assetId);
+    if (timed)
+        return { transcript: timed, source: "timed" };
+    const plain = await readPlainTranscript(engine, assetId);
+    if (plain)
+        return { transcript: plain, source: "plain" };
+    return { transcript: { kind: "missing" }, source: "none" };
+}
+/**
+ * Resolve the best transcript for an artifact: word-timed ElevenLabs cues when
+ * present, otherwise the plain ingest transcription.
+ */
+export async function readAssetTranscript(engine, assetId) {
+    return (await readAssetTranscriptWithSource(engine, assetId)).transcript;
+}
+function countWords(text) {
+    const stripped = text.replace(/\[[^\]]*\]/g, "").trim();
+    return stripped.length === 0 ? 0 : stripped.split(/\s+/).length;
+}
+function transcriptSummary(transcript) {
+    if (transcript.kind === "missing")
+        return null;
+    if (transcript.kind === "silence") {
+        return { kind: "silence", language_code: null, text: "", word_count: 0 };
+    }
+    if (transcript.kind === "plain") {
+        return {
+            kind: "plain",
+            language_code: null,
+            text: transcript.text,
+            word_count: countWords(transcript.text),
+        };
+    }
+    return {
+        kind: "timed",
+        language_code: transcript.languageCode,
+        text: transcript.text,
+        word_count: transcript.words.length,
+    };
+}
+function transcriptSourceSummary(transcript) {
+    const summary = transcriptSummary(transcript);
+    if (!summary)
+        return null;
+    if (transcript.kind !== "timed")
+        return summary;
+    return {
+        ...summary,
+        words: transcript.words.map((word) => ({
+            text: word.text,
+            start: word.start,
+            end: word.end,
+        })),
+    };
+}
+/** Word-timed transcripts are large; asset listings re-read them constantly.
+ *  Their content-addressed hash makes an in-process memo safe and exact. */
+const SUMMARY_CACHE_LIMIT = 512;
+const summaryByObjectHash = new Map();
+function rememberSummary(objectHash, summary) {
+    summaryByObjectHash.delete(objectHash);
+    summaryByObjectHash.set(objectHash, summary);
+    while (summaryByObjectHash.size > SUMMARY_CACHE_LIMIT) {
+        const oldest = summaryByObjectHash.keys().next().value;
+        if (typeof oldest !== "string")
+            return;
+        summaryByObjectHash.delete(oldest);
+    }
+}
+async function readTimedSource(engine, assetId, timedHash) {
+    if (timedHash) {
+        const cached = summaryByObjectHash.get(timedHash);
+        if (cached)
+            return cached;
+    }
+    const timed = await readTimedTranscript(engine, assetId);
+    if (!timed)
+        return null;
+    const summary = transcriptSourceSummary(timed);
+    if (timedHash && summary)
+        rememberSummary(timedHash, summary);
+    return summary;
+}
+/** Both transcript sidecars, independently, or null when neither exists.
+ *  Pass the artifact's manifest files to skip missing sidecars and reuse the
+ *  cached projection of an unchanged word-timed transcript. */
+export async function readAssetTranscriptSources(engine, assetId, files) {
+    const timedFile = files?.find((file) => file.name === TIMED_TRANSCRIPT_FILE);
+    const plainFile = files?.find((file) => file.name === PLAIN_TRANSCRIPT_FILE);
+    if (files && !timedFile && !plainFile)
+        return null;
+    const elevenlabs = !files || timedFile
+        ? await readTimedSource(engine, assetId, timedFile?.objectHash ?? null)
+        : null;
+    const plain = !files || plainFile
+        ? transcriptSourceSummary(await readPlainTranscript(engine, assetId) ?? { kind: "missing" })
+        : null;
+    if (!elevenlabs && !plain)
+        return null;
+    return { elevenlabs, plain };
+}
+export async function readTranscriptSidecarText(engine, assetId, source) {
+    const order = source
+        ? [source]
+        : ["elevenlabs", "plain"];
+    for (const kind of order) {
+        const file = sidecarFile(kind);
+        const read = await engine.files.read(assetId, file);
+        if (!read.ok)
+            continue;
+        return { source: kind, file, text: read.value.toString() };
+    }
+    return null;
+}

package/dist/transcript-srt.js

@@ -0,0 +1,33 @@
+import { chunkSegments } from "./remotion/subtitles/chunker.js";
+import { parseSegments } from "./remotion/subtitles/segments.js";
+import { srtContent } from "./srt.js";
+function asElevenLabsJson(raw) {
+    if (typeof raw !== "object" || raw === null) {
+        throw new Error("ElevenLabs transcript is not valid JSON");
+    }
+    return raw;
+}
+function monotonicCues(chunks) {
+    const cues = [];
+    let lastEnd = 0;
+    for (const chunk of chunks) {
+        const text = chunk.text.trim();
+        if (!text)
+            continue;
+        const start = Math.max(chunk.start, lastEnd);
+        const end = Math.max(chunk.end, start);
+        const safeEnd = end === start ? start + 0.001 : end;
+        cues.push({ startSeconds: start, endSeconds: safeEnd, text });
+        lastEnd = safeEnd;
+    }
+    return cues;
+}
+export function transcriptToSrt(raw, options = {}) {
+    const segments = parseSegments(asElevenLabsJson(raw));
+    const chunks = chunkSegments(segments, options.orientation ?? "landscape", options.style ?? "impact");
+    const cues = monotonicCues(chunks);
+    if (cues.length === 0) {
+        throw new Error("No word-timed cues in the ElevenLabs transcript");
+    }
+    return srtContent(cues);
+}

package/dist/upload.js

@@ -0,0 +1,76 @@
+import { createFreeMediaUpload, uploadToAnyHost } from "@apicity/free-media-upload";
+import { createHash } from "node:crypto";
+import { maybeReencodeImage, maybeUpscaleSmallImage } from "./image-compress.js";
+export function contentHash(data) {
+    return createHash("sha256").update(data).digest("hex");
+}
+const MIME_TYPES = {
+    jpg: "image/jpeg",
+    jpeg: "image/jpeg",
+    png: "image/png",
+    gif: "image/gif",
+    webp: "image/webp",
+    bmp: "image/bmp",
+    svg: "image/svg+xml",
+    mp4: "video/mp4",
+    webm: "video/webm",
+    mov: "video/quicktime",
+    avi: "video/x-msvideo",
+    mp3: "audio/mpeg",
+    wav: "audio/wav",
+    ogg: "audio/ogg",
+    flac: "audio/flac",
+    aac: "audio/aac",
+    m4a: "audio/mp4",
+};
+function inferMimeType(filename) {
+    const ext = filename.split(".").pop()?.toLowerCase();
+    return (ext && MIME_TYPES[ext]) ?? "application/octet-stream";
+}
+// Upload bytes to a free file host and return a publicly accessible download
+// URL. Delegates rotation to @apicity/free-media-upload's `uploadToAnyHost` across
+// litterbox/uguu/tflink.
+//
+// Host selection matters: Alibaba/Kie workers run in China and synchronously
+// fetch the URL server-side during data inspection. Several otherwise-fine
+// hosts (catbox.moe, filebin.net, temp.sh, tmpfiles.org) are unreachable or
+// mis-typed from that side and cause InvalidParameter.DataInspection 400s
+// even when the upload itself succeeded. The three hosts below were verified
+// end-to-end (upload OK + Alibaba submit accepted).
+export async function uploadBytesToFreeHost(data, filename) {
+    const mimeType = inferMimeType(filename);
+    const blob = new Blob([new Uint8Array(data)], { type: mimeType });
+    const file = new File([blob], filename, { type: mimeType });
+    return uploadToAnyHost(createFreeMediaUpload(), {
+        file,
+        filename,
+        hosts: ["litterbox", "uguu", "tflink"],
+        time: "1h",
+    });
+}
+async function prepareBytesForUpload(bytes, filename) {
+    const raw = bytes instanceof Uint8Array ? bytes : new Uint8Array(bytes);
+    const reencoded = await maybeReencodeImage(raw, filename);
+    const { data, filename: outName } = await maybeUpscaleSmallImage(reencoded.data, reencoded.filename);
+    const ext = outName.split(".").pop() ?? "bin";
+    const hashedFilename = `${contentHash(data)}.${ext}`;
+    const contentType = inferMimeType(outName);
+    return { data, hashedFilename, contentType };
+}
+// Read an asset, optionally re-encode it, and hash its final bytes for use as
+// a collision-free remote filename. Shared preamble for every provider-native
+// upload wrapper.
+export async function prepareAssetForUpload(engine, assetId, filename) {
+    const readResult = await engine.files.read(assetId, filename);
+    if (!readResult.ok)
+        throw new Error(readResult.error.message);
+    return prepareBytesForUpload(new Uint8Array(readResult.value), filename);
+}
+// Upload an asset file via the free-host rotation. Kept for
+// provider-agnostic callers (e.g. `transcribe.ts`) that don't target a
+// specific AI provider. Provider-specific jobs should call the matching
+// `uploadAssetTo{Fal,Kie,Alibaba}` wrapper instead.
+export async function uploadAssetFile(engine, assetId, filename) {
+    const { data, hashedFilename } = await prepareAssetForUpload(engine, assetId, filename);
+    return uploadBytesToFreeHost(data, hashedFilename);
+}

package/dist/usage-providers.js

@@ -0,0 +1,25 @@
+import { createOpenAi as openai } from "@apicity/openai";
+import { createXai as xai } from "@apicity/xai";
+import { createKie as kie } from "@apicity/kie";
+import { createFal as fal } from "@apicity/fal";
+import { createAlibaba as alibaba } from "@apicity/alibaba";
+import { createElevenLabs as elevenlabs } from "@apicity/elevenlabs";
+import { providerUsageFetch } from "./provider-usage.js";
+export function createOpenAi(options) {
+    return openai({ ...options, fetch: providerUsageFetch("openai", options.fetch) });
+}
+export function createXai(options) {
+    return xai({ ...options, fetch: providerUsageFetch("xai", options.fetch) });
+}
+export function createKie(options) {
+    return kie({ ...options, fetch: providerUsageFetch("kie", options.fetch) });
+}
+export function createFal(options) {
+    return fal({ ...options, fetch: providerUsageFetch("fal", options.fetch) });
+}
+export function createAlibaba(options) {
+    return alibaba({ ...options, fetch: providerUsageFetch("alibaba", options.fetch) });
+}
+export function createElevenLabs(options) {
+    return elevenlabs({ ...options, fetch: providerUsageFetch("elevenlabs", options.fetch) });
+}

package/dist/video-analysis-projection.js

@@ -0,0 +1,46 @@
+/** Vision analysis sidecar written by the analyze_video job. */
+export const VIDEO_ANALYSIS_FILE = ".original.analysis.json";
+/** Artifact kinds the analyze_video job can describe from sampled frames. */
+export function isAnalyzableVideo(kind) {
+    return kind === "video" || kind === "final";
+}
+function stringOrNull(value) {
+    return typeof value === "string" && value.trim().length > 0 ? value.trim() : null;
+}
+function stringList(value) {
+    return Array.isArray(value)
+        ? value.filter((entry) => typeof entry === "string" && entry.trim().length > 0)
+        : [];
+}
+function videoAnalysisSummaryFrom(raw) {
+    if (typeof raw !== "object" || raw === null)
+        return null;
+    const analysis = raw;
+    const summary = stringOrNull(analysis.summary);
+    const tags = stringList(analysis.tags);
+    const subjects = stringList(analysis.subjects);
+    const sceneCount = Array.isArray(analysis.scenes) ? analysis.scenes.length : 0;
+    if (!summary && tags.length === 0 && subjects.length === 0 && sceneCount === 0) {
+        return null;
+    }
+    return {
+        summary: summary ?? "",
+        mood: stringOrNull(analysis.mood),
+        setting: stringOrNull(analysis.setting),
+        tags,
+        subjects,
+        scene_count: sceneCount,
+    };
+}
+/** Vision analysis projection for an artifact, or null when it has not been analyzed. */
+export async function readVideoAnalysisSummary(engine, assetId) {
+    const read = await engine.files.read(assetId, VIDEO_ANALYSIS_FILE);
+    if (!read.ok)
+        return null;
+    try {
+        return videoAnalysisSummaryFrom(JSON.parse(read.value.toString()));
+    }
+    catch {
+        return null;
+    }
+}

package/dist/video-poster.js

@@ -0,0 +1,33 @@
+import { mkdtemp, readFile, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { isThumbnailBlank, runFfmpeg } from "./ffmpeg.js";
+export async function extractVideoPoster(videoPath, duration, signal) {
+    const scratch = await mkdtemp(join(tmpdir(), "videobook-poster-"));
+    const posterPath = join(scratch, "poster.jpg");
+    try {
+        const extract = async (seconds) => {
+            await runFfmpeg([
+                "-y", "-ss", String(seconds), "-i", videoPath,
+                "-map", "0:v:0", "-frames:v", "1", "-q:v", "2", posterPath,
+            ], { signal });
+            const bytes = await readFile(posterPath);
+            if (bytes.length === 0)
+                throw new Error("Video poster is empty");
+            return bytes;
+        };
+        const first = await extract(0);
+        if (duration > 0 && await isThumbnailBlank(posterPath)) {
+            try {
+                return await extract(duration / 2);
+            }
+            catch {
+                signal?.throwIfAborted();
+            }
+        }
+        return first;
+    }
+    finally {
+        await rm(scratch, { recursive: true, force: true });
+    }
+}