videobook 0.0.0-stage → 0.2.0

package/dist/media/ref-fields.js

@@ -0,0 +1,106 @@
+import { arrayBounds, arrayItems, isArraySchema, isUrlObject, propertyAt } from "./json-schema.js";
+const CANDIDATES = {
+    first_frame: [
+        "first_frame_url",
+        "first_frame_image_url",
+        "start_image_url",
+        "first_frame",
+    ],
+    last_frame: [
+        "last_frame_url",
+        "last_frame_image_url",
+        "end_image_url",
+        "tail_image_url",
+        "last_frame",
+    ],
+    image: [
+        "image_urls",
+        "reference_image_urls",
+        "image_input",
+        "input_urls",
+        "image_references",
+        "reference_images",
+        "reference_image",
+        "imageUrls",
+        "image_url",
+        "reference_image_url",
+        "image",
+        "images",
+    ],
+    video: [
+        "video_urls",
+        "reference_video_urls",
+        "video_list",
+        "reference_video",
+        "video_url",
+        "first_clip_url",
+        "video",
+        "videos",
+    ],
+    audio: [
+        "audio_urls",
+        "reference_audio_urls",
+        "audio_url",
+        "driving_audio_url",
+    ],
+};
+function fieldPresent(input, field) {
+    return propertyAt(input, field) !== null;
+}
+function isRequired(input, field) {
+    return !field.includes(".") && (input.required ?? []).includes(field);
+}
+function expandField(input, field) {
+    const schema = propertyAt(input, field);
+    const required = isRequired(input, field);
+    if (isArraySchema(schema)) {
+        const bounds = arrayBounds(schema);
+        return [{
+                path: field,
+                multi: true,
+                max: bounds.max,
+                min: required ? Math.max(1, bounds.min) : 0,
+                urlObject: isUrlObject(arrayItems(schema)),
+            }];
+    }
+    return [{ path: field, multi: false, max: 1, min: required ? 1 : 0, urlObject: isUrlObject(schema) }];
+}
+function overrideFields(entry, role) {
+    const override = entry.refs?.[role];
+    if (override === null || override === undefined)
+        return override;
+    return typeof override === "string" ? [override] : override;
+}
+export function resolveSlot(role, input, entry) {
+    const override = overrideFields(entry, role);
+    if (override === null)
+        return null;
+    const names = override ?? CANDIDATES[role].filter((candidate) => fieldPresent(input, candidate)).slice(0, 1);
+    if (names.length === 0)
+        return null;
+    for (const name of names) {
+        if (!fieldPresent(input, name))
+            throw new Error(`${entry.id}: ref field ${name} for ${role} is not in the apicity schema`);
+    }
+    const expanded = names.flatMap((name) => expandField(input, name));
+    const fields = expanded.map(({ path, multi, max, urlObject }) => ({ path, multi, max, urlObject }));
+    const max = expanded.some((field) => field.max === null)
+        ? null
+        : expanded.reduce((sum, field) => sum + (field.max ?? 0), 0);
+    return {
+        role,
+        field: names.join("+"),
+        fields,
+        multi: max === null || max > 1,
+        min: expanded.reduce((sum, field) => sum + field.min, 0),
+        max,
+    };
+}
+export function claimedFields(slots) {
+    const claimed = new Set();
+    for (const slot of slots) {
+        for (const field of slot.fields)
+            claimed.add(field.path.split(".")[0]);
+    }
+    return claimed;
+}

package/dist/media/refs.js

@@ -0,0 +1,131 @@
+import { referenceVideoDuration } from "../reference-video-duration.js";
+import { insertVideoTimingError, isInsertVideoModel } from "./insert-video-timing.js";
+import { resolveAudioRef, resolveImageRef, resolveVideoRef } from "../tools/_media-refs.js";
+import { REF_ROLES } from "./types.js";
+function kindOf(role) {
+    if (role === "video")
+        return "video";
+    if (role === "audio")
+        return "audio";
+    return "image";
+}
+function isHttpUrl(value) {
+    return value.startsWith("http://") || value.startsWith("https://");
+}
+export async function resolveRefInput(engine, kind, raw) {
+    const resolved = kind === "video"
+        ? await resolveVideoRef(engine, raw)
+        : kind === "audio"
+            ? await resolveAudioRef(engine, raw)
+            : await resolveImageRef(engine, raw);
+    if (!resolved.ok)
+        throw new Error(resolved.error);
+    return resolved.value;
+}
+function sourceList(result, kind) {
+    if (kind === "video")
+        return result.sourceVideoIds;
+    if (kind === "audio")
+        return result.sourceAudioIds;
+    return result.sourceImageIds;
+}
+async function uploadRef(engine, adapter, kind, raw, result) {
+    if (isHttpUrl(raw))
+        return adapter.uploadUrl(raw);
+    const input = await resolveRefInput(engine, kind, raw);
+    sourceList(result, kind).push(input.assetId);
+    return adapter.upload(engine, input.assetId, input.filename);
+}
+export async function uploadRefs(engine, adapter, refs) {
+    const result = { urls: {}, sourceImageIds: [], sourceVideoIds: [], sourceAudioIds: [] };
+    for (const role of REF_ROLES) {
+        const raws = refs[role] ?? [];
+        if (raws.length === 0)
+            continue;
+        const urls = [];
+        for (const raw of raws)
+            urls.push(await uploadRef(engine, adapter, kindOf(role), raw, result));
+        result.urls[role] = urls;
+    }
+    return result;
+}
+export async function checkRefs(engine, refs) {
+    for (const role of REF_ROLES) {
+        for (const raw of refs[role] ?? []) {
+            if (isHttpUrl(raw))
+                continue;
+            try {
+                await resolveRefInput(engine, kindOf(role), raw);
+            }
+            catch (error) {
+                return `${role} ref ${raw}: ${error instanceof Error ? error.message : String(error)}`;
+            }
+        }
+    }
+    return null;
+}
+export function normalizeRefs(raw) {
+    if (raw === null || typeof raw !== "object")
+        return {};
+    const refs = {};
+    for (const role of REF_ROLES) {
+        const value = raw[role];
+        const list = Array.isArray(value) ? value : value === undefined || value === null ? [] : [value];
+        const strings = list.filter((item) => typeof item === "string" && item.trim() !== "").map((item) => item.trim());
+        if (strings.length > 0)
+            refs[role] = strings;
+    }
+    return refs;
+}
+function seconds(ms) {
+    return String(Math.round(ms / 100) / 10);
+}
+async function knownDurationMs(engine, role, raw) {
+    if (isHttpUrl(raw))
+        return undefined;
+    const input = await resolveRefInput(engine, role, raw);
+    if (!input.filename.startsWith("original"))
+        return undefined;
+    const meta = await engine.metadata.artifacts.read(input.assetId, "original");
+    const duration = meta.ok ? meta.value?.duration : undefined;
+    return typeof duration === "number" && Number.isFinite(duration) && duration > 0 ? Math.round(duration * 1000) : undefined;
+}
+function clipError(modelId, role, raw, ms, window) {
+    if (ms >= window.minimum && ms <= window.maximum)
+        return null;
+    return `${role} ref ${raw} is ${seconds(ms)} s; ${modelId} accepts ${seconds(window.minimum)}-${seconds(window.maximum)} s per ${role} reference`;
+}
+async function checkRoleDurations(engine, modelId, role, raws, window) {
+    let total = 0;
+    for (const raw of raws) {
+        const ms = await knownDurationMs(engine, role, raw);
+        if (ms === undefined)
+            continue;
+        const error = clipError(modelId, role, raw, ms, window);
+        if (error)
+            return error;
+        total += ms;
+    }
+    return total > window.maximumTotal
+        ? `${role} refs total ${seconds(total)} s; ${modelId} accepts at most ${seconds(window.maximumTotal)} s of ${role} references`
+        : null;
+}
+export async function checkRefDurations(engine, model, refs) {
+    const windows = model.entry.rules?.refDurationMs;
+    if (!windows)
+        return null;
+    for (const role of ["video", "audio"]) {
+        const window = windows[role];
+        const error = window ? await checkRoleDurations(engine, model.id, role, refs[role] ?? [], window) : null;
+        if (error)
+            return error;
+    }
+    return null;
+}
+export async function checkInsertVideoTiming(engine, model, refs, options) {
+    if (!isInsertVideoModel(model.id))
+        return null;
+    const source = refs.video?.[0];
+    const duration = source ? await referenceVideoDuration(engine, [source]) : undefined;
+    return insertVideoTimingError(options, duration);
+}

package/dist/media/request.js

@@ -0,0 +1,188 @@
+import { insertVideoTimingError, isInsertVideoModel } from "./insert-video-timing.js";
+function clone(value) {
+    return value === undefined ? value : structuredClone(value);
+}
+function childContainer(current, key, nextIsIndex) {
+    const holder = current;
+    const existing = holder[key];
+    if (existing !== null && typeof existing === "object")
+        return existing;
+    const created = nextIsIndex ? [] : {};
+    holder[key] = created;
+    return created;
+}
+function setPath(target, path, value) {
+    const segments = path.split(".");
+    let current = target;
+    segments.slice(0, -1).forEach((segment, index) => {
+        current = childContainer(current, segment, /^\d+$/.test(segments[index + 1]));
+    });
+    current[segments[segments.length - 1]] = value;
+}
+function coerce(option, value) {
+    if (typeof value !== "string")
+        return value;
+    if ((option.valueType === "number" || option.valueType === "integer") && value.trim() !== "" && Number.isFinite(Number(value))) {
+        return Number(value);
+    }
+    if (option.valueType === "boolean" && (value === "true" || value === "false"))
+        return value === "true";
+    return value;
+}
+function refCount(refs, role) {
+    return refs[role]?.length ?? 0;
+}
+function soleMode(model) {
+    return model.modes.length === 1 ? model.modes[0] : undefined;
+}
+function checkRefs(model, refs, mode) {
+    const modeSpec = mode ? model.modes.find((candidate) => candidate.id === mode) : soleMode(model);
+    if (mode && !modeSpec)
+        return `${model.id} has no mode ${mode}; modes: ${model.modes.map((m) => m.id).join(", ")}`;
+    for (const role of Object.keys(refs)) {
+        const count = refCount(refs, role);
+        if (count === 0)
+            continue;
+        const slot = model.slots.find((candidate) => candidate.role === role);
+        if (!slot)
+            return `${model.id} does not accept ${role} refs`;
+        if (modeSpec && !modeSpec.roles.includes(role))
+            return `${model.id} mode ${modeSpec.id} does not accept ${role} refs`;
+        if (slot.max !== null && count > slot.max)
+            return `${model.id} accepts at most ${slot.max} ${role} ref${slot.max === 1 ? "" : "s"}`;
+    }
+    for (const slot of model.slots) {
+        if (modeSpec && !modeSpec.roles.includes(slot.role))
+            continue;
+        if (refCount(refs, slot.role) < slot.min)
+            return `${model.id} requires at least ${slot.min} ${slot.role} ref${slot.min === 1 ? "" : "s"}`;
+    }
+    const requireAny = modeSpec?.requireAny ?? [];
+    if (requireAny.length > 0 && !requireAny.some((role) => refCount(refs, role) > 0)) {
+        return `${model.id} mode ${modeSpec.id} needs one of: ${requireAny.join(", ")}`;
+    }
+    return null;
+}
+function refValue(field, url, item) {
+    return field.urlObject ? { ...item, url } : url;
+}
+function placeSlot(model, slot, value, urls) {
+    const item = model.entry.refItem?.[slot.role];
+    let cursor = 0;
+    for (const field of slot.fields) {
+        if (cursor >= urls.length)
+            return;
+        if (!field.multi) {
+            setPath(value, field.path, refValue(field, urls[cursor], item));
+            cursor += 1;
+            continue;
+        }
+        const take = field.max === null ? urls.length - cursor : Math.min(field.max, urls.length - cursor);
+        setPath(value, field.path, urls.slice(cursor, cursor + take).map((url) => refValue(field, url, item)));
+        cursor += take;
+    }
+}
+function placeRefs(model, value, refs) {
+    for (const slot of model.slots) {
+        const urls = refs[slot.role] ?? [];
+        if (urls.length > 0)
+            placeSlot(model, slot, value, urls);
+    }
+}
+function applyOptions(model, value, options) {
+    for (const [name, raw] of Object.entries(options)) {
+        if (raw === undefined)
+            continue;
+        const option = model.options.find((candidate) => candidate.name === name);
+        if (!option) {
+            if (model.slots.some((slot) => slot.fields.some((field) => field.path.split(".")[0] === name)))
+                return `options.${name}: pass media through refs`;
+            if (model.promptField?.split(".")[0] === name)
+                return `options.${name}: pass text through prompt`;
+            return `options.${name}: not an option of ${model.id}; options: ${model.options.map((o) => o.name).join(", ")}`;
+        }
+        setPath(value, option.path, coerce(option, raw));
+    }
+    return null;
+}
+function activeMode(model, refs, mode) {
+    if (mode)
+        return model.modes.find((candidate) => candidate.id === mode);
+    const used = Object.keys(refs).filter((role) => refCount(refs, role) > 0);
+    return model.modes.find((candidate) => used.every((role) => candidate.roles.includes(role))
+        && ((candidate.requireAny ?? []).length === 0 || (candidate.requireAny ?? []).some((role) => refCount(refs, role) > 0)));
+}
+function applyModeDefaults(model, value, refs, mode) {
+    for (const [path, option] of Object.entries(activeMode(model, refs, mode)?.defaults ?? {}))
+        setPath(value, path, clone(option));
+}
+function hasFrames(refs) {
+    return refCount(refs, "first_frame") > 0 || refCount(refs, "last_frame") > 0;
+}
+function applyFrameOptions(model, value, refs, mode) {
+    const frameOptions = model.entry.frameOptions;
+    if (!frameOptions || !hasFrames(refs) || mode === "r2v")
+        return;
+    for (const [path, option] of Object.entries(frameOptions))
+        setPath(value, path, option);
+}
+function checkPrompt(model, prompt) {
+    const text = prompt?.trim() ?? "";
+    if (model.promptField === null)
+        return text === "" ? null : `${model.id} takes no prompt`;
+    if (text === "" && model.promptRequired)
+        return `${model.id} requires a prompt`;
+    const max = model.entry.rules?.maxPromptChars;
+    if (max !== undefined && text.length > max)
+        return `${model.id} prompt is ${text.length} characters; the limit is ${max}`;
+    return null;
+}
+function issueLabel(path) {
+    return path
+        .filter((segment) => segment !== "input")
+        .map(String)
+        .join(" ")
+        .replaceAll("_", " ")
+        .replace(/^./u, (character) => character.toUpperCase());
+}
+function issueMessage(issue) {
+    const input = "input" in issue ? issue.input : undefined;
+    const missing = issue.code === "invalid_type" && (input === undefined || input === null);
+    return missing ? "Required" : issue.message;
+}
+function formatIssues(model, error) {
+    const issues = error.issues.map((issue) => {
+        const label = issueLabel(issue.path);
+        return label ? `${label}: ${issueMessage(issue)}` : `${model.title}: ${issueMessage(issue)}`;
+    });
+    return issues.join("; ");
+}
+export function buildWireRequest(model, call) {
+    const refs = call.refUrls ?? {};
+    const refError = checkRefs(model, refs, call.mode);
+    if (refError)
+        return { ok: false, error: refError };
+    const promptError = checkPrompt(model, call.prompt);
+    if (promptError)
+        return { ok: false, error: promptError };
+    const value = clone(model.entry.defaults) ?? {};
+    applyModeDefaults(model, value, refs, call.mode);
+    placeRefs(model, value, refs);
+    const optionError = applyOptions(model, value, call.options ?? {});
+    if (optionError)
+        return { ok: false, error: optionError };
+    applyFrameOptions(model, value, refs, call.mode);
+    const prompt = call.prompt?.trim();
+    if (model.promptField && prompt)
+        setPath(value, model.promptField, prompt);
+    const input = model.wire.input.safeParse(value);
+    if (!input.success)
+        return { ok: false, error: formatIssues(model, input.error) };
+    const timingError = isInsertVideoModel(model.id) ? insertVideoTimingError(input.data) : null;
+    if (timingError)
+        return { ok: false, error: timingError };
+    const parsed = model.wire.request.safeParse(model.wire.wrap(input.data));
+    if (!parsed.success)
+        return { ok: false, error: formatIssues(model, parsed.error) };
+    return { ok: true, request: parsed.data, input: input.data };
+}

package/dist/media/types.js

@@ -0,0 +1,6 @@
+export const OPERATIONS_BY_MEDIA = {
+    image: ["text_to_image", "edit_image", "layer_image"],
+    video: ["text_to_video", "image_to_video", "reference_to_video", "edit_video", "extend_video", "upscale_video", "lip_sync", "motion_control"],
+    audio: ["speech", "dialogue", "music", "sfx", "video_to_music"],
+};
+export const REF_ROLES = ["first_frame", "last_frame", "image", "video", "audio"];

package/dist/media/wire.js

@@ -0,0 +1,161 @@
+import { z } from "zod";
+import { MediaGenerationRequestSchema, SunoGenerateRequestSchema, SunoSoundsRequestSchema, VeoGenerateRequestSchema } from "@apicity/kie/zod";
+import { FAL_ENDPOINT_REQUEST_SCHEMAS } from "@apicity/fal/zod";
+import { XaiVideoGenerateRequestSchema } from "@apicity/xai/zod";
+import { AlibabaMultimodalGenerationParametersSchema, AlibabaQwenImageEditStableModelSchema, AlibabaQwenImageGenerationStableModelSchema, } from "@apicity/alibaba/zod";
+import { ElevenLabsComposeMusicRequestSchema, ElevenLabsSoundGenerationRequestSchema, ElevenLabsTextToSpeechRequestSchema, ElevenLabsVideoToMusicRequestSchema, } from "@apicity/elevenlabs/zod";
+const KIE_VEO_MODELS = new Set(["veo3", "veo3_fast", "veo3_lite"]);
+const SUNO_CALLBACK_URL = "https://localhost/videobook/suno-callback";
+function objectShape(schema) {
+    const shape = schema.shape;
+    return shape && typeof shape === "object" ? shape : null;
+}
+function literalValue(schema) {
+    const value = schema.value;
+    return typeof value === "string" ? value : null;
+}
+function identity(input) {
+    return input;
+}
+function kieCreateTaskSchemas() {
+    const map = new Map();
+    for (const option of MediaGenerationRequestSchema.options) {
+        const shape = objectShape(option);
+        const model = shape ? literalValue(shape.model) : null;
+        if (model)
+            map.set(model, option);
+    }
+    return map;
+}
+let kieSchemas = null;
+function kieCreateTaskWire(apiId) {
+    kieSchemas ??= kieCreateTaskSchemas();
+    const request = kieSchemas.get(apiId);
+    if (!request)
+        return null;
+    const input = request.shape.input;
+    return {
+        input,
+        fixed: {},
+        wrap: (value) => ({ model: apiId, input: value }),
+        request,
+    };
+}
+function omitShape(schema, keys) {
+    const shape = Object.fromEntries(Object.entries(schema.shape).filter(([key]) => !keys.includes(key)));
+    return z.object(shape);
+}
+function kieVeoWire(apiId) {
+    if (!KIE_VEO_MODELS.has(apiId))
+        return null;
+    const request = VeoGenerateRequestSchema;
+    return {
+        input: omitShape(request, ["model", "callBackUrl"]),
+        fixed: { model: apiId },
+        wrap: (value) => ({ ...value, model: apiId }),
+        request,
+    };
+}
+function kieSunoWire(apiId) {
+    const request = apiId === "suno/generate"
+        ? SunoGenerateRequestSchema
+        : apiId === "suno/sounds-generate"
+            ? SunoSoundsRequestSchema
+            : null;
+    if (!request)
+        return null;
+    return {
+        input: omitShape(request, ["callBackUrl"]),
+        fixed: { callBackUrl: SUNO_CALLBACK_URL },
+        wrap: (value) => ({ ...value, callBackUrl: SUNO_CALLBACK_URL }),
+        request,
+    };
+}
+export function isKieVeoModel(apiId) {
+    return KIE_VEO_MODELS.has(apiId);
+}
+export function isKieSunoModel(apiId) {
+    return apiId === "suno/generate" || apiId === "suno/sounds-generate";
+}
+function falWire(apiId) {
+    const schemas = FAL_ENDPOINT_REQUEST_SCHEMAS;
+    const request = schemas[apiId];
+    if (!request || !objectShape(request))
+        return null;
+    return { input: request, fixed: {}, wrap: identity, request };
+}
+function xaiWire(apiId) {
+    if (apiId !== "grok-imagine-video" && !apiId.startsWith("grok-imagine-video-"))
+        return null;
+    const request = XaiVideoGenerateRequestSchema;
+    return {
+        input: omitShape(request, ["model", "storage_options"]),
+        fixed: { model: apiId },
+        wrap: (value) => ({ ...value, model: apiId }),
+        request,
+    };
+}
+const ALIBABA_MODELS = new Set([
+    ...Object.values(AlibabaQwenImageGenerationStableModelSchema._def.entries),
+    ...Object.values(AlibabaQwenImageEditStableModelSchema._def.entries),
+]);
+const ALIBABA_EDIT_MODELS = new Set(Object.values(AlibabaQwenImageEditStableModelSchema._def.entries));
+function alibabaInput(apiId) {
+    const parameters = AlibabaMultimodalGenerationParametersSchema;
+    const base = z.object({ prompt: z.string().min(1), ...parameters.shape });
+    return ALIBABA_EDIT_MODELS.has(apiId) ? base.extend({ images: z.array(z.string()).min(1).max(3) }) : base;
+}
+function alibabaRequest(apiId, value) {
+    const { prompt, images, ...parameters } = value;
+    const content = [];
+    for (const image of Array.isArray(images) ? images : [])
+        content.push({ image: String(image) });
+    content.push({ text: String(prompt ?? "") });
+    return { model: apiId, input: { messages: [{ role: "user", content }] }, parameters };
+}
+function alibabaWire(apiId) {
+    if (!ALIBABA_MODELS.has(apiId))
+        return null;
+    const input = alibabaInput(apiId);
+    return {
+        input,
+        fixed: {},
+        wrap: (value) => alibabaRequest(apiId, value),
+        request: z.record(z.string(), z.unknown()),
+    };
+}
+function elevenLabsWire(apiId) {
+    if (apiId === "sound-generation") {
+        const request = ElevenLabsSoundGenerationRequestSchema;
+        return { input: request, fixed: {}, wrap: identity, request };
+    }
+    if (apiId === "music") {
+        const request = ElevenLabsComposeMusicRequestSchema;
+        return { input: request, fixed: {}, wrap: identity, request };
+    }
+    if (apiId === "video-to-music") {
+        const base = ElevenLabsVideoToMusicRequestSchema;
+        const input = omitShape(base, ["videos"]).extend({ videos: z.array(z.string()).min(1).max(1) });
+        return { input, fixed: {}, wrap: identity, request: input };
+    }
+    if (apiId === "text-to-speech") {
+        const base = ElevenLabsTextToSpeechRequestSchema;
+        const input = base.extend({ voice_id: z.string().min(1).describe("ElevenLabs voice id") });
+        return { input, fixed: {}, wrap: identity, request: input };
+    }
+    return null;
+}
+export function wireSpec(parsed) {
+    switch (parsed.provider) {
+        case "kie":
+            return kieCreateTaskWire(parsed.apiId) ?? kieVeoWire(parsed.apiId) ?? kieSunoWire(parsed.apiId);
+        case "fal":
+            return falWire(parsed.apiId);
+        case "xai":
+            return xaiWire(parsed.apiId);
+        case "alibaba":
+            return alibabaWire(parsed.apiId);
+        case "elevenlabs":
+            return elevenLabsWire(parsed.apiId);
+    }
+}