visualfries 0.1.10120 → 0.3.2

package/dist/agent/audioMixer.js

@@ -0,0 +1,308 @@
+import { spawn } from 'node:child_process';
+import crypto from 'node:crypto';
+import fs from 'node:fs/promises';
+import { existsSync } from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { SceneShape } from '../schemas/scene/index.js';
+const REMOTE_AUDIO_INPUT_OPTIONS = [
+    '-reconnect',
+    '1',
+    '-reconnect_streamed',
+    '1',
+    '-reconnect_on_network_error',
+    '1',
+    '-http_follow_location',
+    '1',
+    '-multiple_requests',
+    '1',
+    '-rw_timeout',
+    '15000000'
+];
+const isRemoteUrl = (value) => /^https?:\/\//i.test(value);
+const toFfmpegInput = (value) => value.startsWith('file://') ? fileURLToPath(value) : value;
+const safeVolume = (input) => {
+    if (typeof input !== 'number' || Number.isNaN(input))
+        return 1;
+    return Math.max(0, Math.min(1, input));
+};
+const sourceUrlForComponent = (scene, component) => {
+    if (!('source' in component))
+        return undefined;
+    const directUrl = component.source?.url ?? component.source?.streamUrl;
+    if (directUrl)
+        return directUrl;
+    const assetId = component.source?.assetId;
+    return assetId ? scene.assets.find((asset) => asset.id === assetId)?.url : undefined;
+};
+const clampRange = (range, boundary) => {
+    const start = Math.max(range.start, boundary.start);
+    const end = Math.min(range.end, boundary.end);
+    return end > start ? { start, end } : null;
+};
+export function resolveAudioMixRanges(sceneInput, chunkStartSec, chunkEndSec) {
+    const scene = SceneShape.parse(sceneInput);
+    const fps = Math.max(1, Number(scene.settings.fps || 30));
+    const oneFrameSec = 1 / fps;
+    const sceneDuration = Math.max(0, Number(scene.settings.duration || 0));
+    const boundary = {
+        start: Math.max(0, Math.min(chunkStartSec, sceneDuration)),
+        end: Math.max(0, Math.min(chunkEndSec, sceneDuration))
+    };
+    if (boundary.end <= boundary.start)
+        return [];
+    let ranges = [boundary];
+    if (typeof scene.settings.startAt === 'number' && scene.settings.startAt > 0) {
+        const startAt = Math.max(0, scene.settings.startAt - oneFrameSec);
+        ranges = ranges.flatMap((range) => range.end <= startAt ? [] : [{ start: Math.max(range.start, startAt), end: range.end }]);
+    }
+    const sceneEndAt = scene.settings.endAt;
+    if (typeof sceneEndAt === 'number') {
+        ranges = ranges.flatMap((range) => range.start >= sceneEndAt
+            ? []
+            : [{ start: range.start, end: Math.min(range.end, sceneEndAt) }]);
+    }
+    // Keep this local-first module conservative: if trim zones exist, split around them.
+    for (const trim of scene.settings.trimZones ?? []) {
+        ranges = ranges.flatMap((range) => {
+            if (trim.end <= range.start || trim.start >= range.end)
+                return [range];
+            const parts = [];
+            if (trim.start > range.start)
+                parts.push({ start: range.start, end: trim.start });
+            if (trim.end < range.end)
+                parts.push({ start: trim.end, end: range.end });
+            return parts;
+        });
+    }
+    return ranges.filter((range) => range.end > range.start);
+}
+function collectAudioSourcesForBoundary(scene, boundary, outputOffsetSec) {
+    const plans = [];
+    for (const layer of scene.layers ?? []) {
+        if (layer.muted)
+            continue;
+        for (const component of layer.components ?? []) {
+            if (component.type !== 'VIDEO' && component.type !== 'AUDIO')
+                continue;
+            const muted = 'muted' in component ? component.muted === true : false;
+            const volume = safeVolume('volume' in component ? component.volume : 1);
+            if (muted || volume <= 0)
+                continue;
+            const url = sourceUrlForComponent(scene, component);
+            if (!url)
+                continue;
+            const timelineStart = component.timeline?.startAt ?? 0;
+            const timelineEnd = component.timeline?.endAt ?? scene.settings.duration;
+            const active = clampRange({ start: timelineStart, end: timelineEnd }, boundary);
+            if (!active)
+                continue;
+            const source = 'source' in component ? component.source : undefined;
+            const sourceStart = source?.startAt ?? 0;
+            const sourceEnd = source?.endAt;
+            const sourceOffsetSec = sourceStart + (active.start - timelineStart);
+            let durationSec = active.end - active.start;
+            if (typeof sourceEnd === 'number') {
+                durationSec = Math.min(durationSec, sourceEnd - sourceOffsetSec);
+            }
+            if (durationSec <= 0)
+                continue;
+            plans.push({
+                id: component.id || `component-${plans.length}`,
+                kind: 'component',
+                url,
+                startInChunkSec: outputOffsetSec + (active.start - boundary.start),
+                durationSec,
+                sourceOffsetSec,
+                volume
+            });
+        }
+    }
+    for (const track of scene.audioTracks ?? []) {
+        if (!track || track.muted)
+            continue;
+        const volume = safeVolume(track.volume);
+        if (volume <= 0 || !track.url)
+            continue;
+        const startAt = track.startAt ?? 0;
+        const endAt = typeof track.endAt === 'number' ? track.endAt : scene.settings.duration;
+        const active = clampRange({ start: startAt, end: endAt }, boundary);
+        if (!active)
+            continue;
+        plans.push({
+            id: track.id || `track-${plans.length}`,
+            kind: 'track',
+            url: track.url,
+            startInChunkSec: outputOffsetSec + (active.start - boundary.start),
+            durationSec: active.end - active.start,
+            sourceOffsetSec: active.start - startAt,
+            volume
+        });
+    }
+    const sceneAudio = scene.settings.audio;
+    if (sceneAudio?.src && sceneAudio.muted !== true && safeVolume(sceneAudio.volume) > 0) {
+        plans.push({
+            id: 'scene-audio',
+            kind: 'scene',
+            url: sceneAudio.src,
+            startInChunkSec: outputOffsetSec,
+            durationSec: boundary.end - boundary.start,
+            sourceOffsetSec: boundary.start,
+            volume: safeVolume(sceneAudio.volume)
+        });
+    }
+    return plans;
+}
+export function collectAudioSourcesForRanges(sceneInput, ranges) {
+    const scene = SceneShape.parse(sceneInput);
+    const plans = [];
+    let outputOffsetSec = 0;
+    for (const range of ranges) {
+        if (range.end <= range.start)
+            continue;
+        plans.push(...collectAudioSourcesForBoundary(scene, range, outputOffsetSec));
+        outputOffsetSec += range.end - range.start;
+    }
+    return plans;
+}
+const runProcess = async (command, args, label, options = {}) => {
+    return await new Promise((resolve, reject) => {
+        const child = spawn(command, args, { stdio: ['ignore', 'pipe', 'pipe'] });
+        let stdout = '';
+        let stderr = '';
+        child.stdout?.on('data', (chunk) => {
+            stdout += String(chunk);
+        });
+        child.stderr?.on('data', (chunk) => {
+            const text = String(chunk);
+            if (options.debug)
+                process.stderr.write(`[${label}] ${text}`);
+            stderr = (stderr + text).slice(-4000);
+        });
+        child.on('error', reject);
+        child.on('close', (code) => {
+            if (code === 0) {
+                resolve({ stdout });
+            }
+            else {
+                reject(new Error(`${label} failed with exit code ${code}. ${stderr}`));
+            }
+        });
+    });
+};
+export async function sourceHasAudio(url) {
+    const input = toFfmpegInput(url);
+    const { stdout } = await runProcess(process.env.FFPROBE_PATH || 'ffprobe', [
+        '-v',
+        'error',
+        '-select_streams',
+        'a',
+        '-show_entries',
+        'stream=index',
+        '-of',
+        'csv=p=0',
+        input
+    ], 'ffprobe-audio');
+    return stdout.trim().length > 0;
+}
+export function createPrepareSourceAudioArgs(source, outputPath) {
+    const input = toFfmpegInput(source.url);
+    return [
+        '-y',
+        ...(isRemoteUrl(source.url) ? REMOTE_AUDIO_INPUT_OPTIONS : []),
+        '-ss',
+        source.sourceOffsetSec.toFixed(3),
+        '-i',
+        input,
+        '-t',
+        source.durationSec.toFixed(3),
+        '-vn',
+        '-ac',
+        '2',
+        '-ar',
+        '48000',
+        '-acodec',
+        'flac',
+        outputPath
+    ];
+}
+export function createMixAudioArgs(inputs, outputPath, chunkDurationSec) {
+    const args = ['-y'];
+    for (const input of inputs) {
+        args.push('-i', input.path);
+    }
+    const filterSegments = [];
+    const mixInputs = [];
+    for (let index = 0; index < inputs.length; index += 1) {
+        const source = inputs[index].plan;
+        const delayMs = Math.max(0, Math.round(source.startInChunkSec * 1000));
+        const label = `a${index}`;
+        filterSegments.push(`[${index}:a]adelay=${delayMs}|${delayMs},volume=${source.volume.toFixed(6)}[${label}]`);
+        mixInputs.push(`[${label}]`);
+    }
+    filterSegments.push(`${mixInputs.join('')}amix=inputs=${mixInputs.length}:duration=longest:normalize=false[mixraw]`);
+    filterSegments.push(`[mixraw]atrim=0:${chunkDurationSec.toFixed(3)},asetpts=PTS-STARTPTS[mixtrim]`);
+    args.push('-filter_complex', filterSegments.join(';'), '-map', '[mixtrim]', '-ac', '2', '-ar', '48000', '-acodec', 'flac', outputPath);
+    return args;
+}
+export async function buildLocalMixedAudioTrack(input) {
+    const scene = SceneShape.parse(input.scene);
+    const chunkDurationSec = input.ranges.reduce((sum, range) => sum + Math.max(0, range.end - range.start), 0);
+    if (chunkDurationSec <= 0) {
+        return { audioPath: null, selectedSources: 0, skippedSources: 0, plannedSources: [] };
+    }
+    const plannedSources = collectAudioSourcesForRanges(scene, input.ranges);
+    if (plannedSources.length === 0) {
+        return { audioPath: null, selectedSources: 0, skippedSources: 0, plannedSources };
+    }
+    await fs.mkdir(input.workDir, { recursive: true });
+    const preparedInputs = [];
+    let skippedSources = 0;
+    for (let index = 0; index < plannedSources.length; index += 1) {
+        const source = plannedSources[index];
+        if (!(await sourceHasAudio(source.url))) {
+            skippedSources += 1;
+            continue;
+        }
+        const hash = crypto
+            .createHash('sha1')
+            .update(JSON.stringify({
+            url: source.url,
+            offset: source.sourceOffsetSec.toFixed(3),
+            duration: source.durationSec.toFixed(3),
+            volume: source.volume.toFixed(3)
+        }))
+            .digest('hex');
+        const preparedPath = path.join(input.workDir, `prepared-${index}-${hash}.flac`);
+        if (!existsSync(preparedPath)) {
+            await runProcess(process.env.FFMPEG_PATH || 'ffmpeg', createPrepareSourceAudioArgs(source, preparedPath), 'visualfries-audio-prepare', { debug: input.debug });
+        }
+        preparedInputs.push({ plan: source, path: preparedPath });
+    }
+    if (preparedInputs.length === 0) {
+        return { audioPath: null, selectedSources: 0, skippedSources, plannedSources };
+    }
+    const checksum = crypto
+        .createHash('sha1')
+        .update(JSON.stringify({
+        renderId: input.renderId ?? 'local',
+        chunkDurationSec: chunkDurationSec.toFixed(3),
+        inputs: preparedInputs.map((entry) => ({
+            id: entry.plan.id,
+            path: entry.path,
+            delay: entry.plan.startInChunkSec.toFixed(3),
+            volume: entry.plan.volume.toFixed(3)
+        }))
+    }))
+        .digest('hex');
+    const mixedPath = path.join(input.workDir, `mixed-${checksum}.flac`);
+    if (!existsSync(mixedPath)) {
+        await runProcess(process.env.FFMPEG_PATH || 'ffmpeg', createMixAudioArgs(preparedInputs, mixedPath, chunkDurationSec), 'visualfries-audio-mix', { debug: input.debug });
+    }
+    return {
+        audioPath: mixedPath,
+        selectedSources: preparedInputs.length,
+        skippedSources,
+        plannedSources
+    };
+}

package/dist/agent/broll.d.ts

@@ -0,0 +1,35 @@
+import { type ComponentInput, type Scene, type SceneAssetInput, type SceneInput } from '../schemas/scene/index.js';
+export type AgentBrollCue = {
+    id?: string;
+    url: string;
+    type?: 'VIDEO' | 'IMAGE' | 'GIF';
+    motion?: 'none' | 'slow-zoom-in' | 'slow-zoom-out' | 'drift-up';
+    start: number;
+    end: number;
+    assetId?: string;
+    name?: string;
+    order?: number;
+    muted?: boolean;
+    volume?: number;
+    fade?: number;
+    x?: number;
+    y?: number;
+    width?: number;
+    height?: number;
+    sourceStart?: number;
+    sourceEnd?: number;
+    playbackRate?: number;
+    loop?: boolean;
+};
+export type AddAgentBrollSequenceInput = {
+    scene: Scene | SceneInput;
+    cues: AgentBrollCue[];
+    layerId?: string;
+    layerName?: string;
+    layerOrder?: number;
+};
+export declare function createAgentBrollComponent(cue: AgentBrollCue, scene: Scene | SceneInput, index?: number): {
+    asset: SceneAssetInput;
+    component: ComponentInput;
+};
+export declare function addAgentBrollSequence(input: AddAgentBrollSequenceInput): Scene;

package/dist/agent/broll.js

@@ -0,0 +1,226 @@
+import path from 'node:path';
+import { pathToFileURL } from 'node:url';
+import { ComponentShape, SceneShape } from '../schemas/scene/index.js';
+function inferAssetType(cue) {
+    if (cue.type)
+        return cue.type;
+    const cleanUrl = cue.url.split('?')[0]?.toLowerCase() ?? cue.url.toLowerCase();
+    if (cleanUrl.endsWith('.gif'))
+        return 'GIF';
+    if (/\.(png|jpe?g|webp|avif)$/i.test(cleanUrl))
+        return 'IMAGE';
+    return 'VIDEO';
+}
+function normalizeAgentMediaUrl(url) {
+    if (/^[a-z]+:\/\//i.test(url) || url.startsWith('data:'))
+        return url;
+    return pathToFileURL(path.resolve(url)).toString();
+}
+function fadeAnimation(id, cue) {
+    const fade = cue.fade ?? 0.2;
+    if (fade <= 0)
+        return [];
+    return [
+        {
+            id: `${id}-fade`,
+            name: 'Agent b-roll fade',
+            animation: {
+                id: `${id}-fade-preset`,
+                timeline: [
+                    {
+                        tweens: [
+                            {
+                                method: 'from',
+                                vars: {
+                                    duration: fade,
+                                    opacity: 0,
+                                    ease: 'power1.out'
+                                }
+                            }
+                        ]
+                    },
+                    {
+                        position: {
+                            anchor: 'componentEnd',
+                            alignTween: 'end',
+                            offset: '0s'
+                        },
+                        tweens: [
+                            {
+                                method: 'to',
+                                vars: {
+                                    duration: fade,
+                                    opacity: 0,
+                                    ease: 'power1.in'
+                                }
+                            }
+                        ]
+                    }
+                ]
+            }
+        }
+    ];
+}
+function motionAnimation(id, cue) {
+    const motion = cue.motion ?? (inferAssetType(cue) === 'IMAGE' ? 'slow-zoom-in' : 'none');
+    const duration = Math.max(0.1, cue.end - cue.start);
+    if (motion === 'none')
+        return [];
+    if (motion === 'slow-zoom-out') {
+        return [
+            {
+                id: `${id}-motion`,
+                name: 'Agent b-roll slow zoom out',
+                animation: {
+                    id: `${id}-motion-preset`,
+                    timeline: [
+                        {
+                            tweens: [
+                                {
+                                    method: 'fromTo',
+                                    vars: {
+                                        from: { scale: 1.08 },
+                                        duration,
+                                        scale: 1,
+                                        ease: 'none'
+                                    }
+                                }
+                            ]
+                        }
+                    ]
+                }
+            }
+        ];
+    }
+    if (motion === 'drift-up') {
+        return [
+            {
+                id: `${id}-motion`,
+                name: 'Agent b-roll drift up',
+                animation: {
+                    id: `${id}-motion-preset`,
+                    timeline: [
+                        {
+                            tweens: [
+                                {
+                                    method: 'fromTo',
+                                    vars: {
+                                        from: { y: cue.y ?? 0 },
+                                        duration,
+                                        y: (cue.y ?? 0) - 36,
+                                        ease: 'none'
+                                    }
+                                }
+                            ]
+                        }
+                    ]
+                }
+            }
+        ];
+    }
+    return [
+        {
+            id: `${id}-motion`,
+            name: 'Agent b-roll slow zoom in',
+            animation: {
+                id: `${id}-motion-preset`,
+                timeline: [
+                    {
+                        tweens: [
+                            {
+                                method: 'fromTo',
+                                vars: {
+                                    from: { scale: 1 },
+                                    duration,
+                                    scale: 1.08,
+                                    ease: 'none'
+                                }
+                            }
+                        ]
+                    }
+                ]
+            }
+        }
+    ];
+}
+export function createAgentBrollComponent(cue, scene, index = 0) {
+    const parsedScene = SceneShape.parse(scene);
+    const type = inferAssetType(cue);
+    const id = cue.id ?? `agent-broll-${index + 1}`;
+    const assetId = cue.assetId ?? `${id}-asset`;
+    const url = normalizeAgentMediaUrl(cue.url);
+    const base = {
+        id,
+        name: cue.name ?? `Agent B-roll ${index + 1}`,
+        type,
+        timeline: { startAt: cue.start, endAt: cue.end },
+        source: {
+            url,
+            assetId,
+            startAt: cue.sourceStart,
+            endAt: cue.sourceEnd
+        },
+        order: cue.order ?? index,
+        appearance: {
+            x: cue.x ?? 0,
+            y: cue.y ?? 0,
+            width: cue.width ?? parsedScene.settings.width,
+            height: cue.height ?? parsedScene.settings.height,
+            opacity: 1
+        },
+        animations: {
+            enabled: true,
+            list: [...fadeAnimation(id, cue), ...motionAnimation(id, cue)]
+        }
+    };
+    const component = type === 'VIDEO'
+        ? {
+            ...base,
+            type: 'VIDEO',
+            volume: cue.volume ?? 0,
+            muted: cue.muted ?? true,
+            playback: {
+                autoplay: true,
+                loop: cue.loop ?? false,
+                playbackRate: cue.playbackRate ?? 1,
+                startAt: cue.sourceStart ?? 0,
+                endAt: cue.sourceEnd
+            },
+            crop: { x: 0, y: 0, width: 1, height: 1 }
+        }
+        : type === 'GIF'
+            ? {
+                ...base,
+                type: 'GIF',
+                playback: { loop: true, speed: 1 }
+            }
+            : {
+                ...base,
+                type: 'IMAGE',
+                crop: { xPercent: 0, yPercent: 0, widthPercent: 1, heightPercent: 1 }
+            };
+    const asset = {
+        id: assetId,
+        type,
+        url
+    };
+    return {
+        asset,
+        component: ComponentShape.parse(component)
+    };
+}
+export function addAgentBrollSequence(input) {
+    const scene = SceneShape.parse(input.scene);
+    const created = input.cues.map((cue, index) => createAgentBrollComponent(cue, scene, index));
+    const layer = {
+        id: input.layerId ?? 'layer-agent-broll',
+        name: input.layerName ?? 'Agent B-roll',
+        order: input.layerOrder ?? 5,
+        components: created.map((item) => item.component)
+    };
+    return SceneShape.parse({
+        ...scene,
+        assets: [...scene.assets, ...created.map((item) => item.asset)],
+        layers: [...scene.layers, layer]
+    });
+}

package/dist/agent/capabilities.d.ts

@@ -0,0 +1,34 @@
+export declare const VISUAL_COMPONENT_TYPES: readonly ["TEXT", "SUBTITLES", "IMAGE", "VIDEO", "GIF", "SHAPE", "COLOR", "GRADIENT"];
+export declare const COMPONENT_TYPES: readonly ["TEXT", "SUBTITLES", "IMAGE", "VIDEO", "GIF", "SHAPE", "COLOR", "GRADIENT", "AUDIO"];
+export type CapabilityComponentType = (typeof COMPONENT_TYPES)[number];
+export type VisualCapabilityComponentType = (typeof VISUAL_COMPONENT_TYPES)[number];
+export type AnimationTransform = 'x' | 'y' | 'opacity' | 'rotation' | 'scale' | 'scaleX' | 'scaleY';
+export type ComponentCapability = {
+    type: CapabilityComponentType;
+    visual: boolean;
+    renderers: {
+        preview: 'html' | 'pixi' | 'native-media' | 'none';
+        final: 'pixi' | 'deterministic-media-pixi' | 'audio-mix' | 'none';
+        parity: 'same-representation' | 'different-representation' | 'media-predecoded' | 'not-applicable';
+    };
+    animation: {
+        attached: boolean;
+        target: 'html-element-or-wrapper' | 'pixi-container' | 'none';
+        transforms: AnimationTransform[];
+        selectors: Array<'container' | 'words' | 'lines' | 'chars' | 'css'>;
+        coordinateSpace: 'css-transform' | 'component-relative-offset' | 'none';
+        transformOrigin: 'css-center-fixed' | 'component-center-fixed' | 'none';
+        seekDeterministic: boolean;
+    };
+    effects: {
+        supported: string[];
+        validationOnly: string[];
+    };
+    backgrounds: string[];
+    textFeatures: string[];
+    caveats: string[];
+};
+export declare const COMPONENT_CAPABILITIES: Record<CapabilityComponentType, ComponentCapability>;
+export declare function getComponentCapability(type: string): ComponentCapability | undefined;
+export declare function getCapabilityCatalog(): ComponentCapability[];
+export declare function componentSupportsRuntimeAnimation(type: string): boolean;