tuna-agent 0.1.133 → 0.1.135

package/dist/daemon/analyze-video-handler.d.ts

@@ -11,6 +11,11 @@ export interface AnalyzeVideoResult {
     language: string;
     transcript: string;
     video_style: string;
+    master_cast_prompt: string;
+    characters: Array<{
+        name: string;
+        description: string;
+    }>;
     segments: Array<{
         start: number;
         end: number;

package/dist/daemon/analyze-video-handler.js

@@ -74,28 +74,46 @@ ${rawText}`,
         return rawText;
     }
 }
-async function visionDescribe(frameB64, voiceoverText) {
+// Accepts 1..N frames sampled across a scene (start → mid → end). Multiple
+// frames let the model observe MOTION direction ("walks left-to-right then
+// turns") instead of guessing from a single frozen midpoint. The model is
+// told the frames are chronological so it describes the action arc, not 3
+// separate moments.
+async function visionDescribe(frameB64s, voiceoverText, castContext = '') {
     if (!OPENAI_KEY)
         return '';
+    const frames = frameB64s.filter(Boolean);
+    if (frames.length === 0)
+        return '';
+    const castBlock = castContext
+        ? `\n\nKNOWN CHARACTER CAST (reuse these EXACT names when a subject appears — do NOT invent new labels for the same subject):\n${castContext}\n`
+        : '';
+    const seqNote = frames.length > 1
+        ? ` The ${frames.length} images are CHRONOLOGICAL samples from the SAME scene (start → middle → end). Treat them as one continuous shot: describe the MOTION ARC across them (direction of movement, camera push/pan, what changes from first to last frame), not three separate moments.`
+        : '';
+    const imageParts = frames.map(b64 => ({
+        type: 'image_url',
+        image_url: { url: `data:image/jpeg;base64,${b64}` },
+    }));
     const res = await fetch('https://api.openai.com/v1/chat/completions', {
         method: 'POST',
         headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${OPENAI_KEY}` },
         body: JSON.stringify({
             model: 'gpt-4o-mini',
-            max_tokens: 300,
+            max_tokens: 350,
             messages: [{
                     role: 'user',
                     content: [
-                        { type: 'text', text: `Describe this frame in detail (4-6 sentences, English). Include:
-- Characters: appearance (shape, color, size), facial expression, what they're doing
+                        { type: 'text', text: `Describe this scene in detail (4-6 sentences, English).${seqNote}${castBlock} Include:
+- Characters: name them using the KNOWN CHARACTER CAST labels above when they appear; appearance (shape, color, size), facial expression, what they're doing
 - Physical connections: Are characters physically joined/attached/fused together (e.g. organs connected at a junction, body parts linked)? Or are they separate/independent? Be VERY specific — "physically attached at Y-junction" is different from "standing next to each other"
 - Spatial positions: exact position of each character (left/right/above/below/center), distance between them
 - Environment: setting, lighting, color palette, atmosphere
-- Camera: angle, framing (close-up, wide, etc.)
-- Action: what is happening in this moment, movement direction
+- Camera: angle, framing (close-up, wide, etc.), and any camera movement across the frames
+- Action: the movement/action arc from first to last frame (direction, what changes)
 
-Voiceover at this moment: "${voiceoverText || '(none)'}"` },
-                        { type: 'image_url', image_url: { url: `data:image/jpeg;base64,${frameB64}` } },
+Voiceover during this scene: "${voiceoverText || '(none)'}"` },
+                        ...imageParts,
                     ],
                 }],
         }),
@@ -105,6 +123,79 @@ Voiceover at this moment: "${voiceoverText || '(none)'}"` },
     const data = await res.json();
     return data.choices?.[0]?.message?.content?.trim() || '';
 }
+// Phase 1 (borrowed from AI_Video_Clone): extract the recurring character
+// cast ONCE from frames sampled across the whole video + the transcript.
+// Returns a master-cast prompt block in the exact [AESTHETIC & STYLE] /
+// [CHARACTER CAST LIST] format that channel-manager's ScriptImporter parses,
+// plus a structured characters[] list. Doing this upfront (a) populates
+// idea.master_cast_prompt so FlowKit has a reference sheet to generate, and
+// (b) gives every per-scene describe call a consistent naming vocabulary so
+// scene 1 and scene 50 refer to "THE BISHOP" instead of "a man in a suit".
+async function visionExtractMasterCast(frames, transcript, videoStyle) {
+    const empty = { master_cast_prompt: '', characters: [] };
+    if (!OPENAI_KEY || frames.length === 0)
+        return empty;
+    try {
+        const content = [
+            {
+                type: 'text',
+                text: `Act as a Master Film Director. These frames are sampled across an entire video. Identify EVERY recurring character/subject (people, anthropomorphic objects, animals, mascots).
+
+Transcript context (may name characters): "${(transcript || '').slice(0, 1500)}"
+
+Return ONLY a JSON object, no markdown fences:
+{
+  "characters": [
+    { "name": "SHORT_UPPERCASE_LABEL", "description": "one-line English visual description: age/build, face, hair, outfit, colors, distinguishing features" }
+  ]
+}
+
+Rules:
+- name: a stable short uppercase label you will reuse for this subject (e.g. "THE BISHOP", "RED CAR", "NARRATOR DOG"). Max 4 words.
+- Only RECURRING subjects worth a reference sheet. Skip one-off background extras.
+- description: ENGLISH only, factual, no camera/action words.
+- Max 6 characters.`,
+            },
+        ];
+        for (const b64 of frames) {
+            content.push({ type: 'image_url', image_url: { url: `data:image/jpeg;base64,${b64}` } });
+        }
+        const res = await fetch('https://api.openai.com/v1/chat/completions', {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${OPENAI_KEY}` },
+            body: JSON.stringify({ model: 'gpt-4o', max_tokens: 1200, messages: [{ role: 'user', content }] }),
+        });
+        if (!res.ok)
+            return empty;
+        const data = await res.json();
+        const raw = (data.choices?.[0]?.message?.content || '').trim();
+        let parsed = {};
+        try {
+            const m = raw.match(/\{[\s\S]*\}/);
+            parsed = JSON.parse(m ? m[0] : raw);
+        }
+        catch {
+            return empty;
+        }
+        const characters = (parsed.characters || [])
+            .filter(c => c && c.name && c.description)
+            .slice(0, 6)
+            .map(c => ({ name: String(c.name).trim(), description: String(c.description).trim() }));
+        if (characters.length === 0)
+            return empty;
+        // Assemble the verbatim-style master cast block ScriptImporter expects.
+        const styleLine = (videoStyle || '').trim() || 'Keep the original video’s visual style, color grading, and lighting.';
+        const castList = characters.map(c => `- ${c.name}: ${c.description}`).join('\n');
+        const master_cast_prompt = `[AESTHETIC & STYLE]\n${styleLine}\n` +
+            `[COMPOSITION & LAYOUT]\nCharacter Reference Sheet. Full-body side-by-side.\n` +
+            `[CHARACTER CAST LIST]\n${castList}\n` +
+            `[TECHNICAL SPECIFICATIONS]\nHigh detail, 8k resolution, consistent facial structures across all frames.`;
+        return { master_cast_prompt, characters };
+    }
+    catch {
+        return empty;
+    }
+}
 async function visionExtractStyle(frames) {
     if (!OPENAI_KEY || frames.length === 0)
         return '';
@@ -164,71 +255,136 @@ export async function analyzeVideo(url, onProgress) {
         const segments = transcript.segments || [];
         const sceneSlots = [];
         const SILENCE_THRESHOLD = 5; // seconds — gaps longer than this become their own scene
-        const MAX_SCENES = 30;
+        // Veo3 clips are 8s. The KPI (borrowed from AI_Video_Clone) is: never
+        // under-segment — a 40-min video must yield ~ceil(duration/8) scenes, not
+        // a fixed 30. Long transcript segments are SPLIT into 8s sub-slots so a
+        // 90s monologue becomes ~11 scenes instead of one giant clip. A hard
+        // ceiling still bounds runaway vision cost on very long videos.
+        const TARGET_SCENE_SEC = 8;
+        const HARD_CAP = 600; // ~80 min @ 8s — safety bound on vision API spend
+        const targetScenes = Math.max(1, Math.ceil(durationSec / TARGET_SCENE_SEC));
+        const MAX_SCENES = Math.min(targetScenes + 20, HARD_CAP);
+        // Split a [start,end] span into ≤TARGET_SCENE_SEC sub-slots, preserving
+        // the voiceover on the FIRST sub-slot (the rest are silent continuations
+        // of the same spoken line so lip-sync isn't duplicated downstream).
+        const pushSplit = (start, end, voiceover) => {
+            const span = end - start;
+            if (span <= TARGET_SCENE_SEC * 1.5) {
+                sceneSlots.push({ start, end, voiceover });
+                return;
+            }
+            const n = Math.ceil(span / TARGET_SCENE_SEC);
+            const step = span / n;
+            for (let k = 0; k < n; k++) {
+                sceneSlots.push({
+                    start: start + k * step,
+                    end: k === n - 1 ? end : start + (k + 1) * step,
+                    voiceover: k === 0 ? voiceover : '',
+                });
+            }
+        };
         if (segments.length > 0) {
-            // Add silence scene before first segment if gap > threshold
             if (segments[0].start > SILENCE_THRESHOLD) {
-                sceneSlots.push({ start: 0, end: segments[0].start, voiceover: '' });
+                pushSplit(0, segments[0].start, '');
             }
             for (let i = 0; i < segments.length; i++) {
                 const seg = segments[i];
-                sceneSlots.push({ start: seg.start, end: seg.end, voiceover: seg.text?.trim() || '' });
-                // Add silence scene between segments if gap > threshold
+                pushSplit(seg.start, seg.end, seg.text?.trim() || '');
                 if (i < segments.length - 1) {
                     const gap = segments[i + 1].start - seg.end;
                     if (gap > SILENCE_THRESHOLD) {
-                        sceneSlots.push({ start: seg.end, end: segments[i + 1].start, voiceover: '' });
+                        pushSplit(seg.end, segments[i + 1].start, '');
                     }
                 }
             }
-            // Add silence scene after last segment if gap > threshold
             const lastEnd = segments[segments.length - 1].end;
             if (durationSec - lastEnd > SILENCE_THRESHOLD) {
-                sceneSlots.push({ start: lastEnd, end: durationSec, voiceover: '' });
+                pushSplit(lastEnd, durationSec, '');
             }
         }
         else {
             // No transcript — split into scenes every 8s (Veo3 clip length)
-            const interval = 8;
-            for (let t = 0; t < durationSec; t += interval) {
-                sceneSlots.push({ start: t, end: Math.min(t + interval, durationSec), voiceover: '' });
+            for (let t = 0; t < durationSec; t += TARGET_SCENE_SEC) {
+                sceneSlots.push({ start: t, end: Math.min(t + TARGET_SCENE_SEC, durationSec), voiceover: '' });
             }
         }
-        // Cap max scenes
+        // Duration-aware cap (was a flat 30 — that silently truncated any video
+        // longer than ~4 min). Re-number after slicing.
         const finalSlots = sceneSlots.slice(0, MAX_SCENES);
         progress(`Đang cắt ${finalSlots.length} frames và phân tích...`);
         console.log('[analyze_video] Building', finalSlots.length, 'scenes (segments:', segments.length, ', duration:', durationSec, 's)');
-        // Step 1: Extract all frames sequentially (ffmpeg can't run in parallel on same file efficiently)
+        // Step 1: Extract frames sequentially. Per scene we grab 3 chronological
+        // frames — start → middle → end — so the vision model can read the motion
+        // arc (direction of movement, camera push) instead of guessing from a
+        // single frozen midpoint. The MIDDLE frame doubles as the UI thumbnail.
+        // Tiny scenes (<1.5s) collapse to just the midpoint (the 3 frames would
+        // be near-identical — no motion info, wasted tokens). Start/end are
+        // nudged ~15% inward to dodge hard-cut / black transition frames.
         const frameBuffers = [];
         for (let i = 0; i < finalSlots.length; i++) {
             const slot = finalSlots[i];
-            const midpoint = (slot.start + slot.end) / 2;
-            const framePath = path.join(framesDir, `scene-${String(i).padStart(3, '0')}.jpg`);
-            try {
-                await run(FFMPEG, ['-y', '-ss', String(midpoint), '-i', videoPath, '-vframes', '1', '-vf', 'scale=640:-1', '-q:v', '5', framePath, '-loglevel', 'error']);
-                const buf = await fs.readFile(framePath);
-                frameBuffers.push({ idx: i, buf, slot });
+            const span = slot.end - slot.start;
+            const mid = (slot.start + slot.end) / 2;
+            const inset = Math.min(0.3, span * 0.15);
+            const stamps = span < 1.5
+                ? [mid]
+                : [slot.start + inset, mid, slot.end - inset];
+            const buffers = [];
+            let thumb = null;
+            for (let k = 0; k < stamps.length; k++) {
+                const framePath = path.join(framesDir, `scene-${String(i).padStart(3, '0')}-${k}.jpg`);
+                try {
+                    await run(FFMPEG, ['-y', '-ss', String(Math.max(0, stamps[k])), '-i', videoPath, '-vframes', '1', '-vf', 'scale=640:-1', '-q:v', '5', framePath, '-loglevel', 'error']);
+                    const buf = await fs.readFile(framePath);
+                    buffers.push(buf);
+                    // Middle frame = thumbnail (index 1 when 3 frames, index 0 when 1).
+                    if (k === Math.floor(stamps.length / 2))
+                        thumb = buf;
+                }
+                catch (err) {
+                    const msg = err instanceof Error ? err.message : String(err);
+                    console.warn('[analyze_video] Frame extract failed for scene', i, 'frame', k, msg);
+                }
             }
-            catch (err) {
-                const msg = err instanceof Error ? err.message : String(err);
-                console.warn('[analyze_video] Frame extract failed for scene', i, msg);
+            if (buffers.length) {
+                frameBuffers.push({ idx: i, frames: buffers, thumb: thumb || buffers[0], slot });
             }
         }
-        // Step 2: Vision describe all frames in parallel (batch of 5)
+        // Step 2: Style + Master Cast FIRST (Phase 1), so per-scene describe can
+        // reuse consistent character labels (the AI_Video_Clone lesson).
+        progress('Đang phân tích video style...');
+        const styleSamples = frameBuffers.slice(0, 3).map(f => f.thumb.toString('base64'));
+        const video_style = await visionExtractStyle(styleSamples);
+        console.log('[analyze_video] Video style:', video_style.substring(0, 100));
+        progress('Đang trích xuất dàn nhân vật (Master Cast)...');
+        // Sample up to 12 frames evenly across the whole video for cast detection.
+        const castSampleCount = Math.min(12, frameBuffers.length);
+        const castStep = Math.max(1, Math.floor(frameBuffers.length / castSampleCount));
+        const castSamples = frameBuffers
+            .filter((_, i) => i % castStep === 0)
+            .slice(0, castSampleCount)
+            .map(f => f.thumb.toString('base64'));
+        const { master_cast_prompt, characters } = await visionExtractMasterCast(castSamples, transcript.text || '', video_style);
+        const castContext = characters.length
+            ? characters.map(c => `- ${c.name}: ${c.description}`).join('\n')
+            : '';
+        console.log('[analyze_video] Master cast:', characters.map(c => c.name).join(', ') || '(none)');
+        // Step 3: Vision describe all frames in parallel (batch of 5), passing the
+        // cast context so naming stays consistent across the whole timeline.
         progress(`Đang phân tích ${frameBuffers.length} scenes song song...`);
         const BATCH_SIZE = 5;
         const sceneResults = [];
         for (let b = 0; b < frameBuffers.length; b += BATCH_SIZE) {
             const batch = frameBuffers.slice(b, b + BATCH_SIZE);
             progress(`Đang phân tích scene ${b + 1}-${Math.min(b + BATCH_SIZE, frameBuffers.length)}/${frameBuffers.length}...`);
-            const results = await Promise.all(batch.map(async ({ idx, buf, slot }) => {
+            const results = await Promise.all(batch.map(async ({ idx, frames, thumb, slot }) => {
                 try {
-                    const visual_description = await visionDescribe(buf.toString('base64'), slot.voiceover);
+                    const visual_description = await visionDescribe(frames.map(f => f.toString('base64')), slot.voiceover, castContext);
                     return {
                         scene_number: idx + 1,
                         timestamp_start: Math.round(slot.start * 10) / 10,
                         timestamp_end: Math.round(slot.end * 10) / 10,
-                        thumbnail_base64: buf.toString('base64'),
+                        thumbnail_base64: thumb.toString('base64'),
                         voiceover: slot.voiceover,
                         visual_description,
                     };
@@ -242,16 +398,13 @@ export async function analyzeVideo(url, onProgress) {
             sceneResults.push(...results.filter((r) => r !== null));
         }
         const scenes = sceneResults.sort((a, b) => a.scene_number - b.scene_number);
-        // Extract video style from 3 sample frames
-        progress('Đang phân tích video style...');
-        const styleSamples = frameBuffers.slice(0, 3).map(f => f.buf.toString('base64'));
-        const video_style = await visionExtractStyle(styleSamples);
-        console.log('[analyze_video] Video style:', video_style.substring(0, 100));
         return {
             duration_sec: Math.round(durationSec),
             language: transcript.language || 'unknown',
             transcript: transcript.text || '',
             video_style,
+            master_cast_prompt,
+            characters,
             segments: segments.map((s) => ({ start: s.start, end: s.end, text: s.text })),
             scenes,
         };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tuna-agent",
-  "version": "0.1.133",
+  "version": "0.1.135",
   "description": "Tuna Agent - Run AI coding tasks on your machine",
   "bin": {
     "tuna-agent": "dist/cli/index.js"