tuna-agent 0.1.147 → 0.1.149

package/dist/daemon/analyze-video-handler.js

@@ -29,6 +29,8 @@ const RATES = {
     'gpt-4o': { in: 2.50, out: 10.0 },
     // Gemini 3 Flash preview: text+image input share one rate, output 6x.
     'gemini-3-flash-preview': { in: 0.50, out: 3.0 },
+    // Gemini 2.5 Flash: cheaper image-heavy reads (used by Phase-1).
+    'gemini-2.5-flash': { in: 0.30, out: 2.50 },
 };
 // Per-run cost+token accumulator. Single-threaded JS → plain mutation is safe
 // even across the parallel visionDescribe calls.
@@ -50,12 +52,12 @@ class CostTracker {
     }
     // Gemini reports usageMetadata.{promptTokenCount,candidatesTokenCount}
     // instead of OpenAI's prompt_tokens/completion_tokens.
-    geminiVision(bucket, usage) {
+    geminiVision(bucket, usage, model = 'gemini-3-flash-preview') {
         if (!usage) {
             this.add(bucket, 0);
             return;
         }
-        const r = RATES['gemini-3-flash-preview'];
+        const r = RATES[model];
         const cost = ((usage.promptTokenCount || 0) / 1e6) * r.in + ((usage.candidatesTokenCount || 0) / 1e6) * r.out;
         this.add(bucket, cost);
     }
@@ -174,7 +176,7 @@ ${rawText}`,
 // One Gemini generateContent call with key rotation + exponential backoff on
 // 429/5xx. A single free-tier key under the 5-way concurrent batch WILL
 // rate-limit; retrying (slower) beats dropping the scene description.
-async function geminiGenerate(parts, maxOutputTokens) {
+async function geminiGenerate(parts, maxOutputTokens, model = GEMINI_MODEL) {
     if (!GEMINI_KEYS.length)
         return { text: '' };
     const body = JSON.stringify({
@@ -187,7 +189,7 @@ async function geminiGenerate(parts, maxOutputTokens) {
     for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
         const key = GEMINI_KEYS[keyIdx % GEMINI_KEYS.length];
         try {
-            const res = await fetch(`https://generativelanguage.googleapis.com/v1beta/models/${encodeURIComponent(GEMINI_MODEL)}:generateContent?key=${key}`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body });
+            const res = await fetch(`https://generativelanguage.googleapis.com/v1beta/models/${encodeURIComponent(model)}:generateContent?key=${key}`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body });
             if (res.status === 429 || res.status >= 500) {
                 lastErr = `Gemini ${res.status}`;
                 keyIdx++; // rotate to the next key before backing off
@@ -259,13 +261,10 @@ async function visionExtractPhase1(frames, transcript, cost) {
         master_cast_prompt: '',
         characters: [],
     };
-    if (!OPENAI_KEY || frames.length === 0)
+    if (!GEMINI_KEYS.length || frames.length === 0)
         return empty;
     try {
-        const content = [
-            {
-                type: 'text',
-                text: `Act as a Master Film Director. These frames are sampled across an ENTIRE video, in order. Use them + the transcript to analyse the whole piece.
+        const promptText = `Act as a Master Film Director. These frames are sampled across an ENTIRE video, in order. Use them + the transcript to analyse the whole piece.
 
 Transcript context: "${(transcript || '').slice(0, 4000)}"
 
@@ -282,22 +281,18 @@ Rules:
 - characters.name: a stable short uppercase label reused for this subject (e.g. "THE BISHOP", "U-94 SUBMARINE"). Max 4 words.
 - RECALL (CRITICAL): list EVERY distinct recurring subject SEPARATELY. If a family or group recurs, include EACH member as its own entry (e.g. adult man, adult woman, older boy, younger girl) — never merge them into one. Skip only true one-off background extras. Be COMPLETE: missing a recurring character is worse than one extra. Up to 8.
 - characters.description: ENGLISH only, factual, no camera/action words.
-- DISTINCT FACES (CRITICAL): every character MUST have a HIGHLY UNIQUE facial structure, a distinct hairstyle, a specific body type and a clearly different age. NEVER reuse the same or a similar facial description for two characters — they must look completely different from one another.`,
-            },
+- DISTINCT FACES (CRITICAL): every character MUST have a HIGHLY UNIQUE facial structure, a distinct hairstyle, a specific body type and a clearly different age. NEVER reuse the same or a similar facial description for two characters — they must look completely different from one another.`;
+        // Phase-1 on Gemini 2.5 Flash: image-heavy read is far cheaper than gpt-4o,
+        // and cast recall is backstopped by the post-Phase-2 reconcile pass, so a
+        // small frame sample suffices here.
+        const parts = [
+            { text: promptText },
+            ...frames.map(b64 => ({ inlineData: { mimeType: 'image/jpeg', data: b64 } })),
         ];
-        for (const b64 of frames) {
-            content.push({ type: 'image_url', image_url: { url: `data:image/jpeg;base64,${b64}` } });
-        }
-        const res = await fetch('https://api.openai.com/v1/chat/completions', {
-            method: 'POST',
-            headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${OPENAI_KEY}` },
-            body: JSON.stringify({ model: 'gpt-4o', max_tokens: 1600, messages: [{ role: 'user', content }] }),
-        });
-        if (!res.ok)
+        const { text: rawTxt, usage } = await geminiGenerate(parts, 1600, 'gemini-2.5-flash');
+        if (!rawTxt)
             return empty;
-        const data = await res.json();
-        cost?.chat('phase1', 'gpt-4o', data.usage);
-        const rawTxt = (data.choices?.[0]?.message?.content || '').trim();
+        cost?.geminiVision('phase1', usage, 'gemini-2.5-flash');
         let parsed = {};
         try {
             const m = rawTxt.match(/\{[\s\S]*\}/);
@@ -312,22 +307,103 @@ Rules:
             .map(c => ({ name: String(c.name).trim(), description: String(c.description).trim() }));
         const video_summary = (parsed.video_summary || '').trim();
         const video_style = (parsed.video_style || '').trim();
-        let master_cast_prompt = '';
-        if (characters.length > 0) {
-            const styleLine = video_style || 'Keep the original video’s visual style, color grading, and lighting.';
-            const castList = characters.map(c => `- ${c.name}: ${c.description}`).join('\n');
-            master_cast_prompt =
-                `[AESTHETIC & STYLE]\n${styleLine}\n` +
-                    `[COMPOSITION & LAYOUT]\nCharacter Reference Sheet. Full-body side-by-side.\n` +
-                    `[CHARACTER CAST LIST]\n${castList}\n` +
-                    `[TECHNICAL SPECIFICATIONS]\nHigh detail, 8k resolution, consistent facial structures across all frames. Each character has a completely distinct face, hairstyle, body type and age — no two characters look alike.`;
-        }
+        const master_cast_prompt = buildMasterCastPrompt(video_style, characters);
         return { video_summary, video_style, master_cast_prompt, characters };
     }
     catch {
         return empty;
     }
 }
+// Single source of truth for the master-cast prompt block (used by Phase-1
+// and the post-Phase-2 cast reconciliation so the format never drifts).
+function buildMasterCastPrompt(videoStyle, characters) {
+    if (!characters.length)
+        return '';
+    const styleLine = videoStyle || 'Keep the original video’s visual style, color grading, and lighting.';
+    const castList = characters.map(c => `- ${c.name}: ${c.description}`).join('\n');
+    return (`[AESTHETIC & STYLE]\n${styleLine}\n` +
+        `[COMPOSITION & LAYOUT]\nCharacter Reference Sheet. Full-body side-by-side.\n` +
+        `[CHARACTER CAST LIST]\n${castList}\n` +
+        `[TECHNICAL SPECIFICATIONS]\nHigh detail, 8k resolution, consistent facial structures across all frames. Each character has a completely distinct face, hairstyle, body type and age — no two characters look alike.`);
+}
+// Post-Phase-2 cast RECONCILE. Phase-1 only sees a 30-frame sample so it can
+// miss a recurring character; the per-scene visionDescribe pass, however,
+// looked at EVERY scene. Feed all per-scene descriptions (+ Phase-1 cast as a
+// trusted seed) to one gpt-4o text call to produce the definitive cast: keep
+// seed entries, merge duplicates, ADD any recurring subject Phase-1 missed.
+// Returns null on any failure → caller keeps the Phase-1 cast.
+async function reconcileCast(seed, sceneDescriptions, transcript, videoStyle, cost) {
+    if (!OPENAI_KEY)
+        return null;
+    const descs = sceneDescriptions.map(s => (s || '').trim()).filter(Boolean);
+    if (descs.length < 3)
+        return null;
+    // Bound tokens: cap each scene line + overall.
+    const joined = descs
+        .map((d, i) => `S${i + 1}: ${d.slice(0, 240)}`)
+        .join('\n')
+        .slice(0, 60000);
+    const seedBlock = seed.length
+        ? seed.map(c => `- ${c.name}: ${c.description}`).join('\n')
+        : '(none detected yet)';
+    try {
+        const res = await fetch('https://api.openai.com/v1/chat/completions', {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${OPENAI_KEY}` },
+            body: JSON.stringify({
+                model: 'gpt-4o',
+                max_tokens: 1200,
+                messages: [{
+                        role: 'user',
+                        content: `You are reconciling the definitive recurring CHARACTER CAST of a video from per-scene visual descriptions that cover EVERY scene (the seed cast below was extracted from only a few sampled frames and may be INCOMPLETE).
+
+SEED CAST (trusted — keep these, refine wording if scenes add detail):
+${seedBlock}
+
+PER-SCENE DESCRIPTIONS (every scene, in order):
+${joined}
+
+Transcript (may name characters): "${(transcript || '').slice(0, 1500)}"
+
+Return ONLY a JSON object, no markdown:
+{ "characters": [ { "name": "SHORT_UPPERCASE_LABEL", "description": "age/build, face, hair, outfit, colors, distinguishing features (English, factual, no camera/action words)" } ] }
+
+Rules:
+- Start from the SEED CAST; KEEP every seed character (don't drop them).
+- ADD any recurring subject that appears in multiple scenes but is missing from the seed (e.g. a family member, a recurring animal/mascot/object). Missing a recurring character is the main failure to avoid.
+- Merge entries that clearly refer to the SAME subject under one name.
+- If a group/family recurs, list EACH member SEPARATELY (e.g. adult man, adult woman, older boy, younger girl) — never merge them.
+- Skip true one-off background extras.
+- DISTINCT FACES: every character must have a UNIQUE facial structure, hairstyle, body type and a clearly different age — never reuse a similar facial description for two characters.
+- Stable short UPPERCASE name, max 4 words. Up to 8 characters total.`,
+                    }],
+            }),
+        });
+        if (!res.ok)
+            return null;
+        const data = await res.json();
+        cost?.chat('cast', 'gpt-4o', data.usage);
+        const raw = (data.choices?.[0]?.message?.content || '').trim();
+        let parsed = {};
+        try {
+            const m = raw.match(/\{[\s\S]*\}/);
+            parsed = JSON.parse(m ? m[0] : raw);
+        }
+        catch {
+            return null;
+        }
+        const characters = (parsed.characters || [])
+            .filter(c => c && c.name && c.description)
+            .slice(0, 8)
+            .map(c => ({ name: String(c.name).trim(), description: String(c.description).trim() }));
+        if (!characters.length)
+            return null;
+        return { characters, master_cast_prompt: buildMasterCastPrompt(videoStyle, characters) };
+    }
+    catch {
+        return null;
+    }
+}
 export async function analyzeVideo(url, onProgress) {
     const progress = onProgress || (() => { });
     const cost = new CostTracker();
@@ -518,16 +594,19 @@ export async function analyzeVideo(url, onProgress) {
         // master cast + characters. Runs before per-scene describe so the cast
         // context keeps naming consistent across the whole timeline.
         progress('Đang phân tích tổng thể (summary + style + master cast)...');
-        // Sample up to 30 frames evenly across the whole video — denser sampling
-        // is critical for cast RECALL on sparse-narration (ASMR) videos where
-        // Phase-1 relies almost entirely on frames (matches AI_Video_Clone).
-        const p1SampleCount = Math.min(30, frameBuffers.length);
+        // Sample up to 10 frames evenly — enough for summary + style + a naming
+        // seed. Cast RECALL no longer depends on this sample: the post-Phase-2
+        // reconcile pass derives the definitive cast from every per-scene
+        // description, so a small sample keeps the (now Gemini 2.5 Flash) Phase-1
+        // call cheap.
+        const p1SampleCount = Math.min(10, frameBuffers.length);
         const p1Step = Math.max(1, Math.floor(frameBuffers.length / p1SampleCount));
         const p1Samples = frameBuffers
             .filter((_, i) => i % p1Step === 0)
             .slice(0, p1SampleCount)
             .map(f => f.thumb.toString('base64'));
-        const { video_summary, video_style, master_cast_prompt, characters } = await visionExtractPhase1(p1Samples, transcript.text || '', cost);
+        // eslint-disable-next-line prefer-const
+        let { video_summary, video_style, master_cast_prompt, characters } = await visionExtractPhase1(p1Samples, transcript.text || '', cost);
         const castContext = characters.length
             ? characters.map(c => `- ${c.name}: ${c.description}`).join('\n')
             : '';
@@ -561,6 +640,16 @@ export async function analyzeVideo(url, onProgress) {
             sceneResults.push(...results.filter((r) => r !== null));
         }
         const scenes = sceneResults.sort((a, b) => a.scene_number - b.scene_number);
+        // Reconcile cast from EVERY per-scene description (recall fix — Phase-1
+        // only saw a sampled set of frames). Falls back to Phase-1 cast on failure.
+        progress('Đang đối soát dàn nhân vật từ toàn bộ scene...');
+        const reconciled = await reconcileCast(characters, scenes.map(s => s.visual_description || ''), transcript.text || '', video_style, cost);
+        if (reconciled && reconciled.characters.length) {
+            console.log('[analyze_video] Cast reconciled:', (characters.map(c => c.name).join(', ') || '(none)'), '→', reconciled.characters.map(c => c.name).join(', '));
+            characters = reconciled.characters;
+            if (reconciled.master_cast_prompt)
+                master_cast_prompt = reconciled.master_cast_prompt;
+        }
         return {
             source_title,
             duration_sec: Math.round(durationSec),

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tuna-agent",
-  "version": "0.1.147",
+  "version": "0.1.149",
   "description": "Tuna Agent - Run AI coding tasks on your machine",
   "bin": {
     "tuna-agent": "dist/cli/index.js"