tuna-agent 0.1.143 → 0.1.145

package/dist/daemon/analyze-video-handler.d.ts

@@ -1,9 +1,11 @@
 /**
  * Analyze Video Handler
  * Downloads YouTube video via yt-dlp, extracts audio, transcribes via Whisper,
- * extracts frames per segment, describes each frame via GPT-4o vision.
+ * extracts frames per segment, describes each scene via Gemini 3 Flash vision
+ * (per-scene; Phase-1 summary/style/cast still uses gpt-4o).
  *
- * Requires on the host machine: yt-dlp, ffmpeg, ffprobe, OPENAI_API_KEY env var.
+ * Requires on the host machine: yt-dlp, ffmpeg, ffprobe, OPENAI_API_KEY
+ * (Whisper + correction + Phase-1) and GEMINI_API_KEY (per-scene vision) env vars.
  */
 import { AgentWebSocketClient } from './ws-client.js';
 export interface AnalyzeVideoResult {

package/dist/daemon/analyze-video-handler.js

@@ -1,9 +1,11 @@
 /**
  * Analyze Video Handler
  * Downloads YouTube video via yt-dlp, extracts audio, transcribes via Whisper,
- * extracts frames per segment, describes each frame via GPT-4o vision.
+ * extracts frames per segment, describes each scene via Gemini 3 Flash vision
+ * (per-scene; Phase-1 summary/style/cast still uses gpt-4o).
  *
- * Requires on the host machine: yt-dlp, ffmpeg, ffprobe, OPENAI_API_KEY env var.
+ * Requires on the host machine: yt-dlp, ffmpeg, ffprobe, OPENAI_API_KEY
+ * (Whisper + correction + Phase-1) and GEMINI_API_KEY (per-scene vision) env vars.
  */
 import { spawn } from 'child_process';
 import { promises as fs } from 'fs';
@@ -11,6 +13,12 @@ import path from 'path';
 import os from 'os';
 import crypto from 'crypto';
 const OPENAI_KEY = process.env.OPENAI_API_KEY || '';
+// Gemini 3 Flash powers per-scene visionDescribe: cheaper image tokens than
+// gpt-4o-mini's ~33x multiplier and a stronger VLM. Comma/newline-separated
+// list → rotate on 429 so a single free-tier key still completes (slower).
+const GEMINI_KEYS = (process.env.GEMINI_API_KEY || '')
+    .split(/[,\n]+/).map(s => s.trim()).filter(Boolean);
+const GEMINI_MODEL = process.env.GEMINI_MODEL || 'gemini-3-flash-preview';
 const YT_DLP = process.env.YT_DLP_BIN || '/home/gatoasang94/.local/bin/yt-dlp';
 const FFMPEG = process.env.FFMPEG_BIN || '/usr/bin/ffmpeg';
 const FFPROBE = process.env.FFPROBE_BIN || '/usr/bin/ffprobe';
@@ -19,6 +27,8 @@ const RATES = {
     whisperPerMin: 0.006,
     'gpt-4o-mini': { in: 0.15, out: 0.60 },
     'gpt-4o': { in: 2.50, out: 10.0 },
+    // Gemini 3 Flash preview: text+image input share one rate, output 6x.
+    'gemini-3-flash-preview': { in: 0.50, out: 3.0 },
 };
 // Per-run cost+token accumulator. Single-threaded JS → plain mutation is safe
 // even across the parallel visionDescribe calls.
@@ -38,6 +48,17 @@ class CostTracker {
         const cost = ((usage.prompt_tokens || 0) / 1e6) * r.in + ((usage.completion_tokens || 0) / 1e6) * r.out;
         this.add(bucket, cost);
     }
+    // Gemini reports usageMetadata.{promptTokenCount,candidatesTokenCount}
+    // instead of OpenAI's prompt_tokens/completion_tokens.
+    geminiVision(bucket, usage) {
+        if (!usage) {
+            this.add(bucket, 0);
+            return;
+        }
+        const r = RATES['gemini-3-flash-preview'];
+        const cost = ((usage.promptTokenCount || 0) / 1e6) * r.in + ((usage.candidatesTokenCount || 0) / 1e6) * r.out;
+        this.add(bucket, cost);
+    }
     whisper(audioSec) { this.add('whisper', (audioSec / 60) * RATES.whisperPerMin); }
     total() { return Object.values(this.breakdown).reduce((s, b) => s + b.cost, 0); }
 }
@@ -150,8 +171,49 @@ ${rawText}`,
 // turns") instead of guessing from a single frozen midpoint. The model is
 // told the frames are chronological so it describes the action arc, not 3
 // separate moments.
+// One Gemini generateContent call with key rotation + exponential backoff on
+// 429/5xx. A single free-tier key under the 5-way concurrent batch WILL
+// rate-limit; retrying (slower) beats dropping the scene description.
+async function geminiGenerate(parts, maxOutputTokens) {
+    if (!GEMINI_KEYS.length)
+        return { text: '' };
+    const body = JSON.stringify({
+        contents: [{ parts }],
+        generationConfig: { maxOutputTokens, temperature: 0.4 },
+    });
+    const MAX_ATTEMPTS = 6;
+    let keyIdx = 0;
+    let lastErr = '';
+    for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
+        const key = GEMINI_KEYS[keyIdx % GEMINI_KEYS.length];
+        try {
+            const res = await fetch(`https://generativelanguage.googleapis.com/v1beta/models/${encodeURIComponent(GEMINI_MODEL)}:generateContent?key=${key}`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body });
+            if (res.status === 429 || res.status >= 500) {
+                lastErr = `Gemini ${res.status}`;
+                keyIdx++; // rotate to the next key before backing off
+                const backoff = Math.min(60000, 1500 * 2 ** attempt) + Math.floor(Math.random() * 1000);
+                await new Promise(r => setTimeout(r, backoff));
+                continue;
+            }
+            if (!res.ok) {
+                lastErr = `Gemini ${res.status}: ${(await res.text()).slice(0, 200)}`;
+                break;
+            }
+            const data = await res.json();
+            const text = (data?.candidates?.[0]?.content?.parts || [])
+                .map(p => p.text || '').join('').trim();
+            return { text, usage: data?.usageMetadata };
+        }
+        catch (e) {
+            lastErr = e instanceof Error ? e.message : String(e);
+            await new Promise(r => setTimeout(r, Math.min(30000, 1000 * 2 ** attempt)));
+        }
+    }
+    console.warn(`[analyze_video] geminiGenerate failed after retries: ${lastErr}`);
+    return { text: '' };
+}
 async function visionDescribe(frameB64s, voiceoverText, castContext = '', cost) {
-    if (!OPENAI_KEY)
+    if (!GEMINI_KEYS.length)
         return '';
     const frames = frameB64s.filter(Boolean);
     if (frames.length === 0)
@@ -162,20 +224,7 @@ async function visionDescribe(frameB64s, voiceoverText, castContext = '', cost)
     const seqNote = frames.length > 1
         ? ` The ${frames.length} images are CHRONOLOGICAL samples from the SAME scene (start → middle → end). Treat them as one continuous shot: describe the MOTION ARC across them (direction of movement, camera push/pan, what changes from first to last frame), not three separate moments.`
         : '';
-    const imageParts = frames.map(b64 => ({
-        type: 'image_url',
-        image_url: { url: `data:image/jpeg;base64,${b64}` },
-    }));
-    const res = await fetch('https://api.openai.com/v1/chat/completions', {
-        method: 'POST',
-        headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${OPENAI_KEY}` },
-        body: JSON.stringify({
-            model: 'gpt-4o-mini',
-            max_tokens: 350,
-            messages: [{
-                    role: 'user',
-                    content: [
-                        { type: 'text', text: `Describe this scene in detail (4-6 sentences, English).${seqNote}${castBlock} Include:
+    const promptText = `Describe this scene in detail (4-6 sentences, English).${seqNote}${castBlock} Include:
 - Characters: name them using the KNOWN CHARACTER CAST labels above when they appear; appearance (shape, color, size), facial expression, what they're doing
 - Physical connections: Are characters physically joined/attached/fused together (e.g. organs connected at a junction, body parts linked)? Or are they separate/independent? Be VERY specific — "physically attached at Y-junction" is different from "standing next to each other"
 - Spatial positions: exact position of each character (left/right/above/below/center), distance between them
@@ -183,17 +232,14 @@ async function visionDescribe(frameB64s, voiceoverText, castContext = '', cost)
 - Camera: angle, framing (close-up, wide, etc.), and any camera movement across the frames
 - Action: the movement/action arc from first to last frame (direction, what changes)
 
-Voiceover during this scene: "${voiceoverText || '(none)'}"` },
-                        ...imageParts,
-                    ],
-                }],
-        }),
-    });
-    if (!res.ok)
-        return '';
-    const data = await res.json();
-    cost?.chat('vision', 'gpt-4o-mini', data.usage);
-    return data.choices?.[0]?.message?.content?.trim() || '';
+Voiceover during this scene: "${voiceoverText || '(none)'}"`;
+    const parts = [
+        { text: promptText },
+        ...frames.map(b64 => ({ inlineData: { mimeType: 'image/jpeg', data: b64 } })),
+    ];
+    const { text, usage } = await geminiGenerate(parts, 512);
+    cost?.geminiVision('vision', usage);
+    return text;
 }
 // Phase 1 (the strong part of AI_Video_Clone, ported): ONE gpt-4o call over
 // frames sampled across the whole video + transcript that returns, together:

package/dist/mcp/idea-server.js

@@ -65,6 +65,8 @@ const TOOLS = [
             properties: {
                 name: { type: 'string', description: 'Name/title of the idea' },
                 category: { type: 'string', description: 'Category of the idea (e.g. SaaS, Mobile App, AI Tool, etc.)' },
+                startup: { type: 'string', enum: ['tuctac', 'capple', 'thinkless'], description: 'Which startup this idea is scoped to (REQUIRED for scoped scans)' },
+                idea_kind: { type: 'string', enum: ['feature', 'improvement', 'expansion', 'counter-competitor'], description: 'Kind of idea relative to the startup (optional)' },
                 source: { type: 'string', enum: ['trending', 'research', 'social', 'creative'], description: 'Where the idea came from' },
                 strategy: { type: 'string', description: 'High-level strategy or approach for this idea (optional)' },
                 url: { type: 'string', description: 'Reference URL related to the idea (optional)' },
@@ -87,6 +89,8 @@ const TOOLS = [
                 idea_id: { type: 'string', description: 'The ID of the idea to update' },
                 name: { type: 'string', description: 'New name (optional)' },
                 category: { type: 'string', description: 'New category (optional)' },
+                startup: { type: 'string', enum: ['tuctac', 'capple', 'thinkless'], description: 'New startup scope (optional)' },
+                idea_kind: { type: 'string', enum: ['feature', 'improvement', 'expansion', 'counter-competitor'], description: 'New idea kind (optional)' },
                 status: { type: 'string', enum: ['new', 'watching', 'validated', 'archived'], description: 'New status (optional)' },
                 score: { type: 'number', description: 'New score (optional)' },
                 notes: { type: 'string', description: 'New notes (optional)' },
@@ -109,6 +113,8 @@ const TOOLS = [
             properties: {
                 status: { type: 'string', enum: ['new', 'watching', 'validated', 'archived'], description: 'Filter by status (optional)' },
                 category: { type: 'string', description: 'Filter by category (optional)' },
+                startup: { type: 'string', enum: ['tuctac', 'capple', 'thinkless'], description: 'Filter by startup (optional)' },
+                idea_kind: { type: 'string', description: 'Filter by idea kind (optional)' },
                 source: { type: 'string', enum: ['trending', 'research', 'social', 'creative'], description: 'Filter by source (optional)' },
                 agent_id: { type: 'string', description: 'Filter by agent ID (optional, defaults to current agent)' },
                 page: { type: 'number', description: 'Page number (optional, default 1)' },
@@ -250,6 +256,10 @@ async function handleToolCall(config, toolName, args) {
                     body.competitors = args.competitors.split(',').map((c) => c.trim());
                 if (args.notes)
                     body.notes = args.notes;
+                if (args.startup)
+                    body.startup = args.startup;
+                if (args.idea_kind)
+                    body.idea_kind = args.idea_kind;
                 const data = await apiCall(config, 'POST', '/agent-idea', body);
                 return { content: [{ type: 'text', text: `Idea "${data.name}" created (ID: ${data._id})` }] };
             }
@@ -282,6 +292,10 @@ async function handleToolCall(config, toolName, args) {
                     body.competitors = args.competitors.split(',').map((c) => c.trim());
                 if (args.times_seen)
                     body.times_seen = Number(args.times_seen);
+                if (args.startup)
+                    body.startup = args.startup;
+                if (args.idea_kind)
+                    body.idea_kind = args.idea_kind;
                 const data = await apiCall(config, 'PUT', `/agent-idea/${args.idea_id}`, body);
                 return { content: [{ type: 'text', text: `Idea "${data.name}" updated (ID: ${data._id})` }] };
             }
@@ -293,6 +307,10 @@ async function handleToolCall(config, toolName, args) {
                     params.set('status', args.status);
                 if (args.category)
                     params.set('category', args.category);
+                if (args.startup)
+                    params.set('startup', args.startup);
+                if (args.idea_kind)
+                    params.set('idea_kind', args.idea_kind);
                 if (args.source)
                     params.set('source', args.source);
                 if (args.page)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tuna-agent",
-  "version": "0.1.143",
+  "version": "0.1.145",
   "description": "Tuna Agent - Run AI coding tasks on your machine",
   "bin": {
     "tuna-agent": "dist/cli/index.js"