tuna-agent 0.1.119 → 0.1.120

package/dist/daemon/analyze-video-handler.d.ts

@@ -0,0 +1,31 @@
+/**
+ * Analyze Video Handler
+ * Downloads YouTube video via yt-dlp, extracts audio, transcribes via Whisper,
+ * extracts frames per segment, describes each frame via GPT-4o vision.
+ *
+ * Requires on the host machine: yt-dlp, ffmpeg, ffprobe, OPENAI_API_KEY env var.
+ */
+import { AgentWebSocketClient } from './ws-client.js';
+export interface AnalyzeVideoResult {
+    duration_sec: number;
+    language: string;
+    transcript: string;
+    segments: Array<{
+        start: number;
+        end: number;
+        text: string;
+    }>;
+    scenes: Array<{
+        scene_number: number;
+        timestamp_start: number;
+        timestamp_end: number;
+        thumbnail_base64: string;
+        voiceover: string;
+        visual_description: string;
+    }>;
+    isError?: boolean;
+    error?: string;
+}
+export declare function analyzeVideo(url: string): Promise<AnalyzeVideoResult>;
+/** Extension task handler — wraps analyzeVideo with WS response */
+export declare function handleAnalyzeVideo(ws: AgentWebSocketClient, code: string, taskId: string, url: string): Promise<void>;

package/dist/daemon/analyze-video-handler.js

@@ -0,0 +1,180 @@
+/**
+ * Analyze Video Handler
+ * Downloads YouTube video via yt-dlp, extracts audio, transcribes via Whisper,
+ * extracts frames per segment, describes each frame via GPT-4o vision.
+ *
+ * Requires on the host machine: yt-dlp, ffmpeg, ffprobe, OPENAI_API_KEY env var.
+ */
+import { spawn } from 'child_process';
+import { promises as fs } from 'fs';
+import path from 'path';
+import os from 'os';
+import crypto from 'crypto';
+const OPENAI_KEY = process.env.OPENAI_API_KEY || '';
+const YT_DLP = process.env.YT_DLP_BIN || '/home/gatoasang94/.local/bin/yt-dlp';
+const FFMPEG = process.env.FFMPEG_BIN || '/usr/bin/ffmpeg';
+const FFPROBE = process.env.FFPROBE_BIN || '/usr/bin/ffprobe';
+function run(cmd, args, opts = {}) {
+    return new Promise((resolve, reject) => {
+        const p = spawn(cmd, args, { stdio: ['ignore', 'pipe', 'pipe'], ...opts });
+        let out = '', err = '';
+        p.stdout.on('data', (d) => out += d);
+        p.stderr.on('data', (d) => err += d);
+        p.on('close', (code) => code === 0 ? resolve({ out, err }) : reject(new Error(`${cmd} exit ${code}: ${err.slice(0, 500)}`)));
+    });
+}
+async function whisperTranscribe(audioPath) {
+    if (!OPENAI_KEY)
+        throw new Error('OPENAI_API_KEY not set');
+    const buf = await fs.readFile(audioPath);
+    const form = new FormData();
+    const blob = new Blob([buf], { type: 'audio/mpeg' });
+    form.append('file', blob, path.basename(audioPath));
+    form.append('model', 'whisper-1');
+    form.append('response_format', 'verbose_json');
+    form.append('timestamp_granularities[]', 'segment');
+    const res = await fetch('https://api.openai.com/v1/audio/transcriptions', {
+        method: 'POST',
+        headers: { Authorization: `Bearer ${OPENAI_KEY}` },
+        body: form,
+    });
+    if (!res.ok)
+        throw new Error(`whisper ${res.status}: ${(await res.text()).slice(0, 300)}`);
+    return res.json();
+}
+async function correctTranscript(rawText, language) {
+    if (!OPENAI_KEY || !rawText || rawText.length < 20)
+        return rawText;
+    try {
+        const res = await fetch('https://api.openai.com/v1/chat/completions', {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${OPENAI_KEY}` },
+            body: JSON.stringify({
+                model: 'gpt-4o-mini',
+                max_tokens: 2000,
+                messages: [{
+                        role: 'user',
+                        content: `Below is a raw transcript from Whisper speech-to-text (language: ${language || 'unknown'}). It may contain recognition errors: wrong words, missing punctuation, or garbled phrases.
+
+Fix any obvious errors while keeping the meaning intact. Add proper punctuation and capitalization. If a word sounds wrong in context, replace it with the most likely correct word.
+
+Return ONLY the corrected transcript text, nothing else.
+
+Raw transcript:
+${rawText}`,
+                    }],
+            }),
+        });
+        if (!res.ok)
+            return rawText;
+        const data = await res.json();
+        return data.choices?.[0]?.message?.content?.trim() || rawText;
+    }
+    catch {
+        return rawText;
+    }
+}
+async function visionDescribe(frameB64, voiceoverText) {
+    if (!OPENAI_KEY)
+        return '';
+    const res = await fetch('https://api.openai.com/v1/chat/completions', {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${OPENAI_KEY}` },
+        body: JSON.stringify({
+            model: 'gpt-4o-mini',
+            max_tokens: 150,
+            messages: [{
+                    role: 'user',
+                    content: [
+                        { type: 'text', text: `Mô tả ngắn gọn (1-2 câu tiếng Việt) khung hình này: nhân vật, bối cảnh, góc quay, tâm trạng. Voiceover đang nói: "${voiceoverText || '(không có)'}"` },
+                        { type: 'image_url', image_url: { url: `data:image/jpeg;base64,${frameB64}` } },
+                    ],
+                }],
+        }),
+    });
+    if (!res.ok)
+        return '';
+    const data = await res.json();
+    return data.choices?.[0]?.message?.content?.trim() || '';
+}
+export async function analyzeVideo(url) {
+    const tmpDir = path.join(os.tmpdir(), 'tuna-analyze-' + crypto.randomBytes(6).toString('hex'));
+    await fs.mkdir(tmpDir, { recursive: true });
+    const videoPath = path.join(tmpDir, 'video.mp4');
+    const audioPath = path.join(tmpDir, 'audio.mp3');
+    const framesDir = path.join(tmpDir, 'frames');
+    await fs.mkdir(framesDir, { recursive: true });
+    try {
+        console.log('[analyze_video] Downloading:', url);
+        await run(YT_DLP, ['-f', 'best[height<=720]/best', '-o', videoPath, '--no-playlist', '--quiet', url]);
+        console.log('[analyze_video] Extracting audio');
+        await run(FFMPEG, ['-y', '-i', videoPath, '-vn', '-ar', '16000', '-ac', '1', '-b:a', '64k', audioPath, '-loglevel', 'error']);
+        console.log('[analyze_video] Probing duration');
+        const durationSec = await (async () => {
+            try {
+                const r = await run(FFPROBE, ['-v', 'error', '-show_entries', 'format=duration', '-of', 'default=nw=1:nk=1', videoPath]);
+                return parseFloat(r.out.trim()) || 0;
+            }
+            catch {
+                return 0;
+            }
+        })();
+        console.log('[analyze_video] Transcribing via Whisper');
+        const rawTranscript = await whisperTranscribe(audioPath);
+        console.log('[analyze_video] AI correcting transcript');
+        const correctedText = await correctTranscript(rawTranscript.text, rawTranscript.language);
+        const transcript = { ...rawTranscript, text: correctedText };
+        const segments = transcript.segments || [];
+        console.log('[analyze_video] Extracting', segments.length, 'frames (1 per segment)');
+        const scenes = [];
+        for (let i = 0; i < segments.length; i++) {
+            const seg = segments[i];
+            const midpoint = (seg.start + seg.end) / 2;
+            const framePath = path.join(framesDir, `seg-${String(i).padStart(3, '0')}.jpg`);
+            try {
+                await run(FFMPEG, ['-y', '-ss', String(midpoint), '-i', videoPath, '-vframes', '1', '-vf', 'scale=640:-1', '-q:v', '5', framePath, '-loglevel', 'error']);
+                const buf = await fs.readFile(framePath);
+                const voiceover = seg.text?.trim() || '';
+                const visual_description = await visionDescribe(buf.toString('base64'), voiceover);
+                scenes.push({
+                    scene_number: i + 1,
+                    timestamp_start: Math.round(seg.start * 10) / 10,
+                    timestamp_end: Math.round(seg.end * 10) / 10,
+                    thumbnail_base64: buf.toString('base64'),
+                    voiceover,
+                    visual_description,
+                });
+            }
+            catch (err) {
+                const msg = err instanceof Error ? err.message : String(err);
+                console.warn('[analyze_video] Frame extract failed for segment', i, msg);
+            }
+        }
+        return {
+            duration_sec: Math.round(durationSec),
+            language: transcript.language || 'unknown',
+            transcript: transcript.text || '',
+            segments: segments.map((s) => ({ start: s.start, end: s.end, text: s.text })),
+            scenes,
+        };
+    }
+    finally {
+        try {
+            await fs.rm(tmpDir, { recursive: true, force: true });
+        }
+        catch { /* ignore */ }
+    }
+}
+/** Extension task handler — wraps analyzeVideo with WS response */
+export async function handleAnalyzeVideo(ws, code, taskId, url) {
+    console.log(`[analyze_video] Starting for ${url}`);
+    try {
+        const result = await analyzeVideo(url);
+        ws.sendExtensionDone(code, taskId, { ...result, isError: false });
+    }
+    catch (err) {
+        const errMsg = err instanceof Error ? err.message : String(err);
+        console.error(`[analyze_video] Error: ${errMsg}`);
+        ws.sendExtensionDone(code, taskId, { error: errMsg, isError: true });
+    }
+}

package/dist/daemon/index.js

@@ -10,6 +10,7 @@ import { chatWithPM } from '../pm/planner.js';
 import { executePlanAndReport, simplifyMarkdown, waitForInput } from '../utils/execution-helpers.js';
 import { runClaude } from '../utils/claude-cli.js';
 import { handleGetHistory, handleRetryVideo, handleGenerateIdeas, handleGenerateScript, handleGenerateScene, handleGenerateScenes, handleRenderVideo, handleListCharacters, handleCreateCharacter, handleSaveCharacterSelection, handleSyncApps, handleClaudePrompt, handleClaudePromptStream, } from './extension-handlers.js';
+import { handleAnalyzeVideo } from './analyze-video-handler.js';
 import { downloadAttachments, cleanupAttachments } from '../utils/image-download.js';
 import { scanSkills } from '../utils/skill-scanner.js';
 import { setupMcpConfig } from '../mcp/setup.js';
@@ -561,19 +562,7 @@ ${skillContent.slice(0, 15000)}`;
                 }
                 if (extTask === 'analyze_video' && msg.url) {
                     (async () => {
-                        try {
-                            // Load custom handler from ~/.tuna-agent/ if available
-                            const handlerPath = path.join(os.homedir(), '.tuna-agent', 'analyze-video-handler.mjs');
-                            const { analyzeVideo } = await import(handlerPath);
-                            console.log(`[Daemon] analyze_video: using custom handler for ${msg.url}`);
-                            const result = await analyzeVideo(msg.url);
-                            ws.sendExtensionDone(extCode, extTaskId, { ...result, isError: false });
-                        }
-                        catch (err) {
-                            const errMsg = err instanceof Error ? err.message : String(err);
-                            console.error(`[Daemon] analyze_video error: ${errMsg}`);
-                            ws.sendExtensionDone(extCode, extTaskId, { error: errMsg, isError: true });
-                        }
+                        await handleAnalyzeVideo(ws, extCode, extTaskId, msg.url);
                     })();
                     break;
                 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tuna-agent",
-  "version": "0.1.119",
+  "version": "0.1.120",
   "description": "Tuna Agent - Run AI coding tasks on your machine",
   "bin": {
     "tuna-agent": "dist/cli/index.js"