project-graph-mcp 2.3.2 → 2.4.0

package/vendor/symbiote-node/engine/packs/ai/replicate-lipsync.handler.js

@@ -1,341 +0,0 @@
-/**
- * ai/replicate-lipsync — Replicate API lip-sync via Kling model
- *
- * Alternative lipsync provider using Replicate API.
- * Model: kwaivgi/kling-lip-sync (~$0.014/second of output)
- * Simpler than direct Kling API — just video URL + audio URL → result.
- *
- * Supports:
- * - Single segment processing
- * - Batch processing with worker pool
- * - Tunnel validation (cloudflared/ngrok)
- *
- * Ported from Mr-Computer/modules/ai-music-video/src/services/replicate-lipsync.js
- *
- * @module agi-graph/packs/ai/replicate-lipsync
- */
-
-import { readFile, writeFile, mkdir } from 'fs/promises';
-import { existsSync } from 'fs';
-import { execSync } from 'child_process';
-import path from 'path';
-
-export default {
-  type: 'ai/replicate-lipsync',
-  category: 'ai',
-  icon: 'mic',
-
-  driver: {
-    description: 'Replicate API lipsync via kwaivgi/kling-lip-sync model',
-    inputs: [
-      { name: 'videoUrl', type: 'string' },
-      { name: 'audioPath', type: 'string' },
-    ],
-    outputs: [
-      { name: 'result', type: 'any' },
-      { name: 'error', type: 'string' },
-    ],
-    params: {
-      operation: { type: 'string', default: 'process', description: 'Operation: process | batch | validate-tunnel' },
-      replicateToken: { type: 'string', default: '', description: 'Replicate API token' },
-      publicBaseUrl: { type: 'string', default: '', description: 'Public URL for file server' },
-      outputDir: { type: 'string', default: '/tmp/replicate-lipsync', description: 'Output directory' },
-      segmentId: { type: 'string', default: '', description: 'Segment identifier' },
-      startTime: { type: 'number', default: 0, description: 'Audio start time (seconds)' },
-      endTime: { type: 'number', default: 0, description: 'Audio end time (seconds)' },
-      maxWaitMs: { type: 'int', default: 300000, description: 'Max poll wait time (ms)' },
-      // Batch params
-      segments: { type: 'any', default: null, description: 'Segments array' },
-      videoMap: { type: 'any', default: null, description: 'Map of promptId → videoUrl' },
-      concurrency: { type: 'int', default: 3, description: 'Max concurrent tasks' },
-    },
-  },
-
-  lifecycle: {
-    validate: (inputs, params) => {
-      if (params.operation === 'validate-tunnel') return !!params.publicBaseUrl;
-      if (!params.replicateToken) return false;
-      if (params.operation === 'process' && (!inputs.videoUrl || !inputs.audioPath)) return false;
-      if (params.operation === 'batch' && (!params.segments || !inputs.audioPath)) return false;
-      return true;
-    },
-
-    cacheKey: (inputs, params) => {
-      return `replicate:${params.operation}:${params.segmentId || ''}`;
-    },
-
-    execute: async (inputs, params) => {
-      try {
-        const op = params.operation;
-
-        if (op === 'validate-tunnel') {
-          const valid = await validateTunnel(params.publicBaseUrl);
-          return { result: { valid }, error: null };
-        }
-
-        if (op === 'process') {
-          const result = await processSegment(inputs, params);
-          return { result, error: null };
-        }
-
-        if (op === 'batch') {
-          const results = await processBatch(inputs, params);
-          return { result: { processed: results.size, results: Object.fromEntries(results) }, error: null };
-        }
-
-        return { result: null, error: `Unknown operation: ${op}` };
-      } catch (err) {
-        return { result: null, error: err.message };
-      }
-    },
-  },
-};
-
-// --- Replicate API ---
-
-/**
- * Create prediction on Replicate
- * @param {string} videoUrl
- * @param {string} audioUrl - Public URL or data URI
- * @param {string} token
- * @returns {Promise<Object>}
- */
-async function createPrediction(videoUrl, audioUrl, token) {
-  const response = await fetch('https://api.replicate.com/v1/predictions', {
-    method: 'POST',
-    headers: {
-      'Authorization': `Bearer ${token}`,
-      'Content-Type': 'application/json',
-      'Prefer': 'wait',
-    },
-    body: JSON.stringify({
-      version: 'kwaivgi/kling-lip-sync',
-      input: {
-        video: videoUrl,
-        audio: audioUrl,
-      },
-    }),
-  });
-
-  if (!response.ok) {
-    const error = await response.text();
-    throw new Error(`Replicate API error ${response.status}: ${error}`);
-  }
-
-  return await response.json();
-}
-
-/**
- * Poll for prediction completion
- * @param {string} predictionId
- * @param {string} token
- * @param {number} maxWaitMs
- * @returns {Promise<Object>}
- */
-async function pollPrediction(predictionId, token, maxWaitMs = 300000) {
-  const startTime = Date.now();
-  const pollInterval = 5000;
-
-  while (Date.now() - startTime < maxWaitMs) {
-    const response = await fetch(`https://api.replicate.com/v1/predictions/${predictionId}`, {
-      headers: { 'Authorization': `Bearer ${token}` },
-    });
-
-    if (!response.ok) {
-      throw new Error(`Replicate poll error: ${response.status}`);
-    }
-
-    const prediction = await response.json();
-
-    if (prediction.status === 'succeeded') {
-      return prediction;
-    }
-    if (prediction.status === 'failed' || prediction.status === 'canceled') {
-      throw new Error(`Replicate prediction ${prediction.status}: ${prediction.error || 'Unknown'}`);
-    }
-
-    await new Promise(resolve => setTimeout(resolve, pollInterval));
-  }
-
-  throw new Error(`Replicate prediction timed out after ${maxWaitMs / 1000}s`);
-}
-
-/**
- * Download result video
- * @param {string} videoUrl
- * @param {string} outputPath
- * @returns {Promise<string>}
- */
-async function downloadResult(videoUrl, outputPath) {
-  const response = await fetch(videoUrl);
-  if (!response.ok) {
-    throw new Error(`Failed to download: ${response.status}`);
-  }
-
-  const buffer = await response.arrayBuffer();
-  await writeFile(outputPath, Buffer.from(buffer));
-  return outputPath;
-}
-
-// --- Utilities ---
-
-/**
- * Convert file to data URI
- * @param {string} filePath
- * @returns {Promise<string>}
- */
-async function fileToDataUri(filePath) {
-  const buffer = await readFile(path.resolve(filePath));
-  const ext = path.extname(filePath).toLowerCase();
-  const mimeTypes = {
-    '.mp3': 'audio/mpeg',
-    '.wav': 'audio/wav',
-    '.mp4': 'video/mp4',
-    '.webm': 'video/webm',
-  };
-  const mime = mimeTypes[ext] || 'application/octet-stream';
-  return `data:${mime};base64,${buffer.toString('base64')}`;
-}
-
-/**
- * Extract audio clip using FFmpeg
- * @param {string} audioPath
- * @param {number} startTime
- * @param {number} endTime
- * @param {string} outputPath
- * @returns {string}
- */
-function extractAudioClip(audioPath, startTime, endTime, outputPath) {
-  if (existsSync(outputPath)) return outputPath;
-
-  const duration = endTime - startTime;
-  const cmd = `ffmpeg -y -i "${path.resolve(audioPath)}" -ss ${startTime.toFixed(3)} -t ${duration.toFixed(3)} ` +
-    `-c:a libmp3lame -q:a 2 "${outputPath}" 2>/dev/null`;
-
-  execSync(cmd, { stdio: 'pipe' });
-  return outputPath;
-}
-
-/**
- * Get public URL for local file
- * @param {string} filePath
- * @param {string} publicBaseUrl
- * @returns {string}
- */
-function getPublicUrl(filePath, publicBaseUrl) {
-  const absPath = path.resolve(filePath);
-  return `${publicBaseUrl}/${encodeURIComponent(absPath)}`;
-}
-
-/**
- * Validate tunnel accessibility
- * @param {string} publicBaseUrl
- * @returns {Promise<boolean>}
- */
-async function validateTunnel(publicBaseUrl) {
-  try {
-    const controller = new AbortController();
-    const timeout = setTimeout(() => controller.abort(), 10000);
-    const response = await fetch(publicBaseUrl, { signal: controller.signal });
-    clearTimeout(timeout);
-    return response.status < 500;
-  } catch {
-    return false;
-  }
-}
-
-// --- Processing pipeline ---
-
-/**
- * Process single segment
- * @param {Object} inputs
- * @param {Object} params
- * @returns {Promise<Object>}
- */
-async function processSegment(inputs, params) {
-  const { segmentId, startTime, endTime, outputDir, replicateToken, publicBaseUrl } = params;
-
-  const lipsyncDir = path.join(outputDir, 'lipsync-videos');
-  const clipsDir = path.join(outputDir, 'audio-clips');
-  await mkdir(lipsyncDir, { recursive: true });
-  await mkdir(clipsDir, { recursive: true });
-
-  const outputPath = path.join(lipsyncDir, `${segmentId}.mp4`);
-  if (existsSync(outputPath)) {
-    return { videoPath: outputPath, cached: true };
-  }
-
-  // Extract audio clip
-  const clipPath = path.join(clipsDir, `${segmentId}.mp3`);
-  extractAudioClip(inputs.audioPath, startTime, endTime, clipPath);
-
-  // Get audio data URI or public URL
-  let audioUrl;
-  if (publicBaseUrl) {
-    audioUrl = getPublicUrl(clipPath, publicBaseUrl);
-  } else {
-    audioUrl = await fileToDataUri(clipPath);
-  }
-
-  // Create prediction
-  const prediction = await createPrediction(inputs.videoUrl, audioUrl, replicateToken);
-
-  // Poll if not already complete
-  let result = prediction;
-  if (prediction.status !== 'succeeded') {
-    result = await pollPrediction(prediction.id, replicateToken, params.maxWaitMs);
-  }
-
-  // Download result
-  const resultUrl = result.output;
-  if (!resultUrl) {
-    throw new Error('No output URL in prediction result');
-  }
-
-  await downloadResult(resultUrl, outputPath);
-  return { videoPath: outputPath, predictionId: prediction.id, cached: false };
-}
-
-/**
- * Process batch with worker pool
- * @param {Object} inputs
- * @param {Object} params
- * @returns {Promise<Map>}
- */
-async function processBatch(inputs, params) {
-  const segments = params.segments;
-  const videoMap = params.videoMap || {};
-  const concurrency = params.concurrency;
-  const results = new Map();
-  const queue = [...segments];
-
-  const workers = Array(Math.min(concurrency, queue.length)).fill(null).map(async () => {
-    while (queue.length > 0) {
-      const segment = queue.shift();
-      if (!segment) break;
-
-      const videoUrl = videoMap[segment.promptId];
-      if (!videoUrl) continue;
-
-      try {
-        const segParams = {
-          ...params,
-          segmentId: segment.promptId,
-          startTime: segment.start,
-          endTime: segment.end,
-        };
-
-        const result = await processSegment(
-          { videoUrl, audioPath: inputs.audioPath },
-          segParams,
-        );
-        results.set(segment.promptId, result.videoPath);
-      } catch (error) {
-        console.error(`[Replicate] Failed: ${segment.promptId} - ${error.message}`);
-      }
-    }
-  });
-
-  await Promise.all(workers);
-  return results;
-}

package/vendor/symbiote-node/engine/packs/ai/tts.handler.js

@@ -1,241 +0,0 @@
-/**
- * ai/tts — Text-to-Speech via Qwen3-TTS
- *
- * Two modes:
- * - SSH: batch script on remote server (mr-agent.rnd-pro.com)
- * - HTTP: POST to Qwen3 TTS HTTP endpoint
- *
- * Supports:
- * - Built-in speakers: ryan, vivian, aiden, dylan, eric, serena, sohee, chelsie, etc.
- * - Voice cloning via ref_audio (reference audio sample)
- * - Language: es (Spanish/Rioplatense), ru (Russian), en (English)
- *
- * Config from Mr-Computer/automations/argentine-spanish-bot:
- *   TTS_SERVER_URL: http://localhost:5008
- *   TTS_VENV_PATH: /home/mr-agent/automations/argentine-spanish-bot/venv
- *   Batch script: utils/generate_qwen3tts_batch.py
- *
- * @module agi-graph/packs/ai/tts
- */
-
-import { execSync } from 'child_process';
-import { promises as fs } from 'fs';
-import path from 'path';
-import os from 'os';
-
-export default {
-  type: 'ai/tts',
-  category: 'ai',
-  icon: 'record_voice_over',
-
-  driver: {
-    description: 'Text-to-Speech via Qwen3-TTS (SSH batch or HTTP)',
-    inputs: [
-      { name: 'text', type: 'string' },
-    ],
-    outputs: [
-      { name: 'audioPath', type: 'string' },
-      { name: 'error', type: 'string' },
-    ],
-    params: {
-      mode: { type: 'string', default: 'http', description: 'ssh | http' },
-      language: { type: 'string', default: 'es', description: 'Language: es, ru, en' },
-      speaker: { type: 'string', default: 'vivian', description: 'Built-in Qwen3 speaker ID' },
-      refAudio: { type: 'string', default: '', description: 'Path to voice reference audio (clone mode)' },
-      outputDir: { type: 'string', default: '', description: 'Output directory for generated audio' },
-      outputFormat: { type: 'string', default: 'wav', description: 'wav | mp3' },
-      exaggeration: { type: 'number', default: 0, description: 'Voice exaggeration (0-1)' },
-      cfg: { type: 'number', default: 0.1, description: 'Classifier-free guidance (0-1)' },
-      // SSH params
-      remoteHost: { type: 'string', default: 'mr-agent@mr-agent.rnd-pro.com', description: 'SSH host' },
-      remotePath: { type: 'string', default: '/home/mr-agent/automations/argentine-spanish-bot', description: 'Remote project path' },
-      remoteVenv: { type: 'string', default: '/home/mr-agent/automations/argentine-spanish-bot/venv', description: 'Remote Python venv path' },
-      device: { type: 'string', default: 'cuda', description: 'cuda | cpu' },
-      // HTTP params
-      endpoint: { type: 'string', default: 'http://localhost:5008', description: 'TTS HTTP endpoint' },
-      timeout: { type: 'int', default: 120000, description: 'Max wait time (ms)' },
-    },
-  },
-
-  lifecycle: {
-    validate: (inputs) => {
-      if (!inputs.text) return false;
-      return true;
-    },
-
-    cacheKey: (inputs, params) =>
-      `tts:${params.mode}:${params.speaker}:${params.language}:${inputs.text}`,
-
-    execute: async (inputs, params) => {
-      const { text } = inputs;
-      const mode = params.mode || 'http';
-
-      if (mode === 'ssh') {
-        return executeSSH(text, params);
-      }
-      return executeHTTP(text, params);
-    },
-  },
-};
-
-/**
- * Qwen3-TTS built-in speaker IDs
- * @type {Set<string>}
- */
-const SPEAKERS = new Set([
-  'aiden', 'dylan', 'eric', 'ono_anna', 'ryan',
-  'serena', 'sohee', 'uncle_fu', 'vivian', 'chelsie',
-]);
-
-/**
- * SSH mode: write batch JSON → scp → remote python exec → scp result back
- * @param {string} text - Text to synthesize
- * @param {Object} params - Node params
- * @returns {Promise<Object>}
- */
-async function executeSSH(text, params) {
-  const host = params.remoteHost || process.env.WHISPER_REMOTE_HOST || 'mr-agent@mr-agent.rnd-pro.com';
-  const remotePath = params.remotePath || process.env.WHISPER_REMOTE_PATH || '/home/mr-agent/automations/argentine-spanish-bot';
-  const venv = params.remoteVenv || process.env.TTS_VENV_PATH || `${remotePath}/venv`;
-  const device = params.device || process.env.PODCAST_TTS_DEVICE || 'cuda';
-
-  const outDir = params.outputDir || path.join(os.tmpdir(), 'agi-graph-tts');
-  const taskId = `tts_${Date.now()}`;
-  const localWav = path.join(outDir, `${taskId}.wav`);
-  const remoteTmpDir = '/tmp/agi-graph-tts';
-
-  try {
-    await fs.mkdir(outDir, { recursive: true });
-
-    // Build batch task
-    const batchTask = [{
-      id: taskId,
-      text,
-      lang: params.language || 'es',
-      prompt: params.refAudio || null,
-      out: `${remoteTmpDir}/${taskId}.wav`,
-      exaggeration: params.exaggeration ?? 0,
-      cfg: params.cfg ?? 0.1,
-    }];
-
-    // If using built-in speaker (no refAudio), add speaker param
-    if (!params.refAudio && SPEAKERS.has(params.speaker)) {
-      batchTask[0].speaker = params.speaker;
-    }
-
-    // Write local batch file
-    const batchFile = path.join(outDir, `${taskId}_batch.json`);
-    await fs.writeFile(batchFile, JSON.stringify(batchTask, null, 2));
-
-    // Ensure remote dir + upload batch
-    execSync(`ssh ${host} "mkdir -p ${remoteTmpDir}"`, {
-      encoding: 'utf-8', stdio: 'pipe', timeout: 10000,
-    });
-
-    const remoteBatch = `${remoteTmpDir}/${taskId}_batch.json`;
-    execSync(`scp "${batchFile}" "${host}:${remoteBatch}"`, {
-      encoding: 'utf-8', stdio: 'pipe', timeout: 30000,
-    });
-
-    try {
-      // Run batch script
-      const pythonCmd = `${venv}/bin/python`;
-      const scriptPath = `${remotePath}/utils/generate_qwen3tts_batch.py`;
-      const cmd = `source "${venv}/bin/activate" && "${pythonCmd}" "${scriptPath}" --batch "${remoteBatch}" --device "${device}"`;
-
-      execSync(`ssh ${host} '${cmd}'`, {
-        encoding: 'utf-8',
-        maxBuffer: 50 * 1024 * 1024,
-        timeout: params.timeout || 120000,
-      });
-
-      // Download result
-      const remoteOut = `${remoteTmpDir}/${taskId}.wav`;
-      execSync(`scp "${host}:${remoteOut}" "${localWav}"`, {
-        encoding: 'utf-8', stdio: 'pipe', timeout: 30000,
-      });
-
-      // Cleanup batch + remote output
-      await fs.unlink(batchFile).catch(() => { });
-      execSync(`ssh ${host} "rm -f ${remoteBatch} ${remoteOut}"`, {
-        encoding: 'utf-8', stdio: 'pipe', timeout: 5000,
-      }).toString();
-
-      return { audioPath: localWav, error: null };
-
-    } catch (err) {
-      await fs.unlink(batchFile).catch(() => { });
-      return { audioPath: null, error: err.message };
-    }
-
-  } catch (err) {
-    return { audioPath: null, error: err.message };
-  }
-}
-
-/**
- * HTTP mode: POST to Qwen3 TTS endpoint
- * @param {string} text - Text to synthesize
- * @param {Object} params - Node params
- * @returns {Promise<Object>}
- */
-async function executeHTTP(text, params) {
-  const endpoint = params.endpoint || process.env.TTS_SERVER_URL || 'http://localhost:5008';
-  const outDir = params.outputDir || path.join(os.tmpdir(), 'agi-graph-tts');
-  const taskId = `tts_${Date.now()}`;
-  const outputPath = path.join(outDir, `${taskId}.wav`);
-
-  try {
-    await fs.mkdir(outDir, { recursive: true });
-
-    const body = {
-      text,
-      language: params.language || 'es',
-      speaker: params.speaker || 'vivian',
-      exaggeration: params.exaggeration ?? 0,
-      cfg: params.cfg ?? 0.1,
-    };
-
-    // Add ref_audio for voice cloning
-    if (params.refAudio) {
-      const refBuffer = await fs.readFile(params.refAudio);
-      body.ref_audio = refBuffer.toString('base64');
-    }
-
-    const response = await fetch(`${endpoint}/synthesize`, {
-      method: 'POST',
-      headers: { 'Content-Type': 'application/json' },
-      body: JSON.stringify(body),
-      signal: AbortSignal.timeout(params.timeout || 120000),
-    });
-
-    if (!response.ok) {
-      return { audioPath: null, error: `TTS API error: ${response.status}` };
-    }
-
-    // Response is audio binary
-    const contentType = response.headers.get('content-type') || '';
-
-    if (contentType.includes('audio') || contentType.includes('octet-stream')) {
-      const buffer = Buffer.from(await response.arrayBuffer());
-      await fs.writeFile(outputPath, buffer);
-      return { audioPath: outputPath, error: null };
-    }
-
-    // JSON response with file path
-    const result = await response.json();
-    if (result.audio_path) {
-      return { audioPath: result.audio_path, error: null };
-    }
-    if (result.audio) {
-      const buffer = Buffer.from(result.audio, 'base64');
-      await fs.writeFile(outputPath, buffer);
-      return { audioPath: outputPath, error: null };
-    }
-
-    return { audioPath: null, error: 'Unexpected TTS response format' };
-
-  } catch (err) {
-    return { audioPath: null, error: err.message };
-  }
-}