voxflow 1.15.3 → 1.15.5

package/lib/stage-core/server.js

@@ -65,6 +65,8 @@ async function startStageServer(opts) {
     cloudRender = null,
     localRender = null,
     deckSaver = null,
+    audition = null,
+    imagine = null,
     publishEvent = null,
     tokenAvailable = false,
     preferredPort = 5180,
@@ -243,8 +245,22 @@ async function startStageServer(opts) {
     function statusForCode(code) {
       if (code === 'not_logged_in') return 401;
       if (code === 'quota_exceeded') return 402;
-      if (code === 'invalid_deck' || code === 'invalid_id') return 400;
-      if (code === 'job_not_found') return 404;
+      if (
+        code === 'invalid_deck' ||
+        code === 'invalid_id' ||
+        code === 'invalid_card_index' ||
+        code === 'invalid_voice' ||
+        code === 'invalid_text' ||
+        code === 'invalid_prompt' ||
+        code === 'invalid_image_id'
+      ) return 400;
+      if (
+        code === 'job_not_found' ||
+        code === 'no_deck' ||
+        code === 'card_not_found' ||
+        code === 'image_not_found'
+      ) return 404;
+      if (code === 'voiceover_disabled') return 409;
       if (code === 'success') return 200;
       return 502; // upstream error
     }
@@ -403,6 +419,98 @@ async function startStageServer(opts) {
       return;
     }
 
+    // ─── TTS audition (per-card ▶) ──────────────────────────────────────────
+    // GET /api/audition?card=<int>[&voice=<id>]
+    //   Resolves card.voiceover/voiceId/narration → calls /api/tts/synthesize
+    //   via the audition bridge → streams audio bytes (default mp3). Content
+    //   hash caches identical (voice, text, speed, format) so iteration is
+    //   free after the first call. The page never sees the JWT.
+    if (audition && req.method === 'GET' && req.url.startsWith('/api/audition')) {
+      let parsed;
+      try { parsed = new URL(req.url, `http://127.0.0.1:${port}`); }
+      catch {
+        return sendJson(400, { code: 'bad_request', message: 'invalid /api/audition url' });
+      }
+      const cardIndexStr = parsed.searchParams.get('card');
+      const cardIndex = Number.parseInt(cardIndexStr, 10);
+      if (!Number.isInteger(cardIndex) || cardIndex < 0) {
+        return sendJson(400, {
+          code: 'invalid_card_index',
+          message: '?card= must be a non-negative integer',
+        });
+      }
+      const voiceOverride = parsed.searchParams.get('voice') || undefined;
+      (async () => {
+        let result;
+        try {
+          result = await audition.play({ cardIndex, voiceOverride });
+        } catch (err) {
+          return sendJson(502, { code: 'upstream_error', message: err.message });
+        }
+        if (result.code !== 'success') {
+          return sendJson(statusForCode(result.code), result);
+        }
+        res.writeHead(200, {
+          'Content-Type': result.contentType || 'audio/mpeg',
+          'Content-Length': result.buf.length,
+          'Cache-Control': 'no-store',
+          'X-Audition-Cache': result.fromCache ? 'HIT' : 'MISS',
+          'X-Audition-Key': result.cacheKey || '',
+        });
+        res.end(result.buf);
+      })();
+      return;
+    }
+
+    // ─── Image generation (per-card 🎨) ─────────────────────────────────────
+    // GET /api/imagine?card=<int>&img=<id>
+    //   Resolves card.images[?] → calls /api/image/generate via the imagine
+    //   bridge → streams PNG/JPG bytes. Content-hash caches identical
+    //   (prompt, aspect, quality) so iterating on visuals is free after the
+    //   first call. Symmetric with /api/audition.
+    if (imagine && req.method === 'GET' && req.url.startsWith('/api/imagine')) {
+      let parsed;
+      try { parsed = new URL(req.url, `http://127.0.0.1:${port}`); }
+      catch {
+        return sendJson(400, { code: 'bad_request', message: 'invalid /api/imagine url' });
+      }
+      const cardIndexStr = parsed.searchParams.get('card');
+      const cardIndex = Number.parseInt(cardIndexStr, 10);
+      if (!Number.isInteger(cardIndex) || cardIndex < 0) {
+        return sendJson(400, {
+          code: 'invalid_card_index',
+          message: '?card= must be a non-negative integer',
+        });
+      }
+      const imageId = parsed.searchParams.get('img') || '';
+      if (typeof imageId !== 'string' || !imageId.trim()) {
+        return sendJson(400, {
+          code: 'invalid_image_id',
+          message: '?img= must be a non-empty image id',
+        });
+      }
+      (async () => {
+        let result;
+        try {
+          result = await imagine.play({ cardIndex, imageId });
+        } catch (err) {
+          return sendJson(502, { code: 'upstream_error', message: err.message });
+        }
+        if (result.code !== 'success') {
+          return sendJson(statusForCode(result.code), result);
+        }
+        res.writeHead(200, {
+          'Content-Type': result.contentType || 'image/png',
+          'Content-Length': result.buf.length,
+          'Cache-Control': 'no-store',
+          'X-Imagine-Cache': result.fromCache ? 'HIT' : 'MISS',
+          'X-Imagine-Key': result.cacheKey || '',
+        });
+        res.end(result.buf);
+      })();
+      return;
+    }
+
     // ─── Inline deck save (Task B) ──────────────────────────────────────────
     // POST /api/deck  body: full deck JSON  → validates + writes to disk.
     // The file watcher picks up the write and broadcasts the deck event, so

package/lib/stage-core/voiceover-mux.js

@@ -0,0 +1,290 @@
+'use strict';
+
+/**
+ * Per-card voiceover audio prep for local Remotion render.
+ *
+ * Reuses the audition cache (~/.config/voxflow/stage-tts-cache/) so a card
+ * the user just listened to via stage's ▶ button doesn't get re-synthesized
+ * at render time. Spins up a tiny localhost HTTP server (auto-picked port)
+ * that serves audio files to the headless Chromium Remotion launches; the
+ * Remotion composition fetches voiceoverSrc URLs from this server while
+ * rendering. Tear the server down after renderMedia() resolves.
+ *
+ *   const aud = createTtsAuditionClient();
+ *   const server = await startVoiceoverServer({ cacheDir: aud.cacheDir });
+ *   const { byIdx, skipped } = await prepareVoiceovers({
+ *     deck, auditionClient: aud, baseUrl: server.url, onProgress,
+ *   });
+ *   // buildInputProps reads byIdx and threads URLs into card.slide.voiceoverSrc
+ *   await renderMedia({ inputProps: buildInputProps(deck, { voiceoverByIdx: byIdx }), ... });
+ *   await server.close();
+ *
+ * When auth is unavailable (no token in CLI cache), prepareVoiceovers
+ * returns an empty map quietly — the resulting mp4 is the Phase 0 silent
+ * video. Callers branch on the empty map to surface a hint to the user.
+ */
+
+const fs = require('fs');
+const http = require('http');
+const path = require('path');
+
+const { contentTypeFor } = require('./tts-audition');
+const { SYNTHESIZE_DEFAULTS } = require('../core/config');
+
+// Mime mapper covering both audio + image extensions so the local media
+// server can serve cached audition mp3s and generated PNG/JPEG/WebP from a
+// single process. Anything unrecognised becomes octet-stream — Remotion's
+// fetch still goes through but the composition decides what to do.
+const MIME_BY_EXT = {
+  mp3: 'audio/mpeg',
+  wav: 'audio/wav',
+  pcm: 'audio/L16',
+  png: 'image/png',
+  jpg: 'image/jpeg',
+  jpeg: 'image/jpeg',
+  webp: 'image/webp',
+};
+function mimeFor(ext) {
+  if (!ext) return 'application/octet-stream';
+  return MIME_BY_EXT[ext.toLowerCase()] || contentTypeFor(ext);
+}
+
+function serveFileFrom(rootDir, urlFname, res) {
+  const fname = urlFname.split('?')[0];
+  if (fname === '' || fname.includes('/') || fname.includes('\\') || fname.includes('..')) {
+    res.writeHead(400, { 'Content-Type': 'text/plain' });
+    res.end('bad filename');
+    return;
+  }
+  const filePath = path.join(rootDir, fname);
+  fs.stat(filePath, (statErr, st) => {
+    if (statErr || !st.isFile()) {
+      res.writeHead(404, { 'Content-Type': 'text/plain' });
+      res.end('not found');
+      return;
+    }
+    const ext = path.extname(fname).slice(1);
+    res.writeHead(200, {
+      'Content-Type': mimeFor(ext),
+      'Content-Length': st.size,
+      'Cache-Control': 'no-store',
+    });
+    fs.createReadStream(filePath).pipe(res);
+  });
+}
+
+/**
+ * Tiny localhost HTTP server serving the audition + image-gen caches.
+ * Responds to GET /audio/<filename> and (optionally) GET /image/<filename>;
+ * everything else is 404. Path traversal is rejected — the cache layouts
+ * are intentionally flat <sha256>.<ext> files.
+ *
+ * @param {object} opts
+ * @param {string} opts.cacheDir          Directory containing audio <hash>.<ext> files.
+ * @param {string} [opts.imageCacheDir]   Optional second root mounted at /image/.
+ * @param {number} [opts.preferredPort=0]  0 lets the OS pick a free port.
+ * @returns {Promise<{server, port, url, close}>}
+ */
+async function startVoiceoverServer({ cacheDir, imageCacheDir = null, preferredPort = 0 }) {
+  if (typeof cacheDir !== 'string' || !cacheDir) {
+    throw new Error('startVoiceoverServer: cacheDir required');
+  }
+  const server = http.createServer((req, res) => {
+    if (req.method !== 'GET') {
+      res.writeHead(404, { 'Content-Type': 'text/plain' });
+      res.end('not found');
+      return;
+    }
+    if (req.url.startsWith('/audio/')) {
+      return serveFileFrom(cacheDir, req.url.slice('/audio/'.length), res);
+    }
+    if (imageCacheDir && req.url.startsWith('/image/')) {
+      return serveFileFrom(imageCacheDir, req.url.slice('/image/'.length), res);
+    }
+    res.writeHead(404, { 'Content-Type': 'text/plain' });
+    res.end('not found');
+  });
+  await new Promise((resolve, reject) => {
+    server.once('error', reject);
+    server.listen(preferredPort, '127.0.0.1', () => {
+      server.removeListener('error', reject);
+      resolve();
+    });
+  });
+  const port = server.address().port;
+  return {
+    server,
+    port,
+    url: `http://127.0.0.1:${port}`,
+    async close() {
+      await new Promise((resolve) => server.close(() => resolve()));
+    },
+  };
+}
+
+/**
+ * Resolve + synthesize (or cache-hit) one mp3 per card, return a map of
+ * { cardIdx: audio URL } that buildInputProps threads into voiceoverSrc.
+ *
+ * @param {object} opts
+ * @param {object} opts.deck                       Validator-shaped deck.
+ * @param {{audition: Function}} opts.auditionClient
+ *   Same client stage's /api/audition uses. Shares the on-disk cache so a
+ *   card the user previewed in the browser doesn't burn quota again at
+ *   render time.
+ * @param {string} opts.baseUrl                    e.g. http://127.0.0.1:54321
+ * @param {(p:object) => void} [opts.onProgress]
+ *   Called once per resolved card: { cardIdx, total, fromCache, voiceId, textLen }.
+ *   Use this to print a one-line "voiceover N/M (cache hit)" log so the
+ *   user knows TTS is happening before the renderer takes over.
+ * @returns {Promise<{ byIdx: Record<number,string>, skipped: Array<{cardIdx, reason, message?}> }>}
+ *   skipped reasons: missing-card | voiceover-disabled | no-text |
+ *   not_logged_in | quota_exceeded | tts_failed | network_error | invalid_voice
+ */
+async function prepareVoiceovers({ deck, auditionClient, baseUrl, onProgress }) {
+  const byIdx = {};
+  const skipped = [];
+  if (!deck || !Array.isArray(deck.cards)) return { byIdx, skipped };
+  if (!auditionClient || typeof auditionClient.audition !== 'function') {
+    throw new Error('prepareVoiceovers: auditionClient.audition is required');
+  }
+  if (typeof baseUrl !== 'string' || !baseUrl) {
+    throw new Error('prepareVoiceovers: baseUrl is required');
+  }
+  const cards = deck.cards;
+
+  for (let i = 0; i < cards.length; i++) {
+    const card = cards[i];
+    if (!card) { skipped.push({ cardIdx: i, reason: 'missing-card' }); continue; }
+    const vo = card.voiceover || {};
+    if (vo.enabled === false) {
+      skipped.push({ cardIdx: i, reason: 'voiceover-disabled' });
+      continue;
+    }
+    const text = (typeof vo.text === 'string' && vo.text.trim())
+      ? vo.text
+      : card.narration;
+    if (typeof text !== 'string' || !text.trim()) {
+      skipped.push({ cardIdx: i, reason: 'no-text' });
+      continue;
+    }
+    const voiceId = vo.voiceId || card.voiceId || SYNTHESIZE_DEFAULTS.voice;
+    const speed = typeof vo.rate === 'number' ? vo.rate : 1.0;
+    const format = 'mp3';
+
+    let r;
+    try {
+      r = await auditionClient.audition({ voiceId, text, speed, format });
+    } catch (err) {
+      skipped.push({ cardIdx: i, reason: 'network_error', message: err.message || String(err) });
+      continue;
+    }
+    if (r.code !== 'success') {
+      skipped.push({ cardIdx: i, reason: r.code, message: r.message });
+      // not_logged_in / quota_exceeded → bail early so the user sees one
+      // clear message rather than N copies of the same root cause.
+      if (r.code === 'not_logged_in' || r.code === 'quota_exceeded') break;
+      continue;
+    }
+    const fname = `${r.cacheKey}.${format}`;
+    byIdx[i] = `${baseUrl.replace(/\/$/, '')}/audio/${fname}`;
+    if (typeof onProgress === 'function') {
+      try {
+        onProgress({
+          cardIdx: i,
+          total: cards.length,
+          fromCache: !!r.fromCache,
+          voiceId,
+          textLen: text.length,
+        });
+      } catch { /* swallow consumer errors */ }
+    }
+  }
+
+  return { byIdx, skipped };
+}
+
+/**
+ * Image counterpart of prepareVoiceovers — resolves the primary image
+ * for each card. Precedence per card:
+ *   (a) card.imageUrl present → use as-is, no API call (external asset)
+ *   (b) card.images[0] present → call image-gen client, route through
+ *       the local server's /image/ mount, return URL.
+ * Returns { byIdx: { cardIdx → URL }, skipped: [{cardIdx, reason, message?}] }.
+ *
+ * @param {object} opts
+ * @param {object} opts.deck
+ * @param {{imagine: Function}} opts.imgClient
+ * @param {string} opts.baseUrl                 e.g. http://127.0.0.1:54321
+ * @param {(p:object) => void} [opts.onProgress]
+ */
+async function prepareImages({ deck, imgClient, baseUrl, onProgress }) {
+  const byIdx = {};
+  const skipped = [];
+  if (!deck || !Array.isArray(deck.cards)) return { byIdx, skipped };
+  if (!imgClient || typeof imgClient.imagine !== 'function') {
+    throw new Error('prepareImages: imgClient.imagine is required');
+  }
+  if (typeof baseUrl !== 'string' || !baseUrl) {
+    throw new Error('prepareImages: baseUrl is required');
+  }
+  const cards = deck.cards;
+
+  for (let i = 0; i < cards.length; i++) {
+    const card = cards[i];
+    if (!card) { skipped.push({ cardIdx: i, reason: 'missing-card' }); continue; }
+
+    // External imageUrl wins — themes that already consume it (photo-feature
+    // / atmospheric) keep working unchanged.
+    if (typeof card.imageUrl === 'string' && card.imageUrl.trim()) {
+      byIdx[i] = card.imageUrl;
+      continue;
+    }
+
+    const images = Array.isArray(card.images) ? card.images : [];
+    const primary = images[0];
+    if (!primary) {
+      skipped.push({ cardIdx: i, reason: 'no-images' });
+      continue;
+    }
+
+    let r;
+    try {
+      r = await imgClient.imagine({
+        prompt: primary.prompt,
+        aspect: primary.aspect,
+        quality: primary.quality,
+      });
+    } catch (err) {
+      skipped.push({ cardIdx: i, reason: 'network_error', message: err.message || String(err) });
+      continue;
+    }
+    if (r.code !== 'success') {
+      skipped.push({ cardIdx: i, reason: r.code, message: r.message });
+      if (r.code === 'not_logged_in' || r.code === 'quota_exceeded') break;
+      continue;
+    }
+    const fname = `${r.cacheKey}.${r.ext}`;
+    byIdx[i] = `${baseUrl.replace(/\/$/, '')}/image/${fname}`;
+    if (typeof onProgress === 'function') {
+      try {
+        onProgress({
+          cardIdx: i,
+          total: cards.length,
+          fromCache: !!r.fromCache,
+          prompt: primary.prompt,
+          aspect: primary.aspect,
+        });
+      } catch { /* swallow */ }
+    }
+  }
+
+  return { byIdx, skipped };
+}
+
+module.exports = {
+  startVoiceoverServer,
+  prepareVoiceovers,
+  prepareImages,
+  mimeFor,
+};