voila-recorder 0.4.0

package/render.js

@@ -0,0 +1,177 @@
+// Renderer: turns captured frames + cursor/zoom timeline into a smooth MP4.
+// For each output frame we compute the eased zoom level and a camera center
+// following the cursor, crop the 2x-DPR source frame, and pipe JPEGs to ffmpeg.
+
+const fs = require('fs');
+const path = require('path');
+const { spawn } = require('child_process');
+const sharp = require('sharp');
+const ffmpegPath = require('ffmpeg-static');
+
+const cubicInOut = p => (p < 0.5 ? 4 * p * p * p : 1 - Math.pow(-2 * p + 2, 3) / 2);
+
+// Piecewise value from timeline events [{t, from, to, dur}] at time t.
+function scalarAt(events, t, initial) {
+  let v = initial;
+  for (const e of events) {
+    if (t < e.t) break;
+    v = e.from + (e.to - e.from) * cubicInOut(Math.min(1, (t - e.t) / e.dur));
+  }
+  return v;
+}
+
+function pointAt(events, t, initial) {
+  let v = { ...initial };
+  for (const e of events) {
+    if (t < e.t) break;
+    const p = cubicInOut(Math.min(1, (t - e.t) / e.dur));
+    v = { x: e.from.x + (e.to.x - e.from.x) * p, y: e.from.y + (e.to.y - e.from.y) * p };
+  }
+  return v;
+}
+
+const CAPTION_MAX_MS = 7000;
+
+function captionAt(segments, t) {
+  let active = null;
+  for (const s of segments || []) {
+    if (s.t > t) break;
+    active = s;
+  }
+  if (!active || !active.caption) return null;
+  // Caption lifetime follows the narration clip when its duration is known.
+  const windowMs = active.dur ? active.dur + 500 : CAPTION_MAX_MS;
+  return t - active.t < windowMs ? active.caption : null;
+}
+
+const escapeXml = s => s.replace(/[<>&'"]/g, c =>
+  ({ '<': '&lt;', '>': '&gt;', '&': '&amp;', "'": '&apos;', '"': '&quot;' }[c]));
+
+function wrapText(text, maxChars = 46) {
+  const words = text.split(/\s+/);
+  const lines = [''];
+  for (const w of words) {
+    const cur = lines[lines.length - 1];
+    if (cur && (cur + ' ' + w).length > maxChars) lines.push(w);
+    else lines[lines.length - 1] = cur ? cur + ' ' + w : w;
+  }
+  return lines.slice(0, 2);
+}
+
+async function captionOverlay(text, outW) {
+  const fontSize = Math.max(19, Math.min(34, Math.round(outW * 0.0177)));
+  const maxChars = Math.min(46, Math.floor((outW * 0.85) / (fontSize * 0.56)));
+  const lines = wrapText(text, maxChars);
+  const lineH = Math.round(fontSize * 1.35), padX = Math.round(fontSize * 0.88), padY = Math.round(fontSize * 0.59);
+  const boxW = Math.min(outW - 80, Math.max(...lines.map(l => l.length)) * fontSize * 0.56 + padX * 2);
+  const boxH = lines.length * lineH + padY * 2 - 8;
+  const svg = `<svg width="${Math.round(boxW)}" height="${boxH}" xmlns="http://www.w3.org/2000/svg">
+    <rect x="0" y="0" width="${Math.round(boxW)}" height="${boxH}" rx="14" fill="rgba(10,10,14,0.74)"/>
+    ${lines.map((l, i) =>
+      `<text x="50%" y="${padY + (i + 0.78) * lineH - Math.round(fontSize * 0.29)}" text-anchor="middle" fill="#ffffff"
+        font-family="Helvetica, Arial, sans-serif" font-size="${fontSize}" font-weight="600">${escapeXml(l)}</text>`
+    ).join('')}
+  </svg>`;
+  return {
+    input: await sharp(Buffer.from(svg)).png().toBuffer(),
+    width: Math.round(boxW),
+    height: boxH,
+  };
+}
+
+async function render(meta, outFile, { fps = 30, outW = null, outH = null, onStatus = () => {} } = {}) {
+  const { frames, moves, zooms, tStart, tEnd, viewport, framesDir } = meta;
+  if (!frames.length) throw new Error('no frames captured');
+
+  // Output follows the recorded viewport's aspect (portrait for mobile).
+  // 1.5x the CSS viewport, rounded to even for yuv420p (desktop → 1920x1200).
+  const even = n => Math.round(n / 2) * 2;
+  if (!outW) outW = even(viewport.width * 1.5);
+  if (!outH) outH = even(viewport.height * 1.5);
+
+  // Actual encoded frame size (screencast can letterbox/scale).
+  const first = await sharp(path.join(framesDir, frames[0].file)).metadata();
+  const W = first.width, H = first.height;
+  const sx = W / viewport.width, sy = H / viewport.height;
+
+  const ffmpeg = spawn(ffmpegPath, [
+    '-y', '-f', 'image2pipe', '-framerate', String(fps), '-i', 'pipe:0',
+    '-c:v', 'libx264', '-pix_fmt', 'yuv420p', '-crf', '18', '-preset', 'veryfast',
+    '-movflags', '+faststart', outFile,
+  ], { stdio: ['pipe', 'ignore', 'pipe'] });
+
+  let ffErr = '';
+  ffmpeg.stderr.on('data', d => { ffErr += d; if (ffErr.length > 20000) ffErr = ffErr.slice(-10000); });
+  const done = new Promise((res, rej) => {
+    ffmpeg.on('close', code => (code === 0 ? res() : rej(new Error(`ffmpeg exited ${code}\n${ffErr.slice(-2000)}`))));
+    ffmpeg.on('error', rej);
+  });
+
+  const writeFrame = buf => new Promise((res, rej) => {
+    ffmpeg.stdin.write(buf, err => (err ? rej(err) : res()));
+  });
+
+  const total = Math.max(1, Math.floor(((tEnd - tStart) / 1000) * fps));
+  let frameIdx = 0;
+  let lastKey = null, lastBuf = null;
+  let srcCache = { file: null, img: null };
+  const overlayCache = new Map();
+
+  for (let i = 0; i < total; i++) {
+    const t = tStart + (i * 1000) / fps;
+
+    while (frameIdx + 1 < frames.length && frames[frameIdx + 1].t <= t) frameIdx++;
+    const srcFile = frames[frameIdx].file;
+
+    const z = Math.max(1, Math.min(3, scalarAt(zooms, t, 1)));
+    const cur = pointAt(moves, t, { x: viewport.width / 2, y: viewport.height / 2 });
+    const caption = captionAt(meta.segments, t);
+
+    const cropW = W / z, cropH = H / z;
+    let cx = cur.x * sx, cy = cur.y * sy;
+    cx = Math.max(cropW / 2, Math.min(W - cropW / 2, cx));
+    cy = Math.max(cropH / 2, Math.min(H - cropH / 2, cy));
+
+    const left = Math.max(0, Math.min(W - Math.round(cropW), Math.round(cx - cropW / 2)));
+    const top = Math.max(0, Math.min(H - Math.round(cropH), Math.round(cy - cropH / 2)));
+
+    const key = `${srcFile}|${left}|${top}|${Math.round(cropW)}|${caption || ''}`;
+    if (key === lastKey && lastBuf) {
+      await writeFrame(lastBuf);
+      continue;
+    }
+
+    if (srcCache.file !== srcFile) {
+      srcCache = { file: srcFile, img: await fs.promises.readFile(path.join(framesDir, srcFile)) };
+    }
+
+    let img = sharp(srcCache.img)
+      .extract({ left, top, width: Math.round(cropW), height: Math.round(cropH) })
+      .resize(outW, outH, { fit: 'fill' });
+
+    if (caption) {
+      if (!overlayCache.has(caption)) overlayCache.set(caption, await captionOverlay(caption, outW));
+      const ov = overlayCache.get(caption);
+      img = sharp(await img.toBuffer()).composite([{
+        input: ov.input,
+        left: Math.round((outW - ov.width) / 2),
+        top: outH - ov.height - 46,
+      }]);
+    }
+
+    const buf = await img.jpeg({ quality: 93 }).toBuffer();
+
+    lastKey = key;
+    lastBuf = buf;
+    await writeFrame(buf);
+
+    if (i % (fps * 2) === 0) onStatus(`rendering ${Math.round((i / total) * 100)}%`);
+  }
+
+  ffmpeg.stdin.end();
+  await done;
+  onStatus('render complete');
+  return outFile;
+}
+
+module.exports = { render };

package/review.js

@@ -0,0 +1,66 @@
+// Review: turn a finished demo back into something an agent can look at.
+// Extracts evenly spaced frames, the embedded recipe, and segment timings —
+// the agent inspects the frames, patches the steps YAML, and re-records.
+
+const fs = require('fs');
+const path = require('path');
+const { execFile } = require('child_process');
+const ffmpegPath = require('ffmpeg-static');
+
+// ffmpeg -i with no output exits non-zero by design; capture both streams and
+// let callers pattern-match (duration lives on stderr, ffmetadata on stdout).
+const ff = args => new Promise(res => {
+  execFile(ffmpegPath, args, { maxBuffer: 1e7 }, (_err, stdout, stderr) =>
+    res({ stdout: String(stdout || ''), stderr: String(stderr || '') }));
+});
+
+async function videoInfo(videoPath) {
+  const info = await ff(['-i', videoPath]);
+  const m = /Duration:\s*(\d+):(\d+):(\d+)\.(\d+)/.exec(info.stderr);
+  const durMs = m ? (+m[1] * 3600 + +m[2] * 60 + +m[3]) * 1000 + +m[4] * 10 : 0;
+  const meta = await ff(['-i', videoPath, '-f', 'ffmetadata', '-']);
+  const r = /voila-recipe:(\{.*)/.exec(meta.stdout);
+  let recipe = null;
+  if (r) { try { recipe = JSON.parse(r[1].split('\n')[0].replace(/\\(.)/g, '$1')); } catch { /* unparseable */ } }
+  return { durMs, recipe };
+}
+
+async function reviewDemo(videoPath, { count = 12, outDir = null } = {}) {
+  if (!fs.existsSync(videoPath)) throw new Error(`video not found: ${videoPath}`);
+  const { durMs, recipe } = await videoInfo(videoPath);
+  if (!durMs) throw new Error('could not read video duration');
+
+  outDir = outDir || path.join(path.dirname(videoPath), 'review');
+  fs.mkdirSync(outDir, { recursive: true });
+
+  const frames = [];
+  for (let i = 0; i < count; i++) {
+    const atSec = +(((i + 0.5) * durMs) / count / 1000).toFixed(1);
+    const file = path.join(outDir, `frame-${String(atSec).padStart(5, '0')}s.png`);
+    await new Promise((res, rej) => {
+      execFile(ffmpegPath, ['-y', '-ss', String(atSec), '-i', videoPath, '-vframes', '1', file],
+        err => (err ? rej(err) : res()));
+    });
+    frames.push({ atSec, file });
+  }
+
+  // Segment timings from the sibling meta.json when available (richer than the recipe).
+  let segments = recipe ? recipe.segments || null : null;
+  const metaPath = path.join(path.dirname(videoPath), 'meta.json');
+  let warnings = [];
+  if (fs.existsSync(metaPath)) {
+    try {
+      const meta = JSON.parse(fs.readFileSync(metaPath, 'utf8'));
+      warnings = meta.warnings || [];
+      segments = (meta.segments || []).map(s => ({
+        at: +((s.t - meta.tStart) / 1000).toFixed(2),
+        caption: s.caption, narration: s.narration,
+        durSec: s.dur ? +(s.dur / 1000).toFixed(2) : null,
+      }));
+    } catch { /* keep recipe segments */ }
+  }
+
+  return { video: videoPath, durationSec: +(durMs / 1000).toFixed(1), frames, segments, recipe, warnings };
+}
+
+module.exports = { reviewDemo };

package/server.js

@@ -0,0 +1,74 @@
+// Voila server: one-click UI + API.
+// POST /api/open   {url}         -> opens the recording browser so the user can sign in
+// POST /api/record {url, mode}   -> runs tour + render, job tracked in /api/status
+// GET  /api/status               -> {state, detail, video?}
+
+const path = require('path');
+const fs = require('fs');
+const express = require('express');
+const yaml = require('js-yaml');
+const { VoilaSession } = require('./recorder');
+const { produceDemo } = require('./pipeline');
+
+const PORT = process.env.PORT || 4477;
+const HEADLESS = process.env.VOILA_HEADLESS === '1';
+const RECORDINGS = path.join(__dirname, 'recordings');
+
+const app = express();
+app.use(express.json());
+app.use(express.static(path.join(__dirname, 'public')));
+app.use('/videos', express.static(RECORDINGS));
+
+let session = new VoilaSession({ headless: HEADLESS });
+let job = { state: 'idle', detail: '', video: null };
+
+async function sessionFor(device = 'desktop') {
+  if (session.device !== device) {
+    await session.close();
+    session = new VoilaSession({ headless: HEADLESS, device });
+  }
+  return session;
+}
+
+app.post('/api/open', async (req, res) => {
+  try {
+    const { url, device } = req.body;
+    if (!url) return res.status(400).json({ error: 'url required' });
+    await (await sessionFor(device || 'desktop')).open(url);
+    res.json({ ok: true, message: 'Browser open — sign in there if the site needs it, then hit Record.' });
+  } catch (e) {
+    res.status(500).json({ error: String(e.message || e) });
+  }
+});
+
+app.post('/api/record', async (req, res) => {
+  if (job.state === 'working') return res.status(409).json({ error: 'a recording is already in progress' });
+  const { url, mode = 'auto', stepsYaml = null } = req.body;
+  if (!url) return res.status(400).json({ error: 'url required' });
+
+  const id = `demo-${Date.now()}`;
+  const workDir = path.join(RECORDINGS, id);
+  job = { state: 'working', detail: 'starting', video: null };
+  res.json({ ok: true, id });
+
+  (async () => {
+    try {
+      const steps = stepsYaml ? yaml.load(stepsYaml) : null;
+      await produceDemo(await sessionFor(req.body.device || 'desktop'), {
+        url, mode, steps, workDir,
+        narrate: req.body.narrate !== false,
+        voice: req.body.voice || null,
+        onStatus: d => { job.detail = d; },
+      });
+      job = { state: 'done', detail: 'ready', video: `/videos/${id}/demo.mp4` };
+    } catch (e) {
+      job = { state: 'error', detail: String(e.message || e), video: null };
+    }
+  })();
+});
+
+app.get('/api/status', (_req, res) => res.json(job));
+
+app.listen(PORT, () => {
+  console.log(`voila running at http://localhost:${PORT} (headless=${HEADLESS})`);
+});

package/skills/voila/SKILL.md

@@ -0,0 +1,59 @@
+---
+name: voila
+description: Record a narrated, auto-zoomed product demo video of any website — no screen-recording permission, fully on-device. Use when the user asks for a product demo video, a site walkthrough recording, a narrated tour of a web app, or to recreate/fork a demo from a voila MP4/recipe. Plans with a page outline, scripts steps YAML (captions + narration + title slides), records, then self-reviews frames and iterates.
+---
+
+# voila — agent-recorded product demos
+
+voila renders a site in its own Chromium and records the viewport (CDP
+screencast) — the user never grants screen-recording permission and nothing
+leaves the machine. ffmpeg is bundled, the Kokoro TTS model and Chromium
+download themselves on first use. The user installs nothing (Node >= 20).
+
+Prefer the MCP tools if registered (`voila_outline`, `voila_record`,
+`voila_review`); otherwise use the CLI via npx:
+
+```bash
+npx -y voila-recorder outline <url>
+npx -y voila-recorder record <url> --steps steps.yaml [--device mobile]
+npx -y voila-recorder review demo.mp4
+```
+
+Register the MCP server once with: `claude mcp add voila -- npx -y voila-recorder mcp`
+Reference: https://voila.anzalabidi.dev/llms.txt · https://github.com/anzal1/voila
+
+## Workflow — always this loop
+
+1. **Outline first** → real nav text, headings, CTAs. Build selectors from it;
+   never guess.
+2. **Write steps YAML**. Every meaningful beat gets a `caption` (burned-in
+   lower-third) and `narration` (spoken; the recording auto-paces to each
+   clip's length — do NOT pad waits for narration). Open and close with a
+   `slide` (animated title card: `title`, `subtitle`, `accent` hex).
+   Actions: goto, click, hover, type, scroll (y), scroll_to (selector),
+   slide, zoom (level), wait. Mark risky steps `optional: true`.
+3. **Record** (`voila_record` / `record --steps`).
+4. **Review your own output** (`voila_review`) → frames + timeline. Read the
+   frames. Check: cursor near what narration discusses; captions not covering
+   key UI; zooms centered on content, not whitespace; every page actually
+   loaded; no dead segments. Patch the YAML, re-record. One review pass
+   minimum before delivering.
+5. Deliver the MP4 (the recipe travels inside it — extract from any voila MP4
+   with `ffmpeg -i demo.mp4 -f ffmetadata -`).
+
+## Hard-won rules
+
+- Selectors: prefer `a[href='/path']`, roles, and ids over text= (hydration
+  makes text selectors flaky). Append `>> visible=true` when duplicates exist
+  (desktop + mobile nav).
+- On selector failure the error includes the live page outline — use it to
+  patch, don't retry blindly.
+- Devices: desktop (1280×800), mobile (390×844, iPhone emulation, zoom
+  auto-disabled — mobile layouts crop badly), tablet (834×1112). Portrait
+  output on mobile.
+- Zoom levels 1.3–1.6 on desktop; always return to 1 before ending.
+- Login-protected apps: ask the user to run the record once with `--headful`,
+  or open the web UI (`npx -y voila-recorder serve`, port 4477) and sign in —
+  the session persists in the local profile. NEVER type credentials yourself.
+- Narration style: short sentences, product language, no "as you can see".
+  8–15 words per beat reads best at Kokoro's pace.