voila-recorder 0.7.0 → 0.8.0

package/AGENTS.md

@@ -58,12 +58,12 @@ npx -y voila-recorder rerender <dir> --voice bf_emma
 - Narration style: short sentences, product language, 8-15 words per beat.
 - Prefer `zoom` with a `selector` over a raw `level`: voila measures the element
   and picks the level and camera centre so nothing gets cropped.
-- Voices: Kokoro is English only (af_heart A, af_bella A-, bf_emma British).
-  `speed` 0.5-1.6 sets pace.
-- Other languages: set `voice:` per narration step, so one demo can mix them.
-  `say:Monica` uses a macOS system voice (~50 languages, `voila voices --all`);
-  `--tts-cmd 'engine -o {out} "{text}"'` plugs in any engine on any platform;
-  `audio: clip.mp3` on a step uses a file you already have.
+- Voices: Kokoro covers English, Spanish, French, Italian, Portuguese and
+  Hindi on every platform (af_heart, ef_dora, ff_siwis, if_sara, pf_dora,
+  hf_alpha). `speed` 0.5-1.6 sets pace. `voila voices` lists all 41.
+- Mixing languages: set `voice:` per narration step.
+- Japanese/Mandarin are gated (espeak mispronounces them). Use `--tts-cmd`
+  with a dedicated engine, a macOS `say:` voice, or `audio: clip.mp3`.
 - Sign-in walls: recording refuses to film a login page. Run `voila login <url>`
   (or the voila_login tool), let the HUMAN sign in in the window that opens, and
   the session persists in a local profile for every later recording.

package/README.md

@@ -73,27 +73,27 @@ on-device, no cloud, no API keys ([audio.js](audio.js)). Falls back to macOS
 `say` if Kokoro can't load. Voices: `af_heart` (default), `af_bella`,
 `am_adam`, … (`voice` param). Disable with `narrate: false` / `--no-narrate`.
 
-**Other languages, and mixing them.** Kokoro's JS port speaks English only, so
-non-English narration comes from one of two other engines, chosen per step:
+**Six languages, mixable in one demo.** Kokoro speaks English (US/UK),
+Spanish, French, Italian, Portuguese and Hindi on every platform, on-device.
+Set `voice:` per step:
 
 ```yaml
 - action: hover
   selector: h1
-  narration: "This part is English."          # Kokoro
+  narration: "This part is English."          # af_heart
 - action: scroll_to
   selector: "#pricing"
-  voice: "say:Mónica"                          # macOS system voice
+  voice: ef_dora                               # Spanish, same model
   narration: "Esta parte está en español."
-- action: wait
-  ms: 500
-  audio: ./clips/intro-ja.mp3                  # a clip you already have
 ```
 
-`voila voices --all` lists the ~180 system voices across ~50 languages on
-macOS. On Linux and Windows use any engine you like:
-`--tts-cmd 'piper -m es.onnx -f {out} -- "{text}"'` ({out}, {text}, {voice}).
-Every step is still paced to its own spoken clip, so mixed-language demos stay
-in sync.
+`voila voices` lists all 41. Every step is paced to its own spoken clip, so
+mixed-language demos stay in sync. Japanese and Mandarin voices ship with the
+model but are gated: espeak mispronounces them badly. For those, and for
+anything else, plug in your own engine with
+`--tts-cmd 'piper -m ja.onnx -f {out} -- "{text}"'`, use a macOS system voice
+(`voice: "say:Kyoko"`, `voila voices --all`), or hand a step a ready-made clip
+with `audio: intro.mp3`.
 
 ## Recipes — demos as code
 

package/audio.js

@@ -166,8 +166,27 @@ async function prepareNarration(items, dir, defaultVoice, onStatus = () => {}, s
         kokoro = await KokoroTTS.from_pretrained('onnx-community/Kokoro-82M-v1.0-ONNX', { dtype: 'q8' });
       }
       const v = voice && voices.isValid(voice) ? voice : 'af_heart';
-      onStatus(`narrating ${i + 1}/${norm.length} with Kokoro (${v})`);
-      const audio = await kokoro.generate(it.text, { voice: v, speed });
+      const { langOf, phonemize } = require('./phonemes');
+      const lang = langOf(v);
+      onStatus(`narrating ${i + 1}/${norm.length} with Kokoro (${v}, ${lang ? lang.name : 'English'})`);
+
+      let audio;
+      if (lang && lang.tier !== 'native') {
+        // kokoro-js only phonemizes English, so do it ourselves with espeak-ng
+        // (WASM, every platform) and feed the model token ids directly.
+        if (lang.tier === 'experimental' && !process.env.VOILA_EXPERIMENTAL_LANGS) {
+          throw new Error(
+            `${lang.name} voices are experimental: espeak mispronounces them badly ` +
+            `(Japanese leaks English words, Mandarin emits numeric tones Kokoro never saw). ` +
+            `Set VOILA_EXPERIMENTAL_LANGS=1 to try anyway, or use --tts-cmd with a ${lang.name} engine.`
+          );
+        }
+        const ipa = await phonemize(it.text, v);
+        const enc = kokoro.tokenizer(ipa, { truncation: true });
+        audio = await kokoro.generate_from_ids(enc.input_ids, { voice: v, speed });
+      } else {
+        audio = await kokoro.generate(it.text, { voice: v, speed });
+      }
       await audio.save(file);
       clips.push({
         file,

package/mcp.js

@@ -37,7 +37,7 @@ function getSession(device) {
   return sessions.get(key);
 }
 
-const server = new McpServer({ name: 'voila', version: '0.7.0' });
+const server = new McpServer({ name: 'voila', version: '0.8.0' });
 const deviceParam = z.enum(['desktop', 'mobile', 'tablet']).optional().default('desktop');
 
 server.tool(
@@ -69,7 +69,7 @@ server.tool(
     url: z.string().url(),
     steps_yaml: z.string().optional(),
     narrate: z.boolean().optional().default(true),
-    voice: z.string().optional().describe('default narration voice. Kokoro is ENGLISH ONLY (af_heart, af_bella, bf_emma). For other languages use a macOS system voice ("say:Monica") or set tts_cmd. Per-step `voice:` overrides this, so one demo can mix languages.'),
+    voice: z.string().optional().describe('default narration voice. Kokoro speaks English (af_heart, af_bella, bf_emma), Spanish (ef_dora), French (ff_siwis), Italian (if_sara), Portuguese (pf_dora) and Hindi (hf_alpha) on every platform. Per-step `voice:` overrides this, so one demo can mix languages. Japanese/Mandarin are gated (mispronounced) - use tts_cmd for those.'),
     speed: z.number().min(0.5).max(1.6).optional().default(1).describe('narration speed; 0.9 reads calmer'),
     tts_cmd: z.string().optional().describe('external TTS engine template for any language/platform, e.g. \'piper -m es.onnx -f {out} -- "{text}"\'. Placeholders: {out} {text} {voice}.'),
     device: deviceParam,
@@ -118,18 +118,20 @@ server.tool(
 
 server.tool(
   'voila_voices',
-  'List narration voices. Kokoro voices are English only, with quality grades. Pass system:true to ' +
-  'also get the machine\'s system voices, which is how you narrate other languages (macOS only; on ' +
-  'Linux or Windows use tts_cmd instead). Use before voila_record when the user asks for a different ' +
-  'voice, an accent, a male or female narrator, or a non-English language.',
+  'List narration voices. Kokoro covers English, Spanish, French, Italian, Portuguese and Hindi on ' +
+  'every platform, on-device; English voices carry quality grades. Japanese and Mandarin voices exist ' +
+  'but are gated because espeak mispronounces them. Pass system:true to also list the machine\'s own ' +
+  'voices (macOS). Use before voila_record when the user asks for a different voice, an accent, a male ' +
+  'or female narrator, or a non-English language.',
   { system: z.boolean().optional().default(false) },
   async ({ system }) => ({
     content: [{ type: 'text', text: JSON.stringify({
       kokoro: voiceCatalogue.ranked(),
-      englishOnly: true,
+      languages: 'English (US/UK), Spanish, French, Italian, Portuguese (BR), Hindi',
+      gated: 'Japanese and Mandarin: espeak mispronounces them; use tts_cmd or a system voice',
       system: system ? voiceCatalogue.systemVoices() : undefined,
       systemLanguages: system ? Object.keys(voiceCatalogue.systemLanguages()) : undefined,
-      note: 'Non-English: use a system voice (say:Name) on macOS, or tts_cmd on any platform.',
+      note: 'Set voice: per step to mix languages in one demo.',
     }, null, 2) }],
   })
 );

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "voila-recorder",
-  "version": "0.7.0",
+  "version": "0.8.0",
   "description": "Permission-free product demo recorder: URL in, narrated auto-zoomed MP4 out — with the recipe embedded in the video. Agent-native (MCP), fully on-device.",
   "license": "MIT",
   "main": "pipeline.js",
@@ -21,6 +21,7 @@
     "modelcache.js",
     "consent.js",
     "doctor.js",
+    "phonemes.js",
     "voices.js",
     "auth.js",
     "public/",
@@ -54,6 +55,7 @@
     "demos-as-code"
   ],
   "dependencies": {
+    "@echogarden/espeak-ng-emscripten": "^0.3.5",
     "@modelcontextprotocol/sdk": "^1.30.0",
     "express": "^4.19.2",
     "ffmpeg-static": "^5.2.0",

package/phonemes.js

@@ -0,0 +1,51 @@
+// Grapheme-to-phoneme for Kokoro's non-English voices.
+//
+// kokoro-js only ships an English phonemizer, which is why its 26 non-English
+// voice files sat unusable. espeak-ng compiled to WASM carries the full
+// language data, runs on every platform, and emits the IPA Kokoro was trained
+// on. That combination gives real multilingual narration with no OS-specific
+// dependency.
+
+// Kokoro voice ids are prefixed by language: af_/am_ = American English,
+// bf_/bm_ = British, ef_/em_ = Spanish, and so on.
+const LANGS = {
+  a: { espeak: 'en-us', name: 'English (US)', tier: 'native' },
+  b: { espeak: 'en-gb', name: 'English (UK)', tier: 'native' },
+  e: { espeak: 'es',    name: 'Spanish',      tier: 'good' },
+  f: { espeak: 'fr-fr', name: 'French',       tier: 'good' },
+  h: { espeak: 'hi',    name: 'Hindi',        tier: 'good' },
+  i: { espeak: 'it',    name: 'Italian',      tier: 'good' },
+  p: { espeak: 'pt-br', name: 'Portuguese (BR)', tier: 'good' },
+  // espeak leaks English words into Japanese kanji, and emits numeric tones
+  // for Mandarin that Kokoro was not trained on. Both need a dedicated G2P
+  // (the Python release uses one); until then they are off by default.
+  j: { espeak: 'ja',    name: 'Japanese',     tier: 'experimental' },
+  z: { espeak: 'cmn',   name: 'Mandarin',     tier: 'experimental' },
+};
+
+const langOf = voiceId => LANGS[String(voiceId || '')[0]] || null;
+
+let worker = null;
+async function getWorker() {
+  if (!worker) {
+    const mod = await require('@echogarden/espeak-ng-emscripten').default();
+    worker = await new mod.eSpeakNGWorker();
+  }
+  return worker;
+}
+
+// espeak separates phonemes with underscores and sentences with newlines.
+// Kokoro wants a plain IPA string.
+function tidy(ipa) {
+  return ipa.replace(/_/g, '').replace(/\s*\n\s*/g, ' ').replace(/\s+/g, ' ').trim();
+}
+
+async function phonemize(text, voiceId) {
+  const lang = langOf(voiceId);
+  if (!lang) throw new Error(`no language mapping for voice "${voiceId}"`);
+  const w = await getWorker();
+  w.set_voice(lang.espeak);
+  return tidy(w.synthesize_ipa(text).ipa);
+}
+
+module.exports = { phonemize, langOf, LANGS };

package/skills/voila/SKILL.md

@@ -64,13 +64,13 @@ Reference: https://voila.anzalabidi.dev/llms.txt · https://github.com/anzal1/vo
   for them in chat.
 - Prefer `zoom` with a `selector` over a bare `level`: voila measures the
   element and picks the level and camera centre, so nothing is cropped.
-- Languages: narration voice is per step, so demos can mix languages. Kokoro
-  is ENGLISH ONLY (af_heart A, af_bella A-, bf_emma British). For other
-  languages use a macOS system voice (`voice: "say:Monica"`, `voila voices
-  --all` lists ~180 across ~50 languages), or `--tts-cmd` with any engine on
-  any platform (placeholders {out} {text} {voice}), or point a step at a
-  ready-made clip with `audio: file.mp3`. On Linux/Windows non-English needs
-  --tts-cmd: tell the user rather than silently narrating in English.
+- Languages: narration voice is per step, so demos can mix languages freely.
+  Kokoro covers English (af_heart A, af_bella A-, bf_emma British), Spanish
+  (ef_dora), French (ff_siwis), Italian (if_sara), Portuguese (pf_dora) and
+  Hindi (hf_alpha) on EVERY platform, on-device. `voila voices` lists all 41.
+  Japanese and Mandarin voices are gated because espeak mispronounces them;
+  for those use `--tts-cmd` with a dedicated engine, a macOS system voice
+  (`voice: "say:Kyoko"`, see `voila voices --all`), or `audio: clip.mp3`.
 - Voices: `voila voices` lists the English voices with quality grades. af_heart
   (A) default, af_bella (A-), af_nicole (B-), bf_emma (B-, British). `--speed`
   or the speed param (0.5-1.6) changes pace; 0.9 reads calmer.

package/voices.js

@@ -3,6 +3,15 @@
 
 let cache = null;
 
+// Voices the model ships, including the non-English ones kokoro-js leaves out
+// of its metadata. Their language comes from the id prefix.
+function shippedVoiceIds() {
+  const fs = require('fs'), path = require('path');
+  const dir = path.join(path.dirname(require.resolve('kokoro-js')), '..', 'voices');
+  try { return fs.readdirSync(dir).filter(f => f.endsWith('.bin')).map(f => f.replace('.bin', '')); }
+  catch { return []; }
+}
+
 function allVoices() {
   if (cache) return cache;
   const fs = require('fs');
@@ -16,11 +25,30 @@ function allVoices() {
   while ((m = re.exec(src))) {
     out.push({ id: m[1], name: m[2], language: m[3], gender: m[4], grade: m[6] });
   }
+  // Fold in the non-English voices, which ship as files but carry no metadata.
+  const { langOf } = require('./phonemes');
+  const known = new Set(out.map(v => v.id));
+  for (const id of shippedVoiceIds()) {
+    if (known.has(id)) continue;
+    const lang = langOf(id);
+    if (!lang || lang.tier === 'native') continue;
+    out.push({
+      id,
+      name: (id.split('_')[1] || id).replace(/^./, c => c.toUpperCase()),
+      language: lang.name,
+      gender: id[1] === 'f' ? 'Female' : id[1] === 'm' ? 'Male' : '',
+      grade: lang.tier === 'experimental' ? 'experimental' : 'unrated',
+      tier: lang.tier,
+    });
+  }
+  for (const v of out) if (!v.tier) v.tier = 'native';
   cache = out;
   return out;
 }
 
 const gradeRank = g => {
+  if (g === 'unrated') return 6;
+  if (g === 'experimental') return 9;
   const base = { A: 0, B: 1, C: 2, D: 3, F: 4 }[g[0]] ?? 5;
   const mod = g[1] === '+' ? -0.3 : g[1] === '-' ? 0.3 : 0;
   return base + mod;
@@ -46,12 +74,15 @@ function format() {
   for (const v of rows) (byLang[v.language] = byLang[v.language] || []).push(v);
   const lines = [];
   for (const [lang, vs] of Object.entries(byLang)) {
-    lines.push(`\n${lang}  (${vs.length} voices, best first)`);
+    const tier = vs[0].tier;
+    const note = tier === 'experimental' ? '  [experimental: pronunciation is unreliable]' : '';
+    lines.push(`\n${lang}  (${vs.length} voices)${note}`);
     for (const v of vs) {
-      lines.push(`  ${v.id.padEnd(13)} ${v.grade.padEnd(3)} ${v.gender.padEnd(7)} ${v.name}`);
+      lines.push(`  ${v.id.padEnd(13)} ${String(v.grade).padEnd(12)} ${v.gender.padEnd(7)} ${v.name}`);
     }
   }
-  lines.push('\nUse with: --voice af_bella   (also --speed 0.9 to slow the delivery)');
+  lines.push('\nAll of these run on-device on every platform.');
+  lines.push('Use with: --voice ef_dora   (--speed 0.9 slows the delivery)');
   return lines.join('\n');
 }