storyshot 0.4.2 → 0.6.0

package/src/kokoro-zh.mjs

@@ -0,0 +1,204 @@
+// Mandarin narration with Kokoro v1.1-zh (voices zf_001…zf_099, zm_009…zm_100), free and local.
+// Text → phonemes runs misaki's Chinese G2P (Python) in storyshot's own venv, once per render for all
+// lines; phonemes → audio runs the v1.1-zh ONNX model through kokoro-js, like the English voice.
+import { execFileSync, spawnSync } from 'node:child_process';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+export const ZH_DEFAULT_VOICE = 'zf_001';
+export const isZhVoice = (voice) => /^z[fm]_\d{3}$/.test(String(voice ?? ''));
+const REPO = 'onnx-community/Kokoro-82M-v1.1-zh-ONNX';
+const CACHE = path.join(os.homedir(), '.cache', 'storyshot');
+const MODELS = path.join(CACHE, 'models');
+export const ZH_VENV = path.join(CACHE, 'zh-venv');
+const HELPER = path.join(path.dirname(fileURLToPath(import.meta.url)), 'zh_g2p.py');
+// misaki 0.9.4 (the v1.1 frontend) needs Python <3.13; regex 2026.x needs >=3.10
+const PY_RANGE = [10, 12];
+const REQUIREMENTS = ['misaki[zh]==0.9.4', 'cn2an==0.5.24', 'proces==0.1.7', 'jieba==0.42.1', 'pypinyin==0.55.0',
+  'pypinyin-dict==0.9.0', 'ordered-set==4.1.0', 'phonemizer-fork==3.3.2', 'espeakng-loader==0.2.4', 'regex==2026.9.29', 'addict==2.4.0'];
+// Part of every cached line's hash (with the rewritten text): change it when the model or the G2P
+// (zh_g2p.py, REQUIREMENTS) changes.
+export const ZH_TTS_ID = `${REPO}:q8|${REQUIREMENTS[0]}|g2p-1`;
+// Pieces longer than this many phonemes are split at sentence ends (the model's hard limit is 510).
+const TARGET = 300, MAX_PS = 510;
+
+// --- text rewrites (TTS only: subtitles keep the original) ----------------------------------------
+const MEASURE = `(?:${['个', '条', '只', '次(?!方|幂|函数|项|根)', '本', '位', '张', '种', '倍', '年(?!级)', '天', '点(?!\\d)', '分钟',
+  '分(?!之)', '小时', '秒', '周', '页', '行', '列', '步', '格', '遍', '份', '组', '句', '段', '件', '台', '层', '部分', '名', '项',
+  '类', '处', '边', '侧', '端', '根', '片', '块', '把', '支', '间', '元', '岁', '米', '公里', '千米', '厘米', '毫米', '公斤', '千克',
+  '克', '斤', '升', '毫升', '度', '万', '千', '百', '亿'].join('|')})`;
+const NUM_SPACE = new RegExp(`(\\d)\\s+(?=${MEASURE})`, 'g');
+const LIANG = new RegExp(`(?<![\\d.,/A-Za-z_\\-−+]|第\\s*|负\\s*)2(?=${MEASURE})`, 'g');
+// 行 after a number is the measure word háng (第一行, 两行, 每一行, 三行代码), which the G2P reads xíng;
+// 航 is a háng with no other reading. Not in 一行人 (a party), 统一行动, 行为, 行李 …
+const HANG = /(?<=[\d一二三四五六七八九十百千两几])行(?![人动为走驶进程政使李星销善事])/g;
+
+export function zhSpeakText(text) {
+  return String(text)
+    // a number reads as one word with the measure word after it: "3 个" → "3个" (no pause; 一 gets its tone change)
+    .replace(NUM_SPACE, '$1')
+    // and a lone 2 there is 两: "2 个单位" → "两个单位" (not 12个, 0.2个, 2.5个, 第2个, 负2个, 2次方, 2年级)
+    .replace(LIANG, '两')
+    .replace(HANG, '航')
+    // math symbols the G2P would drop
+    .replace(/\s*[=＝]\s*/g, '等于').replace(/\s*[+＋]\s*/g, '加').replace(/\s*×\s*/g, '乘').replace(/\s*÷\s*/g, '除以')
+    .replace(/(?<=[\w)²³]\s*)−\s*/g, '减').replace(/−\s*/g, '负').replace(/²/g, '的平方').replace(/³/g, '的立方');
+}
+
+// --- Python for the G2P: found once, then a venv of storyshot's own --------------------------------
+const venvPython = (dir) => (process.platform === 'win32' ? path.join(dir, 'Scripts', 'python.exe') : path.join(dir, 'bin', 'python'));
+const STAMP = 'storyshot-requirements.txt';
+const venvReady = () => {
+  try { return fs.readFileSync(path.join(ZH_VENV, STAMP), 'utf8') === REQUIREMENTS.join('\n') && fs.existsSync(venvPython(ZH_VENV)); } catch { return false; }
+};
+// (the real path: a venv made through a symlink to a relocatable build, like uv's ~/.local/bin/python3.12, can't find its stdlib)
+function probe(cmd, args = []) {
+  try {
+    const [version, exe] = execFileSync(cmd, [...args, '-c', 'import os, sys; print("%d.%d.%d" % sys.version_info[:3]); print(os.path.realpath(sys.executable))'],
+      { stdio: ['ignore', 'pipe', 'ignore'], timeout: 15000 }).toString().trim().split(/\r?\n/);
+    const [maj, min] = version.split('.').map(Number);
+    return { version, exe, ok: maj === 3 && min >= PY_RANGE[0] && min <= PY_RANGE[1] };
+  } catch { return null; }
+}
+let found, seen = [];
+// → { version, exe } of a Python 3.10–3.12, or null (then `seen` lists the unsuitable ones, for the error)
+export function findPython() {
+  if (found !== undefined) return found;
+  const cands = [...(process.env.STORYSHOT_PYTHON ? [[process.env.STORYSHOT_PYTHON]] : []), ['python3.12'], ['python3.11'], ['python3.10'],
+    ...(process.platform === 'win32' ? [['py', '-3.12'], ['py', '-3.11'], ['py', '-3.10']] : []), ['python3'], ['python']];
+  for (const [cmd, ...args] of cands) {
+    const p = probe(cmd, args);
+    if (p?.ok) return (found = p);
+    if (p) seen.push(`${[cmd, ...args].join(' ')} is ${p.version}`);
+  }
+  try { // a uv-managed interpreter (uv python install 3.12)
+    const exe = execFileSync('uv', ['python', 'find', '>=3.10,<3.13'], { stdio: ['ignore', 'pipe', 'ignore'], timeout: 15000 }).toString().trim();
+    const p = exe && probe(exe);
+    if (p?.ok) return (found = p);
+  } catch { /* no uv */ }
+  return (found = null);
+}
+
+const NO_PYTHON = () => 'the Chinese Kokoro voice needs Python 3.10–3.12 for its Mandarin text-to-phoneme step (misaki does not support ' +
+  `Python 3.13+ yet)${seen.length ? `; found only: ${seen.join(', ')}` : ''}. Install one — macOS: brew install python@3.12 · ` +
+  'Ubuntu/Debian: sudo apt install python3.12 python3.12-venv · any OS: uv python install 3.12 — or set STORYSHOT_PYTHON to one. ' +
+  `Storyshot then keeps its packages in its own venv (${ZH_VENV}); nothing is installed globally.`;
+
+// For doctor and check: is the Chinese voice usable, and what's missing?
+export function zhVoiceStatus() {
+  if (venvReady()) return { ready: true, venv: ZH_VENV, python: probe(venvPython(ZH_VENV)) };
+  const python = findPython();
+  return { ready: false, venv: ZH_VENV, python, problem: python ? null : NO_PYTHON() };
+}
+
+function ensureVenv(log) {
+  if (venvReady()) return venvPython(ZH_VENV);
+  const py = findPython();
+  if (!py) throw new Error(NO_PYTHON());
+  log(`setting up the Chinese voice (one time): a Python ${py.version} venv in ${ZH_VENV} with misaki, the Mandarin G2P (~1 min)…`);
+  const run = (cmd, args, what) => {
+    const r = spawnSync(cmd, args, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] });
+    if (r.status !== 0) {
+      throw new Error(`${what} failed: ${(r.stderr || r.stdout || r.error?.message || '').trim().split('\n').slice(-6).join('\n')}` +
+        (what.includes('venv') && process.platform === 'linux' ? '\n  (Debian/Ubuntu: sudo apt install python3-venv, or the python3.X-venv package for your version)' : ''));
+    }
+  };
+  fs.rmSync(ZH_VENV, { recursive: true, force: true });
+  fs.mkdirSync(CACHE, { recursive: true });
+  run(py.exe, ['-m', 'venv', ZH_VENV], `creating the venv with ${py.exe} -m venv`);
+  run(venvPython(ZH_VENV), ['-m', 'pip', 'install', '--disable-pip-version-check', '--no-input', '-q', ...REQUIREMENTS], 'installing misaki into the venv');
+  fs.writeFileSync(path.join(ZH_VENV, STAMP), REQUIREMENTS.join('\n')); // written last: a half-built venv is rebuilt next time
+  return venvPython(ZH_VENV);
+}
+
+// All lines in one Python run (jieba + espeak start once): → per line, a list of phoneme strings
+function phonemize(lines, log) {
+  const py = ensureVenv(log);
+  const r = spawnSync(py, ['-I', HELPER], { input: JSON.stringify({ lines, target: TARGET, max: MAX_PS }), maxBuffer: 1 << 26 });
+  if (r.status !== 0) throw new Error(`Mandarin G2P failed (${py} ${HELPER}): ${String(r.stderr || r.error?.message).trim().split('\n').slice(-8).join('\n')}`);
+  return JSON.parse(r.stdout.toString('utf8')).pieces;
+}
+
+// --- model + voice --------------------------------------------------------------------------------
+let model = null;
+async function loadModel(log) {
+  if (model) return model;
+  let KokoroTTS;
+  try { ({ KokoroTTS } = await import('kokoro-js')); } catch {
+    throw new Error('the Chinese Kokoro voice needs the kokoro-js package: npm i kokoro-js');
+  }
+  const { env } = await import('@huggingface/transformers');
+  env.cacheDir = MODELS;
+  if (!fs.existsSync(path.join(MODELS, REPO, 'onnx', 'model_quantized.onnx'))) log(`downloading the Chinese Kokoro model (one time, ~130 MB) into ${MODELS}…`);
+  model = await KokoroTTS.from_pretrained(REPO, { dtype: 'q8', device: 'cpu' });
+  return model;
+}
+
+const packs = new Map();
+async function voicePack(voice) {
+  if (packs.has(voice)) return packs.get(voice);
+  const file = path.join(MODELS, REPO, 'voices', `${voice}.bin`);
+  const SIZE = 510 * 256 * 4; // one 256-dim style vector per input length
+  if (!fs.existsSync(file) || fs.statSync(file).size !== SIZE) {
+    const res = await fetch(`https://huggingface.co/${REPO}/resolve/main/voices/${voice}.bin`);
+    if (res.status === 404) throw new Error(`Kokoro v1.1-zh has no voice "${voice}" (try zf_001, zf_002, zm_010; list: https://huggingface.co/${REPO}/tree/main/voices)`);
+    if (!res.ok) throw new Error(`downloading Kokoro voice ${voice} failed (${res.status})`);
+    const buf = Buffer.from(await res.arrayBuffer());
+    if (buf.length !== SIZE) throw new Error(`Kokoro voice ${voice}: unexpected size ${buf.length}`);
+    fs.mkdirSync(path.dirname(file), { recursive: true });
+    fs.writeFileSync(`${file}.tmp`, buf);
+    fs.renameSync(`${file}.tmp`, file);
+  }
+  const pack = new Float32Array(new Uint8Array(fs.readFileSync(file)).buffer);
+  packs.set(voice, pack);
+  return pack;
+}
+
+// The model rushes inputs longer than its training lines (~100 tokens): slow those down, as hexgrad's make_zh.py does
+export const zhSpeed = (n) => (n <= 83 ? 1 : n < 183 ? 1 - (n - 83) / 500 : 0.8) * 1.1;
+
+const SR = 24000;
+// Join a long line's pieces: trim each piece's edges, then a pause like a sentence break between them.
+function join(chunks) {
+  if (chunks.length === 1) return chunks[0];
+  const trim = (a) => {
+    const th = 0.01;
+    let s = 0, e = a.length;
+    while (s < e && Math.abs(a[s]) < th) s++;
+    while (e > s && Math.abs(a[e - 1]) < th) e--;
+    return a.subarray(Math.max(0, s - SR * 0.02), Math.min(a.length, e + SR * 0.05));
+  };
+  const parts = chunks.map(trim), gap = Math.round(SR * 0.35);
+  const out = new Float32Array(parts.reduce((n, p) => n + p.length, 0) + gap * (parts.length - 1));
+  let o = 0;
+  for (const p of parts) { out.set(p, o); o += p.length + gap; }
+  return out;
+}
+
+// entries: [{ text, file }] — synthesizes each line (already rewritten by zhSpeakText) into a WAV at `file`.
+export async function synthZh(v, entries, { log = console.log } = {}) {
+  if (!entries.length) return;
+  const pieces = phonemize(entries.map((e) => e.text), log);
+  const tts = await loadModel(log);
+  const pack = await voicePack(v.voice);
+  const { Tensor, RawAudio } = await import('@huggingface/transformers');
+  for (const [i, e] of entries.entries()) {
+    log(`  tts (kokoro/${v.voice}): "${e.text}"`);
+    const chunks = [];
+    for (const ps of pieces[i]) {
+      const { input_ids } = tts.tokenizer(ps, { truncation: true });
+      const off = 256 * Math.min(Math.max(input_ids.dims.at(-1) - 2, 0), 509);
+      const { waveform } = await tts.model({
+        input_ids,
+        style: new Tensor('float32', pack.slice(off, off + 256), [1, 256]),
+        speed: new Tensor('float32', [zhSpeed([...ps].length) * (v.speed ?? 1)], [1]),
+      });
+      chunks.push(waveform.data);
+    }
+    const tmp = `${e.file}.tmp.wav`;
+    await new RawAudio(chunks.length ? join(chunks) : new Float32Array(SR / 10), SR).save(tmp);
+    fs.renameSync(tmp, e.file);
+  }
+}

package/src/qwen-tts.mjs

@@ -0,0 +1,134 @@
+// Narration with Qwen3-TTS 0.6B CustomVoice (Apache-2.0): nine preset voices, five of them Chinese
+// (serena, vivian, uncle_fu, dylan with a Beijing accent, eric with a Sichuan one), free and local.
+// It runs through mlx-audio, so on Apple Silicon only; elsewhere config.mjs falls back to kokoro.
+// Python (qwen_tts.py) runs in storyshot's own venv, once per render for all uncached lines.
+import { execFileSync, spawn, spawnSync } from 'node:child_process';
+import crypto from 'node:crypto';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { findPython } from './kokoro-zh.mjs';
+
+export const QWEN_VOICES = ['serena', 'vivian', 'uncle_fu', 'dylan', 'eric', 'ryan', 'aiden', 'ono_anna', 'sohee'];
+export const qwenSupported = () => process.platform === 'darwin' && process.arch === 'arm64';
+// `Uncle Fu`, `Serena` → uncle_fu, serena (null when it isn't one of the nine)
+export const qwenVoice = (name) => {
+  const v = String(name ?? '').trim().toLowerCase().replace(/[\s-]+/g, '_');
+  return QWEN_VOICES.includes(v) ? v : null;
+};
+
+const REPO = 'mlx-community/Qwen3-TTS-12Hz-0.6B-CustomVoice-bf16';
+const REVISION = '6415d95f88be018ff9e46813119dc3bc12261328';
+// 0.9 (the model's default) now and then runs on into made-up words; 0.7 keeps the same voice steadier
+const TEMPERATURE = 0.7;
+const REQUIREMENTS = ['mlx-audio==0.6.3', 'mlx==0.32.4', 'mlx-metal==0.32.4', 'transformers==5.22.0', 'huggingface-hub==2.4.0',
+  'tokenizers==0.23.3', 'safetensors==0.8.0', 'numpy==2.5.4'];
+// Part of every cached line's hash: change it when the model, the packages or qwen_tts.py's sampling change.
+export const QWEN_TTS_ID = `${REPO}@${REVISION.slice(0, 7)}|${REQUIREMENTS[0]}|t${TEMPERATURE}|guard-1`;
+const CACHE = path.join(os.homedir(), '.cache', 'storyshot');
+export const QWEN_VENV = path.join(CACHE, 'qwen-venv');
+const HELPER = path.join(path.dirname(fileURLToPath(import.meta.url)), 'qwen_tts.py');
+const HUB = process.env.HF_HUB_CACHE ?? path.join(process.env.HF_HOME ?? path.join(os.homedir(), '.cache', 'huggingface'), 'hub');
+const MODEL_FILE = path.join(HUB, `models--${REPO.replace('/', '--')}`, 'snapshots', REVISION, 'model.safetensors');
+
+// --- text (TTS only: subtitles keep the original) --------------------------------------------------
+// The model reads = + 2个 第2行 x² itself; a minus sign before a number it drops, and a number like
+// 2e-105 sets it babbling, so those are spelled out in Chinese lines.
+export function qwenSpeakText(text, lang = qwenLang(text)) {
+  let t = String(text).replace(/\s*\n+\s*/g, ' ');
+  if (lang === 'chinese') {
+    t = t.replace(/(\d(?:\.\d+)?)[eE]\s*([-−+]?)\s*(\d+)/g, (_, m, s, e) => `${m}乘10的${s && s !== '+' ? '负' : ''}${e}次方`)
+      .replace(/(?<![\w)²³]\s*)−\s*(?=\d)/g, '负');
+  }
+  return t;
+}
+
+// The language the model is told: the script decides; Latin text is left to the model.
+export function qwenLang(text) {
+  const s = String(text).replace(/\d+(?:\.\d+)?(?:[eE][-−+]?\d+)?/g, ' '); // the e in 2e-105 is no English word
+  if (/[぀-ヿ]/.test(s)) return 'japanese';
+  if (/[가-힯]/.test(s)) return 'korean';
+  const han = (s.match(/[㐀-鿿]/g) ?? []).length, latin = (s.match(/[A-Za-z]+/g) ?? []).length;
+  return han > latin ? 'chinese' : 'auto';
+}
+
+// --- the venv ---------------------------------------------------------------------------------------
+const venvPython = (dir) => path.join(dir, 'bin', 'python');
+const STAMP = 'storyshot-requirements.txt';
+const venvReady = () => {
+  try { return fs.readFileSync(path.join(QWEN_VENV, STAMP), 'utf8') === REQUIREMENTS.join('\n') && fs.existsSync(venvPython(QWEN_VENV)); } catch { return false; }
+};
+
+const NO_PYTHON = () => 'the Qwen3-TTS voice needs Python 3.10–3.12 to run (brew install python@3.12, or uv python install 3.12, ' +
+  `or set STORYSHOT_PYTHON); storyshot then keeps its packages in its own venv (${QWEN_VENV}), nothing is installed globally`;
+
+// For doctor and check: can this machine use it, and what's still to come on first use?
+export function qwenStatus() {
+  if (!qwenSupported()) return { supported: false };
+  const model = fs.existsSync(MODEL_FILE);
+  if (venvReady()) return { supported: true, ready: true, venv: QWEN_VENV, model };
+  const python = findPython();
+  return { supported: true, ready: false, venv: QWEN_VENV, model, python, problem: python ? null : NO_PYTHON() };
+}
+
+function ensureVenv(log) {
+  if (venvReady()) return venvPython(QWEN_VENV);
+  const py = findPython();
+  if (!py) throw new Error(NO_PYTHON());
+  log(`setting up the Qwen3-TTS voice (one time): a Python ${py.version} venv in ${QWEN_VENV} with mlx-audio (~400 MB, 1–2 min)…`);
+  const run = (cmd, args, what) => {
+    const r = spawnSync(cmd, args, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] });
+    if (r.status !== 0) throw new Error(`${what} failed: ${(r.stderr || r.stdout || r.error?.message || '').trim().split('\n').slice(-6).join('\n')}`);
+  };
+  fs.rmSync(QWEN_VENV, { recursive: true, force: true });
+  fs.mkdirSync(CACHE, { recursive: true });
+  run(py.exe, ['-m', 'venv', QWEN_VENV], `creating the venv with ${py.exe} -m venv`);
+  run(venvPython(QWEN_VENV), ['-m', 'pip', 'install', '--disable-pip-version-check', '--no-input', '-q', ...REQUIREMENTS], 'installing mlx-audio into the venv');
+  fs.writeFileSync(path.join(QWEN_VENV, STAMP), REQUIREMENTS.join('\n')); // written last: a half-built venv is rebuilt next time
+  return venvPython(QWEN_VENV);
+}
+
+// --- synthesis --------------------------------------------------------------------------------------
+// entries: [{ text, file }] (text already through qwenSpeakText) → a WAV at each `file`.
+export async function synthQwen(v, entries, { log = console.log } = {}) {
+  if (!entries.length) return;
+  if (!qwenSupported()) throw new Error('the Qwen3-TTS voice runs on Apple Silicon Macs only (it uses MLX)');
+  const py = ensureVenv(log);
+  if (!fs.existsSync(MODEL_FILE)) log(`downloading the Qwen3-TTS model (one time, ~2.3 GB) into ${HUB}…`);
+  const lines = entries.map((e) => ({
+    text: e.text, file: `${e.file}.raw.wav`, voice: v.voice, lang: qwenLang(e.text),
+    // the same line gets the same take: a seed from the text
+    seed: parseInt(crypto.createHash('sha1').update(e.text).digest('hex').slice(0, 7), 16),
+  }));
+  const job = JSON.stringify({ repo: REPO, revision: REVISION, temperature: TEMPERATURE, lines });
+  const { code, stdout, tail } = await new Promise((resolve, reject) => {
+    const p = spawn(py, ['-I', HELPER], { env: { ...process.env, HF_HUB_DISABLE_TELEMETRY: '1', TRANSFORMERS_VERBOSITY: 'error' } });
+    let out = '', err = '', buf = '';
+    p.stdout.on('data', (d) => { out += d; });
+    p.stderr.on('data', (d) => {
+      err = (err + d).slice(-4000);
+      buf += d;
+      let nl;
+      while ((nl = buf.search(/[\r\n]/)) >= 0) {
+        const m = buf.slice(0, nl).match(/^line (\d+)\/\d+ (\S+)( \d+ tries)?/);
+        buf = buf.slice(nl + 1);
+        if (m) log(`  tts (qwen/${v.voice}, ${m[2]}${m[3] ? `,${m[3]}` : ''}): "${entries[m[1] - 1].text}"`);
+      }
+    });
+    p.on('error', reject);
+    p.on('close', (c) => resolve({ code: c, stdout: out, tail: err }));
+    p.stdin.end(job);
+  });
+  if (code !== 0) throw new Error(`Qwen3-TTS failed (${py} ${HELPER}): ${tail.trim().split('\n').slice(-8).join('\n')}`);
+  const { results } = JSON.parse(stdout);
+  for (const [i, e] of entries.entries()) {
+    if (results[i]?.suspect) log(`  ⚠ qwen: "${e.text}" still sounds off after ${results[i].tries} tries (${results[i].seconds.toFixed(1)}s) — listen to it, or rephrase the line / add a pronounce entry`);
+    const raw = lines[i].file, tmp = `${e.file}.tmp.wav`;
+    // the model has no speed control: ffmpeg's pitch-preserving atempo does it
+    if ((v.speed ?? 1) !== 1) execFileSync('ffmpeg', ['-loglevel', 'error', '-y', '-i', raw, '-af', `atempo=${v.speed}`, tmp]);
+    else fs.renameSync(raw, tmp);
+    fs.rmSync(raw, { force: true });
+    fs.renameSync(tmp, e.file);
+  }
+}

package/src/qwen_tts.py

@@ -0,0 +1,70 @@
+# Qwen3-TTS CustomVoice through mlx-audio (Apple Silicon), for `voice: { provider: qwen }`. Runs in
+# storyshot's own venv (qwen-tts.mjs), one process per render: the model loads once for all lines.
+#
+# stdin:  {"repo", "revision", "temperature", "lines": [{"text", "file", "voice", "lang", "seed"}]}
+# stderr: "line i/n <seconds>s [<tries> tries]" per line, plus whatever the libraries log
+# stdout: {"results": [{"seconds", "tries", "suspect"}]}
+#
+# The model samples its speech tokens, so now and then a line runs on into made-up words (a
+# scientific number like 2e-105 does it) or stops early. A take far longer or shorter than the text
+# should need is sampled again with another seed, and the closest one is kept.
+import contextlib
+import json
+import os
+import re
+import sys
+
+os.environ.setdefault('HF_HUB_DISABLE_TELEMETRY', '1')
+os.environ.setdefault('TRANSFORMERS_VERBOSITY', 'error')
+
+TRIES = 4
+
+
+def expected(text):
+    """Rough speaking time in seconds: ~3.6 Chinese characters a second, Latin words and numbers."""
+    han = len(re.findall(r'[㐀-鿿]', text))
+    words = len(re.findall(r'[A-Za-z]+', text))
+    nums = len(re.findall(r'\d+(?:\.\d+)?', text))
+    return han / 3.6 + words * 0.4 + nums * 0.5 + 0.3
+
+
+def plausible(seconds, est):
+    return 0.35 * est - 0.5 <= seconds <= 2.0 * est + 2.0
+
+
+def main():
+    job = json.load(sys.stdin)
+    out = sys.stdout
+    # the libraries print their own notes on stdout: keep it for the JSON answer
+    with contextlib.redirect_stdout(sys.stderr):
+        import math
+        import mlx.core as mx
+        import numpy as np
+        from huggingface_hub import snapshot_download
+        from mlx_audio.audio_io import write
+        from mlx_audio.tts.utils import load_model
+
+        model = load_model(snapshot_download(job['repo'], revision=job['revision']))
+        results, n = [], len(job['lines'])
+        for i, line in enumerate(job['lines']):
+            est = expected(line['text'])
+            best = None
+            for k in range(TRIES):
+                mx.random.seed(line['seed'] + k * 7919)
+                parts = model.generate(text=line['text'], voice=line['voice'], lang_code=line['lang'],
+                                       temperature=job['temperature'])
+                audio = np.concatenate([np.array(r.audio, dtype=np.float32) for r in parts])
+                seconds = len(audio) / model.sample_rate
+                score = abs(math.log(max(seconds, 0.1) / est))
+                if best is None or score < best[0]:
+                    best = (score, audio, seconds)
+                if plausible(seconds, est):
+                    break
+            write(line['file'], best[1], model.sample_rate)
+            results.append({'seconds': best[2], 'tries': k + 1, 'suspect': not plausible(best[2], est)})
+            print(f'line {i + 1}/{n} {best[2]:.1f}s' + (f' {k + 1} tries' if k else ''), file=sys.stderr, flush=True)
+    out.write(json.dumps({'results': results}))
+
+
+if __name__ == '__main__':
+    main()

package/src/record.mjs

@@ -719,6 +719,9 @@ export async function record(cfg, { log = console.log, inspectAfter = null } = {
     await locate(page, card.logo).screenshot({ path: path.join(cfg.work, file), omitBackground: true, timeout: 5000 })
       .catch((e) => log(`warning: could not capture logo (${e.message.split('\n')[0]})`));
   }
+  // where the intro's logo sits on the first page: a HyperFrames launch intro lands its logo there
+  const introLogo = cfg.intro?.logo && !cfg.intro.logoFile
+    ? await locate(page, cfg.intro.logo).boundingBox({ timeout: 2000 }).catch(() => null) : null;
   // an app window shows the app's icon and name: from the web app manifest, else the page's own
   let app = null;
   if (cfg.frame?.window === 'app' && !fast) {
@@ -824,7 +827,7 @@ export async function record(cfg, { log = console.log, inspectAfter = null } = {
   await cdp.send('Page.stopScreencast');
   await browser.close();
 
-  const timeline = { frames, steps, clicks, navs, gotos, urls, app, scale: cfg.scale };
+  const timeline = { frames, steps, clicks, navs, gotos, urls, app, introLogo, scale: cfg.scale };
   fs.writeFileSync(path.join(cfg.work, 'timeline.json'), JSON.stringify(timeline, null, 1));
   log(`recorded ${frames.length} frames`);
   return timeline;

package/src/voice.mjs

@@ -1,23 +1,52 @@
 // Voiceover: text-to-speech for each step's `say:` line, cached on disk.
-// Providers: kokoro (local, free, English), elevenlabs, openai, say (macOS).
+// Providers: kokoro (local, free: English, and Mandarin with the v1.1-zh voices), qwen (local, free, Apple
+// Silicon: Qwen3-TTS, best for Chinese), elevenlabs, openai, say (macOS).
 import { execFileSync } from 'node:child_process';
 import crypto from 'node:crypto';
 import fs from 'node:fs';
 import os from 'node:os';
 import path from 'node:path';
+import { isZhVoice, synthZh, zhSpeakText, ZH_DEFAULT_VOICE, ZH_TTS_ID } from './kokoro-zh.mjs';
+import { qwenSpeakText, QWEN_TTS_ID, synthQwen } from './qwen-tts.mjs';
 
 const SR = 48000;
 export const VOICE_DEFAULTS = {
   kokoro: { voice: 'af_heart' },
+  qwen: { voice: 'dylan' },
   elevenlabs: { voice: 'JBFqnCBsd6RMkjVDRZzb', model: 'eleven_multilingual_v2' },
   openai: { voice: 'alloy', model: 'gpt-4o-mini-tts' },
   say: { voice: 'Samantha' },
 };
 
+// kokoro without a named voice: the Mandarin one when the narration is mostly Chinese (more Han
+// characters than Latin words; no kana or hangul), or, with no narration to go by, `lang: zh`.
+// English (af_heart) otherwise.
+export function kokoroVoiceFor(raw) {
+  return narrationIsChinese(raw) ? ZH_DEFAULT_VOICE : VOICE_DEFAULTS.kokoro.voice;
+}
+
+// qwen without a named voice, for Chinese: dylan (a steady teacher's voice) in tutorials, lessons and
+// docs, serena (warmer) in promos; ono_anna for Japanese, sohee for Korean, aiden otherwise
+export function qwenVoiceFor(raw) {
+  const says = narration(raw);
+  if (/[\u3040-\u30ff]/.test(says) || /^ja/i.test(String(raw.lang ?? ''))) return 'ono_anna';
+  if (/[\uac00-\ud7af]/.test(says) || /^ko/i.test(String(raw.lang ?? ''))) return 'sohee';
+  if (!narrationIsChinese(raw)) return 'aiden';
+  return (raw.purpose ?? 'pitch') === 'pitch' ? 'serena' : VOICE_DEFAULTS.qwen.voice;
+}
+
+const narration = (raw) => [...(raw.steps ?? []).flatMap((s) => [s?.say, s?.card?.say]), raw.intro?.say, raw.end_card?.say].filter(Boolean).join(' ');
+function narrationIsChinese(raw) {
+  const says = narration(raw);
+  const han = (says.match(/[\u3400-\u9fff]/g) ?? []).length, latin = (says.match(/[A-Za-z]+/g) ?? []).length;
+  return says.trim() ? han > latin && !/[\u3040-\u30ff\uac00-\ud7af]/.test(says) : /^zh/i.test(String(raw.lang ?? ''));
+}
+
 let kokoro = null;
 async function synth(v, text, file) {
   switch (v.provider) {
     case 'kokoro': {
+      if (/^z[fm]_/.test(v.voice)) throw new Error(`Kokoro voice "${v.voice}" is from v1.0, which storyshot does not run; for Mandarin use a v1.1-zh voice such as ${ZH_DEFAULT_VOICE}`);
       if (!kokoro) {
         let KokoroTTS;
         try { ({ KokoroTTS } = await import('kokoro-js')); } catch {
@@ -60,7 +89,7 @@ async function synth(v, text, file) {
       execFileSync('say', ['-v', v.voice, '-r', String(Math.round(185 * v.speed)), '-o', file, text]);
       return;
     default:
-      throw new Error(`unknown voice provider "${v.provider}" (use kokoro, elevenlabs, openai or say)`);
+      throw new Error(`unknown voice provider "${v.provider}" (use kokoro, qwen, elevenlabs, openai or say)`);
   }
 }
 
@@ -74,18 +103,32 @@ function pronounce(v, text) {
   return t;
 }
 
-// Returns stereo Float32 PCM at 48 kHz with leading/trailing silence trimmed.
-export async function speak(cfg, line, { log = console.log } = {}) {
+// The cache file for a line, and the text the engine is given. The Mandarin Kokoro voice reads a
+// rewritten text (2个 → 两个, …) and has its model and G2P version in the hash; so does qwen.
+function cached(cfg, line) {
   const v = cfg.voice;
-  const text = pronounce(v, line);
   const dir = path.join(path.dirname(cfg.work), 'tts-cache');
   fs.mkdirSync(dir, { recursive: true });
-  const ext = { kokoro: 'wav', say: 'aiff' }[v.provider] ?? 'mp3';
-  const hash = crypto.createHash('sha1').update(JSON.stringify([v.provider, v.voice, v.model, v.speed, v.instructions, text])).digest('hex').slice(0, 16);
-  const file = path.join(dir, `${hash}.${ext}`);
+  const zh = v.provider === 'kokoro' && isZhVoice(v.voice), qwen = v.provider === 'qwen';
+  const text = zh ? zhSpeakText(pronounce(v, line)) : qwen ? qwenSpeakText(pronounce(v, line)) : pronounce(v, line);
+  const ext = { kokoro: 'wav', qwen: 'wav', say: 'aiff' }[v.provider] ?? 'mp3';
+  const key = zh ? [v.provider, v.voice, ZH_TTS_ID, v.speed, text] : qwen ? [v.provider, v.voice, QWEN_TTS_ID, v.speed, text]
+    : [v.provider, v.voice, v.model, v.speed, v.instructions, text];
+  const hash = crypto.createHash('sha1').update(JSON.stringify(key)).digest('hex').slice(0, 16);
+  return { text, file: path.join(dir, `${hash}.${ext}`), zh, qwen };
+}
+
+// Returns stereo Float32 PCM at 48 kHz with leading/trailing silence trimmed.
+export async function speak(cfg, line, { log = console.log } = {}) {
+  const v = cfg.voice;
+  const { text, file, zh, qwen } = cached(cfg, line);
   if (!fs.existsSync(file)) {
-    log(`  tts (${v.provider}): "${text}"`);
-    await synth(v, text, file);
+    if (zh) await synthZh(v, [{ text, file }], { log });
+    else if (qwen) await synthQwen(v, [{ text, file }], { log });
+    else {
+      log(`  tts (${v.provider}): "${text}"`);
+      await synth(v, text, file);
+    }
   }
   const trim = 'silenceremove=start_periods=1:start_threshold=-50dB,areverse,silenceremove=start_periods=1:start_threshold=-50dB,areverse';
   const pcm = execFileSync('ffmpeg', ['-loglevel', 'error', '-i', file, '-af', trim, '-f', 'f32le', '-ac', '2', '-ar', `${SR}`, 'pipe:1'], { maxBuffer: 1 << 28 });
@@ -105,6 +148,14 @@ export function tempo(pcm, factor) {
 export async function measureVoice(cfg, { log = console.log } = {}) {
   const lens = { steps: {}, endCard: null, intro: null, cards: {} };
   if (!cfg.voice) return lens;
+  if (cfg.voice.fallback) log(`note: ${cfg.voice.fallback}`);
+  const zh = cfg.voice.provider === 'kokoro' && isZhVoice(cfg.voice.voice);
+  if (zh || cfg.voice.provider === 'qwen') {
+    // synthesize every uncached line in one batch, so Python (the G2P, or the Qwen model) starts once
+    const lines = [...cfg.steps.filter((s) => !s.hidden).flatMap((s) => [s.say, s.card?.say]), cfg.intro?.say, cfg.endCard?.say].filter(Boolean);
+    const todo = new Map(lines.map((l) => cached(cfg, l)).filter((c) => !fs.existsSync(c.file)).map((c) => [c.file, c]));
+    await (zh ? synthZh : synthQwen)(cfg.voice, [...todo.values()], { log });
+  }
   for (const [i, s] of cfg.steps.entries()) {
     if (s.say && !s.hidden) lens.steps[i] = (await speak(cfg, s.say, { log })).length / 2 / SR;
     if (s.card?.say && !s.hidden) lens.cards[i] = (await speak(cfg, s.card.say, { log })).length / 2 / SR;