storyshot 0.6.0 → 0.7.2

package/src/inspect.mjs

@@ -39,12 +39,25 @@ function collect() {
   const pos = (el) => {
     const r = el.getBoundingClientRect();
     return { x: Math.round(r.x + scrollX), y: Math.round(r.y + scrollTop), w: Math.round(r.width), h: Math.round(r.height),
-      cx: Math.round(r.x + r.width / 2), cy: Math.round(r.y + r.height / 2), css: cssOf(el) };
+      vx: Math.round(r.x), vy: Math.round(r.y), cx: Math.round(r.x + r.width / 2), cy: Math.round(r.y + r.height / 2), css: cssOf(el) };
+  };
+  // what to list first when there are more than inspect shows: an open dialog, menu or popover (0), what
+  // is on screen and not covered by something else (1), below the fold (2), covered (3)
+  const LAYER = 'dialog[open], [role=dialog], [role=alertdialog], [aria-modal=true], [role=menu], [role=listbox], [popover], [role=tooltip]';
+  const rank = (el) => {
+    if (el.closest(LAYER)) return 0;
+    const r = el.getBoundingClientRect(), x = r.x + r.width / 2, y = r.y + r.height / 2;
+    if (y < 0 || y > innerHeight || x < 0 || x > innerWidth) return 2;
+    const hit = document.elementFromPoint(x, y), label = el.closest('label');
+    // (inside a shadow root the hit is its host; storyshot's own cursor and highlights don't count)
+    const clear = !hit || el.contains(hit) || hit.contains(el) || (label && hit.closest('label') === label) || el.getRootNode() !== document || hit.closest('[id^=__demo]');
+    return clear ? 1 : 3;
   };
   const clean = (s) => (s || '').replace(/\s+/g, ' ').trim();
   // a short CSS selector (tag + classes, or #id) — the last-resort target when names don't resolve
   const cssOf = (el) => {
-    if (el.id && !/^[0-9:]|:/.test(el.id)) return `#${CSS.escape(el.id)}`;
+    const id = el.getAttribute('id'); // not el.id: a form with an <input name=id> has that input as its .id
+    if (id && !/^[0-9:]|:/.test(id)) return `#${CSS.escape(id)}`;
     const cls = [...el.classList].filter((c) => !/^(css|sc|jsx|svelte|tw)-|\d{3,}|__/.test(c)).slice(0, 2).map((c) => `.${CSS.escape(c)}`).join('');
     return cls ? `${el.tagName.toLowerCase()}${cls}` : null;
   };
@@ -68,28 +81,35 @@ function collect() {
   // a label's own text, without the links/buttons inside it (e.g. a "Clear" link)
   const labelText = (lab) => { if (!lab) return null; const c = lab.cloneNode(true); c.querySelectorAll('a,button,select,input,textarea').forEach((x) => x.remove()); return clean(c.innerText) || null; };
   // styled checkboxes and radios hide the real input and draw their label instead: list them where the label is
+  // a slider's range and value, when it says (an <input type=range>, or aria-valuemin/max/now)
+  const sliderRange = (el) => {
+    const n = (a) => (el.getAttribute(a) == null ? NaN : Number(el.getAttribute(a)));
+    const [lo, hi, now] = el.type === 'range' ? [Number(el.min || 0), Number(el.max || 100), Number(el.value)] : [n('aria-valuemin'), n('aria-valuemax'), n('aria-valuenow')];
+    return [lo, hi, now].every(Number.isFinite) ? { lo, hi, now } : null;
+  };
   const shownBy = (el) => (el.tagName === 'INPUT' && ['checkbox', 'radio'].includes(el.type) && !vis(el) ? [...(el.labels ?? [])].find(vis) : null);
   const interactive = [...document.querySelectorAll(NATIVE)]
     .filter((el) => (vis(el) || shownBy(el)) && el.tagName !== 'LABEL' && !isAd(el) && !(el.isContentEditable && el.parentElement?.isContentEditable))
     .map((el) => ({ kind: 'role', role: roleOf(el), name: (el.isContentEditable || el.tagName === 'TEXTAREA' ? editorName(el) : nameOf(el)).slice(0, 80), shows: clean(el.innerText).slice(0, 60), alt: clean(el.querySelector('img[alt]')?.alt) || null, href: el.getAttribute('href'), label: labelText(el.labels?.[0]),
-      options: el.tagName === 'SELECT' ? [...el.options].map((o) => ({ text: clean(o.text), on: o.selected })).filter((o) => o.text).slice(0, 12) : null, ...pos(shownBy(el) ?? el) }));
+      range: roleOf(el) === 'slider' ? sliderRange(el) : null,
+      options: el.tagName === 'SELECT' ? [...el.options].map((o) => ({ text: clean(o.text), on: o.selected })).filter((o) => o.text).slice(0, 12) : null, rank: rank(shownBy(el) ?? el), ...pos(shownBy(el) ?? el) }));
   // custom clickables: outermost cursor:pointer elements that aren't native controls
   const pointer = [];
   for (const el of document.querySelectorAll('body *')) {
-    if (pointer.length >= 40) break;
+    if (pointer.length >= 60) break;
     if (el.closest(NATIVE) || el.querySelector(NATIVE) || !vis(el) || isAd(el)) continue;
     if (getComputedStyle(el).cursor !== 'pointer' || (el.parentElement && getComputedStyle(el.parentElement).cursor === 'pointer')) continue;
     const img = el.tagName === 'IMG' ? el : el.querySelector('img[alt]');
-    pointer.push({ kind: 'pointer', alt: img?.alt ? clean(img.alt).slice(0, 80) : null, name: clean(el.innerText).slice(0, 80), ...pos(el) });
+    pointer.push({ kind: 'pointer', alt: img?.alt ? clean(img.alt).slice(0, 80) : null, name: clean(el.innerText).slice(0, 80), rank: rank(el), ...pos(el) });
   }
   return {
     title: document.title,
     height: scroller ? scroller.scrollHeight : document.documentElement.scrollHeight,
-    scroller: scroller ? (scroller.id ? `#${scroller.id}` : scroller.tagName.toLowerCase() + (scroller.className && typeof scroller.className === 'string' ? '.' + scroller.className.trim().split(/\s+/)[0] : '')) : null,
+    scroller: scroller ? (scroller.getAttribute('id') ? `#${scroller.getAttribute('id')}` : scroller.tagName.toLowerCase() + (scroller.className && typeof scroller.className === 'string' ? '.' + scroller.className.trim().split(/\s+/)[0] : '')) : null,
     headings: [...document.querySelectorAll('h1, h2, h3')].filter(vis).map((h) => ({ tag: h.tagName.toLowerCase(), text: clean(h.innerText).slice(0, 100), ...pos(h) })),
     anchors: [...document.querySelectorAll('[id]')].filter((el) => vis(el) && el !== scroller && el.getBoundingClientRect().height > 150 &&
-      el.getBoundingClientRect().height < document.documentElement.scrollHeight * 0.8 && !/^(__|radix|headlessui|:)/.test(el.id))
-      .map((el) => ({ id: el.id, ...pos(el) })).slice(0, 30),
+      el.getBoundingClientRect().height < document.documentElement.scrollHeight * 0.8 && !/^(__|radix|headlessui|:)/.test(el.getAttribute('id')))
+      .map((el) => ({ id: el.getAttribute('id'), ...pos(el) })).slice(0, 30),
     images: [...document.querySelectorAll('img[alt]')].filter((el) => vis(el) && el.alt.trim()).map((el) => ({ alt: clean(el.alt).slice(0, 80), ...pos(el) })).slice(0, 30),
     interactive: [...interactive, ...pointer],
   };
@@ -97,24 +117,38 @@ function collect() {
 
 const yaml = (t) => (typeof t === 'string' ? JSON.stringify(t) : `{ ${Object.entries(t).map(([k, v]) => `${k}: ${JSON.stringify(v)}`).join(', ')} }`);
 
+// What can be clicked right now, without verifying targets: `inspect --after N` compares the page before and
+// after step N, to list what the step opened first.
+const keyOf = (el) => `${el.kind}|${el.role ?? ''}|${el.name}|${el.css ?? ''}`;
+export async function clickableKeys(page) {
+  const info = await page.evaluate(collect).catch(() => null);
+  return info && new Set(info.interactive.map(keyOf));
+}
+
 // Describe the page as it is right now. Shared by `inspect` and failure diagnostics.
-export async function describePage(page, viewport, { maxElements = 80 } = {}) {
+// before: clickableKeys() of an earlier state; what is new since then is listed first and marked.
+export async function describePage(page, viewport, { maxElements = 150, before = null } = {}) {
   const info = await page.evaluate(collect);
-  const seen = new Set(), elements = [];
+  const seen = new Set(), targets = new Set(), elements = [];
   const verify = async (locator, spec, el) => {
     const visible = locator.filter({ visible: true });
     const n = await visible.count().catch(() => 0);
     if (n === 1) return spec;
     if (n > 1) {
-      // pick the match whose box is where we saw the element
-      const xs = await visible.evaluateAll((els) => els.map((e) => { const r = e.getBoundingClientRect(); return [Math.round(r.x), Math.round(r.y), Math.round(r.width)]; })).catch(() => []);
-      const k = xs.findIndex(([x, , w]) => Math.abs(x - el.x) <= 2 && Math.abs(w - el.w) <= 2);
-      if (k >= 0) return { ...spec, nth: k };
+      // pick the match whose box is where we saw the element (all four sides: a column of same-named
+      // buttons shares x and width)
+      const xs = await visible.evaluateAll((els) => els.map((e) => { const r = e.getBoundingClientRect(); return [r.x, r.y, r.width, r.height].map(Math.round); })).catch(() => []);
+      const at = xs.map(([x, y, w, h], k) => [k, Math.abs(x - el.vx) + Math.abs(y - el.vy) + Math.abs(w - el.w) + Math.abs(h - el.h)]).filter(([, d]) => d <= 4);
+      if (at.length === 1) return { ...spec, nth: at[0][0] };
     }
     return null;
   };
-  for (const el of info.interactive) {
-    if (elements.length >= maxElements) break;
+  const fresh = (el) => !!before && !before.has(keyOf(el));
+  // the most useful first (new since `before`, open dialogs and menus, on screen), then the page's order
+  const order = info.interactive.map((el, k) => ({ el, k })).sort((a, b) => (fresh(b.el) - fresh(a.el)) || (a.el.rank - b.el.rank) || (a.k - b.k));
+  let left = 0;
+  for (const { el, k } of order) {
+    if (elements.length >= maxElements) { left++; continue; }
     const key = `${el.role ?? el.alt}|${el.name}|${el.x}|${el.y}`;
     if (seen.has(key)) continue;
     seen.add(key);
@@ -135,8 +169,15 @@ export async function describePage(page, viewport, { maxElements = 80 } = {}) {
       if (!target && el.name) target = await verify(page.getByText(el.name, { exact: true }), { text: el.name, exact: true }, el);
     }
     if (!target && el.css) target = await verify(page.locator(el.css), { css: el.css }, el);
-    if (target) elements.push({ ...el, target });
+    // two elements given the same target would both click the first: list neither as verified
+    if (target && targets.has(JSON.stringify(target))) {
+      const dup = elements.findIndex((e) => JSON.stringify(e.target) === JSON.stringify(target));
+      if (dup >= 0) elements.splice(dup, 1);
+      continue;
+    }
+    if (target) { targets.add(JSON.stringify(target)); elements.push({ ...el, k, fresh: fresh(el), target }); }
   }
+  elements.sort((a, b) => a.k - b.k);
 
   const url = page.url(), origin = new URL(url).origin;
   const internal = [...new Set(info.interactive.filter((e) => e.href && !e.href.startsWith('#'))
@@ -160,7 +201,10 @@ export async function describePage(page, viewport, { maxElements = 80 } = {}) {
     '',
     '## Clickable elements — verified targets (each matches exactly one visible element) for click/move/zoom',
     '   Copy targets exactly; `name` is the accessible name, which may differ from the on-screen label.',
-    ...elements.map((e) => `${fold(e.y)} y=${e.y}\tat (${e.cx}, ${e.cy})\t${yaml(e.target)}${e.role === 'textbox' ? `  (text input ${e.w}×${e.h} — click it, then \`type: …\`)` : e.role === 'slider' ? '  (slider — `slide: { target: …, to: 0..1 }`)' : e.options ? `  (dropdown — \`select: { target: …, option: "…" }\`; options: ${e.options.map((o) => `"${o.text}"${o.on ? ' (selected)' : ''}`).join(', ')})` : shows(e)}${e.href ? `  → ${e.href}` : ''}${e.kind === 'pointer' ? '  (custom clickable)' : ''}`),
+    ...(before && elements.some((e) => e.fresh) ? ['   ★ = new since the step before (what this step opened)'] : []),
+    ...elements.map((e) => `${e.fresh ? '★' : fold(e.y)} y=${e.y}\tat (${e.cx}, ${e.cy})\t${yaml(e.target)}${e.role === 'textbox' ? `  (text input ${e.w}×${e.h} — click it, then \`type: …\`)` : e.role === 'slider' ? (e.range ? `  (slider ${e.range.lo}…${e.range.hi}, now ${e.range.now} — \`slide: { target: …, value: ${e.range.now} }\`)` : '  (slider — `slide: { target: …, to: 0..1 }`)') : e.options ? `  (dropdown — \`select: { target: …, option: "…" }\`; options: ${e.options.map((o) => `"${o.text}"${o.on ? ' (selected)' : ''}`).join(', ')})` : shows(e)}${e.href ? `  → ${e.href}` : ''}${e.kind === 'pointer' ? '  (custom clickable)' : ''}${e.rank === 3 ? '  (covered by something on top right now)' : ''}`),
+    ...(left ? [`… and ${left} more not listed (the first ${maxElements} are: ${before ? 'new ones, ' : ''}open dialogs and menus, then what's on screen). ` +
+      'For a panel or menu, open it with a step and run `inspect <storyboard> --after <that step>`; its elements are listed first.'] : []),
     '',
     '## Images with alt text (target: { alt: "..." })',
     ...(info.images.length ? info.images.map((i) => `${fold(i.y)} y=${i.y}\t"${i.alt}"`) : ['(none)']),
@@ -171,7 +215,7 @@ export async function describePage(page, viewport, { maxElements = 80 } = {}) {
   return { text: lines.join('\n'), height: info.height, title: info.title, url, headings: info.headings, anchors: info.anchors, elements };
 }
 
-export async function inspect(url, { viewport = { width: 1600, height: 900 }, storageState, ignoreHTTPSErrors = false, outDir = '.storyshot/inspect', maxElements = 80 } = {}) {
+export async function inspect(url, { viewport = { width: 1600, height: 900 }, storageState, ignoreHTTPSErrors = false, outDir = '.storyshot/inspect', maxElements = 150 } = {}) {
   const browser = await chromium.launch();
   const context = await sessionContext(browser, { viewport, storageState, ignoreHTTPSErrors });
   const page = await context.newPage();

package/src/qwen-tts.mjs

@@ -22,15 +22,20 @@ const REPO = 'mlx-community/Qwen3-TTS-12Hz-0.6B-CustomVoice-bf16';
 const REVISION = '6415d95f88be018ff9e46813119dc3bc12261328';
 // 0.9 (the model's default) now and then runs on into made-up words; 0.7 keeps the same voice steadier
 const TEMPERATURE = 0.7;
+// Each take is checked by ear: Qwen3-ASR (Apache-2.0, ~1 GB) transcribes it, and a line it hears
+// differently is sampled again (pypinyin compares Chinese by sound, so homophones don't count)
+const ASR_REPO = 'mlx-community/Qwen3-ASR-0.6B-8bit';
+const ASR_REVISION = '89e96d92ba34aca20b3e29fb10cc284097d1219f';
 const REQUIREMENTS = ['mlx-audio==0.6.3', 'mlx==0.32.4', 'mlx-metal==0.32.4', 'transformers==5.22.0', 'huggingface-hub==2.4.0',
-  'tokenizers==0.23.3', 'safetensors==0.8.0', 'numpy==2.5.4'];
+  'tokenizers==0.23.3', 'safetensors==0.8.0', 'numpy==2.5.4', 'pypinyin==0.55.0'];
 // Part of every cached line's hash: change it when the model, the packages or qwen_tts.py's sampling change.
-export const QWEN_TTS_ID = `${REPO}@${REVISION.slice(0, 7)}|${REQUIREMENTS[0]}|t${TEMPERATURE}|guard-1`;
+export const QWEN_TTS_ID = `${REPO}@${REVISION.slice(0, 7)}|${REQUIREMENTS[0]}|t${TEMPERATURE}|guard-2|${ASR_REPO}@${ASR_REVISION.slice(0, 7)}`;
 const CACHE = path.join(os.homedir(), '.cache', 'storyshot');
 export const QWEN_VENV = path.join(CACHE, 'qwen-venv');
 const HELPER = path.join(path.dirname(fileURLToPath(import.meta.url)), 'qwen_tts.py');
 const HUB = process.env.HF_HUB_CACHE ?? path.join(process.env.HF_HOME ?? path.join(os.homedir(), '.cache', 'huggingface'), 'hub');
 const MODEL_FILE = path.join(HUB, `models--${REPO.replace('/', '--')}`, 'snapshots', REVISION, 'model.safetensors');
+const ASR_FILE = path.join(HUB, `models--${ASR_REPO.replace('/', '--')}`, 'snapshots', ASR_REVISION, 'model.safetensors');
 
 // --- text (TTS only: subtitles keep the original) --------------------------------------------------
 // The model reads = + 2个 第2行 x² itself; a minus sign before a number it drops, and a number like
@@ -44,13 +49,16 @@ export function qwenSpeakText(text, lang = qwenLang(text)) {
   return t;
 }
 
-// The language the model is told: the script decides; Latin text is left to the model.
-export function qwenLang(text) {
+// The language the model is told: the script decides; a line in Latin letters gets the storyboard's
+// `lang` (es → spanish) when the model knows it, else it's left to the model.
+const LATIN_LANGS = { en: 'english', es: 'spanish', fr: 'french', de: 'german', it: 'italian', pt: 'portuguese' };
+export function qwenLang(text, lang) {
   const s = String(text).replace(/\d+(?:\.\d+)?(?:[eE][-−+]?\d+)?/g, ' '); // the e in 2e-105 is no English word
   if (/[぀-ヿ]/.test(s)) return 'japanese';
   if (/[가-힯]/.test(s)) return 'korean';
-  const han = (s.match(/[㐀-鿿]/g) ?? []).length, latin = (s.match(/[A-Za-z]+/g) ?? []).length;
-  return han > latin ? 'chinese' : 'auto';
+  if (/[А-яЁё]/.test(s)) return 'russian';
+  const han = (s.match(/[㐀-鿿]/g) ?? []).length, latin = (s.match(/[A-Za-zÀ-ÿ]+/g) ?? []).length;
+  return han > latin ? 'chinese' : LATIN_LANGS[String(lang ?? '').toLowerCase().split(/[-_]/)[0]] ?? 'auto';
 }
 
 // --- the venv ---------------------------------------------------------------------------------------
@@ -66,7 +74,7 @@ const NO_PYTHON = () => 'the Qwen3-TTS voice needs Python 3.10–3.12 to run (br
 // For doctor and check: can this machine use it, and what's still to come on first use?
 export function qwenStatus() {
   if (!qwenSupported()) return { supported: false };
-  const model = fs.existsSync(MODEL_FILE);
+  const model = fs.existsSync(MODEL_FILE) && fs.existsSync(ASR_FILE);
   if (venvReady()) return { supported: true, ready: true, venv: QWEN_VENV, model };
   const python = findPython();
   return { supported: true, ready: false, venv: QWEN_VENV, model, python, problem: python ? null : NO_PYTHON() };
@@ -74,13 +82,22 @@ export function qwenStatus() {
 
 function ensureVenv(log) {
   if (venvReady()) return venvPython(QWEN_VENV);
-  const py = findPython();
-  if (!py) throw new Error(NO_PYTHON());
-  log(`setting up the Qwen3-TTS voice (one time): a Python ${py.version} venv in ${QWEN_VENV} with mlx-audio (~400 MB, 1–2 min)…`);
   const run = (cmd, args, what) => {
     const r = spawnSync(cmd, args, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] });
     if (r.status !== 0) throw new Error(`${what} failed: ${(r.stderr || r.stdout || r.error?.message || '').trim().split('\n').slice(-6).join('\n')}`);
   };
+  if (fs.existsSync(venvPython(QWEN_VENV))) {
+    // a venv from an older storyshot: bring its packages up to date in place, rebuild it only if that fails
+    log(`updating the Qwen3-TTS venv in ${QWEN_VENV}…`);
+    try {
+      run(venvPython(QWEN_VENV), ['-m', 'pip', 'install', '--disable-pip-version-check', '--no-input', '-q', ...REQUIREMENTS], 'updating the venv');
+      fs.writeFileSync(path.join(QWEN_VENV, STAMP), REQUIREMENTS.join('\n'));
+      return venvPython(QWEN_VENV);
+    } catch (e) { log(`  ${e.message.split('\n')[0]} — rebuilding it`); }
+  }
+  const py = findPython();
+  if (!py) throw new Error(NO_PYTHON());
+  log(`setting up the Qwen3-TTS voice (one time): a Python ${py.version} venv in ${QWEN_VENV} with mlx-audio (~400 MB, 1–2 min)…`);
   fs.rmSync(QWEN_VENV, { recursive: true, force: true });
   fs.mkdirSync(CACHE, { recursive: true });
   run(py.exe, ['-m', 'venv', QWEN_VENV], `creating the venv with ${py.exe} -m venv`);
@@ -90,18 +107,20 @@ function ensureVenv(log) {
 }
 
 // --- synthesis --------------------------------------------------------------------------------------
-// entries: [{ text, file }] (text already through qwenSpeakText) → a WAV at each `file`.
+// entries: [{ text, written?, file }] (text already through qwenSpeakText; written: the line before
+// `pronounce`, which the take check also accepts) → a WAV at each `file`.
 export async function synthQwen(v, entries, { log = console.log } = {}) {
   if (!entries.length) return;
   if (!qwenSupported()) throw new Error('the Qwen3-TTS voice runs on Apple Silicon Macs only (it uses MLX)');
   const py = ensureVenv(log);
   if (!fs.existsSync(MODEL_FILE)) log(`downloading the Qwen3-TTS model (one time, ~2.3 GB) into ${HUB}…`);
+  if (!fs.existsSync(ASR_FILE)) log(`downloading the Qwen3-ASR model that listens to each take (one time, ~1 GB) into ${HUB}…`);
   const lines = entries.map((e) => ({
-    text: e.text, file: `${e.file}.raw.wav`, voice: v.voice, lang: qwenLang(e.text),
+    text: e.text, written: e.written ?? e.text, file: `${e.file}.raw.wav`, voice: v.voice, lang: qwenLang(e.text, v.lang),
     // the same line gets the same take: a seed from the text
     seed: parseInt(crypto.createHash('sha1').update(e.text).digest('hex').slice(0, 7), 16),
   }));
-  const job = JSON.stringify({ repo: REPO, revision: REVISION, temperature: TEMPERATURE, lines });
+  const job = JSON.stringify({ repo: REPO, revision: REVISION, temperature: TEMPERATURE, asr: { repo: ASR_REPO, revision: ASR_REVISION }, lines });
   const { code, stdout, tail } = await new Promise((resolve, reject) => {
     const p = spawn(py, ['-I', HELPER], { env: { ...process.env, HF_HUB_DISABLE_TELEMETRY: '1', TRANSFORMERS_VERBOSITY: 'error' } });
     let out = '', err = '', buf = '';
@@ -121,9 +140,15 @@ export async function synthQwen(v, entries, { log = console.log } = {}) {
     p.stdin.end(job);
   });
   if (code !== 0) throw new Error(`Qwen3-TTS failed (${py} ${HELPER}): ${tail.trim().split('\n').slice(-8).join('\n')}`);
-  const { results } = JSON.parse(stdout);
+  const { results, asr } = JSON.parse(stdout);
+  if (asr) log(`  ⚠ qwen: the speech check couldn't load (${asr}), so these takes were checked by length only`);
   for (const [i, e] of entries.entries()) {
-    if (results[i]?.suspect) log(`  ⚠ qwen: "${e.text}" still sounds off after ${results[i].tries} tries (${results[i].seconds.toFixed(1)}s) — listen to it, or rephrase the line / add a pronounce entry`);
+    const r = results[i];
+    if (r?.suspect) {
+      log(r.heard != null && r.mismatch > 0
+        ? `  ⚠ qwen: "${e.text}" still doesn't come out right after ${r.tries} tries — it sounds like "${r.heard}". Listen to it; rephrase the line or add a pronounce entry`
+        : `  ⚠ qwen: "${e.text}" still sounds off after ${r.tries} tries (${r.seconds.toFixed(1)}s) — listen to it, or rephrase the line / add a pronounce entry`);
+    }
     const raw = lines[i].file, tmp = `${e.file}.tmp.wav`;
     // the model has no speed control: ffmpeg's pitch-preserving atempo does it
     if ((v.speed ?? 1) !== 1) execFileSync('ffmpeg', ['-loglevel', 'error', '-y', '-i', raw, '-af', `atempo=${v.speed}`, tmp]);

package/src/qwen_tts.py

@@ -1,23 +1,31 @@
 # Qwen3-TTS CustomVoice through mlx-audio (Apple Silicon), for `voice: { provider: qwen }`. Runs in
 # storyshot's own venv (qwen-tts.mjs), one process per render: the model loads once for all lines.
 #
-# stdin:  {"repo", "revision", "temperature", "lines": [{"text", "file", "voice", "lang", "seed"}]}
+# stdin:  {"repo", "revision", "temperature", "asr": {"repo", "revision"} | null,
+#          "lines": [{"text", "written", "file", "voice", "lang", "seed"}]}
+#         (text: what the model reads; written: the line as the storyboard has it, before `pronounce`)
 # stderr: "line i/n <seconds>s [<tries> tries]" per line, plus whatever the libraries log
-# stdout: {"results": [{"seconds", "tries", "suspect"}]}
+# stdout: {"results": [{"seconds", "tries", "suspect", "heard", "mismatch"}], "asr": null | "why it didn't load"}
 #
-# The model samples its speech tokens, so now and then a line runs on into made-up words (a
-# scientific number like 2e-105 does it) or stops early. A take far longer or shorter than the text
-# should need is sampled again with another seed, and the closest one is kept.
+# The model samples its speech tokens, so now and then a take garbles a few words, drops one, runs on
+# into made-up words (a scientific number like 2e-105 does it) or stops early. Each take is checked:
+# its length against the text, and what a speech recognizer (Qwen3-ASR) hears in it against the text.
+# A take that fails is sampled again with another seed, and the best one is kept.
 import contextlib
 import json
 import os
 import re
 import sys
+import unicodedata
 
 os.environ.setdefault('HF_HUB_DISABLE_TELEMETRY', '1')
 os.environ.setdefault('TRANSFORMERS_VERBOSITY', 'error')
 
 TRIES = 4
+# a take is sampled again when the recognizer hears this share of the line differently, or this long a run
+# of mistakes in a row (a garble: 负 2 x → 负二长方荡 is 2.5); a warning if every take is still off
+RETRY_AT, RETRY_RUN = 0.10, 2
+WARN_AT, WARN_RUN = 0.15, 2
 
 
 def expected(text):
@@ -32,6 +40,95 @@ def plausible(seconds, est):
     return 0.35 * est - 0.5 <= seconds <= 2.0 * est + 2.0
 
 
+# --- what was said vs. what was meant ---------------------------------------------------------------
+# Chinese is compared by sound (toneless pinyin), so a homophone the recognizer picks (交点/焦点) is no
+# mistake; numbers are read out first (0.5 → 零点五, 2 → 二), since the recognizer writes either form.
+DIGITS = '零一二三四五六七八九'
+
+
+def cn_int(n):
+    if n >= 10 ** 8:
+        return ''.join(DIGITS[int(c)] for c in str(n))
+
+    def four(x):
+        out, zero = '', False
+        for i, c in enumerate(f'{x:04d}'):
+            d = int(c)
+            if d == 0:
+                zero = bool(out)
+            else:
+                out += ('零' if zero else '') + DIGITS[d] + ['千', '百', '十', ''][i]
+                zero = False
+        return out
+
+    if n < 10:
+        return DIGITS[n]
+    hi, lo = divmod(n, 10000)
+    out = (four(hi) + '万' if hi else '') + (('零' if hi and lo < 1000 else '') + four(lo) if lo else '')
+    return re.sub('^一十', '十', out)
+
+
+def cn_number(m):
+    whole, _, frac = m.group().partition('.')
+    return cn_int(int(whole)) + ('点' + ''.join(DIGITS[int(c)] for c in frac) if frac else '')
+
+
+def units(text, chinese, pinyin):
+    """The line as a list of sounds: pinyin syllables, digits, and Latin words as '#word' without accents
+    (m x and mx as '#m', '#x')."""
+    t = ''.join(c for c in unicodedata.normalize('NFD', text.lower().replace('−', '-')) if unicodedata.category(c) != 'Mn')
+    if chinese:
+        t = re.sub(r'\d+(?:\.\d+)?', cn_number, t)
+    out = []
+    for tok in re.findall(r'[㐀-鿿]|[^\W\d_㐀-鿿]+|\d+', t):  # Han one by one, words (with accents), numbers
+        if chinese and re.match(r'[㐀-鿿]', tok):
+            out += [{'liang': 'er'}.get(p, p) for p in pinyin(tok)]
+        elif tok.isdigit():
+            out += list(tok)  # digit by digit: a wrong number (260 → 45) is several mistakes in a row, not one
+        else:
+            out += ['#' + c for c in tok] if len(tok) <= 2 else ['#' + tok]
+    return out
+
+
+def mismatch(text, heard, chinese, pinyin):
+    """(share of the line heard differently, the longest run of mistakes in a row), both in units of the
+    text. A garbled take has a run of several; a homophone or a misheard letter is one on its own."""
+    a, b = units(text, chinese, pinyin), units(heard, chinese, pinyin)
+    letter = lambda u: len(u) == 2 and u[0] == '#'
+    glue = lambda us: ''.join(u.lstrip('#') for u in us)
+    d = {(0, 0): (0, None)}
+    for i in range(len(a) + 1):
+        for j in range(len(b) + 1):
+            if i == j == 0:
+                continue
+            ways = []
+            if i:
+                ways.append((d[i - 1, j][0] + 1, (i - 1, j, 1)))
+            if j:
+                ways.append((d[i, j - 1][0] + 1, (i, j - 1, 1)))
+            if i and j:
+                x, y = a[i - 1], b[j - 1]
+                # a variable (m, b, x) the recognizer wrote as a character, or a number word it wrote in digits
+                # (tres → 3), is half a mistake
+                cost = 0 if x == y else 0.5 if letter(x) or letter(y) or x.isdigit() != y.isdigit() else 1
+                ways.append((d[i - 1, j - 1][0] + cost, (i - 1, j - 1, cost)))
+                # a name in Latin letters in a Chinese line (Desmos) can come back as a few syllables: one mistake
+                if chinese and len(x) > 3 and x[0] == '#':
+                    ways += [(d[i - 1, j - k][0] + 1, (i - 1, j - k, 1)) for k in range(2, min(5, j) + 1)]
+                # one word written as two or three (escribámosla / escribamos la): no mistake
+                if len(x) > 3 and x[0] == '#':
+                    ways += [(d[i - 1, j - k][0], (i - 1, j - k, 0)) for k in range(2, min(4, j) + 1) if glue(b[j - k:j]) == x[1:]]
+                if len(y) > 3 and y[0] == '#':
+                    ways += [(d[i - k, j - 1][0], (i - k, j - 1, 0)) for k in range(2, min(4, i) + 1) if glue(a[i - k:i]) == y[1:]]
+            d[i, j] = min(ways, key=lambda w: w[0])
+    i, j, run, worst = len(a), len(b), 0, 0
+    while (i, j) != (0, 0):
+        i, j, cost = d[i, j][1]
+        run = run + cost if cost else 0
+        worst = max(worst, run)
+    return d[len(a), len(b)][0] / max(1, len(a)), worst
+
+
 def main():
     job = json.load(sys.stdin)
     out = sys.stdout
@@ -45,6 +142,33 @@ def main():
         from mlx_audio.tts.utils import load_model
 
         model = load_model(snapshot_download(job['repo'], revision=job['revision']))
+        asr, asr_error, pinyin = None, None, None
+        if job.get('asr'):
+            try:
+                from mlx_audio.stt.utils import load_model as load_stt
+                from pypinyin import lazy_pinyin as pinyin
+                asr = load_stt(snapshot_download(job['asr']['repo'], revision=job['asr']['revision']))
+            except Exception as e:  # offline before the first download, say: check by length only
+                asr_error = f'{type(e).__name__}: {e}'.splitlines()[0][:300]
+
+        def hear(audio, line):
+            """(what the recognizer heard, mismatch), or (None, None) for a line it can't check."""
+            chinese = line['lang'] == 'chinese'
+            latin = {'english': 'English', 'spanish': 'Spanish', 'french': 'French', 'german': 'German', 'italian': 'Italian',
+                     'portuguese': 'Portuguese'}.get(line['lang'], 'English' if line['lang'] == 'auto' else None)
+            if asr is None or not (chinese or (latin and not re.search(r'[㐀-鿿]', line['text']))):
+                return None, None
+            tmp = line['file'] + '.check.wav'
+            write(tmp, audio, model.sample_rate)
+            try:
+                heard = asr.generate(tmp, language='Chinese' if chinese else latin).text.strip()
+            finally:
+                os.remove(tmp)
+            # against the line as written too: `pronounce` respells words for the voice (y → ye), and the
+            # recognizer writes what it hears the usual way
+            ms = [mismatch(t, heard, chinese, pinyin) for t in dict.fromkeys([line['text'], line.get('written') or line['text']])]
+            return heard, min(ms, key=lambda m: max(m[0] / RETRY_AT, m[1] / RETRY_RUN))
+
         results, n = [], len(job['lines'])
         for i, line in enumerate(job['lines']):
             est = expected(line['text'])
@@ -55,15 +179,22 @@ def main():
                                        temperature=job['temperature'])
                 audio = np.concatenate([np.array(r.audio, dtype=np.float32) for r in parts])
                 seconds = len(audio) / model.sample_rate
-                score = abs(math.log(max(seconds, 0.1) / est))
-                if best is None or score < best[0]:
-                    best = (score, audio, seconds)
-                if plausible(seconds, est):
+                fits = plausible(seconds, est)
+                heard, m = hear(audio, line) if fits else (None, None)
+                # how far off the recognizer heard it: up to 1 is fine (0 when it couldn't check)
+                worse = max(m[0] / RETRY_AT, m[1] / RETRY_RUN) if m else 0
+                # a take of the right length that says the line beats one that doesn't; then the closest length
+                rank = (not fits, worse if worse > 1 else 0, abs(math.log(max(seconds, 0.1) / est)))
+                if best is None or rank < best['rank']:
+                    best = {'rank': rank, 'audio': audio, 'seconds': seconds, 'fits': fits, 'heard': heard, 'm': m}
+                if fits and worse <= 1:
                     break
-            write(line['file'], best[1], model.sample_rate)
-            results.append({'seconds': best[2], 'tries': k + 1, 'suspect': not plausible(best[2], est)})
-            print(f'line {i + 1}/{n} {best[2]:.1f}s' + (f' {k + 1} tries' if k else ''), file=sys.stderr, flush=True)
-    out.write(json.dumps({'results': results}))
+            write(line['file'], best['audio'], model.sample_rate)
+            m = best['m']
+            results.append({'seconds': best['seconds'], 'tries': k + 1, 'heard': best['heard'], 'mismatch': m and m[0],
+                            'suspect': not best['fits'] or bool(m and (m[0] > WARN_AT or m[1] > WARN_RUN))})
+            print(f'line {i + 1}/{n} {best["seconds"]:.1f}s' + (f' {k + 1} tries' if k else ''), file=sys.stderr, flush=True)
+    out.write(json.dumps({'results': results, 'asr': asr_error}))
 
 
 if __name__ == '__main__':