talking-head-studio 0.2.8 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. package/dist/TalkingHead.d.ts +8 -0
  2. package/dist/TalkingHead.js +104 -7
  3. package/dist/TalkingHead.web.d.ts +3 -2
  4. package/dist/TalkingHead.web.js +17 -2
  5. package/dist/TalkingHeadVisualization.d.ts +35 -0
  6. package/dist/TalkingHeadVisualization.js +277 -0
  7. package/dist/api/index.d.ts +2 -0
  8. package/dist/api/index.js +18 -0
  9. package/dist/api/studioApi.d.ts +38 -0
  10. package/dist/api/studioApi.js +235 -0
  11. package/dist/api/types.d.ts +87 -0
  12. package/dist/api/types.js +5 -0
  13. package/dist/assets/face-squeeze-local.glb +0 -0
  14. package/dist/filament/FilamentAvatar.d.ts +41 -0
  15. package/dist/filament/FilamentAvatar.js +737 -0
  16. package/dist/filament/faceSqueezeAssets.d.ts +1 -0
  17. package/dist/filament/faceSqueezeAssets.js +5 -0
  18. package/dist/filament/index.d.ts +5 -0
  19. package/dist/filament/index.js +22 -0
  20. package/dist/filament/morphTables.d.ts +5 -0
  21. package/dist/filament/morphTables.js +93 -0
  22. package/dist/filament/useAuthedFilamentUri.d.ts +11 -0
  23. package/dist/filament/useAuthedFilamentUri.js +126 -0
  24. package/dist/html.d.ts +7 -0
  25. package/dist/html.js +255 -56
  26. package/dist/index.d.ts +9 -2
  27. package/dist/index.js +13 -2
  28. package/dist/index.web.d.ts +6 -2
  29. package/dist/index.web.js +10 -2
  30. package/dist/utils/avatarUtils.d.ts +13 -0
  31. package/dist/utils/avatarUtils.js +56 -0
  32. package/dist/wardrobe/index.d.ts +2 -0
  33. package/dist/wardrobe/index.js +20 -0
  34. package/dist/wardrobe/useAvatarWardrobeHydration.d.ts +7 -0
  35. package/dist/wardrobe/useAvatarWardrobeHydration.js +34 -0
  36. package/dist/wardrobe/wardrobeStore.d.ts +30 -0
  37. package/dist/wardrobe/wardrobeStore.js +106 -0
  38. package/package.json +33 -4
package/dist/html.js CHANGED
@@ -13,6 +13,13 @@ const UPSTREAM_SAFE_MOOD_MAP = {
13
13
  };
14
14
  function buildAvatarHtml(config) {
15
15
  const safeMood = UPSTREAM_SAFE_MOOD_MAP[config.mood] ?? 'neutral';
16
+ const v = config.vendorBaseUrl ? config.vendorBaseUrl.replace(/\/$/, '') : null;
17
+ const threeUrl = v ? `${v}/three.module.js` : 'https://cdn.jsdelivr.net/npm/three@0.180.0/build/three.module.js';
18
+ const threeAddonsUrl = v ? `${v}/three-addons/` : 'https://cdn.jsdelivr.net/npm/three@0.180.0/examples/jsm/';
19
+ const talkingHeadUrl = v ? `${v}/talkinghead.mjs` : 'https://cdn.jsdelivr.net/gh/met4citizen/TalkingHead@1.7/modules/talkinghead.mjs';
20
+ const headAudioUrl = v ? `${v}/headaudio.min.mjs` : 'https://cdn.jsdelivr.net/gh/met4citizen/HeadAudio@v0.1.0-alpha/dist/headaudio.min.mjs';
21
+ const headWorkletUrl = v ? `${v}/headworklet.min.mjs` : 'https://cdn.jsdelivr.net/gh/met4citizen/HeadAudio@v0.1.0-alpha/dist/headworklet.min.mjs';
22
+ const headModelUrl = v ? `${v}/model-en-mixed.bin` : 'https://cdn.jsdelivr.net/gh/met4citizen/HeadAudio@v0.1.0-alpha/dist/model-en-mixed.bin';
16
23
  return `
17
24
  <!DOCTYPE html>
18
25
  <html>
@@ -26,21 +33,47 @@ function buildAvatarHtml(config) {
26
33
  <script type="importmap">
27
34
  {
28
35
  "imports": {
29
- "three": "https://cdn.jsdelivr.net/npm/three@0.180.0/build/three.module.js",
30
- "three/addons/": "https://cdn.jsdelivr.net/npm/three@0.180.0/examples/jsm/",
31
- "talkinghead": "https://cdn.jsdelivr.net/gh/met4citizen/TalkingHead@1.7/modules/talkinghead.mjs"
36
+ "three": ${JSON.stringify(threeUrl)},
37
+ "three/addons/": ${JSON.stringify(threeAddonsUrl)},
38
+ "talkinghead": ${JSON.stringify(talkingHeadUrl)}
32
39
  }
33
40
  }
34
41
  </script>
42
+ <script>
43
+ window.ReactNativeWebView?.postMessage(
44
+ JSON.stringify({ type: 'log', message: '[bootstrap] inline script start' })
45
+ );
46
+ function postBootstrapError(kind, message) {
47
+ window.ReactNativeWebView?.postMessage(
48
+ JSON.stringify({ type: 'error', message: '[' + kind + '] ' + String(message || 'Unknown error') })
49
+ );
50
+ }
51
+ document.addEventListener('DOMContentLoaded', function() {
52
+ window.ReactNativeWebView?.postMessage(
53
+ JSON.stringify({ type: 'log', message: '[bootstrap] DOMContentLoaded' })
54
+ );
55
+ });
56
+ window.addEventListener('error', function(event) {
57
+ postBootstrapError('window.error', event?.message || event?.error?.message || 'Script error');
58
+ });
59
+ window.addEventListener('unhandledrejection', function(event) {
60
+ const reason = event?.reason;
61
+ postBootstrapError('unhandledrejection', reason?.message || reason || 'Unhandled promise rejection');
62
+ });
63
+ </script>
35
64
  </head>
36
65
  <body>
37
66
  <div id="avatar"></div>
38
67
  <script type="module">
68
+ (async function() {
69
+ window.ReactNativeWebView?.postMessage(
70
+ JSON.stringify({ type: 'log', message: '[module] script start' })
71
+ );
39
72
  const AUTH_TOKEN = ${JSON.stringify(config.authToken ?? null)};
40
- const TALKING_HEAD_URL = 'https://cdn.jsdelivr.net/gh/met4citizen/TalkingHead@1.7/modules/talkinghead.mjs';
41
- const HEAD_AUDIO_URL = 'https://cdn.jsdelivr.net/gh/met4citizen/HeadAudio@v0.1.0-alpha/dist/headaudio.min.mjs';
42
- const HEAD_AUDIO_WORKLET = 'https://cdn.jsdelivr.net/gh/met4citizen/HeadAudio@v0.1.0-alpha/dist/headworklet.min.mjs';
43
- const HEAD_AUDIO_MODEL = 'https://cdn.jsdelivr.net/gh/met4citizen/HeadAudio@v0.1.0-alpha/dist/model-en-mixed.bin';
73
+ const TALKING_HEAD_URL = ${JSON.stringify(talkingHeadUrl)};
74
+ const HEAD_AUDIO_URL = ${JSON.stringify(headAudioUrl)};
75
+ const HEAD_AUDIO_WORKLET = ${JSON.stringify(headWorkletUrl)};
76
+ const HEAD_AUDIO_MODEL = ${JSON.stringify(headModelUrl)};
44
77
  const MOOD_MAP = ${JSON.stringify(UPSTREAM_SAFE_MOOD_MAP)};
45
78
 
46
79
  let AVATAR_URL = ${JSON.stringify(config.avatarUrl)};
@@ -64,9 +97,16 @@ function log(msg) {
64
97
  window.ReactNativeWebView?.postMessage(JSON.stringify({ type: 'log', message: msg }));
65
98
  }
66
99
 
100
+ function emitLoading(stage, progress = null) {
101
+ window.ReactNativeWebView?.postMessage(
102
+ JSON.stringify({ type: 'loading', stage, progress }),
103
+ );
104
+ }
105
+
67
106
  async function loadWithAuth(url) {
68
107
  if (!url) throw new Error('Avatar URL is empty');
69
108
  if (AUTH_TOKEN && !url.startsWith('https://cdn.jsdelivr.net')) {
109
+ emitLoading('fetching_model');
70
110
  log('Fetching authenticated model: ' + url);
71
111
  const resp = await fetch(url, { headers: { 'Authorization': 'Bearer ' + AUTH_TOKEN } });
72
112
  if (!resp.ok) throw new Error('Failed to fetch model: ' + resp.status + ' ' + resp.statusText);
@@ -162,6 +202,7 @@ async function loadStaticFallback(loadedAvatarUrl) {
162
202
  staticModel.position.sub(scaledCenter);
163
203
 
164
204
  applyAccessories(pendingAccessoriesList);
205
+ emitLoading('ready', 100);
165
206
  window.ReactNativeWebView?.postMessage(JSON.stringify({ type: 'ready' }));
166
207
 
167
208
  window.addEventListener('resize', () => {
@@ -178,7 +219,11 @@ async function loadStaticFallback(loadedAvatarUrl) {
178
219
  renderer.render(scene, camera);
179
220
  });
180
221
  }, (ev) => {
181
- if (ev.lengthComputable) log('Fallback Loading: ' + Math.round((ev.loaded / ev.total) * 100) + '%');
222
+ if (ev.lengthComputable) {
223
+ const progress = Math.round((ev.loaded / ev.total) * 100);
224
+ emitLoading('loading_fallback', progress);
225
+ log('Fallback Loading: ' + progress + '%');
226
+ }
182
227
  }, (err) => {
183
228
  log('Fallback Error: ' + err.message);
184
229
  window.ReactNativeWebView?.postMessage(JSON.stringify({ type: 'error', message: err.message }));
@@ -203,7 +248,8 @@ async function init() {
203
248
 
204
249
  try {
205
250
  log('Loading TalkingHead...');
206
- const module = await import(TALKING_HEAD_URL);
251
+ emitLoading('booting');
252
+ const module = await import('talkinghead');
207
253
 
208
254
  head = new module.TalkingHead(container, {
209
255
  ttsEndpoint: null,
@@ -233,7 +279,9 @@ async function init() {
233
279
  lipsyncLang: 'en',
234
280
  }, (ev) => {
235
281
  if (ev.lengthComputable) {
236
- log('Loading: ' + Math.round((ev.loaded / ev.total) * 100) + '%');
282
+ const progress = Math.round((ev.loaded / ev.total) * 100);
283
+ emitLoading('loading_avatar', progress);
284
+ log('Loading: ' + progress + '%');
237
285
  }
238
286
  });
239
287
  if (loadedAvatarUrl.startsWith('blob:')) URL.revokeObjectURL(loadedAvatarUrl);
@@ -271,18 +319,22 @@ async function init() {
271
319
  Object.assign(head.mtAvatar[key], { newvalue: value, needsUpdate: true });
272
320
  }
273
321
  };
274
- head.opt.update = headaudio.update.bind(headaudio);
322
+ const headaudioUpdate = headaudio.update.bind(headaudio);
323
+ head.opt.update = (dt) => { headaudioUpdate(dt); tickVisemeDecay(); };
275
324
  log('HeadAudio ready (phoneme lip sync)');
276
325
  } else {
277
326
  log('HeadAudio skipped: AudioWorklet not supported in this WebView. Use sendViseme() from native TTS callbacks.');
327
+ head.opt.update = () => tickVisemeDecay();
278
328
  }
279
329
  } catch (err) {
280
330
  log('HeadAudio unavailable, viseme/amplitude fallback active: ' + err.message);
331
+ head.opt.update = () => tickVisemeDecay();
281
332
  }
282
333
 
283
334
  startAudioInterception();
284
335
  log('[ACC] init() complete, calling applyAccessories with ' + pendingAccessoriesList.length + ' pending items');
285
336
  applyAccessories(pendingAccessoriesList);
337
+ emitLoading('ready', 100);
286
338
  window.ReactNativeWebView?.postMessage(JSON.stringify({ type: 'ready' }));
287
339
  }
288
340
  } catch (err) {
@@ -332,34 +384,79 @@ let visemeMorphCache = null;
332
384
  const visemeState = {};
333
385
 
334
386
  const VISEME_MORPH_ALIASES = {
335
- sil: ['viseme_sil', 'sil'],
336
- PP: ['viseme_PP', 'pp', 'viseme_pp'],
337
- FF: ['viseme_FF', 'ff', 'viseme_ff'],
338
- TH: ['viseme_TH', 'th', 'viseme_th'],
339
- DD: ['viseme_DD', 'dd', 'viseme_dd'],
340
- kk: ['viseme_kk', 'kk', 'viseme_k'],
341
- CH: ['viseme_CH', 'ch', 'viseme_ch'],
342
- SS: ['viseme_SS', 'ss', 'viseme_s'],
343
- nn: ['viseme_nn', 'nn', 'viseme_n'],
344
- RR: ['viseme_RR', 'rr', 'viseme_r'],
387
+ sil: ['viseme_sil', 'sil', 'mouthClose', 'mouth_close'],
388
+ PP: ['viseme_PP', 'pp', 'viseme_pp', 'mouthPucker', 'mouth_pucker'],
389
+ FF: ['viseme_FF', 'ff', 'viseme_ff', 'mouthLowerLipIn', 'mouth_lower_lip_in', 'mouthRollLower', 'mouthShrugLower'],
390
+ TH: ['viseme_TH', 'th', 'viseme_th', 'tongueOut', 'tongue_out'],
391
+ DD: ['viseme_DD', 'dd', 'viseme_dd', 'mouthShrugUpper', 'mouth_shrug_upper'],
392
+ kk: ['viseme_kk', 'kk', 'viseme_k', 'mouthStretchLeft', 'mouth_stretch_left'],
393
+ CH: ['viseme_CH', 'ch', 'viseme_ch', 'mouthSmile', 'mouth_smile', 'mouthSmileLeft', 'mouth_smile_left'],
394
+ SS: ['viseme_SS', 'ss', 'viseme_ss', 'mouthStretchRight', 'mouth_stretch_right'],
395
+ nn: ['viseme_nn', 'nn', 'viseme_n', 'mouthDimpleLeft', 'mouth_dimple_left'],
396
+ RR: ['viseme_RR', 'rr', 'viseme_r', 'mouthDimpleRight', 'mouth_dimple_right'],
345
397
  aa: ['viseme_aa', 'viseme_AA', 'aa', 'jawOpen', 'jaw_open', 'jawopen', 'mouthOpen', 'mouth_open', 'mouthopen'],
346
- ee: ['viseme_ee', 'viseme_E', 'ee'],
347
- ih: ['viseme_ih', 'viseme_I', 'ih'],
348
- oh: ['viseme_oh', 'viseme_O', 'oh'],
349
- ou: ['viseme_ou', 'viseme_U', 'ou'],
398
+ ee: ['viseme_ee', 'viseme_E', 'ee', 'mouthSmileLeft', 'mouth_smile_left'],
399
+ ih: ['viseme_ih', 'viseme_I', 'ih', 'mouthSmileRight', 'mouth_smile_right'],
400
+ oh: ['viseme_oh', 'viseme_O', 'oh', 'mouthFunnel', 'mouth_funnel'],
401
+ ou: ['viseme_ou', 'viseme_U', 'ou', 'mouthRollLower', 'mouth_roll_lower'],
402
+ };
403
+
404
+ // For ARKit models, each viseme may need multiple blend shapes driven together.
405
+ // Each entry is a list of morph names to combine for that viseme.
406
+ // The first alias list that has ANY match on the model is used.
407
+ const VISEME_COMPOUND_ARKIT = {
408
+ aa: [['jawOpen', 'mouthLowerDownLeft', 'mouthLowerDownRight'], ['jawOpen', 'mouthOpen']],
409
+ oh: [['mouthFunnel', 'jawOpen'], ['mouthFunnel']],
410
+ ou: [['mouthPucker', 'mouthRollLower'], ['mouthPucker']],
411
+ PP: [['mouthPucker', 'mouthClose'], ['mouthPucker']],
412
+ FF: [['mouthRollLower', 'mouthLowerDownLeft', 'mouthLowerDownRight'], ['mouthShrugLower']],
413
+ CH: [['mouthSmileLeft', 'mouthSmileRight', 'mouthStretchLeft', 'mouthStretchRight'], ['mouthSmileLeft', 'mouthSmileRight']],
414
+ ee: [['mouthSmileLeft', 'mouthSmileRight'], ['mouthSmileLeft']],
415
+ ih: [['mouthSmileLeft', 'mouthSmileRight'], ['mouthSmileRight']],
350
416
  };
351
417
 
352
418
  function buildVisemeMorphCache() {
353
419
  visemeMorphCache = {};
354
420
  for (const [visemeKey, aliases] of Object.entries(VISEME_MORPH_ALIASES)) {
355
421
  const entries = [];
422
+ // Check if we have a compound ARKit mapping for this viseme
423
+ const compoundOptions = VISEME_COMPOUND_ARKIT[visemeKey];
424
+ if (compoundOptions) {
425
+ // Try each compound option; use the first one where all names exist on at least one mesh
426
+ let usedCompound = false;
427
+ for (const nameList of compoundOptions) {
428
+ // Collect entries for all names across all meshes
429
+ const compoundEntries = [];
430
+ for (const mesh of mouthMeshes) {
431
+ if (!mesh.morphTargetDictionary) continue;
432
+ const dict = mesh.morphTargetDictionary;
433
+ const dictKeysLower = Object.fromEntries(Object.keys(dict).map(k => [k.toLowerCase(), k]));
434
+ for (const name of nameList) {
435
+ const found = dictKeysLower[name.toLowerCase()];
436
+ if (found !== undefined) {
437
+ compoundEntries.push({ influences: mesh.morphTargetInfluences, idx: dict[found], morphName: found });
438
+ }
439
+ }
440
+ }
441
+ if (compoundEntries.length > 0) {
442
+ entries.push(...compoundEntries);
443
+ usedCompound = true;
444
+ break;
445
+ }
446
+ }
447
+ if (usedCompound) {
448
+ visemeMorphCache[visemeKey] = entries;
449
+ continue;
450
+ }
451
+ }
452
+ // Fallback: single-alias lookup
356
453
  for (const mesh of mouthMeshes) {
357
454
  if (!mesh.morphTargetDictionary) continue;
358
455
  const dictKeys = Object.keys(mesh.morphTargetDictionary);
359
456
  for (const alias of aliases) {
360
457
  const found = dictKeys.find(k => k.toLowerCase() === alias.toLowerCase());
361
458
  if (found !== undefined) {
362
- entries.push({ influences: mesh.morphTargetInfluences, idx: mesh.morphTargetDictionary[found] });
459
+ entries.push({ influences: mesh.morphTargetInfluences, idx: mesh.morphTargetDictionary[found], morphName: found });
363
460
  break;
364
461
  }
365
462
  }
@@ -368,6 +465,12 @@ function buildVisemeMorphCache() {
368
465
  }
369
466
  const found = Object.keys(visemeMorphCache);
370
467
  log('Viseme cache: ' + (found.length > 0 ? found.join(', ') : 'none — check morph target names'));
468
+
469
+ // Always log all available morphs so we can see what the model actually has
470
+ if (mouthMeshes.length > 0) {
471
+ const allMorphs = Object.keys(mouthMeshes[0].morphTargetDictionary || {});
472
+ log('Available morphs: ' + (allMorphs.length > 0 ? allMorphs.join(', ') : 'none'));
473
+ }
371
474
  }
372
475
 
373
476
  function applyViseme(visemeKey, weight) {
@@ -377,39 +480,42 @@ function applyViseme(visemeKey, weight) {
377
480
  return;
378
481
  }
379
482
  visemeState[visemeKey] = Math.min(1, weight);
483
+ visemeStateLastSet.set(visemeKey, Date.now());
380
484
  }
381
485
 
382
- function tickVisemeDecay() {
383
- if (!visemeMorphCache) return;
384
- for (const [key, weight] of Object.entries(visemeState)) {
385
- const decayed = weight * 0.82;
386
- visemeState[key] = decayed < 0.01 ? 0 : decayed;
387
- const entries = visemeMorphCache[key];
388
- if (!entries) continue;
389
- for (const e of entries) e.influences[e.idx] = visemeState[key];
390
- }
391
- }
392
-
393
- // ============ RHUBARB VISEME SCHEDULER ============
394
- // Rhubarb mouth shapes A-H, X mapped to the closest standard viseme morph targets.
395
- // Amplitude fallback is gated while a schedule is active (visemeModeUntil).
486
+ const RHUBARB_DEFAULT_VISEME_WEIGHT = 0.72;
487
+ const RHUBARB_LABIAL_VISEME_WEIGHT = 0.85;
488
+ const RHUBARB_AA_VISEME_WEIGHT = 0.72;
489
+ const RHUBARB_ROUNDED_VISEME_WEIGHT = 0.62;
490
+ const RHUBARB_FALLBACK_AMPLITUDE_CAP = 0.72;
491
+ const RHUBARB_FALLBACK_AMPLITUDE_GAIN = 0.75;
492
+ const RHUBARB_VISEME_WEIGHTS = {
493
+ PP: RHUBARB_LABIAL_VISEME_WEIGHT,
494
+ FF: 0.78,
495
+ ee: 0.72,
496
+ ih: 0.68,
497
+ oh: RHUBARB_ROUNDED_VISEME_WEIGHT,
498
+ ou: 0.58,
499
+ aa: RHUBARB_AA_VISEME_WEIGHT,
500
+ };
396
501
 
397
502
  const RHUBARB_TO_VISEME = {
398
- X: 'sil', // silence — mouth closed
399
- A: 'PP', // m, b, p — lips together
400
- B: 'kk', // k, s, t — slightly open
401
- C: 'ee', // e as in bed — open with smile
402
- D: 'aa', // aa as in father — wide open
403
- E: 'oh', // eh/uh — rounded open
404
- F: 'ou', // oo/w — puckered
405
- G: 'FF', // f, v — teeth on lip
406
- H: 'ih', // ee as in see — wide smile
503
+ A: 'aa',
504
+ B: 'PP',
505
+ C: 'ih',
506
+ D: 'FF',
507
+ E: 'ee',
508
+ F: 'oh',
509
+ G: 'ou',
510
+ H: 'nn',
511
+ X: 'sil',
407
512
  };
408
513
 
409
514
  let rhubarbMorphCache = null;
410
515
  let visemeTimers = [];
411
516
  let activeVisemeScheduleId = 0;
412
517
  let visemeModeUntil = 0;
518
+ const visemeStateLastSet = new Map();
413
519
 
414
520
  function buildRhubarbMorphCache() {
415
521
  if (!visemeMorphCache) buildVisemeMorphCache();
@@ -428,7 +534,8 @@ function applyRhubarbCue(shape) {
428
534
  if (shape === 'X' || !rhubarbMorphCache[shape]) return;
429
535
  const visemeKey = RHUBARB_TO_VISEME[shape];
430
536
  if (visemeKey && visemeKey !== 'sil') {
431
- visemeState[visemeKey] = 1.0;
537
+ visemeState[visemeKey] = RHUBARB_VISEME_WEIGHTS[visemeKey] || RHUBARB_DEFAULT_VISEME_WEIGHT;
538
+ visemeStateLastSet.set(visemeKey, Date.now());
432
539
  }
433
540
  }
434
541
 
@@ -441,20 +548,98 @@ function clearScheduledVisemes() {
441
548
  for (const key of Object.keys(visemeState)) visemeState[key] = 0;
442
549
  }
443
550
 
551
+ function tickVisemeDecay() {
552
+ if (!visemeMorphCache) return;
553
+
554
+ const isScheduled = Date.now() < visemeModeUntil;
555
+ const hasSpecificLipShape =
556
+ visemeState.PP > 0.05 ||
557
+ visemeState.FF > 0.05 ||
558
+ visemeState.kk > 0.05 ||
559
+ visemeState.ee > 0.05 ||
560
+ visemeState.ih > 0.05;
561
+
562
+ for (const [key, weight] of Object.entries(visemeState)) {
563
+ // Only decay if we aren't in the middle of a viseme schedule.
564
+ // Scheduled visemes are cleared manually by timeouts.
565
+ if (!isScheduled) {
566
+ const decayed = weight * 0.82;
567
+ visemeState[key] = decayed < 0.01 ? 0 : decayed;
568
+ }
569
+
570
+ const entries = visemeMorphCache[key];
571
+ if (!entries) continue;
572
+
573
+ let targetWeight = visemeState[key];
574
+ if (key === 'aa' && hasSpecificLipShape) targetWeight = Math.min(targetWeight, 0.45);
575
+
576
+ for (const e of entries) {
577
+ // When TalkingHead is active, write through its morph API so the internal
578
+ // render loop doesn't overwrite our values every frame.
579
+ // Use realtime (not newvalue) — newvalue is consumed and cleared after
580
+ // a single frame, so scheduled visemes would vanish immediately.
581
+ // realtime persists until explicitly set to null.
582
+ if (head?.mtAvatar && e.morphName && head.mtAvatar[e.morphName]) {
583
+ const mt = head.mtAvatar[e.morphName];
584
+ mt.realtime = targetWeight > 0 ? targetWeight : null;
585
+ mt.needsUpdate = true;
586
+ } else {
587
+ e.influences[e.idx] = targetWeight;
588
+ }
589
+ }
590
+ }
591
+ }
592
+
444
593
  function scheduleVisemes(schedule) {
445
594
  clearScheduledVisemes();
595
+
596
+ // Prune visemeState keys that haven't been written in the last 2 seconds to
597
+ // prevent unbounded accumulation across many utterances.
598
+ const staleThreshold = Date.now() - 2000;
599
+ for (const key of Object.keys(visemeState)) {
600
+ if ((visemeStateLastSet.get(key) ?? 0) < staleThreshold) {
601
+ delete visemeState[key];
602
+ visemeStateLastSet.delete(key);
603
+ }
604
+ }
605
+
446
606
  if (!schedule || !Array.isArray(schedule.cues) || schedule.cues.length === 0) return;
447
607
 
448
608
  const myScheduleId = activeVisemeScheduleId;
449
- const startedAt = schedule.startedAtMs || Date.now();
609
+ // The startedAtMs anchor is set when tts_request_start arrives on the data
610
+ // channel. Audio doesn't play until ~300ms later (LiveKit audio buffering).
611
+ // TTS generation delay is no longer included here since visemes now arrive
612
+ // via direct ref call before the React render cycle.
613
+ const AUDIO_PIPELINE_DELAY_MS = 300;
614
+ let startedAt = (schedule.startedAtMs || Date.now()) + AUDIO_PIPELINE_DELAY_MS;
450
615
  const durationMs = schedule.durationMs || 0;
616
+ const now = Date.now();
617
+ let elapsedMs = Math.max(0, now - startedAt);
618
+
619
+ // If the schedule still arrives late after the pipeline offset, shift further
620
+ if (elapsedMs > 300 && schedule.cues.length > 3) {
621
+ const shift = Math.min(elapsedMs - 50, 500);
622
+ startedAt += shift;
623
+ elapsedMs -= shift;
624
+ log('Viseme schedule arrived late, shifting anchor forward by ' + shift + 'ms');
625
+ }
451
626
 
452
- // Gate amplitude fallback for the full duration plus a small buffer
453
- visemeModeUntil = startedAt + durationMs + 200;
627
+ const remainingMs = Math.max(0, durationMs - elapsedMs);
628
+ let scheduledCueCount = 0;
629
+ let skippedCueCount = 0;
630
+
631
+ // Gate amplitude fallback for the locally remaining duration plus a small buffer.
632
+ // If the schedule arrives a bit late, keep amplitude out of the way for the rest
633
+ // of the utterance instead of expiring immediately from the original timestamp.
634
+ visemeModeUntil = now + remainingMs + 200;
454
635
 
455
636
  for (const cue of schedule.cues) {
456
637
  const delay = cue.startMs - (Date.now() - startedAt);
457
- if (delay < -50) continue; // already in the past, skip
638
+ if (delay < -50) {
639
+ skippedCueCount++;
640
+ continue; // already in the past, skip
641
+ }
642
+ scheduledCueCount++;
458
643
 
459
644
  const applyId = setTimeout(() => {
460
645
  if (activeVisemeScheduleId !== myScheduleId) return;
@@ -470,6 +655,16 @@ function scheduleVisemes(schedule) {
470
655
  visemeTimers.push(applyId, clearId);
471
656
  }
472
657
 
658
+ log(
659
+ 'Viseme schedule received: requestId=' +
660
+ (schedule.requestId || 'unknown') +
661
+ ' cues=' + schedule.cues.length +
662
+ ' scheduled=' + scheduledCueCount +
663
+ ' skipped=' + skippedCueCount +
664
+ ' elapsedMs=' + elapsedMs +
665
+ ' remainingMs=' + remainingMs,
666
+ );
667
+
473
668
  // Ensure silence at schedule end
474
669
  const endDelay = durationMs - (Date.now() - startedAt);
475
670
  if (endDelay > 0) {
@@ -639,7 +834,10 @@ function onIncomingMessage(event) {
639
834
  if (jawKey !== undefined) jawMorphCache.push({ influences: mesh.morphTargetInfluences, idx: mesh.morphTargetDictionary[jawKey] });
640
835
  }
641
836
  }
642
- const val = Math.min(1, msg.value * 1.8);
837
+ const val = Math.min(
838
+ RHUBARB_FALLBACK_AMPLITUDE_CAP,
839
+ msg.value * RHUBARB_FALLBACK_AMPLITUDE_GAIN,
840
+ );
643
841
  amplitudeDecay = Math.max(amplitudeDecay * 0.7, val);
644
842
  for (let i = 0; i < jawMorphCache.length; i++) jawMorphCache[i].influences[jawMorphCache[i].idx] = amplitudeDecay;
645
843
  } else if (msg.type === 'viseme') {
@@ -667,7 +865,8 @@ function onIncomingMessage(event) {
667
865
  window.addEventListener('message', onIncomingMessage);
668
866
  document.addEventListener('message', onIncomingMessage);
669
867
 
670
- init();
868
+ await init();
869
+ })();
671
870
  </script>
672
871
  </body>
673
872
  </html>
package/dist/index.d.ts CHANGED
@@ -1,3 +1,10 @@
1
- export type { TalkingHeadAccessory, TalkingHeadMood, TalkingHeadProps, TalkingHeadRef, TalkingHeadViseme, TalkingHeadVisemeCue, TalkingHeadVisemeSchedule, } from './TalkingHead';
1
+ export type { TalkingHeadAccessory, TalkingHeadLoadingState, TalkingHeadLoadingStage, TalkingHeadMood, TalkingHeadProps, TalkingHeadRef, TalkingHeadViseme, TalkingHeadVisemeCue, TalkingHeadVisemeSchedule, } from './TalkingHead';
2
2
  export { TalkingHead } from './TalkingHead';
3
- export * from './appearance';
3
+ export { applyAppearanceToObject3D } from './appearance/apply';
4
+ export { pickTargetForMaterialName } from './appearance/matchers';
5
+ export { normalizeAppearance } from './appearance/schema';
6
+ export type { AppearanceTarget } from './appearance/matchers';
7
+ export * from './api';
8
+ export * from './wardrobe';
9
+ export { TalkingHeadVisualization } from './TalkingHeadVisualization';
10
+ export type { TalkingHeadVisualizationRef } from './TalkingHeadVisualization';
package/dist/index.js CHANGED
@@ -14,7 +14,18 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
14
14
  for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
15
15
  };
16
16
  Object.defineProperty(exports, "__esModule", { value: true });
17
- exports.TalkingHead = void 0;
17
+ exports.TalkingHeadVisualization = exports.normalizeAppearance = exports.pickTargetForMaterialName = exports.applyAppearanceToObject3D = exports.TalkingHead = void 0;
18
18
  var TalkingHead_1 = require("./TalkingHead");
19
19
  Object.defineProperty(exports, "TalkingHead", { enumerable: true, get: function () { return TalkingHead_1.TalkingHead; } });
20
- __exportStar(require("./appearance"), exports);
20
+ // Export appearance utilities, but exclude AvatarAppearance — the canonical
21
+ // AvatarAppearance (with equippedAccessories) comes from ./api below.
22
+ var apply_1 = require("./appearance/apply");
23
+ Object.defineProperty(exports, "applyAppearanceToObject3D", { enumerable: true, get: function () { return apply_1.applyAppearanceToObject3D; } });
24
+ var matchers_1 = require("./appearance/matchers");
25
+ Object.defineProperty(exports, "pickTargetForMaterialName", { enumerable: true, get: function () { return matchers_1.pickTargetForMaterialName; } });
26
+ var schema_1 = require("./appearance/schema");
27
+ Object.defineProperty(exports, "normalizeAppearance", { enumerable: true, get: function () { return schema_1.normalizeAppearance; } });
28
+ __exportStar(require("./api"), exports);
29
+ __exportStar(require("./wardrobe"), exports);
30
+ var TalkingHeadVisualization_1 = require("./TalkingHeadVisualization");
31
+ Object.defineProperty(exports, "TalkingHeadVisualization", { enumerable: true, get: function () { return TalkingHeadVisualization_1.TalkingHeadVisualization; } });
package/dist/index.web.d.ts CHANGED
@@ -1,3 +1,7 @@
1
- export type { TalkingHeadAccessory, TalkingHeadMood, TalkingHeadProps, TalkingHeadRef, } from './TalkingHead.web';
1
+ export type { TalkingHeadAccessory, TalkingHeadLoadingState, TalkingHeadMood, TalkingHeadProps, TalkingHeadRef, } from './TalkingHead.web';
2
2
  export { TalkingHead } from './TalkingHead.web';
3
- export * from './appearance';
3
+ export { applyAppearanceToObject3D } from './appearance/apply';
4
+ export { pickTargetForMaterialName } from './appearance/matchers';
5
+ export { normalizeAppearance } from './appearance/schema';
6
+ export type { AppearanceTarget } from './appearance/matchers';
7
+ export * from './api';
package/dist/index.web.js CHANGED
@@ -14,7 +14,15 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
14
14
  for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
15
15
  };
16
16
  Object.defineProperty(exports, "__esModule", { value: true });
17
- exports.TalkingHead = void 0;
17
+ exports.normalizeAppearance = exports.pickTargetForMaterialName = exports.applyAppearanceToObject3D = exports.TalkingHead = void 0;
18
18
  var TalkingHead_web_1 = require("./TalkingHead.web");
19
19
  Object.defineProperty(exports, "TalkingHead", { enumerable: true, get: function () { return TalkingHead_web_1.TalkingHead; } });
20
- __exportStar(require("./appearance"), exports);
20
+ // Export appearance utilities, but exclude AvatarAppearance — the canonical
21
+ // AvatarAppearance (with equippedAccessories) comes from ./api below.
22
+ var apply_1 = require("./appearance/apply");
23
+ Object.defineProperty(exports, "applyAppearanceToObject3D", { enumerable: true, get: function () { return apply_1.applyAppearanceToObject3D; } });
24
+ var matchers_1 = require("./appearance/matchers");
25
+ Object.defineProperty(exports, "pickTargetForMaterialName", { enumerable: true, get: function () { return matchers_1.pickTargetForMaterialName; } });
26
+ var schema_1 = require("./appearance/schema");
27
+ Object.defineProperty(exports, "normalizeAppearance", { enumerable: true, get: function () { return schema_1.normalizeAppearance; } });
28
+ __exportStar(require("./api"), exports);
package/dist/utils/avatarUtils.d.ts ADDED
@@ -0,0 +1,13 @@
1
+ /**
2
+ * Resolves a local Expo asset module into a file:// URI for use with
3
+ * react-native-filament's useModel(). Never base64-encodes — Filament reads
4
+ * file:// directly on both iOS and Android.
5
+ */
6
+ export declare function resolveFilamentAssetUri(module: unknown): Promise<string | null>;
7
+ /**
8
+ * Resolves a local Expo asset module (from require()) into a usable URL string.
9
+ * On web, returns an absolute URL.
10
+ * On Android/iOS, converts to a base64 data URI so the WebView can load it
11
+ * without a cross-origin file:// fetch (which is blocked on Android).
12
+ */
13
+ export declare function resolveLocalAssetUrl(module: unknown): Promise<string | null>;
package/dist/utils/avatarUtils.js ADDED
@@ -0,0 +1,56 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.resolveFilamentAssetUri = resolveFilamentAssetUri;
4
+ exports.resolveLocalAssetUrl = resolveLocalAssetUrl;
5
+ const expo_asset_1 = require("expo-asset");
6
+ const react_native_1 = require("react-native");
7
+ const expo_file_system_1 = require("expo-file-system");
8
+ /**
9
+ * Resolves a local Expo asset module into a file:// URI for use with
10
+ * react-native-filament's useModel(). Never base64-encodes — Filament reads
11
+ * file:// directly on both iOS and Android.
12
+ */
13
+ async function resolveFilamentAssetUri(module) {
14
+ try {
15
+ const asset = expo_asset_1.Asset.fromModule(module);
16
+ await asset.downloadAsync();
17
+ const uri = asset.localUri ?? asset.uri;
18
+ return uri ?? null;
19
+ }
20
+ catch (e) {
21
+ console.error('[AssetUtils] Failed to resolve Filament asset:', e);
22
+ return null;
23
+ }
24
+ }
25
+ /**
26
+ * Resolves a local Expo asset module (from require()) into a usable URL string.
27
+ * On web, returns an absolute URL.
28
+ * On Android/iOS, converts to a base64 data URI so the WebView can load it
29
+ * without a cross-origin file:// fetch (which is blocked on Android).
30
+ */
31
+ async function resolveLocalAssetUrl(module) {
32
+ try {
33
+ const asset = expo_asset_1.Asset.fromModule(module);
34
+ await asset.downloadAsync();
35
+ const uri = asset.localUri ?? asset.uri;
36
+ if (!uri)
37
+ return null;
38
+ if (react_native_1.Platform.OS === 'web') {
39
+ if (uri.startsWith('/')) {
40
+ return window.location.origin + uri;
41
+ }
42
+ return uri;
43
+ }
44
+ // On Android the WebView cannot fetch file:// URIs — convert to data URI.
45
+ if (react_native_1.Platform.OS === 'android' && uri.startsWith('file://')) {
46
+ const file = new expo_file_system_1.File(uri);
47
+ const base64 = await file.base64();
48
+ return `data:model/gltf-binary;base64,${base64}`;
49
+ }
50
+ return uri;
51
+ }
52
+ catch (e) {
53
+ console.error('[AssetUtils] Failed to resolve local asset:', e);
54
+ return null;
55
+ }
56
+ }
package/dist/wardrobe/index.d.ts ADDED
@@ -0,0 +1,2 @@
1
+ export * from './wardrobeStore';
2
+ export { useAvatarWardrobeHydration } from './useAvatarWardrobeHydration';
package/dist/wardrobe/index.js ADDED
@@ -0,0 +1,20 @@
1
+ "use strict";
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ var desc = Object.getOwnPropertyDescriptor(m, k);
5
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
+ desc = { enumerable: true, get: function() { return m[k]; } };
7
+ }
8
+ Object.defineProperty(o, k2, desc);
9
+ }) : (function(o, m, k, k2) {
10
+ if (k2 === undefined) k2 = k;
11
+ o[k2] = m[k];
12
+ }));
13
+ var __exportStar = (this && this.__exportStar) || function(m, exports) {
14
+ for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
15
+ };
16
+ Object.defineProperty(exports, "__esModule", { value: true });
17
+ exports.useAvatarWardrobeHydration = void 0;
18
+ __exportStar(require("./wardrobeStore"), exports);
19
+ var useAvatarWardrobeHydration_1 = require("./useAvatarWardrobeHydration");
20
+ Object.defineProperty(exports, "useAvatarWardrobeHydration", { enumerable: true, get: function () { return useAvatarWardrobeHydration_1.useAvatarWardrobeHydration; } });
package/dist/wardrobe/useAvatarWardrobeHydration.d.ts ADDED
@@ -0,0 +1,7 @@
1
+ import type { EquippedAccessory } from '../api/types';
2
+ type UseAvatarWardrobeHydrationArgs = {
3
+ avatarId: string | null | undefined;
4
+ accessories: EquippedAccessory[] | null | undefined;
5
+ };
6
+ export declare function useAvatarWardrobeHydration({ avatarId, accessories, }: UseAvatarWardrobeHydrationArgs): void;
7
+ export {};