npm - talking-head-studio - Versions diffs - 0.2.3 → 0.2.5 - Mend

talking-head-studio 0.2.3 → 0.2.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (15) hide show

package/dist/TalkingHead.d.ts +54 -0
package/dist/TalkingHead.d.ts.map +1 -1
package/dist/TalkingHead.js +36 -29
package/dist/TalkingHead.web.d.ts +4 -0
package/dist/TalkingHead.web.d.ts.map +1 -1
package/dist/TalkingHead.web.js +2 -0
package/dist/html.d.ts.map +1 -1
package/dist/html.js +232 -145
package/dist/index.d.ts +1 -1
package/dist/index.d.ts.map +1 -1
package/package.json +1 -1
package/src/TalkingHead.tsx +100 -31
package/src/TalkingHead.web.tsx +10 -0
package/src/html.ts +234 -145
package/src/index.ts +3 -0

package/src/html.ts CHANGED Viewed

@@ -10,7 +10,12 @@ export type AvatarConfig = {
   initialEyeColor?: string;
 };
+const VALID_MOODS = new Set(['neutral','happy','sad','angry','excited','thinking','concerned','surprised']);
 export function buildAvatarHtml(config: AvatarConfig): string {
+  // Sanitize mood at build time so the WebView never receives an invalid value
+  const safeMood = VALID_MOODS.has(config.mood) ? config.mood : 'neutral';
   return `
 <!DOCTYPE html>
 <html>
@@ -40,8 +45,8 @@ const HEAD_AUDIO_URL = 'https://cdn.jsdelivr.net/gh/met4citizen/HeadAudio@main/d
 const HEAD_AUDIO_WORKLET = 'https://cdn.jsdelivr.net/gh/met4citizen/HeadAudio@main/dist/headworklet.min.mjs';
 const HEAD_AUDIO_MODEL = 'https://cdn.jsdelivr.net/gh/met4citizen/HeadAudio@main/dist/model-en-mixed.bin';
-const AVATAR_URL = ${JSON.stringify(config.avatarUrl)};
-const INITIAL_MOOD = ${JSON.stringify(config.mood)};
+let AVATAR_URL = ${JSON.stringify(config.avatarUrl)};
+const INITIAL_MOOD = ${JSON.stringify(safeMood)};
 const CAMERA_VIEW = ${JSON.stringify(config.cameraView)};
 const CAMERA_DISTANCE = ${config.cameraDistance};
 let HAIR_COLOR = ${JSON.stringify(config.initialHairColor ?? null)};
@@ -54,11 +59,15 @@ let headaudio = null;
 let mouthMeshes = [];
 let staticModel = null;
+// Guard: prevent multiple concurrent init() calls (React StrictMode / rapid remounts)
+let initStarted = false;
 function log(msg) {
   window.ReactNativeWebView?.postMessage(JSON.stringify({ type: 'log', message: msg }));
 }
 async function loadWithAuth(url) {
+  if (!url) throw new Error('Avatar URL is empty');
   if (AUTH_TOKEN && !url.startsWith('https://cdn.jsdelivr.net')) {
     log('Fetching authenticated model: ' + url);
     const resp = await fetch(url, { headers: { 'Authorization': 'Bearer ' + AUTH_TOKEN } });
@@ -105,9 +114,9 @@ async function loadStaticFallback(loadedAvatarUrl) {
     const camera = new THREE.PerspectiveCamera(45, window.innerWidth / window.innerHeight, 0.1, 100);
     camera.position.set(0, 0, 3);
-    const renderer = new THREE.WebGLRenderer({ alpha: true, antialias: true });
+    const renderer = new THREE.WebGLRenderer({ alpha: true, antialias: true, powerPreference: 'low-power' });
     renderer.setSize(window.innerWidth, window.innerHeight);
-    renderer.setPixelRatio(window.devicePixelRatio);
+    renderer.setPixelRatio(Math.min(window.devicePixelRatio, 2));
     container.appendChild(renderer.domElement);
     const controls = new OrbitControls(camera, renderer.domElement);
@@ -125,45 +134,36 @@ async function loadStaticFallback(loadedAvatarUrl) {
     loader.load(loadedAvatarUrl, (gltf) => {
       staticModel = gltf.scene;
       applyColorOverrides();
       scene.add(staticModel);
-      // Play any embedded animations (walk cycles, idle, etc.)
       if (gltf.animations && gltf.animations.length > 0) {
         staticMixer = new THREE.AnimationMixer(staticModel);
-        for (const clip of gltf.animations) {
-          staticMixer.clipAction(clip).play();
-        }
+        for (const clip of gltf.animations) staticMixer.clipAction(clip).play();
         log('Playing ' + gltf.animations.length + ' animation(s)');
       }
-      // Collect morph targets for amplitude-driven jaw/mouth
-      jawMorphCache = null; // invalidate cache whenever meshes are repopulated
+      jawMorphCache = null;
+      visemeMorphCache = null;
+      rhubarbMorphCache = null;
       staticModel.traverse((child) => {
         if (child.isMesh && child.morphTargetDictionary && child.morphTargetInfluences) {
           mouthMeshes.push(child);
         }
       });
-      if (mouthMeshes.length > 0) {
-        log('Found ' + mouthMeshes.length + ' mesh(es) with morph targets');
-      }
+      if (mouthMeshes.length > 0) log('Found ' + mouthMeshes.length + ' mesh(es) with morph targets');
-      // Auto-frame the model
       const box = new THREE.Box3().setFromObject(staticModel);
       const size = box.getSize(new THREE.Vector3());
       const maxDim = Math.max(size.x, size.y, size.z);
       if (maxDim > 0 && maxDim !== Infinity) {
         const scale = 1.5 / maxDim;
         staticModel.scale.set(scale, scale, scale);
       }
       const scaledBox = new THREE.Box3().setFromObject(staticModel);
       const scaledCenter = scaledBox.getCenter(new THREE.Vector3());
       staticModel.position.sub(scaledCenter);
       applyAccessories(pendingAccessoriesList);
       window.ReactNativeWebView?.postMessage(JSON.stringify({ type: 'ready' }));
       window.addEventListener('resize', () => {
@@ -175,14 +175,12 @@ async function loadStaticFallback(loadedAvatarUrl) {
       renderer.setAnimationLoop(() => {
         const delta = clock.getDelta();
         if (staticMixer) staticMixer.update(delta);
+        tickVisemeDecay();
         controls.update();
         renderer.render(scene, camera);
       });
     }, (ev) => {
-      if (ev.lengthComputable) {
-        const pct = Math.round((ev.loaded / ev.total) * 100);
-        log('Fallback Loading: ' + pct + '%');
-      }
+      if (ev.lengthComputable) log('Fallback Loading: ' + Math.round((ev.loaded / ev.total) * 100) + '%');
     }, (err) => {
       log('Fallback Error: ' + err.message);
       window.ReactNativeWebView?.postMessage(JSON.stringify({ type: 'error', message: err.message }));
@@ -194,6 +192,17 @@ async function loadStaticFallback(loadedAvatarUrl) {
 }
 async function init() {
+  // Prevent concurrent inits from React StrictMode double-invoke or rapid remounts
+  if (initStarted) { log('init() already running, skipping duplicate'); return; }
+  initStarted = true;
+  // If avatar URL is empty wait for a set_avatar_url message rather than crashing
+  if (!AVATAR_URL) {
+    log('No avatarUrl yet — waiting for set_avatar_url message');
+    initStarted = false;
+    return;
+  }
   try {
     log('Loading TalkingHead...');
     const module = await import(TALKING_HEAD_URL);
@@ -222,26 +231,26 @@ async function init() {
       await head.showAvatar({
         url: loadedAvatarUrl,
         body: 'F',
-        avatarMood: INITIAL_MOOD,
+        avatarMood: INITIAL_MOOD, // always 'neutral' or a valid mood — sanitized above
         lipsyncLang: 'en',
       }, (ev) => {
         if (ev.lengthComputable) {
-          const pct = Math.round((ev.loaded / ev.total) * 100);
-          log('Loading: ' + pct + '%');
+          log('Loading: ' + Math.round((ev.loaded / ev.total) * 100) + '%');
         }
       });
       if (loadedAvatarUrl.startsWith('blob:')) URL.revokeObjectURL(loadedAvatarUrl);
       avatarLoaded = true;
     } catch (avatarErr) {
-      // Non-rigged avatars (no Armature) are expected — fall back to static viewer silently
       log('No armature found, using static viewer: ' + avatarErr.message);
       await loadStaticFallback(loadedAvatarUrl);
       if (loadedAvatarUrl.startsWith('blob:')) URL.revokeObjectURL(loadedAvatarUrl);
-      return; // fallback posts 'ready' itself; don't post 'error'
+      return;
     }
     if (avatarLoaded) {
-      jawMorphCache = null; // invalidate cache whenever meshes are repopulated
+      jawMorphCache = null;
+      visemeMorphCache = null;
+      rhubarbMorphCache = null;
       head.armature?.traverse((child) => {
         if (child.isMesh && child.morphTargetDictionary && child.morphTargetInfluences) {
           mouthMeshes.push(child);
@@ -253,13 +262,12 @@ async function init() {
       log('Initializing HeadAudio...');
       try {
         const haModule = await import(HEAD_AUDIO_URL);
-        if (haModule.HeadAudio && head.audioCtx) {
+        // AudioWorklet is not supported in React Native WebViews (iOS/Android).
+        if (haModule.HeadAudio && head.audioCtx && head.audioCtx.audioWorklet) {
           await head.audioCtx.audioWorklet.addModule(HEAD_AUDIO_WORKLET);
           headaudio = new haModule.HeadAudio(head.audioCtx);
           await headaudio.loadModel(HEAD_AUDIO_MODEL);
-          if (head.audioSpeechGainNode) {
-            head.audioSpeechGainNode.connect(headaudio);
-          }
+          if (head.audioSpeechGainNode) head.audioSpeechGainNode.connect(headaudio);
           headaudio.onvalue = (key, value) => {
             if (head.mtAvatar && head.mtAvatar[key]) {
               Object.assign(head.mtAvatar[key], { newvalue: value, needsUpdate: true });
@@ -267,9 +275,11 @@ async function init() {
           };
           head.opt.update = headaudio.update.bind(headaudio);
           log('HeadAudio ready (phoneme lip sync)');
+        } else {
+          log('HeadAudio skipped: AudioWorklet not supported in this WebView. Use sendViseme() from native TTS callbacks.');
         }
       } catch (err) {
-        log('HeadAudio unavailable, amplitude fallback active: ' + err.message);
+        log('HeadAudio unavailable, viseme/amplitude fallback active: ' + err.message);
       }
       startAudioInterception();
@@ -278,6 +288,7 @@ async function init() {
       window.ReactNativeWebView?.postMessage(JSON.stringify({ type: 'ready' }));
     }
   } catch (err) {
+    initStarted = false; // allow retry on error
     log('Error: ' + err.message);
     window.ReactNativeWebView?.postMessage(JSON.stringify({ type: 'error', message: err.message }));
   }
@@ -294,27 +305,19 @@ function startAudioInterception() {
     const stream = el.srcObject;
     if (!stream || connected.has(stream)) return !!stream;
     connected.add(stream);
-    try {
-      audioCtx.createMediaStreamSource(stream).connect(gainNode);
-    } catch (e) {
-      log('Audio routing error: ' + e);
-    }
+    try { audioCtx.createMediaStreamSource(stream).connect(gainNode); } catch (e) { log('Audio routing error: ' + e); }
     return true;
   }
   function track(el) {
     if (!tryConnect(el)) pending.add(el);
-    // Listen for play events which happen when stream is ready to avoid polling
-    const onReady = () => {
-      if (tryConnect(el)) pending.delete(el);
-    };
+    const onReady = () => { if (tryConnect(el)) pending.delete(el); };
     el.addEventListener('play', onReady);
     el.addEventListener('playing', onReady);
     el.addEventListener('loadedmetadata', onReady);
   }
   document.querySelectorAll('audio').forEach((el) => track(el));
   new MutationObserver((mutations) => {
     for (const mut of mutations) {
       for (const node of mut.addedNodes) {
@@ -326,11 +329,164 @@ function startAudioInterception() {
 }
 let amplitudeDecay = 0;
-// Cached jaw morph-target indices — built once after model load, used every amplitude tick.
-let jawMorphCache = null; // [{influences, idx}] or null = not yet built
+let jawMorphCache = null;
+let visemeMorphCache = null;
+const visemeState = {};
+const VISEME_MORPH_ALIASES = {
+  sil: ['viseme_sil', 'sil'],
+  PP:  ['viseme_PP',  'pp',  'viseme_pp'],
+  FF:  ['viseme_FF',  'ff',  'viseme_ff'],
+  TH:  ['viseme_TH',  'th',  'viseme_th'],
+  DD:  ['viseme_DD',  'dd',  'viseme_dd'],
+  kk:  ['viseme_kk',  'kk',  'viseme_k'],
+  CH:  ['viseme_CH',  'ch',  'viseme_ch'],
+  SS:  ['viseme_SS',  'ss',  'viseme_s'],
+  nn:  ['viseme_nn',  'nn',  'viseme_n'],
+  RR:  ['viseme_RR',  'rr',  'viseme_r'],
+  aa:  ['viseme_aa',  'viseme_AA', 'aa', 'jawOpen', 'jaw_open', 'jawopen', 'mouthOpen', 'mouth_open', 'mouthopen'],
+  ee:  ['viseme_ee',  'viseme_E',  'ee'],
+  ih:  ['viseme_ih',  'viseme_I',  'ih'],
+  oh:  ['viseme_oh',  'viseme_O',  'oh'],
+  ou:  ['viseme_ou',  'viseme_U',  'ou'],
+};
+function buildVisemeMorphCache() {
+  visemeMorphCache = {};
+  for (const [visemeKey, aliases] of Object.entries(VISEME_MORPH_ALIASES)) {
+    const entries = [];
+    for (const mesh of mouthMeshes) {
+      if (!mesh.morphTargetDictionary) continue;
+      const dictKeys = Object.keys(mesh.morphTargetDictionary);
+      for (const alias of aliases) {
+        const found = dictKeys.find(k => k.toLowerCase() === alias.toLowerCase());
+        if (found !== undefined) {
+          entries.push({ influences: mesh.morphTargetInfluences, idx: mesh.morphTargetDictionary[found] });
+          break;
+        }
+      }
+    }
+    if (entries.length > 0) visemeMorphCache[visemeKey] = entries;
+  }
+  const found = Object.keys(visemeMorphCache);
+  log('Viseme cache: ' + (found.length > 0 ? found.join(', ') : 'none — check morph target names'));
+}
+function applyViseme(visemeKey, weight) {
+  if (!visemeMorphCache) buildVisemeMorphCache();
+  if (visemeKey === 'sil' || weight <= 0) {
+    for (const key of Object.keys(visemeState)) visemeState[key] = 0;
+    return;
+  }
+  visemeState[visemeKey] = Math.min(1, weight);
+}
+function tickVisemeDecay() {
+  if (!visemeMorphCache) return;
+  for (const [key, weight] of Object.entries(visemeState)) {
+    const decayed = weight * 0.82;
+    visemeState[key] = decayed < 0.01 ? 0 : decayed;
+    const entries = visemeMorphCache[key];
+    if (!entries) continue;
+    for (const e of entries) e.influences[e.idx] = visemeState[key];
+  }
+}
+// ============ RHUBARB VISEME SCHEDULER ============
+// Rhubarb mouth shapes A-H, X mapped to the closest standard viseme morph targets.
+// Amplitude fallback is gated while a schedule is active (visemeModeUntil).
+const RHUBARB_TO_VISEME = {
+  X: 'sil',  // silence — mouth closed
+  A: 'PP',   // m, b, p — lips together
+  B: 'kk',   // k, s, t — slightly open
+  C: 'ee',   // e as in bed — open with smile
+  D: 'aa',   // aa as in father — wide open
+  E: 'oh',   // eh/uh — rounded open
+  F: 'ou',   // oo/w — puckered
+  G: 'FF',   // f, v — teeth on lip
+  H: 'ih',   // ee as in see — wide smile
+};
+let rhubarbMorphCache = null;
+let visemeTimers = [];
+let activeVisemeScheduleId = 0;
+let visemeModeUntil = 0;
+function buildRhubarbMorphCache() {
+  if (!visemeMorphCache) buildVisemeMorphCache();
+  rhubarbMorphCache = {};
+  for (const [rhubarbShape, visemeKey] of Object.entries(RHUBARB_TO_VISEME)) {
+    if (visemeKey === 'sil') { rhubarbMorphCache[rhubarbShape] = null; continue; }
+    rhubarbMorphCache[rhubarbShape] = visemeMorphCache[visemeKey] || null;
+  }
+  log('Rhubarb morph cache built');
+}
+function applyRhubarbCue(shape) {
+  if (!rhubarbMorphCache) buildRhubarbMorphCache();
+  // Zero all active viseme channels first
+  for (const key of Object.keys(visemeState)) visemeState[key] = 0;
+  if (shape === 'X' || !rhubarbMorphCache[shape]) return;
+  const visemeKey = RHUBARB_TO_VISEME[shape];
+  if (visemeKey && visemeKey !== 'sil') {
+    visemeState[visemeKey] = 1.0;
+  }
+}
+function clearScheduledVisemes() {
+  activeVisemeScheduleId++;
+  for (const id of visemeTimers) clearTimeout(id);
+  visemeTimers = [];
+  visemeModeUntil = 0;
+  // Zero all mouth morphs
+  for (const key of Object.keys(visemeState)) visemeState[key] = 0;
+}
+function scheduleVisemes(schedule) {
+  clearScheduledVisemes();
+  if (!schedule || !Array.isArray(schedule.cues) || schedule.cues.length === 0) return;
+  const myScheduleId = activeVisemeScheduleId;
+  const startedAt = schedule.startedAtMs || Date.now();
+  const durationMs = schedule.durationMs || 0;
+  // Gate amplitude fallback for the full duration plus a small buffer
+  visemeModeUntil = startedAt + durationMs + 200;
+  for (const cue of schedule.cues) {
+    const delay = cue.startMs - (Date.now() - startedAt);
+    if (delay < -50) continue; // already in the past, skip
+    const applyId = setTimeout(() => {
+      if (activeVisemeScheduleId !== myScheduleId) return;
+      applyRhubarbCue(cue.viseme);
+    }, Math.max(0, delay));
+    const clearId = setTimeout(() => {
+      if (activeVisemeScheduleId !== myScheduleId) return;
+      // Only clear if the next cue hasn't already overwritten state
+      visemeState[RHUBARB_TO_VISEME[cue.viseme]] = 0;
+    }, Math.max(0, delay + (cue.endMs - cue.startMs)));
+    visemeTimers.push(applyId, clearId);
+  }
+  // Ensure silence at schedule end
+  const endDelay = durationMs - (Date.now() - startedAt);
+  if (endDelay > 0) {
+    visemeTimers.push(setTimeout(() => {
+      if (activeVisemeScheduleId !== myScheduleId) return;
+      for (const key of Object.keys(visemeState)) visemeState[key] = 0;
+      visemeModeUntil = 0;
+    }, endDelay + 100));
+  }
+}
+// ============ END RHUBARB VISEME SCHEDULER ============
 let THREE_REF = null;
 let gltfLoaderInstance = null;
-const currentAccessories = {}; // { [id]: { model: THREE.Group | null, url, bone, position, rotation, scale, isLoading: boolean, latestData: any } }
+const currentAccessories = {};
 let pendingAccessoriesList = [];
 function disposeHierarchy(node) {
@@ -359,7 +515,6 @@ async function ensureThree() {
     THREE_REF = await import('three');
     const { GLTFLoader } = await import('three/addons/loaders/GLTFLoader.js');
     const { DRACOLoader } = await import('three/addons/loaders/DRACOLoader.js');
     gltfLoaderInstance = new GLTFLoader();
     const dracoLoader = new DRACOLoader();
     dracoLoader.setDecoderPath('https://www.gstatic.com/draco/versioned/decoders/1.5.6/');
@@ -375,21 +530,16 @@ async function applyAccessories(accessoriesList) {
   log('[ACC] root=' + (root ? root.constructor.name + '/' + root.name : 'NULL') + ' head=' + !!head + ' head.armature=' + !!(head && head.armature) + ' staticModel=' + !!staticModel);
   if (!root) { log('[ACC] ABORT: no root'); return; }
-  // Debug: list all bones in the root
   const boneNames = [];
   root.traverse((child) => { if (child.isBone) boneNames.push(child.name); });
   log('[ACC] Bones found: ' + boneNames.join(', '));
   const newAccessoryIds = new Set(accessoriesList.map(a => a.id));
-  // Remove old ones
   for (const id in currentAccessories) {
     if (!newAccessoryIds.has(id)) {
       const acc = currentAccessories[id];
       if (acc.model) {
-        if (acc.model.parent) {
-          acc.model.parent.remove(acc.model);
-        }
+        if (acc.model.parent) acc.model.parent.remove(acc.model);
         disposeHierarchy(acc.model);
       }
       delete currentAccessories[id];
@@ -397,26 +547,17 @@ async function applyAccessories(accessoriesList) {
     }
   }
-  // Add or update
   for (const accData of accessoriesList) {
     log('[ACC] Processing: id=' + accData.id + ' url=' + accData.url + ' bone=' + accData.bone);
     const existing = currentAccessories[accData.id];
-    // If URL changed or it's new, we need to load it
     if (!existing || existing.url !== accData.url) {
       if (existing && existing.model) {
-        if (existing.model.parent) {
-          existing.model.parent.remove(existing.model);
-        }
+        if (existing.model.parent) existing.model.parent.remove(existing.model);
         disposeHierarchy(existing.model);
       }
-      // Mark as loading and store the latest transform data
       currentAccessories[accData.id] = { ...accData, model: null, isLoading: true, latestData: accData };
       log('[ACC] Starting GLB load: ' + accData.url);
-      // Use an IIFE to handle the async load with auth
       (async () => {
         try {
           const loadedUrl = await loadWithAuth(accData.url);
@@ -424,41 +565,20 @@ async function applyAccessories(accessoriesList) {
             if (loadedUrl.startsWith('blob:')) URL.revokeObjectURL(loadedUrl);
             log('[ACC] GLB loaded OK for ' + accData.id);
             const model = gltf.scene;
-            // Grab the *latest* data that might have arrived while we were loading
             const latestData = currentAccessories[accData.id]?.latestData || accData;
             let targetBone = null;
-            // Exact match first, then prefix match for Sketchfab exports (e.g. Head_5)
             let prefixCandidate = null;
             root.traverse((child) => {
-              if (child.isBone && child.name === latestData.bone) {
-                targetBone = child;
-              } else if (!prefixCandidate && child.name.replace(/_\\d+$/, '') === latestData.bone) {
-                prefixCandidate = child;
-              }
+              if (child.isBone && child.name === latestData.bone) targetBone = child;
+              else if (!prefixCandidate && child.name.replace(/_\\d+$/, '') === latestData.bone) prefixCandidate = child;
             });
             if (!targetBone && prefixCandidate) {
               targetBone = prefixCandidate;
-              log('[ACC] Exact bone "' + accData.bone + '" not found, using prefix match: ' + targetBone.name);
-            }
-            // Prevent frustum culling from making the accessory disappear when bone moves
-            model.traverse((child) => {
-              if (child.isMesh) {
-                child.frustumCulled = false;
-              }
-            });
-            if (!targetBone) {
-              log('[ACC] Bone not found: ' + latestData.bone + '. Falling back to root.');
-              targetBone = root;
-            } else {
-              log('[ACC] Found bone: ' + targetBone.name);
+              log('[ACC] Prefix match bone: ' + targetBone.name);
             }
-            // Apply scale and position directly from stored data.
-            // Filament editor is the single source of truth for placement.
+            model.traverse((child) => { if (child.isMesh) child.frustumCulled = false; });
+            if (!targetBone) { log('[ACC] Bone not found: ' + latestData.bone + '. Using root.'); targetBone = root; }
+            else log('[ACC] Found bone: ' + targetBone.name);
             const modelScale = latestData.scale !== undefined ? latestData.scale : 1.0;
             model.scale.set(modelScale, modelScale, modelScale);
             model.position.set(
@@ -466,52 +586,29 @@ async function applyAccessories(accessoriesList) {
               latestData.position ? latestData.position[1] : 0,
               latestData.position ? latestData.position[2] : 0
             );
-            if (latestData.rotation) {
-              model.rotation.set(...latestData.rotation);
-            }
-            log('[ACC] Final pos=' + model.position.x.toFixed(3) + ',' + model.position.y.toFixed(3) + ',' + model.position.z.toFixed(3) + ' scale=' + model.scale.x.toFixed(3));
-            // Ensure the accessory wasn't removed or changed while loading
+            if (latestData.rotation) model.rotation.set(...latestData.rotation);
             if (!currentAccessories[accData.id] || currentAccessories[accData.id].url !== accData.url) {
-              log('[ACC] Aborting attachment: ' + accData.id + ' was removed or changed while loading.');
+              log('[ACC] Aborting: ' + accData.id + ' changed while loading.');
               return;
             }
             targetBone.add(model);
-            log('[ACC] Model attached to bone ' + targetBone.name + '. Children count: ' + targetBone.children.length);
+            log('[ACC] Attached to ' + targetBone.name);
             currentAccessories[accData.id].model = model;
             currentAccessories[accData.id].isLoading = false;
-          }, (progress) => {
-            if (progress.lengthComputable) {
-              // log('[ACC] Load progress ' + accData.id + ': ' + Math.round((progress.loaded / progress.total) * 100) + '%');
-            }
-          }, (err) => {
+          }, () => {}, (err) => {
             if (loadedUrl.startsWith('blob:')) URL.revokeObjectURL(loadedUrl);
-            log('[ACC] FAILED to load accessory ' + accData.id + ': ' + err.message);
-            if (currentAccessories[accData.id]) {
-              currentAccessories[accData.id].isLoading = false;
-            }
+            log('[ACC] FAILED: ' + accData.id + ': ' + err.message);
+            if (currentAccessories[accData.id]) currentAccessories[accData.id].isLoading = false;
           });
         } catch (authErr) {
-          log('[ACC] FAILED to fetch accessory blob ' + accData.id + ': ' + authErr.message);
-          if (currentAccessories[accData.id]) {
-            currentAccessories[accData.id].isLoading = false;
-          }
+          log('[ACC] Auth fetch failed ' + accData.id + ': ' + authErr.message);
+          if (currentAccessories[accData.id]) currentAccessories[accData.id].isLoading = false;
         }
       })();
     } else if (existing && existing.isLoading) {
-      // It's already loading, just update the latestData so when it finishes it uses these transforms
-      log('[ACC] Still loading ' + accData.id + '... buffering latest transforms');
       existing.latestData = accData;
     } else if (existing && existing.model) {
-      log('[ACC] Updating existing model for ' + accData.id);
-      // Just update transform if model already loaded
       const model = existing.model;
-      // Apply scale and position directly from stored data.
       const accScale = accData.scale !== undefined ? accData.scale : 1.0;
       model.scale.set(accScale, accScale, accScale);
       model.position.set(
@@ -519,9 +616,7 @@ async function applyAccessories(accessoriesList) {
         accData.position ? accData.position[1] : 0,
         accData.position ? accData.position[2] : 0
       );
-      if (accData.rotation) {
-        model.rotation.set(...accData.rotation);
-      }
+      if (accData.rotation) model.rotation.set(...accData.rotation);
     }
   }
 }
@@ -530,7 +625,8 @@ function onIncomingMessage(event) {
   try {
     const msg = JSON.parse(event.data);
     if (msg.type === 'amplitude' && mouthMeshes.length > 0) {
-      // Build jaw index cache once, reuse on every subsequent tick
+      // Gate: do not fight an active Rhubarb viseme schedule
+      if (Date.now() < visemeModeUntil) return;
       if (!jawMorphCache) {
         jawMorphCache = [];
         for (const mesh of mouthMeshes) {
@@ -542,35 +638,28 @@ function onIncomingMessage(event) {
                    lk.includes('mouthopen') || lk.includes('mouth_open') ||
                    lk.includes('viseme_aa') || lk === 'aa';
           });
-          if (jawKey !== undefined) {
-            jawMorphCache.push({ influences: mesh.morphTargetInfluences, idx: mesh.morphTargetDictionary[jawKey] });
-          }
+          if (jawKey !== undefined) jawMorphCache.push({ influences: mesh.morphTargetInfluences, idx: mesh.morphTargetDictionary[jawKey] });
         }
       }
       const val = Math.min(1, msg.value * 2.5);
       amplitudeDecay = Math.max(amplitudeDecay * 0.7, val);
-      for (let i = 0; i < jawMorphCache.length; i++) {
-        jawMorphCache[i].influences[jawMorphCache[i].idx] = amplitudeDecay;
-      }
+      for (let i = 0; i < jawMorphCache.length; i++) jawMorphCache[i].influences[jawMorphCache[i].idx] = amplitudeDecay;
+    } else if (msg.type === 'viseme') {
+      applyViseme(msg.viseme, msg.weight !== undefined ? msg.weight : 1.0);
+    } else if (msg.type === 'schedule_visemes') {
+      scheduleVisemes(msg.schedule || { cues: [] });
+    } else if (msg.type === 'clear_visemes') {
+      clearScheduledVisemes();
     } else if (msg.type === 'mood' && head) {
-      // Map agent moods to library moods (library accepts: neutral,happy,angry,sad,fear,disgust,love,sleep)
-      const moodMap = {
-        neutral: 'neutral', happy: 'happy', sad: 'sad', angry: 'angry',
-        excited: 'happy', thinking: 'neutral', concerned: 'sad', surprised: 'happy',
-      };
-      const m = moodMap[msg.value] || 'neutral';
-      head.setMood(m);
+      const moodMap = { neutral:'neutral', happy:'happy', sad:'sad', angry:'angry', excited:'happy', thinking:'neutral', concerned:'sad', surprised:'happy' };
+      head.setMood(moodMap[msg.value] || 'neutral');
     } else if (msg.type === 'hair_color') {
-      HAIR_COLOR = msg.value;
-      applyColorOverrides();
+      HAIR_COLOR = msg.value; applyColorOverrides();
     } else if (msg.type === 'skin_color') {
-      SKIN_COLOR = msg.value;
-      applyColorOverrides();
+      SKIN_COLOR = msg.value; applyColorOverrides();
     } else if (msg.type === 'eye_color') {
-      EYE_COLOR = msg.value;
-      applyColorOverrides();
+      EYE_COLOR = msg.value; applyColorOverrides();
     } else if (msg.type === 'set_accessories') {
-      log('[ACC] Received set_accessories message with ' + (msg.accessories ? msg.accessories.length : 0) + ' items');
       applyAccessories(msg.accessories || []);
     }
   } catch (err) {

package/src/index.ts CHANGED Viewed

@@ -3,6 +3,9 @@ export type {
   TalkingHeadMood,
   TalkingHeadProps,
   TalkingHeadRef,
+  TalkingHeadViseme,
+  TalkingHeadVisemeCue,
+  TalkingHeadVisemeSchedule,
 } from './TalkingHead';
 export { TalkingHead } from './TalkingHead';
 export * from './appearance';