talking-head-studio 0.4.5 → 0.4.8

package/dist/TalkingHead.d.ts

@@ -61,6 +61,7 @@ export interface TalkingHeadProps {
     onReady?: () => void;
     onError?: (message: string) => void;
     onAvatarState?: (state: string) => void;
+    onVoiceMood?: (mood: TalkingHeadMood) => void;
     style?: StyleProp<ViewStyle>;
     /** Base URL for vendored assets. When set, replaces all cdn.jsdelivr.net references. */
     vendorBaseUrl?: string | null;

package/dist/TalkingHead.js

@@ -6,7 +6,7 @@ const react_1 = require("react");
 const react_native_1 = require("react-native");
 const react_native_webview_1 = require("react-native-webview");
 const html_1 = require("./html");
-exports.TalkingHead = (0, react_1.forwardRef)(({ avatarUrl, authToken, mood = 'neutral', cameraView = 'upper', cameraDistance = -0.5, hairColor, skinColor, eyeColor, accessories, onLoadingChange, onReady, onError, onAvatarState, style, vendorBaseUrl, }, ref) => {
+exports.TalkingHead = (0, react_1.forwardRef)(({ avatarUrl, authToken, mood = 'neutral', cameraView = 'upper', cameraDistance = -0.5, hairColor, skinColor, eyeColor, accessories, onLoadingChange, onReady, onError, onAvatarState, onVoiceMood, style, vendorBaseUrl, }, ref) => {
     const webViewRef = (0, react_1.useRef)(null);
     const readyRef = (0, react_1.useRef)(false);
     const firstMessageSeenRef = (0, react_1.useRef)(false);
@@ -25,6 +25,8 @@ exports.TalkingHead = (0, react_1.forwardRef)(({ avatarUrl, authToken, mood = 'n
     onErrorRef.current = onError;
     const onAvatarStateRef = (0, react_1.useRef)(onAvatarState);
     onAvatarStateRef.current = onAvatarState;
+    const onVoiceMoodRef = (0, react_1.useRef)(onVoiceMood);
+    onVoiceMoodRef.current = onVoiceMood;
     // The WebView HTML is built once from stable initial values.
     // avatarUrl + authToken changing causes a controlled key-based remount.
     // All other prop changes (mood, colors, accessories) go via postMessage.
@@ -226,6 +228,9 @@ exports.TalkingHead = (0, react_1.forwardRef)(({ avatarUrl, authToken, mood = 'n
             else if (msg.type === 'avatarState') {
                 onAvatarStateRef.current?.(msg.state);
             }
+            else if (msg.type === 'voiceMood') {
+                onVoiceMoodRef.current?.(msg.mood);
+            }
             else if (msg.type === 'log') {
                 console.log('[TalkingHead]', msg.message);
             }

package/dist/TalkingHeadVisualization.d.ts

@@ -16,6 +16,7 @@ interface TalkingHeadVisualizationProps {
         requestId: string | null;
         appliedAtMs: number;
     }) => void;
+    onVoiceMood?: (mood: TalkingHeadMood) => void;
     vendorBaseUrl?: string | null;
 }
 export interface TalkingHeadVisualizationRef {
@@ -24,6 +25,8 @@ export interface TalkingHeadVisualizationRef {
     sendViseme: (viseme: TalkingHeadViseme, weight?: number) => void;
     scheduleVisemes: (schedule: TalkingHeadVisemeSchedule) => void;
     clearVisemes: () => void;
+    /** Trigger a named motion on the avatar (e.g. 'celebrate', 'groove', 'wave') */
+    playMotion: (name: string) => void;
 }
 /**
  * TalkingHeadVisualization — optimized component for rendering the 3D avatar.

package/dist/TalkingHeadVisualization.js

@@ -44,10 +44,12 @@ function getLoadingLabel(stage) {
  * On native: uses WgpuAvatar (direct morph writes, no WebView bridge).
  * On web: uses TalkingHead WebView renderer.
  */
-exports.TalkingHeadVisualization = (0, react_1.forwardRef)(({ style, avatarUrl, authToken, cameraView = 'head', cameraDistance = 0.2, accessories, mood: initialMood = 'neutral', aspect, focalLength, visemeSchedule, onVisemeScheduleApplied, vendorBaseUrl }, ref) => {
+exports.TalkingHeadVisualization = (0, react_1.forwardRef)(({ style, avatarUrl, authToken, cameraView = 'head', cameraDistance = 0.2, accessories, mood: initialMood = 'neutral', aspect, focalLength, visemeSchedule, onVisemeScheduleApplied, onVoiceMood, vendorBaseUrl }, ref) => {
     const avatarRef = (0, react_1.useRef)(null);
-    // On native, WgpuAvatar ref is wired via callback ref — store it here so
-    // scheduleVisemes / sendAmplitude can route to it.
+    // On native, WgpuAvatar ref is stored here so scheduleVisemes / sendAmplitude can route to it.
+    // useImperativeHandle is the sole owner of the forwarded ref on all platforms —
+    // do NOT manually forward ref inside the WgpuAvatar callback ref, as that bypasses
+    // the dedup key check in scheduleVisemes and causes double-fire.
     const wgpuRef = (0, react_1.useRef)(null);
     // Unified accessor — WgpuAvatar on native, WebView on web
     const activeAvatar = (0, react_1.useCallback)(() => (wgpuRef.current ?? avatarRef.current), []);
@@ -109,6 +111,7 @@ exports.TalkingHeadVisualization = (0, react_1.forwardRef)(({ style, avatarUrl,
         sendAmplitude: (a) => activeAvatar()?.sendAmplitude(a),
         sendViseme: (viseme, weight) => activeAvatar()?.sendViseme(viseme, weight),
         clearVisemes: () => activeAvatar()?.clearVisemes(),
+        playMotion: (name) => avatarRef.current?.dispatchMotion(name),
         scheduleVisemes: (schedule) => {
             const scheduleKey = `${schedule.requestId ?? 'anonymous'}:${schedule.startedAtMs ?? 0}`;
             if (lastScheduledVisemeKeyRef.current === scheduleKey)
@@ -182,26 +185,15 @@ exports.TalkingHeadVisualization = (0, react_1.forwardRef)(({ style, avatarUrl,
         lastScheduledVisemeKeyRef.current = scheduleKey;
     }, [isAvatarReady, onVisemeScheduleApplied, visemeSchedule, activeAvatar]);
     // On native use WgpuAvatar — direct morph writes, no WebView bridge.
+    // NOTE: Only store wgpuRef here — do NOT forward `ref` manually. useImperativeHandle
+    // above is the sole owner of the forwarded ref and handles the dedup key check.
     if (react_native_1.Platform.OS !== 'web') {
-        return ((0, jsx_runtime_1.jsx)(WgpuAvatar_1.WgpuAvatar, { focalLength: focalLength, ref: (fr) => {
-                wgpuRef.current = fr;
-                if (typeof ref === 'function')
-                    ref(fr);
-                else if (ref)
-                    ref.current = fr;
-                if (fr && pendingVisemeScheduleRef.current) {
-                    fr.scheduleVisemes(pendingVisemeScheduleRef.current);
-                    pendingVisemeScheduleRef.current = null;
-                }
-            }, style: style, avatarUrl: avatarUrl ?? null, aspect: aspect, mood: mood, accessories: accessories, onReady: handleReady, onError: handleError }));
+        return ((0, jsx_runtime_1.jsx)(WgpuAvatar_1.WgpuAvatar, { focalLength: focalLength, ref: (fr) => { wgpuRef.current = fr; }, style: style, avatarUrl: avatarUrl ?? null, aspect: aspect, mood: mood, accessories: accessories, onReady: handleReady, onError: handleError }));
     }
     if (!effectiveAvatarUrl) {
         return ((0, jsx_runtime_1.jsx)(react_native_1.View, { style: [style, styles.placeholder] }));
     }
-    return ((0, jsx_runtime_1.jsxs)(react_native_1.View, { style: [style, styles.container], pointerEvents: "box-none", children: [(0, jsx_runtime_1.jsx)(TalkingHead_1.TalkingHead, { ref: avatarRef, avatarUrl: effectiveAvatarUrl, authToken: authToken, cameraView: cameraView, cameraDistance: cameraDistance, accessories: accessories, mood: mood, onLoadingChange: handleLoadingChange, onReady: handleReady, onError: handleError, style: react_native_1.StyleSheet.absoluteFill, vendorBaseUrl: vendorBaseUrl }), !isAvatarReady && ((0, jsx_runtime_1.jsx)(react_native_1.View, { testID: "talking-head-loading", style: styles.loadingOverlay, pointerEvents: "none", children: (0, jsx_runtime_1.jsxs)(react_native_1.View, { style: styles.loadingCard, children: [(0, jsx_runtime_1.jsx)(react_native_1.View, { style: styles.loadingBadge, children: (0, jsx_runtime_1.jsx)(react_native_1.Text, { style: styles.loadingBadgeText, children: "AVATAR" }) }), (0, jsx_runtime_1.jsx)(react_native_1.Text, { style: styles.loadingTitle, children: avatarError ? 'Avatar failed to load' : getLoadingLabel(loadingState.stage) }), avatarError ? ((0, jsx_runtime_1.jsx)(react_native_1.Text, { style: styles.loadingHint, children: avatarError })) : typeof loadingState.progress === 'number' ? ((0, jsx_runtime_1.jsxs)(jsx_runtime_1.Fragment, { children: [(0, jsx_runtime_1.jsxs)(react_native_1.Text, { style: styles.loadingPercent, children: [loadingState.progress, "%"] }), (0, jsx_runtime_1.jsx)(react_native_1.View, { style: styles.progressTrack, children: (0, jsx_runtime_1.jsx)(react_native_1.View, { style: [
-                                            styles.progressFill,
-                                            { width: `${Math.max(6, loadingState.progress)}%` },
-                                        ] }) })] })) : ((0, jsx_runtime_1.jsx)(react_native_1.Text, { style: styles.loadingHint, children: "Preparing the avatar scene\u2026" }))] }) }))] }));
+    return ((0, jsx_runtime_1.jsxs)(react_native_1.View, { style: [style, styles.container], pointerEvents: "box-none", children: [(0, jsx_runtime_1.jsx)(TalkingHead_1.TalkingHead, { ref: avatarRef, avatarUrl: effectiveAvatarUrl, authToken: authToken, cameraView: cameraView, cameraDistance: cameraDistance, accessories: accessories, mood: mood, onLoadingChange: handleLoadingChange, onReady: handleReady, onError: handleError, onVoiceMood: onVoiceMood, style: react_native_1.StyleSheet.absoluteFill, vendorBaseUrl: vendorBaseUrl }), !isAvatarReady && ((0, jsx_runtime_1.jsx)(react_native_1.View, { testID: "talking-head-loading", style: styles.loadingOverlay, pointerEvents: "none", children: (0, jsx_runtime_1.jsxs)(react_native_1.View, { style: styles.loadingCard, children: [(0, jsx_runtime_1.jsx)(react_native_1.View, { style: styles.loadingBadge, children: (0, jsx_runtime_1.jsx)(react_native_1.Text, { style: styles.loadingBadgeText, children: "AVATAR" }) }), (0, jsx_runtime_1.jsx)(react_native_1.Text, { style: styles.loadingTitle, children: avatarError ? 'Avatar failed to load' : getLoadingLabel(loadingState.stage) }), avatarError ? ((0, jsx_runtime_1.jsx)(react_native_1.Text, { style: styles.loadingHint, children: avatarError })) : typeof loadingState.progress === 'number' ? ((0, jsx_runtime_1.jsxs)(jsx_runtime_1.Fragment, { children: [(0, jsx_runtime_1.jsxs)(react_native_1.Text, { style: styles.loadingPercent, children: [loadingState.progress, "%"] }), (0, jsx_runtime_1.jsxs)(react_native_1.View, { style: styles.progressTrack, children: [(0, jsx_runtime_1.jsx)(react_native_1.View, { style: { flex: Math.max(6, loadingState.progress ?? 0), height: '100%', borderRadius: 999, backgroundColor: '#5eead4' } }), (0, jsx_runtime_1.jsx)(react_native_1.View, { style: { flex: 100 - Math.max(6, loadingState.progress ?? 0) } })] })] })) : ((0, jsx_runtime_1.jsx)(react_native_1.Text, { style: styles.loadingHint, children: "Preparing the avatar scene\u2026" }))] }) }))] }));
 });
 exports.TalkingHeadVisualization.displayName = 'TalkingHeadVisualization';
 const styles = react_native_1.StyleSheet.create({
@@ -263,11 +255,7 @@ const styles = react_native_1.StyleSheet.create({
         backgroundColor: 'rgba(148, 163, 184, 0.18)',
         marginTop: 12,
         overflow: 'hidden',
-    },
-    progressFill: {
-        height: '100%',
-        borderRadius: 999,
-        backgroundColor: '#5eead4',
+        flexDirection: 'row',
     },
     placeholder: {
         backgroundColor: 'rgba(30,34,53,0.5)',

package/dist/html.js

@@ -40,25 +40,26 @@ function buildAvatarHtml(config) {
 }
 </script>
 <script>
-window.ReactNativeWebView?.postMessage(
+function rnPost(d) { window.ReactNativeWebView && window.ReactNativeWebView.postMessage(d); }
+rnPost(
   JSON.stringify({ type: 'log', message: '[bootstrap] inline script start' })
 );
 function postBootstrapError(kind, message) {
-  window.ReactNativeWebView?.postMessage(
+  rnPost(
     JSON.stringify({ type: 'error', message: '[' + kind + '] ' + String(message || 'Unknown error') })
   );
 }
 document.addEventListener('DOMContentLoaded', function() {
-  window.ReactNativeWebView?.postMessage(
+  rnPost(
     JSON.stringify({ type: 'log', message: '[bootstrap] DOMContentLoaded' })
   );
 });
 window.addEventListener('error', function(event) {
-  postBootstrapError('window.error', event?.message || event?.error?.message || 'Script error');
+  postBootstrapError('window.error', (event && event.message) || (event && event.error && event.error.message) || 'Script error');
 });
 window.addEventListener('unhandledrejection', function(event) {
-  const reason = event?.reason;
-  postBootstrapError('unhandledrejection', reason?.message || reason || 'Unhandled promise rejection');
+  const reason = event && event.reason;
+  postBootstrapError('unhandledrejection', (reason && reason.message) || reason || 'Unhandled promise rejection');
 });
 </script>
 </head>
@@ -66,7 +67,7 @@ window.addEventListener('unhandledrejection', function(event) {
 <div id="avatar"></div>
 <script type="module">
 (async function() {
-window.ReactNativeWebView?.postMessage(
+rnPost(
   JSON.stringify({ type: 'log', message: '[module] script start' })
 );
 const AUTH_TOKEN = ${JSON.stringify(config.authToken ?? null)};
@@ -94,11 +95,11 @@ let staticModel = null;
 let initStarted = false;
 
 function log(msg) {
-  window.ReactNativeWebView?.postMessage(JSON.stringify({ type: 'log', message: msg }));
+  rnPost(JSON.stringify({ type: 'log', message: msg }));
 }
 
 function emitLoading(stage, progress = null) {
-  window.ReactNativeWebView?.postMessage(
+  rnPost(
     JSON.stringify({ type: 'loading', stage, progress }),
   );
 }
@@ -125,13 +126,13 @@ function applyColorOverrides() {
     mats.forEach((mat) => {
       const name = (mat.name || '').toLowerCase();
       if (HAIR_COLOR && (name.includes('hair') || name.includes('fur'))) {
-        mat.color?.set(HAIR_COLOR);
+        mat.color && mat.color.set(HAIR_COLOR);
       }
       if (SKIN_COLOR && (name.includes('skin') || name.includes('body') || name.includes('face'))) {
-        mat.color?.set(SKIN_COLOR);
+        mat.color && mat.color.set(SKIN_COLOR);
       }
       if (EYE_COLOR && (name.includes('eye') || name.includes('iris'))) {
-        mat.color?.set(EYE_COLOR);
+        mat.color && mat.color.set(EYE_COLOR);
       }
     });
   });
@@ -204,7 +205,7 @@ async function loadStaticFallback(loadedAvatarUrl) {
       applyAccessories(pendingAccessoriesList);
       motionInitFromRoot(staticModel);
       emitLoading('ready', 100);
-      window.ReactNativeWebView?.postMessage(JSON.stringify({ type: 'ready' }));
+      rnPost(JSON.stringify({ type: 'ready' }));
 
       window.addEventListener('resize', () => {
         camera.aspect = window.innerWidth / window.innerHeight;
@@ -228,11 +229,11 @@ async function loadStaticFallback(loadedAvatarUrl) {
       }
     }, (err) => {
       log('Fallback Error: ' + err.message);
-      window.ReactNativeWebView?.postMessage(JSON.stringify({ type: 'error', message: err.message }));
+      rnPost(JSON.stringify({ type: 'error', message: err.message }));
     });
   } catch (err) {
     log('Fallback setup error: ' + err.message);
-    window.ReactNativeWebView?.postMessage(JSON.stringify({ type: 'error', message: err.message }));
+    rnPost(JSON.stringify({ type: 'error', message: err.message }));
   }
 }
 
@@ -299,7 +300,7 @@ async function init() {
       jawMorphCache = null;
       visemeMorphCache = null;
       rhubarbMorphCache = null;
-      head.armature?.traverse((child) => {
+      head.armature && head.armature.traverse(function(child) {
         if (child.isMesh && child.morphTargetDictionary && child.morphTargetInfluences) {
           mouthMeshes.push(child);
         }
@@ -338,17 +339,17 @@ async function init() {
       applyAccessories(pendingAccessoriesList);
       if (head && head.armature) motionInitFromRoot(head.armature);
       emitLoading('ready', 100);
-      window.ReactNativeWebView?.postMessage(JSON.stringify({ type: 'ready' }));
+      rnPost(JSON.stringify({ type: 'ready' }));
     }
   } catch (err) {
     initStarted = false; // allow retry on error
     log('Error: ' + err.message);
-    window.ReactNativeWebView?.postMessage(JSON.stringify({ type: 'error', message: err.message }));
+    rnPost(JSON.stringify({ type: 'error', message: err.message }));
   }
 }
 
 function startAudioInterception() {
-  if (!head?.audioCtx || !head?.audioSpeechGainNode) return;
+  if (!head || !head.audioCtx || !head.audioSpeechGainNode) return;
   const audioCtx = head.audioCtx;
   const gainNode = head.audioSpeechGainNode;
   const connected = new WeakSet();
@@ -384,6 +385,33 @@ function startAudioInterception() {
 let amplitudeDecay = 0;
 let jawMorphCache = null;
 let visemeMorphCache = null;
+
+// Voice-driven mood detection
+let voiceEnergySmoothed = 0;
+let voiceMoodCurrent = 'neutral';
+let voiceMoodFramesAbove = 0;
+let voiceMoodFramesBelow = 0;
+const VOICE_ENERGY_EXCITED_THRESH = 0.45;
+const VOICE_ENERGY_NEUTRAL_THRESH = 0.15;
+const VOICE_MOOD_FRAMES_REQUIRED = 12; // ~0.2s at 60fps
+function updateVoiceMood(rawAmplitude) {
+  voiceEnergySmoothed = voiceEnergySmoothed * 0.85 + rawAmplitude * 0.15;
+  if (voiceEnergySmoothed > VOICE_ENERGY_EXCITED_THRESH) {
+    voiceMoodFramesAbove++;
+    voiceMoodFramesBelow = 0;
+    if (voiceMoodCurrent !== 'excited' && voiceMoodFramesAbove >= VOICE_MOOD_FRAMES_REQUIRED) {
+      voiceMoodCurrent = 'excited';
+      rnPost(JSON.stringify({ type: 'voiceMood', mood: 'excited' }));
+    }
+  } else if (voiceEnergySmoothed < VOICE_ENERGY_NEUTRAL_THRESH) {
+    voiceMoodFramesBelow++;
+    voiceMoodFramesAbove = 0;
+    if (voiceMoodCurrent !== 'neutral' && voiceMoodFramesBelow >= VOICE_MOOD_FRAMES_REQUIRED * 3) {
+      voiceMoodCurrent = 'neutral';
+      rnPost(JSON.stringify({ type: 'voiceMood', mood: 'neutral' }));
+    }
+  }
+}
 const visemeState = {};
 
 const VISEME_MORPH_ALIASES = {
@@ -551,7 +579,7 @@ function clearScheduledVisemes() {
   for (const key of Object.keys(visemeState)) visemeState[key] = 0;
 }
 
-function tickVisemeDecay(deltaSeconds?: number) {
+function tickVisemeDecay(deltaSeconds) {
   if (!visemeMorphCache) return;
 
   const isScheduled = Date.now() < visemeModeUntil;
@@ -587,7 +615,7 @@ function tickVisemeDecay(deltaSeconds?: number) {
       // Use realtime (not newvalue) — newvalue is consumed and cleared after
       // a single frame, so scheduled visemes would vanish immediately.
       // realtime persists until explicitly set to null.
-      if (head?.mtAvatar && e.morphName && head.mtAvatar[e.morphName]) {
+      if (head && head.mtAvatar && e.morphName && head.mtAvatar[e.morphName]) {
         const mt = head.mtAvatar[e.morphName];
         mt.realtime = targetWeight > 0 ? targetWeight : null;
         mt.needsUpdate = true;
@@ -771,7 +799,7 @@ async function applyAccessories(accessoriesList) {
             if (loadedUrl.startsWith('blob:')) URL.revokeObjectURL(loadedUrl);
             log('[ACC] GLB loaded OK for ' + accData.id);
             const model = gltf.scene;
-            const latestData = currentAccessories[accData.id]?.latestData || accData;
+            const latestData = (currentAccessories[accData.id] && currentAccessories[accData.id].latestData) || accData;
             let targetBone = null;
             let prefixCandidate = null;
             root.traverse((child) => {
@@ -1109,6 +1137,7 @@ function onIncomingMessage(event) {
             msg.value * RHUBARB_FALLBACK_AMPLITUDE_GAIN,
           );
       amplitudeDecay = Math.max(amplitudeDecay * 0.7, val);
+      updateVoiceMood(msg.value);
       for (let i = 0; i < jawMorphCache.length; i++) jawMorphCache[i].influences[jawMorphCache[i].idx] = amplitudeDecay;
     } else if (msg.type === 'viseme') {
       applyViseme(msg.viseme, msg.weight !== undefined ? msg.weight : 1.0);
@@ -1130,7 +1159,7 @@ function onIncomingMessage(event) {
       if (typeof window.playMotion === 'function') {
         window.playMotion(msg.name);
       }
-      window.ReactNativeWebView?.postMessage(JSON.stringify({ type: 'avatarState', state: 'motion:' + msg.name }));
+      rnPost(JSON.stringify({ type: 'avatarState', state: 'motion:' + msg.name }));
       log('motion dispatched: ' + msg.name);
     }
   } catch (err) {

package/dist/wgpu/WgpuAvatar.js

@@ -232,7 +232,7 @@ exports.WgpuAvatar = (0, react_1.forwardRef)(({ style, avatarUrl, focalLength, m
     (0, react_1.useEffect)(() => () => clearScheduleTimers(), [clearScheduleTimers]);
     (0, react_1.useImperativeHandle)(ref, () => ({
         setMood: (m) => { morphStateRef.current.moodBase = { ...(morphTables_1.MOOD_MORPHS[m] ?? {}) }; },
-        sendAmplitude: (_amplitude) => { },
+        sendAmplitude: () => { },
         sendViseme: (viseme, weight) => {
             const aliases = morphTables_1.VISEME_MORPH_ALIASES[viseme];
             if (!aliases)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "talking-head-studio",
-  "version": "0.4.5",
+  "version": "0.4.8",
   "description": "Cross-platform 3D avatar component for React Native & web — lip-sync, gestures, accessories, and LLM integration. Powered by TalkingHead + Three.js.",
   "main": "dist/index.web.js",
   "browser": "dist/index.web.js",