talking-head-studio 0.2.4 → 0.2.6

package/src/TalkingHead.tsx

@@ -5,6 +5,7 @@ import React, {
   useImperativeHandle,
   useMemo,
   useRef,
+  useState,
 } from 'react';
 import { type StyleProp, StyleSheet, View, type ViewStyle } from 'react-native';
 import { WebView, type WebViewMessageEvent } from 'react-native-webview';
@@ -20,6 +21,38 @@ export type TalkingHeadMood =
   | 'concerned'
   | 'surprised';
 
+/**
+ * Standard viseme keys supported by the avatar.
+ * Use with sendViseme() from your TTS viseme callbacks.
+ */
+export type TalkingHeadViseme =
+  | 'sil' | 'PP' | 'FF' | 'TH' | 'DD' | 'kk' | 'CH' | 'SS' | 'nn' | 'RR'
+  | 'aa' | 'ee' | 'ih' | 'oh' | 'ou';
+
+/** Rhubarb mouth shape cue (Preston Blair set: A-H, X) */
+export interface TalkingHeadVisemeCue {
+  startMs: number;
+  endMs: number;
+  viseme: 'A' | 'B' | 'C' | 'D' | 'E' | 'F' | 'G' | 'H' | 'X';
+}
+
+/**
+ * A full viseme schedule from the Rhubarb sidecar endpoint.
+ * Pass to scheduleVisemes() when agent_visemes arrives on the data channel.
+ */
+export interface TalkingHeadVisemeSchedule {
+  /** Matches X-TTS-Request-Id / agent_visemes.requestId */
+  requestId?: string;
+  /**
+   * Wall-clock ms at which audio playback started.
+   * Anchor this to the moment you observe agent_state: speaking.
+   * Used to skip cues that are already in the past on late delivery.
+   */
+  startedAtMs?: number;
+  durationMs?: number;
+  cues: TalkingHeadVisemeCue[];
+}
+
 export interface TalkingHeadAccessory {
   id: string;
   url: string;
@@ -46,6 +79,33 @@ export interface TalkingHeadProps {
 
 export interface TalkingHeadRef {
   sendAmplitude: (amplitude: number) => void;
+  /**
+   * Drive a viseme morph target from your TTS pipeline.
+   * Works on iOS/Android WebViews where AudioWorklet is unavailable.
+   *
+   * @example
+   * // ElevenLabs websocket:
+   * ws.on('viseme', ({ visemeId }) => avatarRef.current?.sendViseme(ELEVENLABS_MAP[visemeId], 1.0));
+   * // Azure TTS:
+   * synthesizer.visemeReceived = (s, e) => avatarRef.current?.sendViseme(AZURE_MAP[e.visemeId], 1.0);
+   */
+  sendViseme: (viseme: TalkingHeadViseme, weight?: number) => void;
+  /**
+   * Schedule a full Rhubarb viseme payload for playback.
+   * Call this when agent_visemes arrives on the LiveKit data channel.
+   * The scheduler gates amplitude fallback while visemes are active.
+   *
+   * @example
+   * room.on(RoomEvent.DataReceived, (payload) => {
+   *   const msg = JSON.parse(new TextDecoder().decode(payload));
+   *   if (msg.type === 'agent_visemes') {
+   *     avatarRef.current?.scheduleVisemes({ ...msg, startedAtMs: speakingStartedAt });
+   *   }
+   * });
+   */
+  scheduleVisemes: (schedule: TalkingHeadVisemeSchedule) => void;
+  /** Cancel any running viseme schedule and return to amplitude fallback. */
+  clearVisemes: () => void;
   setMood: (mood: TalkingHeadMood) => void;
   setHairColor: (color: string) => void;
   setSkinColor: (color: string) => void;
@@ -72,6 +132,23 @@ export const TalkingHead = forwardRef<TalkingHeadRef, TalkingHeadProps>(
     ref,
   ) => {
     const webViewRef = useRef<WebView>(null);
+    const readyRef = useRef(false);
+    const accessoriesRef = useRef(accessories);
+
+    // The WebView HTML is built once from stable initial values.
+    // avatarUrl + authToken changing causes a controlled key-based remount.
+    // All other prop changes (mood, colors, accessories) go via postMessage.
+    const [webViewKey, setWebViewKey] = useState(() => `${avatarUrl}__${authToken ?? ''}`);
+    const prevAvatarRef = useRef({ avatarUrl, authToken });
+
+    useEffect(() => {
+      const prev = prevAvatarRef.current;
+      if (prev.avatarUrl !== avatarUrl || prev.authToken !== authToken) {
+        prevAvatarRef.current = { avatarUrl, authToken };
+        readyRef.current = false;
+        setWebViewKey(`${avatarUrl}__${authToken ?? ''}`);
+      }
+    }, [avatarUrl, authToken]);
 
     const post = useCallback((msg: object) => {
       webViewRef.current?.postMessage(JSON.stringify(msg));
@@ -81,6 +158,9 @@ export const TalkingHead = forwardRef<TalkingHeadRef, TalkingHeadProps>(
       ref,
       () => ({
         sendAmplitude: (amplitude) => post({ type: 'amplitude', value: amplitude }),
+        sendViseme: (viseme, weight = 1.0) => post({ type: 'viseme', viseme, weight }),
+        scheduleVisemes: (schedule) => post({ type: 'schedule_visemes', schedule }),
+        clearVisemes: () => post({ type: 'clear_visemes' }),
         setMood: (nextMood) => post({ type: 'mood', value: nextMood }),
         setHairColor: (color) => post({ type: 'hair_color', value: color }),
         setSkinColor: (color) => post({ type: 'skin_color', value: color }),
@@ -91,20 +171,13 @@ export const TalkingHead = forwardRef<TalkingHeadRef, TalkingHeadProps>(
       [post],
     );
 
+    // Sync mood via postMessage only — never causes a WebView reload
     useEffect(() => {
       if (readyRef.current) post({ type: 'mood', value: mood });
     }, [mood, post]);
 
-    // Track whether the WebView JS is ready to receive messages
-    const readyRef = useRef(false);
-    // Always hold the latest accessories so the ready handler can send them
-    const accessoriesRef = useRef(accessories);
     useEffect(() => {
       accessoriesRef.current = accessories;
-    }, [accessories]);
-
-    useEffect(() => {
-      // Only post if the WebView is already ready; otherwise the ready handler sends them
       if (accessories && readyRef.current) {
         post({ type: 'set_accessories', accessories });
       }
@@ -122,13 +195,13 @@ export const TalkingHead = forwardRef<TalkingHeadRef, TalkingHeadProps>(
       if (eyeColor && readyRef.current) post({ type: 'eye_color', value: eyeColor });
     }, [eyeColor, post]);
 
-    // Color props are intentionally excluded from deps — live updates go via postMessage.
-    // Only avatarUrl, authToken, cameraView, cameraDistance cause a full WebView reload.
-    const [initialMood] = React.useState(mood);
-    const [initialHairColor] = React.useState(hairColor);
-    const [initialSkinColor] = React.useState(skinColor);
-    const [initialEyeColor] = React.useState(eyeColor);
+    // Capture stable initial values at first mount only
+    const [initialMood] = useState(mood);
+    const [initialHairColor] = useState(hairColor);
+    const [initialSkinColor] = useState(skinColor);
+    const [initialEyeColor] = useState(eyeColor);
 
+    // html is stable — only rebuilds when webViewKey changes (avatarUrl/authToken)
     const html = useMemo(
       () =>
         buildAvatarHtml({
@@ -137,20 +210,12 @@ export const TalkingHead = forwardRef<TalkingHeadRef, TalkingHeadProps>(
           mood: initialMood,
           cameraView,
           cameraDistance,
-          initialHairColor: initialHairColor,
-          initialSkinColor: initialSkinColor,
-          initialEyeColor: initialEyeColor,
+          initialHairColor,
+          initialSkinColor,
+          initialEyeColor,
         }),
-      [
-        avatarUrl,
-        authToken,
-        cameraView,
-        cameraDistance,
-        initialMood,
-        initialHairColor,
-        initialSkinColor,
-        initialEyeColor,
-      ],
+      // eslint-disable-next-line react-hooks/exhaustive-deps
+      [webViewKey], // intentionally keyed only on webViewKey, not every prop
     );
 
     const onMessage = useCallback(
@@ -159,15 +224,18 @@ export const TalkingHead = forwardRef<TalkingHeadRef, TalkingHeadProps>(
           const msg = JSON.parse(event.nativeEvent.data);
           if (msg.type === 'ready') {
             readyRef.current = true;
-            // Flush any accessories that arrived before the WebView was ready
+            // Flush pending props that may have arrived before WebView was ready
             if (accessoriesRef.current?.length) {
               post({ type: 'set_accessories', accessories: accessoriesRef.current });
             }
             onReady?.();
-          } else if (msg.type === 'error') onError?.(msg.message);
-          else if (msg.type === 'log') console.log('[TalkingHead]', msg.message);
+          } else if (msg.type === 'error') {
+            onError?.(msg.message);
+          } else if (msg.type === 'log') {
+            console.log('[TalkingHead]', msg.message);
+          }
         } catch (err) {
-          console.warn('[TalkingHead] Invalid message received from WebView:', err);
+          console.warn('[TalkingHead] Invalid message from WebView:', err);
         }
       },
       [onReady, onError, post],
@@ -176,6 +244,7 @@ export const TalkingHead = forwardRef<TalkingHeadRef, TalkingHeadProps>(
     return (
       <View style={[styles.container, style]}>
         <WebView
+          key={webViewKey}
           ref={webViewRef}
           source={{ html }}
           style={styles.webview}

package/src/TalkingHead.web.tsx

@@ -8,6 +8,12 @@ import React, {
 } from 'react';
 import { type StyleProp, StyleSheet, View, type ViewStyle } from 'react-native';
 import { buildAvatarHtml } from './html';
+import type {
+  TalkingHeadVisemeCue,
+  TalkingHeadVisemeSchedule,
+} from './TalkingHead';
+
+export type { TalkingHeadVisemeCue, TalkingHeadVisemeSchedule };
 
 export type TalkingHeadMood =
   | 'neutral'
@@ -45,6 +51,8 @@ export interface TalkingHeadProps {
 
 export interface TalkingHeadRef {
   sendAmplitude: (amplitude: number) => void;
+  scheduleVisemes: (schedule: TalkingHeadVisemeSchedule) => void;
+  clearVisemes: () => void;
   setMood: (mood: TalkingHeadMood) => void;
   setHairColor: (color: string) => void;
   setSkinColor: (color: string) => void;
@@ -86,6 +94,8 @@ export const TalkingHead = forwardRef<TalkingHeadRef, TalkingHeadProps>(
       ref,
       () => ({
         sendAmplitude: (amplitude) => post({ type: 'amplitude', value: amplitude }),
+        scheduleVisemes: (schedule) => post({ type: 'schedule_visemes', schedule }),
+        clearVisemes: () => post({ type: 'clear_visemes' }),
         setMood: (nextMood) => post({ type: 'mood', value: nextMood }),
         setHairColor: (color) => post({ type: 'hair_color', value: color }),
         setSkinColor: (color) => post({ type: 'skin_color', value: color }),