talking-head-studio 0.2.7 → 0.2.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,11 @@
1
1
  import React from 'react';
2
2
  import { type StyleProp, type ViewStyle } from 'react-native';
3
3
  export type TalkingHeadMood = 'neutral' | 'happy' | 'sad' | 'angry' | 'excited' | 'thinking' | 'concerned' | 'surprised';
4
+ export type TalkingHeadLoadingStage = 'booting' | 'fetching_model' | 'loading_avatar' | 'loading_fallback' | 'ready';
5
+ export interface TalkingHeadLoadingState {
6
+ stage: TalkingHeadLoadingStage;
7
+ progress?: number | null;
8
+ }
4
9
  /**
5
10
  * Standard viseme keys supported by the avatar.
6
11
  * Use with sendViseme() from your TTS viseme callbacks.
@@ -46,9 +51,12 @@ export interface TalkingHeadProps {
46
51
  skinColor?: string;
47
52
  eyeColor?: string;
48
53
  accessories?: TalkingHeadAccessory[];
54
+ onLoadingChange?: (state: TalkingHeadLoadingState) => void;
49
55
  onReady?: () => void;
50
56
  onError?: (message: string) => void;
51
57
  style?: StyleProp<ViewStyle>;
58
+ /** Base URL for vendored assets. When set, replaces all cdn.jsdelivr.net references. */
59
+ vendorBaseUrl?: string | null;
52
60
  }
53
61
  export interface TalkingHeadRef {
54
62
  sendAmplitude: (amplitude: number) => void;
@@ -6,10 +6,23 @@ const react_1 = require("react");
6
6
  const react_native_1 = require("react-native");
7
7
  const react_native_webview_1 = require("react-native-webview");
8
8
  const html_1 = require("./html");
9
- exports.TalkingHead = (0, react_1.forwardRef)(({ avatarUrl, authToken, mood = 'neutral', cameraView = 'upper', cameraDistance = -0.5, hairColor, skinColor, eyeColor, accessories, onReady, onError, style, }, ref) => {
9
+ exports.TalkingHead = (0, react_1.forwardRef)(({ avatarUrl, authToken, mood = 'neutral', cameraView = 'upper', cameraDistance = -0.5, hairColor, skinColor, eyeColor, accessories, onLoadingChange, onReady, onError, style, vendorBaseUrl, }, ref) => {
10
10
  const webViewRef = (0, react_1.useRef)(null);
11
11
  const readyRef = (0, react_1.useRef)(false);
12
+ const firstMessageSeenRef = (0, react_1.useRef)(false);
13
+ const bootTimerRef = (0, react_1.useRef)(null);
14
+ const pendingMoodRef = (0, react_1.useRef)(mood);
15
+ const pendingHairColorRef = (0, react_1.useRef)(hairColor);
16
+ const pendingSkinColorRef = (0, react_1.useRef)(skinColor);
17
+ const pendingEyeColorRef = (0, react_1.useRef)(eyeColor);
12
18
  const accessoriesRef = (0, react_1.useRef)(accessories);
19
+ // Stable refs for callbacks so they never retrigger the boot effect.
20
+ const onLoadingChangeRef = (0, react_1.useRef)(onLoadingChange);
21
+ onLoadingChangeRef.current = onLoadingChange;
22
+ const onReadyRef = (0, react_1.useRef)(onReady);
23
+ onReadyRef.current = onReady;
24
+ const onErrorRef = (0, react_1.useRef)(onError);
25
+ onErrorRef.current = onError;
13
26
  // The WebView HTML is built once from stable initial values.
14
27
  // avatarUrl + authToken changing causes a controlled key-based remount.
15
28
  // All other prop changes (mood, colors, accessories) go via postMessage.
@@ -23,22 +36,70 @@ exports.TalkingHead = (0, react_1.forwardRef)(({ avatarUrl, authToken, mood = 'n
23
36
  setWebViewKey(`${avatarUrl}__${authToken ?? ''}`);
24
37
  }
25
38
  }, [avatarUrl, authToken]);
39
+ (0, react_1.useEffect)(() => {
40
+ if (!avatarUrl)
41
+ return;
42
+ firstMessageSeenRef.current = false;
43
+ onLoadingChangeRef.current?.({ stage: 'booting', progress: null });
44
+ if (bootTimerRef.current)
45
+ clearTimeout(bootTimerRef.current);
46
+ bootTimerRef.current = setTimeout(() => {
47
+ if (readyRef.current || firstMessageSeenRef.current)
48
+ return;
49
+ const vendorLabel = vendorBaseUrl ? ` vendor=${vendorBaseUrl}` : ' vendor=<cdn>';
50
+ const avatarLabel = avatarUrl && avatarUrl.length > 80 ? avatarUrl.slice(0, 80) + '…' : avatarUrl;
51
+ onErrorRef.current?.(`[boot timeout] No WebView messages received after 8s.${vendorLabel} avatar=${avatarLabel}`);
52
+ }, 8000);
53
+ return () => {
54
+ if (bootTimerRef.current) {
55
+ clearTimeout(bootTimerRef.current);
56
+ bootTimerRef.current = null;
57
+ }
58
+ };
59
+ }, [avatarUrl, authToken, vendorBaseUrl]);
26
60
  const post = (0, react_1.useCallback)((msg) => {
27
- webViewRef.current?.postMessage(JSON.stringify(msg));
61
+ try {
62
+ webViewRef.current?.postMessage(JSON.stringify(msg));
63
+ }
64
+ catch {
65
+ // WebView ref frozen/invalidated during unmount — ignore
66
+ }
28
67
  }, []);
29
68
  (0, react_1.useImperativeHandle)(ref, () => ({
30
69
  sendAmplitude: (amplitude) => post({ type: 'amplitude', value: amplitude }),
31
70
  sendViseme: (viseme, weight = 1.0) => post({ type: 'viseme', viseme, weight }),
32
71
  scheduleVisemes: (schedule) => post({ type: 'schedule_visemes', schedule }),
33
72
  clearVisemes: () => post({ type: 'clear_visemes' }),
34
- setMood: (nextMood) => post({ type: 'mood', value: nextMood }),
35
- setHairColor: (color) => post({ type: 'hair_color', value: color }),
36
- setSkinColor: (color) => post({ type: 'skin_color', value: color }),
37
- setEyeColor: (color) => post({ type: 'eye_color', value: color }),
38
- setAccessories: (newAccessories) => post({ type: 'set_accessories', accessories: newAccessories }),
73
+ setMood: (nextMood) => {
74
+ pendingMoodRef.current = nextMood;
75
+ if (readyRef.current)
76
+ post({ type: 'mood', value: nextMood });
77
+ },
78
+ setHairColor: (color) => {
79
+ pendingHairColorRef.current = color;
80
+ if (readyRef.current)
81
+ post({ type: 'hair_color', value: color });
82
+ },
83
+ setSkinColor: (color) => {
84
+ pendingSkinColorRef.current = color;
85
+ if (readyRef.current)
86
+ post({ type: 'skin_color', value: color });
87
+ },
88
+ setEyeColor: (color) => {
89
+ pendingEyeColorRef.current = color;
90
+ if (readyRef.current)
91
+ post({ type: 'eye_color', value: color });
92
+ },
93
+ setAccessories: (newAccessories) => {
94
+ accessoriesRef.current = newAccessories;
95
+ if (readyRef.current) {
96
+ post({ type: 'set_accessories', accessories: newAccessories });
97
+ }
98
+ },
39
99
  }), [post]);
40
100
  // Sync mood via postMessage only — never causes a WebView reload
41
101
  (0, react_1.useEffect)(() => {
102
+ pendingMoodRef.current = mood;
42
103
  if (readyRef.current)
43
104
  post({ type: 'mood', value: mood });
44
105
  }, [mood, post]);
@@ -49,14 +110,17 @@ exports.TalkingHead = (0, react_1.forwardRef)(({ avatarUrl, authToken, mood = 'n
49
110
  }
50
111
  }, [accessories, post]);
51
112
  (0, react_1.useEffect)(() => {
113
+ pendingHairColorRef.current = hairColor;
52
114
  if (hairColor && readyRef.current)
53
115
  post({ type: 'hair_color', value: hairColor });
54
116
  }, [hairColor, post]);
55
117
  (0, react_1.useEffect)(() => {
118
+ pendingSkinColorRef.current = skinColor;
56
119
  if (skinColor && readyRef.current)
57
120
  post({ type: 'skin_color', value: skinColor });
58
121
  }, [skinColor, post]);
59
122
  (0, react_1.useEffect)(() => {
123
+ pendingEyeColorRef.current = eyeColor;
60
124
  if (eyeColor && readyRef.current)
61
125
  post({ type: 'eye_color', value: eyeColor });
62
126
  }, [eyeColor, post]);
@@ -65,6 +129,18 @@ exports.TalkingHead = (0, react_1.forwardRef)(({ avatarUrl, authToken, mood = 'n
65
129
  const [initialHairColor] = (0, react_1.useState)(hairColor);
66
130
  const [initialSkinColor] = (0, react_1.useState)(skinColor);
67
131
  const [initialEyeColor] = (0, react_1.useState)(eyeColor);
132
+ // Use the vendor origin as baseUrl so dynamic module imports are same-origin
133
+ // and don't get blocked by the WebView's security model.
134
+ const webViewBaseUrl = (0, react_1.useMemo)(() => {
135
+ if (!vendorBaseUrl)
136
+ return 'https://localhost/';
137
+ try {
138
+ return new URL(vendorBaseUrl).origin + '/';
139
+ }
140
+ catch {
141
+ return 'https://localhost/';
142
+ }
143
+ }, [vendorBaseUrl]);
68
144
  // html is stable — only rebuilds when webViewKey changes (avatarUrl/authToken)
69
145
  const html = (0, react_1.useMemo)(() => (0, html_1.buildAvatarHtml)({
70
146
  avatarUrl,
@@ -75,22 +151,46 @@ exports.TalkingHead = (0, react_1.forwardRef)(({ avatarUrl, authToken, mood = 'n
75
151
  initialHairColor,
76
152
  initialSkinColor,
77
153
  initialEyeColor,
154
+ vendorBaseUrl,
78
155
  }),
79
156
  // eslint-disable-next-line react-hooks/exhaustive-deps
80
157
  [webViewKey]);
81
158
  const onMessage = (0, react_1.useCallback)((event) => {
82
159
  try {
160
+ firstMessageSeenRef.current = true;
83
161
  const msg = JSON.parse(event.nativeEvent.data);
162
+ if (msg.type === 'loading' && typeof msg.stage === 'string') {
163
+ onLoadingChangeRef.current?.({
164
+ stage: msg.stage,
165
+ progress: typeof msg.progress === 'number' && Number.isFinite(msg.progress)
166
+ ? msg.progress
167
+ : null,
168
+ });
169
+ return;
170
+ }
84
171
  if (msg.type === 'ready') {
85
172
  readyRef.current = true;
86
- // Flush pending props that may have arrived before WebView was ready
173
+ onLoadingChangeRef.current?.({ stage: 'ready', progress: 100 });
174
+ // Flush pending appearance updates that arrived before the WebView was ready.
175
+ if (pendingMoodRef.current) {
176
+ post({ type: 'mood', value: pendingMoodRef.current });
177
+ }
178
+ if (pendingHairColorRef.current) {
179
+ post({ type: 'hair_color', value: pendingHairColorRef.current });
180
+ }
181
+ if (pendingSkinColorRef.current) {
182
+ post({ type: 'skin_color', value: pendingSkinColorRef.current });
183
+ }
184
+ if (pendingEyeColorRef.current) {
185
+ post({ type: 'eye_color', value: pendingEyeColorRef.current });
186
+ }
87
187
  if (accessoriesRef.current?.length) {
88
188
  post({ type: 'set_accessories', accessories: accessoriesRef.current });
89
189
  }
90
- onReady?.();
190
+ onReadyRef.current?.();
91
191
  }
92
192
  else if (msg.type === 'error') {
93
- onError?.(msg.message);
193
+ onErrorRef.current?.(msg.message);
94
194
  }
95
195
  else if (msg.type === 'log') {
96
196
  console.log('[TalkingHead]', msg.message);
@@ -99,8 +199,18 @@ exports.TalkingHead = (0, react_1.forwardRef)(({ avatarUrl, authToken, mood = 'n
99
199
  catch (err) {
100
200
  console.warn('[TalkingHead] Invalid message from WebView:', err);
101
201
  }
102
- }, [onReady, onError, post]);
103
- return ((0, jsx_runtime_1.jsx)(react_native_1.View, { style: [styles.container, style], children: (0, jsx_runtime_1.jsx)(react_native_webview_1.WebView, { ref: webViewRef, source: { html }, style: styles.webview, javaScriptEnabled: true, domStorageEnabled: true, allowsInlineMediaPlayback: true, mediaPlaybackRequiresUserAction: false, onMessage: onMessage, originWhitelist: ['*'], mixedContentMode: "always" }, webViewKey) }));
202
+ }, [post]);
203
+ const handleWebViewError = (0, react_1.useCallback)((event) => {
204
+ firstMessageSeenRef.current = true;
205
+ const description = event.nativeEvent.description || 'WebView failed to load avatar';
206
+ onErrorRef.current?.(`[webview] ${description}`);
207
+ }, []);
208
+ const handleWebViewHttpError = (0, react_1.useCallback)((event) => {
209
+ firstMessageSeenRef.current = true;
210
+ const { statusCode, description, url } = event.nativeEvent;
211
+ onErrorRef.current?.(`[http ${statusCode}] ${description || url || 'Avatar request failed'}`);
212
+ }, []);
213
+ return ((0, jsx_runtime_1.jsx)(react_native_1.View, { style: [styles.container, style], children: (0, jsx_runtime_1.jsx)(react_native_webview_1.WebView, { ref: webViewRef, source: { html, baseUrl: webViewBaseUrl }, style: styles.webview, javaScriptEnabled: true, domStorageEnabled: true, allowsInlineMediaPlayback: true, mediaPlaybackRequiresUserAction: false, onMessage: onMessage, onError: handleWebViewError, onHttpError: handleWebViewHttpError, onLoadStart: () => console.log('[TalkingHead] WebView load start'), onLoadEnd: () => console.log('[TalkingHead] WebView load end'), onLoadProgress: (event) => console.log('[TalkingHead] WebView progress', event.nativeEvent.progress), originWhitelist: ['*'], allowFileAccess: true, allowFileAccessFromFileURLs: true, allowUniversalAccessFromFileURLs: true, mixedContentMode: "always" }, webViewKey) }));
104
214
  });
105
215
  exports.TalkingHead.displayName = 'TalkingHead';
106
216
  const styles = react_native_1.StyleSheet.create({
@@ -1,6 +1,6 @@
1
1
  import React from 'react';
2
- import type { TalkingHeadVisemeCue, TalkingHeadVisemeSchedule } from './TalkingHead';
3
- export type { TalkingHeadVisemeCue, TalkingHeadVisemeSchedule };
2
+ import type { TalkingHeadLoadingState, TalkingHeadVisemeCue, TalkingHeadVisemeSchedule } from './TalkingHead';
3
+ export type { TalkingHeadLoadingState, TalkingHeadVisemeCue, TalkingHeadVisemeSchedule };
4
4
  export type TalkingHeadMood = 'neutral' | 'happy' | 'sad' | 'angry' | 'excited' | 'thinking' | 'concerned' | 'surprised';
5
5
  export interface TalkingHeadAccessory {
6
6
  id: string;
@@ -20,6 +20,7 @@ export interface TalkingHeadProps {
20
20
  skinColor?: string;
21
21
  eyeColor?: string;
22
22
  accessories?: TalkingHeadAccessory[];
23
+ onLoadingChange?: (state: TalkingHeadLoadingState) => void;
23
24
  onReady?: () => void;
24
25
  onError?: (message: string) => void;
25
26
  style?: React.CSSProperties;
@@ -48,13 +48,22 @@ const iframeStyle = {
48
48
  border: 'none',
49
49
  backgroundColor: 'transparent',
50
50
  };
51
- exports.TalkingHead = (0, react_1.forwardRef)(({ avatarUrl, authToken, mood = 'neutral', cameraView = 'upper', cameraDistance = -0.5, hairColor, skinColor, eyeColor, accessories, onReady, onError, style, }, ref) => {
51
+ exports.TalkingHead = (0, react_1.forwardRef)(({ avatarUrl, authToken, mood = 'neutral', cameraView = 'upper', cameraDistance = -0.5, hairColor, skinColor, eyeColor, accessories, onLoadingChange, onReady, onError, style, }, ref) => {
52
52
  const iframeRef = (0, react_1.useRef)(null);
53
53
  const readyRef = (0, react_1.useRef)(false);
54
+ const pendingMoodRef = (0, react_1.useRef)(mood);
55
+ const pendingHairColorRef = (0, react_1.useRef)(hairColor);
56
+ const pendingSkinColorRef = (0, react_1.useRef)(skinColor);
57
+ const pendingEyeColorRef = (0, react_1.useRef)(eyeColor);
54
58
  const accessoriesRef = (0, react_1.useRef)(accessories);
55
59
  (0, react_1.useEffect)(() => {
56
60
  accessoriesRef.current = accessories;
57
61
  }, [accessories]);
62
+ (0, react_1.useEffect)(() => {
63
+ if (!avatarUrl)
64
+ return;
65
+ onLoadingChange?.({ stage: 'booting', progress: null });
66
+ }, [avatarUrl, authToken, onLoadingChange]);
58
67
  const post = (0, react_1.useCallback)((msg) => {
59
68
  iframeRef.current?.contentWindow?.postMessage(JSON.stringify(msg), '*');
60
69
  }, []);
@@ -62,13 +71,35 @@ exports.TalkingHead = (0, react_1.forwardRef)(({ avatarUrl, authToken, mood = 'n
62
71
  sendAmplitude: (amplitude) => post({ type: 'amplitude', value: amplitude }),
63
72
  scheduleVisemes: (schedule) => post({ type: 'schedule_visemes', schedule }),
64
73
  clearVisemes: () => post({ type: 'clear_visemes' }),
65
- setMood: (nextMood) => post({ type: 'mood', value: nextMood }),
66
- setHairColor: (color) => post({ type: 'hair_color', value: color }),
67
- setSkinColor: (color) => post({ type: 'skin_color', value: color }),
68
- setEyeColor: (color) => post({ type: 'eye_color', value: color }),
69
- setAccessories: (newAccessories) => post({ type: 'set_accessories', accessories: newAccessories }),
74
+ setMood: (nextMood) => {
75
+ pendingMoodRef.current = nextMood;
76
+ if (readyRef.current)
77
+ post({ type: 'mood', value: nextMood });
78
+ },
79
+ setHairColor: (color) => {
80
+ pendingHairColorRef.current = color;
81
+ if (readyRef.current)
82
+ post({ type: 'hair_color', value: color });
83
+ },
84
+ setSkinColor: (color) => {
85
+ pendingSkinColorRef.current = color;
86
+ if (readyRef.current)
87
+ post({ type: 'skin_color', value: color });
88
+ },
89
+ setEyeColor: (color) => {
90
+ pendingEyeColorRef.current = color;
91
+ if (readyRef.current)
92
+ post({ type: 'eye_color', value: color });
93
+ },
94
+ setAccessories: (newAccessories) => {
95
+ accessoriesRef.current = newAccessories;
96
+ if (readyRef.current) {
97
+ post({ type: 'set_accessories', accessories: newAccessories });
98
+ }
99
+ },
70
100
  }), [post]);
71
101
  (0, react_1.useEffect)(() => {
102
+ pendingMoodRef.current = mood;
72
103
  if (readyRef.current)
73
104
  post({ type: 'mood', value: mood });
74
105
  }, [mood, post]);
@@ -78,14 +109,17 @@ exports.TalkingHead = (0, react_1.forwardRef)(({ avatarUrl, authToken, mood = 'n
78
109
  }
79
110
  }, [accessories, post]);
80
111
  (0, react_1.useEffect)(() => {
112
+ pendingHairColorRef.current = hairColor;
81
113
  if (hairColor && readyRef.current)
82
114
  post({ type: 'hair_color', value: hairColor });
83
115
  }, [hairColor, post]);
84
116
  (0, react_1.useEffect)(() => {
117
+ pendingSkinColorRef.current = skinColor;
85
118
  if (skinColor && readyRef.current)
86
119
  post({ type: 'skin_color', value: skinColor });
87
120
  }, [skinColor, post]);
88
121
  (0, react_1.useEffect)(() => {
122
+ pendingEyeColorRef.current = eyeColor;
89
123
  if (eyeColor && readyRef.current)
90
124
  post({ type: 'eye_color', value: eyeColor });
91
125
  }, [eyeColor, post]);
@@ -122,8 +156,30 @@ exports.TalkingHead = (0, react_1.forwardRef)(({ avatarUrl, authToken, mood = 'n
122
156
  return;
123
157
  try {
124
158
  const msg = typeof event.data === 'string' ? JSON.parse(event.data) : event.data;
159
+ if (msg.type === 'loading' && typeof msg.stage === 'string') {
160
+ onLoadingChange?.({
161
+ stage: msg.stage,
162
+ progress: typeof msg.progress === 'number' && Number.isFinite(msg.progress)
163
+ ? msg.progress
164
+ : null,
165
+ });
166
+ return;
167
+ }
125
168
  if (msg.type === 'ready') {
126
169
  readyRef.current = true;
170
+ onLoadingChange?.({ stage: 'ready', progress: 100 });
171
+ if (pendingMoodRef.current) {
172
+ post({ type: 'mood', value: pendingMoodRef.current });
173
+ }
174
+ if (pendingHairColorRef.current) {
175
+ post({ type: 'hair_color', value: pendingHairColorRef.current });
176
+ }
177
+ if (pendingSkinColorRef.current) {
178
+ post({ type: 'skin_color', value: pendingSkinColorRef.current });
179
+ }
180
+ if (pendingEyeColorRef.current) {
181
+ post({ type: 'eye_color', value: pendingEyeColorRef.current });
182
+ }
127
183
  if (accessoriesRef.current?.length) {
128
184
  post({ type: 'set_accessories', accessories: accessoriesRef.current });
129
185
  }
@@ -142,7 +198,7 @@ exports.TalkingHead = (0, react_1.forwardRef)(({ avatarUrl, authToken, mood = 'n
142
198
  };
143
199
  window.addEventListener('message', onMessage);
144
200
  return () => window.removeEventListener('message', onMessage);
145
- }, [onReady, onError, post]);
201
+ }, [onLoadingChange, onReady, onError, post]);
146
202
  return ((0, jsx_runtime_1.jsx)("div", { style: { ...containerStyle, ...style }, children: (0, jsx_runtime_1.jsx)("iframe", { ref: iframeRef, srcDoc: srcdoc, style: iframeStyle, sandbox: "allow-scripts allow-same-origin", title: "TalkingHead Avatar" }) }));
147
203
  });
148
204
  exports.TalkingHead.displayName = 'TalkingHead';
package/dist/html.d.ts CHANGED
@@ -8,5 +8,12 @@ export type AvatarConfig = {
8
8
  initialHairColor?: string;
9
9
  initialSkinColor?: string;
10
10
  initialEyeColor?: string;
11
+ /**
12
+ * Base URL for vendored static assets (three.js, talkinghead.mjs, headaudio).
13
+ * When set, replaces all cdn.jsdelivr.net references so the WebView loads
14
+ * assets from your own server instead of an external CDN.
15
+ * Example: "https://studio.sitebay.org/vendor"
16
+ */
17
+ vendorBaseUrl?: string | null;
11
18
  };
12
19
  export declare function buildAvatarHtml(config: AvatarConfig): string;
package/dist/html.js CHANGED
@@ -1,10 +1,25 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
3
  exports.buildAvatarHtml = buildAvatarHtml;
4
- const VALID_MOODS = new Set(['neutral', 'happy', 'sad', 'angry', 'excited', 'thinking', 'concerned', 'surprised']);
4
+ const UPSTREAM_SAFE_MOOD_MAP = {
5
+ neutral: 'neutral',
6
+ happy: 'happy',
7
+ sad: 'sad',
8
+ angry: 'angry',
9
+ excited: 'happy',
10
+ thinking: 'neutral',
11
+ concerned: 'sad',
12
+ surprised: 'happy',
13
+ };
5
14
  function buildAvatarHtml(config) {
6
- // Sanitize mood at build time so the WebView never receives an invalid value
7
- const safeMood = VALID_MOODS.has(config.mood) ? config.mood : 'neutral';
15
+ const safeMood = UPSTREAM_SAFE_MOOD_MAP[config.mood] ?? 'neutral';
16
+ const v = config.vendorBaseUrl ? config.vendorBaseUrl.replace(/\/$/, '') : null;
17
+ const threeUrl = v ? `${v}/three.module.js` : 'https://cdn.jsdelivr.net/npm/three@0.180.0/build/three.module.js';
18
+ const threeAddonsUrl = v ? `${v}/three-addons/` : 'https://cdn.jsdelivr.net/npm/three@0.180.0/examples/jsm/';
19
+ const talkingHeadUrl = v ? `${v}/talkinghead.mjs` : 'https://cdn.jsdelivr.net/gh/met4citizen/TalkingHead@1.7/modules/talkinghead.mjs';
20
+ const headAudioUrl = v ? `${v}/headaudio.min.mjs` : 'https://cdn.jsdelivr.net/gh/met4citizen/HeadAudio@v0.1.0-alpha/dist/headaudio.min.mjs';
21
+ const headWorkletUrl = v ? `${v}/headworklet.min.mjs` : 'https://cdn.jsdelivr.net/gh/met4citizen/HeadAudio@v0.1.0-alpha/dist/headworklet.min.mjs';
22
+ const headModelUrl = v ? `${v}/model-en-mixed.bin` : 'https://cdn.jsdelivr.net/gh/met4citizen/HeadAudio@v0.1.0-alpha/dist/model-en-mixed.bin';
8
23
  return `
9
24
  <!DOCTYPE html>
10
25
  <html>
@@ -18,21 +33,48 @@ function buildAvatarHtml(config) {
18
33
  <script type="importmap">
19
34
  {
20
35
  "imports": {
21
- "three": "https://cdn.jsdelivr.net/npm/three@0.180.0/build/three.module.js",
22
- "three/addons/": "https://cdn.jsdelivr.net/npm/three@0.180.0/examples/jsm/",
23
- "talkinghead": "https://cdn.jsdelivr.net/gh/met4citizen/TalkingHead@1.7/modules/talkinghead.mjs"
36
+ "three": ${JSON.stringify(threeUrl)},
37
+ "three/addons/": ${JSON.stringify(threeAddonsUrl)},
38
+ "talkinghead": ${JSON.stringify(talkingHeadUrl)}
24
39
  }
25
40
  }
26
41
  </script>
42
+ <script>
43
+ window.ReactNativeWebView?.postMessage(
44
+ JSON.stringify({ type: 'log', message: '[bootstrap] inline script start' })
45
+ );
46
+ function postBootstrapError(kind, message) {
47
+ window.ReactNativeWebView?.postMessage(
48
+ JSON.stringify({ type: 'error', message: '[' + kind + '] ' + String(message || 'Unknown error') })
49
+ );
50
+ }
51
+ document.addEventListener('DOMContentLoaded', function() {
52
+ window.ReactNativeWebView?.postMessage(
53
+ JSON.stringify({ type: 'log', message: '[bootstrap] DOMContentLoaded' })
54
+ );
55
+ });
56
+ window.addEventListener('error', function(event) {
57
+ postBootstrapError('window.error', event?.message || event?.error?.message || 'Script error');
58
+ });
59
+ window.addEventListener('unhandledrejection', function(event) {
60
+ const reason = event?.reason;
61
+ postBootstrapError('unhandledrejection', reason?.message || reason || 'Unhandled promise rejection');
62
+ });
63
+ </script>
27
64
  </head>
28
65
  <body>
29
66
  <div id="avatar"></div>
30
67
  <script type="module">
68
+ (async function() {
69
+ window.ReactNativeWebView?.postMessage(
70
+ JSON.stringify({ type: 'log', message: '[module] script start' })
71
+ );
31
72
  const AUTH_TOKEN = ${JSON.stringify(config.authToken ?? null)};
32
- const TALKING_HEAD_URL = 'https://cdn.jsdelivr.net/gh/met4citizen/TalkingHead@1.7/modules/talkinghead.mjs';
33
- const HEAD_AUDIO_URL = 'https://cdn.jsdelivr.net/gh/met4citizen/HeadAudio@v0.1.0-alpha/dist/headaudio.min.mjs';
34
- const HEAD_AUDIO_WORKLET = 'https://cdn.jsdelivr.net/gh/met4citizen/HeadAudio@v0.1.0-alpha/dist/headworklet.min.mjs';
35
- const HEAD_AUDIO_MODEL = 'https://cdn.jsdelivr.net/gh/met4citizen/HeadAudio@v0.1.0-alpha/dist/model-en-mixed.bin';
73
+ const TALKING_HEAD_URL = ${JSON.stringify(talkingHeadUrl)};
74
+ const HEAD_AUDIO_URL = ${JSON.stringify(headAudioUrl)};
75
+ const HEAD_AUDIO_WORKLET = ${JSON.stringify(headWorkletUrl)};
76
+ const HEAD_AUDIO_MODEL = ${JSON.stringify(headModelUrl)};
77
+ const MOOD_MAP = ${JSON.stringify(UPSTREAM_SAFE_MOOD_MAP)};
36
78
 
37
79
  let AVATAR_URL = ${JSON.stringify(config.avatarUrl)};
38
80
  const INITIAL_MOOD = ${JSON.stringify(safeMood)};
@@ -55,9 +97,16 @@ function log(msg) {
55
97
  window.ReactNativeWebView?.postMessage(JSON.stringify({ type: 'log', message: msg }));
56
98
  }
57
99
 
100
+ function emitLoading(stage, progress = null) {
101
+ window.ReactNativeWebView?.postMessage(
102
+ JSON.stringify({ type: 'loading', stage, progress }),
103
+ );
104
+ }
105
+
58
106
  async function loadWithAuth(url) {
59
107
  if (!url) throw new Error('Avatar URL is empty');
60
108
  if (AUTH_TOKEN && !url.startsWith('https://cdn.jsdelivr.net')) {
109
+ emitLoading('fetching_model');
61
110
  log('Fetching authenticated model: ' + url);
62
111
  const resp = await fetch(url, { headers: { 'Authorization': 'Bearer ' + AUTH_TOKEN } });
63
112
  if (!resp.ok) throw new Error('Failed to fetch model: ' + resp.status + ' ' + resp.statusText);
@@ -153,6 +202,7 @@ async function loadStaticFallback(loadedAvatarUrl) {
153
202
  staticModel.position.sub(scaledCenter);
154
203
 
155
204
  applyAccessories(pendingAccessoriesList);
205
+ emitLoading('ready', 100);
156
206
  window.ReactNativeWebView?.postMessage(JSON.stringify({ type: 'ready' }));
157
207
 
158
208
  window.addEventListener('resize', () => {
@@ -169,7 +219,11 @@ async function loadStaticFallback(loadedAvatarUrl) {
169
219
  renderer.render(scene, camera);
170
220
  });
171
221
  }, (ev) => {
172
- if (ev.lengthComputable) log('Fallback Loading: ' + Math.round((ev.loaded / ev.total) * 100) + '%');
222
+ if (ev.lengthComputable) {
223
+ const progress = Math.round((ev.loaded / ev.total) * 100);
224
+ emitLoading('loading_fallback', progress);
225
+ log('Fallback Loading: ' + progress + '%');
226
+ }
173
227
  }, (err) => {
174
228
  log('Fallback Error: ' + err.message);
175
229
  window.ReactNativeWebView?.postMessage(JSON.stringify({ type: 'error', message: err.message }));
@@ -194,7 +248,8 @@ async function init() {
194
248
 
195
249
  try {
196
250
  log('Loading TalkingHead...');
197
- const module = await import(TALKING_HEAD_URL);
251
+ emitLoading('booting');
252
+ const module = await import('talkinghead');
198
253
 
199
254
  head = new module.TalkingHead(container, {
200
255
  ttsEndpoint: null,
@@ -224,7 +279,9 @@ async function init() {
224
279
  lipsyncLang: 'en',
225
280
  }, (ev) => {
226
281
  if (ev.lengthComputable) {
227
- log('Loading: ' + Math.round((ev.loaded / ev.total) * 100) + '%');
282
+ const progress = Math.round((ev.loaded / ev.total) * 100);
283
+ emitLoading('loading_avatar', progress);
284
+ log('Loading: ' + progress + '%');
228
285
  }
229
286
  });
230
287
  if (loadedAvatarUrl.startsWith('blob:')) URL.revokeObjectURL(loadedAvatarUrl);
@@ -262,18 +319,22 @@ async function init() {
262
319
  Object.assign(head.mtAvatar[key], { newvalue: value, needsUpdate: true });
263
320
  }
264
321
  };
265
- head.opt.update = headaudio.update.bind(headaudio);
322
+ const headaudioUpdate = headaudio.update.bind(headaudio);
323
+ head.opt.update = (dt) => { headaudioUpdate(dt); tickVisemeDecay(); };
266
324
  log('HeadAudio ready (phoneme lip sync)');
267
325
  } else {
268
326
  log('HeadAudio skipped: AudioWorklet not supported in this WebView. Use sendViseme() from native TTS callbacks.');
327
+ head.opt.update = () => tickVisemeDecay();
269
328
  }
270
329
  } catch (err) {
271
330
  log('HeadAudio unavailable, viseme/amplitude fallback active: ' + err.message);
331
+ head.opt.update = () => tickVisemeDecay();
272
332
  }
273
333
 
274
334
  startAudioInterception();
275
335
  log('[ACC] init() complete, calling applyAccessories with ' + pendingAccessoriesList.length + ' pending items');
276
336
  applyAccessories(pendingAccessoriesList);
337
+ emitLoading('ready', 100);
277
338
  window.ReactNativeWebView?.postMessage(JSON.stringify({ type: 'ready' }));
278
339
  }
279
340
  } catch (err) {
@@ -323,34 +384,79 @@ let visemeMorphCache = null;
323
384
  const visemeState = {};
324
385
 
325
386
  const VISEME_MORPH_ALIASES = {
326
- sil: ['viseme_sil', 'sil'],
327
- PP: ['viseme_PP', 'pp', 'viseme_pp'],
328
- FF: ['viseme_FF', 'ff', 'viseme_ff'],
329
- TH: ['viseme_TH', 'th', 'viseme_th'],
330
- DD: ['viseme_DD', 'dd', 'viseme_dd'],
331
- kk: ['viseme_kk', 'kk', 'viseme_k'],
332
- CH: ['viseme_CH', 'ch', 'viseme_ch'],
333
- SS: ['viseme_SS', 'ss', 'viseme_s'],
334
- nn: ['viseme_nn', 'nn', 'viseme_n'],
335
- RR: ['viseme_RR', 'rr', 'viseme_r'],
387
+ sil: ['viseme_sil', 'sil', 'mouthClose', 'mouth_close'],
388
+ PP: ['viseme_PP', 'pp', 'viseme_pp', 'mouthPucker', 'mouth_pucker'],
389
+ FF: ['viseme_FF', 'ff', 'viseme_ff', 'mouthLowerLipIn', 'mouth_lower_lip_in', 'mouthRollLower', 'mouthShrugLower'],
390
+ TH: ['viseme_TH', 'th', 'viseme_th', 'tongueOut', 'tongue_out'],
391
+ DD: ['viseme_DD', 'dd', 'viseme_dd', 'mouthShrugUpper', 'mouth_shrug_upper'],
392
+ kk: ['viseme_kk', 'kk', 'viseme_k', 'mouthStretchLeft', 'mouth_stretch_left'],
393
+ CH: ['viseme_CH', 'ch', 'viseme_ch', 'mouthSmile', 'mouth_smile', 'mouthSmileLeft', 'mouth_smile_left'],
394
+ SS: ['viseme_SS', 'ss', 'viseme_ss', 'mouthStretchRight', 'mouth_stretch_right'],
395
+ nn: ['viseme_nn', 'nn', 'viseme_n', 'mouthDimpleLeft', 'mouth_dimple_left'],
396
+ RR: ['viseme_RR', 'rr', 'viseme_r', 'mouthDimpleRight', 'mouth_dimple_right'],
336
397
  aa: ['viseme_aa', 'viseme_AA', 'aa', 'jawOpen', 'jaw_open', 'jawopen', 'mouthOpen', 'mouth_open', 'mouthopen'],
337
- ee: ['viseme_ee', 'viseme_E', 'ee'],
338
- ih: ['viseme_ih', 'viseme_I', 'ih'],
339
- oh: ['viseme_oh', 'viseme_O', 'oh'],
340
- ou: ['viseme_ou', 'viseme_U', 'ou'],
398
+ ee: ['viseme_ee', 'viseme_E', 'ee', 'mouthSmileLeft', 'mouth_smile_left'],
399
+ ih: ['viseme_ih', 'viseme_I', 'ih', 'mouthSmileRight', 'mouth_smile_right'],
400
+ oh: ['viseme_oh', 'viseme_O', 'oh', 'mouthFunnel', 'mouth_funnel'],
401
+ ou: ['viseme_ou', 'viseme_U', 'ou', 'mouthRollLower', 'mouth_roll_lower'],
402
+ };
403
+
404
+ // For ARKit models, each viseme may need multiple blend shapes driven together.
405
+ // Each entry is a list of morph names to combine for that viseme.
406
+ // The first alias list that has ANY match on the model is used.
407
+ const VISEME_COMPOUND_ARKIT = {
408
+ aa: [['jawOpen', 'mouthLowerDownLeft', 'mouthLowerDownRight'], ['jawOpen', 'mouthOpen']],
409
+ oh: [['mouthFunnel', 'jawOpen'], ['mouthFunnel']],
410
+ ou: [['mouthPucker', 'mouthRollLower'], ['mouthPucker']],
411
+ PP: [['mouthPucker', 'mouthClose'], ['mouthPucker']],
412
+ FF: [['mouthRollLower', 'mouthLowerDownLeft', 'mouthLowerDownRight'], ['mouthShrugLower']],
413
+ CH: [['mouthSmileLeft', 'mouthSmileRight', 'mouthStretchLeft', 'mouthStretchRight'], ['mouthSmileLeft', 'mouthSmileRight']],
414
+ ee: [['mouthSmileLeft', 'mouthSmileRight'], ['mouthSmileLeft']],
415
+ ih: [['mouthSmileLeft', 'mouthSmileRight'], ['mouthSmileRight']],
341
416
  };
342
417
 
343
418
  function buildVisemeMorphCache() {
344
419
  visemeMorphCache = {};
345
420
  for (const [visemeKey, aliases] of Object.entries(VISEME_MORPH_ALIASES)) {
346
421
  const entries = [];
422
+ // Check if we have a compound ARKit mapping for this viseme
423
+ const compoundOptions = VISEME_COMPOUND_ARKIT[visemeKey];
424
+ if (compoundOptions) {
425
+ // Try each compound option; use the first one where all names exist on at least one mesh
426
+ let usedCompound = false;
427
+ for (const nameList of compoundOptions) {
428
+ // Collect entries for all names across all meshes
429
+ const compoundEntries = [];
430
+ for (const mesh of mouthMeshes) {
431
+ if (!mesh.morphTargetDictionary) continue;
432
+ const dict = mesh.morphTargetDictionary;
433
+ const dictKeysLower = Object.fromEntries(Object.keys(dict).map(k => [k.toLowerCase(), k]));
434
+ for (const name of nameList) {
435
+ const found = dictKeysLower[name.toLowerCase()];
436
+ if (found !== undefined) {
437
+ compoundEntries.push({ influences: mesh.morphTargetInfluences, idx: dict[found], morphName: found });
438
+ }
439
+ }
440
+ }
441
+ if (compoundEntries.length > 0) {
442
+ entries.push(...compoundEntries);
443
+ usedCompound = true;
444
+ break;
445
+ }
446
+ }
447
+ if (usedCompound) {
448
+ visemeMorphCache[visemeKey] = entries;
449
+ continue;
450
+ }
451
+ }
452
+ // Fallback: single-alias lookup
347
453
  for (const mesh of mouthMeshes) {
348
454
  if (!mesh.morphTargetDictionary) continue;
349
455
  const dictKeys = Object.keys(mesh.morphTargetDictionary);
350
456
  for (const alias of aliases) {
351
457
  const found = dictKeys.find(k => k.toLowerCase() === alias.toLowerCase());
352
458
  if (found !== undefined) {
353
- entries.push({ influences: mesh.morphTargetInfluences, idx: mesh.morphTargetDictionary[found] });
459
+ entries.push({ influences: mesh.morphTargetInfluences, idx: mesh.morphTargetDictionary[found], morphName: found });
354
460
  break;
355
461
  }
356
462
  }
@@ -359,6 +465,12 @@ function buildVisemeMorphCache() {
359
465
  }
360
466
  const found = Object.keys(visemeMorphCache);
361
467
  log('Viseme cache: ' + (found.length > 0 ? found.join(', ') : 'none — check morph target names'));
468
+
469
+ // Always log all available morphs so we can see what the model actually has
470
+ if (mouthMeshes.length > 0) {
471
+ const allMorphs = Object.keys(mouthMeshes[0].morphTargetDictionary || {});
472
+ log('Available morphs: ' + (allMorphs.length > 0 ? allMorphs.join(', ') : 'none'));
473
+ }
362
474
  }
363
475
 
364
476
  function applyViseme(visemeKey, weight) {
@@ -368,39 +480,42 @@ function applyViseme(visemeKey, weight) {
368
480
  return;
369
481
  }
370
482
  visemeState[visemeKey] = Math.min(1, weight);
483
+ visemeStateLastSet.set(visemeKey, Date.now());
371
484
  }
372
485
 
373
- function tickVisemeDecay() {
374
- if (!visemeMorphCache) return;
375
- for (const [key, weight] of Object.entries(visemeState)) {
376
- const decayed = weight * 0.82;
377
- visemeState[key] = decayed < 0.01 ? 0 : decayed;
378
- const entries = visemeMorphCache[key];
379
- if (!entries) continue;
380
- for (const e of entries) e.influences[e.idx] = visemeState[key];
381
- }
382
- }
383
-
384
- // ============ RHUBARB VISEME SCHEDULER ============
385
- // Rhubarb mouth shapes A-H, X mapped to the closest standard viseme morph targets.
386
- // Amplitude fallback is gated while a schedule is active (visemeModeUntil).
486
+ const RHUBARB_DEFAULT_VISEME_WEIGHT = 0.72;
487
+ const RHUBARB_LABIAL_VISEME_WEIGHT = 0.85;
488
+ const RHUBARB_AA_VISEME_WEIGHT = 0.72;
489
+ const RHUBARB_ROUNDED_VISEME_WEIGHT = 0.62;
490
+ const RHUBARB_FALLBACK_AMPLITUDE_CAP = 0.72;
491
+ const RHUBARB_FALLBACK_AMPLITUDE_GAIN = 0.75;
492
+ const RHUBARB_VISEME_WEIGHTS = {
493
+ PP: RHUBARB_LABIAL_VISEME_WEIGHT,
494
+ FF: 0.78,
495
+ ee: 0.72,
496
+ ih: 0.68,
497
+ oh: RHUBARB_ROUNDED_VISEME_WEIGHT,
498
+ ou: 0.58,
499
+ aa: RHUBARB_AA_VISEME_WEIGHT,
500
+ };
387
501
 
388
502
  const RHUBARB_TO_VISEME = {
389
- X: 'sil', // silence — mouth closed
390
- A: 'PP', // m, b, p — lips together
391
- B: 'kk', // k, s, t — slightly open
392
- C: 'ee', // e as in bed — open with smile
393
- D: 'aa', // aa as in father — wide open
394
- E: 'oh', // eh/uh — rounded open
395
- F: 'ou', // oo/w — puckered
396
- G: 'FF', // f, v — teeth on lip
397
- H: 'ih', // ee as in see — wide smile
503
+ A: 'aa',
504
+ B: 'PP',
505
+ C: 'ih',
506
+ D: 'FF',
507
+ E: 'ee',
508
+ F: 'oh',
509
+ G: 'ou',
510
+ H: 'nn',
511
+ X: 'sil',
398
512
  };
399
513
 
400
514
  let rhubarbMorphCache = null;
401
515
  let visemeTimers = [];
402
516
  let activeVisemeScheduleId = 0;
403
517
  let visemeModeUntil = 0;
518
+ const visemeStateLastSet = new Map();
404
519
 
405
520
  function buildRhubarbMorphCache() {
406
521
  if (!visemeMorphCache) buildVisemeMorphCache();
@@ -419,7 +534,8 @@ function applyRhubarbCue(shape) {
419
534
  if (shape === 'X' || !rhubarbMorphCache[shape]) return;
420
535
  const visemeKey = RHUBARB_TO_VISEME[shape];
421
536
  if (visemeKey && visemeKey !== 'sil') {
422
- visemeState[visemeKey] = 1.0;
537
+ visemeState[visemeKey] = RHUBARB_VISEME_WEIGHTS[visemeKey] || RHUBARB_DEFAULT_VISEME_WEIGHT;
538
+ visemeStateLastSet.set(visemeKey, Date.now());
423
539
  }
424
540
  }
425
541
 
@@ -432,20 +548,98 @@ function clearScheduledVisemes() {
432
548
  for (const key of Object.keys(visemeState)) visemeState[key] = 0;
433
549
  }
434
550
 
551
+ function tickVisemeDecay() {
552
+ if (!visemeMorphCache) return;
553
+
554
+ const isScheduled = Date.now() < visemeModeUntil;
555
+ const hasSpecificLipShape =
556
+ visemeState.PP > 0.05 ||
557
+ visemeState.FF > 0.05 ||
558
+ visemeState.kk > 0.05 ||
559
+ visemeState.ee > 0.05 ||
560
+ visemeState.ih > 0.05;
561
+
562
+ for (const [key, weight] of Object.entries(visemeState)) {
563
+ // Only decay if we aren't in the middle of a viseme schedule.
564
+ // Scheduled visemes are cleared manually by timeouts.
565
+ if (!isScheduled) {
566
+ const decayed = weight * 0.82;
567
+ visemeState[key] = decayed < 0.01 ? 0 : decayed;
568
+ }
569
+
570
+ const entries = visemeMorphCache[key];
571
+ if (!entries) continue;
572
+
573
+ let targetWeight = visemeState[key];
574
+ if (key === 'aa' && hasSpecificLipShape) targetWeight = Math.min(targetWeight, 0.45);
575
+
576
+ for (const e of entries) {
577
+ // When TalkingHead is active, write through its morph API so the internal
578
+ // render loop doesn't overwrite our values every frame.
579
+ // Use realtime (not newvalue) — newvalue is consumed and cleared after
580
+ // a single frame, so scheduled visemes would vanish immediately.
581
+ // realtime persists until explicitly set to null.
582
+ if (head?.mtAvatar && e.morphName && head.mtAvatar[e.morphName]) {
583
+ const mt = head.mtAvatar[e.morphName];
584
+ mt.realtime = targetWeight > 0 ? targetWeight : null;
585
+ mt.needsUpdate = true;
586
+ } else {
587
+ e.influences[e.idx] = targetWeight;
588
+ }
589
+ }
590
+ }
591
+ }
592
+
435
593
  function scheduleVisemes(schedule) {
436
594
  clearScheduledVisemes();
595
+
596
+ // Prune visemeState keys that haven't been written in the last 2 seconds to
597
+ // prevent unbounded accumulation across many utterances.
598
+ const staleThreshold = Date.now() - 2000;
599
+ for (const key of Object.keys(visemeState)) {
600
+ if ((visemeStateLastSet.get(key) ?? 0) < staleThreshold) {
601
+ delete visemeState[key];
602
+ visemeStateLastSet.delete(key);
603
+ }
604
+ }
605
+
437
606
  if (!schedule || !Array.isArray(schedule.cues) || schedule.cues.length === 0) return;
438
607
 
439
608
  const myScheduleId = activeVisemeScheduleId;
440
- const startedAt = schedule.startedAtMs || Date.now();
609
+ // The startedAtMs anchor is set when tts_request_start arrives on the data
610
+ // channel. Audio doesn't play until ~300ms later (LiveKit audio buffering).
611
+ // TTS generation delay is no longer included here since visemes now arrive
612
+ // via direct ref call before the React render cycle.
613
+ const AUDIO_PIPELINE_DELAY_MS = 300;
614
+ let startedAt = (schedule.startedAtMs || Date.now()) + AUDIO_PIPELINE_DELAY_MS;
441
615
  const durationMs = schedule.durationMs || 0;
616
+ const now = Date.now();
617
+ let elapsedMs = Math.max(0, now - startedAt);
618
+
619
+ // If the schedule still arrives late after the pipeline offset, shift further
620
+ if (elapsedMs > 300 && schedule.cues.length > 3) {
621
+ const shift = Math.min(elapsedMs - 50, 500);
622
+ startedAt += shift;
623
+ elapsedMs -= shift;
624
+ log('Viseme schedule arrived late, shifting anchor forward by ' + shift + 'ms');
625
+ }
442
626
 
443
- // Gate amplitude fallback for the full duration plus a small buffer
444
- visemeModeUntil = startedAt + durationMs + 200;
627
+ const remainingMs = Math.max(0, durationMs - elapsedMs);
628
+ let scheduledCueCount = 0;
629
+ let skippedCueCount = 0;
630
+
631
+ // Gate amplitude fallback for the locally remaining duration plus a small buffer.
632
+ // If the schedule arrives a bit late, keep amplitude out of the way for the rest
633
+ // of the utterance instead of expiring immediately from the original timestamp.
634
+ visemeModeUntil = now + remainingMs + 200;
445
635
 
446
636
  for (const cue of schedule.cues) {
447
637
  const delay = cue.startMs - (Date.now() - startedAt);
448
- if (delay < -50) continue; // already in the past, skip
638
+ if (delay < -50) {
639
+ skippedCueCount++;
640
+ continue; // already in the past, skip
641
+ }
642
+ scheduledCueCount++;
449
643
 
450
644
  const applyId = setTimeout(() => {
451
645
  if (activeVisemeScheduleId !== myScheduleId) return;
@@ -461,6 +655,16 @@ function scheduleVisemes(schedule) {
461
655
  visemeTimers.push(applyId, clearId);
462
656
  }
463
657
 
658
+ log(
659
+ 'Viseme schedule received: requestId=' +
660
+ (schedule.requestId || 'unknown') +
661
+ ' cues=' + schedule.cues.length +
662
+ ' scheduled=' + scheduledCueCount +
663
+ ' skipped=' + skippedCueCount +
664
+ ' elapsedMs=' + elapsedMs +
665
+ ' remainingMs=' + remainingMs,
666
+ );
667
+
464
668
  // Ensure silence at schedule end
465
669
  const endDelay = durationMs - (Date.now() - startedAt);
466
670
  if (endDelay > 0) {
@@ -630,7 +834,10 @@ function onIncomingMessage(event) {
630
834
  if (jawKey !== undefined) jawMorphCache.push({ influences: mesh.morphTargetInfluences, idx: mesh.morphTargetDictionary[jawKey] });
631
835
  }
632
836
  }
633
- const val = Math.min(1, msg.value * 1.8);
837
+ const val = Math.min(
838
+ RHUBARB_FALLBACK_AMPLITUDE_CAP,
839
+ msg.value * RHUBARB_FALLBACK_AMPLITUDE_GAIN,
840
+ );
634
841
  amplitudeDecay = Math.max(amplitudeDecay * 0.7, val);
635
842
  for (let i = 0; i < jawMorphCache.length; i++) jawMorphCache[i].influences[jawMorphCache[i].idx] = amplitudeDecay;
636
843
  } else if (msg.type === 'viseme') {
@@ -640,8 +847,7 @@ function onIncomingMessage(event) {
640
847
  } else if (msg.type === 'clear_visemes') {
641
848
  clearScheduledVisemes();
642
849
  } else if (msg.type === 'mood' && head) {
643
- const moodMap = { neutral:'neutral', happy:'happy', sad:'sad', angry:'angry', excited:'happy', thinking:'neutral', concerned:'sad', surprised:'happy' };
644
- head.setMood(moodMap[msg.value] || 'neutral');
850
+ head.setMood(MOOD_MAP[msg.value] || 'neutral');
645
851
  } else if (msg.type === 'hair_color') {
646
852
  HAIR_COLOR = msg.value; applyColorOverrides();
647
853
  } else if (msg.type === 'skin_color') {
@@ -659,7 +865,8 @@ function onIncomingMessage(event) {
659
865
  window.addEventListener('message', onIncomingMessage);
660
866
  document.addEventListener('message', onIncomingMessage);
661
867
 
662
- init();
868
+ await init();
869
+ })();
663
870
  </script>
664
871
  </body>
665
872
  </html>
package/dist/index.d.ts CHANGED
@@ -1,3 +1,3 @@
1
- export type { TalkingHeadAccessory, TalkingHeadMood, TalkingHeadProps, TalkingHeadRef, TalkingHeadViseme, TalkingHeadVisemeCue, TalkingHeadVisemeSchedule, } from './TalkingHead';
1
+ export type { TalkingHeadAccessory, TalkingHeadLoadingState, TalkingHeadLoadingStage, TalkingHeadMood, TalkingHeadProps, TalkingHeadRef, TalkingHeadViseme, TalkingHeadVisemeCue, TalkingHeadVisemeSchedule, } from './TalkingHead';
2
2
  export { TalkingHead } from './TalkingHead';
3
3
  export * from './appearance';
@@ -1,3 +1,3 @@
1
- export type { TalkingHeadAccessory, TalkingHeadMood, TalkingHeadProps, TalkingHeadRef, } from './TalkingHead.web';
1
+ export type { TalkingHeadAccessory, TalkingHeadLoadingState, TalkingHeadMood, TalkingHeadProps, TalkingHeadRef, } from './TalkingHead.web';
2
2
  export { TalkingHead } from './TalkingHead.web';
3
3
  export * from './appearance';
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "talking-head-studio",
3
- "version": "0.2.7",
3
+ "version": "0.2.9",
4
4
  "description": "Cross-platform 3D avatar component for React Native & web — lip-sync, gestures, accessories, and LLM integration. Powered by TalkingHead + Three.js.",
5
5
  "main": "dist/index.web.js",
6
6
  "browser": "dist/index.web.js",
@@ -79,11 +79,11 @@
79
79
  },
80
80
  "sideEffects": false,
81
81
  "peerDependencies": {
82
+ "@react-three/drei": ">=9",
83
+ "@react-three/fiber": ">=8",
82
84
  "react": ">=18",
83
85
  "react-native": ">=0.73",
84
86
  "react-native-webview": ">=13",
85
- "@react-three/fiber": ">=8",
86
- "@react-three/drei": ">=9",
87
87
  "three": ">=0.170"
88
88
  },
89
89
  "peerDependenciesMeta": {
@@ -126,6 +126,7 @@
126
126
  "metro-react-native-babel-preset": "^0.77.0",
127
127
  "multer": "^2.1.0",
128
128
  "prettier": "^3.8.1",
129
+ "react-native-webview": "^13.16.0",
129
130
  "react-test-renderer": "^19.2.4",
130
131
  "ts-jest": "^29.4.6"
131
132
  }