talking-head-studio 0.2.7 → 0.2.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/TalkingHead.d.ts +8 -0
- package/dist/TalkingHead.js +122 -12
- package/dist/TalkingHead.web.d.ts +3 -2
- package/dist/TalkingHead.web.js +63 -7
- package/dist/html.d.ts +7 -0
- package/dist/html.js +268 -61
- package/dist/index.d.ts +1 -1
- package/dist/index.web.d.ts +1 -1
- package/package.json +4 -3
package/dist/TalkingHead.d.ts
CHANGED
|
@@ -1,6 +1,11 @@
|
|
|
1
1
|
import React from 'react';
|
|
2
2
|
import { type StyleProp, type ViewStyle } from 'react-native';
|
|
3
3
|
export type TalkingHeadMood = 'neutral' | 'happy' | 'sad' | 'angry' | 'excited' | 'thinking' | 'concerned' | 'surprised';
|
|
4
|
+
export type TalkingHeadLoadingStage = 'booting' | 'fetching_model' | 'loading_avatar' | 'loading_fallback' | 'ready';
|
|
5
|
+
export interface TalkingHeadLoadingState {
|
|
6
|
+
stage: TalkingHeadLoadingStage;
|
|
7
|
+
progress?: number | null;
|
|
8
|
+
}
|
|
4
9
|
/**
|
|
5
10
|
* Standard viseme keys supported by the avatar.
|
|
6
11
|
* Use with sendViseme() from your TTS viseme callbacks.
|
|
@@ -46,9 +51,12 @@ export interface TalkingHeadProps {
|
|
|
46
51
|
skinColor?: string;
|
|
47
52
|
eyeColor?: string;
|
|
48
53
|
accessories?: TalkingHeadAccessory[];
|
|
54
|
+
onLoadingChange?: (state: TalkingHeadLoadingState) => void;
|
|
49
55
|
onReady?: () => void;
|
|
50
56
|
onError?: (message: string) => void;
|
|
51
57
|
style?: StyleProp<ViewStyle>;
|
|
58
|
+
/** Base URL for vendored assets. When set, replaces all cdn.jsdelivr.net references. */
|
|
59
|
+
vendorBaseUrl?: string | null;
|
|
52
60
|
}
|
|
53
61
|
export interface TalkingHeadRef {
|
|
54
62
|
sendAmplitude: (amplitude: number) => void;
|
package/dist/TalkingHead.js
CHANGED
|
@@ -6,10 +6,23 @@ const react_1 = require("react");
|
|
|
6
6
|
const react_native_1 = require("react-native");
|
|
7
7
|
const react_native_webview_1 = require("react-native-webview");
|
|
8
8
|
const html_1 = require("./html");
|
|
9
|
-
exports.TalkingHead = (0, react_1.forwardRef)(({ avatarUrl, authToken, mood = 'neutral', cameraView = 'upper', cameraDistance = -0.5, hairColor, skinColor, eyeColor, accessories, onReady, onError, style, }, ref) => {
|
|
9
|
+
exports.TalkingHead = (0, react_1.forwardRef)(({ avatarUrl, authToken, mood = 'neutral', cameraView = 'upper', cameraDistance = -0.5, hairColor, skinColor, eyeColor, accessories, onLoadingChange, onReady, onError, style, vendorBaseUrl, }, ref) => {
|
|
10
10
|
const webViewRef = (0, react_1.useRef)(null);
|
|
11
11
|
const readyRef = (0, react_1.useRef)(false);
|
|
12
|
+
const firstMessageSeenRef = (0, react_1.useRef)(false);
|
|
13
|
+
const bootTimerRef = (0, react_1.useRef)(null);
|
|
14
|
+
const pendingMoodRef = (0, react_1.useRef)(mood);
|
|
15
|
+
const pendingHairColorRef = (0, react_1.useRef)(hairColor);
|
|
16
|
+
const pendingSkinColorRef = (0, react_1.useRef)(skinColor);
|
|
17
|
+
const pendingEyeColorRef = (0, react_1.useRef)(eyeColor);
|
|
12
18
|
const accessoriesRef = (0, react_1.useRef)(accessories);
|
|
19
|
+
// Stable refs for callbacks so they never retrigger the boot effect.
|
|
20
|
+
const onLoadingChangeRef = (0, react_1.useRef)(onLoadingChange);
|
|
21
|
+
onLoadingChangeRef.current = onLoadingChange;
|
|
22
|
+
const onReadyRef = (0, react_1.useRef)(onReady);
|
|
23
|
+
onReadyRef.current = onReady;
|
|
24
|
+
const onErrorRef = (0, react_1.useRef)(onError);
|
|
25
|
+
onErrorRef.current = onError;
|
|
13
26
|
// The WebView HTML is built once from stable initial values.
|
|
14
27
|
// avatarUrl + authToken changing causes a controlled key-based remount.
|
|
15
28
|
// All other prop changes (mood, colors, accessories) go via postMessage.
|
|
@@ -23,22 +36,70 @@ exports.TalkingHead = (0, react_1.forwardRef)(({ avatarUrl, authToken, mood = 'n
|
|
|
23
36
|
setWebViewKey(`${avatarUrl}__${authToken ?? ''}`);
|
|
24
37
|
}
|
|
25
38
|
}, [avatarUrl, authToken]);
|
|
39
|
+
(0, react_1.useEffect)(() => {
|
|
40
|
+
if (!avatarUrl)
|
|
41
|
+
return;
|
|
42
|
+
firstMessageSeenRef.current = false;
|
|
43
|
+
onLoadingChangeRef.current?.({ stage: 'booting', progress: null });
|
|
44
|
+
if (bootTimerRef.current)
|
|
45
|
+
clearTimeout(bootTimerRef.current);
|
|
46
|
+
bootTimerRef.current = setTimeout(() => {
|
|
47
|
+
if (readyRef.current || firstMessageSeenRef.current)
|
|
48
|
+
return;
|
|
49
|
+
const vendorLabel = vendorBaseUrl ? ` vendor=${vendorBaseUrl}` : ' vendor=<cdn>';
|
|
50
|
+
const avatarLabel = avatarUrl && avatarUrl.length > 80 ? avatarUrl.slice(0, 80) + '…' : avatarUrl;
|
|
51
|
+
onErrorRef.current?.(`[boot timeout] No WebView messages received after 8s.${vendorLabel} avatar=${avatarLabel}`);
|
|
52
|
+
}, 8000);
|
|
53
|
+
return () => {
|
|
54
|
+
if (bootTimerRef.current) {
|
|
55
|
+
clearTimeout(bootTimerRef.current);
|
|
56
|
+
bootTimerRef.current = null;
|
|
57
|
+
}
|
|
58
|
+
};
|
|
59
|
+
}, [avatarUrl, authToken, vendorBaseUrl]);
|
|
26
60
|
const post = (0, react_1.useCallback)((msg) => {
|
|
27
|
-
|
|
61
|
+
try {
|
|
62
|
+
webViewRef.current?.postMessage(JSON.stringify(msg));
|
|
63
|
+
}
|
|
64
|
+
catch {
|
|
65
|
+
// WebView ref frozen/invalidated during unmount — ignore
|
|
66
|
+
}
|
|
28
67
|
}, []);
|
|
29
68
|
(0, react_1.useImperativeHandle)(ref, () => ({
|
|
30
69
|
sendAmplitude: (amplitude) => post({ type: 'amplitude', value: amplitude }),
|
|
31
70
|
sendViseme: (viseme, weight = 1.0) => post({ type: 'viseme', viseme, weight }),
|
|
32
71
|
scheduleVisemes: (schedule) => post({ type: 'schedule_visemes', schedule }),
|
|
33
72
|
clearVisemes: () => post({ type: 'clear_visemes' }),
|
|
34
|
-
setMood: (nextMood) =>
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
73
|
+
setMood: (nextMood) => {
|
|
74
|
+
pendingMoodRef.current = nextMood;
|
|
75
|
+
if (readyRef.current)
|
|
76
|
+
post({ type: 'mood', value: nextMood });
|
|
77
|
+
},
|
|
78
|
+
setHairColor: (color) => {
|
|
79
|
+
pendingHairColorRef.current = color;
|
|
80
|
+
if (readyRef.current)
|
|
81
|
+
post({ type: 'hair_color', value: color });
|
|
82
|
+
},
|
|
83
|
+
setSkinColor: (color) => {
|
|
84
|
+
pendingSkinColorRef.current = color;
|
|
85
|
+
if (readyRef.current)
|
|
86
|
+
post({ type: 'skin_color', value: color });
|
|
87
|
+
},
|
|
88
|
+
setEyeColor: (color) => {
|
|
89
|
+
pendingEyeColorRef.current = color;
|
|
90
|
+
if (readyRef.current)
|
|
91
|
+
post({ type: 'eye_color', value: color });
|
|
92
|
+
},
|
|
93
|
+
setAccessories: (newAccessories) => {
|
|
94
|
+
accessoriesRef.current = newAccessories;
|
|
95
|
+
if (readyRef.current) {
|
|
96
|
+
post({ type: 'set_accessories', accessories: newAccessories });
|
|
97
|
+
}
|
|
98
|
+
},
|
|
39
99
|
}), [post]);
|
|
40
100
|
// Sync mood via postMessage only — never causes a WebView reload
|
|
41
101
|
(0, react_1.useEffect)(() => {
|
|
102
|
+
pendingMoodRef.current = mood;
|
|
42
103
|
if (readyRef.current)
|
|
43
104
|
post({ type: 'mood', value: mood });
|
|
44
105
|
}, [mood, post]);
|
|
@@ -49,14 +110,17 @@ exports.TalkingHead = (0, react_1.forwardRef)(({ avatarUrl, authToken, mood = 'n
|
|
|
49
110
|
}
|
|
50
111
|
}, [accessories, post]);
|
|
51
112
|
(0, react_1.useEffect)(() => {
|
|
113
|
+
pendingHairColorRef.current = hairColor;
|
|
52
114
|
if (hairColor && readyRef.current)
|
|
53
115
|
post({ type: 'hair_color', value: hairColor });
|
|
54
116
|
}, [hairColor, post]);
|
|
55
117
|
(0, react_1.useEffect)(() => {
|
|
118
|
+
pendingSkinColorRef.current = skinColor;
|
|
56
119
|
if (skinColor && readyRef.current)
|
|
57
120
|
post({ type: 'skin_color', value: skinColor });
|
|
58
121
|
}, [skinColor, post]);
|
|
59
122
|
(0, react_1.useEffect)(() => {
|
|
123
|
+
pendingEyeColorRef.current = eyeColor;
|
|
60
124
|
if (eyeColor && readyRef.current)
|
|
61
125
|
post({ type: 'eye_color', value: eyeColor });
|
|
62
126
|
}, [eyeColor, post]);
|
|
@@ -65,6 +129,18 @@ exports.TalkingHead = (0, react_1.forwardRef)(({ avatarUrl, authToken, mood = 'n
|
|
|
65
129
|
const [initialHairColor] = (0, react_1.useState)(hairColor);
|
|
66
130
|
const [initialSkinColor] = (0, react_1.useState)(skinColor);
|
|
67
131
|
const [initialEyeColor] = (0, react_1.useState)(eyeColor);
|
|
132
|
+
// Use the vendor origin as baseUrl so dynamic module imports are same-origin
|
|
133
|
+
// and don't get blocked by the WebView's security model.
|
|
134
|
+
const webViewBaseUrl = (0, react_1.useMemo)(() => {
|
|
135
|
+
if (!vendorBaseUrl)
|
|
136
|
+
return 'https://localhost/';
|
|
137
|
+
try {
|
|
138
|
+
return new URL(vendorBaseUrl).origin + '/';
|
|
139
|
+
}
|
|
140
|
+
catch {
|
|
141
|
+
return 'https://localhost/';
|
|
142
|
+
}
|
|
143
|
+
}, [vendorBaseUrl]);
|
|
68
144
|
// html is stable — only rebuilds when webViewKey changes (avatarUrl/authToken)
|
|
69
145
|
const html = (0, react_1.useMemo)(() => (0, html_1.buildAvatarHtml)({
|
|
70
146
|
avatarUrl,
|
|
@@ -75,22 +151,46 @@ exports.TalkingHead = (0, react_1.forwardRef)(({ avatarUrl, authToken, mood = 'n
|
|
|
75
151
|
initialHairColor,
|
|
76
152
|
initialSkinColor,
|
|
77
153
|
initialEyeColor,
|
|
154
|
+
vendorBaseUrl,
|
|
78
155
|
}),
|
|
79
156
|
// eslint-disable-next-line react-hooks/exhaustive-deps
|
|
80
157
|
[webViewKey]);
|
|
81
158
|
const onMessage = (0, react_1.useCallback)((event) => {
|
|
82
159
|
try {
|
|
160
|
+
firstMessageSeenRef.current = true;
|
|
83
161
|
const msg = JSON.parse(event.nativeEvent.data);
|
|
162
|
+
if (msg.type === 'loading' && typeof msg.stage === 'string') {
|
|
163
|
+
onLoadingChangeRef.current?.({
|
|
164
|
+
stage: msg.stage,
|
|
165
|
+
progress: typeof msg.progress === 'number' && Number.isFinite(msg.progress)
|
|
166
|
+
? msg.progress
|
|
167
|
+
: null,
|
|
168
|
+
});
|
|
169
|
+
return;
|
|
170
|
+
}
|
|
84
171
|
if (msg.type === 'ready') {
|
|
85
172
|
readyRef.current = true;
|
|
86
|
-
|
|
173
|
+
onLoadingChangeRef.current?.({ stage: 'ready', progress: 100 });
|
|
174
|
+
// Flush pending appearance updates that arrived before the WebView was ready.
|
|
175
|
+
if (pendingMoodRef.current) {
|
|
176
|
+
post({ type: 'mood', value: pendingMoodRef.current });
|
|
177
|
+
}
|
|
178
|
+
if (pendingHairColorRef.current) {
|
|
179
|
+
post({ type: 'hair_color', value: pendingHairColorRef.current });
|
|
180
|
+
}
|
|
181
|
+
if (pendingSkinColorRef.current) {
|
|
182
|
+
post({ type: 'skin_color', value: pendingSkinColorRef.current });
|
|
183
|
+
}
|
|
184
|
+
if (pendingEyeColorRef.current) {
|
|
185
|
+
post({ type: 'eye_color', value: pendingEyeColorRef.current });
|
|
186
|
+
}
|
|
87
187
|
if (accessoriesRef.current?.length) {
|
|
88
188
|
post({ type: 'set_accessories', accessories: accessoriesRef.current });
|
|
89
189
|
}
|
|
90
|
-
|
|
190
|
+
onReadyRef.current?.();
|
|
91
191
|
}
|
|
92
192
|
else if (msg.type === 'error') {
|
|
93
|
-
|
|
193
|
+
onErrorRef.current?.(msg.message);
|
|
94
194
|
}
|
|
95
195
|
else if (msg.type === 'log') {
|
|
96
196
|
console.log('[TalkingHead]', msg.message);
|
|
@@ -99,8 +199,18 @@ exports.TalkingHead = (0, react_1.forwardRef)(({ avatarUrl, authToken, mood = 'n
|
|
|
99
199
|
catch (err) {
|
|
100
200
|
console.warn('[TalkingHead] Invalid message from WebView:', err);
|
|
101
201
|
}
|
|
102
|
-
}, [
|
|
103
|
-
|
|
202
|
+
}, [post]);
|
|
203
|
+
const handleWebViewError = (0, react_1.useCallback)((event) => {
|
|
204
|
+
firstMessageSeenRef.current = true;
|
|
205
|
+
const description = event.nativeEvent.description || 'WebView failed to load avatar';
|
|
206
|
+
onErrorRef.current?.(`[webview] ${description}`);
|
|
207
|
+
}, []);
|
|
208
|
+
const handleWebViewHttpError = (0, react_1.useCallback)((event) => {
|
|
209
|
+
firstMessageSeenRef.current = true;
|
|
210
|
+
const { statusCode, description, url } = event.nativeEvent;
|
|
211
|
+
onErrorRef.current?.(`[http ${statusCode}] ${description || url || 'Avatar request failed'}`);
|
|
212
|
+
}, []);
|
|
213
|
+
return ((0, jsx_runtime_1.jsx)(react_native_1.View, { style: [styles.container, style], children: (0, jsx_runtime_1.jsx)(react_native_webview_1.WebView, { ref: webViewRef, source: { html, baseUrl: webViewBaseUrl }, style: styles.webview, javaScriptEnabled: true, domStorageEnabled: true, allowsInlineMediaPlayback: true, mediaPlaybackRequiresUserAction: false, onMessage: onMessage, onError: handleWebViewError, onHttpError: handleWebViewHttpError, onLoadStart: () => console.log('[TalkingHead] WebView load start'), onLoadEnd: () => console.log('[TalkingHead] WebView load end'), onLoadProgress: (event) => console.log('[TalkingHead] WebView progress', event.nativeEvent.progress), originWhitelist: ['*'], allowFileAccess: true, allowFileAccessFromFileURLs: true, allowUniversalAccessFromFileURLs: true, mixedContentMode: "always" }, webViewKey) }));
|
|
104
214
|
});
|
|
105
215
|
exports.TalkingHead.displayName = 'TalkingHead';
|
|
106
216
|
const styles = react_native_1.StyleSheet.create({
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import React from 'react';
|
|
2
|
-
import type { TalkingHeadVisemeCue, TalkingHeadVisemeSchedule } from './TalkingHead';
|
|
3
|
-
export type { TalkingHeadVisemeCue, TalkingHeadVisemeSchedule };
|
|
2
|
+
import type { TalkingHeadLoadingState, TalkingHeadVisemeCue, TalkingHeadVisemeSchedule } from './TalkingHead';
|
|
3
|
+
export type { TalkingHeadLoadingState, TalkingHeadVisemeCue, TalkingHeadVisemeSchedule };
|
|
4
4
|
export type TalkingHeadMood = 'neutral' | 'happy' | 'sad' | 'angry' | 'excited' | 'thinking' | 'concerned' | 'surprised';
|
|
5
5
|
export interface TalkingHeadAccessory {
|
|
6
6
|
id: string;
|
|
@@ -20,6 +20,7 @@ export interface TalkingHeadProps {
|
|
|
20
20
|
skinColor?: string;
|
|
21
21
|
eyeColor?: string;
|
|
22
22
|
accessories?: TalkingHeadAccessory[];
|
|
23
|
+
onLoadingChange?: (state: TalkingHeadLoadingState) => void;
|
|
23
24
|
onReady?: () => void;
|
|
24
25
|
onError?: (message: string) => void;
|
|
25
26
|
style?: React.CSSProperties;
|
package/dist/TalkingHead.web.js
CHANGED
|
@@ -48,13 +48,22 @@ const iframeStyle = {
|
|
|
48
48
|
border: 'none',
|
|
49
49
|
backgroundColor: 'transparent',
|
|
50
50
|
};
|
|
51
|
-
exports.TalkingHead = (0, react_1.forwardRef)(({ avatarUrl, authToken, mood = 'neutral', cameraView = 'upper', cameraDistance = -0.5, hairColor, skinColor, eyeColor, accessories, onReady, onError, style, }, ref) => {
|
|
51
|
+
exports.TalkingHead = (0, react_1.forwardRef)(({ avatarUrl, authToken, mood = 'neutral', cameraView = 'upper', cameraDistance = -0.5, hairColor, skinColor, eyeColor, accessories, onLoadingChange, onReady, onError, style, }, ref) => {
|
|
52
52
|
const iframeRef = (0, react_1.useRef)(null);
|
|
53
53
|
const readyRef = (0, react_1.useRef)(false);
|
|
54
|
+
const pendingMoodRef = (0, react_1.useRef)(mood);
|
|
55
|
+
const pendingHairColorRef = (0, react_1.useRef)(hairColor);
|
|
56
|
+
const pendingSkinColorRef = (0, react_1.useRef)(skinColor);
|
|
57
|
+
const pendingEyeColorRef = (0, react_1.useRef)(eyeColor);
|
|
54
58
|
const accessoriesRef = (0, react_1.useRef)(accessories);
|
|
55
59
|
(0, react_1.useEffect)(() => {
|
|
56
60
|
accessoriesRef.current = accessories;
|
|
57
61
|
}, [accessories]);
|
|
62
|
+
(0, react_1.useEffect)(() => {
|
|
63
|
+
if (!avatarUrl)
|
|
64
|
+
return;
|
|
65
|
+
onLoadingChange?.({ stage: 'booting', progress: null });
|
|
66
|
+
}, [avatarUrl, authToken, onLoadingChange]);
|
|
58
67
|
const post = (0, react_1.useCallback)((msg) => {
|
|
59
68
|
iframeRef.current?.contentWindow?.postMessage(JSON.stringify(msg), '*');
|
|
60
69
|
}, []);
|
|
@@ -62,13 +71,35 @@ exports.TalkingHead = (0, react_1.forwardRef)(({ avatarUrl, authToken, mood = 'n
|
|
|
62
71
|
sendAmplitude: (amplitude) => post({ type: 'amplitude', value: amplitude }),
|
|
63
72
|
scheduleVisemes: (schedule) => post({ type: 'schedule_visemes', schedule }),
|
|
64
73
|
clearVisemes: () => post({ type: 'clear_visemes' }),
|
|
65
|
-
setMood: (nextMood) =>
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
74
|
+
setMood: (nextMood) => {
|
|
75
|
+
pendingMoodRef.current = nextMood;
|
|
76
|
+
if (readyRef.current)
|
|
77
|
+
post({ type: 'mood', value: nextMood });
|
|
78
|
+
},
|
|
79
|
+
setHairColor: (color) => {
|
|
80
|
+
pendingHairColorRef.current = color;
|
|
81
|
+
if (readyRef.current)
|
|
82
|
+
post({ type: 'hair_color', value: color });
|
|
83
|
+
},
|
|
84
|
+
setSkinColor: (color) => {
|
|
85
|
+
pendingSkinColorRef.current = color;
|
|
86
|
+
if (readyRef.current)
|
|
87
|
+
post({ type: 'skin_color', value: color });
|
|
88
|
+
},
|
|
89
|
+
setEyeColor: (color) => {
|
|
90
|
+
pendingEyeColorRef.current = color;
|
|
91
|
+
if (readyRef.current)
|
|
92
|
+
post({ type: 'eye_color', value: color });
|
|
93
|
+
},
|
|
94
|
+
setAccessories: (newAccessories) => {
|
|
95
|
+
accessoriesRef.current = newAccessories;
|
|
96
|
+
if (readyRef.current) {
|
|
97
|
+
post({ type: 'set_accessories', accessories: newAccessories });
|
|
98
|
+
}
|
|
99
|
+
},
|
|
70
100
|
}), [post]);
|
|
71
101
|
(0, react_1.useEffect)(() => {
|
|
102
|
+
pendingMoodRef.current = mood;
|
|
72
103
|
if (readyRef.current)
|
|
73
104
|
post({ type: 'mood', value: mood });
|
|
74
105
|
}, [mood, post]);
|
|
@@ -78,14 +109,17 @@ exports.TalkingHead = (0, react_1.forwardRef)(({ avatarUrl, authToken, mood = 'n
|
|
|
78
109
|
}
|
|
79
110
|
}, [accessories, post]);
|
|
80
111
|
(0, react_1.useEffect)(() => {
|
|
112
|
+
pendingHairColorRef.current = hairColor;
|
|
81
113
|
if (hairColor && readyRef.current)
|
|
82
114
|
post({ type: 'hair_color', value: hairColor });
|
|
83
115
|
}, [hairColor, post]);
|
|
84
116
|
(0, react_1.useEffect)(() => {
|
|
117
|
+
pendingSkinColorRef.current = skinColor;
|
|
85
118
|
if (skinColor && readyRef.current)
|
|
86
119
|
post({ type: 'skin_color', value: skinColor });
|
|
87
120
|
}, [skinColor, post]);
|
|
88
121
|
(0, react_1.useEffect)(() => {
|
|
122
|
+
pendingEyeColorRef.current = eyeColor;
|
|
89
123
|
if (eyeColor && readyRef.current)
|
|
90
124
|
post({ type: 'eye_color', value: eyeColor });
|
|
91
125
|
}, [eyeColor, post]);
|
|
@@ -122,8 +156,30 @@ exports.TalkingHead = (0, react_1.forwardRef)(({ avatarUrl, authToken, mood = 'n
|
|
|
122
156
|
return;
|
|
123
157
|
try {
|
|
124
158
|
const msg = typeof event.data === 'string' ? JSON.parse(event.data) : event.data;
|
|
159
|
+
if (msg.type === 'loading' && typeof msg.stage === 'string') {
|
|
160
|
+
onLoadingChange?.({
|
|
161
|
+
stage: msg.stage,
|
|
162
|
+
progress: typeof msg.progress === 'number' && Number.isFinite(msg.progress)
|
|
163
|
+
? msg.progress
|
|
164
|
+
: null,
|
|
165
|
+
});
|
|
166
|
+
return;
|
|
167
|
+
}
|
|
125
168
|
if (msg.type === 'ready') {
|
|
126
169
|
readyRef.current = true;
|
|
170
|
+
onLoadingChange?.({ stage: 'ready', progress: 100 });
|
|
171
|
+
if (pendingMoodRef.current) {
|
|
172
|
+
post({ type: 'mood', value: pendingMoodRef.current });
|
|
173
|
+
}
|
|
174
|
+
if (pendingHairColorRef.current) {
|
|
175
|
+
post({ type: 'hair_color', value: pendingHairColorRef.current });
|
|
176
|
+
}
|
|
177
|
+
if (pendingSkinColorRef.current) {
|
|
178
|
+
post({ type: 'skin_color', value: pendingSkinColorRef.current });
|
|
179
|
+
}
|
|
180
|
+
if (pendingEyeColorRef.current) {
|
|
181
|
+
post({ type: 'eye_color', value: pendingEyeColorRef.current });
|
|
182
|
+
}
|
|
127
183
|
if (accessoriesRef.current?.length) {
|
|
128
184
|
post({ type: 'set_accessories', accessories: accessoriesRef.current });
|
|
129
185
|
}
|
|
@@ -142,7 +198,7 @@ exports.TalkingHead = (0, react_1.forwardRef)(({ avatarUrl, authToken, mood = 'n
|
|
|
142
198
|
};
|
|
143
199
|
window.addEventListener('message', onMessage);
|
|
144
200
|
return () => window.removeEventListener('message', onMessage);
|
|
145
|
-
}, [onReady, onError, post]);
|
|
201
|
+
}, [onLoadingChange, onReady, onError, post]);
|
|
146
202
|
return ((0, jsx_runtime_1.jsx)("div", { style: { ...containerStyle, ...style }, children: (0, jsx_runtime_1.jsx)("iframe", { ref: iframeRef, srcDoc: srcdoc, style: iframeStyle, sandbox: "allow-scripts allow-same-origin", title: "TalkingHead Avatar" }) }));
|
|
147
203
|
});
|
|
148
204
|
exports.TalkingHead.displayName = 'TalkingHead';
|
package/dist/html.d.ts
CHANGED
|
@@ -8,5 +8,12 @@ export type AvatarConfig = {
|
|
|
8
8
|
initialHairColor?: string;
|
|
9
9
|
initialSkinColor?: string;
|
|
10
10
|
initialEyeColor?: string;
|
|
11
|
+
/**
|
|
12
|
+
* Base URL for vendored static assets (three.js, talkinghead.mjs, headaudio).
|
|
13
|
+
* When set, replaces all cdn.jsdelivr.net references so the WebView loads
|
|
14
|
+
* assets from your own server instead of an external CDN.
|
|
15
|
+
* Example: "https://studio.sitebay.org/vendor"
|
|
16
|
+
*/
|
|
17
|
+
vendorBaseUrl?: string | null;
|
|
11
18
|
};
|
|
12
19
|
export declare function buildAvatarHtml(config: AvatarConfig): string;
|
package/dist/html.js
CHANGED
|
@@ -1,10 +1,25 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.buildAvatarHtml = buildAvatarHtml;
|
|
4
|
-
const
|
|
4
|
+
const UPSTREAM_SAFE_MOOD_MAP = {
|
|
5
|
+
neutral: 'neutral',
|
|
6
|
+
happy: 'happy',
|
|
7
|
+
sad: 'sad',
|
|
8
|
+
angry: 'angry',
|
|
9
|
+
excited: 'happy',
|
|
10
|
+
thinking: 'neutral',
|
|
11
|
+
concerned: 'sad',
|
|
12
|
+
surprised: 'happy',
|
|
13
|
+
};
|
|
5
14
|
function buildAvatarHtml(config) {
|
|
6
|
-
|
|
7
|
-
const
|
|
15
|
+
const safeMood = UPSTREAM_SAFE_MOOD_MAP[config.mood] ?? 'neutral';
|
|
16
|
+
const v = config.vendorBaseUrl ? config.vendorBaseUrl.replace(/\/$/, '') : null;
|
|
17
|
+
const threeUrl = v ? `${v}/three.module.js` : 'https://cdn.jsdelivr.net/npm/three@0.180.0/build/three.module.js';
|
|
18
|
+
const threeAddonsUrl = v ? `${v}/three-addons/` : 'https://cdn.jsdelivr.net/npm/three@0.180.0/examples/jsm/';
|
|
19
|
+
const talkingHeadUrl = v ? `${v}/talkinghead.mjs` : 'https://cdn.jsdelivr.net/gh/met4citizen/TalkingHead@1.7/modules/talkinghead.mjs';
|
|
20
|
+
const headAudioUrl = v ? `${v}/headaudio.min.mjs` : 'https://cdn.jsdelivr.net/gh/met4citizen/HeadAudio@v0.1.0-alpha/dist/headaudio.min.mjs';
|
|
21
|
+
const headWorkletUrl = v ? `${v}/headworklet.min.mjs` : 'https://cdn.jsdelivr.net/gh/met4citizen/HeadAudio@v0.1.0-alpha/dist/headworklet.min.mjs';
|
|
22
|
+
const headModelUrl = v ? `${v}/model-en-mixed.bin` : 'https://cdn.jsdelivr.net/gh/met4citizen/HeadAudio@v0.1.0-alpha/dist/model-en-mixed.bin';
|
|
8
23
|
return `
|
|
9
24
|
<!DOCTYPE html>
|
|
10
25
|
<html>
|
|
@@ -18,21 +33,48 @@ function buildAvatarHtml(config) {
|
|
|
18
33
|
<script type="importmap">
|
|
19
34
|
{
|
|
20
35
|
"imports": {
|
|
21
|
-
"three":
|
|
22
|
-
"three/addons/":
|
|
23
|
-
"talkinghead":
|
|
36
|
+
"three": ${JSON.stringify(threeUrl)},
|
|
37
|
+
"three/addons/": ${JSON.stringify(threeAddonsUrl)},
|
|
38
|
+
"talkinghead": ${JSON.stringify(talkingHeadUrl)}
|
|
24
39
|
}
|
|
25
40
|
}
|
|
26
41
|
</script>
|
|
42
|
+
<script>
|
|
43
|
+
window.ReactNativeWebView?.postMessage(
|
|
44
|
+
JSON.stringify({ type: 'log', message: '[bootstrap] inline script start' })
|
|
45
|
+
);
|
|
46
|
+
function postBootstrapError(kind, message) {
|
|
47
|
+
window.ReactNativeWebView?.postMessage(
|
|
48
|
+
JSON.stringify({ type: 'error', message: '[' + kind + '] ' + String(message || 'Unknown error') })
|
|
49
|
+
);
|
|
50
|
+
}
|
|
51
|
+
document.addEventListener('DOMContentLoaded', function() {
|
|
52
|
+
window.ReactNativeWebView?.postMessage(
|
|
53
|
+
JSON.stringify({ type: 'log', message: '[bootstrap] DOMContentLoaded' })
|
|
54
|
+
);
|
|
55
|
+
});
|
|
56
|
+
window.addEventListener('error', function(event) {
|
|
57
|
+
postBootstrapError('window.error', event?.message || event?.error?.message || 'Script error');
|
|
58
|
+
});
|
|
59
|
+
window.addEventListener('unhandledrejection', function(event) {
|
|
60
|
+
const reason = event?.reason;
|
|
61
|
+
postBootstrapError('unhandledrejection', reason?.message || reason || 'Unhandled promise rejection');
|
|
62
|
+
});
|
|
63
|
+
</script>
|
|
27
64
|
</head>
|
|
28
65
|
<body>
|
|
29
66
|
<div id="avatar"></div>
|
|
30
67
|
<script type="module">
|
|
68
|
+
(async function() {
|
|
69
|
+
window.ReactNativeWebView?.postMessage(
|
|
70
|
+
JSON.stringify({ type: 'log', message: '[module] script start' })
|
|
71
|
+
);
|
|
31
72
|
const AUTH_TOKEN = ${JSON.stringify(config.authToken ?? null)};
|
|
32
|
-
const TALKING_HEAD_URL =
|
|
33
|
-
const HEAD_AUDIO_URL =
|
|
34
|
-
const HEAD_AUDIO_WORKLET =
|
|
35
|
-
const HEAD_AUDIO_MODEL =
|
|
73
|
+
const TALKING_HEAD_URL = ${JSON.stringify(talkingHeadUrl)};
|
|
74
|
+
const HEAD_AUDIO_URL = ${JSON.stringify(headAudioUrl)};
|
|
75
|
+
const HEAD_AUDIO_WORKLET = ${JSON.stringify(headWorkletUrl)};
|
|
76
|
+
const HEAD_AUDIO_MODEL = ${JSON.stringify(headModelUrl)};
|
|
77
|
+
const MOOD_MAP = ${JSON.stringify(UPSTREAM_SAFE_MOOD_MAP)};
|
|
36
78
|
|
|
37
79
|
let AVATAR_URL = ${JSON.stringify(config.avatarUrl)};
|
|
38
80
|
const INITIAL_MOOD = ${JSON.stringify(safeMood)};
|
|
@@ -55,9 +97,16 @@ function log(msg) {
|
|
|
55
97
|
window.ReactNativeWebView?.postMessage(JSON.stringify({ type: 'log', message: msg }));
|
|
56
98
|
}
|
|
57
99
|
|
|
100
|
+
function emitLoading(stage, progress = null) {
|
|
101
|
+
window.ReactNativeWebView?.postMessage(
|
|
102
|
+
JSON.stringify({ type: 'loading', stage, progress }),
|
|
103
|
+
);
|
|
104
|
+
}
|
|
105
|
+
|
|
58
106
|
async function loadWithAuth(url) {
|
|
59
107
|
if (!url) throw new Error('Avatar URL is empty');
|
|
60
108
|
if (AUTH_TOKEN && !url.startsWith('https://cdn.jsdelivr.net')) {
|
|
109
|
+
emitLoading('fetching_model');
|
|
61
110
|
log('Fetching authenticated model: ' + url);
|
|
62
111
|
const resp = await fetch(url, { headers: { 'Authorization': 'Bearer ' + AUTH_TOKEN } });
|
|
63
112
|
if (!resp.ok) throw new Error('Failed to fetch model: ' + resp.status + ' ' + resp.statusText);
|
|
@@ -153,6 +202,7 @@ async function loadStaticFallback(loadedAvatarUrl) {
|
|
|
153
202
|
staticModel.position.sub(scaledCenter);
|
|
154
203
|
|
|
155
204
|
applyAccessories(pendingAccessoriesList);
|
|
205
|
+
emitLoading('ready', 100);
|
|
156
206
|
window.ReactNativeWebView?.postMessage(JSON.stringify({ type: 'ready' }));
|
|
157
207
|
|
|
158
208
|
window.addEventListener('resize', () => {
|
|
@@ -169,7 +219,11 @@ async function loadStaticFallback(loadedAvatarUrl) {
|
|
|
169
219
|
renderer.render(scene, camera);
|
|
170
220
|
});
|
|
171
221
|
}, (ev) => {
|
|
172
|
-
if (ev.lengthComputable)
|
|
222
|
+
if (ev.lengthComputable) {
|
|
223
|
+
const progress = Math.round((ev.loaded / ev.total) * 100);
|
|
224
|
+
emitLoading('loading_fallback', progress);
|
|
225
|
+
log('Fallback Loading: ' + progress + '%');
|
|
226
|
+
}
|
|
173
227
|
}, (err) => {
|
|
174
228
|
log('Fallback Error: ' + err.message);
|
|
175
229
|
window.ReactNativeWebView?.postMessage(JSON.stringify({ type: 'error', message: err.message }));
|
|
@@ -194,7 +248,8 @@ async function init() {
|
|
|
194
248
|
|
|
195
249
|
try {
|
|
196
250
|
log('Loading TalkingHead...');
|
|
197
|
-
|
|
251
|
+
emitLoading('booting');
|
|
252
|
+
const module = await import('talkinghead');
|
|
198
253
|
|
|
199
254
|
head = new module.TalkingHead(container, {
|
|
200
255
|
ttsEndpoint: null,
|
|
@@ -224,7 +279,9 @@ async function init() {
|
|
|
224
279
|
lipsyncLang: 'en',
|
|
225
280
|
}, (ev) => {
|
|
226
281
|
if (ev.lengthComputable) {
|
|
227
|
-
|
|
282
|
+
const progress = Math.round((ev.loaded / ev.total) * 100);
|
|
283
|
+
emitLoading('loading_avatar', progress);
|
|
284
|
+
log('Loading: ' + progress + '%');
|
|
228
285
|
}
|
|
229
286
|
});
|
|
230
287
|
if (loadedAvatarUrl.startsWith('blob:')) URL.revokeObjectURL(loadedAvatarUrl);
|
|
@@ -262,18 +319,22 @@ async function init() {
|
|
|
262
319
|
Object.assign(head.mtAvatar[key], { newvalue: value, needsUpdate: true });
|
|
263
320
|
}
|
|
264
321
|
};
|
|
265
|
-
|
|
322
|
+
const headaudioUpdate = headaudio.update.bind(headaudio);
|
|
323
|
+
head.opt.update = (dt) => { headaudioUpdate(dt); tickVisemeDecay(); };
|
|
266
324
|
log('HeadAudio ready (phoneme lip sync)');
|
|
267
325
|
} else {
|
|
268
326
|
log('HeadAudio skipped: AudioWorklet not supported in this WebView. Use sendViseme() from native TTS callbacks.');
|
|
327
|
+
head.opt.update = () => tickVisemeDecay();
|
|
269
328
|
}
|
|
270
329
|
} catch (err) {
|
|
271
330
|
log('HeadAudio unavailable, viseme/amplitude fallback active: ' + err.message);
|
|
331
|
+
head.opt.update = () => tickVisemeDecay();
|
|
272
332
|
}
|
|
273
333
|
|
|
274
334
|
startAudioInterception();
|
|
275
335
|
log('[ACC] init() complete, calling applyAccessories with ' + pendingAccessoriesList.length + ' pending items');
|
|
276
336
|
applyAccessories(pendingAccessoriesList);
|
|
337
|
+
emitLoading('ready', 100);
|
|
277
338
|
window.ReactNativeWebView?.postMessage(JSON.stringify({ type: 'ready' }));
|
|
278
339
|
}
|
|
279
340
|
} catch (err) {
|
|
@@ -323,34 +384,79 @@ let visemeMorphCache = null;
|
|
|
323
384
|
const visemeState = {};
|
|
324
385
|
|
|
325
386
|
const VISEME_MORPH_ALIASES = {
|
|
326
|
-
sil: ['viseme_sil', 'sil'],
|
|
327
|
-
PP: ['viseme_PP', 'pp', 'viseme_pp'],
|
|
328
|
-
FF: ['viseme_FF', 'ff', 'viseme_ff'],
|
|
329
|
-
TH: ['viseme_TH', 'th', 'viseme_th'],
|
|
330
|
-
DD: ['viseme_DD', 'dd', 'viseme_dd'],
|
|
331
|
-
kk: ['viseme_kk', 'kk', 'viseme_k'],
|
|
332
|
-
CH: ['viseme_CH', 'ch', 'viseme_ch'],
|
|
333
|
-
SS: ['viseme_SS', 'ss', '
|
|
334
|
-
nn: ['viseme_nn', 'nn', 'viseme_n'],
|
|
335
|
-
RR: ['viseme_RR', 'rr', 'viseme_r'],
|
|
387
|
+
sil: ['viseme_sil', 'sil', 'mouthClose', 'mouth_close'],
|
|
388
|
+
PP: ['viseme_PP', 'pp', 'viseme_pp', 'mouthPucker', 'mouth_pucker'],
|
|
389
|
+
FF: ['viseme_FF', 'ff', 'viseme_ff', 'mouthLowerLipIn', 'mouth_lower_lip_in', 'mouthRollLower', 'mouthShrugLower'],
|
|
390
|
+
TH: ['viseme_TH', 'th', 'viseme_th', 'tongueOut', 'tongue_out'],
|
|
391
|
+
DD: ['viseme_DD', 'dd', 'viseme_dd', 'mouthShrugUpper', 'mouth_shrug_upper'],
|
|
392
|
+
kk: ['viseme_kk', 'kk', 'viseme_k', 'mouthStretchLeft', 'mouth_stretch_left'],
|
|
393
|
+
CH: ['viseme_CH', 'ch', 'viseme_ch', 'mouthSmile', 'mouth_smile', 'mouthSmileLeft', 'mouth_smile_left'],
|
|
394
|
+
SS: ['viseme_SS', 'ss', 'viseme_ss', 'mouthStretchRight', 'mouth_stretch_right'],
|
|
395
|
+
nn: ['viseme_nn', 'nn', 'viseme_n', 'mouthDimpleLeft', 'mouth_dimple_left'],
|
|
396
|
+
RR: ['viseme_RR', 'rr', 'viseme_r', 'mouthDimpleRight', 'mouth_dimple_right'],
|
|
336
397
|
aa: ['viseme_aa', 'viseme_AA', 'aa', 'jawOpen', 'jaw_open', 'jawopen', 'mouthOpen', 'mouth_open', 'mouthopen'],
|
|
337
|
-
ee: ['viseme_ee', 'viseme_E', 'ee'],
|
|
338
|
-
ih: ['viseme_ih', 'viseme_I', 'ih'],
|
|
339
|
-
oh: ['viseme_oh', 'viseme_O', 'oh'],
|
|
340
|
-
ou: ['viseme_ou', 'viseme_U', 'ou'],
|
|
398
|
+
ee: ['viseme_ee', 'viseme_E', 'ee', 'mouthSmileLeft', 'mouth_smile_left'],
|
|
399
|
+
ih: ['viseme_ih', 'viseme_I', 'ih', 'mouthSmileRight', 'mouth_smile_right'],
|
|
400
|
+
oh: ['viseme_oh', 'viseme_O', 'oh', 'mouthFunnel', 'mouth_funnel'],
|
|
401
|
+
ou: ['viseme_ou', 'viseme_U', 'ou', 'mouthRollLower', 'mouth_roll_lower'],
|
|
402
|
+
};
|
|
403
|
+
|
|
404
|
+
// For ARKit models, each viseme may need multiple blend shapes driven together.
|
|
405
|
+
// Each entry is a list of morph names to combine for that viseme.
|
|
406
|
+
// The first alias list that has ANY match on the model is used.
|
|
407
|
+
const VISEME_COMPOUND_ARKIT = {
|
|
408
|
+
aa: [['jawOpen', 'mouthLowerDownLeft', 'mouthLowerDownRight'], ['jawOpen', 'mouthOpen']],
|
|
409
|
+
oh: [['mouthFunnel', 'jawOpen'], ['mouthFunnel']],
|
|
410
|
+
ou: [['mouthPucker', 'mouthRollLower'], ['mouthPucker']],
|
|
411
|
+
PP: [['mouthPucker', 'mouthClose'], ['mouthPucker']],
|
|
412
|
+
FF: [['mouthRollLower', 'mouthLowerDownLeft', 'mouthLowerDownRight'], ['mouthShrugLower']],
|
|
413
|
+
CH: [['mouthSmileLeft', 'mouthSmileRight', 'mouthStretchLeft', 'mouthStretchRight'], ['mouthSmileLeft', 'mouthSmileRight']],
|
|
414
|
+
ee: [['mouthSmileLeft', 'mouthSmileRight'], ['mouthSmileLeft']],
|
|
415
|
+
ih: [['mouthSmileLeft', 'mouthSmileRight'], ['mouthSmileRight']],
|
|
341
416
|
};
|
|
342
417
|
|
|
343
418
|
function buildVisemeMorphCache() {
|
|
344
419
|
visemeMorphCache = {};
|
|
345
420
|
for (const [visemeKey, aliases] of Object.entries(VISEME_MORPH_ALIASES)) {
|
|
346
421
|
const entries = [];
|
|
422
|
+
// Check if we have a compound ARKit mapping for this viseme
|
|
423
|
+
const compoundOptions = VISEME_COMPOUND_ARKIT[visemeKey];
|
|
424
|
+
if (compoundOptions) {
|
|
425
|
+
// Try each compound option; use the first one where all names exist on at least one mesh
|
|
426
|
+
let usedCompound = false;
|
|
427
|
+
for (const nameList of compoundOptions) {
|
|
428
|
+
// Collect entries for all names across all meshes
|
|
429
|
+
const compoundEntries = [];
|
|
430
|
+
for (const mesh of mouthMeshes) {
|
|
431
|
+
if (!mesh.morphTargetDictionary) continue;
|
|
432
|
+
const dict = mesh.morphTargetDictionary;
|
|
433
|
+
const dictKeysLower = Object.fromEntries(Object.keys(dict).map(k => [k.toLowerCase(), k]));
|
|
434
|
+
for (const name of nameList) {
|
|
435
|
+
const found = dictKeysLower[name.toLowerCase()];
|
|
436
|
+
if (found !== undefined) {
|
|
437
|
+
compoundEntries.push({ influences: mesh.morphTargetInfluences, idx: dict[found], morphName: found });
|
|
438
|
+
}
|
|
439
|
+
}
|
|
440
|
+
}
|
|
441
|
+
if (compoundEntries.length > 0) {
|
|
442
|
+
entries.push(...compoundEntries);
|
|
443
|
+
usedCompound = true;
|
|
444
|
+
break;
|
|
445
|
+
}
|
|
446
|
+
}
|
|
447
|
+
if (usedCompound) {
|
|
448
|
+
visemeMorphCache[visemeKey] = entries;
|
|
449
|
+
continue;
|
|
450
|
+
}
|
|
451
|
+
}
|
|
452
|
+
// Fallback: single-alias lookup
|
|
347
453
|
for (const mesh of mouthMeshes) {
|
|
348
454
|
if (!mesh.morphTargetDictionary) continue;
|
|
349
455
|
const dictKeys = Object.keys(mesh.morphTargetDictionary);
|
|
350
456
|
for (const alias of aliases) {
|
|
351
457
|
const found = dictKeys.find(k => k.toLowerCase() === alias.toLowerCase());
|
|
352
458
|
if (found !== undefined) {
|
|
353
|
-
entries.push({ influences: mesh.morphTargetInfluences, idx: mesh.morphTargetDictionary[found] });
|
|
459
|
+
entries.push({ influences: mesh.morphTargetInfluences, idx: mesh.morphTargetDictionary[found], morphName: found });
|
|
354
460
|
break;
|
|
355
461
|
}
|
|
356
462
|
}
|
|
@@ -359,6 +465,12 @@ function buildVisemeMorphCache() {
|
|
|
359
465
|
}
|
|
360
466
|
const found = Object.keys(visemeMorphCache);
|
|
361
467
|
log('Viseme cache: ' + (found.length > 0 ? found.join(', ') : 'none — check morph target names'));
|
|
468
|
+
|
|
469
|
+
// Always log all available morphs so we can see what the model actually has
|
|
470
|
+
if (mouthMeshes.length > 0) {
|
|
471
|
+
const allMorphs = Object.keys(mouthMeshes[0].morphTargetDictionary || {});
|
|
472
|
+
log('Available morphs: ' + (allMorphs.length > 0 ? allMorphs.join(', ') : 'none'));
|
|
473
|
+
}
|
|
362
474
|
}
|
|
363
475
|
|
|
364
476
|
function applyViseme(visemeKey, weight) {
|
|
@@ -368,39 +480,42 @@ function applyViseme(visemeKey, weight) {
|
|
|
368
480
|
return;
|
|
369
481
|
}
|
|
370
482
|
visemeState[visemeKey] = Math.min(1, weight);
|
|
483
|
+
visemeStateLastSet.set(visemeKey, Date.now());
|
|
371
484
|
}
|
|
372
485
|
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
486
|
+
const RHUBARB_DEFAULT_VISEME_WEIGHT = 0.72;
|
|
487
|
+
const RHUBARB_LABIAL_VISEME_WEIGHT = 0.85;
|
|
488
|
+
const RHUBARB_AA_VISEME_WEIGHT = 0.72;
|
|
489
|
+
const RHUBARB_ROUNDED_VISEME_WEIGHT = 0.62;
|
|
490
|
+
const RHUBARB_FALLBACK_AMPLITUDE_CAP = 0.72;
|
|
491
|
+
const RHUBARB_FALLBACK_AMPLITUDE_GAIN = 0.75;
|
|
492
|
+
const RHUBARB_VISEME_WEIGHTS = {
|
|
493
|
+
PP: RHUBARB_LABIAL_VISEME_WEIGHT,
|
|
494
|
+
FF: 0.78,
|
|
495
|
+
ee: 0.72,
|
|
496
|
+
ih: 0.68,
|
|
497
|
+
oh: RHUBARB_ROUNDED_VISEME_WEIGHT,
|
|
498
|
+
ou: 0.58,
|
|
499
|
+
aa: RHUBARB_AA_VISEME_WEIGHT,
|
|
500
|
+
};
|
|
387
501
|
|
|
388
502
|
const RHUBARB_TO_VISEME = {
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
503
|
+
A: 'aa',
|
|
504
|
+
B: 'PP',
|
|
505
|
+
C: 'ih',
|
|
506
|
+
D: 'FF',
|
|
507
|
+
E: 'ee',
|
|
508
|
+
F: 'oh',
|
|
509
|
+
G: 'ou',
|
|
510
|
+
H: 'nn',
|
|
511
|
+
X: 'sil',
|
|
398
512
|
};
|
|
399
513
|
|
|
400
514
|
let rhubarbMorphCache = null;
|
|
401
515
|
let visemeTimers = [];
|
|
402
516
|
let activeVisemeScheduleId = 0;
|
|
403
517
|
let visemeModeUntil = 0;
|
|
518
|
+
const visemeStateLastSet = new Map();
|
|
404
519
|
|
|
405
520
|
function buildRhubarbMorphCache() {
|
|
406
521
|
if (!visemeMorphCache) buildVisemeMorphCache();
|
|
@@ -419,7 +534,8 @@ function applyRhubarbCue(shape) {
|
|
|
419
534
|
if (shape === 'X' || !rhubarbMorphCache[shape]) return;
|
|
420
535
|
const visemeKey = RHUBARB_TO_VISEME[shape];
|
|
421
536
|
if (visemeKey && visemeKey !== 'sil') {
|
|
422
|
-
visemeState[visemeKey] =
|
|
537
|
+
visemeState[visemeKey] = RHUBARB_VISEME_WEIGHTS[visemeKey] || RHUBARB_DEFAULT_VISEME_WEIGHT;
|
|
538
|
+
visemeStateLastSet.set(visemeKey, Date.now());
|
|
423
539
|
}
|
|
424
540
|
}
|
|
425
541
|
|
|
@@ -432,20 +548,98 @@ function clearScheduledVisemes() {
|
|
|
432
548
|
for (const key of Object.keys(visemeState)) visemeState[key] = 0;
|
|
433
549
|
}
|
|
434
550
|
|
|
551
|
+
function tickVisemeDecay() {
|
|
552
|
+
if (!visemeMorphCache) return;
|
|
553
|
+
|
|
554
|
+
const isScheduled = Date.now() < visemeModeUntil;
|
|
555
|
+
const hasSpecificLipShape =
|
|
556
|
+
visemeState.PP > 0.05 ||
|
|
557
|
+
visemeState.FF > 0.05 ||
|
|
558
|
+
visemeState.kk > 0.05 ||
|
|
559
|
+
visemeState.ee > 0.05 ||
|
|
560
|
+
visemeState.ih > 0.05;
|
|
561
|
+
|
|
562
|
+
for (const [key, weight] of Object.entries(visemeState)) {
|
|
563
|
+
// Only decay if we aren't in the middle of a viseme schedule.
|
|
564
|
+
// Scheduled visemes are cleared manually by timeouts.
|
|
565
|
+
if (!isScheduled) {
|
|
566
|
+
const decayed = weight * 0.82;
|
|
567
|
+
visemeState[key] = decayed < 0.01 ? 0 : decayed;
|
|
568
|
+
}
|
|
569
|
+
|
|
570
|
+
const entries = visemeMorphCache[key];
|
|
571
|
+
if (!entries) continue;
|
|
572
|
+
|
|
573
|
+
let targetWeight = visemeState[key];
|
|
574
|
+
if (key === 'aa' && hasSpecificLipShape) targetWeight = Math.min(targetWeight, 0.45);
|
|
575
|
+
|
|
576
|
+
for (const e of entries) {
|
|
577
|
+
// When TalkingHead is active, write through its morph API so the internal
|
|
578
|
+
// render loop doesn't overwrite our values every frame.
|
|
579
|
+
// Use realtime (not newvalue) — newvalue is consumed and cleared after
|
|
580
|
+
// a single frame, so scheduled visemes would vanish immediately.
|
|
581
|
+
// realtime persists until explicitly set to null.
|
|
582
|
+
if (head?.mtAvatar && e.morphName && head.mtAvatar[e.morphName]) {
|
|
583
|
+
const mt = head.mtAvatar[e.morphName];
|
|
584
|
+
mt.realtime = targetWeight > 0 ? targetWeight : null;
|
|
585
|
+
mt.needsUpdate = true;
|
|
586
|
+
} else {
|
|
587
|
+
e.influences[e.idx] = targetWeight;
|
|
588
|
+
}
|
|
589
|
+
}
|
|
590
|
+
}
|
|
591
|
+
}
|
|
592
|
+
|
|
435
593
|
function scheduleVisemes(schedule) {
|
|
436
594
|
clearScheduledVisemes();
|
|
595
|
+
|
|
596
|
+
// Prune visemeState keys that haven't been written in the last 2 seconds to
|
|
597
|
+
// prevent unbounded accumulation across many utterances.
|
|
598
|
+
const staleThreshold = Date.now() - 2000;
|
|
599
|
+
for (const key of Object.keys(visemeState)) {
|
|
600
|
+
if ((visemeStateLastSet.get(key) ?? 0) < staleThreshold) {
|
|
601
|
+
delete visemeState[key];
|
|
602
|
+
visemeStateLastSet.delete(key);
|
|
603
|
+
}
|
|
604
|
+
}
|
|
605
|
+
|
|
437
606
|
if (!schedule || !Array.isArray(schedule.cues) || schedule.cues.length === 0) return;
|
|
438
607
|
|
|
439
608
|
const myScheduleId = activeVisemeScheduleId;
|
|
440
|
-
|
|
609
|
+
// The startedAtMs anchor is set when tts_request_start arrives on the data
|
|
610
|
+
// channel. Audio doesn't play until ~300ms later (LiveKit audio buffering).
|
|
611
|
+
// TTS generation delay is no longer included here since visemes now arrive
|
|
612
|
+
// via direct ref call before the React render cycle.
|
|
613
|
+
const AUDIO_PIPELINE_DELAY_MS = 300;
|
|
614
|
+
let startedAt = (schedule.startedAtMs || Date.now()) + AUDIO_PIPELINE_DELAY_MS;
|
|
441
615
|
const durationMs = schedule.durationMs || 0;
|
|
616
|
+
const now = Date.now();
|
|
617
|
+
let elapsedMs = Math.max(0, now - startedAt);
|
|
618
|
+
|
|
619
|
+
// If the schedule still arrives late after the pipeline offset, shift further
|
|
620
|
+
if (elapsedMs > 300 && schedule.cues.length > 3) {
|
|
621
|
+
const shift = Math.min(elapsedMs - 50, 500);
|
|
622
|
+
startedAt += shift;
|
|
623
|
+
elapsedMs -= shift;
|
|
624
|
+
log('Viseme schedule arrived late, shifting anchor forward by ' + shift + 'ms');
|
|
625
|
+
}
|
|
442
626
|
|
|
443
|
-
|
|
444
|
-
|
|
627
|
+
const remainingMs = Math.max(0, durationMs - elapsedMs);
|
|
628
|
+
let scheduledCueCount = 0;
|
|
629
|
+
let skippedCueCount = 0;
|
|
630
|
+
|
|
631
|
+
// Gate amplitude fallback for the locally remaining duration plus a small buffer.
|
|
632
|
+
// If the schedule arrives a bit late, keep amplitude out of the way for the rest
|
|
633
|
+
// of the utterance instead of expiring immediately from the original timestamp.
|
|
634
|
+
visemeModeUntil = now + remainingMs + 200;
|
|
445
635
|
|
|
446
636
|
for (const cue of schedule.cues) {
|
|
447
637
|
const delay = cue.startMs - (Date.now() - startedAt);
|
|
448
|
-
if (delay < -50)
|
|
638
|
+
if (delay < -50) {
|
|
639
|
+
skippedCueCount++;
|
|
640
|
+
continue; // already in the past, skip
|
|
641
|
+
}
|
|
642
|
+
scheduledCueCount++;
|
|
449
643
|
|
|
450
644
|
const applyId = setTimeout(() => {
|
|
451
645
|
if (activeVisemeScheduleId !== myScheduleId) return;
|
|
@@ -461,6 +655,16 @@ function scheduleVisemes(schedule) {
|
|
|
461
655
|
visemeTimers.push(applyId, clearId);
|
|
462
656
|
}
|
|
463
657
|
|
|
658
|
+
log(
|
|
659
|
+
'Viseme schedule received: requestId=' +
|
|
660
|
+
(schedule.requestId || 'unknown') +
|
|
661
|
+
' cues=' + schedule.cues.length +
|
|
662
|
+
' scheduled=' + scheduledCueCount +
|
|
663
|
+
' skipped=' + skippedCueCount +
|
|
664
|
+
' elapsedMs=' + elapsedMs +
|
|
665
|
+
' remainingMs=' + remainingMs,
|
|
666
|
+
);
|
|
667
|
+
|
|
464
668
|
// Ensure silence at schedule end
|
|
465
669
|
const endDelay = durationMs - (Date.now() - startedAt);
|
|
466
670
|
if (endDelay > 0) {
|
|
@@ -630,7 +834,10 @@ function onIncomingMessage(event) {
|
|
|
630
834
|
if (jawKey !== undefined) jawMorphCache.push({ influences: mesh.morphTargetInfluences, idx: mesh.morphTargetDictionary[jawKey] });
|
|
631
835
|
}
|
|
632
836
|
}
|
|
633
|
-
|
|
837
|
+
const val = Math.min(
|
|
838
|
+
RHUBARB_FALLBACK_AMPLITUDE_CAP,
|
|
839
|
+
msg.value * RHUBARB_FALLBACK_AMPLITUDE_GAIN,
|
|
840
|
+
);
|
|
634
841
|
amplitudeDecay = Math.max(amplitudeDecay * 0.7, val);
|
|
635
842
|
for (let i = 0; i < jawMorphCache.length; i++) jawMorphCache[i].influences[jawMorphCache[i].idx] = amplitudeDecay;
|
|
636
843
|
} else if (msg.type === 'viseme') {
|
|
@@ -640,8 +847,7 @@ function onIncomingMessage(event) {
|
|
|
640
847
|
} else if (msg.type === 'clear_visemes') {
|
|
641
848
|
clearScheduledVisemes();
|
|
642
849
|
} else if (msg.type === 'mood' && head) {
|
|
643
|
-
|
|
644
|
-
head.setMood(moodMap[msg.value] || 'neutral');
|
|
850
|
+
head.setMood(MOOD_MAP[msg.value] || 'neutral');
|
|
645
851
|
} else if (msg.type === 'hair_color') {
|
|
646
852
|
HAIR_COLOR = msg.value; applyColorOverrides();
|
|
647
853
|
} else if (msg.type === 'skin_color') {
|
|
@@ -659,7 +865,8 @@ function onIncomingMessage(event) {
|
|
|
659
865
|
window.addEventListener('message', onIncomingMessage);
|
|
660
866
|
document.addEventListener('message', onIncomingMessage);
|
|
661
867
|
|
|
662
|
-
init();
|
|
868
|
+
await init();
|
|
869
|
+
})();
|
|
663
870
|
</script>
|
|
664
871
|
</body>
|
|
665
872
|
</html>
|
package/dist/index.d.ts
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
export type { TalkingHeadAccessory, TalkingHeadMood, TalkingHeadProps, TalkingHeadRef, TalkingHeadViseme, TalkingHeadVisemeCue, TalkingHeadVisemeSchedule, } from './TalkingHead';
|
|
1
|
+
export type { TalkingHeadAccessory, TalkingHeadLoadingState, TalkingHeadLoadingStage, TalkingHeadMood, TalkingHeadProps, TalkingHeadRef, TalkingHeadViseme, TalkingHeadVisemeCue, TalkingHeadVisemeSchedule, } from './TalkingHead';
|
|
2
2
|
export { TalkingHead } from './TalkingHead';
|
|
3
3
|
export * from './appearance';
|
package/dist/index.web.d.ts
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
export type { TalkingHeadAccessory, TalkingHeadMood, TalkingHeadProps, TalkingHeadRef, } from './TalkingHead.web';
|
|
1
|
+
export type { TalkingHeadAccessory, TalkingHeadLoadingState, TalkingHeadMood, TalkingHeadProps, TalkingHeadRef, } from './TalkingHead.web';
|
|
2
2
|
export { TalkingHead } from './TalkingHead.web';
|
|
3
3
|
export * from './appearance';
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "talking-head-studio",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.9",
|
|
4
4
|
"description": "Cross-platform 3D avatar component for React Native & web — lip-sync, gestures, accessories, and LLM integration. Powered by TalkingHead + Three.js.",
|
|
5
5
|
"main": "dist/index.web.js",
|
|
6
6
|
"browser": "dist/index.web.js",
|
|
@@ -79,11 +79,11 @@
|
|
|
79
79
|
},
|
|
80
80
|
"sideEffects": false,
|
|
81
81
|
"peerDependencies": {
|
|
82
|
+
"@react-three/drei": ">=9",
|
|
83
|
+
"@react-three/fiber": ">=8",
|
|
82
84
|
"react": ">=18",
|
|
83
85
|
"react-native": ">=0.73",
|
|
84
86
|
"react-native-webview": ">=13",
|
|
85
|
-
"@react-three/fiber": ">=8",
|
|
86
|
-
"@react-three/drei": ">=9",
|
|
87
87
|
"three": ">=0.170"
|
|
88
88
|
},
|
|
89
89
|
"peerDependenciesMeta": {
|
|
@@ -126,6 +126,7 @@
|
|
|
126
126
|
"metro-react-native-babel-preset": "^0.77.0",
|
|
127
127
|
"multer": "^2.1.0",
|
|
128
128
|
"prettier": "^3.8.1",
|
|
129
|
+
"react-native-webview": "^13.16.0",
|
|
129
130
|
"react-test-renderer": "^19.2.4",
|
|
130
131
|
"ts-jest": "^29.4.6"
|
|
131
132
|
}
|