talking-head-studio 0.4.6 → 0.4.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -7
- package/dist/TalkingHead.web.js +18 -8
- package/dist/TalkingHeadVisualization.js +9 -22
- package/dist/editor/FaceSqueezeEditor.d.ts +17 -0
- package/dist/editor/FaceSqueezeEditor.js +553 -0
- package/dist/editor/FaceSqueezeEditor.web.d.ts +6 -0
- package/dist/editor/FaceSqueezeEditor.web.js +54 -0
- package/dist/editor/index.d.ts +2 -1
- package/dist/editor/index.js +4 -1
- package/dist/editor/types.d.ts +3 -0
- package/dist/html.d.ts +5 -0
- package/dist/html.js +29 -26
- package/dist/tts/useDirectVisemeStream.js +1 -1
- package/dist/wgpu/WgpuAvatar.js +6 -1
- package/package.json +5 -1
package/README.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# talking-head-studio
|
|
2
2
|
|
|
3
|
-
**
|
|
3
|
+
**The missing UI layer for AI Agents. Drop-in, lip-syncing 3D avatars for Web, React, and React Native.**
|
|
4
4
|
|
|
5
5
|
[](https://www.npmjs.com/package/talking-head-studio)
|
|
6
6
|
[](https://opensource.org/licenses/MIT)
|
|
@@ -10,10 +10,11 @@
|
|
|
10
10
|
|
|
11
11
|
## Why this?
|
|
12
12
|
|
|
13
|
-
- **
|
|
14
|
-
- **Bring any GLB
|
|
15
|
-
- **Built for
|
|
16
|
-
- **
|
|
13
|
+
- **Zero-Jank React Native & Web:** True cross-platform rendering. React Native gets a blazing fast wgpu-accelerated native render loop, skipping WebView bridge latency entirely. React on web gets a robust `react-three-fiber` setup. Same API, same props.
|
|
14
|
+
- **Universal GLB Compatibility:** Bring any GLB. Out-of-the-box support for standard ARKit blendshapes. Rigged models get full phoneme-based lip-sync. Non-rigged models get an amplitude-driven jaw animation fallback.
|
|
15
|
+
- **Built for AI & Voice Pipelines:** Wire `sendAmplitude` or visemes directly to LiveKit, Web Audio, ElevenLabs, OpenAI Realtime, or any audio source.
|
|
16
|
+
- **Always Alive:** Procedural idle animations (breathing, nodding, swaying) keep your avatar from feeling like a static doll.
|
|
17
|
+
- **Dynamic Wardrobe & Accessories:** Swap hair, skin, and eye colors on the fly. Attach hats, glasses, or backpacks to any bone at runtime.
|
|
17
18
|
|
|
18
19
|
---
|
|
19
20
|
|
|
@@ -68,7 +69,7 @@ export default function Avatar() {
|
|
|
68
69
|
return (
|
|
69
70
|
<TalkingHead
|
|
70
71
|
ref={ref}
|
|
71
|
-
avatarUrl="https://
|
|
72
|
+
avatarUrl="https://example.com/your-model.glb"
|
|
72
73
|
mood="happy"
|
|
73
74
|
cameraView="upper"
|
|
74
75
|
hairColor="#1a1a2e"
|
|
@@ -378,7 +379,7 @@ For the complete experience -- phoneme lip-sync, expressions, moods, gestures --
|
|
|
378
379
|
- **ARKit blend shapes** and/or **Oculus viseme blend shapes** for lip-sync
|
|
379
380
|
- Standard Three.js-compatible GLB format
|
|
380
381
|
|
|
381
|
-
Models from [
|
|
382
|
+
Models from [Avaturn](https://avaturn.me/) or any Mixamo-rigged source work out of the box.
|
|
382
383
|
|
|
383
384
|
### Non-rigged models (static fallback)
|
|
384
385
|
|
|
@@ -457,4 +458,11 @@ This project builds on excellent open-source work:
|
|
|
457
458
|
|
|
458
459
|
## License
|
|
459
460
|
|
|
461
|
+
MIT
|
|
462
|
+
at runtime.
|
|
463
|
+
|
|
464
|
+
---
|
|
465
|
+
|
|
466
|
+
## License
|
|
467
|
+
|
|
460
468
|
MIT
|
package/dist/TalkingHead.web.js
CHANGED
|
@@ -46,14 +46,24 @@ exports.TalkingHead = (0, react_1.forwardRef)(({ avatarUrl, authToken, mood = 'n
|
|
|
46
46
|
const pendingSkinColorRef = (0, react_1.useRef)(skinColor);
|
|
47
47
|
const pendingEyeColorRef = (0, react_1.useRef)(eyeColor);
|
|
48
48
|
const accessoriesRef = (0, react_1.useRef)(accessories);
|
|
49
|
+
const onLoadingChangeRef = (0, react_1.useRef)(onLoadingChange);
|
|
50
|
+
const onReadyRef = (0, react_1.useRef)(onReady);
|
|
51
|
+
const onErrorRef = (0, react_1.useRef)(onError);
|
|
52
|
+
const onAvatarStateRef = (0, react_1.useRef)(onAvatarState);
|
|
53
|
+
(0, react_1.useEffect)(() => {
|
|
54
|
+
onLoadingChangeRef.current = onLoadingChange;
|
|
55
|
+
onReadyRef.current = onReady;
|
|
56
|
+
onErrorRef.current = onError;
|
|
57
|
+
onAvatarStateRef.current = onAvatarState;
|
|
58
|
+
});
|
|
49
59
|
(0, react_1.useEffect)(() => {
|
|
50
60
|
accessoriesRef.current = accessories;
|
|
51
61
|
}, [accessories]);
|
|
52
62
|
(0, react_1.useEffect)(() => {
|
|
53
63
|
if (!avatarUrl)
|
|
54
64
|
return;
|
|
55
|
-
|
|
56
|
-
}, [avatarUrl, authToken
|
|
65
|
+
onLoadingChangeRef.current?.({ stage: 'booting', progress: null });
|
|
66
|
+
}, [avatarUrl, authToken]);
|
|
57
67
|
const post = (0, react_1.useCallback)((msg) => {
|
|
58
68
|
iframeRef.current?.contentWindow?.postMessage(JSON.stringify(msg), '*');
|
|
59
69
|
}, []);
|
|
@@ -148,7 +158,7 @@ exports.TalkingHead = (0, react_1.forwardRef)(({ avatarUrl, authToken, mood = 'n
|
|
|
148
158
|
try {
|
|
149
159
|
const msg = typeof event.data === 'string' ? JSON.parse(event.data) : event.data;
|
|
150
160
|
if (msg.type === 'loading' && typeof msg.stage === 'string') {
|
|
151
|
-
|
|
161
|
+
onLoadingChangeRef.current?.({
|
|
152
162
|
stage: msg.stage,
|
|
153
163
|
progress: typeof msg.progress === 'number' && Number.isFinite(msg.progress)
|
|
154
164
|
? msg.progress
|
|
@@ -158,7 +168,7 @@ exports.TalkingHead = (0, react_1.forwardRef)(({ avatarUrl, authToken, mood = 'n
|
|
|
158
168
|
}
|
|
159
169
|
if (msg.type === 'ready') {
|
|
160
170
|
readyRef.current = true;
|
|
161
|
-
|
|
171
|
+
onLoadingChangeRef.current?.({ stage: 'ready', progress: 100 });
|
|
162
172
|
if (pendingMoodRef.current) {
|
|
163
173
|
post({ type: 'mood', value: pendingMoodRef.current });
|
|
164
174
|
}
|
|
@@ -174,13 +184,13 @@ exports.TalkingHead = (0, react_1.forwardRef)(({ avatarUrl, authToken, mood = 'n
|
|
|
174
184
|
if (accessoriesRef.current?.length) {
|
|
175
185
|
post({ type: 'set_accessories', accessories: accessoriesRef.current });
|
|
176
186
|
}
|
|
177
|
-
|
|
187
|
+
onReadyRef.current?.();
|
|
178
188
|
}
|
|
179
189
|
else if (msg.type === 'error') {
|
|
180
|
-
|
|
190
|
+
onErrorRef.current?.(msg.message);
|
|
181
191
|
}
|
|
182
192
|
else if (msg.type === 'avatarState') {
|
|
183
|
-
|
|
193
|
+
onAvatarStateRef.current?.(msg.state);
|
|
184
194
|
}
|
|
185
195
|
else if (msg.type === 'log') {
|
|
186
196
|
console.log('[TalkingHead]', msg.message);
|
|
@@ -192,7 +202,7 @@ exports.TalkingHead = (0, react_1.forwardRef)(({ avatarUrl, authToken, mood = 'n
|
|
|
192
202
|
};
|
|
193
203
|
window.addEventListener('message', onMessage);
|
|
194
204
|
return () => window.removeEventListener('message', onMessage);
|
|
195
|
-
}, [
|
|
205
|
+
}, [post]);
|
|
196
206
|
return ((0, jsx_runtime_1.jsx)("div", { style: { ...containerStyle, ...style }, children: (0, jsx_runtime_1.jsx)("iframe", { ref: iframeRef, srcDoc: srcdoc, style: iframeStyle, sandbox: "allow-scripts allow-same-origin", title: "TalkingHead Avatar" }) }));
|
|
197
207
|
});
|
|
198
208
|
exports.TalkingHead.displayName = 'TalkingHead';
|
|
@@ -46,8 +46,10 @@ function getLoadingLabel(stage) {
|
|
|
46
46
|
*/
|
|
47
47
|
exports.TalkingHeadVisualization = (0, react_1.forwardRef)(({ style, avatarUrl, authToken, cameraView = 'head', cameraDistance = 0.2, accessories, mood: initialMood = 'neutral', aspect, focalLength, visemeSchedule, onVisemeScheduleApplied, onVoiceMood, vendorBaseUrl }, ref) => {
|
|
48
48
|
const avatarRef = (0, react_1.useRef)(null);
|
|
49
|
-
// On native, WgpuAvatar ref is
|
|
50
|
-
//
|
|
49
|
+
// On native, WgpuAvatar ref is stored here so scheduleVisemes / sendAmplitude can route to it.
|
|
50
|
+
// useImperativeHandle is the sole owner of the forwarded ref on all platforms —
|
|
51
|
+
// do NOT manually forward ref inside the WgpuAvatar callback ref, as that bypasses
|
|
52
|
+
// the dedup key check in scheduleVisemes and causes double-fire.
|
|
51
53
|
const wgpuRef = (0, react_1.useRef)(null);
|
|
52
54
|
// Unified accessor — WgpuAvatar on native, WebView on web
|
|
53
55
|
const activeAvatar = (0, react_1.useCallback)(() => (wgpuRef.current ?? avatarRef.current), []);
|
|
@@ -183,26 +185,15 @@ exports.TalkingHeadVisualization = (0, react_1.forwardRef)(({ style, avatarUrl,
|
|
|
183
185
|
lastScheduledVisemeKeyRef.current = scheduleKey;
|
|
184
186
|
}, [isAvatarReady, onVisemeScheduleApplied, visemeSchedule, activeAvatar]);
|
|
185
187
|
// On native use WgpuAvatar — direct morph writes, no WebView bridge.
|
|
188
|
+
// NOTE: Only store wgpuRef here — do NOT forward `ref` manually. useImperativeHandle
|
|
189
|
+
// above is the sole owner of the forwarded ref and handles the dedup key check.
|
|
186
190
|
if (react_native_1.Platform.OS !== 'web') {
|
|
187
|
-
return ((0, jsx_runtime_1.jsx)(WgpuAvatar_1.WgpuAvatar, { focalLength: focalLength, ref: (fr) => {
|
|
188
|
-
wgpuRef.current = fr;
|
|
189
|
-
if (typeof ref === 'function')
|
|
190
|
-
ref(fr);
|
|
191
|
-
else if (ref)
|
|
192
|
-
ref.current = fr;
|
|
193
|
-
if (fr && pendingVisemeScheduleRef.current) {
|
|
194
|
-
fr.scheduleVisemes(pendingVisemeScheduleRef.current);
|
|
195
|
-
pendingVisemeScheduleRef.current = null;
|
|
196
|
-
}
|
|
197
|
-
}, style: style, avatarUrl: avatarUrl ?? null, aspect: aspect, mood: mood, accessories: accessories, onReady: handleReady, onError: handleError }));
|
|
191
|
+
return ((0, jsx_runtime_1.jsx)(WgpuAvatar_1.WgpuAvatar, { focalLength: focalLength, ref: (fr) => { wgpuRef.current = fr; }, style: style, avatarUrl: avatarUrl ?? null, aspect: aspect, mood: mood, accessories: accessories, onReady: handleReady, onError: handleError }));
|
|
198
192
|
}
|
|
199
193
|
if (!effectiveAvatarUrl) {
|
|
200
194
|
return ((0, jsx_runtime_1.jsx)(react_native_1.View, { style: [style, styles.placeholder] }));
|
|
201
195
|
}
|
|
202
|
-
return ((0, jsx_runtime_1.jsxs)(react_native_1.View, { style: [style, styles.container], pointerEvents: "box-none", children: [(0, jsx_runtime_1.jsx)(TalkingHead_1.TalkingHead, { ref: avatarRef, avatarUrl: effectiveAvatarUrl, authToken: authToken, cameraView: cameraView, cameraDistance: cameraDistance, accessories: accessories, mood: mood, onLoadingChange: handleLoadingChange, onReady: handleReady, onError: handleError, onVoiceMood: onVoiceMood, style: react_native_1.StyleSheet.absoluteFill, vendorBaseUrl: vendorBaseUrl }), !isAvatarReady && ((0, jsx_runtime_1.jsx)(react_native_1.View, { testID: "talking-head-loading", style: styles.loadingOverlay, pointerEvents: "none", children: (0, jsx_runtime_1.jsxs)(react_native_1.View, { style: styles.loadingCard, children: [(0, jsx_runtime_1.jsx)(react_native_1.View, { style: styles.loadingBadge, children: (0, jsx_runtime_1.jsx)(react_native_1.Text, { style: styles.loadingBadgeText, children: "AVATAR" }) }), (0, jsx_runtime_1.jsx)(react_native_1.Text, { style: styles.loadingTitle, children: avatarError ? 'Avatar failed to load' : getLoadingLabel(loadingState.stage) }), avatarError ? ((0, jsx_runtime_1.jsx)(react_native_1.Text, { style: styles.loadingHint, children: avatarError })) : typeof loadingState.progress === 'number' ? ((0, jsx_runtime_1.jsxs)(jsx_runtime_1.Fragment, { children: [(0, jsx_runtime_1.jsxs)(react_native_1.Text, { style: styles.loadingPercent, children: [loadingState.progress, "%"] }), (0, jsx_runtime_1.
|
|
203
|
-
styles.progressFill,
|
|
204
|
-
{ width: `${Math.max(6, loadingState.progress)}%` },
|
|
205
|
-
] }) })] })) : ((0, jsx_runtime_1.jsx)(react_native_1.Text, { style: styles.loadingHint, children: "Preparing the avatar scene\u2026" }))] }) }))] }));
|
|
196
|
+
return ((0, jsx_runtime_1.jsxs)(react_native_1.View, { style: [style, styles.container], pointerEvents: "box-none", children: [(0, jsx_runtime_1.jsx)(TalkingHead_1.TalkingHead, { ref: avatarRef, avatarUrl: effectiveAvatarUrl, authToken: authToken, cameraView: cameraView, cameraDistance: cameraDistance, accessories: accessories, mood: mood, onLoadingChange: handleLoadingChange, onReady: handleReady, onError: handleError, onVoiceMood: onVoiceMood, style: react_native_1.StyleSheet.absoluteFill, vendorBaseUrl: vendorBaseUrl }), !isAvatarReady && ((0, jsx_runtime_1.jsx)(react_native_1.View, { testID: "talking-head-loading", style: styles.loadingOverlay, pointerEvents: "none", children: (0, jsx_runtime_1.jsxs)(react_native_1.View, { style: styles.loadingCard, children: [(0, jsx_runtime_1.jsx)(react_native_1.View, { style: styles.loadingBadge, children: (0, jsx_runtime_1.jsx)(react_native_1.Text, { style: styles.loadingBadgeText, children: "AVATAR" }) }), (0, jsx_runtime_1.jsx)(react_native_1.Text, { style: styles.loadingTitle, children: avatarError ? 'Avatar failed to load' : getLoadingLabel(loadingState.stage) }), avatarError ? ((0, jsx_runtime_1.jsx)(react_native_1.Text, { style: styles.loadingHint, children: avatarError })) : typeof loadingState.progress === 'number' ? ((0, jsx_runtime_1.jsxs)(jsx_runtime_1.Fragment, { children: [(0, jsx_runtime_1.jsxs)(react_native_1.Text, { style: styles.loadingPercent, children: [loadingState.progress, "%"] }), (0, jsx_runtime_1.jsxs)(react_native_1.View, { style: styles.progressTrack, children: [(0, jsx_runtime_1.jsx)(react_native_1.View, { style: { flex: Math.max(6, loadingState.progress ?? 0), height: '100%', borderRadius: 999, backgroundColor: '#5eead4' } }), (0, jsx_runtime_1.jsx)(react_native_1.View, { style: { flex: 100 - Math.max(6, loadingState.progress ?? 0) } })] })] })) : ((0, jsx_runtime_1.jsx)(react_native_1.Text, { style: styles.loadingHint, children: "Preparing the avatar scene\u2026" }))] }) }))] }));
|
|
206
197
|
});
|
|
207
198
|
exports.TalkingHeadVisualization.displayName = 'TalkingHeadVisualization';
|
|
208
199
|
const styles = react_native_1.StyleSheet.create({
|
|
@@ -264,11 +255,7 @@ const styles = react_native_1.StyleSheet.create({
|
|
|
264
255
|
backgroundColor: 'rgba(148, 163, 184, 0.18)',
|
|
265
256
|
marginTop: 12,
|
|
266
257
|
overflow: 'hidden',
|
|
267
|
-
|
|
268
|
-
progressFill: {
|
|
269
|
-
height: '100%',
|
|
270
|
-
borderRadius: 999,
|
|
271
|
-
backgroundColor: '#5eead4',
|
|
258
|
+
flexDirection: 'row',
|
|
272
259
|
},
|
|
273
260
|
placeholder: {
|
|
274
261
|
backgroundColor: 'rgba(30,34,53,0.5)',
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* WgpuFaceSqueezeEditor — R3F/Three.js port of FaceSqueezeEditor.
|
|
3
|
+
*
|
|
4
|
+
* Same gesture zones, spring physics, idle emotions, lip sync, and sound
|
|
5
|
+
* as the Filament version. Only the render scaffolding changes:
|
|
6
|
+
*
|
|
7
|
+
* FilamentScene + FilamentView + renderableManager.setMorphWeights
|
|
8
|
+
* → @react-three/fiber Canvas + mesh.morphTargetInfluences[]
|
|
9
|
+
*
|
|
10
|
+
* Drop-in replacement: same FaceSqueezeEditorProps interface.
|
|
11
|
+
* The .web.tsx stub is shared — import from FaceSqueezeEditor.web.tsx.
|
|
12
|
+
*/
|
|
13
|
+
export declare const FACE_SQUEEZE_LOCAL_MODULE: number;
|
|
14
|
+
export interface FaceSqueezeEditorProps {
|
|
15
|
+
onClose: () => void;
|
|
16
|
+
}
|
|
17
|
+
export declare function FaceSqueezeEditor({ onClose }: FaceSqueezeEditorProps): import("react/jsx-runtime").JSX.Element;
|
|
@@ -0,0 +1,553 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || function (mod) {
|
|
19
|
+
if (mod && mod.__esModule) return mod;
|
|
20
|
+
var result = {};
|
|
21
|
+
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
22
|
+
__setModuleDefault(result, mod);
|
|
23
|
+
return result;
|
|
24
|
+
};
|
|
25
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
26
|
+
exports.FaceSqueezeEditor = exports.FACE_SQUEEZE_LOCAL_MODULE = void 0;
|
|
27
|
+
const jsx_runtime_1 = require("react/jsx-runtime");
|
|
28
|
+
/**
|
|
29
|
+
* WgpuFaceSqueezeEditor — R3F/Three.js port of FaceSqueezeEditor.
|
|
30
|
+
*
|
|
31
|
+
* Same gesture zones, spring physics, idle emotions, lip sync, and sound
|
|
32
|
+
* as the Filament version. Only the render scaffolding changes:
|
|
33
|
+
*
|
|
34
|
+
* FilamentScene + FilamentView + renderableManager.setMorphWeights
|
|
35
|
+
* → @react-three/fiber Canvas + mesh.morphTargetInfluences[]
|
|
36
|
+
*
|
|
37
|
+
* Drop-in replacement: same FaceSqueezeEditorProps interface.
|
|
38
|
+
* The .web.tsx stub is shared — import from FaceSqueezeEditor.web.tsx.
|
|
39
|
+
*/
|
|
40
|
+
const react_1 = require("react");
|
|
41
|
+
const react_native_1 = require("react-native");
|
|
42
|
+
const native_1 = require("@react-three/fiber/native");
|
|
43
|
+
const native_2 = require("@react-three/drei/native");
|
|
44
|
+
const THREE = __importStar(require("three"));
|
|
45
|
+
const react_native_gesture_handler_1 = require("react-native-gesture-handler");
|
|
46
|
+
const react_native_reanimated_1 = require("react-native-reanimated");
|
|
47
|
+
const Haptics = __importStar(require("expo-haptics"));
|
|
48
|
+
const expo_audio_1 = require("expo-audio");
|
|
49
|
+
const expo_asset_1 = require("expo-asset");
|
|
50
|
+
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
|
51
|
+
exports.FACE_SQUEEZE_LOCAL_MODULE = require('../assets/face-squeeze-local.glb');
|
|
52
|
+
// ---------------------------------------------------------------------------
|
|
53
|
+
// Constants (identical to Filament version)
|
|
54
|
+
// ---------------------------------------------------------------------------
|
|
55
|
+
const SPRING_DURATION_MS = 300;
|
|
56
|
+
const EMOTION_HOLD_MS = 900;
|
|
57
|
+
const IDLE_EMOTION_INTERVAL_MS = 30000;
|
|
58
|
+
const INITIAL_IDLE_EMOTION_DELAY_MS = 1200;
|
|
59
|
+
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
|
60
|
+
const SOUND_OWIE = require('./sounds/owie.wav');
|
|
61
|
+
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
|
62
|
+
const SOUND_STOP = require('./sounds/stop.wav');
|
|
63
|
+
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
|
64
|
+
const SOUND_HAHA = require('./sounds/haha.wav');
|
|
65
|
+
const REACTION_SOUNDS = [
|
|
66
|
+
SOUND_OWIE,
|
|
67
|
+
SOUND_STOP,
|
|
68
|
+
SOUND_HAHA,
|
|
69
|
+
];
|
|
70
|
+
const EMOTION_PRESETS = {
|
|
71
|
+
owie: {
|
|
72
|
+
eyeBlinkLeft: 1.0, eyeBlinkRight: 1.0,
|
|
73
|
+
eyeSquintLeft: 0.9, eyeSquintRight: 0.9,
|
|
74
|
+
browDownLeft: 0.8, browDownRight: 0.8,
|
|
75
|
+
browInnerUp: 0.6,
|
|
76
|
+
noseSneerLeft: 0.7, noseSneerRight: 0.7,
|
|
77
|
+
jawOpen: 0.35,
|
|
78
|
+
mouthStretchLeft: 0.6, mouthStretchRight: 0.6,
|
|
79
|
+
},
|
|
80
|
+
happy: {
|
|
81
|
+
mouthSmileLeft: 0.85, mouthSmileRight: 0.85,
|
|
82
|
+
cheekSquintLeft: 0.6, cheekSquintRight: 0.6,
|
|
83
|
+
mouthDimpleLeft: 0.4, mouthDimpleRight: 0.4,
|
|
84
|
+
},
|
|
85
|
+
surprised: {
|
|
86
|
+
eyeWideLeft: 0.9, eyeWideRight: 0.9,
|
|
87
|
+
browInnerUp: 0.8,
|
|
88
|
+
browOuterUpLeft: 0.7, browOuterUpRight: 0.7,
|
|
89
|
+
jawOpen: 0.4, mouthFunnel: 0.3,
|
|
90
|
+
},
|
|
91
|
+
angry: {
|
|
92
|
+
browDownLeft: 0.9, browDownRight: 0.9,
|
|
93
|
+
noseSneerLeft: 0.7, noseSneerRight: 0.7,
|
|
94
|
+
eyeSquintLeft: 0.6, eyeSquintRight: 0.6,
|
|
95
|
+
mouthPressLeft: 0.5, mouthPressRight: 0.5,
|
|
96
|
+
},
|
|
97
|
+
smirk: {
|
|
98
|
+
mouthSmileLeft: 0.9, mouthSmileRight: 0.1,
|
|
99
|
+
cheekSquintLeft: 0.5, mouthDimpleLeft: 0.35,
|
|
100
|
+
},
|
|
101
|
+
sad: {
|
|
102
|
+
browInnerUp: 0.9,
|
|
103
|
+
browDownLeft: 0.3, browDownRight: 0.3,
|
|
104
|
+
mouthFrownLeft: 0.8, mouthFrownRight: 0.8,
|
|
105
|
+
mouthPressLeft: 0.3, mouthPressRight: 0.3,
|
|
106
|
+
eyeLookDownLeft: 0.5, eyeLookDownRight: 0.5,
|
|
107
|
+
},
|
|
108
|
+
disgust: {
|
|
109
|
+
noseSneerLeft: 0.9, noseSneerRight: 0.9,
|
|
110
|
+
browDownLeft: 0.7, browDownRight: 0.7,
|
|
111
|
+
mouthShrugUpper: 0.6, mouthShrugLower: 0.4,
|
|
112
|
+
mouthLeft: 0.3,
|
|
113
|
+
eyeSquintLeft: 0.4, eyeSquintRight: 0.4,
|
|
114
|
+
},
|
|
115
|
+
wink: {
|
|
116
|
+
eyeBlinkLeft: 1.0,
|
|
117
|
+
mouthSmileLeft: 0.5, mouthSmileRight: 0.3,
|
|
118
|
+
cheekSquintLeft: 0.4,
|
|
119
|
+
},
|
|
120
|
+
sleepy: {
|
|
121
|
+
eyeBlinkLeft: 0.7, eyeBlinkRight: 0.7,
|
|
122
|
+
browDownLeft: 0.4, browDownRight: 0.4,
|
|
123
|
+
mouthShrugLower: 0.3, jawOpen: 0.15,
|
|
124
|
+
},
|
|
125
|
+
contempt: {
|
|
126
|
+
mouthSmileLeft: 0.6, mouthPressRight: 0.5,
|
|
127
|
+
mouthStretchRight: 0.3, browDownRight: 0.4,
|
|
128
|
+
eyeSquintRight: 0.3,
|
|
129
|
+
},
|
|
130
|
+
};
|
|
131
|
+
const IDLE_EMOTIONS = [
|
|
132
|
+
{ key: 'happy', weight: 5 },
|
|
133
|
+
{ key: 'surprised', weight: 2 },
|
|
134
|
+
{ key: 'smirk', weight: 2 },
|
|
135
|
+
{ key: 'sad', weight: 1 },
|
|
136
|
+
{ key: 'wink', weight: 2 },
|
|
137
|
+
{ key: 'sleepy', weight: 1 },
|
|
138
|
+
{ key: 'contempt', weight: 1 },
|
|
139
|
+
];
|
|
140
|
+
function pickIdleEmotion() {
|
|
141
|
+
const total = IDLE_EMOTIONS.reduce((s, e) => s + e.weight, 0);
|
|
142
|
+
let r = Math.random() * total;
|
|
143
|
+
for (const e of IDLE_EMOTIONS) {
|
|
144
|
+
r -= e.weight;
|
|
145
|
+
if (r <= 0)
|
|
146
|
+
return e.key;
|
|
147
|
+
}
|
|
148
|
+
return IDLE_EMOTIONS[0].key;
|
|
149
|
+
}
|
|
150
|
+
// ---------------------------------------------------------------------------
|
|
151
|
+
// Inner R3F scene — runs inside Canvas
|
|
152
|
+
// ---------------------------------------------------------------------------
|
|
153
|
+
function AvatarScene({ uri, morphState, onReady, }) {
|
|
154
|
+
const { camera } = (0, native_1.useThree)();
|
|
155
|
+
const gltf = (0, native_2.useGLTF)(uri);
|
|
156
|
+
const readyFiredRef = (0, react_1.useRef)(false);
|
|
157
|
+
// Position camera to match Filament version
|
|
158
|
+
(0, react_1.useLayoutEffect)(() => {
|
|
159
|
+
if (!(camera instanceof THREE.PerspectiveCamera))
|
|
160
|
+
return;
|
|
161
|
+
camera.fov = 34;
|
|
162
|
+
camera.position.set(0, 1.55, 1.05);
|
|
163
|
+
camera.lookAt(0, 1.55, 0);
|
|
164
|
+
camera.updateProjectionMatrix();
|
|
165
|
+
}, [camera]);
|
|
166
|
+
// Find best mesh and build morph index
|
|
167
|
+
(0, react_1.useEffect)(() => {
|
|
168
|
+
if (!gltf?.scene)
|
|
169
|
+
return;
|
|
170
|
+
let best = null;
|
|
171
|
+
let bestCount = 0;
|
|
172
|
+
gltf.scene.traverse((node) => {
|
|
173
|
+
if (!(node instanceof THREE.Mesh))
|
|
174
|
+
return;
|
|
175
|
+
const count = Object.keys(node.morphTargetDictionary ?? {}).length;
|
|
176
|
+
if (count > bestCount) {
|
|
177
|
+
bestCount = count;
|
|
178
|
+
best = node;
|
|
179
|
+
}
|
|
180
|
+
});
|
|
181
|
+
if (!best)
|
|
182
|
+
return;
|
|
183
|
+
const mesh = best;
|
|
184
|
+
morphState.current.mesh = mesh;
|
|
185
|
+
const idx = new Map();
|
|
186
|
+
for (const [name, i] of Object.entries(mesh.morphTargetDictionary ?? {})) {
|
|
187
|
+
idx.set(name, i);
|
|
188
|
+
idx.set(name.toLowerCase(), i);
|
|
189
|
+
}
|
|
190
|
+
morphState.current.index = idx;
|
|
191
|
+
if (!readyFiredRef.current) {
|
|
192
|
+
readyFiredRef.current = true;
|
|
193
|
+
onReady();
|
|
194
|
+
}
|
|
195
|
+
}, [gltf, morphState, onReady]);
|
|
196
|
+
// Apply morphState.weights to mesh every frame
|
|
197
|
+
(0, native_1.useFrame)(() => {
|
|
198
|
+
const { mesh, weights, index } = morphState.current;
|
|
199
|
+
if (!mesh?.morphTargetInfluences)
|
|
200
|
+
return;
|
|
201
|
+
for (const [name, w] of Object.entries(weights)) {
|
|
202
|
+
const i = index.get(name) ?? index.get(name.toLowerCase());
|
|
203
|
+
if (i !== undefined)
|
|
204
|
+
mesh.morphTargetInfluences[i] = w;
|
|
205
|
+
}
|
|
206
|
+
});
|
|
207
|
+
return ((0, jsx_runtime_1.jsxs)(jsx_runtime_1.Fragment, { children: [(0, jsx_runtime_1.jsx)("ambientLight", { intensity: 0.65 }), (0, jsx_runtime_1.jsx)("directionalLight", { position: [1, 3, 2], intensity: 1.3 }), (0, jsx_runtime_1.jsx)("primitive", { object: gltf.scene })] }));
|
|
208
|
+
}
|
|
209
|
+
// ---------------------------------------------------------------------------
|
|
210
|
+
// Main component
|
|
211
|
+
// ---------------------------------------------------------------------------
|
|
212
|
+
function FaceSqueezeEditor({ onClose }) {
|
|
213
|
+
const [localUri, setLocalUri] = (0, react_1.useState)(null);
|
|
214
|
+
const [isReady, setIsReady] = (0, react_1.useState)(false);
|
|
215
|
+
(0, react_1.useEffect)(() => {
|
|
216
|
+
let cancelled = false;
|
|
217
|
+
(async () => {
|
|
218
|
+
try {
|
|
219
|
+
const asset = expo_asset_1.Asset.fromModule(exports.FACE_SQUEEZE_LOCAL_MODULE);
|
|
220
|
+
await asset.downloadAsync();
|
|
221
|
+
const uri = asset.localUri ?? asset.uri;
|
|
222
|
+
if (!cancelled && uri)
|
|
223
|
+
setLocalUri(uri);
|
|
224
|
+
}
|
|
225
|
+
catch (e) {
|
|
226
|
+
console.error('[WgpuFaceSqueezeEditor] Failed to load local GLB:', e);
|
|
227
|
+
}
|
|
228
|
+
})();
|
|
229
|
+
return () => { cancelled = true; };
|
|
230
|
+
}, []);
|
|
231
|
+
// Shared morph state — mutated directly, never triggers re-render
|
|
232
|
+
const morphState = (0, react_1.useRef)({
|
|
233
|
+
weights: {},
|
|
234
|
+
index: new Map(),
|
|
235
|
+
mesh: null,
|
|
236
|
+
});
|
|
237
|
+
// ---- Audio ----
|
|
238
|
+
const reactionSoundIdx = (0, react_1.useRef)(0);
|
|
239
|
+
const player = (0, expo_audio_1.useAudioPlayer)(REACTION_SOUNDS[0]);
|
|
240
|
+
// ---- Timers ----
|
|
241
|
+
const springRef = (0, react_1.useRef)(null);
|
|
242
|
+
const emotionHoldTimerRef = (0, react_1.useRef)(null);
|
|
243
|
+
const idleEmotionTimerRef = (0, react_1.useRef)(null);
|
|
244
|
+
const lipSyncIntervalRef = (0, react_1.useRef)(null);
|
|
245
|
+
// ---- View geometry (for gesture zone math) ----
|
|
246
|
+
const viewSize = (0, react_1.useRef)({ width: 1, height: 1 });
|
|
247
|
+
const viewWidthSv = (0, react_native_reanimated_1.useSharedValue)(1);
|
|
248
|
+
const viewHeightSv = (0, react_native_reanimated_1.useSharedValue)(1);
|
|
249
|
+
const touchStartXSv = (0, react_native_reanimated_1.useSharedValue)(0);
|
|
250
|
+
const touchStartYSv = (0, react_native_reanimated_1.useSharedValue)(0);
|
|
251
|
+
const touchZoneSv = (0, react_native_reanimated_1.useSharedValue)('cheek_left');
|
|
252
|
+
const lastPanUpdateTsSv = (0, react_native_reanimated_1.useSharedValue)(0);
|
|
253
|
+
const onLayout = (0, react_1.useCallback)((e) => {
|
|
254
|
+
const w = Math.max(1, e.nativeEvent.layout.width);
|
|
255
|
+
const h = Math.max(1, e.nativeEvent.layout.height);
|
|
256
|
+
viewSize.current = { width: w, height: h };
|
|
257
|
+
viewWidthSv.value = w;
|
|
258
|
+
viewHeightSv.value = h;
|
|
259
|
+
}, [viewHeightSv, viewWidthSv]);
|
|
260
|
+
// ---- Weight helpers ----
|
|
261
|
+
const applyWeights = (0, react_1.useCallback)((targets) => {
|
|
262
|
+
for (const [name, w] of Object.entries(targets)) {
|
|
263
|
+
morphState.current.weights[name] = Math.max(0, Math.min(1, w));
|
|
264
|
+
}
|
|
265
|
+
}, []);
|
|
266
|
+
const stopSpring = (0, react_1.useCallback)(() => {
|
|
267
|
+
if (springRef.current !== null) {
|
|
268
|
+
cancelAnimationFrame(springRef.current);
|
|
269
|
+
springRef.current = null;
|
|
270
|
+
}
|
|
271
|
+
}, []);
|
|
272
|
+
const springToZero = (0, react_1.useCallback)(() => {
|
|
273
|
+
stopSpring();
|
|
274
|
+
const snapshot = { ...morphState.current.weights };
|
|
275
|
+
const startTime = Date.now();
|
|
276
|
+
const tick = () => {
|
|
277
|
+
const t = Math.min(1, (Date.now() - startTime) / SPRING_DURATION_MS);
|
|
278
|
+
const eased = 1 - Math.pow(1 - t, 3);
|
|
279
|
+
for (const name of Object.keys(snapshot)) {
|
|
280
|
+
morphState.current.weights[name] = snapshot[name] * (1 - eased);
|
|
281
|
+
}
|
|
282
|
+
if (t >= 1) {
|
|
283
|
+
springRef.current = null;
|
|
284
|
+
morphState.current.weights = {};
|
|
285
|
+
return;
|
|
286
|
+
}
|
|
287
|
+
springRef.current = requestAnimationFrame(tick);
|
|
288
|
+
};
|
|
289
|
+
springRef.current = requestAnimationFrame(tick);
|
|
290
|
+
}, [stopSpring]);
|
|
291
|
+
const playEmotionBurst = (0, react_1.useCallback)((emotionKey) => {
|
|
292
|
+
const preset = EMOTION_PRESETS[emotionKey];
|
|
293
|
+
if (!preset)
|
|
294
|
+
return;
|
|
295
|
+
if (emotionHoldTimerRef.current !== null) {
|
|
296
|
+
clearTimeout(emotionHoldTimerRef.current);
|
|
297
|
+
emotionHoldTimerRef.current = null;
|
|
298
|
+
}
|
|
299
|
+
stopSpring();
|
|
300
|
+
applyWeights(preset);
|
|
301
|
+
emotionHoldTimerRef.current = setTimeout(() => {
|
|
302
|
+
emotionHoldTimerRef.current = null;
|
|
303
|
+
springToZero();
|
|
304
|
+
}, EMOTION_HOLD_MS);
|
|
305
|
+
}, [applyWeights, springToZero, stopSpring]);
|
|
306
|
+
const stopLipSync = (0, react_1.useCallback)(() => {
|
|
307
|
+
if (lipSyncIntervalRef.current !== null) {
|
|
308
|
+
clearInterval(lipSyncIntervalRef.current);
|
|
309
|
+
lipSyncIntervalRef.current = null;
|
|
310
|
+
}
|
|
311
|
+
applyWeights({ jawOpen: 0, mouthFunnel: 0, mouthPucker: 0,
|
|
312
|
+
mouthUpperUpLeft: 0, mouthUpperUpRight: 0,
|
|
313
|
+
mouthLowerDownLeft: 0, mouthLowerDownRight: 0,
|
|
314
|
+
mouthShrugLower: 0, mouthShrugUpper: 0 });
|
|
315
|
+
}, [applyWeights]);
|
|
316
|
+
const startLipSync = (0, react_1.useCallback)((durationMs) => {
|
|
317
|
+
if (lipSyncIntervalRef.current !== null)
|
|
318
|
+
clearInterval(lipSyncIntervalRef.current);
|
|
319
|
+
const startTime = Date.now();
|
|
320
|
+
const TICK_MS = 60;
|
|
321
|
+
lipSyncIntervalRef.current = setInterval(() => {
|
|
322
|
+
const elapsed = Date.now() - startTime;
|
|
323
|
+
if (elapsed >= durationMs) {
|
|
324
|
+
stopLipSync();
|
|
325
|
+
return;
|
|
326
|
+
}
|
|
327
|
+
const fadeIn = Math.min(1, elapsed / 80);
|
|
328
|
+
const fadeOut = Math.min(1, (durationMs - elapsed) / 150);
|
|
329
|
+
const envelope = Math.min(fadeIn, fadeOut);
|
|
330
|
+
const t = elapsed / 1000;
|
|
331
|
+
const jaw = envelope * (0.45 + 0.35 * Math.abs(Math.sin(t * 9.5 + Math.random() * 0.3)));
|
|
332
|
+
const funnel = envelope * Math.max(0, Math.sin(t * 7.2) * 0.5);
|
|
333
|
+
const shrug = envelope * Math.max(0, Math.sin(t * 5.8 + 1.2) * 0.35);
|
|
334
|
+
applyWeights({
|
|
335
|
+
jawOpen: jaw, mouthFunnel: funnel,
|
|
336
|
+
mouthShrugLower: shrug, mouthShrugUpper: shrug * 0.6,
|
|
337
|
+
mouthLowerDownLeft: jaw * 0.6, mouthLowerDownRight: jaw * 0.6,
|
|
338
|
+
mouthUpperUpLeft: jaw * 0.35, mouthUpperUpRight: jaw * 0.35,
|
|
339
|
+
});
|
|
340
|
+
}, TICK_MS);
|
|
341
|
+
}, [applyWeights, stopLipSync]);
|
|
342
|
+
const playReaction = (0, react_1.useCallback)((durationMs = 1400) => {
|
|
343
|
+
const idx = reactionSoundIdx.current % REACTION_SOUNDS.length;
|
|
344
|
+
reactionSoundIdx.current += 1;
|
|
345
|
+
try {
|
|
346
|
+
player.replace(REACTION_SOUNDS[idx]);
|
|
347
|
+
player.play();
|
|
348
|
+
}
|
|
349
|
+
catch { /* ignore */ }
|
|
350
|
+
startLipSync(durationMs);
|
|
351
|
+
}, [player, startLipSync]);
|
|
352
|
+
// Cleanup
|
|
353
|
+
(0, react_1.useEffect)(() => () => {
|
|
354
|
+
if (springRef.current !== null)
|
|
355
|
+
cancelAnimationFrame(springRef.current);
|
|
356
|
+
if (emotionHoldTimerRef.current !== null)
|
|
357
|
+
clearTimeout(emotionHoldTimerRef.current);
|
|
358
|
+
if (idleEmotionTimerRef.current !== null)
|
|
359
|
+
clearInterval(idleEmotionTimerRef.current);
|
|
360
|
+
if (lipSyncIntervalRef.current !== null)
|
|
361
|
+
clearInterval(lipSyncIntervalRef.current);
|
|
362
|
+
}, []);
|
|
363
|
+
// Idle emotion bursts (start once ready)
|
|
364
|
+
(0, react_1.useEffect)(() => {
|
|
365
|
+
if (!isReady)
|
|
366
|
+
return;
|
|
367
|
+
const initialKickoff = setTimeout(() => playEmotionBurst(pickIdleEmotion()), INITIAL_IDLE_EMOTION_DELAY_MS);
|
|
368
|
+
idleEmotionTimerRef.current = setInterval(() => playEmotionBurst(pickIdleEmotion()), IDLE_EMOTION_INTERVAL_MS);
|
|
369
|
+
return () => {
|
|
370
|
+
clearTimeout(initialKickoff);
|
|
371
|
+
if (idleEmotionTimerRef.current !== null)
|
|
372
|
+
clearInterval(idleEmotionTimerRef.current);
|
|
373
|
+
};
|
|
374
|
+
}, [isReady, playEmotionBurst]);
|
|
375
|
+
// ---- Gesture callbacks (JS side — called via runOnJS) ----
|
|
376
|
+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
377
|
+
const onPanBeginJs = (0, react_1.useCallback)((_x, _y, _zone) => {
|
|
378
|
+
stopSpring();
|
|
379
|
+
Haptics.impactAsync(Haptics.ImpactFeedbackStyle.Light);
|
|
380
|
+
}, [stopSpring]);
|
|
381
|
+
const onPanUpdateJs = (0, react_1.useCallback)((zone, dx, dy, isLeftHalf) => {
|
|
382
|
+
const u = {};
|
|
383
|
+
if (zone === 'brow') {
|
|
384
|
+
if (dy < 0) {
|
|
385
|
+
u['browInnerUp'] = Math.abs(dy) * 10;
|
|
386
|
+
u['browOuterUpLeft'] = Math.abs(dy) * 10;
|
|
387
|
+
u['browOuterUpRight'] = Math.abs(dy) * 10;
|
|
388
|
+
u['eyeWideLeft'] = Math.abs(dy) * 8;
|
|
389
|
+
u['eyeWideRight'] = Math.abs(dy) * 8;
|
|
390
|
+
}
|
|
391
|
+
else {
|
|
392
|
+
u['browDownLeft'] = Math.abs(dy) * 10;
|
|
393
|
+
u['browDownRight'] = Math.abs(dy) * 10;
|
|
394
|
+
u['eyeSquintLeft'] = Math.abs(dy) * 8;
|
|
395
|
+
u['eyeSquintRight'] = Math.abs(dy) * 8;
|
|
396
|
+
u['noseSneerLeft'] = Math.abs(dy) * 6;
|
|
397
|
+
u['noseSneerRight'] = Math.abs(dy) * 6;
|
|
398
|
+
}
|
|
399
|
+
u['browOuterUpLeft'] = (u['browOuterUpLeft'] ?? 0) + (dx < 0 ? Math.abs(dx) * 8 : 0);
|
|
400
|
+
u['browOuterUpRight'] = (u['browOuterUpRight'] ?? 0) + (dx > 0 ? Math.abs(dx) * 8 : 0);
|
|
401
|
+
}
|
|
402
|
+
else if (zone === 'eye') {
|
|
403
|
+
u[isLeftHalf ? 'eyeBlinkLeft' : 'eyeBlinkRight'] = dy > 0 ? Math.min(1, Math.abs(dy) * 10) : 0;
|
|
404
|
+
u[isLeftHalf ? 'eyeSquintLeft' : 'eyeSquintRight'] = dy > 0 ? Math.abs(dy) * 6 : 0;
|
|
405
|
+
u[isLeftHalf ? 'eyeWideLeft' : 'eyeWideRight'] = dy < 0 ? Math.abs(dy) * 8 : 0;
|
|
406
|
+
u[isLeftHalf ? 'eyeLookOutLeft' : 'eyeLookInRight'] = dx < 0 ? Math.abs(dx) * 7 : 0;
|
|
407
|
+
u[isLeftHalf ? 'eyeLookInLeft' : 'eyeLookOutRight'] = dx > 0 ? Math.abs(dx) * 7 : 0;
|
|
408
|
+
u[isLeftHalf ? 'eyeLookUpLeft' : 'eyeLookUpRight'] = dy < 0 ? Math.abs(dy) * 5 : 0;
|
|
409
|
+
u[isLeftHalf ? 'eyeLookDownLeft' : 'eyeLookDownRight'] = dy > 0 ? Math.abs(dy) * 5 : 0;
|
|
410
|
+
}
|
|
411
|
+
else if (zone === 'cheek_left') {
|
|
412
|
+
u['cheekPuff'] = Math.abs(dx) * 6;
|
|
413
|
+
u['cheekSquintLeft'] = Math.abs(dy) * 6;
|
|
414
|
+
u['cheekSquintRight'] = 0;
|
|
415
|
+
u['mouthSmileLeft'] = dy < 0 ? Math.abs(dy) * 5 : 0;
|
|
416
|
+
u['mouthDimpleLeft'] = dy < 0 ? Math.abs(dy) * 4 : 0;
|
|
417
|
+
u['mouthStretchLeft'] = dy > 0 ? Math.abs(dy) * 5 : 0;
|
|
418
|
+
u['mouthFrownLeft'] = dy > 0 ? Math.abs(dy) * 4 : 0;
|
|
419
|
+
u['mouthPressLeft'] = dy > 0 ? Math.abs(dy) * 3 : 0;
|
|
420
|
+
}
|
|
421
|
+
else if (zone === 'cheek_right') {
|
|
422
|
+
u['cheekPuff'] = Math.abs(dx) * 6;
|
|
423
|
+
u['cheekSquintRight'] = Math.abs(dy) * 6;
|
|
424
|
+
u['cheekSquintLeft'] = 0;
|
|
425
|
+
u['mouthSmileRight'] = dy < 0 ? Math.abs(dy) * 5 : 0;
|
|
426
|
+
u['mouthDimpleRight'] = dy < 0 ? Math.abs(dy) * 4 : 0;
|
|
427
|
+
u['mouthStretchRight'] = dy > 0 ? Math.abs(dy) * 5 : 0;
|
|
428
|
+
u['mouthFrownRight'] = dy > 0 ? Math.abs(dy) * 4 : 0;
|
|
429
|
+
u['mouthPressRight'] = dy > 0 ? Math.abs(dy) * 3 : 0;
|
|
430
|
+
}
|
|
431
|
+
else if (zone === 'jaw') {
|
|
432
|
+
u['jawOpen'] = dy > 0 ? Math.abs(dy) * 10 : 0;
|
|
433
|
+
u['tongueOut'] = dy > 0 ? Math.max(0, Math.abs(dy) * 10 - 0.7) : 0;
|
|
434
|
+
u['mouthLowerDownLeft'] = dy > 0 ? Math.abs(dy) * 6 : 0;
|
|
435
|
+
u['mouthLowerDownRight'] = dy > 0 ? Math.abs(dy) * 6 : 0;
|
|
436
|
+
u['mouthUpperUpLeft'] = dy > 0 ? Math.abs(dy) * 4 : 0;
|
|
437
|
+
u['mouthUpperUpRight'] = dy > 0 ? Math.abs(dy) * 4 : 0;
|
|
438
|
+
u['mouthPressLeft'] = dy < 0 ? Math.abs(dy) * 6 : 0;
|
|
439
|
+
u['mouthPressRight'] = dy < 0 ? Math.abs(dy) * 6 : 0;
|
|
440
|
+
u['jawLeft'] = dx < 0 ? Math.abs(dx) * 8 : 0;
|
|
441
|
+
u['jawRight'] = dx > 0 ? Math.abs(dx) * 8 : 0;
|
|
442
|
+
u['jawForward'] = Math.abs(dx) * 4;
|
|
443
|
+
u['mouthLeft'] = dx < 0 ? Math.abs(dx) * 5 : 0;
|
|
444
|
+
u['mouthRight'] = dx > 0 ? Math.abs(dx) * 5 : 0;
|
|
445
|
+
}
|
|
446
|
+
applyWeights(u);
|
|
447
|
+
}, [applyWeights]);
|
|
448
|
+
const onPanEndJs = (0, react_1.useCallback)(() => {
|
|
449
|
+
if (emotionHoldTimerRef.current !== null)
|
|
450
|
+
return;
|
|
451
|
+
springToZero();
|
|
452
|
+
}, [springToZero]);
|
|
453
|
+
const onPinchBeginJs = (0, react_1.useCallback)(() => {
|
|
454
|
+
stopSpring();
|
|
455
|
+
Haptics.impactAsync(Haptics.ImpactFeedbackStyle.Medium);
|
|
456
|
+
}, [stopSpring]);
|
|
457
|
+
const onPinchUpdateJs = (0, react_1.useCallback)((scale, delta) => {
|
|
458
|
+
if (scale < 1) {
|
|
459
|
+
applyWeights({ mouthPucker: delta * 2, mouthFunnel: 0, mouthRollLower: delta * 1.2, mouthRollUpper: delta * 1.2 });
|
|
460
|
+
}
|
|
461
|
+
else {
|
|
462
|
+
applyWeights({ mouthFunnel: delta * 2, mouthPucker: 0, mouthShrugLower: delta * 0.8, mouthShrugUpper: delta * 0.8, mouthRollLower: 0, mouthRollUpper: 0 });
|
|
463
|
+
}
|
|
464
|
+
}, [applyWeights]);
|
|
465
|
+
const onPinchEndJs = (0, react_1.useCallback)(() => {
|
|
466
|
+
if (emotionHoldTimerRef.current !== null)
|
|
467
|
+
return;
|
|
468
|
+
springToZero();
|
|
469
|
+
}, [springToZero]);
|
|
470
|
+
const onEyeTapJs = (0, react_1.useCallback)((x, y) => {
|
|
471
|
+
const relY = y / Math.max(1, viewSize.current.height);
|
|
472
|
+
if (relY < 0.16 || relY > 0.52)
|
|
473
|
+
return;
|
|
474
|
+
Haptics.impactAsync(Haptics.ImpactFeedbackStyle.Heavy);
|
|
475
|
+
playReaction();
|
|
476
|
+
playEmotionBurst('owie');
|
|
477
|
+
}, [playEmotionBurst, playReaction]);
|
|
478
|
+
// ---- Gestures ----
|
|
479
|
+
const panGesture = react_native_gesture_handler_1.Gesture.Pan()
|
|
480
|
+
.minDistance(0)
|
|
481
|
+
.onBegin((e) => {
|
|
482
|
+
'worklet';
|
|
483
|
+
touchStartXSv.value = e.x;
|
|
484
|
+
touchStartYSv.value = e.y;
|
|
485
|
+
const relY = e.y / Math.max(1, viewHeightSv.value);
|
|
486
|
+
let zone;
|
|
487
|
+
if (relY < 0.20)
|
|
488
|
+
zone = 'brow';
|
|
489
|
+
else if (relY < 0.38)
|
|
490
|
+
zone = 'eye';
|
|
491
|
+
else if (relY > 0.72)
|
|
492
|
+
zone = 'jaw';
|
|
493
|
+
else
|
|
494
|
+
zone = e.x < viewWidthSv.value / 2 ? 'cheek_left' : 'cheek_right';
|
|
495
|
+
touchZoneSv.value = zone;
|
|
496
|
+
(0, react_native_reanimated_1.runOnJS)(onPanBeginJs)(e.x, e.y, zone);
|
|
497
|
+
})
|
|
498
|
+
.onUpdate((e) => {
|
|
499
|
+
'worklet';
|
|
500
|
+
const now = Date.now();
|
|
501
|
+
if (now - lastPanUpdateTsSv.value < 33)
|
|
502
|
+
return;
|
|
503
|
+
lastPanUpdateTsSv.value = now;
|
|
504
|
+
const w = Math.max(1, viewWidthSv.value);
|
|
505
|
+
const h = Math.max(1, viewHeightSv.value);
|
|
506
|
+
const dx = (e.x - touchStartXSv.value) / w;
|
|
507
|
+
const dy = (e.y - touchStartYSv.value) / h;
|
|
508
|
+
(0, react_native_reanimated_1.runOnJS)(onPanUpdateJs)(touchZoneSv.value, dx, dy, touchStartXSv.value < w / 2);
|
|
509
|
+
})
|
|
510
|
+
.onEnd(() => {
|
|
511
|
+
'worklet';
|
|
512
|
+
(0, react_native_reanimated_1.runOnJS)(onPanEndJs)();
|
|
513
|
+
});
|
|
514
|
+
const pinchGesture = react_native_gesture_handler_1.Gesture.Pinch()
|
|
515
|
+
.onBegin(() => {
|
|
516
|
+
'worklet';
|
|
517
|
+
(0, react_native_reanimated_1.runOnJS)(onPinchBeginJs)();
|
|
518
|
+
})
|
|
519
|
+
.onUpdate((e) => {
|
|
520
|
+
'worklet';
|
|
521
|
+
(0, react_native_reanimated_1.runOnJS)(onPinchUpdateJs)(e.scale, Math.abs(1 - e.scale));
|
|
522
|
+
})
|
|
523
|
+
.onEnd(() => {
|
|
524
|
+
'worklet';
|
|
525
|
+
(0, react_native_reanimated_1.runOnJS)(onPinchEndJs)();
|
|
526
|
+
});
|
|
527
|
+
const tapGesture = react_native_gesture_handler_1.Gesture.Tap()
|
|
528
|
+
.onEnd((e, success) => {
|
|
529
|
+
'worklet';
|
|
530
|
+
if (success)
|
|
531
|
+
(0, react_native_reanimated_1.runOnJS)(onEyeTapJs)(e.x, e.y);
|
|
532
|
+
});
|
|
533
|
+
const composed = react_native_gesture_handler_1.Gesture.Simultaneous(panGesture, pinchGesture, tapGesture);
|
|
534
|
+
const handleReady = (0, react_1.useCallback)(() => setIsReady(true), []);
|
|
535
|
+
return ((0, jsx_runtime_1.jsx)(react_native_gesture_handler_1.GestureHandlerRootView, { style: { flex: 1 }, children: (0, jsx_runtime_1.jsxs)(react_native_1.View, { style: styles.container, children: [(0, jsx_runtime_1.jsx)(react_native_1.View, { style: styles.scene, children: localUri ? ((0, jsx_runtime_1.jsx)(native_1.Canvas, { style: react_native_1.StyleSheet.absoluteFill, camera: { fov: 34, position: [0, 1.55, 1.05], near: 0.01, far: 50 }, gl: { antialias: true, alpha: false }, onCreated: ({ gl }) => gl.setClearColor(new THREE.Color('#0a0a0a')), children: (0, jsx_runtime_1.jsx)(AvatarScene, { uri: localUri, morphState: morphState, onReady: handleReady }) })) : ((0, jsx_runtime_1.jsx)(react_native_1.View, { style: styles.loading, children: (0, jsx_runtime_1.jsx)(react_native_1.Text, { style: styles.loadingText, children: "Loading face\u2026" }) })) }), (0, jsx_runtime_1.jsx)(react_native_gesture_handler_1.GestureDetector, { gesture: composed, children: (0, jsx_runtime_1.jsx)(react_native_1.View, { style: react_native_1.StyleSheet.absoluteFill, onLayout: onLayout, collapsable: false }) }), (0, jsx_runtime_1.jsx)(react_native_1.Pressable, { style: styles.closeBtn, onPress: onClose, children: (0, jsx_runtime_1.jsx)(react_native_1.Text, { style: styles.closeBtnText, children: "\u2715 Done squeezing" }) }), (0, jsx_runtime_1.jsx)(react_native_1.Text, { style: styles.hint, children: "Drag to stretch \u00B7 Pinch lips \u00B7 Poke an eye \uD83D\uDC41" })] }) }));
|
|
536
|
+
}
|
|
537
|
+
exports.FaceSqueezeEditor = FaceSqueezeEditor;
|
|
538
|
+
const styles = react_native_1.StyleSheet.create({
|
|
539
|
+
container: { flex: 1, backgroundColor: '#0a0a0a' },
|
|
540
|
+
scene: { flex: 1 },
|
|
541
|
+
loading: { flex: 1, alignItems: 'center', justifyContent: 'center' },
|
|
542
|
+
loadingText: { color: '#97a3b5', fontSize: 14 },
|
|
543
|
+
closeBtn: {
|
|
544
|
+
position: 'absolute', top: 52, right: 16,
|
|
545
|
+
backgroundColor: '#6c63ff', borderRadius: 20,
|
|
546
|
+
paddingHorizontal: 16, paddingVertical: 8,
|
|
547
|
+
},
|
|
548
|
+
closeBtnText: { color: '#fff', fontWeight: '700', fontSize: 14 },
|
|
549
|
+
hint: {
|
|
550
|
+
position: 'absolute', bottom: 40, alignSelf: 'center',
|
|
551
|
+
color: '#97a3b5', fontSize: 13,
|
|
552
|
+
},
|
|
553
|
+
});
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
export declare const FACE_SQUEEZE_LOCAL_MODULE: number;
|
|
2
|
+
export interface FaceSqueezeEditorProps {
|
|
3
|
+
onClose: () => void;
|
|
4
|
+
}
|
|
5
|
+
export declare function FaceSqueezeEditor({ onClose }: FaceSqueezeEditorProps): import("react/jsx-runtime").JSX.Element;
|
|
6
|
+
export default FaceSqueezeEditor;
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.FaceSqueezeEditor = exports.FACE_SQUEEZE_LOCAL_MODULE = void 0;
|
|
4
|
+
const jsx_runtime_1 = require("react/jsx-runtime");
|
|
5
|
+
const react_native_1 = require("react-native");
|
|
6
|
+
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
|
7
|
+
exports.FACE_SQUEEZE_LOCAL_MODULE = require('../assets/face-squeeze-local.glb');
|
|
8
|
+
function FaceSqueezeEditor({ onClose }) {
|
|
9
|
+
return ((0, jsx_runtime_1.jsx)(react_native_1.View, { style: styles.container, children: (0, jsx_runtime_1.jsxs)(react_native_1.View, { style: styles.card, children: [(0, jsx_runtime_1.jsx)(react_native_1.Text, { style: styles.title, children: "Face squeeze editor is mobile-only." }), (0, jsx_runtime_1.jsx)(react_native_1.Text, { style: styles.subtitle, children: "This placeholder keeps web builds safe while preserving native functionality." }), (0, jsx_runtime_1.jsx)(react_native_1.Pressable, { style: styles.closeButton, onPress: onClose, children: (0, jsx_runtime_1.jsx)(react_native_1.Text, { style: styles.closeButtonText, children: "Close" }) })] }) }));
|
|
10
|
+
}
|
|
11
|
+
exports.FaceSqueezeEditor = FaceSqueezeEditor;
|
|
12
|
+
const styles = react_native_1.StyleSheet.create({
|
|
13
|
+
container: {
|
|
14
|
+
flex: 1,
|
|
15
|
+
alignItems: "center",
|
|
16
|
+
justifyContent: "center",
|
|
17
|
+
backgroundColor: "#0f1115",
|
|
18
|
+
padding: 20,
|
|
19
|
+
},
|
|
20
|
+
card: {
|
|
21
|
+
width: "100%",
|
|
22
|
+
maxWidth: 460,
|
|
23
|
+
borderRadius: 12,
|
|
24
|
+
borderWidth: 1,
|
|
25
|
+
borderColor: "#2d3340",
|
|
26
|
+
backgroundColor: "#171b23",
|
|
27
|
+
padding: 16,
|
|
28
|
+
gap: 10,
|
|
29
|
+
},
|
|
30
|
+
title: {
|
|
31
|
+
color: "#f1f4f8",
|
|
32
|
+
fontSize: 18,
|
|
33
|
+
fontWeight: "700",
|
|
34
|
+
},
|
|
35
|
+
subtitle: {
|
|
36
|
+
color: "#98a1b3",
|
|
37
|
+
fontSize: 14,
|
|
38
|
+
lineHeight: 20,
|
|
39
|
+
},
|
|
40
|
+
closeButton: {
|
|
41
|
+
marginTop: 6,
|
|
42
|
+
alignSelf: "flex-start",
|
|
43
|
+
borderRadius: 8,
|
|
44
|
+
backgroundColor: "#2e8f5b",
|
|
45
|
+
paddingHorizontal: 12,
|
|
46
|
+
paddingVertical: 8,
|
|
47
|
+
},
|
|
48
|
+
closeButtonText: {
|
|
49
|
+
color: "#ffffff",
|
|
50
|
+
fontSize: 14,
|
|
51
|
+
fontWeight: "600",
|
|
52
|
+
},
|
|
53
|
+
});
|
|
54
|
+
exports.default = FaceSqueezeEditor;
|
package/dist/editor/index.d.ts
CHANGED
|
@@ -3,7 +3,8 @@ export { AvatarEditor } from './AvatarEditor';
|
|
|
3
3
|
export { AvatarModel } from './AvatarModel';
|
|
4
4
|
export { RigidAccessory } from './RigidAccessory';
|
|
5
5
|
export { SkinnedClothing } from './SkinnedClothing';
|
|
6
|
+
export { FaceSqueezeEditor, FACE_SQUEEZE_LOCAL_MODULE } from './FaceSqueezeEditor';
|
|
6
7
|
export { studioTheme, withAlpha } from './studioTheme';
|
|
7
8
|
export { useBoneSnap, findNearestBone, getWorldPositionForBone, getWorldPositionForPlacement, snapPlacementToNearestBone, HUMANOID_BONES } from './boneSnap';
|
|
8
|
-
export type { AvatarEditorProps, AssetPlacement, EquippedAsset } from './types';
|
|
9
|
+
export type { AvatarEditorProps, AssetPlacement, EquippedAsset, FaceSqueezeEditorProps } from './types';
|
|
9
10
|
export type { BoneSnapResult, HumanoidBone } from './boneSnap';
|
package/dist/editor/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.HUMANOID_BONES = exports.snapPlacementToNearestBone = exports.getWorldPositionForPlacement = exports.getWorldPositionForBone = exports.findNearestBone = exports.useBoneSnap = exports.withAlpha = exports.studioTheme = exports.SkinnedClothing = exports.RigidAccessory = exports.AvatarModel = exports.AvatarEditor = exports.AvatarCanvas = void 0;
|
|
3
|
+
exports.HUMANOID_BONES = exports.snapPlacementToNearestBone = exports.getWorldPositionForPlacement = exports.getWorldPositionForBone = exports.findNearestBone = exports.useBoneSnap = exports.withAlpha = exports.studioTheme = exports.FACE_SQUEEZE_LOCAL_MODULE = exports.FaceSqueezeEditor = exports.SkinnedClothing = exports.RigidAccessory = exports.AvatarModel = exports.AvatarEditor = exports.AvatarCanvas = void 0;
|
|
4
4
|
var AvatarCanvas_1 = require("./AvatarCanvas");
|
|
5
5
|
Object.defineProperty(exports, "AvatarCanvas", { enumerable: true, get: function () { return AvatarCanvas_1.AvatarCanvas; } });
|
|
6
6
|
var AvatarEditor_1 = require("./AvatarEditor");
|
|
@@ -11,6 +11,9 @@ var RigidAccessory_1 = require("./RigidAccessory");
|
|
|
11
11
|
Object.defineProperty(exports, "RigidAccessory", { enumerable: true, get: function () { return RigidAccessory_1.RigidAccessory; } });
|
|
12
12
|
var SkinnedClothing_1 = require("./SkinnedClothing");
|
|
13
13
|
Object.defineProperty(exports, "SkinnedClothing", { enumerable: true, get: function () { return SkinnedClothing_1.SkinnedClothing; } });
|
|
14
|
+
var FaceSqueezeEditor_1 = require("./FaceSqueezeEditor");
|
|
15
|
+
Object.defineProperty(exports, "FaceSqueezeEditor", { enumerable: true, get: function () { return FaceSqueezeEditor_1.FaceSqueezeEditor; } });
|
|
16
|
+
Object.defineProperty(exports, "FACE_SQUEEZE_LOCAL_MODULE", { enumerable: true, get: function () { return FaceSqueezeEditor_1.FACE_SQUEEZE_LOCAL_MODULE; } });
|
|
14
17
|
var studioTheme_1 = require("./studioTheme");
|
|
15
18
|
Object.defineProperty(exports, "studioTheme", { enumerable: true, get: function () { return studioTheme_1.studioTheme; } });
|
|
16
19
|
Object.defineProperty(exports, "withAlpha", { enumerable: true, get: function () { return studioTheme_1.withAlpha; } });
|
package/dist/editor/types.d.ts
CHANGED
package/dist/html.d.ts
CHANGED
|
@@ -15,5 +15,10 @@ export type AvatarConfig = {
|
|
|
15
15
|
* Example: "https://studio.sitebay.org/vendor"
|
|
16
16
|
*/
|
|
17
17
|
vendorBaseUrl?: string | null;
|
|
18
|
+
/**
|
|
19
|
+
* URL to the DRACO decoder WASM/JS folder.
|
|
20
|
+
* Defaults to gstatic's v1.5.6 CDN.
|
|
21
|
+
*/
|
|
22
|
+
dracoDecoderUrl?: string | null;
|
|
18
23
|
};
|
|
19
24
|
export declare function buildAvatarHtml(config: AvatarConfig): string;
|
package/dist/html.js
CHANGED
|
@@ -20,6 +20,7 @@ function buildAvatarHtml(config) {
|
|
|
20
20
|
const headAudioUrl = v ? `${v}/headaudio.min.mjs` : 'https://cdn.jsdelivr.net/gh/met4citizen/HeadAudio@v0.1.0-alpha/dist/headaudio.min.mjs';
|
|
21
21
|
const headWorkletUrl = v ? `${v}/headworklet.min.mjs` : 'https://cdn.jsdelivr.net/gh/met4citizen/HeadAudio@v0.1.0-alpha/dist/headworklet.min.mjs';
|
|
22
22
|
const headModelUrl = v ? `${v}/model-en-mixed.bin` : 'https://cdn.jsdelivr.net/gh/met4citizen/HeadAudio@v0.1.0-alpha/dist/model-en-mixed.bin';
|
|
23
|
+
const dracoUrl = config.dracoDecoderUrl ?? (v ? `${v}/draco/` : 'https://www.gstatic.com/draco/versioned/decoders/1.5.6/');
|
|
23
24
|
return `
|
|
24
25
|
<!DOCTYPE html>
|
|
25
26
|
<html>
|
|
@@ -40,25 +41,26 @@ function buildAvatarHtml(config) {
|
|
|
40
41
|
}
|
|
41
42
|
</script>
|
|
42
43
|
<script>
|
|
43
|
-
window.ReactNativeWebView
|
|
44
|
+
function rnPost(d) { window.ReactNativeWebView && window.ReactNativeWebView.postMessage(d); }
|
|
45
|
+
rnPost(
|
|
44
46
|
JSON.stringify({ type: 'log', message: '[bootstrap] inline script start' })
|
|
45
47
|
);
|
|
46
48
|
function postBootstrapError(kind, message) {
|
|
47
|
-
|
|
49
|
+
rnPost(
|
|
48
50
|
JSON.stringify({ type: 'error', message: '[' + kind + '] ' + String(message || 'Unknown error') })
|
|
49
51
|
);
|
|
50
52
|
}
|
|
51
53
|
document.addEventListener('DOMContentLoaded', function() {
|
|
52
|
-
|
|
54
|
+
rnPost(
|
|
53
55
|
JSON.stringify({ type: 'log', message: '[bootstrap] DOMContentLoaded' })
|
|
54
56
|
);
|
|
55
57
|
});
|
|
56
58
|
window.addEventListener('error', function(event) {
|
|
57
|
-
postBootstrapError('window.error', event
|
|
59
|
+
postBootstrapError('window.error', (event && event.message) || (event && event.error && event.error.message) || 'Script error');
|
|
58
60
|
});
|
|
59
61
|
window.addEventListener('unhandledrejection', function(event) {
|
|
60
|
-
const reason = event
|
|
61
|
-
postBootstrapError('unhandledrejection', reason
|
|
62
|
+
const reason = event && event.reason;
|
|
63
|
+
postBootstrapError('unhandledrejection', (reason && reason.message) || reason || 'Unhandled promise rejection');
|
|
62
64
|
});
|
|
63
65
|
</script>
|
|
64
66
|
</head>
|
|
@@ -66,10 +68,11 @@ window.addEventListener('unhandledrejection', function(event) {
|
|
|
66
68
|
<div id="avatar"></div>
|
|
67
69
|
<script type="module">
|
|
68
70
|
(async function() {
|
|
69
|
-
|
|
71
|
+
rnPost(
|
|
70
72
|
JSON.stringify({ type: 'log', message: '[module] script start' })
|
|
71
73
|
);
|
|
72
74
|
const AUTH_TOKEN = ${JSON.stringify(config.authToken ?? null)};
|
|
75
|
+
const DRACO_URL = ${JSON.stringify(dracoUrl)};
|
|
73
76
|
const TALKING_HEAD_URL = ${JSON.stringify(talkingHeadUrl)};
|
|
74
77
|
const HEAD_AUDIO_URL = ${JSON.stringify(headAudioUrl)};
|
|
75
78
|
const HEAD_AUDIO_WORKLET = ${JSON.stringify(headWorkletUrl)};
|
|
@@ -94,11 +97,11 @@ let staticModel = null;
|
|
|
94
97
|
let initStarted = false;
|
|
95
98
|
|
|
96
99
|
function log(msg) {
|
|
97
|
-
|
|
100
|
+
rnPost(JSON.stringify({ type: 'log', message: msg }));
|
|
98
101
|
}
|
|
99
102
|
|
|
100
103
|
function emitLoading(stage, progress = null) {
|
|
101
|
-
|
|
104
|
+
rnPost(
|
|
102
105
|
JSON.stringify({ type: 'loading', stage, progress }),
|
|
103
106
|
);
|
|
104
107
|
}
|
|
@@ -125,13 +128,13 @@ function applyColorOverrides() {
|
|
|
125
128
|
mats.forEach((mat) => {
|
|
126
129
|
const name = (mat.name || '').toLowerCase();
|
|
127
130
|
if (HAIR_COLOR && (name.includes('hair') || name.includes('fur'))) {
|
|
128
|
-
mat.color
|
|
131
|
+
mat.color && mat.color.set(HAIR_COLOR);
|
|
129
132
|
}
|
|
130
133
|
if (SKIN_COLOR && (name.includes('skin') || name.includes('body') || name.includes('face'))) {
|
|
131
|
-
mat.color
|
|
134
|
+
mat.color && mat.color.set(SKIN_COLOR);
|
|
132
135
|
}
|
|
133
136
|
if (EYE_COLOR && (name.includes('eye') || name.includes('iris'))) {
|
|
134
|
-
mat.color
|
|
137
|
+
mat.color && mat.color.set(EYE_COLOR);
|
|
135
138
|
}
|
|
136
139
|
});
|
|
137
140
|
});
|
|
@@ -204,7 +207,7 @@ async function loadStaticFallback(loadedAvatarUrl) {
|
|
|
204
207
|
applyAccessories(pendingAccessoriesList);
|
|
205
208
|
motionInitFromRoot(staticModel);
|
|
206
209
|
emitLoading('ready', 100);
|
|
207
|
-
|
|
210
|
+
rnPost(JSON.stringify({ type: 'ready' }));
|
|
208
211
|
|
|
209
212
|
window.addEventListener('resize', () => {
|
|
210
213
|
camera.aspect = window.innerWidth / window.innerHeight;
|
|
@@ -228,11 +231,11 @@ async function loadStaticFallback(loadedAvatarUrl) {
|
|
|
228
231
|
}
|
|
229
232
|
}, (err) => {
|
|
230
233
|
log('Fallback Error: ' + err.message);
|
|
231
|
-
|
|
234
|
+
rnPost(JSON.stringify({ type: 'error', message: err.message }));
|
|
232
235
|
});
|
|
233
236
|
} catch (err) {
|
|
234
237
|
log('Fallback setup error: ' + err.message);
|
|
235
|
-
|
|
238
|
+
rnPost(JSON.stringify({ type: 'error', message: err.message }));
|
|
236
239
|
}
|
|
237
240
|
}
|
|
238
241
|
|
|
@@ -299,7 +302,7 @@ async function init() {
|
|
|
299
302
|
jawMorphCache = null;
|
|
300
303
|
visemeMorphCache = null;
|
|
301
304
|
rhubarbMorphCache = null;
|
|
302
|
-
head.armature
|
|
305
|
+
head.armature && head.armature.traverse(function(child) {
|
|
303
306
|
if (child.isMesh && child.morphTargetDictionary && child.morphTargetInfluences) {
|
|
304
307
|
mouthMeshes.push(child);
|
|
305
308
|
}
|
|
@@ -338,17 +341,17 @@ async function init() {
|
|
|
338
341
|
applyAccessories(pendingAccessoriesList);
|
|
339
342
|
if (head && head.armature) motionInitFromRoot(head.armature);
|
|
340
343
|
emitLoading('ready', 100);
|
|
341
|
-
|
|
344
|
+
rnPost(JSON.stringify({ type: 'ready' }));
|
|
342
345
|
}
|
|
343
346
|
} catch (err) {
|
|
344
347
|
initStarted = false; // allow retry on error
|
|
345
348
|
log('Error: ' + err.message);
|
|
346
|
-
|
|
349
|
+
rnPost(JSON.stringify({ type: 'error', message: err.message }));
|
|
347
350
|
}
|
|
348
351
|
}
|
|
349
352
|
|
|
350
353
|
function startAudioInterception() {
|
|
351
|
-
if (!head
|
|
354
|
+
if (!head || !head.audioCtx || !head.audioSpeechGainNode) return;
|
|
352
355
|
const audioCtx = head.audioCtx;
|
|
353
356
|
const gainNode = head.audioSpeechGainNode;
|
|
354
357
|
const connected = new WeakSet();
|
|
@@ -400,14 +403,14 @@ function updateVoiceMood(rawAmplitude) {
|
|
|
400
403
|
voiceMoodFramesBelow = 0;
|
|
401
404
|
if (voiceMoodCurrent !== 'excited' && voiceMoodFramesAbove >= VOICE_MOOD_FRAMES_REQUIRED) {
|
|
402
405
|
voiceMoodCurrent = 'excited';
|
|
403
|
-
|
|
406
|
+
rnPost(JSON.stringify({ type: 'voiceMood', mood: 'excited' }));
|
|
404
407
|
}
|
|
405
408
|
} else if (voiceEnergySmoothed < VOICE_ENERGY_NEUTRAL_THRESH) {
|
|
406
409
|
voiceMoodFramesBelow++;
|
|
407
410
|
voiceMoodFramesAbove = 0;
|
|
408
411
|
if (voiceMoodCurrent !== 'neutral' && voiceMoodFramesBelow >= VOICE_MOOD_FRAMES_REQUIRED * 3) {
|
|
409
412
|
voiceMoodCurrent = 'neutral';
|
|
410
|
-
|
|
413
|
+
rnPost(JSON.stringify({ type: 'voiceMood', mood: 'neutral' }));
|
|
411
414
|
}
|
|
412
415
|
}
|
|
413
416
|
}
|
|
@@ -578,7 +581,7 @@ function clearScheduledVisemes() {
|
|
|
578
581
|
for (const key of Object.keys(visemeState)) visemeState[key] = 0;
|
|
579
582
|
}
|
|
580
583
|
|
|
581
|
-
function tickVisemeDecay(deltaSeconds
|
|
584
|
+
function tickVisemeDecay(deltaSeconds) {
|
|
582
585
|
if (!visemeMorphCache) return;
|
|
583
586
|
|
|
584
587
|
const isScheduled = Date.now() < visemeModeUntil;
|
|
@@ -614,7 +617,7 @@ function tickVisemeDecay(deltaSeconds?: number) {
|
|
|
614
617
|
// Use realtime (not newvalue) — newvalue is consumed and cleared after
|
|
615
618
|
// a single frame, so scheduled visemes would vanish immediately.
|
|
616
619
|
// realtime persists until explicitly set to null.
|
|
617
|
-
if (head
|
|
620
|
+
if (head && head.mtAvatar && e.morphName && head.mtAvatar[e.morphName]) {
|
|
618
621
|
const mt = head.mtAvatar[e.morphName];
|
|
619
622
|
mt.realtime = targetWeight > 0 ? targetWeight : null;
|
|
620
623
|
mt.needsUpdate = true;
|
|
@@ -750,7 +753,7 @@ async function ensureThree() {
|
|
|
750
753
|
const { DRACOLoader } = await import('three/addons/loaders/DRACOLoader.js');
|
|
751
754
|
gltfLoaderInstance = new GLTFLoader();
|
|
752
755
|
const dracoLoader = new DRACOLoader();
|
|
753
|
-
dracoLoader.setDecoderPath(
|
|
756
|
+
dracoLoader.setDecoderPath(DRACO_URL);
|
|
754
757
|
gltfLoaderInstance.setDRACOLoader(dracoLoader);
|
|
755
758
|
}
|
|
756
759
|
}
|
|
@@ -798,7 +801,7 @@ async function applyAccessories(accessoriesList) {
|
|
|
798
801
|
if (loadedUrl.startsWith('blob:')) URL.revokeObjectURL(loadedUrl);
|
|
799
802
|
log('[ACC] GLB loaded OK for ' + accData.id);
|
|
800
803
|
const model = gltf.scene;
|
|
801
|
-
const latestData = currentAccessories[accData.id]
|
|
804
|
+
const latestData = (currentAccessories[accData.id] && currentAccessories[accData.id].latestData) || accData;
|
|
802
805
|
let targetBone = null;
|
|
803
806
|
let prefixCandidate = null;
|
|
804
807
|
root.traverse((child) => {
|
|
@@ -1158,7 +1161,7 @@ function onIncomingMessage(event) {
|
|
|
1158
1161
|
if (typeof window.playMotion === 'function') {
|
|
1159
1162
|
window.playMotion(msg.name);
|
|
1160
1163
|
}
|
|
1161
|
-
|
|
1164
|
+
rnPost(JSON.stringify({ type: 'avatarState', state: 'motion:' + msg.name }));
|
|
1162
1165
|
log('motion dispatched: ' + msg.name);
|
|
1163
1166
|
}
|
|
1164
1167
|
} catch (err) {
|
|
@@ -180,7 +180,7 @@ function sleep(ms, signal) {
|
|
|
180
180
|
const id = setTimeout(resolve, ms);
|
|
181
181
|
signal.addEventListener("abort", () => {
|
|
182
182
|
clearTimeout(id);
|
|
183
|
-
reject(
|
|
183
|
+
reject(new DOMException("The operation was aborted", "AbortError"));
|
|
184
184
|
}, { once: true });
|
|
185
185
|
});
|
|
186
186
|
}
|
package/dist/wgpu/WgpuAvatar.js
CHANGED
|
@@ -178,6 +178,11 @@ exports.WgpuAvatar = (0, react_1.forwardRef)(({ style, avatarUrl, focalLength, m
|
|
|
178
178
|
? avatarUrl : null;
|
|
179
179
|
const fileResult = (0, useAuthedModelUri_1.useAuthedModelUri)(remoteUrl);
|
|
180
180
|
const lockedUriRef = (0, react_1.useRef)(null);
|
|
181
|
+
const lastUrlRef = (0, react_1.useRef)(avatarUrl);
|
|
182
|
+
if (lastUrlRef.current !== avatarUrl) {
|
|
183
|
+
lastUrlRef.current = avatarUrl;
|
|
184
|
+
lockedUriRef.current = null;
|
|
185
|
+
}
|
|
181
186
|
const currentUri = fileResult?.uri ?? null;
|
|
182
187
|
if (lockedUriRef.current === null && currentUri)
|
|
183
188
|
lockedUriRef.current = currentUri;
|
|
@@ -232,7 +237,7 @@ exports.WgpuAvatar = (0, react_1.forwardRef)(({ style, avatarUrl, focalLength, m
|
|
|
232
237
|
(0, react_1.useEffect)(() => () => clearScheduleTimers(), [clearScheduleTimers]);
|
|
233
238
|
(0, react_1.useImperativeHandle)(ref, () => ({
|
|
234
239
|
setMood: (m) => { morphStateRef.current.moodBase = { ...(morphTables_1.MOOD_MORPHS[m] ?? {}) }; },
|
|
235
|
-
sendAmplitude: (
|
|
240
|
+
sendAmplitude: () => { },
|
|
236
241
|
sendViseme: (viseme, weight) => {
|
|
237
242
|
const aliases = morphTables_1.VISEME_MORPH_ALIASES[viseme];
|
|
238
243
|
if (!aliases)
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "talking-head-studio",
|
|
3
|
-
"version": "0.4.
|
|
3
|
+
"version": "0.4.9",
|
|
4
4
|
"description": "Cross-platform 3D avatar component for React Native & web — lip-sync, gestures, accessories, and LLM integration. Powered by TalkingHead + Three.js.",
|
|
5
5
|
"main": "dist/index.web.js",
|
|
6
6
|
"browser": "dist/index.web.js",
|
|
@@ -103,6 +103,7 @@
|
|
|
103
103
|
"@react-three/fiber": ">=8",
|
|
104
104
|
"expo": ">=51",
|
|
105
105
|
"expo-asset": ">=10",
|
|
106
|
+
"expo-audio": ">=0.3",
|
|
106
107
|
"expo-file-system": ">=17",
|
|
107
108
|
"react": ">=18",
|
|
108
109
|
"react-native": ">=0.73",
|
|
@@ -126,6 +127,9 @@
|
|
|
126
127
|
"expo-asset": {
|
|
127
128
|
"optional": true
|
|
128
129
|
},
|
|
130
|
+
"expo-audio": {
|
|
131
|
+
"optional": true
|
|
132
|
+
},
|
|
129
133
|
"expo-file-system": {
|
|
130
134
|
"optional": true
|
|
131
135
|
},
|