talking-head-studio 0.4.8 → 0.4.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -7
- package/dist/TalkingHead.web.js +18 -8
- package/dist/editor/FaceSqueezeEditor.d.ts +17 -0
- package/dist/editor/FaceSqueezeEditor.js +553 -0
- package/dist/editor/FaceSqueezeEditor.web.d.ts +6 -0
- package/dist/editor/FaceSqueezeEditor.web.js +54 -0
- package/dist/editor/index.d.ts +2 -1
- package/dist/editor/index.js +4 -1
- package/dist/editor/types.d.ts +3 -0
- package/dist/html.d.ts +5 -0
- package/dist/html.js +3 -1
- package/dist/tts/useDirectVisemeStream.js +1 -1
- package/dist/wgpu/WgpuAvatar.js +5 -0
- package/package.json +5 -1
package/README.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# talking-head-studio
|
|
2
2
|
|
|
3
|
-
**
|
|
3
|
+
**The missing UI layer for AI Agents. Drop-in, lip-syncing 3D avatars for Web, React, and React Native.**
|
|
4
4
|
|
|
5
5
|
[](https://www.npmjs.com/package/talking-head-studio)
|
|
6
6
|
[](https://opensource.org/licenses/MIT)
|
|
@@ -10,10 +10,11 @@
|
|
|
10
10
|
|
|
11
11
|
## Why this?
|
|
12
12
|
|
|
13
|
-
- **
|
|
14
|
-
- **Bring any GLB
|
|
15
|
-
- **Built for
|
|
16
|
-
- **
|
|
13
|
+
- **Zero-Jank React Native & Web:** True cross-platform rendering. React Native gets a blazing fast wgpu-accelerated native render loop, skipping WebView bridge latency entirely. React on web gets a robust `react-three-fiber` setup. Same API, same props.
|
|
14
|
+
- **Universal GLB Compatibility:** Bring any GLB. Out-of-the-box support for standard ARKit blendshapes. Rigged models get full phoneme-based lip-sync. Non-rigged models get an amplitude-driven jaw animation fallback.
|
|
15
|
+
- **Built for AI & Voice Pipelines:** Wire `sendAmplitude` or visemes directly to LiveKit, Web Audio, ElevenLabs, OpenAI Realtime, or any audio source.
|
|
16
|
+
- **Always Alive:** Procedural idle animations (breathing, nodding, swaying) keep your avatar from feeling like a static doll.
|
|
17
|
+
- **Dynamic Wardrobe & Accessories:** Swap hair, skin, and eye colors on the fly. Attach hats, glasses, or backpacks to any bone at runtime.
|
|
17
18
|
|
|
18
19
|
---
|
|
19
20
|
|
|
@@ -68,7 +69,7 @@ export default function Avatar() {
|
|
|
68
69
|
return (
|
|
69
70
|
<TalkingHead
|
|
70
71
|
ref={ref}
|
|
71
|
-
avatarUrl="https://
|
|
72
|
+
avatarUrl="https://example.com/your-model.glb"
|
|
72
73
|
mood="happy"
|
|
73
74
|
cameraView="upper"
|
|
74
75
|
hairColor="#1a1a2e"
|
|
@@ -378,7 +379,7 @@ For the complete experience -- phoneme lip-sync, expressions, moods, gestures --
|
|
|
378
379
|
- **ARKit blend shapes** and/or **Oculus viseme blend shapes** for lip-sync
|
|
379
380
|
- Standard Three.js-compatible GLB format
|
|
380
381
|
|
|
381
|
-
Models from [
|
|
382
|
+
Models from [Avaturn](https://avaturn.me/) or any Mixamo-rigged source work out of the box.
|
|
382
383
|
|
|
383
384
|
### Non-rigged models (static fallback)
|
|
384
385
|
|
|
@@ -457,4 +458,11 @@ This project builds on excellent open-source work:
|
|
|
457
458
|
|
|
458
459
|
## License
|
|
459
460
|
|
|
461
|
+
MIT
|
|
462
|
+
at runtime.
|
|
463
|
+
|
|
464
|
+
---
|
|
465
|
+
|
|
466
|
+
## License
|
|
467
|
+
|
|
460
468
|
MIT
|
package/dist/TalkingHead.web.js
CHANGED
|
@@ -46,14 +46,24 @@ exports.TalkingHead = (0, react_1.forwardRef)(({ avatarUrl, authToken, mood = 'n
|
|
|
46
46
|
const pendingSkinColorRef = (0, react_1.useRef)(skinColor);
|
|
47
47
|
const pendingEyeColorRef = (0, react_1.useRef)(eyeColor);
|
|
48
48
|
const accessoriesRef = (0, react_1.useRef)(accessories);
|
|
49
|
+
const onLoadingChangeRef = (0, react_1.useRef)(onLoadingChange);
|
|
50
|
+
const onReadyRef = (0, react_1.useRef)(onReady);
|
|
51
|
+
const onErrorRef = (0, react_1.useRef)(onError);
|
|
52
|
+
const onAvatarStateRef = (0, react_1.useRef)(onAvatarState);
|
|
53
|
+
(0, react_1.useEffect)(() => {
|
|
54
|
+
onLoadingChangeRef.current = onLoadingChange;
|
|
55
|
+
onReadyRef.current = onReady;
|
|
56
|
+
onErrorRef.current = onError;
|
|
57
|
+
onAvatarStateRef.current = onAvatarState;
|
|
58
|
+
});
|
|
49
59
|
(0, react_1.useEffect)(() => {
|
|
50
60
|
accessoriesRef.current = accessories;
|
|
51
61
|
}, [accessories]);
|
|
52
62
|
(0, react_1.useEffect)(() => {
|
|
53
63
|
if (!avatarUrl)
|
|
54
64
|
return;
|
|
55
|
-
|
|
56
|
-
}, [avatarUrl, authToken
|
|
65
|
+
onLoadingChangeRef.current?.({ stage: 'booting', progress: null });
|
|
66
|
+
}, [avatarUrl, authToken]);
|
|
57
67
|
const post = (0, react_1.useCallback)((msg) => {
|
|
58
68
|
iframeRef.current?.contentWindow?.postMessage(JSON.stringify(msg), '*');
|
|
59
69
|
}, []);
|
|
@@ -148,7 +158,7 @@ exports.TalkingHead = (0, react_1.forwardRef)(({ avatarUrl, authToken, mood = 'n
|
|
|
148
158
|
try {
|
|
149
159
|
const msg = typeof event.data === 'string' ? JSON.parse(event.data) : event.data;
|
|
150
160
|
if (msg.type === 'loading' && typeof msg.stage === 'string') {
|
|
151
|
-
|
|
161
|
+
onLoadingChangeRef.current?.({
|
|
152
162
|
stage: msg.stage,
|
|
153
163
|
progress: typeof msg.progress === 'number' && Number.isFinite(msg.progress)
|
|
154
164
|
? msg.progress
|
|
@@ -158,7 +168,7 @@ exports.TalkingHead = (0, react_1.forwardRef)(({ avatarUrl, authToken, mood = 'n
|
|
|
158
168
|
}
|
|
159
169
|
if (msg.type === 'ready') {
|
|
160
170
|
readyRef.current = true;
|
|
161
|
-
|
|
171
|
+
onLoadingChangeRef.current?.({ stage: 'ready', progress: 100 });
|
|
162
172
|
if (pendingMoodRef.current) {
|
|
163
173
|
post({ type: 'mood', value: pendingMoodRef.current });
|
|
164
174
|
}
|
|
@@ -174,13 +184,13 @@ exports.TalkingHead = (0, react_1.forwardRef)(({ avatarUrl, authToken, mood = 'n
|
|
|
174
184
|
if (accessoriesRef.current?.length) {
|
|
175
185
|
post({ type: 'set_accessories', accessories: accessoriesRef.current });
|
|
176
186
|
}
|
|
177
|
-
|
|
187
|
+
onReadyRef.current?.();
|
|
178
188
|
}
|
|
179
189
|
else if (msg.type === 'error') {
|
|
180
|
-
|
|
190
|
+
onErrorRef.current?.(msg.message);
|
|
181
191
|
}
|
|
182
192
|
else if (msg.type === 'avatarState') {
|
|
183
|
-
|
|
193
|
+
onAvatarStateRef.current?.(msg.state);
|
|
184
194
|
}
|
|
185
195
|
else if (msg.type === 'log') {
|
|
186
196
|
console.log('[TalkingHead]', msg.message);
|
|
@@ -192,7 +202,7 @@ exports.TalkingHead = (0, react_1.forwardRef)(({ avatarUrl, authToken, mood = 'n
|
|
|
192
202
|
};
|
|
193
203
|
window.addEventListener('message', onMessage);
|
|
194
204
|
return () => window.removeEventListener('message', onMessage);
|
|
195
|
-
}, [
|
|
205
|
+
}, [post]);
|
|
196
206
|
return ((0, jsx_runtime_1.jsx)("div", { style: { ...containerStyle, ...style }, children: (0, jsx_runtime_1.jsx)("iframe", { ref: iframeRef, srcDoc: srcdoc, style: iframeStyle, sandbox: "allow-scripts allow-same-origin", title: "TalkingHead Avatar" }) }));
|
|
197
207
|
});
|
|
198
208
|
exports.TalkingHead.displayName = 'TalkingHead';
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* WgpuFaceSqueezeEditor — R3F/Three.js port of FaceSqueezeEditor.
|
|
3
|
+
*
|
|
4
|
+
* Same gesture zones, spring physics, idle emotions, lip sync, and sound
|
|
5
|
+
* as the Filament version. Only the render scaffolding changes:
|
|
6
|
+
*
|
|
7
|
+
* FilamentScene + FilamentView + renderableManager.setMorphWeights
|
|
8
|
+
* → @react-three/fiber Canvas + mesh.morphTargetInfluences[]
|
|
9
|
+
*
|
|
10
|
+
* Drop-in replacement: same FaceSqueezeEditorProps interface.
|
|
11
|
+
* The .web.tsx stub is shared — import from FaceSqueezeEditor.web.tsx.
|
|
12
|
+
*/
|
|
13
|
+
export declare const FACE_SQUEEZE_LOCAL_MODULE: number;
|
|
14
|
+
export interface FaceSqueezeEditorProps {
|
|
15
|
+
onClose: () => void;
|
|
16
|
+
}
|
|
17
|
+
export declare function FaceSqueezeEditor({ onClose }: FaceSqueezeEditorProps): import("react/jsx-runtime").JSX.Element;
|
|
@@ -0,0 +1,553 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || function (mod) {
|
|
19
|
+
if (mod && mod.__esModule) return mod;
|
|
20
|
+
var result = {};
|
|
21
|
+
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
22
|
+
__setModuleDefault(result, mod);
|
|
23
|
+
return result;
|
|
24
|
+
};
|
|
25
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
26
|
+
exports.FaceSqueezeEditor = exports.FACE_SQUEEZE_LOCAL_MODULE = void 0;
|
|
27
|
+
const jsx_runtime_1 = require("react/jsx-runtime");
|
|
28
|
+
/**
|
|
29
|
+
* WgpuFaceSqueezeEditor — R3F/Three.js port of FaceSqueezeEditor.
|
|
30
|
+
*
|
|
31
|
+
* Same gesture zones, spring physics, idle emotions, lip sync, and sound
|
|
32
|
+
* as the Filament version. Only the render scaffolding changes:
|
|
33
|
+
*
|
|
34
|
+
* FilamentScene + FilamentView + renderableManager.setMorphWeights
|
|
35
|
+
* → @react-three/fiber Canvas + mesh.morphTargetInfluences[]
|
|
36
|
+
*
|
|
37
|
+
* Drop-in replacement: same FaceSqueezeEditorProps interface.
|
|
38
|
+
* The .web.tsx stub is shared — import from FaceSqueezeEditor.web.tsx.
|
|
39
|
+
*/
|
|
40
|
+
const react_1 = require("react");
|
|
41
|
+
const react_native_1 = require("react-native");
|
|
42
|
+
const native_1 = require("@react-three/fiber/native");
|
|
43
|
+
const native_2 = require("@react-three/drei/native");
|
|
44
|
+
const THREE = __importStar(require("three"));
|
|
45
|
+
const react_native_gesture_handler_1 = require("react-native-gesture-handler");
|
|
46
|
+
const react_native_reanimated_1 = require("react-native-reanimated");
|
|
47
|
+
const Haptics = __importStar(require("expo-haptics"));
|
|
48
|
+
const expo_audio_1 = require("expo-audio");
|
|
49
|
+
const expo_asset_1 = require("expo-asset");
|
|
50
|
+
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
|
51
|
+
exports.FACE_SQUEEZE_LOCAL_MODULE = require('../assets/face-squeeze-local.glb');
|
|
52
|
+
// ---------------------------------------------------------------------------
|
|
53
|
+
// Constants (identical to Filament version)
|
|
54
|
+
// ---------------------------------------------------------------------------
|
|
55
|
+
const SPRING_DURATION_MS = 300;
|
|
56
|
+
const EMOTION_HOLD_MS = 900;
|
|
57
|
+
const IDLE_EMOTION_INTERVAL_MS = 30000;
|
|
58
|
+
const INITIAL_IDLE_EMOTION_DELAY_MS = 1200;
|
|
59
|
+
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
|
60
|
+
const SOUND_OWIE = require('./sounds/owie.wav');
|
|
61
|
+
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
|
62
|
+
const SOUND_STOP = require('./sounds/stop.wav');
|
|
63
|
+
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
|
64
|
+
const SOUND_HAHA = require('./sounds/haha.wav');
|
|
65
|
+
const REACTION_SOUNDS = [
|
|
66
|
+
SOUND_OWIE,
|
|
67
|
+
SOUND_STOP,
|
|
68
|
+
SOUND_HAHA,
|
|
69
|
+
];
|
|
70
|
+
const EMOTION_PRESETS = {
|
|
71
|
+
owie: {
|
|
72
|
+
eyeBlinkLeft: 1.0, eyeBlinkRight: 1.0,
|
|
73
|
+
eyeSquintLeft: 0.9, eyeSquintRight: 0.9,
|
|
74
|
+
browDownLeft: 0.8, browDownRight: 0.8,
|
|
75
|
+
browInnerUp: 0.6,
|
|
76
|
+
noseSneerLeft: 0.7, noseSneerRight: 0.7,
|
|
77
|
+
jawOpen: 0.35,
|
|
78
|
+
mouthStretchLeft: 0.6, mouthStretchRight: 0.6,
|
|
79
|
+
},
|
|
80
|
+
happy: {
|
|
81
|
+
mouthSmileLeft: 0.85, mouthSmileRight: 0.85,
|
|
82
|
+
cheekSquintLeft: 0.6, cheekSquintRight: 0.6,
|
|
83
|
+
mouthDimpleLeft: 0.4, mouthDimpleRight: 0.4,
|
|
84
|
+
},
|
|
85
|
+
surprised: {
|
|
86
|
+
eyeWideLeft: 0.9, eyeWideRight: 0.9,
|
|
87
|
+
browInnerUp: 0.8,
|
|
88
|
+
browOuterUpLeft: 0.7, browOuterUpRight: 0.7,
|
|
89
|
+
jawOpen: 0.4, mouthFunnel: 0.3,
|
|
90
|
+
},
|
|
91
|
+
angry: {
|
|
92
|
+
browDownLeft: 0.9, browDownRight: 0.9,
|
|
93
|
+
noseSneerLeft: 0.7, noseSneerRight: 0.7,
|
|
94
|
+
eyeSquintLeft: 0.6, eyeSquintRight: 0.6,
|
|
95
|
+
mouthPressLeft: 0.5, mouthPressRight: 0.5,
|
|
96
|
+
},
|
|
97
|
+
smirk: {
|
|
98
|
+
mouthSmileLeft: 0.9, mouthSmileRight: 0.1,
|
|
99
|
+
cheekSquintLeft: 0.5, mouthDimpleLeft: 0.35,
|
|
100
|
+
},
|
|
101
|
+
sad: {
|
|
102
|
+
browInnerUp: 0.9,
|
|
103
|
+
browDownLeft: 0.3, browDownRight: 0.3,
|
|
104
|
+
mouthFrownLeft: 0.8, mouthFrownRight: 0.8,
|
|
105
|
+
mouthPressLeft: 0.3, mouthPressRight: 0.3,
|
|
106
|
+
eyeLookDownLeft: 0.5, eyeLookDownRight: 0.5,
|
|
107
|
+
},
|
|
108
|
+
disgust: {
|
|
109
|
+
noseSneerLeft: 0.9, noseSneerRight: 0.9,
|
|
110
|
+
browDownLeft: 0.7, browDownRight: 0.7,
|
|
111
|
+
mouthShrugUpper: 0.6, mouthShrugLower: 0.4,
|
|
112
|
+
mouthLeft: 0.3,
|
|
113
|
+
eyeSquintLeft: 0.4, eyeSquintRight: 0.4,
|
|
114
|
+
},
|
|
115
|
+
wink: {
|
|
116
|
+
eyeBlinkLeft: 1.0,
|
|
117
|
+
mouthSmileLeft: 0.5, mouthSmileRight: 0.3,
|
|
118
|
+
cheekSquintLeft: 0.4,
|
|
119
|
+
},
|
|
120
|
+
sleepy: {
|
|
121
|
+
eyeBlinkLeft: 0.7, eyeBlinkRight: 0.7,
|
|
122
|
+
browDownLeft: 0.4, browDownRight: 0.4,
|
|
123
|
+
mouthShrugLower: 0.3, jawOpen: 0.15,
|
|
124
|
+
},
|
|
125
|
+
contempt: {
|
|
126
|
+
mouthSmileLeft: 0.6, mouthPressRight: 0.5,
|
|
127
|
+
mouthStretchRight: 0.3, browDownRight: 0.4,
|
|
128
|
+
eyeSquintRight: 0.3,
|
|
129
|
+
},
|
|
130
|
+
};
|
|
131
|
+
const IDLE_EMOTIONS = [
|
|
132
|
+
{ key: 'happy', weight: 5 },
|
|
133
|
+
{ key: 'surprised', weight: 2 },
|
|
134
|
+
{ key: 'smirk', weight: 2 },
|
|
135
|
+
{ key: 'sad', weight: 1 },
|
|
136
|
+
{ key: 'wink', weight: 2 },
|
|
137
|
+
{ key: 'sleepy', weight: 1 },
|
|
138
|
+
{ key: 'contempt', weight: 1 },
|
|
139
|
+
];
|
|
140
|
+
function pickIdleEmotion() {
|
|
141
|
+
const total = IDLE_EMOTIONS.reduce((s, e) => s + e.weight, 0);
|
|
142
|
+
let r = Math.random() * total;
|
|
143
|
+
for (const e of IDLE_EMOTIONS) {
|
|
144
|
+
r -= e.weight;
|
|
145
|
+
if (r <= 0)
|
|
146
|
+
return e.key;
|
|
147
|
+
}
|
|
148
|
+
return IDLE_EMOTIONS[0].key;
|
|
149
|
+
}
|
|
150
|
+
// ---------------------------------------------------------------------------
|
|
151
|
+
// Inner R3F scene — runs inside Canvas
|
|
152
|
+
// ---------------------------------------------------------------------------
|
|
153
|
+
function AvatarScene({ uri, morphState, onReady, }) {
|
|
154
|
+
const { camera } = (0, native_1.useThree)();
|
|
155
|
+
const gltf = (0, native_2.useGLTF)(uri);
|
|
156
|
+
const readyFiredRef = (0, react_1.useRef)(false);
|
|
157
|
+
// Position camera to match Filament version
|
|
158
|
+
(0, react_1.useLayoutEffect)(() => {
|
|
159
|
+
if (!(camera instanceof THREE.PerspectiveCamera))
|
|
160
|
+
return;
|
|
161
|
+
camera.fov = 34;
|
|
162
|
+
camera.position.set(0, 1.55, 1.05);
|
|
163
|
+
camera.lookAt(0, 1.55, 0);
|
|
164
|
+
camera.updateProjectionMatrix();
|
|
165
|
+
}, [camera]);
|
|
166
|
+
// Find best mesh and build morph index
|
|
167
|
+
(0, react_1.useEffect)(() => {
|
|
168
|
+
if (!gltf?.scene)
|
|
169
|
+
return;
|
|
170
|
+
let best = null;
|
|
171
|
+
let bestCount = 0;
|
|
172
|
+
gltf.scene.traverse((node) => {
|
|
173
|
+
if (!(node instanceof THREE.Mesh))
|
|
174
|
+
return;
|
|
175
|
+
const count = Object.keys(node.morphTargetDictionary ?? {}).length;
|
|
176
|
+
if (count > bestCount) {
|
|
177
|
+
bestCount = count;
|
|
178
|
+
best = node;
|
|
179
|
+
}
|
|
180
|
+
});
|
|
181
|
+
if (!best)
|
|
182
|
+
return;
|
|
183
|
+
const mesh = best;
|
|
184
|
+
morphState.current.mesh = mesh;
|
|
185
|
+
const idx = new Map();
|
|
186
|
+
for (const [name, i] of Object.entries(mesh.morphTargetDictionary ?? {})) {
|
|
187
|
+
idx.set(name, i);
|
|
188
|
+
idx.set(name.toLowerCase(), i);
|
|
189
|
+
}
|
|
190
|
+
morphState.current.index = idx;
|
|
191
|
+
if (!readyFiredRef.current) {
|
|
192
|
+
readyFiredRef.current = true;
|
|
193
|
+
onReady();
|
|
194
|
+
}
|
|
195
|
+
}, [gltf, morphState, onReady]);
|
|
196
|
+
// Apply morphState.weights to mesh every frame
|
|
197
|
+
(0, native_1.useFrame)(() => {
|
|
198
|
+
const { mesh, weights, index } = morphState.current;
|
|
199
|
+
if (!mesh?.morphTargetInfluences)
|
|
200
|
+
return;
|
|
201
|
+
for (const [name, w] of Object.entries(weights)) {
|
|
202
|
+
const i = index.get(name) ?? index.get(name.toLowerCase());
|
|
203
|
+
if (i !== undefined)
|
|
204
|
+
mesh.morphTargetInfluences[i] = w;
|
|
205
|
+
}
|
|
206
|
+
});
|
|
207
|
+
return ((0, jsx_runtime_1.jsxs)(jsx_runtime_1.Fragment, { children: [(0, jsx_runtime_1.jsx)("ambientLight", { intensity: 0.65 }), (0, jsx_runtime_1.jsx)("directionalLight", { position: [1, 3, 2], intensity: 1.3 }), (0, jsx_runtime_1.jsx)("primitive", { object: gltf.scene })] }));
|
|
208
|
+
}
|
|
209
|
+
// ---------------------------------------------------------------------------
|
|
210
|
+
// Main component
|
|
211
|
+
// ---------------------------------------------------------------------------
|
|
212
|
+
function FaceSqueezeEditor({ onClose }) {
|
|
213
|
+
const [localUri, setLocalUri] = (0, react_1.useState)(null);
|
|
214
|
+
const [isReady, setIsReady] = (0, react_1.useState)(false);
|
|
215
|
+
(0, react_1.useEffect)(() => {
|
|
216
|
+
let cancelled = false;
|
|
217
|
+
(async () => {
|
|
218
|
+
try {
|
|
219
|
+
const asset = expo_asset_1.Asset.fromModule(exports.FACE_SQUEEZE_LOCAL_MODULE);
|
|
220
|
+
await asset.downloadAsync();
|
|
221
|
+
const uri = asset.localUri ?? asset.uri;
|
|
222
|
+
if (!cancelled && uri)
|
|
223
|
+
setLocalUri(uri);
|
|
224
|
+
}
|
|
225
|
+
catch (e) {
|
|
226
|
+
console.error('[WgpuFaceSqueezeEditor] Failed to load local GLB:', e);
|
|
227
|
+
}
|
|
228
|
+
})();
|
|
229
|
+
return () => { cancelled = true; };
|
|
230
|
+
}, []);
|
|
231
|
+
// Shared morph state — mutated directly, never triggers re-render
|
|
232
|
+
const morphState = (0, react_1.useRef)({
|
|
233
|
+
weights: {},
|
|
234
|
+
index: new Map(),
|
|
235
|
+
mesh: null,
|
|
236
|
+
});
|
|
237
|
+
// ---- Audio ----
|
|
238
|
+
const reactionSoundIdx = (0, react_1.useRef)(0);
|
|
239
|
+
const player = (0, expo_audio_1.useAudioPlayer)(REACTION_SOUNDS[0]);
|
|
240
|
+
// ---- Timers ----
|
|
241
|
+
const springRef = (0, react_1.useRef)(null);
|
|
242
|
+
const emotionHoldTimerRef = (0, react_1.useRef)(null);
|
|
243
|
+
const idleEmotionTimerRef = (0, react_1.useRef)(null);
|
|
244
|
+
const lipSyncIntervalRef = (0, react_1.useRef)(null);
|
|
245
|
+
// ---- View geometry (for gesture zone math) ----
|
|
246
|
+
const viewSize = (0, react_1.useRef)({ width: 1, height: 1 });
|
|
247
|
+
const viewWidthSv = (0, react_native_reanimated_1.useSharedValue)(1);
|
|
248
|
+
const viewHeightSv = (0, react_native_reanimated_1.useSharedValue)(1);
|
|
249
|
+
const touchStartXSv = (0, react_native_reanimated_1.useSharedValue)(0);
|
|
250
|
+
const touchStartYSv = (0, react_native_reanimated_1.useSharedValue)(0);
|
|
251
|
+
const touchZoneSv = (0, react_native_reanimated_1.useSharedValue)('cheek_left');
|
|
252
|
+
const lastPanUpdateTsSv = (0, react_native_reanimated_1.useSharedValue)(0);
|
|
253
|
+
const onLayout = (0, react_1.useCallback)((e) => {
|
|
254
|
+
const w = Math.max(1, e.nativeEvent.layout.width);
|
|
255
|
+
const h = Math.max(1, e.nativeEvent.layout.height);
|
|
256
|
+
viewSize.current = { width: w, height: h };
|
|
257
|
+
viewWidthSv.value = w;
|
|
258
|
+
viewHeightSv.value = h;
|
|
259
|
+
}, [viewHeightSv, viewWidthSv]);
|
|
260
|
+
// ---- Weight helpers ----
|
|
261
|
+
const applyWeights = (0, react_1.useCallback)((targets) => {
|
|
262
|
+
for (const [name, w] of Object.entries(targets)) {
|
|
263
|
+
morphState.current.weights[name] = Math.max(0, Math.min(1, w));
|
|
264
|
+
}
|
|
265
|
+
}, []);
|
|
266
|
+
const stopSpring = (0, react_1.useCallback)(() => {
|
|
267
|
+
if (springRef.current !== null) {
|
|
268
|
+
cancelAnimationFrame(springRef.current);
|
|
269
|
+
springRef.current = null;
|
|
270
|
+
}
|
|
271
|
+
}, []);
|
|
272
|
+
const springToZero = (0, react_1.useCallback)(() => {
|
|
273
|
+
stopSpring();
|
|
274
|
+
const snapshot = { ...morphState.current.weights };
|
|
275
|
+
const startTime = Date.now();
|
|
276
|
+
const tick = () => {
|
|
277
|
+
const t = Math.min(1, (Date.now() - startTime) / SPRING_DURATION_MS);
|
|
278
|
+
const eased = 1 - Math.pow(1 - t, 3);
|
|
279
|
+
for (const name of Object.keys(snapshot)) {
|
|
280
|
+
morphState.current.weights[name] = snapshot[name] * (1 - eased);
|
|
281
|
+
}
|
|
282
|
+
if (t >= 1) {
|
|
283
|
+
springRef.current = null;
|
|
284
|
+
morphState.current.weights = {};
|
|
285
|
+
return;
|
|
286
|
+
}
|
|
287
|
+
springRef.current = requestAnimationFrame(tick);
|
|
288
|
+
};
|
|
289
|
+
springRef.current = requestAnimationFrame(tick);
|
|
290
|
+
}, [stopSpring]);
|
|
291
|
+
const playEmotionBurst = (0, react_1.useCallback)((emotionKey) => {
|
|
292
|
+
const preset = EMOTION_PRESETS[emotionKey];
|
|
293
|
+
if (!preset)
|
|
294
|
+
return;
|
|
295
|
+
if (emotionHoldTimerRef.current !== null) {
|
|
296
|
+
clearTimeout(emotionHoldTimerRef.current);
|
|
297
|
+
emotionHoldTimerRef.current = null;
|
|
298
|
+
}
|
|
299
|
+
stopSpring();
|
|
300
|
+
applyWeights(preset);
|
|
301
|
+
emotionHoldTimerRef.current = setTimeout(() => {
|
|
302
|
+
emotionHoldTimerRef.current = null;
|
|
303
|
+
springToZero();
|
|
304
|
+
}, EMOTION_HOLD_MS);
|
|
305
|
+
}, [applyWeights, springToZero, stopSpring]);
|
|
306
|
+
const stopLipSync = (0, react_1.useCallback)(() => {
|
|
307
|
+
if (lipSyncIntervalRef.current !== null) {
|
|
308
|
+
clearInterval(lipSyncIntervalRef.current);
|
|
309
|
+
lipSyncIntervalRef.current = null;
|
|
310
|
+
}
|
|
311
|
+
applyWeights({ jawOpen: 0, mouthFunnel: 0, mouthPucker: 0,
|
|
312
|
+
mouthUpperUpLeft: 0, mouthUpperUpRight: 0,
|
|
313
|
+
mouthLowerDownLeft: 0, mouthLowerDownRight: 0,
|
|
314
|
+
mouthShrugLower: 0, mouthShrugUpper: 0 });
|
|
315
|
+
}, [applyWeights]);
|
|
316
|
+
const startLipSync = (0, react_1.useCallback)((durationMs) => {
|
|
317
|
+
if (lipSyncIntervalRef.current !== null)
|
|
318
|
+
clearInterval(lipSyncIntervalRef.current);
|
|
319
|
+
const startTime = Date.now();
|
|
320
|
+
const TICK_MS = 60;
|
|
321
|
+
lipSyncIntervalRef.current = setInterval(() => {
|
|
322
|
+
const elapsed = Date.now() - startTime;
|
|
323
|
+
if (elapsed >= durationMs) {
|
|
324
|
+
stopLipSync();
|
|
325
|
+
return;
|
|
326
|
+
}
|
|
327
|
+
const fadeIn = Math.min(1, elapsed / 80);
|
|
328
|
+
const fadeOut = Math.min(1, (durationMs - elapsed) / 150);
|
|
329
|
+
const envelope = Math.min(fadeIn, fadeOut);
|
|
330
|
+
const t = elapsed / 1000;
|
|
331
|
+
const jaw = envelope * (0.45 + 0.35 * Math.abs(Math.sin(t * 9.5 + Math.random() * 0.3)));
|
|
332
|
+
const funnel = envelope * Math.max(0, Math.sin(t * 7.2) * 0.5);
|
|
333
|
+
const shrug = envelope * Math.max(0, Math.sin(t * 5.8 + 1.2) * 0.35);
|
|
334
|
+
applyWeights({
|
|
335
|
+
jawOpen: jaw, mouthFunnel: funnel,
|
|
336
|
+
mouthShrugLower: shrug, mouthShrugUpper: shrug * 0.6,
|
|
337
|
+
mouthLowerDownLeft: jaw * 0.6, mouthLowerDownRight: jaw * 0.6,
|
|
338
|
+
mouthUpperUpLeft: jaw * 0.35, mouthUpperUpRight: jaw * 0.35,
|
|
339
|
+
});
|
|
340
|
+
}, TICK_MS);
|
|
341
|
+
}, [applyWeights, stopLipSync]);
|
|
342
|
+
const playReaction = (0, react_1.useCallback)((durationMs = 1400) => {
|
|
343
|
+
const idx = reactionSoundIdx.current % REACTION_SOUNDS.length;
|
|
344
|
+
reactionSoundIdx.current += 1;
|
|
345
|
+
try {
|
|
346
|
+
player.replace(REACTION_SOUNDS[idx]);
|
|
347
|
+
player.play();
|
|
348
|
+
}
|
|
349
|
+
catch { /* ignore */ }
|
|
350
|
+
startLipSync(durationMs);
|
|
351
|
+
}, [player, startLipSync]);
|
|
352
|
+
// Cleanup
|
|
353
|
+
(0, react_1.useEffect)(() => () => {
|
|
354
|
+
if (springRef.current !== null)
|
|
355
|
+
cancelAnimationFrame(springRef.current);
|
|
356
|
+
if (emotionHoldTimerRef.current !== null)
|
|
357
|
+
clearTimeout(emotionHoldTimerRef.current);
|
|
358
|
+
if (idleEmotionTimerRef.current !== null)
|
|
359
|
+
clearInterval(idleEmotionTimerRef.current);
|
|
360
|
+
if (lipSyncIntervalRef.current !== null)
|
|
361
|
+
clearInterval(lipSyncIntervalRef.current);
|
|
362
|
+
}, []);
|
|
363
|
+
// Idle emotion bursts (start once ready)
|
|
364
|
+
(0, react_1.useEffect)(() => {
|
|
365
|
+
if (!isReady)
|
|
366
|
+
return;
|
|
367
|
+
const initialKickoff = setTimeout(() => playEmotionBurst(pickIdleEmotion()), INITIAL_IDLE_EMOTION_DELAY_MS);
|
|
368
|
+
idleEmotionTimerRef.current = setInterval(() => playEmotionBurst(pickIdleEmotion()), IDLE_EMOTION_INTERVAL_MS);
|
|
369
|
+
return () => {
|
|
370
|
+
clearTimeout(initialKickoff);
|
|
371
|
+
if (idleEmotionTimerRef.current !== null)
|
|
372
|
+
clearInterval(idleEmotionTimerRef.current);
|
|
373
|
+
};
|
|
374
|
+
}, [isReady, playEmotionBurst]);
|
|
375
|
+
// ---- Gesture callbacks (JS side — called via runOnJS) ----
|
|
376
|
+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
377
|
+
const onPanBeginJs = (0, react_1.useCallback)((_x, _y, _zone) => {
|
|
378
|
+
stopSpring();
|
|
379
|
+
Haptics.impactAsync(Haptics.ImpactFeedbackStyle.Light);
|
|
380
|
+
}, [stopSpring]);
|
|
381
|
+
const onPanUpdateJs = (0, react_1.useCallback)((zone, dx, dy, isLeftHalf) => {
|
|
382
|
+
const u = {};
|
|
383
|
+
if (zone === 'brow') {
|
|
384
|
+
if (dy < 0) {
|
|
385
|
+
u['browInnerUp'] = Math.abs(dy) * 10;
|
|
386
|
+
u['browOuterUpLeft'] = Math.abs(dy) * 10;
|
|
387
|
+
u['browOuterUpRight'] = Math.abs(dy) * 10;
|
|
388
|
+
u['eyeWideLeft'] = Math.abs(dy) * 8;
|
|
389
|
+
u['eyeWideRight'] = Math.abs(dy) * 8;
|
|
390
|
+
}
|
|
391
|
+
else {
|
|
392
|
+
u['browDownLeft'] = Math.abs(dy) * 10;
|
|
393
|
+
u['browDownRight'] = Math.abs(dy) * 10;
|
|
394
|
+
u['eyeSquintLeft'] = Math.abs(dy) * 8;
|
|
395
|
+
u['eyeSquintRight'] = Math.abs(dy) * 8;
|
|
396
|
+
u['noseSneerLeft'] = Math.abs(dy) * 6;
|
|
397
|
+
u['noseSneerRight'] = Math.abs(dy) * 6;
|
|
398
|
+
}
|
|
399
|
+
u['browOuterUpLeft'] = (u['browOuterUpLeft'] ?? 0) + (dx < 0 ? Math.abs(dx) * 8 : 0);
|
|
400
|
+
u['browOuterUpRight'] = (u['browOuterUpRight'] ?? 0) + (dx > 0 ? Math.abs(dx) * 8 : 0);
|
|
401
|
+
}
|
|
402
|
+
else if (zone === 'eye') {
|
|
403
|
+
u[isLeftHalf ? 'eyeBlinkLeft' : 'eyeBlinkRight'] = dy > 0 ? Math.min(1, Math.abs(dy) * 10) : 0;
|
|
404
|
+
u[isLeftHalf ? 'eyeSquintLeft' : 'eyeSquintRight'] = dy > 0 ? Math.abs(dy) * 6 : 0;
|
|
405
|
+
u[isLeftHalf ? 'eyeWideLeft' : 'eyeWideRight'] = dy < 0 ? Math.abs(dy) * 8 : 0;
|
|
406
|
+
u[isLeftHalf ? 'eyeLookOutLeft' : 'eyeLookInRight'] = dx < 0 ? Math.abs(dx) * 7 : 0;
|
|
407
|
+
u[isLeftHalf ? 'eyeLookInLeft' : 'eyeLookOutRight'] = dx > 0 ? Math.abs(dx) * 7 : 0;
|
|
408
|
+
u[isLeftHalf ? 'eyeLookUpLeft' : 'eyeLookUpRight'] = dy < 0 ? Math.abs(dy) * 5 : 0;
|
|
409
|
+
u[isLeftHalf ? 'eyeLookDownLeft' : 'eyeLookDownRight'] = dy > 0 ? Math.abs(dy) * 5 : 0;
|
|
410
|
+
}
|
|
411
|
+
else if (zone === 'cheek_left') {
|
|
412
|
+
u['cheekPuff'] = Math.abs(dx) * 6;
|
|
413
|
+
u['cheekSquintLeft'] = Math.abs(dy) * 6;
|
|
414
|
+
u['cheekSquintRight'] = 0;
|
|
415
|
+
u['mouthSmileLeft'] = dy < 0 ? Math.abs(dy) * 5 : 0;
|
|
416
|
+
u['mouthDimpleLeft'] = dy < 0 ? Math.abs(dy) * 4 : 0;
|
|
417
|
+
u['mouthStretchLeft'] = dy > 0 ? Math.abs(dy) * 5 : 0;
|
|
418
|
+
u['mouthFrownLeft'] = dy > 0 ? Math.abs(dy) * 4 : 0;
|
|
419
|
+
u['mouthPressLeft'] = dy > 0 ? Math.abs(dy) * 3 : 0;
|
|
420
|
+
}
|
|
421
|
+
else if (zone === 'cheek_right') {
|
|
422
|
+
u['cheekPuff'] = Math.abs(dx) * 6;
|
|
423
|
+
u['cheekSquintRight'] = Math.abs(dy) * 6;
|
|
424
|
+
u['cheekSquintLeft'] = 0;
|
|
425
|
+
u['mouthSmileRight'] = dy < 0 ? Math.abs(dy) * 5 : 0;
|
|
426
|
+
u['mouthDimpleRight'] = dy < 0 ? Math.abs(dy) * 4 : 0;
|
|
427
|
+
u['mouthStretchRight'] = dy > 0 ? Math.abs(dy) * 5 : 0;
|
|
428
|
+
u['mouthFrownRight'] = dy > 0 ? Math.abs(dy) * 4 : 0;
|
|
429
|
+
u['mouthPressRight'] = dy > 0 ? Math.abs(dy) * 3 : 0;
|
|
430
|
+
}
|
|
431
|
+
else if (zone === 'jaw') {
|
|
432
|
+
u['jawOpen'] = dy > 0 ? Math.abs(dy) * 10 : 0;
|
|
433
|
+
u['tongueOut'] = dy > 0 ? Math.max(0, Math.abs(dy) * 10 - 0.7) : 0;
|
|
434
|
+
u['mouthLowerDownLeft'] = dy > 0 ? Math.abs(dy) * 6 : 0;
|
|
435
|
+
u['mouthLowerDownRight'] = dy > 0 ? Math.abs(dy) * 6 : 0;
|
|
436
|
+
u['mouthUpperUpLeft'] = dy > 0 ? Math.abs(dy) * 4 : 0;
|
|
437
|
+
u['mouthUpperUpRight'] = dy > 0 ? Math.abs(dy) * 4 : 0;
|
|
438
|
+
u['mouthPressLeft'] = dy < 0 ? Math.abs(dy) * 6 : 0;
|
|
439
|
+
u['mouthPressRight'] = dy < 0 ? Math.abs(dy) * 6 : 0;
|
|
440
|
+
u['jawLeft'] = dx < 0 ? Math.abs(dx) * 8 : 0;
|
|
441
|
+
u['jawRight'] = dx > 0 ? Math.abs(dx) * 8 : 0;
|
|
442
|
+
u['jawForward'] = Math.abs(dx) * 4;
|
|
443
|
+
u['mouthLeft'] = dx < 0 ? Math.abs(dx) * 5 : 0;
|
|
444
|
+
u['mouthRight'] = dx > 0 ? Math.abs(dx) * 5 : 0;
|
|
445
|
+
}
|
|
446
|
+
applyWeights(u);
|
|
447
|
+
}, [applyWeights]);
|
|
448
|
+
const onPanEndJs = (0, react_1.useCallback)(() => {
|
|
449
|
+
if (emotionHoldTimerRef.current !== null)
|
|
450
|
+
return;
|
|
451
|
+
springToZero();
|
|
452
|
+
}, [springToZero]);
|
|
453
|
+
const onPinchBeginJs = (0, react_1.useCallback)(() => {
|
|
454
|
+
stopSpring();
|
|
455
|
+
Haptics.impactAsync(Haptics.ImpactFeedbackStyle.Medium);
|
|
456
|
+
}, [stopSpring]);
|
|
457
|
+
const onPinchUpdateJs = (0, react_1.useCallback)((scale, delta) => {
|
|
458
|
+
if (scale < 1) {
|
|
459
|
+
applyWeights({ mouthPucker: delta * 2, mouthFunnel: 0, mouthRollLower: delta * 1.2, mouthRollUpper: delta * 1.2 });
|
|
460
|
+
}
|
|
461
|
+
else {
|
|
462
|
+
applyWeights({ mouthFunnel: delta * 2, mouthPucker: 0, mouthShrugLower: delta * 0.8, mouthShrugUpper: delta * 0.8, mouthRollLower: 0, mouthRollUpper: 0 });
|
|
463
|
+
}
|
|
464
|
+
}, [applyWeights]);
|
|
465
|
+
const onPinchEndJs = (0, react_1.useCallback)(() => {
|
|
466
|
+
if (emotionHoldTimerRef.current !== null)
|
|
467
|
+
return;
|
|
468
|
+
springToZero();
|
|
469
|
+
}, [springToZero]);
|
|
470
|
+
const onEyeTapJs = (0, react_1.useCallback)((x, y) => {
|
|
471
|
+
const relY = y / Math.max(1, viewSize.current.height);
|
|
472
|
+
if (relY < 0.16 || relY > 0.52)
|
|
473
|
+
return;
|
|
474
|
+
Haptics.impactAsync(Haptics.ImpactFeedbackStyle.Heavy);
|
|
475
|
+
playReaction();
|
|
476
|
+
playEmotionBurst('owie');
|
|
477
|
+
}, [playEmotionBurst, playReaction]);
|
|
478
|
+
// ---- Gestures ----
|
|
479
|
+
const panGesture = react_native_gesture_handler_1.Gesture.Pan()
|
|
480
|
+
.minDistance(0)
|
|
481
|
+
.onBegin((e) => {
|
|
482
|
+
'worklet';
|
|
483
|
+
touchStartXSv.value = e.x;
|
|
484
|
+
touchStartYSv.value = e.y;
|
|
485
|
+
const relY = e.y / Math.max(1, viewHeightSv.value);
|
|
486
|
+
let zone;
|
|
487
|
+
if (relY < 0.20)
|
|
488
|
+
zone = 'brow';
|
|
489
|
+
else if (relY < 0.38)
|
|
490
|
+
zone = 'eye';
|
|
491
|
+
else if (relY > 0.72)
|
|
492
|
+
zone = 'jaw';
|
|
493
|
+
else
|
|
494
|
+
zone = e.x < viewWidthSv.value / 2 ? 'cheek_left' : 'cheek_right';
|
|
495
|
+
touchZoneSv.value = zone;
|
|
496
|
+
(0, react_native_reanimated_1.runOnJS)(onPanBeginJs)(e.x, e.y, zone);
|
|
497
|
+
})
|
|
498
|
+
.onUpdate((e) => {
|
|
499
|
+
'worklet';
|
|
500
|
+
const now = Date.now();
|
|
501
|
+
if (now - lastPanUpdateTsSv.value < 33)
|
|
502
|
+
return;
|
|
503
|
+
lastPanUpdateTsSv.value = now;
|
|
504
|
+
const w = Math.max(1, viewWidthSv.value);
|
|
505
|
+
const h = Math.max(1, viewHeightSv.value);
|
|
506
|
+
const dx = (e.x - touchStartXSv.value) / w;
|
|
507
|
+
const dy = (e.y - touchStartYSv.value) / h;
|
|
508
|
+
(0, react_native_reanimated_1.runOnJS)(onPanUpdateJs)(touchZoneSv.value, dx, dy, touchStartXSv.value < w / 2);
|
|
509
|
+
})
|
|
510
|
+
.onEnd(() => {
|
|
511
|
+
'worklet';
|
|
512
|
+
(0, react_native_reanimated_1.runOnJS)(onPanEndJs)();
|
|
513
|
+
});
|
|
514
|
+
const pinchGesture = react_native_gesture_handler_1.Gesture.Pinch()
|
|
515
|
+
.onBegin(() => {
|
|
516
|
+
'worklet';
|
|
517
|
+
(0, react_native_reanimated_1.runOnJS)(onPinchBeginJs)();
|
|
518
|
+
})
|
|
519
|
+
.onUpdate((e) => {
|
|
520
|
+
'worklet';
|
|
521
|
+
(0, react_native_reanimated_1.runOnJS)(onPinchUpdateJs)(e.scale, Math.abs(1 - e.scale));
|
|
522
|
+
})
|
|
523
|
+
.onEnd(() => {
|
|
524
|
+
'worklet';
|
|
525
|
+
(0, react_native_reanimated_1.runOnJS)(onPinchEndJs)();
|
|
526
|
+
});
|
|
527
|
+
const tapGesture = react_native_gesture_handler_1.Gesture.Tap()
|
|
528
|
+
.onEnd((e, success) => {
|
|
529
|
+
'worklet';
|
|
530
|
+
if (success)
|
|
531
|
+
(0, react_native_reanimated_1.runOnJS)(onEyeTapJs)(e.x, e.y);
|
|
532
|
+
});
|
|
533
|
+
const composed = react_native_gesture_handler_1.Gesture.Simultaneous(panGesture, pinchGesture, tapGesture);
|
|
534
|
+
const handleReady = (0, react_1.useCallback)(() => setIsReady(true), []);
|
|
535
|
+
return ((0, jsx_runtime_1.jsx)(react_native_gesture_handler_1.GestureHandlerRootView, { style: { flex: 1 }, children: (0, jsx_runtime_1.jsxs)(react_native_1.View, { style: styles.container, children: [(0, jsx_runtime_1.jsx)(react_native_1.View, { style: styles.scene, children: localUri ? ((0, jsx_runtime_1.jsx)(native_1.Canvas, { style: react_native_1.StyleSheet.absoluteFill, camera: { fov: 34, position: [0, 1.55, 1.05], near: 0.01, far: 50 }, gl: { antialias: true, alpha: false }, onCreated: ({ gl }) => gl.setClearColor(new THREE.Color('#0a0a0a')), children: (0, jsx_runtime_1.jsx)(AvatarScene, { uri: localUri, morphState: morphState, onReady: handleReady }) })) : ((0, jsx_runtime_1.jsx)(react_native_1.View, { style: styles.loading, children: (0, jsx_runtime_1.jsx)(react_native_1.Text, { style: styles.loadingText, children: "Loading face\u2026" }) })) }), (0, jsx_runtime_1.jsx)(react_native_gesture_handler_1.GestureDetector, { gesture: composed, children: (0, jsx_runtime_1.jsx)(react_native_1.View, { style: react_native_1.StyleSheet.absoluteFill, onLayout: onLayout, collapsable: false }) }), (0, jsx_runtime_1.jsx)(react_native_1.Pressable, { style: styles.closeBtn, onPress: onClose, children: (0, jsx_runtime_1.jsx)(react_native_1.Text, { style: styles.closeBtnText, children: "\u2715 Done squeezing" }) }), (0, jsx_runtime_1.jsx)(react_native_1.Text, { style: styles.hint, children: "Drag to stretch \u00B7 Pinch lips \u00B7 Poke an eye \uD83D\uDC41" })] }) }));
|
|
536
|
+
}
|
|
537
|
+
exports.FaceSqueezeEditor = FaceSqueezeEditor;
|
|
538
|
+
const styles = react_native_1.StyleSheet.create({
|
|
539
|
+
container: { flex: 1, backgroundColor: '#0a0a0a' },
|
|
540
|
+
scene: { flex: 1 },
|
|
541
|
+
loading: { flex: 1, alignItems: 'center', justifyContent: 'center' },
|
|
542
|
+
loadingText: { color: '#97a3b5', fontSize: 14 },
|
|
543
|
+
closeBtn: {
|
|
544
|
+
position: 'absolute', top: 52, right: 16,
|
|
545
|
+
backgroundColor: '#6c63ff', borderRadius: 20,
|
|
546
|
+
paddingHorizontal: 16, paddingVertical: 8,
|
|
547
|
+
},
|
|
548
|
+
closeBtnText: { color: '#fff', fontWeight: '700', fontSize: 14 },
|
|
549
|
+
hint: {
|
|
550
|
+
position: 'absolute', bottom: 40, alignSelf: 'center',
|
|
551
|
+
color: '#97a3b5', fontSize: 13,
|
|
552
|
+
},
|
|
553
|
+
});
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
export declare const FACE_SQUEEZE_LOCAL_MODULE: number;
|
|
2
|
+
export interface FaceSqueezeEditorProps {
|
|
3
|
+
onClose: () => void;
|
|
4
|
+
}
|
|
5
|
+
export declare function FaceSqueezeEditor({ onClose }: FaceSqueezeEditorProps): import("react/jsx-runtime").JSX.Element;
|
|
6
|
+
export default FaceSqueezeEditor;
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.FaceSqueezeEditor = exports.FACE_SQUEEZE_LOCAL_MODULE = void 0;
|
|
4
|
+
const jsx_runtime_1 = require("react/jsx-runtime");
|
|
5
|
+
const react_native_1 = require("react-native");
|
|
6
|
+
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
|
7
|
+
exports.FACE_SQUEEZE_LOCAL_MODULE = require('../assets/face-squeeze-local.glb');
|
|
8
|
+
function FaceSqueezeEditor({ onClose }) {
|
|
9
|
+
return ((0, jsx_runtime_1.jsx)(react_native_1.View, { style: styles.container, children: (0, jsx_runtime_1.jsxs)(react_native_1.View, { style: styles.card, children: [(0, jsx_runtime_1.jsx)(react_native_1.Text, { style: styles.title, children: "Face squeeze editor is mobile-only." }), (0, jsx_runtime_1.jsx)(react_native_1.Text, { style: styles.subtitle, children: "This placeholder keeps web builds safe while preserving native functionality." }), (0, jsx_runtime_1.jsx)(react_native_1.Pressable, { style: styles.closeButton, onPress: onClose, children: (0, jsx_runtime_1.jsx)(react_native_1.Text, { style: styles.closeButtonText, children: "Close" }) })] }) }));
|
|
10
|
+
}
|
|
11
|
+
exports.FaceSqueezeEditor = FaceSqueezeEditor;
|
|
12
|
+
const styles = react_native_1.StyleSheet.create({
|
|
13
|
+
container: {
|
|
14
|
+
flex: 1,
|
|
15
|
+
alignItems: "center",
|
|
16
|
+
justifyContent: "center",
|
|
17
|
+
backgroundColor: "#0f1115",
|
|
18
|
+
padding: 20,
|
|
19
|
+
},
|
|
20
|
+
card: {
|
|
21
|
+
width: "100%",
|
|
22
|
+
maxWidth: 460,
|
|
23
|
+
borderRadius: 12,
|
|
24
|
+
borderWidth: 1,
|
|
25
|
+
borderColor: "#2d3340",
|
|
26
|
+
backgroundColor: "#171b23",
|
|
27
|
+
padding: 16,
|
|
28
|
+
gap: 10,
|
|
29
|
+
},
|
|
30
|
+
title: {
|
|
31
|
+
color: "#f1f4f8",
|
|
32
|
+
fontSize: 18,
|
|
33
|
+
fontWeight: "700",
|
|
34
|
+
},
|
|
35
|
+
subtitle: {
|
|
36
|
+
color: "#98a1b3",
|
|
37
|
+
fontSize: 14,
|
|
38
|
+
lineHeight: 20,
|
|
39
|
+
},
|
|
40
|
+
closeButton: {
|
|
41
|
+
marginTop: 6,
|
|
42
|
+
alignSelf: "flex-start",
|
|
43
|
+
borderRadius: 8,
|
|
44
|
+
backgroundColor: "#2e8f5b",
|
|
45
|
+
paddingHorizontal: 12,
|
|
46
|
+
paddingVertical: 8,
|
|
47
|
+
},
|
|
48
|
+
closeButtonText: {
|
|
49
|
+
color: "#ffffff",
|
|
50
|
+
fontSize: 14,
|
|
51
|
+
fontWeight: "600",
|
|
52
|
+
},
|
|
53
|
+
});
|
|
54
|
+
exports.default = FaceSqueezeEditor;
|
package/dist/editor/index.d.ts
CHANGED
|
@@ -3,7 +3,8 @@ export { AvatarEditor } from './AvatarEditor';
|
|
|
3
3
|
export { AvatarModel } from './AvatarModel';
|
|
4
4
|
export { RigidAccessory } from './RigidAccessory';
|
|
5
5
|
export { SkinnedClothing } from './SkinnedClothing';
|
|
6
|
+
export { FaceSqueezeEditor, FACE_SQUEEZE_LOCAL_MODULE } from './FaceSqueezeEditor';
|
|
6
7
|
export { studioTheme, withAlpha } from './studioTheme';
|
|
7
8
|
export { useBoneSnap, findNearestBone, getWorldPositionForBone, getWorldPositionForPlacement, snapPlacementToNearestBone, HUMANOID_BONES } from './boneSnap';
|
|
8
|
-
export type { AvatarEditorProps, AssetPlacement, EquippedAsset } from './types';
|
|
9
|
+
export type { AvatarEditorProps, AssetPlacement, EquippedAsset, FaceSqueezeEditorProps } from './types';
|
|
9
10
|
export type { BoneSnapResult, HumanoidBone } from './boneSnap';
|
package/dist/editor/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.HUMANOID_BONES = exports.snapPlacementToNearestBone = exports.getWorldPositionForPlacement = exports.getWorldPositionForBone = exports.findNearestBone = exports.useBoneSnap = exports.withAlpha = exports.studioTheme = exports.SkinnedClothing = exports.RigidAccessory = exports.AvatarModel = exports.AvatarEditor = exports.AvatarCanvas = void 0;
|
|
3
|
+
exports.HUMANOID_BONES = exports.snapPlacementToNearestBone = exports.getWorldPositionForPlacement = exports.getWorldPositionForBone = exports.findNearestBone = exports.useBoneSnap = exports.withAlpha = exports.studioTheme = exports.FACE_SQUEEZE_LOCAL_MODULE = exports.FaceSqueezeEditor = exports.SkinnedClothing = exports.RigidAccessory = exports.AvatarModel = exports.AvatarEditor = exports.AvatarCanvas = void 0;
|
|
4
4
|
var AvatarCanvas_1 = require("./AvatarCanvas");
|
|
5
5
|
Object.defineProperty(exports, "AvatarCanvas", { enumerable: true, get: function () { return AvatarCanvas_1.AvatarCanvas; } });
|
|
6
6
|
var AvatarEditor_1 = require("./AvatarEditor");
|
|
@@ -11,6 +11,9 @@ var RigidAccessory_1 = require("./RigidAccessory");
|
|
|
11
11
|
Object.defineProperty(exports, "RigidAccessory", { enumerable: true, get: function () { return RigidAccessory_1.RigidAccessory; } });
|
|
12
12
|
var SkinnedClothing_1 = require("./SkinnedClothing");
|
|
13
13
|
Object.defineProperty(exports, "SkinnedClothing", { enumerable: true, get: function () { return SkinnedClothing_1.SkinnedClothing; } });
|
|
14
|
+
var FaceSqueezeEditor_1 = require("./FaceSqueezeEditor");
|
|
15
|
+
Object.defineProperty(exports, "FaceSqueezeEditor", { enumerable: true, get: function () { return FaceSqueezeEditor_1.FaceSqueezeEditor; } });
|
|
16
|
+
Object.defineProperty(exports, "FACE_SQUEEZE_LOCAL_MODULE", { enumerable: true, get: function () { return FaceSqueezeEditor_1.FACE_SQUEEZE_LOCAL_MODULE; } });
|
|
14
17
|
var studioTheme_1 = require("./studioTheme");
|
|
15
18
|
Object.defineProperty(exports, "studioTheme", { enumerable: true, get: function () { return studioTheme_1.studioTheme; } });
|
|
16
19
|
Object.defineProperty(exports, "withAlpha", { enumerable: true, get: function () { return studioTheme_1.withAlpha; } });
|
package/dist/editor/types.d.ts
CHANGED
package/dist/html.d.ts
CHANGED
|
@@ -15,5 +15,10 @@ export type AvatarConfig = {
|
|
|
15
15
|
* Example: "https://studio.sitebay.org/vendor"
|
|
16
16
|
*/
|
|
17
17
|
vendorBaseUrl?: string | null;
|
|
18
|
+
/**
|
|
19
|
+
* URL to the DRACO decoder WASM/JS folder.
|
|
20
|
+
* Defaults to gstatic's v1.5.6 CDN.
|
|
21
|
+
*/
|
|
22
|
+
dracoDecoderUrl?: string | null;
|
|
18
23
|
};
|
|
19
24
|
export declare function buildAvatarHtml(config: AvatarConfig): string;
|
package/dist/html.js
CHANGED
|
@@ -20,6 +20,7 @@ function buildAvatarHtml(config) {
|
|
|
20
20
|
const headAudioUrl = v ? `${v}/headaudio.min.mjs` : 'https://cdn.jsdelivr.net/gh/met4citizen/HeadAudio@v0.1.0-alpha/dist/headaudio.min.mjs';
|
|
21
21
|
const headWorkletUrl = v ? `${v}/headworklet.min.mjs` : 'https://cdn.jsdelivr.net/gh/met4citizen/HeadAudio@v0.1.0-alpha/dist/headworklet.min.mjs';
|
|
22
22
|
const headModelUrl = v ? `${v}/model-en-mixed.bin` : 'https://cdn.jsdelivr.net/gh/met4citizen/HeadAudio@v0.1.0-alpha/dist/model-en-mixed.bin';
|
|
23
|
+
const dracoUrl = config.dracoDecoderUrl ?? (v ? `${v}/draco/` : 'https://www.gstatic.com/draco/versioned/decoders/1.5.6/');
|
|
23
24
|
return `
|
|
24
25
|
<!DOCTYPE html>
|
|
25
26
|
<html>
|
|
@@ -71,6 +72,7 @@ rnPost(
|
|
|
71
72
|
JSON.stringify({ type: 'log', message: '[module] script start' })
|
|
72
73
|
);
|
|
73
74
|
const AUTH_TOKEN = ${JSON.stringify(config.authToken ?? null)};
|
|
75
|
+
const DRACO_URL = ${JSON.stringify(dracoUrl)};
|
|
74
76
|
const TALKING_HEAD_URL = ${JSON.stringify(talkingHeadUrl)};
|
|
75
77
|
const HEAD_AUDIO_URL = ${JSON.stringify(headAudioUrl)};
|
|
76
78
|
const HEAD_AUDIO_WORKLET = ${JSON.stringify(headWorkletUrl)};
|
|
@@ -751,7 +753,7 @@ async function ensureThree() {
|
|
|
751
753
|
const { DRACOLoader } = await import('three/addons/loaders/DRACOLoader.js');
|
|
752
754
|
gltfLoaderInstance = new GLTFLoader();
|
|
753
755
|
const dracoLoader = new DRACOLoader();
|
|
754
|
-
dracoLoader.setDecoderPath(
|
|
756
|
+
dracoLoader.setDecoderPath(DRACO_URL);
|
|
755
757
|
gltfLoaderInstance.setDRACOLoader(dracoLoader);
|
|
756
758
|
}
|
|
757
759
|
}
|
|
@@ -180,7 +180,7 @@ function sleep(ms, signal) {
|
|
|
180
180
|
const id = setTimeout(resolve, ms);
|
|
181
181
|
signal.addEventListener("abort", () => {
|
|
182
182
|
clearTimeout(id);
|
|
183
|
-
reject(
|
|
183
|
+
reject(new DOMException("The operation was aborted", "AbortError"));
|
|
184
184
|
}, { once: true });
|
|
185
185
|
});
|
|
186
186
|
}
|
package/dist/wgpu/WgpuAvatar.js
CHANGED
|
@@ -178,6 +178,11 @@ exports.WgpuAvatar = (0, react_1.forwardRef)(({ style, avatarUrl, focalLength, m
|
|
|
178
178
|
? avatarUrl : null;
|
|
179
179
|
const fileResult = (0, useAuthedModelUri_1.useAuthedModelUri)(remoteUrl);
|
|
180
180
|
const lockedUriRef = (0, react_1.useRef)(null);
|
|
181
|
+
const lastUrlRef = (0, react_1.useRef)(avatarUrl);
|
|
182
|
+
if (lastUrlRef.current !== avatarUrl) {
|
|
183
|
+
lastUrlRef.current = avatarUrl;
|
|
184
|
+
lockedUriRef.current = null;
|
|
185
|
+
}
|
|
181
186
|
const currentUri = fileResult?.uri ?? null;
|
|
182
187
|
if (lockedUriRef.current === null && currentUri)
|
|
183
188
|
lockedUriRef.current = currentUri;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "talking-head-studio",
|
|
3
|
-
"version": "0.4.
|
|
3
|
+
"version": "0.4.9",
|
|
4
4
|
"description": "Cross-platform 3D avatar component for React Native & web — lip-sync, gestures, accessories, and LLM integration. Powered by TalkingHead + Three.js.",
|
|
5
5
|
"main": "dist/index.web.js",
|
|
6
6
|
"browser": "dist/index.web.js",
|
|
@@ -103,6 +103,7 @@
|
|
|
103
103
|
"@react-three/fiber": ">=8",
|
|
104
104
|
"expo": ">=51",
|
|
105
105
|
"expo-asset": ">=10",
|
|
106
|
+
"expo-audio": ">=0.3",
|
|
106
107
|
"expo-file-system": ">=17",
|
|
107
108
|
"react": ">=18",
|
|
108
109
|
"react-native": ">=0.73",
|
|
@@ -126,6 +127,9 @@
|
|
|
126
127
|
"expo-asset": {
|
|
127
128
|
"optional": true
|
|
128
129
|
},
|
|
130
|
+
"expo-audio": {
|
|
131
|
+
"optional": true
|
|
132
|
+
},
|
|
129
133
|
"expo-file-system": {
|
|
130
134
|
"optional": true
|
|
131
135
|
},
|