talking-head-studio 0.4.6 → 0.4.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -1,6 +1,6 @@
1
1
  # talking-head-studio
2
2
 
3
- **Drop a talking, lip-syncing 3D avatar into any React app -- native or web -- in under five minutes.**
3
+ **The missing UI layer for AI Agents. Drop-in, lip-syncing 3D avatars for Web, React, and React Native.**
4
4
 
5
5
  [![npm version](https://img.shields.io/npm/v/talking-head-studio.svg)](https://www.npmjs.com/package/talking-head-studio)
6
6
  [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT)
@@ -10,10 +10,11 @@
10
10
 
11
11
  ## Why this?
12
12
 
13
- - **Truly cross-platform.** One component, two renderers. React Native gets a WebView; React on web gets an iframe with `srcdoc`. Same API, same props, same ref.
14
- - **Bring any GLB.** Rigged models with ARKit/Oculus blend shapes get full phoneme-based lip-sync via HeadAudio. Non-rigged models still work -- they get a static viewer with amplitude-driven jaw animation as a fallback. No vendor lock-in to a single avatar format.
15
- - **Built for LLM voice pipelines.** Wire `sendAmplitude` to LiveKit, Web Audio, ElevenLabs, or any audio source. The avatar speaks when your AI speaks.
16
- - **Accessory system.** Attach hats, glasses, backpacks, or any GLB to any bone at runtime. Position, rotate, and scale each piece independently.
13
+ - **Zero-Jank React Native & Web:** True cross-platform rendering. React Native gets a blazing fast wgpu-accelerated native render loop, skipping WebView bridge latency entirely. React on web gets a robust `react-three-fiber` setup. Same API, same props.
14
+ - **Universal GLB Compatibility:** Bring any GLB. Out-of-the-box support for standard ARKit blendshapes. Rigged models get full phoneme-based lip-sync. Non-rigged models get an amplitude-driven jaw animation fallback.
15
+ - **Built for AI & Voice Pipelines:** Wire `sendAmplitude` or visemes directly to LiveKit, Web Audio, ElevenLabs, OpenAI Realtime, or any audio source.
16
+ - **Always Alive:** Procedural idle animations (breathing, nodding, swaying) keep your avatar from feeling like a static doll.
17
+ - **Dynamic Wardrobe & Accessories:** Swap hair, skin, and eye colors on the fly. Attach hats, glasses, or backpacks to any bone at runtime.
17
18
 
18
19
  ---
19
20
 
@@ -68,7 +69,7 @@ export default function Avatar() {
68
69
  return (
69
70
  <TalkingHead
70
71
  ref={ref}
71
- avatarUrl="https://models.readyplayer.me/your-model.glb"
72
+ avatarUrl="https://example.com/your-model.glb"
72
73
  mood="happy"
73
74
  cameraView="upper"
74
75
  hairColor="#1a1a2e"
@@ -378,7 +379,7 @@ For the complete experience -- phoneme lip-sync, expressions, moods, gestures --
378
379
  - **ARKit blend shapes** and/or **Oculus viseme blend shapes** for lip-sync
379
380
  - Standard Three.js-compatible GLB format
380
381
 
381
- Models from [Ready Player Me](https://readyplayer.me/), [Avaturn](https://avaturn.me/), or any Mixamo-rigged source work out of the box.
382
+ Models from [Avaturn](https://avaturn.me/) or any Mixamo-rigged source work out of the box.
382
383
 
383
384
  ### Non-rigged models (static fallback)
384
385
 
@@ -457,4 +458,11 @@ This project builds on excellent open-source work:
457
458
 
458
459
  ## License
459
460
 
461
+ MIT
462
+ at runtime.
463
+
464
+ ---
465
+
466
+ ## License
467
+
460
468
  MIT
@@ -46,14 +46,24 @@ exports.TalkingHead = (0, react_1.forwardRef)(({ avatarUrl, authToken, mood = 'n
46
46
  const pendingSkinColorRef = (0, react_1.useRef)(skinColor);
47
47
  const pendingEyeColorRef = (0, react_1.useRef)(eyeColor);
48
48
  const accessoriesRef = (0, react_1.useRef)(accessories);
49
+ const onLoadingChangeRef = (0, react_1.useRef)(onLoadingChange);
50
+ const onReadyRef = (0, react_1.useRef)(onReady);
51
+ const onErrorRef = (0, react_1.useRef)(onError);
52
+ const onAvatarStateRef = (0, react_1.useRef)(onAvatarState);
53
+ (0, react_1.useEffect)(() => {
54
+ onLoadingChangeRef.current = onLoadingChange;
55
+ onReadyRef.current = onReady;
56
+ onErrorRef.current = onError;
57
+ onAvatarStateRef.current = onAvatarState;
58
+ });
49
59
  (0, react_1.useEffect)(() => {
50
60
  accessoriesRef.current = accessories;
51
61
  }, [accessories]);
52
62
  (0, react_1.useEffect)(() => {
53
63
  if (!avatarUrl)
54
64
  return;
55
- onLoadingChange?.({ stage: 'booting', progress: null });
56
- }, [avatarUrl, authToken, onLoadingChange]);
65
+ onLoadingChangeRef.current?.({ stage: 'booting', progress: null });
66
+ }, [avatarUrl, authToken]);
57
67
  const post = (0, react_1.useCallback)((msg) => {
58
68
  iframeRef.current?.contentWindow?.postMessage(JSON.stringify(msg), '*');
59
69
  }, []);
@@ -148,7 +158,7 @@ exports.TalkingHead = (0, react_1.forwardRef)(({ avatarUrl, authToken, mood = 'n
148
158
  try {
149
159
  const msg = typeof event.data === 'string' ? JSON.parse(event.data) : event.data;
150
160
  if (msg.type === 'loading' && typeof msg.stage === 'string') {
151
- onLoadingChange?.({
161
+ onLoadingChangeRef.current?.({
152
162
  stage: msg.stage,
153
163
  progress: typeof msg.progress === 'number' && Number.isFinite(msg.progress)
154
164
  ? msg.progress
@@ -158,7 +168,7 @@ exports.TalkingHead = (0, react_1.forwardRef)(({ avatarUrl, authToken, mood = 'n
158
168
  }
159
169
  if (msg.type === 'ready') {
160
170
  readyRef.current = true;
161
- onLoadingChange?.({ stage: 'ready', progress: 100 });
171
+ onLoadingChangeRef.current?.({ stage: 'ready', progress: 100 });
162
172
  if (pendingMoodRef.current) {
163
173
  post({ type: 'mood', value: pendingMoodRef.current });
164
174
  }
@@ -174,13 +184,13 @@ exports.TalkingHead = (0, react_1.forwardRef)(({ avatarUrl, authToken, mood = 'n
174
184
  if (accessoriesRef.current?.length) {
175
185
  post({ type: 'set_accessories', accessories: accessoriesRef.current });
176
186
  }
177
- onReady?.();
187
+ onReadyRef.current?.();
178
188
  }
179
189
  else if (msg.type === 'error') {
180
- onError?.(msg.message);
190
+ onErrorRef.current?.(msg.message);
181
191
  }
182
192
  else if (msg.type === 'avatarState') {
183
- onAvatarState?.(msg.state);
193
+ onAvatarStateRef.current?.(msg.state);
184
194
  }
185
195
  else if (msg.type === 'log') {
186
196
  console.log('[TalkingHead]', msg.message);
@@ -192,7 +202,7 @@ exports.TalkingHead = (0, react_1.forwardRef)(({ avatarUrl, authToken, mood = 'n
192
202
  };
193
203
  window.addEventListener('message', onMessage);
194
204
  return () => window.removeEventListener('message', onMessage);
195
- }, [onLoadingChange, onReady, onError, post]);
205
+ }, [post]);
196
206
  return ((0, jsx_runtime_1.jsx)("div", { style: { ...containerStyle, ...style }, children: (0, jsx_runtime_1.jsx)("iframe", { ref: iframeRef, srcDoc: srcdoc, style: iframeStyle, sandbox: "allow-scripts allow-same-origin", title: "TalkingHead Avatar" }) }));
197
207
  });
198
208
  exports.TalkingHead.displayName = 'TalkingHead';
@@ -46,8 +46,10 @@ function getLoadingLabel(stage) {
46
46
  */
47
47
  exports.TalkingHeadVisualization = (0, react_1.forwardRef)(({ style, avatarUrl, authToken, cameraView = 'head', cameraDistance = 0.2, accessories, mood: initialMood = 'neutral', aspect, focalLength, visemeSchedule, onVisemeScheduleApplied, onVoiceMood, vendorBaseUrl }, ref) => {
48
48
  const avatarRef = (0, react_1.useRef)(null);
49
- // On native, WgpuAvatar ref is wired via callback ref store it here so
50
- // scheduleVisemes / sendAmplitude can route to it.
49
+ // On native, WgpuAvatar ref is stored here so scheduleVisemes / sendAmplitude can route to it.
50
+ // useImperativeHandle is the sole owner of the forwarded ref on all platforms —
51
+ // do NOT manually forward ref inside the WgpuAvatar callback ref, as that bypasses
52
+ // the dedup key check in scheduleVisemes and causes double-fire.
51
53
  const wgpuRef = (0, react_1.useRef)(null);
52
54
  // Unified accessor — WgpuAvatar on native, WebView on web
53
55
  const activeAvatar = (0, react_1.useCallback)(() => (wgpuRef.current ?? avatarRef.current), []);
@@ -183,26 +185,15 @@ exports.TalkingHeadVisualization = (0, react_1.forwardRef)(({ style, avatarUrl,
183
185
  lastScheduledVisemeKeyRef.current = scheduleKey;
184
186
  }, [isAvatarReady, onVisemeScheduleApplied, visemeSchedule, activeAvatar]);
185
187
  // On native use WgpuAvatar — direct morph writes, no WebView bridge.
188
+ // NOTE: Only store wgpuRef here — do NOT forward `ref` manually. useImperativeHandle
189
+ // above is the sole owner of the forwarded ref and handles the dedup key check.
186
190
  if (react_native_1.Platform.OS !== 'web') {
187
- return ((0, jsx_runtime_1.jsx)(WgpuAvatar_1.WgpuAvatar, { focalLength: focalLength, ref: (fr) => {
188
- wgpuRef.current = fr;
189
- if (typeof ref === 'function')
190
- ref(fr);
191
- else if (ref)
192
- ref.current = fr;
193
- if (fr && pendingVisemeScheduleRef.current) {
194
- fr.scheduleVisemes(pendingVisemeScheduleRef.current);
195
- pendingVisemeScheduleRef.current = null;
196
- }
197
- }, style: style, avatarUrl: avatarUrl ?? null, aspect: aspect, mood: mood, accessories: accessories, onReady: handleReady, onError: handleError }));
191
+ return ((0, jsx_runtime_1.jsx)(WgpuAvatar_1.WgpuAvatar, { focalLength: focalLength, ref: (fr) => { wgpuRef.current = fr; }, style: style, avatarUrl: avatarUrl ?? null, aspect: aspect, mood: mood, accessories: accessories, onReady: handleReady, onError: handleError }));
198
192
  }
199
193
  if (!effectiveAvatarUrl) {
200
194
  return ((0, jsx_runtime_1.jsx)(react_native_1.View, { style: [style, styles.placeholder] }));
201
195
  }
202
- return ((0, jsx_runtime_1.jsxs)(react_native_1.View, { style: [style, styles.container], pointerEvents: "box-none", children: [(0, jsx_runtime_1.jsx)(TalkingHead_1.TalkingHead, { ref: avatarRef, avatarUrl: effectiveAvatarUrl, authToken: authToken, cameraView: cameraView, cameraDistance: cameraDistance, accessories: accessories, mood: mood, onLoadingChange: handleLoadingChange, onReady: handleReady, onError: handleError, onVoiceMood: onVoiceMood, style: react_native_1.StyleSheet.absoluteFill, vendorBaseUrl: vendorBaseUrl }), !isAvatarReady && ((0, jsx_runtime_1.jsx)(react_native_1.View, { testID: "talking-head-loading", style: styles.loadingOverlay, pointerEvents: "none", children: (0, jsx_runtime_1.jsxs)(react_native_1.View, { style: styles.loadingCard, children: [(0, jsx_runtime_1.jsx)(react_native_1.View, { style: styles.loadingBadge, children: (0, jsx_runtime_1.jsx)(react_native_1.Text, { style: styles.loadingBadgeText, children: "AVATAR" }) }), (0, jsx_runtime_1.jsx)(react_native_1.Text, { style: styles.loadingTitle, children: avatarError ? 'Avatar failed to load' : getLoadingLabel(loadingState.stage) }), avatarError ? ((0, jsx_runtime_1.jsx)(react_native_1.Text, { style: styles.loadingHint, children: avatarError })) : typeof loadingState.progress === 'number' ? ((0, jsx_runtime_1.jsxs)(jsx_runtime_1.Fragment, { children: [(0, jsx_runtime_1.jsxs)(react_native_1.Text, { style: styles.loadingPercent, children: [loadingState.progress, "%"] }), (0, jsx_runtime_1.jsx)(react_native_1.View, { style: styles.progressTrack, children: (0, jsx_runtime_1.jsx)(react_native_1.View, { style: [
203
- styles.progressFill,
204
- { width: `${Math.max(6, loadingState.progress)}%` },
205
- ] }) })] })) : ((0, jsx_runtime_1.jsx)(react_native_1.Text, { style: styles.loadingHint, children: "Preparing the avatar scene\u2026" }))] }) }))] }));
196
+ return ((0, jsx_runtime_1.jsxs)(react_native_1.View, { style: [style, styles.container], pointerEvents: "box-none", children: [(0, jsx_runtime_1.jsx)(TalkingHead_1.TalkingHead, { ref: avatarRef, avatarUrl: effectiveAvatarUrl, authToken: authToken, cameraView: cameraView, cameraDistance: cameraDistance, accessories: accessories, mood: mood, onLoadingChange: handleLoadingChange, onReady: handleReady, onError: handleError, onVoiceMood: onVoiceMood, style: react_native_1.StyleSheet.absoluteFill, vendorBaseUrl: vendorBaseUrl }), !isAvatarReady && ((0, jsx_runtime_1.jsx)(react_native_1.View, { testID: "talking-head-loading", style: styles.loadingOverlay, pointerEvents: "none", children: (0, jsx_runtime_1.jsxs)(react_native_1.View, { style: styles.loadingCard, children: [(0, jsx_runtime_1.jsx)(react_native_1.View, { style: styles.loadingBadge, children: (0, jsx_runtime_1.jsx)(react_native_1.Text, { style: styles.loadingBadgeText, children: "AVATAR" }) }), (0, jsx_runtime_1.jsx)(react_native_1.Text, { style: styles.loadingTitle, children: avatarError ? 'Avatar failed to load' : getLoadingLabel(loadingState.stage) }), avatarError ? ((0, jsx_runtime_1.jsx)(react_native_1.Text, { style: styles.loadingHint, children: avatarError })) : typeof loadingState.progress === 'number' ? ((0, jsx_runtime_1.jsxs)(jsx_runtime_1.Fragment, { children: [(0, jsx_runtime_1.jsxs)(react_native_1.Text, { style: styles.loadingPercent, children: [loadingState.progress, "%"] }), (0, jsx_runtime_1.jsxs)(react_native_1.View, { style: styles.progressTrack, children: [(0, jsx_runtime_1.jsx)(react_native_1.View, { style: { flex: Math.max(6, loadingState.progress ?? 0), height: '100%', borderRadius: 999, backgroundColor: '#5eead4' } }), (0, jsx_runtime_1.jsx)(react_native_1.View, { style: { flex: 100 - Math.max(6, loadingState.progress ?? 0) } })] })] })) : ((0, jsx_runtime_1.jsx)(react_native_1.Text, { style: styles.loadingHint, children: "Preparing the avatar scene\u2026" }))] }) }))] }));
206
197
  });
207
198
  exports.TalkingHeadVisualization.displayName = 'TalkingHeadVisualization';
208
199
  const styles = react_native_1.StyleSheet.create({
@@ -264,11 +255,7 @@ const styles = react_native_1.StyleSheet.create({
264
255
  backgroundColor: 'rgba(148, 163, 184, 0.18)',
265
256
  marginTop: 12,
266
257
  overflow: 'hidden',
267
- },
268
- progressFill: {
269
- height: '100%',
270
- borderRadius: 999,
271
- backgroundColor: '#5eead4',
258
+ flexDirection: 'row',
272
259
  },
273
260
  placeholder: {
274
261
  backgroundColor: 'rgba(30,34,53,0.5)',
@@ -0,0 +1,17 @@
1
+ /**
2
+ * WgpuFaceSqueezeEditor — R3F/Three.js port of FaceSqueezeEditor.
3
+ *
4
+ * Same gesture zones, spring physics, idle emotions, lip sync, and sound
5
+ * as the Filament version. Only the render scaffolding changes:
6
+ *
7
+ * FilamentScene + FilamentView + renderableManager.setMorphWeights
8
+ * → @react-three/fiber Canvas + mesh.morphTargetInfluences[]
9
+ *
10
+ * Drop-in replacement: same FaceSqueezeEditorProps interface.
11
+ * The .web.tsx stub is shared — import from FaceSqueezeEditor.web.tsx.
12
+ */
13
+ export declare const FACE_SQUEEZE_LOCAL_MODULE: number;
14
+ export interface FaceSqueezeEditorProps {
15
+ onClose: () => void;
16
+ }
17
+ export declare function FaceSqueezeEditor({ onClose }: FaceSqueezeEditorProps): import("react/jsx-runtime").JSX.Element;
@@ -0,0 +1,553 @@
1
+ "use strict";
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ var desc = Object.getOwnPropertyDescriptor(m, k);
5
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
+ desc = { enumerable: true, get: function() { return m[k]; } };
7
+ }
8
+ Object.defineProperty(o, k2, desc);
9
+ }) : (function(o, m, k, k2) {
10
+ if (k2 === undefined) k2 = k;
11
+ o[k2] = m[k];
12
+ }));
13
+ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
+ Object.defineProperty(o, "default", { enumerable: true, value: v });
15
+ }) : function(o, v) {
16
+ o["default"] = v;
17
+ });
18
+ var __importStar = (this && this.__importStar) || function (mod) {
19
+ if (mod && mod.__esModule) return mod;
20
+ var result = {};
21
+ if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
22
+ __setModuleDefault(result, mod);
23
+ return result;
24
+ };
25
+ Object.defineProperty(exports, "__esModule", { value: true });
26
+ exports.FaceSqueezeEditor = exports.FACE_SQUEEZE_LOCAL_MODULE = void 0;
27
+ const jsx_runtime_1 = require("react/jsx-runtime");
28
+ /**
29
+ * WgpuFaceSqueezeEditor — R3F/Three.js port of FaceSqueezeEditor.
30
+ *
31
+ * Same gesture zones, spring physics, idle emotions, lip sync, and sound
32
+ * as the Filament version. Only the render scaffolding changes:
33
+ *
34
+ * FilamentScene + FilamentView + renderableManager.setMorphWeights
35
+ * → @react-three/fiber Canvas + mesh.morphTargetInfluences[]
36
+ *
37
+ * Drop-in replacement: same FaceSqueezeEditorProps interface.
38
+ * The .web.tsx stub is shared — import from FaceSqueezeEditor.web.tsx.
39
+ */
40
+ const react_1 = require("react");
41
+ const react_native_1 = require("react-native");
42
+ const native_1 = require("@react-three/fiber/native");
43
+ const native_2 = require("@react-three/drei/native");
44
+ const THREE = __importStar(require("three"));
45
+ const react_native_gesture_handler_1 = require("react-native-gesture-handler");
46
+ const react_native_reanimated_1 = require("react-native-reanimated");
47
+ const Haptics = __importStar(require("expo-haptics"));
48
+ const expo_audio_1 = require("expo-audio");
49
+ const expo_asset_1 = require("expo-asset");
50
+ // eslint-disable-next-line @typescript-eslint/no-require-imports
51
+ exports.FACE_SQUEEZE_LOCAL_MODULE = require('../assets/face-squeeze-local.glb');
52
+ // ---------------------------------------------------------------------------
53
+ // Constants (identical to Filament version)
54
+ // ---------------------------------------------------------------------------
55
+ const SPRING_DURATION_MS = 300;
56
+ const EMOTION_HOLD_MS = 900;
57
+ const IDLE_EMOTION_INTERVAL_MS = 30000;
58
+ const INITIAL_IDLE_EMOTION_DELAY_MS = 1200;
59
+ // eslint-disable-next-line @typescript-eslint/no-require-imports
60
+ const SOUND_OWIE = require('./sounds/owie.wav');
61
+ // eslint-disable-next-line @typescript-eslint/no-require-imports
62
+ const SOUND_STOP = require('./sounds/stop.wav');
63
+ // eslint-disable-next-line @typescript-eslint/no-require-imports
64
+ const SOUND_HAHA = require('./sounds/haha.wav');
65
+ const REACTION_SOUNDS = [
66
+ SOUND_OWIE,
67
+ SOUND_STOP,
68
+ SOUND_HAHA,
69
+ ];
70
+ const EMOTION_PRESETS = {
71
+ owie: {
72
+ eyeBlinkLeft: 1.0, eyeBlinkRight: 1.0,
73
+ eyeSquintLeft: 0.9, eyeSquintRight: 0.9,
74
+ browDownLeft: 0.8, browDownRight: 0.8,
75
+ browInnerUp: 0.6,
76
+ noseSneerLeft: 0.7, noseSneerRight: 0.7,
77
+ jawOpen: 0.35,
78
+ mouthStretchLeft: 0.6, mouthStretchRight: 0.6,
79
+ },
80
+ happy: {
81
+ mouthSmileLeft: 0.85, mouthSmileRight: 0.85,
82
+ cheekSquintLeft: 0.6, cheekSquintRight: 0.6,
83
+ mouthDimpleLeft: 0.4, mouthDimpleRight: 0.4,
84
+ },
85
+ surprised: {
86
+ eyeWideLeft: 0.9, eyeWideRight: 0.9,
87
+ browInnerUp: 0.8,
88
+ browOuterUpLeft: 0.7, browOuterUpRight: 0.7,
89
+ jawOpen: 0.4, mouthFunnel: 0.3,
90
+ },
91
+ angry: {
92
+ browDownLeft: 0.9, browDownRight: 0.9,
93
+ noseSneerLeft: 0.7, noseSneerRight: 0.7,
94
+ eyeSquintLeft: 0.6, eyeSquintRight: 0.6,
95
+ mouthPressLeft: 0.5, mouthPressRight: 0.5,
96
+ },
97
+ smirk: {
98
+ mouthSmileLeft: 0.9, mouthSmileRight: 0.1,
99
+ cheekSquintLeft: 0.5, mouthDimpleLeft: 0.35,
100
+ },
101
+ sad: {
102
+ browInnerUp: 0.9,
103
+ browDownLeft: 0.3, browDownRight: 0.3,
104
+ mouthFrownLeft: 0.8, mouthFrownRight: 0.8,
105
+ mouthPressLeft: 0.3, mouthPressRight: 0.3,
106
+ eyeLookDownLeft: 0.5, eyeLookDownRight: 0.5,
107
+ },
108
+ disgust: {
109
+ noseSneerLeft: 0.9, noseSneerRight: 0.9,
110
+ browDownLeft: 0.7, browDownRight: 0.7,
111
+ mouthShrugUpper: 0.6, mouthShrugLower: 0.4,
112
+ mouthLeft: 0.3,
113
+ eyeSquintLeft: 0.4, eyeSquintRight: 0.4,
114
+ },
115
+ wink: {
116
+ eyeBlinkLeft: 1.0,
117
+ mouthSmileLeft: 0.5, mouthSmileRight: 0.3,
118
+ cheekSquintLeft: 0.4,
119
+ },
120
+ sleepy: {
121
+ eyeBlinkLeft: 0.7, eyeBlinkRight: 0.7,
122
+ browDownLeft: 0.4, browDownRight: 0.4,
123
+ mouthShrugLower: 0.3, jawOpen: 0.15,
124
+ },
125
+ contempt: {
126
+ mouthSmileLeft: 0.6, mouthPressRight: 0.5,
127
+ mouthStretchRight: 0.3, browDownRight: 0.4,
128
+ eyeSquintRight: 0.3,
129
+ },
130
+ };
131
+ const IDLE_EMOTIONS = [
132
+ { key: 'happy', weight: 5 },
133
+ { key: 'surprised', weight: 2 },
134
+ { key: 'smirk', weight: 2 },
135
+ { key: 'sad', weight: 1 },
136
+ { key: 'wink', weight: 2 },
137
+ { key: 'sleepy', weight: 1 },
138
+ { key: 'contempt', weight: 1 },
139
+ ];
140
+ function pickIdleEmotion() {
141
+ const total = IDLE_EMOTIONS.reduce((s, e) => s + e.weight, 0);
142
+ let r = Math.random() * total;
143
+ for (const e of IDLE_EMOTIONS) {
144
+ r -= e.weight;
145
+ if (r <= 0)
146
+ return e.key;
147
+ }
148
+ return IDLE_EMOTIONS[0].key;
149
+ }
150
+ // ---------------------------------------------------------------------------
151
+ // Inner R3F scene — runs inside Canvas
152
+ // ---------------------------------------------------------------------------
153
+ function AvatarScene({ uri, morphState, onReady, }) {
154
+ const { camera } = (0, native_1.useThree)();
155
+ const gltf = (0, native_2.useGLTF)(uri);
156
+ const readyFiredRef = (0, react_1.useRef)(false);
157
+ // Position camera to match Filament version
158
+ (0, react_1.useLayoutEffect)(() => {
159
+ if (!(camera instanceof THREE.PerspectiveCamera))
160
+ return;
161
+ camera.fov = 34;
162
+ camera.position.set(0, 1.55, 1.05);
163
+ camera.lookAt(0, 1.55, 0);
164
+ camera.updateProjectionMatrix();
165
+ }, [camera]);
166
+ // Find best mesh and build morph index
167
+ (0, react_1.useEffect)(() => {
168
+ if (!gltf?.scene)
169
+ return;
170
+ let best = null;
171
+ let bestCount = 0;
172
+ gltf.scene.traverse((node) => {
173
+ if (!(node instanceof THREE.Mesh))
174
+ return;
175
+ const count = Object.keys(node.morphTargetDictionary ?? {}).length;
176
+ if (count > bestCount) {
177
+ bestCount = count;
178
+ best = node;
179
+ }
180
+ });
181
+ if (!best)
182
+ return;
183
+ const mesh = best;
184
+ morphState.current.mesh = mesh;
185
+ const idx = new Map();
186
+ for (const [name, i] of Object.entries(mesh.morphTargetDictionary ?? {})) {
187
+ idx.set(name, i);
188
+ idx.set(name.toLowerCase(), i);
189
+ }
190
+ morphState.current.index = idx;
191
+ if (!readyFiredRef.current) {
192
+ readyFiredRef.current = true;
193
+ onReady();
194
+ }
195
+ }, [gltf, morphState, onReady]);
196
+ // Apply morphState.weights to mesh every frame
197
+ (0, native_1.useFrame)(() => {
198
+ const { mesh, weights, index } = morphState.current;
199
+ if (!mesh?.morphTargetInfluences)
200
+ return;
201
+ for (const [name, w] of Object.entries(weights)) {
202
+ const i = index.get(name) ?? index.get(name.toLowerCase());
203
+ if (i !== undefined)
204
+ mesh.morphTargetInfluences[i] = w;
205
+ }
206
+ });
207
+ return ((0, jsx_runtime_1.jsxs)(jsx_runtime_1.Fragment, { children: [(0, jsx_runtime_1.jsx)("ambientLight", { intensity: 0.65 }), (0, jsx_runtime_1.jsx)("directionalLight", { position: [1, 3, 2], intensity: 1.3 }), (0, jsx_runtime_1.jsx)("primitive", { object: gltf.scene })] }));
208
+ }
209
+ // ---------------------------------------------------------------------------
210
+ // Main component
211
+ // ---------------------------------------------------------------------------
212
+ function FaceSqueezeEditor({ onClose }) {
213
+ const [localUri, setLocalUri] = (0, react_1.useState)(null);
214
+ const [isReady, setIsReady] = (0, react_1.useState)(false);
215
+ (0, react_1.useEffect)(() => {
216
+ let cancelled = false;
217
+ (async () => {
218
+ try {
219
+ const asset = expo_asset_1.Asset.fromModule(exports.FACE_SQUEEZE_LOCAL_MODULE);
220
+ await asset.downloadAsync();
221
+ const uri = asset.localUri ?? asset.uri;
222
+ if (!cancelled && uri)
223
+ setLocalUri(uri);
224
+ }
225
+ catch (e) {
226
+ console.error('[WgpuFaceSqueezeEditor] Failed to load local GLB:', e);
227
+ }
228
+ })();
229
+ return () => { cancelled = true; };
230
+ }, []);
231
+ // Shared morph state — mutated directly, never triggers re-render
232
+ const morphState = (0, react_1.useRef)({
233
+ weights: {},
234
+ index: new Map(),
235
+ mesh: null,
236
+ });
237
+ // ---- Audio ----
238
+ const reactionSoundIdx = (0, react_1.useRef)(0);
239
+ const player = (0, expo_audio_1.useAudioPlayer)(REACTION_SOUNDS[0]);
240
+ // ---- Timers ----
241
+ const springRef = (0, react_1.useRef)(null);
242
+ const emotionHoldTimerRef = (0, react_1.useRef)(null);
243
+ const idleEmotionTimerRef = (0, react_1.useRef)(null);
244
+ const lipSyncIntervalRef = (0, react_1.useRef)(null);
245
+ // ---- View geometry (for gesture zone math) ----
246
+ const viewSize = (0, react_1.useRef)({ width: 1, height: 1 });
247
+ const viewWidthSv = (0, react_native_reanimated_1.useSharedValue)(1);
248
+ const viewHeightSv = (0, react_native_reanimated_1.useSharedValue)(1);
249
+ const touchStartXSv = (0, react_native_reanimated_1.useSharedValue)(0);
250
+ const touchStartYSv = (0, react_native_reanimated_1.useSharedValue)(0);
251
+ const touchZoneSv = (0, react_native_reanimated_1.useSharedValue)('cheek_left');
252
+ const lastPanUpdateTsSv = (0, react_native_reanimated_1.useSharedValue)(0);
253
+ const onLayout = (0, react_1.useCallback)((e) => {
254
+ const w = Math.max(1, e.nativeEvent.layout.width);
255
+ const h = Math.max(1, e.nativeEvent.layout.height);
256
+ viewSize.current = { width: w, height: h };
257
+ viewWidthSv.value = w;
258
+ viewHeightSv.value = h;
259
+ }, [viewHeightSv, viewWidthSv]);
260
+ // ---- Weight helpers ----
261
+ const applyWeights = (0, react_1.useCallback)((targets) => {
262
+ for (const [name, w] of Object.entries(targets)) {
263
+ morphState.current.weights[name] = Math.max(0, Math.min(1, w));
264
+ }
265
+ }, []);
266
+ const stopSpring = (0, react_1.useCallback)(() => {
267
+ if (springRef.current !== null) {
268
+ cancelAnimationFrame(springRef.current);
269
+ springRef.current = null;
270
+ }
271
+ }, []);
272
+ const springToZero = (0, react_1.useCallback)(() => {
273
+ stopSpring();
274
+ const snapshot = { ...morphState.current.weights };
275
+ const startTime = Date.now();
276
+ const tick = () => {
277
+ const t = Math.min(1, (Date.now() - startTime) / SPRING_DURATION_MS);
278
+ const eased = 1 - Math.pow(1 - t, 3);
279
+ for (const name of Object.keys(snapshot)) {
280
+ morphState.current.weights[name] = snapshot[name] * (1 - eased);
281
+ }
282
+ if (t >= 1) {
283
+ springRef.current = null;
284
+ morphState.current.weights = {};
285
+ return;
286
+ }
287
+ springRef.current = requestAnimationFrame(tick);
288
+ };
289
+ springRef.current = requestAnimationFrame(tick);
290
+ }, [stopSpring]);
291
+ const playEmotionBurst = (0, react_1.useCallback)((emotionKey) => {
292
+ const preset = EMOTION_PRESETS[emotionKey];
293
+ if (!preset)
294
+ return;
295
+ if (emotionHoldTimerRef.current !== null) {
296
+ clearTimeout(emotionHoldTimerRef.current);
297
+ emotionHoldTimerRef.current = null;
298
+ }
299
+ stopSpring();
300
+ applyWeights(preset);
301
+ emotionHoldTimerRef.current = setTimeout(() => {
302
+ emotionHoldTimerRef.current = null;
303
+ springToZero();
304
+ }, EMOTION_HOLD_MS);
305
+ }, [applyWeights, springToZero, stopSpring]);
306
+ const stopLipSync = (0, react_1.useCallback)(() => {
307
+ if (lipSyncIntervalRef.current !== null) {
308
+ clearInterval(lipSyncIntervalRef.current);
309
+ lipSyncIntervalRef.current = null;
310
+ }
311
+ applyWeights({ jawOpen: 0, mouthFunnel: 0, mouthPucker: 0,
312
+ mouthUpperUpLeft: 0, mouthUpperUpRight: 0,
313
+ mouthLowerDownLeft: 0, mouthLowerDownRight: 0,
314
+ mouthShrugLower: 0, mouthShrugUpper: 0 });
315
+ }, [applyWeights]);
316
+ const startLipSync = (0, react_1.useCallback)((durationMs) => {
317
+ if (lipSyncIntervalRef.current !== null)
318
+ clearInterval(lipSyncIntervalRef.current);
319
+ const startTime = Date.now();
320
+ const TICK_MS = 60;
321
+ lipSyncIntervalRef.current = setInterval(() => {
322
+ const elapsed = Date.now() - startTime;
323
+ if (elapsed >= durationMs) {
324
+ stopLipSync();
325
+ return;
326
+ }
327
+ const fadeIn = Math.min(1, elapsed / 80);
328
+ const fadeOut = Math.min(1, (durationMs - elapsed) / 150);
329
+ const envelope = Math.min(fadeIn, fadeOut);
330
+ const t = elapsed / 1000;
331
+ const jaw = envelope * (0.45 + 0.35 * Math.abs(Math.sin(t * 9.5 + Math.random() * 0.3)));
332
+ const funnel = envelope * Math.max(0, Math.sin(t * 7.2) * 0.5);
333
+ const shrug = envelope * Math.max(0, Math.sin(t * 5.8 + 1.2) * 0.35);
334
+ applyWeights({
335
+ jawOpen: jaw, mouthFunnel: funnel,
336
+ mouthShrugLower: shrug, mouthShrugUpper: shrug * 0.6,
337
+ mouthLowerDownLeft: jaw * 0.6, mouthLowerDownRight: jaw * 0.6,
338
+ mouthUpperUpLeft: jaw * 0.35, mouthUpperUpRight: jaw * 0.35,
339
+ });
340
+ }, TICK_MS);
341
+ }, [applyWeights, stopLipSync]);
342
+ const playReaction = (0, react_1.useCallback)((durationMs = 1400) => {
343
+ const idx = reactionSoundIdx.current % REACTION_SOUNDS.length;
344
+ reactionSoundIdx.current += 1;
345
+ try {
346
+ player.replace(REACTION_SOUNDS[idx]);
347
+ player.play();
348
+ }
349
+ catch { /* ignore */ }
350
+ startLipSync(durationMs);
351
+ }, [player, startLipSync]);
352
+ // Cleanup
353
+ (0, react_1.useEffect)(() => () => {
354
+ if (springRef.current !== null)
355
+ cancelAnimationFrame(springRef.current);
356
+ if (emotionHoldTimerRef.current !== null)
357
+ clearTimeout(emotionHoldTimerRef.current);
358
+ if (idleEmotionTimerRef.current !== null)
359
+ clearInterval(idleEmotionTimerRef.current);
360
+ if (lipSyncIntervalRef.current !== null)
361
+ clearInterval(lipSyncIntervalRef.current);
362
+ }, []);
363
+ // Idle emotion bursts (start once ready)
364
+ (0, react_1.useEffect)(() => {
365
+ if (!isReady)
366
+ return;
367
+ const initialKickoff = setTimeout(() => playEmotionBurst(pickIdleEmotion()), INITIAL_IDLE_EMOTION_DELAY_MS);
368
+ idleEmotionTimerRef.current = setInterval(() => playEmotionBurst(pickIdleEmotion()), IDLE_EMOTION_INTERVAL_MS);
369
+ return () => {
370
+ clearTimeout(initialKickoff);
371
+ if (idleEmotionTimerRef.current !== null)
372
+ clearInterval(idleEmotionTimerRef.current);
373
+ };
374
+ }, [isReady, playEmotionBurst]);
375
+ // ---- Gesture callbacks (JS side — called via runOnJS) ----
376
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars
377
+ const onPanBeginJs = (0, react_1.useCallback)((_x, _y, _zone) => {
378
+ stopSpring();
379
+ Haptics.impactAsync(Haptics.ImpactFeedbackStyle.Light);
380
+ }, [stopSpring]);
381
+ const onPanUpdateJs = (0, react_1.useCallback)((zone, dx, dy, isLeftHalf) => {
382
+ const u = {};
383
+ if (zone === 'brow') {
384
+ if (dy < 0) {
385
+ u['browInnerUp'] = Math.abs(dy) * 10;
386
+ u['browOuterUpLeft'] = Math.abs(dy) * 10;
387
+ u['browOuterUpRight'] = Math.abs(dy) * 10;
388
+ u['eyeWideLeft'] = Math.abs(dy) * 8;
389
+ u['eyeWideRight'] = Math.abs(dy) * 8;
390
+ }
391
+ else {
392
+ u['browDownLeft'] = Math.abs(dy) * 10;
393
+ u['browDownRight'] = Math.abs(dy) * 10;
394
+ u['eyeSquintLeft'] = Math.abs(dy) * 8;
395
+ u['eyeSquintRight'] = Math.abs(dy) * 8;
396
+ u['noseSneerLeft'] = Math.abs(dy) * 6;
397
+ u['noseSneerRight'] = Math.abs(dy) * 6;
398
+ }
399
+ u['browOuterUpLeft'] = (u['browOuterUpLeft'] ?? 0) + (dx < 0 ? Math.abs(dx) * 8 : 0);
400
+ u['browOuterUpRight'] = (u['browOuterUpRight'] ?? 0) + (dx > 0 ? Math.abs(dx) * 8 : 0);
401
+ }
402
+ else if (zone === 'eye') {
403
+ u[isLeftHalf ? 'eyeBlinkLeft' : 'eyeBlinkRight'] = dy > 0 ? Math.min(1, Math.abs(dy) * 10) : 0;
404
+ u[isLeftHalf ? 'eyeSquintLeft' : 'eyeSquintRight'] = dy > 0 ? Math.abs(dy) * 6 : 0;
405
+ u[isLeftHalf ? 'eyeWideLeft' : 'eyeWideRight'] = dy < 0 ? Math.abs(dy) * 8 : 0;
406
+ u[isLeftHalf ? 'eyeLookOutLeft' : 'eyeLookInRight'] = dx < 0 ? Math.abs(dx) * 7 : 0;
407
+ u[isLeftHalf ? 'eyeLookInLeft' : 'eyeLookOutRight'] = dx > 0 ? Math.abs(dx) * 7 : 0;
408
+ u[isLeftHalf ? 'eyeLookUpLeft' : 'eyeLookUpRight'] = dy < 0 ? Math.abs(dy) * 5 : 0;
409
+ u[isLeftHalf ? 'eyeLookDownLeft' : 'eyeLookDownRight'] = dy > 0 ? Math.abs(dy) * 5 : 0;
410
+ }
411
+ else if (zone === 'cheek_left') {
412
+ u['cheekPuff'] = Math.abs(dx) * 6;
413
+ u['cheekSquintLeft'] = Math.abs(dy) * 6;
414
+ u['cheekSquintRight'] = 0;
415
+ u['mouthSmileLeft'] = dy < 0 ? Math.abs(dy) * 5 : 0;
416
+ u['mouthDimpleLeft'] = dy < 0 ? Math.abs(dy) * 4 : 0;
417
+ u['mouthStretchLeft'] = dy > 0 ? Math.abs(dy) * 5 : 0;
418
+ u['mouthFrownLeft'] = dy > 0 ? Math.abs(dy) * 4 : 0;
419
+ u['mouthPressLeft'] = dy > 0 ? Math.abs(dy) * 3 : 0;
420
+ }
421
+ else if (zone === 'cheek_right') {
422
+ u['cheekPuff'] = Math.abs(dx) * 6;
423
+ u['cheekSquintRight'] = Math.abs(dy) * 6;
424
+ u['cheekSquintLeft'] = 0;
425
+ u['mouthSmileRight'] = dy < 0 ? Math.abs(dy) * 5 : 0;
426
+ u['mouthDimpleRight'] = dy < 0 ? Math.abs(dy) * 4 : 0;
427
+ u['mouthStretchRight'] = dy > 0 ? Math.abs(dy) * 5 : 0;
428
+ u['mouthFrownRight'] = dy > 0 ? Math.abs(dy) * 4 : 0;
429
+ u['mouthPressRight'] = dy > 0 ? Math.abs(dy) * 3 : 0;
430
+ }
431
+ else if (zone === 'jaw') {
432
+ u['jawOpen'] = dy > 0 ? Math.abs(dy) * 10 : 0;
433
+ u['tongueOut'] = dy > 0 ? Math.max(0, Math.abs(dy) * 10 - 0.7) : 0;
434
+ u['mouthLowerDownLeft'] = dy > 0 ? Math.abs(dy) * 6 : 0;
435
+ u['mouthLowerDownRight'] = dy > 0 ? Math.abs(dy) * 6 : 0;
436
+ u['mouthUpperUpLeft'] = dy > 0 ? Math.abs(dy) * 4 : 0;
437
+ u['mouthUpperUpRight'] = dy > 0 ? Math.abs(dy) * 4 : 0;
438
+ u['mouthPressLeft'] = dy < 0 ? Math.abs(dy) * 6 : 0;
439
+ u['mouthPressRight'] = dy < 0 ? Math.abs(dy) * 6 : 0;
440
+ u['jawLeft'] = dx < 0 ? Math.abs(dx) * 8 : 0;
441
+ u['jawRight'] = dx > 0 ? Math.abs(dx) * 8 : 0;
442
+ u['jawForward'] = Math.abs(dx) * 4;
443
+ u['mouthLeft'] = dx < 0 ? Math.abs(dx) * 5 : 0;
444
+ u['mouthRight'] = dx > 0 ? Math.abs(dx) * 5 : 0;
445
+ }
446
+ applyWeights(u);
447
+ }, [applyWeights]);
448
+ const onPanEndJs = (0, react_1.useCallback)(() => {
449
+ if (emotionHoldTimerRef.current !== null)
450
+ return;
451
+ springToZero();
452
+ }, [springToZero]);
453
+ const onPinchBeginJs = (0, react_1.useCallback)(() => {
454
+ stopSpring();
455
+ Haptics.impactAsync(Haptics.ImpactFeedbackStyle.Medium);
456
+ }, [stopSpring]);
457
+ const onPinchUpdateJs = (0, react_1.useCallback)((scale, delta) => {
458
+ if (scale < 1) {
459
+ applyWeights({ mouthPucker: delta * 2, mouthFunnel: 0, mouthRollLower: delta * 1.2, mouthRollUpper: delta * 1.2 });
460
+ }
461
+ else {
462
+ applyWeights({ mouthFunnel: delta * 2, mouthPucker: 0, mouthShrugLower: delta * 0.8, mouthShrugUpper: delta * 0.8, mouthRollLower: 0, mouthRollUpper: 0 });
463
+ }
464
+ }, [applyWeights]);
465
+ const onPinchEndJs = (0, react_1.useCallback)(() => {
466
+ if (emotionHoldTimerRef.current !== null)
467
+ return;
468
+ springToZero();
469
+ }, [springToZero]);
470
+ const onEyeTapJs = (0, react_1.useCallback)((x, y) => {
471
+ const relY = y / Math.max(1, viewSize.current.height);
472
+ if (relY < 0.16 || relY > 0.52)
473
+ return;
474
+ Haptics.impactAsync(Haptics.ImpactFeedbackStyle.Heavy);
475
+ playReaction();
476
+ playEmotionBurst('owie');
477
+ }, [playEmotionBurst, playReaction]);
478
+ // ---- Gestures ----
479
+ const panGesture = react_native_gesture_handler_1.Gesture.Pan()
480
+ .minDistance(0)
481
+ .onBegin((e) => {
482
+ 'worklet';
483
+ touchStartXSv.value = e.x;
484
+ touchStartYSv.value = e.y;
485
+ const relY = e.y / Math.max(1, viewHeightSv.value);
486
+ let zone;
487
+ if (relY < 0.20)
488
+ zone = 'brow';
489
+ else if (relY < 0.38)
490
+ zone = 'eye';
491
+ else if (relY > 0.72)
492
+ zone = 'jaw';
493
+ else
494
+ zone = e.x < viewWidthSv.value / 2 ? 'cheek_left' : 'cheek_right';
495
+ touchZoneSv.value = zone;
496
+ (0, react_native_reanimated_1.runOnJS)(onPanBeginJs)(e.x, e.y, zone);
497
+ })
498
+ .onUpdate((e) => {
499
+ 'worklet';
500
+ const now = Date.now();
501
+ if (now - lastPanUpdateTsSv.value < 33)
502
+ return;
503
+ lastPanUpdateTsSv.value = now;
504
+ const w = Math.max(1, viewWidthSv.value);
505
+ const h = Math.max(1, viewHeightSv.value);
506
+ const dx = (e.x - touchStartXSv.value) / w;
507
+ const dy = (e.y - touchStartYSv.value) / h;
508
+ (0, react_native_reanimated_1.runOnJS)(onPanUpdateJs)(touchZoneSv.value, dx, dy, touchStartXSv.value < w / 2);
509
+ })
510
+ .onEnd(() => {
511
+ 'worklet';
512
+ (0, react_native_reanimated_1.runOnJS)(onPanEndJs)();
513
+ });
514
+ const pinchGesture = react_native_gesture_handler_1.Gesture.Pinch()
515
+ .onBegin(() => {
516
+ 'worklet';
517
+ (0, react_native_reanimated_1.runOnJS)(onPinchBeginJs)();
518
+ })
519
+ .onUpdate((e) => {
520
+ 'worklet';
521
+ (0, react_native_reanimated_1.runOnJS)(onPinchUpdateJs)(e.scale, Math.abs(1 - e.scale));
522
+ })
523
+ .onEnd(() => {
524
+ 'worklet';
525
+ (0, react_native_reanimated_1.runOnJS)(onPinchEndJs)();
526
+ });
527
+ const tapGesture = react_native_gesture_handler_1.Gesture.Tap()
528
+ .onEnd((e, success) => {
529
+ 'worklet';
530
+ if (success)
531
+ (0, react_native_reanimated_1.runOnJS)(onEyeTapJs)(e.x, e.y);
532
+ });
533
+ const composed = react_native_gesture_handler_1.Gesture.Simultaneous(panGesture, pinchGesture, tapGesture);
534
+ const handleReady = (0, react_1.useCallback)(() => setIsReady(true), []);
535
+ return ((0, jsx_runtime_1.jsx)(react_native_gesture_handler_1.GestureHandlerRootView, { style: { flex: 1 }, children: (0, jsx_runtime_1.jsxs)(react_native_1.View, { style: styles.container, children: [(0, jsx_runtime_1.jsx)(react_native_1.View, { style: styles.scene, children: localUri ? ((0, jsx_runtime_1.jsx)(native_1.Canvas, { style: react_native_1.StyleSheet.absoluteFill, camera: { fov: 34, position: [0, 1.55, 1.05], near: 0.01, far: 50 }, gl: { antialias: true, alpha: false }, onCreated: ({ gl }) => gl.setClearColor(new THREE.Color('#0a0a0a')), children: (0, jsx_runtime_1.jsx)(AvatarScene, { uri: localUri, morphState: morphState, onReady: handleReady }) })) : ((0, jsx_runtime_1.jsx)(react_native_1.View, { style: styles.loading, children: (0, jsx_runtime_1.jsx)(react_native_1.Text, { style: styles.loadingText, children: "Loading face\u2026" }) })) }), (0, jsx_runtime_1.jsx)(react_native_gesture_handler_1.GestureDetector, { gesture: composed, children: (0, jsx_runtime_1.jsx)(react_native_1.View, { style: react_native_1.StyleSheet.absoluteFill, onLayout: onLayout, collapsable: false }) }), (0, jsx_runtime_1.jsx)(react_native_1.Pressable, { style: styles.closeBtn, onPress: onClose, children: (0, jsx_runtime_1.jsx)(react_native_1.Text, { style: styles.closeBtnText, children: "\u2715 Done squeezing" }) }), (0, jsx_runtime_1.jsx)(react_native_1.Text, { style: styles.hint, children: "Drag to stretch \u00B7 Pinch lips \u00B7 Poke an eye \uD83D\uDC41" })] }) }));
536
+ }
537
+ exports.FaceSqueezeEditor = FaceSqueezeEditor;
538
+ const styles = react_native_1.StyleSheet.create({
539
+ container: { flex: 1, backgroundColor: '#0a0a0a' },
540
+ scene: { flex: 1 },
541
+ loading: { flex: 1, alignItems: 'center', justifyContent: 'center' },
542
+ loadingText: { color: '#97a3b5', fontSize: 14 },
543
+ closeBtn: {
544
+ position: 'absolute', top: 52, right: 16,
545
+ backgroundColor: '#6c63ff', borderRadius: 20,
546
+ paddingHorizontal: 16, paddingVertical: 8,
547
+ },
548
+ closeBtnText: { color: '#fff', fontWeight: '700', fontSize: 14 },
549
+ hint: {
550
+ position: 'absolute', bottom: 40, alignSelf: 'center',
551
+ color: '#97a3b5', fontSize: 13,
552
+ },
553
+ });
@@ -0,0 +1,6 @@
1
+ export declare const FACE_SQUEEZE_LOCAL_MODULE: number;
2
+ export interface FaceSqueezeEditorProps {
3
+ onClose: () => void;
4
+ }
5
+ export declare function FaceSqueezeEditor({ onClose }: FaceSqueezeEditorProps): import("react/jsx-runtime").JSX.Element;
6
+ export default FaceSqueezeEditor;
@@ -0,0 +1,54 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.FaceSqueezeEditor = exports.FACE_SQUEEZE_LOCAL_MODULE = void 0;
4
+ const jsx_runtime_1 = require("react/jsx-runtime");
5
+ const react_native_1 = require("react-native");
6
+ // eslint-disable-next-line @typescript-eslint/no-require-imports
7
+ exports.FACE_SQUEEZE_LOCAL_MODULE = require('../assets/face-squeeze-local.glb');
8
+ function FaceSqueezeEditor({ onClose }) {
9
+ return ((0, jsx_runtime_1.jsx)(react_native_1.View, { style: styles.container, children: (0, jsx_runtime_1.jsxs)(react_native_1.View, { style: styles.card, children: [(0, jsx_runtime_1.jsx)(react_native_1.Text, { style: styles.title, children: "Face squeeze editor is mobile-only." }), (0, jsx_runtime_1.jsx)(react_native_1.Text, { style: styles.subtitle, children: "This placeholder keeps web builds safe while preserving native functionality." }), (0, jsx_runtime_1.jsx)(react_native_1.Pressable, { style: styles.closeButton, onPress: onClose, children: (0, jsx_runtime_1.jsx)(react_native_1.Text, { style: styles.closeButtonText, children: "Close" }) })] }) }));
10
+ }
11
+ exports.FaceSqueezeEditor = FaceSqueezeEditor;
12
+ const styles = react_native_1.StyleSheet.create({
13
+ container: {
14
+ flex: 1,
15
+ alignItems: "center",
16
+ justifyContent: "center",
17
+ backgroundColor: "#0f1115",
18
+ padding: 20,
19
+ },
20
+ card: {
21
+ width: "100%",
22
+ maxWidth: 460,
23
+ borderRadius: 12,
24
+ borderWidth: 1,
25
+ borderColor: "#2d3340",
26
+ backgroundColor: "#171b23",
27
+ padding: 16,
28
+ gap: 10,
29
+ },
30
+ title: {
31
+ color: "#f1f4f8",
32
+ fontSize: 18,
33
+ fontWeight: "700",
34
+ },
35
+ subtitle: {
36
+ color: "#98a1b3",
37
+ fontSize: 14,
38
+ lineHeight: 20,
39
+ },
40
+ closeButton: {
41
+ marginTop: 6,
42
+ alignSelf: "flex-start",
43
+ borderRadius: 8,
44
+ backgroundColor: "#2e8f5b",
45
+ paddingHorizontal: 12,
46
+ paddingVertical: 8,
47
+ },
48
+ closeButtonText: {
49
+ color: "#ffffff",
50
+ fontSize: 14,
51
+ fontWeight: "600",
52
+ },
53
+ });
54
+ exports.default = FaceSqueezeEditor;
@@ -3,7 +3,8 @@ export { AvatarEditor } from './AvatarEditor';
3
3
  export { AvatarModel } from './AvatarModel';
4
4
  export { RigidAccessory } from './RigidAccessory';
5
5
  export { SkinnedClothing } from './SkinnedClothing';
6
+ export { FaceSqueezeEditor, FACE_SQUEEZE_LOCAL_MODULE } from './FaceSqueezeEditor';
6
7
  export { studioTheme, withAlpha } from './studioTheme';
7
8
  export { useBoneSnap, findNearestBone, getWorldPositionForBone, getWorldPositionForPlacement, snapPlacementToNearestBone, HUMANOID_BONES } from './boneSnap';
8
- export type { AvatarEditorProps, AssetPlacement, EquippedAsset } from './types';
9
+ export type { AvatarEditorProps, AssetPlacement, EquippedAsset, FaceSqueezeEditorProps } from './types';
9
10
  export type { BoneSnapResult, HumanoidBone } from './boneSnap';
@@ -1,6 +1,6 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.HUMANOID_BONES = exports.snapPlacementToNearestBone = exports.getWorldPositionForPlacement = exports.getWorldPositionForBone = exports.findNearestBone = exports.useBoneSnap = exports.withAlpha = exports.studioTheme = exports.SkinnedClothing = exports.RigidAccessory = exports.AvatarModel = exports.AvatarEditor = exports.AvatarCanvas = void 0;
3
+ exports.HUMANOID_BONES = exports.snapPlacementToNearestBone = exports.getWorldPositionForPlacement = exports.getWorldPositionForBone = exports.findNearestBone = exports.useBoneSnap = exports.withAlpha = exports.studioTheme = exports.FACE_SQUEEZE_LOCAL_MODULE = exports.FaceSqueezeEditor = exports.SkinnedClothing = exports.RigidAccessory = exports.AvatarModel = exports.AvatarEditor = exports.AvatarCanvas = void 0;
4
4
  var AvatarCanvas_1 = require("./AvatarCanvas");
5
5
  Object.defineProperty(exports, "AvatarCanvas", { enumerable: true, get: function () { return AvatarCanvas_1.AvatarCanvas; } });
6
6
  var AvatarEditor_1 = require("./AvatarEditor");
@@ -11,6 +11,9 @@ var RigidAccessory_1 = require("./RigidAccessory");
11
11
  Object.defineProperty(exports, "RigidAccessory", { enumerable: true, get: function () { return RigidAccessory_1.RigidAccessory; } });
12
12
  var SkinnedClothing_1 = require("./SkinnedClothing");
13
13
  Object.defineProperty(exports, "SkinnedClothing", { enumerable: true, get: function () { return SkinnedClothing_1.SkinnedClothing; } });
14
+ var FaceSqueezeEditor_1 = require("./FaceSqueezeEditor");
15
+ Object.defineProperty(exports, "FaceSqueezeEditor", { enumerable: true, get: function () { return FaceSqueezeEditor_1.FaceSqueezeEditor; } });
16
+ Object.defineProperty(exports, "FACE_SQUEEZE_LOCAL_MODULE", { enumerable: true, get: function () { return FaceSqueezeEditor_1.FACE_SQUEEZE_LOCAL_MODULE; } });
14
17
  var studioTheme_1 = require("./studioTheme");
15
18
  Object.defineProperty(exports, "studioTheme", { enumerable: true, get: function () { return studioTheme_1.studioTheme; } });
16
19
  Object.defineProperty(exports, "withAlpha", { enumerable: true, get: function () { return studioTheme_1.withAlpha; } });
@@ -39,3 +39,6 @@ export interface AvatarEditorProps {
39
39
  onDone?: () => void;
40
40
  skinColor?: string;
41
41
  }
42
+ export interface FaceSqueezeEditorProps {
43
+ onClose: () => void;
44
+ }
package/dist/html.d.ts CHANGED
@@ -15,5 +15,10 @@ export type AvatarConfig = {
15
15
  * Example: "https://studio.sitebay.org/vendor"
16
16
  */
17
17
  vendorBaseUrl?: string | null;
18
+ /**
19
+ * URL to the DRACO decoder WASM/JS folder.
20
+ * Defaults to gstatic's v1.5.6 CDN.
21
+ */
22
+ dracoDecoderUrl?: string | null;
18
23
  };
19
24
  export declare function buildAvatarHtml(config: AvatarConfig): string;
package/dist/html.js CHANGED
@@ -20,6 +20,7 @@ function buildAvatarHtml(config) {
20
20
  const headAudioUrl = v ? `${v}/headaudio.min.mjs` : 'https://cdn.jsdelivr.net/gh/met4citizen/HeadAudio@v0.1.0-alpha/dist/headaudio.min.mjs';
21
21
  const headWorkletUrl = v ? `${v}/headworklet.min.mjs` : 'https://cdn.jsdelivr.net/gh/met4citizen/HeadAudio@v0.1.0-alpha/dist/headworklet.min.mjs';
22
22
  const headModelUrl = v ? `${v}/model-en-mixed.bin` : 'https://cdn.jsdelivr.net/gh/met4citizen/HeadAudio@v0.1.0-alpha/dist/model-en-mixed.bin';
23
+ const dracoUrl = config.dracoDecoderUrl ?? (v ? `${v}/draco/` : 'https://www.gstatic.com/draco/versioned/decoders/1.5.6/');
23
24
  return `
24
25
  <!DOCTYPE html>
25
26
  <html>
@@ -40,25 +41,26 @@ function buildAvatarHtml(config) {
40
41
  }
41
42
  </script>
42
43
  <script>
43
- window.ReactNativeWebView?.postMessage(
44
+ function rnPost(d) { window.ReactNativeWebView && window.ReactNativeWebView.postMessage(d); }
45
+ rnPost(
44
46
  JSON.stringify({ type: 'log', message: '[bootstrap] inline script start' })
45
47
  );
46
48
  function postBootstrapError(kind, message) {
47
- window.ReactNativeWebView?.postMessage(
49
+ rnPost(
48
50
  JSON.stringify({ type: 'error', message: '[' + kind + '] ' + String(message || 'Unknown error') })
49
51
  );
50
52
  }
51
53
  document.addEventListener('DOMContentLoaded', function() {
52
- window.ReactNativeWebView?.postMessage(
54
+ rnPost(
53
55
  JSON.stringify({ type: 'log', message: '[bootstrap] DOMContentLoaded' })
54
56
  );
55
57
  });
56
58
  window.addEventListener('error', function(event) {
57
- postBootstrapError('window.error', event?.message || event?.error?.message || 'Script error');
59
+ postBootstrapError('window.error', (event && event.message) || (event && event.error && event.error.message) || 'Script error');
58
60
  });
59
61
  window.addEventListener('unhandledrejection', function(event) {
60
- const reason = event?.reason;
61
- postBootstrapError('unhandledrejection', reason?.message || reason || 'Unhandled promise rejection');
62
+ const reason = event && event.reason;
63
+ postBootstrapError('unhandledrejection', (reason && reason.message) || reason || 'Unhandled promise rejection');
62
64
  });
63
65
  </script>
64
66
  </head>
@@ -66,10 +68,11 @@ window.addEventListener('unhandledrejection', function(event) {
66
68
  <div id="avatar"></div>
67
69
  <script type="module">
68
70
  (async function() {
69
- window.ReactNativeWebView?.postMessage(
71
+ rnPost(
70
72
  JSON.stringify({ type: 'log', message: '[module] script start' })
71
73
  );
72
74
  const AUTH_TOKEN = ${JSON.stringify(config.authToken ?? null)};
75
+ const DRACO_URL = ${JSON.stringify(dracoUrl)};
73
76
  const TALKING_HEAD_URL = ${JSON.stringify(talkingHeadUrl)};
74
77
  const HEAD_AUDIO_URL = ${JSON.stringify(headAudioUrl)};
75
78
  const HEAD_AUDIO_WORKLET = ${JSON.stringify(headWorkletUrl)};
@@ -94,11 +97,11 @@ let staticModel = null;
94
97
  let initStarted = false;
95
98
 
96
99
  function log(msg) {
97
- window.ReactNativeWebView?.postMessage(JSON.stringify({ type: 'log', message: msg }));
100
+ rnPost(JSON.stringify({ type: 'log', message: msg }));
98
101
  }
99
102
 
100
103
  function emitLoading(stage, progress = null) {
101
- window.ReactNativeWebView?.postMessage(
104
+ rnPost(
102
105
  JSON.stringify({ type: 'loading', stage, progress }),
103
106
  );
104
107
  }
@@ -125,13 +128,13 @@ function applyColorOverrides() {
125
128
  mats.forEach((mat) => {
126
129
  const name = (mat.name || '').toLowerCase();
127
130
  if (HAIR_COLOR && (name.includes('hair') || name.includes('fur'))) {
128
- mat.color?.set(HAIR_COLOR);
131
+ mat.color && mat.color.set(HAIR_COLOR);
129
132
  }
130
133
  if (SKIN_COLOR && (name.includes('skin') || name.includes('body') || name.includes('face'))) {
131
- mat.color?.set(SKIN_COLOR);
134
+ mat.color && mat.color.set(SKIN_COLOR);
132
135
  }
133
136
  if (EYE_COLOR && (name.includes('eye') || name.includes('iris'))) {
134
- mat.color?.set(EYE_COLOR);
137
+ mat.color && mat.color.set(EYE_COLOR);
135
138
  }
136
139
  });
137
140
  });
@@ -204,7 +207,7 @@ async function loadStaticFallback(loadedAvatarUrl) {
204
207
  applyAccessories(pendingAccessoriesList);
205
208
  motionInitFromRoot(staticModel);
206
209
  emitLoading('ready', 100);
207
- window.ReactNativeWebView?.postMessage(JSON.stringify({ type: 'ready' }));
210
+ rnPost(JSON.stringify({ type: 'ready' }));
208
211
 
209
212
  window.addEventListener('resize', () => {
210
213
  camera.aspect = window.innerWidth / window.innerHeight;
@@ -228,11 +231,11 @@ async function loadStaticFallback(loadedAvatarUrl) {
228
231
  }
229
232
  }, (err) => {
230
233
  log('Fallback Error: ' + err.message);
231
- window.ReactNativeWebView?.postMessage(JSON.stringify({ type: 'error', message: err.message }));
234
+ rnPost(JSON.stringify({ type: 'error', message: err.message }));
232
235
  });
233
236
  } catch (err) {
234
237
  log('Fallback setup error: ' + err.message);
235
- window.ReactNativeWebView?.postMessage(JSON.stringify({ type: 'error', message: err.message }));
238
+ rnPost(JSON.stringify({ type: 'error', message: err.message }));
236
239
  }
237
240
  }
238
241
 
@@ -299,7 +302,7 @@ async function init() {
299
302
  jawMorphCache = null;
300
303
  visemeMorphCache = null;
301
304
  rhubarbMorphCache = null;
302
- head.armature?.traverse((child) => {
305
+ head.armature && head.armature.traverse(function(child) {
303
306
  if (child.isMesh && child.morphTargetDictionary && child.morphTargetInfluences) {
304
307
  mouthMeshes.push(child);
305
308
  }
@@ -338,17 +341,17 @@ async function init() {
338
341
  applyAccessories(pendingAccessoriesList);
339
342
  if (head && head.armature) motionInitFromRoot(head.armature);
340
343
  emitLoading('ready', 100);
341
- window.ReactNativeWebView?.postMessage(JSON.stringify({ type: 'ready' }));
344
+ rnPost(JSON.stringify({ type: 'ready' }));
342
345
  }
343
346
  } catch (err) {
344
347
  initStarted = false; // allow retry on error
345
348
  log('Error: ' + err.message);
346
- window.ReactNativeWebView?.postMessage(JSON.stringify({ type: 'error', message: err.message }));
349
+ rnPost(JSON.stringify({ type: 'error', message: err.message }));
347
350
  }
348
351
  }
349
352
 
350
353
  function startAudioInterception() {
351
- if (!head?.audioCtx || !head?.audioSpeechGainNode) return;
354
+ if (!head || !head.audioCtx || !head.audioSpeechGainNode) return;
352
355
  const audioCtx = head.audioCtx;
353
356
  const gainNode = head.audioSpeechGainNode;
354
357
  const connected = new WeakSet();
@@ -400,14 +403,14 @@ function updateVoiceMood(rawAmplitude) {
400
403
  voiceMoodFramesBelow = 0;
401
404
  if (voiceMoodCurrent !== 'excited' && voiceMoodFramesAbove >= VOICE_MOOD_FRAMES_REQUIRED) {
402
405
  voiceMoodCurrent = 'excited';
403
- window.ReactNativeWebView?.postMessage(JSON.stringify({ type: 'voiceMood', mood: 'excited' }));
406
+ rnPost(JSON.stringify({ type: 'voiceMood', mood: 'excited' }));
404
407
  }
405
408
  } else if (voiceEnergySmoothed < VOICE_ENERGY_NEUTRAL_THRESH) {
406
409
  voiceMoodFramesBelow++;
407
410
  voiceMoodFramesAbove = 0;
408
411
  if (voiceMoodCurrent !== 'neutral' && voiceMoodFramesBelow >= VOICE_MOOD_FRAMES_REQUIRED * 3) {
409
412
  voiceMoodCurrent = 'neutral';
410
- window.ReactNativeWebView?.postMessage(JSON.stringify({ type: 'voiceMood', mood: 'neutral' }));
413
+ rnPost(JSON.stringify({ type: 'voiceMood', mood: 'neutral' }));
411
414
  }
412
415
  }
413
416
  }
@@ -578,7 +581,7 @@ function clearScheduledVisemes() {
578
581
  for (const key of Object.keys(visemeState)) visemeState[key] = 0;
579
582
  }
580
583
 
581
- function tickVisemeDecay(deltaSeconds?: number) {
584
+ function tickVisemeDecay(deltaSeconds) {
582
585
  if (!visemeMorphCache) return;
583
586
 
584
587
  const isScheduled = Date.now() < visemeModeUntil;
@@ -614,7 +617,7 @@ function tickVisemeDecay(deltaSeconds?: number) {
614
617
  // Use realtime (not newvalue) — newvalue is consumed and cleared after
615
618
  // a single frame, so scheduled visemes would vanish immediately.
616
619
  // realtime persists until explicitly set to null.
617
- if (head?.mtAvatar && e.morphName && head.mtAvatar[e.morphName]) {
620
+ if (head && head.mtAvatar && e.morphName && head.mtAvatar[e.morphName]) {
618
621
  const mt = head.mtAvatar[e.morphName];
619
622
  mt.realtime = targetWeight > 0 ? targetWeight : null;
620
623
  mt.needsUpdate = true;
@@ -750,7 +753,7 @@ async function ensureThree() {
750
753
  const { DRACOLoader } = await import('three/addons/loaders/DRACOLoader.js');
751
754
  gltfLoaderInstance = new GLTFLoader();
752
755
  const dracoLoader = new DRACOLoader();
753
- dracoLoader.setDecoderPath('https://www.gstatic.com/draco/versioned/decoders/1.5.6/');
756
+ dracoLoader.setDecoderPath(DRACO_URL);
754
757
  gltfLoaderInstance.setDRACOLoader(dracoLoader);
755
758
  }
756
759
  }
@@ -798,7 +801,7 @@ async function applyAccessories(accessoriesList) {
798
801
  if (loadedUrl.startsWith('blob:')) URL.revokeObjectURL(loadedUrl);
799
802
  log('[ACC] GLB loaded OK for ' + accData.id);
800
803
  const model = gltf.scene;
801
- const latestData = currentAccessories[accData.id]?.latestData || accData;
804
+ const latestData = (currentAccessories[accData.id] && currentAccessories[accData.id].latestData) || accData;
802
805
  let targetBone = null;
803
806
  let prefixCandidate = null;
804
807
  root.traverse((child) => {
@@ -1158,7 +1161,7 @@ function onIncomingMessage(event) {
1158
1161
  if (typeof window.playMotion === 'function') {
1159
1162
  window.playMotion(msg.name);
1160
1163
  }
1161
- window.ReactNativeWebView?.postMessage(JSON.stringify({ type: 'avatarState', state: 'motion:' + msg.name }));
1164
+ rnPost(JSON.stringify({ type: 'avatarState', state: 'motion:' + msg.name }));
1162
1165
  log('motion dispatched: ' + msg.name);
1163
1166
  }
1164
1167
  } catch (err) {
@@ -180,7 +180,7 @@ function sleep(ms, signal) {
180
180
  const id = setTimeout(resolve, ms);
181
181
  signal.addEventListener("abort", () => {
182
182
  clearTimeout(id);
183
- reject(Object.assign(new Error("AbortError"), { name: "AbortError" }));
183
+ reject(new DOMException("The operation was aborted", "AbortError"));
184
184
  }, { once: true });
185
185
  });
186
186
  }
@@ -178,6 +178,11 @@ exports.WgpuAvatar = (0, react_1.forwardRef)(({ style, avatarUrl, focalLength, m
178
178
  ? avatarUrl : null;
179
179
  const fileResult = (0, useAuthedModelUri_1.useAuthedModelUri)(remoteUrl);
180
180
  const lockedUriRef = (0, react_1.useRef)(null);
181
+ const lastUrlRef = (0, react_1.useRef)(avatarUrl);
182
+ if (lastUrlRef.current !== avatarUrl) {
183
+ lastUrlRef.current = avatarUrl;
184
+ lockedUriRef.current = null;
185
+ }
181
186
  const currentUri = fileResult?.uri ?? null;
182
187
  if (lockedUriRef.current === null && currentUri)
183
188
  lockedUriRef.current = currentUri;
@@ -232,7 +237,7 @@ exports.WgpuAvatar = (0, react_1.forwardRef)(({ style, avatarUrl, focalLength, m
232
237
  (0, react_1.useEffect)(() => () => clearScheduleTimers(), [clearScheduleTimers]);
233
238
  (0, react_1.useImperativeHandle)(ref, () => ({
234
239
  setMood: (m) => { morphStateRef.current.moodBase = { ...(morphTables_1.MOOD_MORPHS[m] ?? {}) }; },
235
- sendAmplitude: (_amplitude) => { },
240
+ sendAmplitude: () => { },
236
241
  sendViseme: (viseme, weight) => {
237
242
  const aliases = morphTables_1.VISEME_MORPH_ALIASES[viseme];
238
243
  if (!aliases)
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "talking-head-studio",
3
- "version": "0.4.6",
3
+ "version": "0.4.9",
4
4
  "description": "Cross-platform 3D avatar component for React Native & web — lip-sync, gestures, accessories, and LLM integration. Powered by TalkingHead + Three.js.",
5
5
  "main": "dist/index.web.js",
6
6
  "browser": "dist/index.web.js",
@@ -103,6 +103,7 @@
103
103
  "@react-three/fiber": ">=8",
104
104
  "expo": ">=51",
105
105
  "expo-asset": ">=10",
106
+ "expo-audio": ">=0.3",
106
107
  "expo-file-system": ">=17",
107
108
  "react": ">=18",
108
109
  "react-native": ">=0.73",
@@ -126,6 +127,9 @@
126
127
  "expo-asset": {
127
128
  "optional": true
128
129
  },
130
+ "expo-audio": {
131
+ "optional": true
132
+ },
129
133
  "expo-file-system": {
130
134
  "optional": true
131
135
  },