talking-head-studio 0.4.0 → 0.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,755 +0,0 @@
1
- "use strict";
2
- var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
- if (k2 === undefined) k2 = k;
4
- var desc = Object.getOwnPropertyDescriptor(m, k);
5
- if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
- desc = { enumerable: true, get: function() { return m[k]; } };
7
- }
8
- Object.defineProperty(o, k2, desc);
9
- }) : (function(o, m, k, k2) {
10
- if (k2 === undefined) k2 = k;
11
- o[k2] = m[k];
12
- }));
13
- var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
- Object.defineProperty(o, "default", { enumerable: true, value: v });
15
- }) : function(o, v) {
16
- o["default"] = v;
17
- });
18
- var __importStar = (this && this.__importStar) || function (mod) {
19
- if (mod && mod.__esModule) return mod;
20
- var result = {};
21
- if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
22
- __setModuleDefault(result, mod);
23
- return result;
24
- };
25
- Object.defineProperty(exports, "__esModule", { value: true });
26
- exports.FilamentAvatar = exports.CAMERA_FOCAL_PIP = exports.CAMERA_FOCAL_FULL = void 0;
27
- const jsx_runtime_1 = require("react/jsx-runtime");
28
- /* eslint-disable @typescript-eslint/no-unused-expressions */
29
- /* eslint-disable react-hooks/refs */
30
- /**
31
- * FilamentAvatar — native Filament-based avatar renderer.
32
- *
33
- * Replaces the TalkingHead WebView on iOS/Android with direct Filament
34
- * morph target writes — no WebView bridge, no JS→native→JS roundtrip,
35
- * visemes applied at 60fps in React Native JS.
36
- *
37
- * Implements the full TalkingHeadRef interface so it can be swapped in
38
- * transparently for the WebView renderer.
39
- */
40
- const react_1 = __importStar(require("react"));
41
- const react_native_1 = require("react-native");
42
- const react_native_filament_1 = require("react-native-filament");
43
- // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-require-imports
44
- const { Skybox } = require('react-native-filament');
45
- const expo_asset_1 = require("expo-asset");
46
- const useAuthedFilamentUri_1 = require("./useAuthedFilamentUri");
47
- const faceSqueezeAssets_1 = require("./faceSqueezeAssets");
48
- const morphTables_1 = require("./morphTables");
49
- // ---------------------------------------------------------------------------
50
- // Material keyword lists for color tinting
51
- // ---------------------------------------------------------------------------
52
- const SKIN_MATERIAL_KEYWORDS = ['skin', 'body', 'flesh', 'face', 'head'];
53
- const HAIR_MATERIAL_KEYWORDS = ['hair', 'beard', 'eyebrow', 'eyelash', 'brow', 'lash'];
54
- const EYE_MATERIAL_KEYWORDS = ['eye', 'iris', 'cornea', 'sclera', 'pupil'];
55
- // ---------------------------------------------------------------------------
56
- // Idle blink timing
57
- // ---------------------------------------------------------------------------
58
- const BLINK_INTERVAL_MIN_MS = 2800;
59
- const BLINK_INTERVAL_MAX_MS = 6000;
60
- const BLINK_HOLD_MS = 80;
61
- // ---------------------------------------------------------------------------
62
- // Idle emotion bursts — matches FaceSqueezeEditor's weighted random system
63
- // ---------------------------------------------------------------------------
64
- const IDLE_EMOTION_PRESETS = {
65
- happy: { mouthSmileLeft: 0.85, mouthSmileRight: 0.85, cheekSquintLeft: 0.6, cheekSquintRight: 0.6, mouthDimpleLeft: 0.4, mouthDimpleRight: 0.4 },
66
- surprised: { eyeWideLeft: 0.9, eyeWideRight: 0.9, browInnerUp: 0.8, browOuterUpLeft: 0.7, browOuterUpRight: 0.7, jawOpen: 0.4, mouthFunnel: 0.3 },
67
- smirk: { mouthSmileLeft: 0.9, mouthSmileRight: 0.1, cheekSquintLeft: 0.5, mouthDimpleLeft: 0.35 },
68
- sad: { browInnerUp: 0.9, browDownLeft: 0.3, browDownRight: 0.3, mouthFrownLeft: 0.8, mouthFrownRight: 0.8, eyeLookDownLeft: 0.5, eyeLookDownRight: 0.5 },
69
- wink: { eyeBlinkLeft: 1.0, mouthSmileLeft: 0.5, mouthSmileRight: 0.3, cheekSquintLeft: 0.4 },
70
- sleepy: { eyeBlinkLeft: 0.7, eyeBlinkRight: 0.7, browDownLeft: 0.4, browDownRight: 0.4, mouthShrugLower: 0.3, jawOpen: 0.15 },
71
- contempt: { mouthSmileLeft: 0.6, mouthPressRight: 0.5, mouthStretchRight: 0.3, browDownRight: 0.4, eyeSquintRight: 0.3 },
72
- };
73
- const IDLE_EMOTIONS_WEIGHTED = [
74
- { key: 'happy', weight: 5 }, { key: 'surprised', weight: 2 },
75
- { key: 'smirk', weight: 2 }, { key: 'sad', weight: 1 },
76
- { key: 'wink', weight: 2 }, { key: 'sleepy', weight: 1 },
77
- { key: 'contempt', weight: 1 },
78
- ];
79
- const IDLE_EMOTION_INTERVAL_MS = 6000;
80
- const IDLE_EMOTION_INITIAL_DELAY_MS = 2000;
81
- const IDLE_EMOTION_BLEND_MS = 300;
82
- const IDLE_EMOTION_HOLD_MS = 1500;
83
- function pickIdleEmotion() {
84
- const total = IDLE_EMOTIONS_WEIGHTED.reduce((s, e) => s + e.weight, 0);
85
- let r = Math.random() * total;
86
- for (const e of IDLE_EMOTIONS_WEIGHTED) {
87
- r -= e.weight;
88
- if (r <= 0)
89
- return e.key;
90
- }
91
- return 'happy';
92
- }
93
- // ---------------------------------------------------------------------------
94
- // Camera presets — tight head framing matching TalkingHead cameraView='head'
95
- // Avatar head is at approximately y=1.62 in Ready Player Me scale.
96
- // ---------------------------------------------------------------------------
97
- // Camera sits ~1.0m back, targeting the face for a tight crop.
98
- const CAMERA_POSITION = [0, 1.52, 0.85];
99
- const CAMERA_TARGET = [0, 1.52, 0];
100
- const CAMERA_UP = [0, 1, 0];
101
- // 50mm gives natural portrait framing at 1.0m distance.
102
- // Pip: 85mm telephoto for a tight face crop in the small bubble.
103
- exports.CAMERA_FOCAL_FULL = 50;
104
- exports.CAMERA_FOCAL_PIP = 85;
105
- // ---------------------------------------------------------------------------
106
- // Idle micro-expression cycles (MotionEngine-style procedural layer)
107
- // Runs at ~60ms ticks, drives subtle head/face movement independent of speech
108
- // ---------------------------------------------------------------------------
109
- // Micro-sway: slow sinusoidal variation on subtle morphs to suggest breathing/life
110
- const IDLE_BREATHE_PERIOD_MS = 4200; // one breath cycle
111
- // ---------------------------------------------------------------------------
112
- // Morph helpers
113
- // ---------------------------------------------------------------------------
114
- function normalizeName(name) {
115
- return name.toLowerCase().replace(/[^a-z0-9]/g, '');
116
- }
117
- function hexToFloat4(hex) {
118
- const cleaned = hex.replace('#', '').padEnd(6, '0');
119
- return [
120
- parseInt(cleaned.slice(0, 2), 16) / 255,
121
- parseInt(cleaned.slice(2, 4), 16) / 255,
122
- parseInt(cleaned.slice(4, 6), 16) / 255,
123
- 1.0,
124
- ];
125
- }
126
- function buildVisemeMorphCache(dispatchMap) {
127
- const cache = new Map();
128
- for (const [visemeKey, aliases] of Object.entries(morphTables_1.VISEME_MORPH_ALIASES)) {
129
- if (visemeKey === 'sil')
130
- continue;
131
- for (const alias of aliases) {
132
- const entries = dispatchMap.get(alias) ?? dispatchMap.get(normalizeName(alias));
133
- if (entries && entries.length > 0) {
134
- cache.set(visemeKey, entries);
135
- break;
136
- }
137
- }
138
- }
139
- return cache;
140
- }
141
- function FilamentAvatarInner({ localUri, mood: initialMood = 'neutral', hairColor: initialHairColor, skinColor: initialSkinColor, eyeColor: initialEyeColor, onReady, onError: _onError, // eslint-disable-line @typescript-eslint/no-unused-vars
142
- innerRef, aspect, focalLength = exports.CAMERA_FOCAL_FULL, }) {
143
- // Memoize source object to avoid new reference on every render → useModel reload → SIGSEGV.
144
- // When localUri is empty (fallback not yet resolved) we still pass a source so
145
- // FilamentAvatarInner stays mounted and FilamentView's SwapChain stays alive.
146
- // useModel will fail gracefully for the empty string; ModelRenderer is gated on state==='loaded'.
147
- const modelSource = react_1.default.useMemo(() => (localUri ? { uri: localUri } : null), [localUri]);
148
- (0, react_1.useEffect)(() => {
149
- console.log('[FilamentAvatar] useModel source:', localUri || '(empty — waiting for fallback)');
150
- }, [localUri]);
151
- const model = (0, react_native_filament_1.useModel)(modelSource);
152
- const modelRef = (0, react_1.useRef)(model);
153
- modelRef.current = model;
154
- const { renderableManager } = (0, react_native_filament_1.useFilamentContext)();
155
- // morph dispatch map — keyed by name and normalized name
156
- const dispatchMap = (0, react_1.useRef)(new Map());
157
- const visemeCacheRef = (0, react_1.useRef)(new Map());
158
- const weightsMap = (0, react_1.useRef)(new Map());
159
- const entityById = (0, react_1.useRef)(new Map());
160
- // viseme scheduler state
161
- const visemeState = (0, react_1.useRef)({});
162
- const visemeTimers = (0, react_1.useRef)([]);
163
- const visemeModeUntil = (0, react_1.useRef)(0);
164
- const activeScheduleId = (0, react_1.useRef)(0);
165
- const tickIntervalRef = (0, react_1.useRef)(null);
166
- // last received schedule — replayed if model loads after the schedule arrives
167
- const lastScheduleRef = (0, react_1.useRef)(null);
168
- // stable ref to scheduleVisemes so the model-load effect can call it after definition
169
- const scheduleVisemesRef = (0, react_1.useRef)(null);
170
- // amplitude fallback
171
- const amplitudeRef = (0, react_1.useRef)(0);
172
- // mood baseline weights
173
- const moodWeightsRef = (0, react_1.useRef)(morphTables_1.MOOD_MORPHS[initialMood] ?? {});
174
- // idle breathing phase (radians, incremented per tick)
175
- const breathPhaseRef = (0, react_1.useRef)(0);
176
- // blink
177
- const blinkTimerRef = (0, react_1.useRef)(null);
178
- const isBlinkingRef = (0, react_1.useRef)(false);
179
- // idle emotion burst — blended overlay on top of mood baseline
180
- const idleEmotionWeightsRef = (0, react_1.useRef)({});
181
- const idleEmotionTimerRef = (0, react_1.useRef)(null);
182
- const idleEmotionIntervalRef = (0, react_1.useRef)(null);
183
- // mounted guard — prevents renderableManager calls after unmount.
184
- // useLayoutEffect so it fires synchronously before any pending timers/intervals.
185
- const mountedRef = (0, react_1.useRef)(true);
186
- (0, react_1.useLayoutEffect)(() => {
187
- mountedRef.current = true;
188
- return () => {
189
- mountedRef.current = false;
190
- // Synchronous cleanup — kill timers immediately on unmount
191
- for (const id of visemeTimers.current)
192
- clearTimeout(id);
193
- visemeTimers.current = [];
194
- if (tickIntervalRef.current) {
195
- clearInterval(tickIntervalRef.current);
196
- tickIntervalRef.current = null;
197
- }
198
- if (blinkTimerRef.current) {
199
- clearTimeout(blinkTimerRef.current);
200
- blinkTimerRef.current = null;
201
- }
202
- if (idleEmotionIntervalRef.current) {
203
- clearInterval(idleEmotionIntervalRef.current);
204
- idleEmotionIntervalRef.current = null;
205
- }
206
- if (idleEmotionTimerRef.current) {
207
- clearTimeout(idleEmotionTimerRef.current);
208
- idleEmotionTimerRef.current = null;
209
- }
210
- if (blendIntervalRef.current) {
211
- clearInterval(blendIntervalRef.current);
212
- blendIntervalRef.current = null;
213
- }
214
- };
215
- }, []);
216
- // ---------------------------------------------------------------------------
217
- // Apply weights to Filament meshes
218
- // ---------------------------------------------------------------------------
219
- // Entities that were non-zero last tick — must be flushed even if zero this tick
220
- const prevDirtyRef = (0, react_1.useRef)(new Set());
221
- /**
222
- * Zero all weight arrays, compose layers, flush to Filament once.
223
- * Always flushes entities that were dirty last tick so zeroed values
224
- * actually reach the GPU (prevents one-frame stale morph flashes).
225
- */
226
- const flushWeights = (0, react_1.useCallback)((morphTargets, visemeTargets) => {
227
- if (!mountedRef.current)
228
- return;
229
- // Zero all weight arrays
230
- for (const arr of weightsMap.current.values())
231
- arr.fill(0);
232
- const dirty = new Set(prevDirtyRef.current);
233
- // Helper: write a morph-name keyed entry (direct dispatchMap lookup)
234
- const writeMorph = (name, weight) => {
235
- const dispatch = dispatchMap.current.get(name) ?? dispatchMap.current.get(normalizeName(name));
236
- if (!dispatch)
237
- return;
238
- const clamped = Math.max(0, Math.min(1, weight));
239
- for (const entry of dispatch) {
240
- const arr = weightsMap.current.get(entry.entityId);
241
- if (!arr)
242
- continue;
243
- arr[entry.index] = Math.max(arr[entry.index], clamped);
244
- dirty.add(entry.entityId);
245
- }
246
- };
247
- // Layer 1+2: mood + breathe + blink (morph target names)
248
- for (const [name, weight] of Object.entries(morphTargets)) {
249
- writeMorph(name, weight);
250
- }
251
- // Layer 3: viseme state (viseme keys → viseme cache)
252
- for (const [visemeKey, weight] of Object.entries(visemeTargets)) {
253
- if (weight <= 0)
254
- continue;
255
- const entries = visemeCacheRef.current.get(visemeKey);
256
- if (!entries)
257
- continue;
258
- const clamped = Math.min(1, weight);
259
- for (const entry of entries) {
260
- const arr = weightsMap.current.get(entry.entityId);
261
- if (!arr)
262
- continue;
263
- arr[entry.index] = Math.max(arr[entry.index], clamped);
264
- dirty.add(entry.entityId);
265
- }
266
- }
267
- // Single flush per entity — includes prev-dirty so zeroed morphs reach GPU
268
- for (const id of dirty) {
269
- const entity = entityById.current.get(id);
270
- const arr = weightsMap.current.get(id);
271
- if (entity && arr) {
272
- try {
273
- renderableManager.setMorphWeights(entity, arr, 0);
274
- }
275
- catch { /* noop */ }
276
- }
277
- }
278
- prevDirtyRef.current = dirty;
279
- }, [renderableManager]);
280
- // ---------------------------------------------------------------------------
281
- // Apply material color tint to a keyword-matched set of materials
282
- // ---------------------------------------------------------------------------
283
- const applyMaterialColor = (0, react_1.useCallback)((keywords, color) => {
284
- const m = modelRef.current;
285
- if (m.state !== 'loaded')
286
- return;
287
- const float4 = hexToFloat4(color);
288
- const entities = m.asset.getRenderableEntities();
289
- let matched = false;
290
- for (const entity of entities) {
291
- const count = renderableManager.getPrimitiveCount(entity);
292
- for (let i = 0; i < count; i++) {
293
- try {
294
- const mi = renderableManager.getMaterialInstanceAt(entity, i);
295
- const lower = (mi.name ?? '').toLowerCase();
296
- if (keywords.some((k) => lower.includes(k))) {
297
- mi.setFloat4Parameter('baseColorFactor', float4);
298
- matched = true;
299
- }
300
- }
301
- catch { /* noop — some entities may not have materials */ }
302
- }
303
- }
304
- // Fallback: tint everything if no keyword match
305
- if (!matched) {
306
- for (const entity of entities) {
307
- const count = renderableManager.getPrimitiveCount(entity);
308
- for (let i = 0; i < count; i++) {
309
- try {
310
- const mi = renderableManager.getMaterialInstanceAt(entity, i);
311
- mi.setFloat4Parameter('baseColorFactor', float4);
312
- }
313
- catch { /* noop */ }
314
- }
315
- }
316
- }
317
- }, [renderableManager]);
318
- // ---------------------------------------------------------------------------
319
- // Build morph cache when model loads
320
- // ---------------------------------------------------------------------------
321
- (0, react_1.useEffect)(() => {
322
- const m = modelRef.current;
323
- if (m.state !== 'loaded') {
324
- // Model is unloading or reloading — clear stale entity refs immediately so
325
- // the 16ms tick interval never calls setMorphWeights on freed entities.
326
- // Leaving stale entity IDs in the maps causes use-after-free in FEngine::loop.
327
- dispatchMap.current = new Map();
328
- weightsMap.current = new Map();
329
- entityById.current = new Map();
330
- visemeCacheRef.current = new Map();
331
- visemeState.current = {};
332
- prevDirtyRef.current = new Set();
333
- return;
334
- }
335
- console.log('[FilamentAvatar] model loaded, boundingBox:', JSON.stringify(m.boundingBox));
336
- const newDispatchMap = new Map();
337
- const newWeightsMap = new Map();
338
- const newEntityById = new Map();
339
- const entities = m.asset.getRenderableEntities();
340
- for (const entity of entities) {
341
- let count = 0;
342
- try {
343
- count = m.asset.getMorphTargetCountAt(entity);
344
- }
345
- catch { /* noop */ }
346
- if (count <= 0) {
347
- try {
348
- count = renderableManager.getMorphTargetCount(entity);
349
- }
350
- catch { /* noop */ }
351
- }
352
- if (count === 0)
353
- continue;
354
- newWeightsMap.set(entity.id, new Array(count).fill(0));
355
- newEntityById.set(entity.id, entity);
356
- for (let i = 0; i < count; i++) {
357
- let name = '';
358
- try {
359
- name = m.asset.getMorphTargetNameAt(entity, i) ?? '';
360
- }
361
- catch { /* noop */ }
362
- if (!name) {
363
- try {
364
- name = renderableManager.getMorphTargetNameAt(entity, i) ?? '';
365
- }
366
- catch { /* noop */ }
367
- }
368
- if (!name)
369
- continue;
370
- const entry = { entityId: entity.id, index: i };
371
- const addEntry = (key) => {
372
- const list = newDispatchMap.get(key);
373
- if (list)
374
- list.push(entry);
375
- else
376
- newDispatchMap.set(key, [entry]);
377
- };
378
- addEntry(name);
379
- addEntry(normalizeName(name));
380
- }
381
- }
382
- dispatchMap.current = newDispatchMap;
383
- weightsMap.current = newWeightsMap;
384
- entityById.current = newEntityById;
385
- visemeCacheRef.current = buildVisemeMorphCache(newDispatchMap);
386
- visemeState.current = {};
387
- // Apply initial appearance
388
- if (initialSkinColor)
389
- applyMaterialColor(SKIN_MATERIAL_KEYWORDS, initialSkinColor);
390
- if (initialHairColor)
391
- applyMaterialColor(HAIR_MATERIAL_KEYWORDS, initialHairColor);
392
- if (initialEyeColor)
393
- applyMaterialColor(EYE_MATERIAL_KEYWORDS, initialEyeColor);
394
- onReady?.();
395
- // Replay any pending schedule that arrived before the model was ready.
396
- // (scheduleVisemes stores to lastScheduleRef even when morphMapSize is 0)
397
- const pending = lastScheduleRef.current;
398
- if (pending && pending.cues && pending.cues.length > 0) {
399
- const startedAt = pending.startedAtMs ?? Date.now();
400
- const durationMs = pending.durationMs ?? 0;
401
- const ageMs = Date.now() - startedAt;
402
- // Only replay if audio is likely still playing
403
- if (durationMs <= 0 || ageMs < durationMs + 300) {
404
- __DEV__ && console.log('[FilamentAvatar] replaying pending schedule after model load', {
405
- requestId: pending.requestId ?? null,
406
- ageMs,
407
- cues: pending.cues.length,
408
- });
409
- const scheduleToReplay = pending;
410
- lastScheduleRef.current = null;
411
- // scheduleVisemesRef.current is populated during render (before effects run)
412
- scheduleVisemesRef.current?.(scheduleToReplay);
413
- }
414
- else {
415
- lastScheduleRef.current = null;
416
- }
417
- }
418
- // eslint-disable-next-line react-hooks/exhaustive-deps
419
- }, [model.state]);
420
- // ---------------------------------------------------------------------------
421
- // Viseme tick — runs every 16ms, decays state and writes to morphs
422
- // ---------------------------------------------------------------------------
423
- const tickViseme = (0, react_1.useCallback)(() => {
424
- const state = visemeState.current;
425
- const now = Date.now();
426
- const isScheduled = now < visemeModeUntil.current;
427
- if (!isScheduled) {
428
- // Amplitude fallback: jaw bobble when no viseme schedule active
429
- const amp = amplitudeRef.current;
430
- if (amp > 0.01) {
431
- state['aa'] = amp * 0.6;
432
- }
433
- else {
434
- for (const key of Object.keys(state)) {
435
- const decayed = (state[key] ?? 0) * 0.82;
436
- state[key] = decayed < 0.01 ? 0 : decayed;
437
- }
438
- }
439
- }
440
- // Idle breathing — subtle cheek/nose puff on inhale
441
- breathPhaseRef.current += (2 * Math.PI * 16) / IDLE_BREATHE_PERIOD_MS;
442
- const breathe = Math.max(0, Math.sin(breathPhaseRef.current)) * 0.06;
443
- // Mood baseline + breathe + blink (morph target names)
444
- const morphTargets = { ...moodWeightsRef.current };
445
- if (breathe > 0.005) {
446
- morphTargets['noseSneerLeft'] = Math.max(morphTargets['noseSneerLeft'] ?? 0, breathe);
447
- morphTargets['noseSneerRight'] = Math.max(morphTargets['noseSneerRight'] ?? 0, breathe);
448
- morphTargets['cheekPuff'] = Math.max(morphTargets['cheekPuff'] ?? 0, breathe * 0.5);
449
- }
450
- if (isBlinkingRef.current) {
451
- morphTargets['eyeBlinkLeft'] = 1;
452
- morphTargets['eyeBlinkRight'] = 1;
453
- }
454
- // Idle emotion overlay — blend on top of mood baseline
455
- for (const [name, weight] of Object.entries(idleEmotionWeightsRef.current)) {
456
- if (weight > 0)
457
- morphTargets[name] = Math.max(morphTargets[name] ?? 0, weight);
458
- }
459
- // Single flush: zeros all arrays, writes mood/breathe, writes visemes on top, pushes to GPU once
460
- flushWeights(morphTargets, state);
461
- }, [flushWeights]);
462
- (0, react_1.useEffect)(() => {
463
- tickIntervalRef.current = setInterval(tickViseme, 16);
464
- return () => {
465
- if (tickIntervalRef.current)
466
- clearInterval(tickIntervalRef.current);
467
- };
468
- }, [tickViseme]);
469
- // ---------------------------------------------------------------------------
470
- // Viseme scheduler
471
- // ---------------------------------------------------------------------------
472
- const clearScheduledVisemes = (0, react_1.useCallback)(() => {
473
- for (const id of visemeTimers.current)
474
- clearTimeout(id);
475
- visemeTimers.current = [];
476
- visemeModeUntil.current = 0;
477
- visemeState.current = {};
478
- }, []);
479
- const scheduleVisemes = (0, react_1.useCallback)((schedule) => {
480
- const modelLoaded = modelRef.current.state === 'loaded';
481
- const morphMapSize = dispatchMap.current.size;
482
- __DEV__ && console.log('[FilamentAvatar] scheduleVisemes', {
483
- cues: schedule.cues?.length ?? 0,
484
- modelLoaded,
485
- morphMapSize,
486
- requestId: schedule.requestId ?? null,
487
- });
488
- // Always save — if model isn't loaded yet, we replay when it loads
489
- lastScheduleRef.current = schedule;
490
- clearScheduledVisemes();
491
- activeScheduleId.current += 1;
492
- const myId = activeScheduleId.current;
493
- if (!schedule.cues || schedule.cues.length === 0)
494
- return;
495
- const startedAt = schedule.startedAtMs ?? Date.now();
496
- const durationMs = schedule.durationMs ?? 0;
497
- const now = Date.now();
498
- const elapsedMs = Math.max(0, now - startedAt);
499
- visemeModeUntil.current = now + Math.max(0, durationMs - elapsedMs) + 200;
500
- for (const cue of schedule.cues) {
501
- const delay = cue.startMs - elapsedMs;
502
- // Skip cues that are more than one cue-duration in the past
503
- if (delay < -(cue.endMs - cue.startMs) - 50)
504
- continue;
505
- const visemeKey = morphTables_1.RHUBARB_TO_VISEME[cue.viseme] ?? 'sil';
506
- const weight = morphTables_1.VISEME_WEIGHTS[visemeKey] ?? morphTables_1.DEFAULT_VISEME_WEIGHT;
507
- const applyId = setTimeout(() => {
508
- if (activeScheduleId.current !== myId)
509
- return;
510
- for (const k of Object.keys(visemeState.current))
511
- visemeState.current[k] = 0;
512
- if (visemeKey !== 'sil')
513
- visemeState.current[visemeKey] = weight;
514
- }, Math.max(0, delay));
515
- const clearId = setTimeout(() => {
516
- if (activeScheduleId.current !== myId)
517
- return;
518
- if (visemeState.current[visemeKey] !== undefined)
519
- visemeState.current[visemeKey] = 0;
520
- }, Math.max(0, delay + (cue.endMs - cue.startMs)));
521
- visemeTimers.current.push(applyId, clearId);
522
- }
523
- // Silence at end — always schedule a hard reset
524
- const endDelay = durationMs > 0
525
- ? Math.max(0, durationMs - (Date.now() - startedAt)) + 100
526
- : (schedule.cues[schedule.cues.length - 1]?.endMs ?? 0) - elapsedMs + 100;
527
- visemeTimers.current.push(setTimeout(() => {
528
- if (activeScheduleId.current !== myId)
529
- return;
530
- // Hard reset: zero all viseme state and exit scheduled mode
531
- visemeState.current = {};
532
- visemeModeUntil.current = 0;
533
- }, Math.max(0, endDelay)));
534
- }, [clearScheduledVisemes]);
535
- // Keep ref in sync so the model-load effect can call scheduleVisemes
536
- scheduleVisemesRef.current = scheduleVisemes;
537
- // Single-shot viseme (for real-time streaming TTS like ElevenLabs)
538
- const sendViseme = (0, react_1.useCallback)((viseme, weight = morphTables_1.DEFAULT_VISEME_WEIGHT) => {
539
- for (const k of Object.keys(visemeState.current))
540
- visemeState.current[k] = 0;
541
- if (viseme !== 'sil')
542
- visemeState.current[viseme] = morphTables_1.VISEME_WEIGHTS[viseme] ?? weight;
543
- visemeModeUntil.current = Date.now() + 120; // hold for ~120ms
544
- }, []);
545
- // ---------------------------------------------------------------------------
546
- // Idle blink
547
- // ---------------------------------------------------------------------------
548
- const scheduleBlink = (0, react_1.useCallback)(() => {
549
- const delay = BLINK_INTERVAL_MIN_MS + Math.random() * (BLINK_INTERVAL_MAX_MS - BLINK_INTERVAL_MIN_MS);
550
- blinkTimerRef.current = setTimeout(() => {
551
- isBlinkingRef.current = true;
552
- setTimeout(() => {
553
- isBlinkingRef.current = false;
554
- scheduleBlink();
555
- }, BLINK_HOLD_MS);
556
- }, delay);
557
- }, []);
558
- (0, react_1.useEffect)(() => {
559
- if (model.state !== 'loaded')
560
- return;
561
- scheduleBlink();
562
- return () => {
563
- if (blinkTimerRef.current)
564
- clearTimeout(blinkTimerRef.current);
565
- };
566
- }, [model.state, scheduleBlink]);
567
- // ---------------------------------------------------------------------------
568
- // Idle emotion bursts — weighted random expressions matching FaceSqueezeEditor
569
- // ---------------------------------------------------------------------------
570
- // Tracks the active blend-in/blend-out intervals so they can be cleaned up on unmount
571
- const blendIntervalRef = (0, react_1.useRef)(null);
572
- const playIdleEmotion = (0, react_1.useCallback)((key) => {
573
- if (!mountedRef.current)
574
- return;
575
- const target = IDLE_EMOTION_PRESETS[key] ?? {};
576
- const steps = Math.max(1, Math.round(IDLE_EMOTION_BLEND_MS / 16));
577
- // Clear any in-flight blend interval from a previous emotion
578
- if (blendIntervalRef.current) {
579
- clearInterval(blendIntervalRef.current);
580
- blendIntervalRef.current = null;
581
- }
582
- // Blend in
583
- let step = 0;
584
- blendIntervalRef.current = setInterval(() => {
585
- if (!mountedRef.current) {
586
- clearInterval(blendIntervalRef.current);
587
- blendIntervalRef.current = null;
588
- return;
589
- }
590
- step++;
591
- const t = Math.min(1, step / steps);
592
- const blended = {};
593
- for (const [k, v] of Object.entries(target))
594
- blended[k] = v * t;
595
- idleEmotionWeightsRef.current = blended;
596
- if (step >= steps) {
597
- clearInterval(blendIntervalRef.current);
598
- blendIntervalRef.current = null;
599
- }
600
- }, 16);
601
- // Hold then blend out
602
- idleEmotionTimerRef.current = setTimeout(() => {
603
- if (!mountedRef.current)
604
- return;
605
- // Clear blend-in if it somehow outlasted the hold period
606
- if (blendIntervalRef.current) {
607
- clearInterval(blendIntervalRef.current);
608
- blendIntervalRef.current = null;
609
- }
610
- let outStep = 0;
611
- blendIntervalRef.current = setInterval(() => {
612
- if (!mountedRef.current) {
613
- clearInterval(blendIntervalRef.current);
614
- blendIntervalRef.current = null;
615
- return;
616
- }
617
- outStep++;
618
- const t = Math.max(0, 1 - outStep / steps);
619
- const blended = {};
620
- for (const [k, v] of Object.entries(target))
621
- blended[k] = v * t;
622
- idleEmotionWeightsRef.current = blended;
623
- if (outStep >= steps) {
624
- clearInterval(blendIntervalRef.current);
625
- blendIntervalRef.current = null;
626
- idleEmotionWeightsRef.current = {};
627
- }
628
- }, 16);
629
- }, IDLE_EMOTION_HOLD_MS);
630
- }, []);
631
- (0, react_1.useEffect)(() => {
632
- if (model.state !== 'loaded')
633
- return;
634
- const initial = setTimeout(() => playIdleEmotion(pickIdleEmotion()), IDLE_EMOTION_INITIAL_DELAY_MS);
635
- idleEmotionIntervalRef.current = setInterval(() => playIdleEmotion(pickIdleEmotion()), IDLE_EMOTION_INTERVAL_MS);
636
- return () => {
637
- clearTimeout(initial);
638
- if (idleEmotionIntervalRef.current)
639
- clearInterval(idleEmotionIntervalRef.current);
640
- if (idleEmotionTimerRef.current)
641
- clearTimeout(idleEmotionTimerRef.current);
642
- if (blendIntervalRef.current) {
643
- clearInterval(blendIntervalRef.current);
644
- blendIntervalRef.current = null;
645
- }
646
- idleEmotionWeightsRef.current = {};
647
- };
648
- }, [model.state, playIdleEmotion]);
649
- // ---------------------------------------------------------------------------
650
- // Imperative ref — full TalkingHeadRef parity
651
- // ---------------------------------------------------------------------------
652
- (0, react_1.useImperativeHandle)(innerRef, () => ({
653
- sendViseme,
654
- scheduleVisemes,
655
- clearVisemes: clearScheduledVisemes,
656
- sendAmplitude: (amp) => { amplitudeRef.current = amp; },
657
- setMood: (mood) => { moodWeightsRef.current = morphTables_1.MOOD_MORPHS[mood] ?? {}; },
658
- setHairColor: (color) => applyMaterialColor(HAIR_MATERIAL_KEYWORDS, color),
659
- setSkinColor: (color) => applyMaterialColor(SKIN_MATERIAL_KEYWORDS, color),
660
- setEyeColor: (color) => applyMaterialColor(EYE_MATERIAL_KEYWORDS, color),
661
- setAccessories: (_accessories) => {
662
- // Future: load accessory GLBs and attach to skeleton bones
663
- },
664
- }), [sendViseme, scheduleVisemes, clearScheduledVisemes, applyMaterialColor]);
665
- // ---------------------------------------------------------------------------
666
- // Cleanup
667
- // ---------------------------------------------------------------------------
668
- (0, react_1.useEffect)(() => {
669
- return () => {
670
- clearScheduledVisemes();
671
- if (blinkTimerRef.current)
672
- clearTimeout(blinkTimerRef.current);
673
- if (tickIntervalRef.current)
674
- clearInterval(tickIntervalRef.current);
675
- };
676
- }, [clearScheduledVisemes]);
677
- // Pause Filament render loop when app backgrounds to prevent native
678
- // render thread from accessing freed GPU/surface resources → SIGSEGV.
679
- const filamentViewRef = (0, react_1.useRef)(null);
680
- (0, react_1.useEffect)(() => {
681
- const sub = react_native_1.AppState.addEventListener('change', (state) => {
682
- if (!filamentViewRef.current)
683
- return;
684
- if (state === 'background' || state === 'inactive') {
685
- filamentViewRef.current.pause?.();
686
- }
687
- else if (state === 'active') {
688
- filamentViewRef.current.resume?.();
689
- }
690
- });
691
- return () => sub.remove();
692
- }, []);
693
- return ((0, jsx_runtime_1.jsxs)(react_native_filament_1.FilamentView, { ref: filamentViewRef, style: react_native_1.StyleSheet.absoluteFill, enableTransparentRendering: false, children: [(0, jsx_runtime_1.jsx)(Skybox, { colorInHex: "#1a1a2e" }), (0, jsx_runtime_1.jsx)(react_native_filament_1.DefaultLight, {}), (0, jsx_runtime_1.jsx)(react_native_filament_1.Camera, { cameraPosition: CAMERA_POSITION, cameraTarget: CAMERA_TARGET, cameraUp: CAMERA_UP, aspect: aspect ?? 1, focalLengthInMillimeters: focalLength }), model.state === 'loaded' && ((0, jsx_runtime_1.jsx)(react_native_filament_1.ModelRenderer, { model: model }))] }));
694
- }
695
- // ---------------------------------------------------------------------------
696
- // Outer component — handles URL resolution + FilamentScene wrapper
697
- // ---------------------------------------------------------------------------
698
- // Resolve the fallback GLB to a file:// URI once at module load.
699
- // useBuffer (inside useModel) resolves require() numbers via Image.resolveAssetSource
700
- // which returns an http://localhost Metro URL in dev — Filament's native loader can't
701
- // fetch that. We must pre-resolve to a file:// path via expo-asset.
702
- let _fallbackUri = null;
703
- const _fallbackReady = (async () => {
704
- const a = expo_asset_1.Asset.fromModule(faceSqueezeAssets_1.FACE_SQUEEZE_LOCAL_MODULE);
705
- await a.downloadAsync();
706
- _fallbackUri = a.localUri ?? a.uri ?? '';
707
- return _fallbackUri;
708
- })();
709
- exports.FilamentAvatar = (0, react_1.forwardRef)(({ style, avatarUrl, aspect: aspectProp, focalLength, mood, hairColor, skinColor, eyeColor, accessories, onReady, onError }, ref) => {
710
- // Only download avatarUrl if it's a real remote studio URL (http/https).
711
- const remoteUrl = (avatarUrl && (avatarUrl.startsWith('http://') || avatarUrl.startsWith('https://')))
712
- ? avatarUrl : null;
713
- const fileResult = (0, useAuthedFilamentUri_1.useAuthedFilamentUri)(remoteUrl);
714
- const currentUri = fileResult?.uri ?? null;
715
- // Fallback: resolved file:// URI of the bundled face-squeeze GLB.
716
- // Starts null, populates async (fast — already kicked off at module load).
717
- const [fallbackUri, setFallbackUri] = react_1.default.useState(_fallbackUri);
718
- react_1.default.useEffect(() => {
719
- if (!_fallbackUri) {
720
- _fallbackReady.then(setFallbackUri).catch(() => { });
721
- }
722
- }, []);
723
- // Lock-in URI: once we pick a URI for this mount, NEVER change it.
724
- // Switching from fallback → studio mid-session causes useModel to reload,
725
- // which tears down GPU assets while FEngine::loop still holds references → SIGSEGV.
726
- // Strategy: if the studio URI is ready before/at mount time, use it directly.
727
- // If not, use the fallback and stay on it for the lifetime of this component.
728
- const lockedUriRef = react_1.default.useRef(null);
729
- if (lockedUriRef.current === null) {
730
- // First render: pick whichever URI is available, preferring studio.
731
- lockedUriRef.current = currentUri ?? fallbackUri;
732
- }
733
- const localUri = lockedUriRef.current;
734
- // Compute aspect ratio from layout if not provided by caller.
735
- const [measuredAspect, setMeasuredAspect] = react_1.default.useState(undefined);
736
- const aspect = aspectProp ?? measuredAspect;
737
- // FilamentScene MUST stay mounted for the lifetime of this component.
738
- // Unmounting it disposes GPU assets on the JS thread while Filament's native
739
- // render thread (FEngine::loop) and surface thread (JNISurfaceTextu) still hold
740
- // references to those objects → SIGSEGV / corrupted-PC use-after-free crashes.
741
- // FilamentAvatarInner keeps FilamentView permanently mounted; only ModelRenderer
742
- // is conditional on the model being ready.
743
- return ((0, jsx_runtime_1.jsx)(react_native_1.View, { style: [style, styles.fill], onLayout: (e) => {
744
- const { width, height } = e.nativeEvent.layout;
745
- if (height > 0)
746
- setMeasuredAspect(width / height);
747
- }, children: (0, jsx_runtime_1.jsx)(react_native_filament_1.FilamentScene, { children: (0, jsx_runtime_1.jsx)(FilamentAvatarInner, { localUri: localUri ?? '', avatarUrl: avatarUrl, aspect: aspect, focalLength: focalLength, mood: mood, hairColor: hairColor, skinColor: skinColor, eyeColor: eyeColor, accessories: accessories, innerRef: ref, onReady: onReady, onError: onError }) }) }));
748
- });
749
- exports.FilamentAvatar.displayName = 'FilamentAvatar';
750
- const styles = react_native_1.StyleSheet.create({
751
- fill: { flex: 1 },
752
- empty: { backgroundColor: 'transparent' },
753
- center: { alignItems: 'center', justifyContent: 'center' },
754
- label: { color: 'rgba(226,232,240,0.6)', fontSize: 13 },
755
- });