talking-head-studio 0.3.9 → 0.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,715 +0,0 @@
1
- "use strict";
2
- var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
- if (k2 === undefined) k2 = k;
4
- var desc = Object.getOwnPropertyDescriptor(m, k);
5
- if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
- desc = { enumerable: true, get: function() { return m[k]; } };
7
- }
8
- Object.defineProperty(o, k2, desc);
9
- }) : (function(o, m, k, k2) {
10
- if (k2 === undefined) k2 = k;
11
- o[k2] = m[k];
12
- }));
13
- var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
- Object.defineProperty(o, "default", { enumerable: true, value: v });
15
- }) : function(o, v) {
16
- o["default"] = v;
17
- });
18
- var __importStar = (this && this.__importStar) || function (mod) {
19
- if (mod && mod.__esModule) return mod;
20
- var result = {};
21
- if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
22
- __setModuleDefault(result, mod);
23
- return result;
24
- };
25
- Object.defineProperty(exports, "__esModule", { value: true });
26
- exports.FilamentAvatar = exports.CAMERA_FOCAL_PIP = exports.CAMERA_FOCAL_FULL = void 0;
27
- const jsx_runtime_1 = require("react/jsx-runtime");
28
- /* eslint-disable @typescript-eslint/no-unused-expressions */
29
- /* eslint-disable react-hooks/refs */
30
- /**
31
- * FilamentAvatar — native Filament-based avatar renderer.
32
- *
33
- * Replaces the TalkingHead WebView on iOS/Android with direct Filament
34
- * morph target writes — no WebView bridge, no JS→native→JS roundtrip,
35
- * visemes applied at 60fps in React Native JS.
36
- *
37
- * Implements the full TalkingHeadRef interface so it can be swapped in
38
- * transparently for the WebView renderer.
39
- */
40
- const react_1 = __importStar(require("react"));
41
- const react_native_1 = require("react-native");
42
- const react_native_filament_1 = require("react-native-filament");
43
- const useAuthedFilamentUri_1 = require("./useAuthedFilamentUri");
44
- const faceSqueezeAssets_1 = require("./faceSqueezeAssets");
45
- const morphTables_1 = require("./morphTables");
46
- // ---------------------------------------------------------------------------
47
- // Material keyword lists for color tinting
48
- // ---------------------------------------------------------------------------
49
- const SKIN_MATERIAL_KEYWORDS = ['skin', 'body', 'flesh', 'face', 'head'];
50
- const HAIR_MATERIAL_KEYWORDS = ['hair', 'beard', 'eyebrow', 'eyelash', 'brow', 'lash'];
51
- const EYE_MATERIAL_KEYWORDS = ['eye', 'iris', 'cornea', 'sclera', 'pupil'];
52
- // ---------------------------------------------------------------------------
53
- // Idle blink timing
54
- // ---------------------------------------------------------------------------
55
- const BLINK_INTERVAL_MIN_MS = 2800;
56
- const BLINK_INTERVAL_MAX_MS = 6000;
57
- const BLINK_HOLD_MS = 80;
58
- // ---------------------------------------------------------------------------
59
- // Idle emotion bursts — matches FaceSqueezeEditor's weighted random system
60
- // ---------------------------------------------------------------------------
61
- const IDLE_EMOTION_PRESETS = {
62
- happy: { mouthSmileLeft: 0.85, mouthSmileRight: 0.85, cheekSquintLeft: 0.6, cheekSquintRight: 0.6, mouthDimpleLeft: 0.4, mouthDimpleRight: 0.4 },
63
- surprised: { eyeWideLeft: 0.9, eyeWideRight: 0.9, browInnerUp: 0.8, browOuterUpLeft: 0.7, browOuterUpRight: 0.7, jawOpen: 0.4, mouthFunnel: 0.3 },
64
- smirk: { mouthSmileLeft: 0.9, mouthSmileRight: 0.1, cheekSquintLeft: 0.5, mouthDimpleLeft: 0.35 },
65
- sad: { browInnerUp: 0.9, browDownLeft: 0.3, browDownRight: 0.3, mouthFrownLeft: 0.8, mouthFrownRight: 0.8, eyeLookDownLeft: 0.5, eyeLookDownRight: 0.5 },
66
- wink: { eyeBlinkLeft: 1.0, mouthSmileLeft: 0.5, mouthSmileRight: 0.3, cheekSquintLeft: 0.4 },
67
- sleepy: { eyeBlinkLeft: 0.7, eyeBlinkRight: 0.7, browDownLeft: 0.4, browDownRight: 0.4, mouthShrugLower: 0.3, jawOpen: 0.15 },
68
- contempt: { mouthSmileLeft: 0.6, mouthPressRight: 0.5, mouthStretchRight: 0.3, browDownRight: 0.4, eyeSquintRight: 0.3 },
69
- };
70
- const IDLE_EMOTIONS_WEIGHTED = [
71
- { key: 'happy', weight: 5 }, { key: 'surprised', weight: 2 },
72
- { key: 'smirk', weight: 2 }, { key: 'sad', weight: 1 },
73
- { key: 'wink', weight: 2 }, { key: 'sleepy', weight: 1 },
74
- { key: 'contempt', weight: 1 },
75
- ];
76
- const IDLE_EMOTION_INTERVAL_MS = 6000;
77
- const IDLE_EMOTION_INITIAL_DELAY_MS = 2000;
78
- const IDLE_EMOTION_BLEND_MS = 300;
79
- const IDLE_EMOTION_HOLD_MS = 1500;
80
- function pickIdleEmotion() {
81
- const total = IDLE_EMOTIONS_WEIGHTED.reduce((s, e) => s + e.weight, 0);
82
- let r = Math.random() * total;
83
- for (const e of IDLE_EMOTIONS_WEIGHTED) {
84
- r -= e.weight;
85
- if (r <= 0)
86
- return e.key;
87
- }
88
- return 'happy';
89
- }
90
- // ---------------------------------------------------------------------------
91
- // Camera presets — tight head framing matching TalkingHead cameraView='head'
92
- // Avatar head is at approximately y=1.62 in Ready Player Me scale.
93
- // ---------------------------------------------------------------------------
94
- // z=1.4 gives head+shoulders framing at 50mm; z=1.05 was too tight (face only).
95
- const CAMERA_POSITION = [0, 1.55, 1.4];
96
- const CAMERA_TARGET = [0, 1.55, 0];
97
- const CAMERA_UP = [0, 1, 0];
98
- // Full/assistant view: 50mm is a natural portrait focal length — head is not distorted.
99
- // Pip: 85mm telephoto for a tight face crop in the small bubble.
100
- exports.CAMERA_FOCAL_FULL = 28;
101
- exports.CAMERA_FOCAL_PIP = 50;
102
- // ---------------------------------------------------------------------------
103
- // Idle micro-expression cycles (MotionEngine-style procedural layer)
104
- // Runs at ~60ms ticks, drives subtle head/face movement independent of speech
105
- // ---------------------------------------------------------------------------
106
- // Micro-sway: slow sinusoidal variation on subtle morphs to suggest breathing/life
107
- const IDLE_BREATHE_PERIOD_MS = 4200; // one breath cycle
108
- // ---------------------------------------------------------------------------
109
- // Morph helpers
110
- // ---------------------------------------------------------------------------
111
- function normalizeName(name) {
112
- return name.toLowerCase().replace(/[^a-z0-9]/g, '');
113
- }
114
- function hexToFloat4(hex) {
115
- const cleaned = hex.replace('#', '').padEnd(6, '0');
116
- return [
117
- parseInt(cleaned.slice(0, 2), 16) / 255,
118
- parseInt(cleaned.slice(2, 4), 16) / 255,
119
- parseInt(cleaned.slice(4, 6), 16) / 255,
120
- 1.0,
121
- ];
122
- }
123
- function buildVisemeMorphCache(dispatchMap) {
124
- const cache = new Map();
125
- for (const [visemeKey, aliases] of Object.entries(morphTables_1.VISEME_MORPH_ALIASES)) {
126
- if (visemeKey === 'sil')
127
- continue;
128
- for (const alias of aliases) {
129
- const entries = dispatchMap.get(alias) ?? dispatchMap.get(normalizeName(alias));
130
- if (entries && entries.length > 0) {
131
- cache.set(visemeKey, entries);
132
- break;
133
- }
134
- }
135
- }
136
- return cache;
137
- }
138
- function FilamentAvatarInner({ localUri, mood: initialMood = 'neutral', hairColor: initialHairColor, skinColor: initialSkinColor, eyeColor: initialEyeColor, onReady, onError: _onError, // eslint-disable-line @typescript-eslint/no-unused-vars
139
- innerRef, aspect, focalLength = exports.CAMERA_FOCAL_FULL, }) {
140
- // localUri is either a file:// string (remote download) or a require() number (bundled fallback).
141
- // useModel accepts both as BufferSource (number | { uri: string }) — cast to any since
142
- // tsconfig resolves to an older type declaration that only shows { uri: string }.
143
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
144
- const model = (0, react_native_filament_1.useModel)(typeof localUri === 'number' ? localUri : { uri: localUri });
145
- const modelRef = (0, react_1.useRef)(model);
146
- modelRef.current = model;
147
- const { renderableManager } = (0, react_native_filament_1.useFilamentContext)();
148
- // morph dispatch map — keyed by name and normalized name
149
- const dispatchMap = (0, react_1.useRef)(new Map());
150
- const visemeCacheRef = (0, react_1.useRef)(new Map());
151
- const weightsMap = (0, react_1.useRef)(new Map());
152
- const entityById = (0, react_1.useRef)(new Map());
153
- // viseme scheduler state
154
- const visemeState = (0, react_1.useRef)({});
155
- const visemeTimers = (0, react_1.useRef)([]);
156
- const visemeModeUntil = (0, react_1.useRef)(0);
157
- const activeScheduleId = (0, react_1.useRef)(0);
158
- const tickIntervalRef = (0, react_1.useRef)(null);
159
- // last received schedule — replayed if model loads after the schedule arrives
160
- const lastScheduleRef = (0, react_1.useRef)(null);
161
- // stable ref to scheduleVisemes so the model-load effect can call it after definition
162
- const scheduleVisemesRef = (0, react_1.useRef)(null);
163
- // amplitude fallback
164
- const amplitudeRef = (0, react_1.useRef)(0);
165
- // mood baseline weights
166
- const moodWeightsRef = (0, react_1.useRef)(morphTables_1.MOOD_MORPHS[initialMood] ?? {});
167
- // idle breathing phase (radians, incremented per tick)
168
- const breathPhaseRef = (0, react_1.useRef)(0);
169
- // blink
170
- const blinkTimerRef = (0, react_1.useRef)(null);
171
- const isBlinkingRef = (0, react_1.useRef)(false);
172
- // idle emotion burst — blended overlay on top of mood baseline
173
- const idleEmotionWeightsRef = (0, react_1.useRef)({});
174
- const idleEmotionTimerRef = (0, react_1.useRef)(null);
175
- const idleEmotionIntervalRef = (0, react_1.useRef)(null);
176
- // mounted guard — prevents renderableManager calls after unmount.
177
- // useLayoutEffect so it fires synchronously before any pending timers/intervals.
178
- const mountedRef = (0, react_1.useRef)(true);
179
- (0, react_1.useLayoutEffect)(() => {
180
- mountedRef.current = true;
181
- return () => {
182
- mountedRef.current = false;
183
- // Synchronous cleanup — kill timers immediately on unmount
184
- for (const id of visemeTimers.current)
185
- clearTimeout(id);
186
- visemeTimers.current = [];
187
- if (tickIntervalRef.current) {
188
- clearInterval(tickIntervalRef.current);
189
- tickIntervalRef.current = null;
190
- }
191
- if (blinkTimerRef.current) {
192
- clearTimeout(blinkTimerRef.current);
193
- blinkTimerRef.current = null;
194
- }
195
- if (idleEmotionIntervalRef.current) {
196
- clearInterval(idleEmotionIntervalRef.current);
197
- idleEmotionIntervalRef.current = null;
198
- }
199
- if (idleEmotionTimerRef.current) {
200
- clearTimeout(idleEmotionTimerRef.current);
201
- idleEmotionTimerRef.current = null;
202
- }
203
- if (blendIntervalRef.current) {
204
- clearInterval(blendIntervalRef.current);
205
- blendIntervalRef.current = null;
206
- }
207
- };
208
- }, []);
209
- // ---------------------------------------------------------------------------
210
- // Apply weights to Filament meshes
211
- // ---------------------------------------------------------------------------
212
- // Entities that were non-zero last tick — must be flushed even if zero this tick
213
- const prevDirtyRef = (0, react_1.useRef)(new Set());
214
- /**
215
- * Zero all weight arrays, compose layers, flush to Filament once.
216
- * Always flushes entities that were dirty last tick so zeroed values
217
- * actually reach the GPU (prevents one-frame stale morph flashes).
218
- */
219
- const flushWeights = (0, react_1.useCallback)((morphTargets, visemeTargets) => {
220
- if (!mountedRef.current)
221
- return;
222
- // Zero all weight arrays
223
- for (const arr of weightsMap.current.values())
224
- arr.fill(0);
225
- const dirty = new Set(prevDirtyRef.current);
226
- // Helper: write a morph-name keyed entry (direct dispatchMap lookup)
227
- const writeMorph = (name, weight) => {
228
- const dispatch = dispatchMap.current.get(name) ?? dispatchMap.current.get(normalizeName(name));
229
- if (!dispatch)
230
- return;
231
- const clamped = Math.max(0, Math.min(1, weight));
232
- for (const entry of dispatch) {
233
- const arr = weightsMap.current.get(entry.entityId);
234
- if (!arr)
235
- continue;
236
- arr[entry.index] = Math.max(arr[entry.index], clamped);
237
- dirty.add(entry.entityId);
238
- }
239
- };
240
- // Layer 1+2: mood + breathe + blink (morph target names)
241
- for (const [name, weight] of Object.entries(morphTargets)) {
242
- writeMorph(name, weight);
243
- }
244
- // Layer 3: viseme state (viseme keys → viseme cache)
245
- for (const [visemeKey, weight] of Object.entries(visemeTargets)) {
246
- if (weight <= 0)
247
- continue;
248
- const entries = visemeCacheRef.current.get(visemeKey);
249
- if (!entries)
250
- continue;
251
- const clamped = Math.min(1, weight);
252
- for (const entry of entries) {
253
- const arr = weightsMap.current.get(entry.entityId);
254
- if (!arr)
255
- continue;
256
- arr[entry.index] = Math.max(arr[entry.index], clamped);
257
- dirty.add(entry.entityId);
258
- }
259
- }
260
- // Single flush per entity — includes prev-dirty so zeroed morphs reach GPU
261
- for (const id of dirty) {
262
- const entity = entityById.current.get(id);
263
- const arr = weightsMap.current.get(id);
264
- if (entity && arr) {
265
- try {
266
- renderableManager.setMorphWeights(entity, arr, 0);
267
- }
268
- catch { /* noop */ }
269
- }
270
- }
271
- prevDirtyRef.current = dirty;
272
- }, [renderableManager]);
273
- // ---------------------------------------------------------------------------
274
- // Apply material color tint to a keyword-matched set of materials
275
- // ---------------------------------------------------------------------------
276
- const applyMaterialColor = (0, react_1.useCallback)((keywords, color) => {
277
- const m = modelRef.current;
278
- if (m.state !== 'loaded')
279
- return;
280
- const float4 = hexToFloat4(color);
281
- const entities = m.asset.getRenderableEntities();
282
- let matched = false;
283
- for (const entity of entities) {
284
- const count = renderableManager.getPrimitiveCount(entity);
285
- for (let i = 0; i < count; i++) {
286
- try {
287
- const mi = renderableManager.getMaterialInstanceAt(entity, i);
288
- const lower = (mi.name ?? '').toLowerCase();
289
- if (keywords.some((k) => lower.includes(k))) {
290
- mi.setFloat4Parameter('baseColorFactor', float4);
291
- matched = true;
292
- }
293
- }
294
- catch { /* noop — some entities may not have materials */ }
295
- }
296
- }
297
- // Fallback: tint everything if no keyword match
298
- if (!matched) {
299
- for (const entity of entities) {
300
- const count = renderableManager.getPrimitiveCount(entity);
301
- for (let i = 0; i < count; i++) {
302
- try {
303
- const mi = renderableManager.getMaterialInstanceAt(entity, i);
304
- mi.setFloat4Parameter('baseColorFactor', float4);
305
- }
306
- catch { /* noop */ }
307
- }
308
- }
309
- }
310
- }, [renderableManager]);
311
- // ---------------------------------------------------------------------------
312
- // Build morph cache when model loads
313
- // ---------------------------------------------------------------------------
314
- (0, react_1.useEffect)(() => {
315
- const m = modelRef.current;
316
- if (m.state !== 'loaded') {
317
- // Model is unloading or reloading — clear stale entity refs immediately so
318
- // the 16ms tick interval never calls setMorphWeights on freed entities.
319
- // Leaving stale entity IDs in the maps causes use-after-free in FEngine::loop.
320
- dispatchMap.current = new Map();
321
- weightsMap.current = new Map();
322
- entityById.current = new Map();
323
- visemeCacheRef.current = new Map();
324
- visemeState.current = {};
325
- prevDirtyRef.current = new Set();
326
- return;
327
- }
328
- const newDispatchMap = new Map();
329
- const newWeightsMap = new Map();
330
- const newEntityById = new Map();
331
- const entities = m.asset.getRenderableEntities();
332
- for (const entity of entities) {
333
- let count = 0;
334
- try {
335
- count = m.asset.getMorphTargetCountAt(entity);
336
- }
337
- catch { /* noop */ }
338
- if (count <= 0) {
339
- try {
340
- count = renderableManager.getMorphTargetCount(entity);
341
- }
342
- catch { /* noop */ }
343
- }
344
- if (count === 0)
345
- continue;
346
- newWeightsMap.set(entity.id, new Array(count).fill(0));
347
- newEntityById.set(entity.id, entity);
348
- for (let i = 0; i < count; i++) {
349
- let name = '';
350
- try {
351
- name = m.asset.getMorphTargetNameAt(entity, i) ?? '';
352
- }
353
- catch { /* noop */ }
354
- if (!name) {
355
- try {
356
- name = renderableManager.getMorphTargetNameAt(entity, i) ?? '';
357
- }
358
- catch { /* noop */ }
359
- }
360
- if (!name)
361
- continue;
362
- const entry = { entityId: entity.id, index: i };
363
- const addEntry = (key) => {
364
- const list = newDispatchMap.get(key);
365
- if (list)
366
- list.push(entry);
367
- else
368
- newDispatchMap.set(key, [entry]);
369
- };
370
- addEntry(name);
371
- addEntry(normalizeName(name));
372
- }
373
- }
374
- dispatchMap.current = newDispatchMap;
375
- weightsMap.current = newWeightsMap;
376
- entityById.current = newEntityById;
377
- visemeCacheRef.current = buildVisemeMorphCache(newDispatchMap);
378
- visemeState.current = {};
379
- // Apply initial appearance
380
- if (initialSkinColor)
381
- applyMaterialColor(SKIN_MATERIAL_KEYWORDS, initialSkinColor);
382
- if (initialHairColor)
383
- applyMaterialColor(HAIR_MATERIAL_KEYWORDS, initialHairColor);
384
- if (initialEyeColor)
385
- applyMaterialColor(EYE_MATERIAL_KEYWORDS, initialEyeColor);
386
- onReady?.();
387
- // Replay any pending schedule that arrived before the model was ready.
388
- // (scheduleVisemes stores to lastScheduleRef even when morphMapSize is 0)
389
- const pending = lastScheduleRef.current;
390
- if (pending && pending.cues && pending.cues.length > 0) {
391
- const startedAt = pending.startedAtMs ?? Date.now();
392
- const durationMs = pending.durationMs ?? 0;
393
- const ageMs = Date.now() - startedAt;
394
- // Only replay if audio is likely still playing
395
- if (durationMs <= 0 || ageMs < durationMs + 300) {
396
- __DEV__ && console.log('[FilamentAvatar] replaying pending schedule after model load', {
397
- requestId: pending.requestId ?? null,
398
- ageMs,
399
- cues: pending.cues.length,
400
- });
401
- const scheduleToReplay = pending;
402
- lastScheduleRef.current = null;
403
- // scheduleVisemesRef.current is populated during render (before effects run)
404
- scheduleVisemesRef.current?.(scheduleToReplay);
405
- }
406
- else {
407
- lastScheduleRef.current = null;
408
- }
409
- }
410
- // eslint-disable-next-line react-hooks/exhaustive-deps
411
- }, [model.state]);
412
- // ---------------------------------------------------------------------------
413
- // Viseme tick — runs every 16ms, decays state and writes to morphs
414
- // ---------------------------------------------------------------------------
415
- const tickViseme = (0, react_1.useCallback)(() => {
416
- const state = visemeState.current;
417
- const now = Date.now();
418
- const isScheduled = now < visemeModeUntil.current;
419
- if (!isScheduled) {
420
- // Amplitude fallback: jaw bobble when no viseme schedule active
421
- const amp = amplitudeRef.current;
422
- if (amp > 0.01) {
423
- state['aa'] = amp * 0.6;
424
- }
425
- else {
426
- for (const key of Object.keys(state)) {
427
- const decayed = (state[key] ?? 0) * 0.82;
428
- state[key] = decayed < 0.01 ? 0 : decayed;
429
- }
430
- }
431
- }
432
- // Idle breathing — subtle cheek/nose puff on inhale
433
- breathPhaseRef.current += (2 * Math.PI * 16) / IDLE_BREATHE_PERIOD_MS;
434
- const breathe = Math.max(0, Math.sin(breathPhaseRef.current)) * 0.06;
435
- // Mood baseline + breathe + blink (morph target names)
436
- const morphTargets = { ...moodWeightsRef.current };
437
- if (breathe > 0.005) {
438
- morphTargets['noseSneerLeft'] = Math.max(morphTargets['noseSneerLeft'] ?? 0, breathe);
439
- morphTargets['noseSneerRight'] = Math.max(morphTargets['noseSneerRight'] ?? 0, breathe);
440
- morphTargets['cheekPuff'] = Math.max(morphTargets['cheekPuff'] ?? 0, breathe * 0.5);
441
- }
442
- if (isBlinkingRef.current) {
443
- morphTargets['eyeBlinkLeft'] = 1;
444
- morphTargets['eyeBlinkRight'] = 1;
445
- }
446
- // Idle emotion overlay — blend on top of mood baseline
447
- for (const [name, weight] of Object.entries(idleEmotionWeightsRef.current)) {
448
- if (weight > 0)
449
- morphTargets[name] = Math.max(morphTargets[name] ?? 0, weight);
450
- }
451
- // Single flush: zeros all arrays, writes mood/breathe, writes visemes on top, pushes to GPU once
452
- flushWeights(morphTargets, state);
453
- }, [flushWeights]);
454
- (0, react_1.useEffect)(() => {
455
- tickIntervalRef.current = setInterval(tickViseme, 16);
456
- return () => {
457
- if (tickIntervalRef.current)
458
- clearInterval(tickIntervalRef.current);
459
- };
460
- }, [tickViseme]);
461
- // ---------------------------------------------------------------------------
462
- // Viseme scheduler
463
- // ---------------------------------------------------------------------------
464
- const clearScheduledVisemes = (0, react_1.useCallback)(() => {
465
- for (const id of visemeTimers.current)
466
- clearTimeout(id);
467
- visemeTimers.current = [];
468
- visemeModeUntil.current = 0;
469
- visemeState.current = {};
470
- }, []);
471
- const scheduleVisemes = (0, react_1.useCallback)((schedule) => {
472
- const modelLoaded = modelRef.current.state === 'loaded';
473
- const morphMapSize = dispatchMap.current.size;
474
- __DEV__ && console.log('[FilamentAvatar] scheduleVisemes', {
475
- cues: schedule.cues?.length ?? 0,
476
- modelLoaded,
477
- morphMapSize,
478
- requestId: schedule.requestId ?? null,
479
- });
480
- // Always save — if model isn't loaded yet, we replay when it loads
481
- lastScheduleRef.current = schedule;
482
- clearScheduledVisemes();
483
- activeScheduleId.current += 1;
484
- const myId = activeScheduleId.current;
485
- if (!schedule.cues || schedule.cues.length === 0)
486
- return;
487
- const startedAt = schedule.startedAtMs ?? Date.now();
488
- const durationMs = schedule.durationMs ?? 0;
489
- const now = Date.now();
490
- const elapsedMs = Math.max(0, now - startedAt);
491
- visemeModeUntil.current = now + Math.max(0, durationMs - elapsedMs) + 200;
492
- for (const cue of schedule.cues) {
493
- const delay = cue.startMs - elapsedMs;
494
- // Skip cues that are more than one cue-duration in the past
495
- if (delay < -(cue.endMs - cue.startMs) - 50)
496
- continue;
497
- const visemeKey = morphTables_1.RHUBARB_TO_VISEME[cue.viseme] ?? 'sil';
498
- const weight = morphTables_1.VISEME_WEIGHTS[visemeKey] ?? morphTables_1.DEFAULT_VISEME_WEIGHT;
499
- const applyId = setTimeout(() => {
500
- if (activeScheduleId.current !== myId)
501
- return;
502
- for (const k of Object.keys(visemeState.current))
503
- visemeState.current[k] = 0;
504
- if (visemeKey !== 'sil')
505
- visemeState.current[visemeKey] = weight;
506
- }, Math.max(0, delay));
507
- const clearId = setTimeout(() => {
508
- if (activeScheduleId.current !== myId)
509
- return;
510
- if (visemeState.current[visemeKey] !== undefined)
511
- visemeState.current[visemeKey] = 0;
512
- }, Math.max(0, delay + (cue.endMs - cue.startMs)));
513
- visemeTimers.current.push(applyId, clearId);
514
- }
515
- // Silence at end — always schedule a hard reset
516
- const endDelay = durationMs > 0
517
- ? Math.max(0, durationMs - (Date.now() - startedAt)) + 100
518
- : (schedule.cues[schedule.cues.length - 1]?.endMs ?? 0) - elapsedMs + 100;
519
- visemeTimers.current.push(setTimeout(() => {
520
- if (activeScheduleId.current !== myId)
521
- return;
522
- // Hard reset: zero all viseme state and exit scheduled mode
523
- visemeState.current = {};
524
- visemeModeUntil.current = 0;
525
- }, Math.max(0, endDelay)));
526
- }, [clearScheduledVisemes]);
527
- // Keep ref in sync so the model-load effect can call scheduleVisemes
528
- scheduleVisemesRef.current = scheduleVisemes;
529
- // Single-shot viseme (for real-time streaming TTS like ElevenLabs)
530
- const sendViseme = (0, react_1.useCallback)((viseme, weight = morphTables_1.DEFAULT_VISEME_WEIGHT) => {
531
- for (const k of Object.keys(visemeState.current))
532
- visemeState.current[k] = 0;
533
- if (viseme !== 'sil')
534
- visemeState.current[viseme] = morphTables_1.VISEME_WEIGHTS[viseme] ?? weight;
535
- visemeModeUntil.current = Date.now() + 120; // hold for ~120ms
536
- }, []);
537
- // ---------------------------------------------------------------------------
538
- // Idle blink
539
- // ---------------------------------------------------------------------------
540
- const scheduleBlink = (0, react_1.useCallback)(() => {
541
- const delay = BLINK_INTERVAL_MIN_MS + Math.random() * (BLINK_INTERVAL_MAX_MS - BLINK_INTERVAL_MIN_MS);
542
- blinkTimerRef.current = setTimeout(() => {
543
- isBlinkingRef.current = true;
544
- setTimeout(() => {
545
- isBlinkingRef.current = false;
546
- scheduleBlink();
547
- }, BLINK_HOLD_MS);
548
- }, delay);
549
- }, []);
550
- (0, react_1.useEffect)(() => {
551
- if (model.state !== 'loaded')
552
- return;
553
- scheduleBlink();
554
- return () => {
555
- if (blinkTimerRef.current)
556
- clearTimeout(blinkTimerRef.current);
557
- };
558
- }, [model.state, scheduleBlink]);
559
- // ---------------------------------------------------------------------------
560
- // Idle emotion bursts — weighted random expressions matching FaceSqueezeEditor
561
- // ---------------------------------------------------------------------------
562
- // Tracks the active blend-in/blend-out intervals so they can be cleaned up on unmount
563
- const blendIntervalRef = (0, react_1.useRef)(null);
564
- const playIdleEmotion = (0, react_1.useCallback)((key) => {
565
- if (!mountedRef.current)
566
- return;
567
- const target = IDLE_EMOTION_PRESETS[key] ?? {};
568
- const steps = Math.max(1, Math.round(IDLE_EMOTION_BLEND_MS / 16));
569
- // Clear any in-flight blend interval from a previous emotion
570
- if (blendIntervalRef.current) {
571
- clearInterval(blendIntervalRef.current);
572
- blendIntervalRef.current = null;
573
- }
574
- // Blend in
575
- let step = 0;
576
- blendIntervalRef.current = setInterval(() => {
577
- if (!mountedRef.current) {
578
- clearInterval(blendIntervalRef.current);
579
- blendIntervalRef.current = null;
580
- return;
581
- }
582
- step++;
583
- const t = Math.min(1, step / steps);
584
- const blended = {};
585
- for (const [k, v] of Object.entries(target))
586
- blended[k] = v * t;
587
- idleEmotionWeightsRef.current = blended;
588
- if (step >= steps) {
589
- clearInterval(blendIntervalRef.current);
590
- blendIntervalRef.current = null;
591
- }
592
- }, 16);
593
- // Hold then blend out
594
- idleEmotionTimerRef.current = setTimeout(() => {
595
- if (!mountedRef.current)
596
- return;
597
- // Clear blend-in if it somehow outlasted the hold period
598
- if (blendIntervalRef.current) {
599
- clearInterval(blendIntervalRef.current);
600
- blendIntervalRef.current = null;
601
- }
602
- let outStep = 0;
603
- blendIntervalRef.current = setInterval(() => {
604
- if (!mountedRef.current) {
605
- clearInterval(blendIntervalRef.current);
606
- blendIntervalRef.current = null;
607
- return;
608
- }
609
- outStep++;
610
- const t = Math.max(0, 1 - outStep / steps);
611
- const blended = {};
612
- for (const [k, v] of Object.entries(target))
613
- blended[k] = v * t;
614
- idleEmotionWeightsRef.current = blended;
615
- if (outStep >= steps) {
616
- clearInterval(blendIntervalRef.current);
617
- blendIntervalRef.current = null;
618
- idleEmotionWeightsRef.current = {};
619
- }
620
- }, 16);
621
- }, IDLE_EMOTION_HOLD_MS);
622
- }, []);
623
- (0, react_1.useEffect)(() => {
624
- if (model.state !== 'loaded')
625
- return;
626
- const initial = setTimeout(() => playIdleEmotion(pickIdleEmotion()), IDLE_EMOTION_INITIAL_DELAY_MS);
627
- idleEmotionIntervalRef.current = setInterval(() => playIdleEmotion(pickIdleEmotion()), IDLE_EMOTION_INTERVAL_MS);
628
- return () => {
629
- clearTimeout(initial);
630
- if (idleEmotionIntervalRef.current)
631
- clearInterval(idleEmotionIntervalRef.current);
632
- if (idleEmotionTimerRef.current)
633
- clearTimeout(idleEmotionTimerRef.current);
634
- if (blendIntervalRef.current) {
635
- clearInterval(blendIntervalRef.current);
636
- blendIntervalRef.current = null;
637
- }
638
- idleEmotionWeightsRef.current = {};
639
- };
640
- }, [model.state, playIdleEmotion]);
641
- // ---------------------------------------------------------------------------
642
- // Imperative ref — full TalkingHeadRef parity
643
- // ---------------------------------------------------------------------------
644
- (0, react_1.useImperativeHandle)(innerRef, () => ({
645
- sendViseme,
646
- scheduleVisemes,
647
- clearVisemes: clearScheduledVisemes,
648
- sendAmplitude: (amp) => { amplitudeRef.current = amp; },
649
- setMood: (mood) => { moodWeightsRef.current = morphTables_1.MOOD_MORPHS[mood] ?? {}; },
650
- setHairColor: (color) => applyMaterialColor(HAIR_MATERIAL_KEYWORDS, color),
651
- setSkinColor: (color) => applyMaterialColor(SKIN_MATERIAL_KEYWORDS, color),
652
- setEyeColor: (color) => applyMaterialColor(EYE_MATERIAL_KEYWORDS, color),
653
- setAccessories: (_accessories) => {
654
- // Future: load accessory GLBs and attach to skeleton bones
655
- },
656
- }), [sendViseme, scheduleVisemes, clearScheduledVisemes, applyMaterialColor]);
657
- // ---------------------------------------------------------------------------
658
- // Cleanup
659
- // ---------------------------------------------------------------------------
660
- (0, react_1.useEffect)(() => {
661
- return () => {
662
- clearScheduledVisemes();
663
- if (blinkTimerRef.current)
664
- clearTimeout(blinkTimerRef.current);
665
- if (tickIntervalRef.current)
666
- clearInterval(tickIntervalRef.current);
667
- };
668
- }, [clearScheduledVisemes]);
669
- // Pause Filament render loop when app backgrounds to prevent native
670
- // render thread from accessing freed GPU/surface resources → SIGSEGV.
671
- const filamentViewRef = (0, react_1.useRef)(null);
672
- (0, react_1.useEffect)(() => {
673
- const sub = react_native_1.AppState.addEventListener('change', (state) => {
674
- if (!filamentViewRef.current)
675
- return;
676
- if (state === 'background' || state === 'inactive') {
677
- filamentViewRef.current.pause?.();
678
- }
679
- else if (state === 'active') {
680
- filamentViewRef.current.resume?.();
681
- }
682
- });
683
- return () => sub.remove();
684
- }, []);
685
- return ((0, jsx_runtime_1.jsxs)(react_native_filament_1.FilamentView, { ref: filamentViewRef, style: react_native_1.StyleSheet.absoluteFill, children: [(0, jsx_runtime_1.jsx)(react_native_filament_1.DefaultLight, {}), (0, jsx_runtime_1.jsx)(react_native_filament_1.Camera, { cameraPosition: CAMERA_POSITION, cameraTarget: CAMERA_TARGET, cameraUp: CAMERA_UP, aspect: aspect ?? 1, focalLengthInMillimeters: focalLength }), model.state === 'loaded' && ((0, jsx_runtime_1.jsx)(react_native_filament_1.ModelRenderer, { model: model, multiplyWithCurrentTransform: false }))] }));
686
- }
687
- // ---------------------------------------------------------------------------
688
- // Outer component — handles URL resolution + FilamentScene wrapper
689
- // ---------------------------------------------------------------------------
690
- // The fallback is the bundled require() number — useModel accepts this directly
691
- // (BufferSource = number | { uri: string }), so no async asset resolution needed.
692
- exports.FilamentAvatar = (0, react_1.forwardRef)(({ style, avatarUrl, aspect, focalLength, mood, hairColor, skinColor, eyeColor, accessories, onReady, onError }, ref) => {
693
- const fileResult = (0, useAuthedFilamentUri_1.useAuthedFilamentUri)(avatarUrl ?? null);
694
- const currentUri = fileResult?.uri ?? null;
695
- // Stabilize localUri: once a valid remote file URI resolves, keep it until a new
696
- // one arrives. Falls back to the bundled require() number — always available sync.
697
- const stableUriRef = react_1.default.useRef(null);
698
- if (currentUri)
699
- stableUriRef.current = currentUri;
700
- const localUri = stableUriRef.current ?? faceSqueezeAssets_1.FACE_SQUEEZE_LOCAL_MODULE;
701
- // FilamentScene MUST stay mounted for the lifetime of this component.
702
- // Unmounting it disposes GPU assets on the JS thread while Filament's native
703
- // render thread (FEngine::loop) and surface thread (JNISurfaceTextu) still hold
704
- // references to those objects → SIGSEGV / corrupted-PC use-after-free crashes.
705
- // FilamentAvatarInner keeps FilamentView permanently mounted; only ModelRenderer
706
- // is conditional on the model being ready.
707
- return ((0, jsx_runtime_1.jsx)(react_native_1.View, { style: [style, styles.fill], children: (0, jsx_runtime_1.jsx)(react_native_filament_1.FilamentScene, { children: (0, jsx_runtime_1.jsx)(FilamentAvatarInner, { localUri: localUri, avatarUrl: avatarUrl, aspect: aspect, focalLength: focalLength, mood: mood, hairColor: hairColor, skinColor: skinColor, eyeColor: eyeColor, accessories: accessories, innerRef: ref, onReady: onReady, onError: onError }) }) }));
708
- });
709
- exports.FilamentAvatar.displayName = 'FilamentAvatar';
710
- const styles = react_native_1.StyleSheet.create({
711
- fill: { flex: 1 },
712
- empty: { backgroundColor: 'transparent' },
713
- center: { alignItems: 'center', justifyContent: 'center' },
714
- label: { color: 'rgba(226,232,240,0.6)', fontSize: 13 },
715
- });