talking-head-studio 0.4.11 → 0.4.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +279 -193
- package/dist/TalkingHead.d.ts +28 -3
- package/dist/TalkingHead.js +21 -2
- package/dist/TalkingHead.web.d.ts +31 -4
- package/dist/TalkingHead.web.js +11 -1
- package/dist/TalkingHeadVisualization.d.ts +22 -0
- package/dist/TalkingHeadVisualization.js +30 -10
- package/dist/api/studioApi.d.ts +12 -1
- package/dist/api/studioApi.js +16 -2
- package/dist/contract.d.ts +14 -0
- package/dist/contract.js +30 -0
- package/dist/core/avatar/avatarCapabilities.d.ts +60 -0
- package/dist/core/avatar/avatarCapabilities.js +100 -0
- package/dist/core/avatar/backends/gaussian.js +6 -4
- package/dist/core/avatar/motion.d.ts +1713 -0
- package/dist/core/avatar/motion.js +550 -0
- package/dist/core/avatar/motionRuntime.d.ts +46 -0
- package/dist/core/avatar/motionRuntime.js +84 -0
- package/dist/core/avatar/schema.d.ts +33 -5
- package/dist/core/avatar/visemes.d.ts +16 -1
- package/dist/core/avatar/visemes.js +48 -1
- package/dist/editor/AvatarCanvas.js +92 -1
- package/dist/editor/AvatarEditor.native.js +1 -0
- package/dist/editor/AvatarModel.js +1 -0
- package/dist/editor/FaceSqueezeEditor.d.ts +3 -1
- package/dist/editor/FaceSqueezeEditor.js +176 -112
- package/dist/editor/FaceSqueezeEditor.web.d.ts +3 -1
- package/dist/editor/FaceSqueezeEditor.web.js +30 -28
- package/dist/editor/RigidAccessory.js +17 -2
- package/dist/editor/SkinnedClothing.js +1 -0
- package/dist/editor/boneLockedDrag.d.ts +11 -0
- package/dist/editor/boneLockedDrag.js +68 -0
- package/dist/editor/boneSnap.web.d.ts +27 -0
- package/dist/editor/boneSnap.web.js +99 -0
- package/dist/editor/index.web.d.ts +10 -0
- package/dist/editor/index.web.js +26 -0
- package/dist/editor/sounds/haha.wav +0 -0
- package/dist/editor/sounds/owie.wav +0 -0
- package/dist/editor/sounds/stop.wav +0 -0
- package/dist/editor/studioTheme.d.ts +14 -14
- package/dist/editor/studioTheme.js +17 -14
- package/dist/editor/types.d.ts +1 -0
- package/dist/html/accessories.d.ts +7 -0
- package/dist/html/accessories.js +149 -0
- package/dist/html/motion.d.ts +1 -0
- package/dist/html/motion.js +189 -0
- package/dist/html/visemes.d.ts +7 -0
- package/dist/html/visemes.js +348 -0
- package/dist/html.d.ts +1 -1
- package/dist/html.js +55 -732
- package/dist/index.d.ts +7 -3
- package/dist/index.js +17 -1
- package/dist/index.web.d.ts +18 -1
- package/dist/index.web.js +36 -3
- package/dist/sketchfab/api.js +1 -0
- package/dist/sketchfab/glbInspect.d.ts +22 -0
- package/dist/sketchfab/glbInspect.js +58 -0
- package/dist/sketchfab/index.d.ts +3 -0
- package/dist/sketchfab/index.js +8 -1
- package/dist/sketchfab/inspectRemote.d.ts +13 -0
- package/dist/sketchfab/inspectRemote.js +77 -0
- package/dist/sketchfab/types.d.ts +10 -0
- package/dist/studio/AccessoryBrowserScreen.d.ts +6 -0
- package/dist/studio/AccessoryBrowserScreen.js +626 -0
- package/dist/studio/AccessoryPanel.d.ts +10 -0
- package/dist/studio/AccessoryPanel.js +396 -0
- package/dist/studio/AppearancePanel.d.ts +9 -0
- package/dist/studio/AppearancePanel.js +77 -0
- package/dist/studio/AvatarCreatorScreen.d.ts +5 -0
- package/dist/studio/AvatarCreatorScreen.js +806 -0
- package/dist/studio/AvatarEditorScreen.d.ts +14 -0
- package/dist/studio/AvatarEditorScreen.js +510 -0
- package/dist/studio/AvatarGrid.d.ts +23 -0
- package/dist/studio/AvatarGrid.js +257 -0
- package/dist/studio/ColorSwatch.d.ts +8 -0
- package/dist/studio/ColorSwatch.js +100 -0
- package/dist/studio/CreateVoiceProfileSheet.d.ts +8 -0
- package/dist/studio/CreateVoiceProfileSheet.js +242 -0
- package/dist/studio/DetailsPanel.d.ts +15 -0
- package/dist/studio/DetailsPanel.js +239 -0
- package/dist/studio/FilamentEditor.d.ts +2 -0
- package/dist/studio/FilamentEditor.js +6 -0
- package/dist/studio/PrecisionPanel.d.ts +2 -0
- package/dist/studio/PrecisionPanel.js +7 -0
- package/dist/studio/PublicGalleryScreen.d.ts +5 -0
- package/dist/studio/PublicGalleryScreen.js +358 -0
- package/dist/studio/SketchfabModelCard.d.ts +20 -0
- package/dist/studio/SketchfabModelCard.js +104 -0
- package/dist/studio/StudioBrowseHeader.d.ts +9 -0
- package/dist/studio/StudioBrowseHeader.js +28 -0
- package/dist/studio/StudioEmptyState.d.ts +8 -0
- package/dist/studio/StudioEmptyState.js +29 -0
- package/dist/studio/StudioFloatingAction.d.ts +13 -0
- package/dist/studio/StudioFloatingAction.js +42 -0
- package/dist/studio/StudioSectionHeader.d.ts +7 -0
- package/dist/studio/StudioSectionHeader.js +27 -0
- package/dist/studio/StudioSurfaceCard.d.ts +8 -0
- package/dist/studio/StudioSurfaceCard.js +20 -0
- package/dist/studio/VoicePanel.d.ts +15 -0
- package/dist/studio/VoicePanel.js +305 -0
- package/dist/studio/constants.d.ts +3 -0
- package/dist/studio/constants.js +6 -0
- package/dist/studio/index.d.ts +29 -0
- package/dist/studio/index.js +54 -0
- package/dist/studio/useSketchfabCapabilities.d.ts +31 -0
- package/dist/studio/useSketchfabCapabilities.js +82 -0
- package/dist/tts/useDirectVisemeStream.js +15 -10
- package/dist/utils/avatarUtils.js +92 -5
- package/dist/utils/faceLandmarkerToShapeWeights.js +2 -4
- package/dist/voice/useAudioPlayer.js +17 -4
- package/dist/voice/useVoicePreview.js +4 -2
- package/dist/wardrobe/index.d.ts +1 -0
- package/dist/wardrobe/index.js +6 -1
- package/dist/wardrobe/useAccessoryGestures.d.ts +20 -0
- package/dist/wardrobe/useAccessoryGestures.js +94 -0
- package/dist/wardrobe/useAvatarWardrobeHydration.js +8 -2
- package/dist/wardrobe/useStudioAvatar.js +11 -2
- package/dist/wardrobe/wardrobeStore.d.ts +2 -0
- package/dist/wardrobe/wardrobeStore.js +12 -2
- package/dist/wgpu/R3FWebGpuCanvas.d.ts +15 -0
- package/dist/wgpu/R3FWebGpuCanvas.js +176 -0
- package/dist/wgpu/WgpuAvatar.d.ts +26 -2
- package/dist/wgpu/WgpuAvatar.js +296 -39
- package/dist/wgpu/accessoryDefaults.d.ts +12 -0
- package/dist/wgpu/accessoryDefaults.js +19 -0
- package/dist/wgpu/blobShim.d.ts +2 -0
- package/dist/wgpu/blobShim.js +191 -0
- package/dist/wgpu/index.d.ts +1 -0
- package/dist/wgpu/index.js +4 -1
- package/dist/wgpu/loadGLTFFromUri.d.ts +2 -0
- package/dist/wgpu/loadGLTFFromUri.js +75 -0
- package/dist/wgpu/morphTables.js +21 -10
- package/dist/wgpu/motionState.d.ts +20 -0
- package/dist/wgpu/motionState.js +31 -0
- package/dist/wgpu/patchThreeForRN.d.ts +28 -0
- package/dist/wgpu/patchThreeForRN.js +292 -0
- package/dist/wgpu/scenePlacement.d.ts +5 -0
- package/dist/wgpu/scenePlacement.js +50 -0
- package/dist/wgpu/useAuthedModelUri.js +4 -2
- package/dist/wgpu/useNativeGLTF.d.ts +7 -0
- package/dist/wgpu/useNativeGLTF.js +36 -0
- package/package.json +97 -31
package/dist/wgpu/WgpuAvatar.js
CHANGED
|
@@ -37,24 +37,38 @@ exports.WgpuAvatar = void 0;
|
|
|
37
37
|
const jsx_runtime_1 = require("react/jsx-runtime");
|
|
38
38
|
/**
|
|
39
39
|
* WgpuAvatar — drop-in replacement for FilamentAvatar using react-three-fiber
|
|
40
|
-
* + react-native-wgpu
|
|
40
|
+
* + react-native-wgpu. Exposes the same ref interface as
|
|
41
41
|
* FilamentAvatarRef so siteclaw can swap renderers without changing call sites.
|
|
42
42
|
*
|
|
43
43
|
* No SurfaceTexture, no choreographer, no JNI surface lifecycle bugs.
|
|
44
44
|
*
|
|
45
45
|
* Peer deps required by the host app:
|
|
46
|
-
* react-native-wgpu
|
|
46
|
+
* react-native-wgpu
|
|
47
47
|
* @react-three/fiber >= 8
|
|
48
48
|
* @react-three/drei >= 9
|
|
49
49
|
* three >= 0.170
|
|
50
50
|
*/
|
|
51
|
+
// IMPORTANT: blobShim MUST be the very first import — before any three/R3F
|
|
52
|
+
// imports — so the Blob+createObjectURL patch is in place before R3F's
|
|
53
|
+
// polyfills() smoke-tests createObjectURL at module evaluation time.
|
|
54
|
+
require("./blobShim");
|
|
51
55
|
const react_1 = __importStar(require("react"));
|
|
52
56
|
const react_native_1 = require("react-native");
|
|
53
57
|
const THREE = __importStar(require("three"));
|
|
54
|
-
const
|
|
55
|
-
const native_2 = require("@react-three/drei/native");
|
|
58
|
+
const fiber_1 = require("@react-three/fiber");
|
|
56
59
|
const morphTables_1 = require("./morphTables");
|
|
57
60
|
const useAuthedModelUri_1 = require("./useAuthedModelUri");
|
|
61
|
+
const patchThreeForRN_1 = require("./patchThreeForRN");
|
|
62
|
+
const R3FWebGpuCanvas_1 = require("./R3FWebGpuCanvas");
|
|
63
|
+
const useNativeGLTF_1 = require("./useNativeGLTF");
|
|
64
|
+
const scenePlacement_1 = require("./scenePlacement");
|
|
65
|
+
const motion_1 = require("../core/avatar/motion");
|
|
66
|
+
const motionRuntime_1 = require("../core/avatar/motionRuntime");
|
|
67
|
+
const motionState_1 = require("./motionState");
|
|
68
|
+
// Patch Three.js loaders before any GLTF/texture loading happens.
|
|
69
|
+
// This fixes "Cannot create URL for blob!" on React Native — URL.createObjectURL
|
|
70
|
+
// doesn't exist in Hermes/JSC, but GLTFLoader calls it for embedded textures.
|
|
71
|
+
(0, patchThreeForRN_1.installThreeRNPatches)(false);
|
|
58
72
|
// ---------------------------------------------------------------------------
|
|
59
73
|
// Camera defaults — match FilamentAvatar constants
|
|
60
74
|
// ---------------------------------------------------------------------------
|
|
@@ -65,7 +79,7 @@ const CAMERA_FOV_FULL = 38;
|
|
|
65
79
|
// Scene/camera ref capture — runs inside Canvas so it can access R3F context
|
|
66
80
|
// ---------------------------------------------------------------------------
|
|
67
81
|
function SceneCapture({ onSceneReady }) {
|
|
68
|
-
const { scene, camera } = (0,
|
|
82
|
+
const { scene, camera } = (0, fiber_1.useThree)();
|
|
69
83
|
(0, react_1.useEffect)(() => {
|
|
70
84
|
onSceneReady(scene, camera);
|
|
71
85
|
// Only fire once on mount — scene/camera identity is stable
|
|
@@ -73,6 +87,13 @@ function SceneCapture({ onSceneReady }) {
|
|
|
73
87
|
}, []);
|
|
74
88
|
return null;
|
|
75
89
|
}
|
|
90
|
+
/** Walks up to the topmost ancestor so a bone scan covers the whole skeleton. */
|
|
91
|
+
function findSkeletonRoot(node) {
|
|
92
|
+
let root = node;
|
|
93
|
+
while (root.parent)
|
|
94
|
+
root = root.parent;
|
|
95
|
+
return root;
|
|
96
|
+
}
|
|
76
97
|
function buildMorphIndex(mesh) {
|
|
77
98
|
const map = new Map();
|
|
78
99
|
const dict = mesh.morphTargetDictionary;
|
|
@@ -92,12 +113,39 @@ function resolveIndex(morphIndex, aliases) {
|
|
|
92
113
|
}
|
|
93
114
|
return undefined;
|
|
94
115
|
}
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
116
|
+
// ---------------------------------------------------------------------------
|
|
117
|
+
// Accessory mesh — loads a GLB and parents it to a named bone on the avatar
|
|
118
|
+
// ---------------------------------------------------------------------------
|
|
119
|
+
function AccessoryMesh({ accessory, avatarScene, }) {
|
|
120
|
+
const { gltf } = (0, useNativeGLTF_1.useNativeGLTF)(accessory.url);
|
|
121
|
+
(0, react_1.useEffect)(() => {
|
|
122
|
+
if (!gltf?.scene)
|
|
123
|
+
return;
|
|
124
|
+
const bone = avatarScene.getObjectByName(accessory.bone ?? 'Head');
|
|
125
|
+
const target = bone ?? avatarScene;
|
|
126
|
+
const clone = gltf.scene.clone(true);
|
|
127
|
+
if (accessory.position)
|
|
128
|
+
clone.position.set(...accessory.position);
|
|
129
|
+
if (accessory.rotation)
|
|
130
|
+
clone.rotation.set(...accessory.rotation);
|
|
131
|
+
if (accessory.scale != null)
|
|
132
|
+
clone.scale.setScalar(accessory.scale);
|
|
133
|
+
target.add(clone);
|
|
134
|
+
return () => { target.remove(clone); };
|
|
135
|
+
}, [gltf, avatarScene, accessory]);
|
|
136
|
+
return null;
|
|
137
|
+
}
|
|
138
|
+
function AvatarScene({ uri, morphStateRef, motionStateRef, fov, accessories, animationRequest, onReady, onError, onAvatarState, onSceneReady, }) {
|
|
139
|
+
const { camera } = (0, fiber_1.useThree)();
|
|
140
|
+
const { gltf, error } = (0, useNativeGLTF_1.useNativeGLTF)(uri);
|
|
141
|
+
const { gltf: animationGltf, error: animationError } = (0, useNativeGLTF_1.useNativeGLTF)(animationRequest?.url ?? null);
|
|
142
|
+
const scene = gltf?.scene ?? null;
|
|
143
|
+
const placement = (0, react_1.useMemo)(() => (scene ? (0, scenePlacement_1.computeObjectPlacement)(scene, CAMERA_TARGET) : null), [scene]);
|
|
98
144
|
const headMeshRef = (0, react_1.useRef)(null);
|
|
99
145
|
const morphIndexRef = (0, react_1.useRef)(new Map());
|
|
100
146
|
const readyFiredRef = (0, react_1.useRef)(false);
|
|
147
|
+
const animationMixerRef = (0, react_1.useRef)(null);
|
|
148
|
+
const animationStopTimerRef = (0, react_1.useRef)(null);
|
|
101
149
|
(0, react_1.useEffect)(() => {
|
|
102
150
|
if (!(camera instanceof THREE.PerspectiveCamera))
|
|
103
151
|
return;
|
|
@@ -107,13 +155,25 @@ function AvatarScene({ uri, morphStateRef, fov, onReady, onError, onSceneReady }
|
|
|
107
155
|
camera.updateProjectionMatrix();
|
|
108
156
|
}, [camera, fov]);
|
|
109
157
|
(0, react_1.useEffect)(() => {
|
|
110
|
-
if (!
|
|
158
|
+
if (!error)
|
|
159
|
+
return;
|
|
160
|
+
onError(error.message);
|
|
161
|
+
}, [error, onError]);
|
|
162
|
+
(0, react_1.useEffect)(() => {
|
|
163
|
+
if (!animationError || !animationRequest)
|
|
164
|
+
return;
|
|
165
|
+
onAvatarState?.(`animation_error:${animationRequest.url}`);
|
|
166
|
+
onError(animationError.message);
|
|
167
|
+
}, [animationError, animationRequest, onAvatarState, onError]);
|
|
168
|
+
(0, react_1.useEffect)(() => {
|
|
169
|
+
if (!scene)
|
|
111
170
|
return;
|
|
112
171
|
let bestMesh = null;
|
|
113
172
|
let bestCount = 0;
|
|
114
|
-
|
|
173
|
+
scene.traverse((node) => {
|
|
115
174
|
if (!(node instanceof THREE.Mesh))
|
|
116
175
|
return;
|
|
176
|
+
node.frustumCulled = false;
|
|
117
177
|
const count = Object.keys(node.morphTargetDictionary ?? {}).length;
|
|
118
178
|
if (count > bestCount) {
|
|
119
179
|
bestCount = count;
|
|
@@ -121,8 +181,16 @@ function AvatarScene({ uri, morphStateRef, fov, onReady, onError, onSceneReady }
|
|
|
121
181
|
}
|
|
122
182
|
});
|
|
123
183
|
if (!bestMesh) {
|
|
124
|
-
|
|
125
|
-
|
|
184
|
+
// Fallback: no morph targets — find any mesh and use amplitude-only jaw drive
|
|
185
|
+
scene.traverse((node) => {
|
|
186
|
+
if (node instanceof THREE.Mesh && !bestMesh)
|
|
187
|
+
bestMesh = node;
|
|
188
|
+
});
|
|
189
|
+
if (!bestMesh) {
|
|
190
|
+
onError('No mesh found in GLB');
|
|
191
|
+
return;
|
|
192
|
+
}
|
|
193
|
+
console.warn('[WgpuAvatar] No morph targets found — using amplitude-only mode');
|
|
126
194
|
}
|
|
127
195
|
headMeshRef.current = bestMesh;
|
|
128
196
|
morphIndexRef.current = buildMorphIndex(bestMesh);
|
|
@@ -130,14 +198,64 @@ function AvatarScene({ uri, morphStateRef, fov, onReady, onError, onSceneReady }
|
|
|
130
198
|
readyFiredRef.current = true;
|
|
131
199
|
onReady();
|
|
132
200
|
}
|
|
133
|
-
}, [
|
|
134
|
-
(0,
|
|
201
|
+
}, [scene, onReady, onError]);
|
|
202
|
+
(0, react_1.useEffect)(() => {
|
|
203
|
+
if (animationStopTimerRef.current) {
|
|
204
|
+
clearTimeout(animationStopTimerRef.current);
|
|
205
|
+
animationStopTimerRef.current = null;
|
|
206
|
+
}
|
|
207
|
+
animationMixerRef.current?.stopAllAction();
|
|
208
|
+
animationMixerRef.current = null;
|
|
209
|
+
if (!scene || !animationRequest)
|
|
210
|
+
return;
|
|
211
|
+
const source = animationGltf ?? (animationRequest.url === uri ? gltf : null);
|
|
212
|
+
if (!source)
|
|
213
|
+
return;
|
|
214
|
+
const clip = source.animations[animationRequest.index] ?? source.animations[0];
|
|
215
|
+
if (!clip) {
|
|
216
|
+
onAvatarState?.(`animation_error:no_clip:${animationRequest.url}`);
|
|
217
|
+
return;
|
|
218
|
+
}
|
|
219
|
+
const mixer = new THREE.AnimationMixer(scene);
|
|
220
|
+
const action = mixer.clipAction(clip);
|
|
221
|
+
action.reset().play();
|
|
222
|
+
animationMixerRef.current = mixer;
|
|
223
|
+
onAvatarState?.(`animation:${animationRequest.url}`);
|
|
224
|
+
if (animationRequest.dur && animationRequest.dur > 0) {
|
|
225
|
+
animationStopTimerRef.current = setTimeout(() => {
|
|
226
|
+
mixer.stopAllAction();
|
|
227
|
+
if (animationMixerRef.current === mixer)
|
|
228
|
+
animationMixerRef.current = null;
|
|
229
|
+
onAvatarState?.('animation:stopped');
|
|
230
|
+
}, animationRequest.dur * 1000);
|
|
231
|
+
}
|
|
232
|
+
return () => {
|
|
233
|
+
if (animationStopTimerRef.current) {
|
|
234
|
+
clearTimeout(animationStopTimerRef.current);
|
|
235
|
+
animationStopTimerRef.current = null;
|
|
236
|
+
}
|
|
237
|
+
mixer.stopAllAction();
|
|
238
|
+
if (animationMixerRef.current === mixer)
|
|
239
|
+
animationMixerRef.current = null;
|
|
240
|
+
};
|
|
241
|
+
}, [animationGltf, animationRequest, gltf, onAvatarState, scene, uri]);
|
|
242
|
+
(0, fiber_1.useFrame)((_, delta) => {
|
|
135
243
|
const mesh = headMeshRef.current;
|
|
136
244
|
if (!mesh?.morphTargetInfluences)
|
|
137
245
|
return;
|
|
138
246
|
const state = morphStateRef.current;
|
|
139
|
-
|
|
140
|
-
const
|
|
247
|
+
// Frame-rate independent lerp: same convergence speed at 30 Hz and 120 Hz
|
|
248
|
+
const alpha = 1 - Math.exp(-state.alpha * delta * 60);
|
|
249
|
+
// Idle animation — subtle breathing / brow life at ~0.3 Hz
|
|
250
|
+
state.idleTime += delta;
|
|
251
|
+
const breathe = Math.sin(state.idleTime * 2 * Math.PI * 0.3) * 0.04;
|
|
252
|
+
const idleIdle = {
|
|
253
|
+
browRaiserL: Math.max(0, breathe),
|
|
254
|
+
browRaiserR: Math.max(0, breathe),
|
|
255
|
+
browRaiser_L: Math.max(0, breathe),
|
|
256
|
+
browRaiser_R: Math.max(0, breathe),
|
|
257
|
+
};
|
|
258
|
+
const combined = { ...state.moodBase, ...idleIdle };
|
|
141
259
|
for (const [name, w] of Object.entries(state.visemeTarget)) {
|
|
142
260
|
combined[name] = Math.max(combined[name] ?? 0, w);
|
|
143
261
|
}
|
|
@@ -151,19 +269,61 @@ function AvatarScene({ uri, morphStateRef, fov, onReady, onError, onSceneReady }
|
|
|
151
269
|
if (idx !== undefined)
|
|
152
270
|
mesh.morphTargetInfluences[idx] = next;
|
|
153
271
|
}
|
|
272
|
+
// Viseme decay: hold peak for 80 ms then decay (avoids eating fast cues)
|
|
273
|
+
const now = performance.now();
|
|
154
274
|
for (const name of Object.keys(state.visemeTarget)) {
|
|
155
|
-
state.
|
|
156
|
-
if (
|
|
157
|
-
|
|
275
|
+
const cueAge = (now - (state.visemeCueTime[name] ?? now)) / 1000;
|
|
276
|
+
if (cueAge > 0.08) {
|
|
277
|
+
state.visemeTarget[name] *= Math.pow(0.05, delta);
|
|
278
|
+
if (state.visemeTarget[name] < 0.001) {
|
|
279
|
+
delete state.visemeTarget[name];
|
|
280
|
+
delete state.visemeCueTime[name];
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
// Procedural body motion — drives the skeleton bones (independent of the
|
|
285
|
+
// face morphs above). Bones are scanned lazily on first active motion.
|
|
286
|
+
const motion = motionStateRef.current;
|
|
287
|
+
if (motion.key) {
|
|
288
|
+
if (!motion.bones) {
|
|
289
|
+
const root = headMeshRef.current?.parent ?? mesh;
|
|
290
|
+
const skeletonRoot = root ? findSkeletonRoot(root) : null;
|
|
291
|
+
if (skeletonRoot) {
|
|
292
|
+
const scanned = (0, motionRuntime_1.scanMotionBones)((visit) => {
|
|
293
|
+
skeletonRoot.traverse((node) => {
|
|
294
|
+
if (node.isBone) {
|
|
295
|
+
visit(node);
|
|
296
|
+
}
|
|
297
|
+
});
|
|
298
|
+
});
|
|
299
|
+
motion.bones = scanned.bones;
|
|
300
|
+
motion.rest = scanned.rest;
|
|
301
|
+
}
|
|
302
|
+
else {
|
|
303
|
+
// No skeleton — nothing to drive. Clear so we don't rescan every frame.
|
|
304
|
+
motion.key = null;
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
if (motion.key && motion.bones && motion.rest) {
|
|
308
|
+
const tSec = (now - motion.startTime) / 1000;
|
|
309
|
+
(0, motionRuntime_1.applyMotionFrame)(motion.key, tSec, motion.bones, motion.rest);
|
|
310
|
+
}
|
|
158
311
|
}
|
|
312
|
+
animationMixerRef.current?.update(delta);
|
|
159
313
|
});
|
|
160
|
-
|
|
314
|
+
if (!scene) {
|
|
315
|
+
return ((0, jsx_runtime_1.jsxs)(jsx_runtime_1.Fragment, { children: [onSceneReady && (0, jsx_runtime_1.jsx)(SceneCapture, { onSceneReady: onSceneReady }), (0, jsx_runtime_1.jsx)("ambientLight", { intensity: 0.6 }), (0, jsx_runtime_1.jsx)("directionalLight", { position: [1, 3, 2], intensity: 1.2, castShadow: false })] }));
|
|
316
|
+
}
|
|
317
|
+
return ((0, jsx_runtime_1.jsxs)(jsx_runtime_1.Fragment, { children: [onSceneReady && (0, jsx_runtime_1.jsx)(SceneCapture, { onSceneReady: onSceneReady }), (0, jsx_runtime_1.jsx)("ambientLight", { intensity: 0.6 }), (0, jsx_runtime_1.jsx)("directionalLight", { position: [1, 3, 2], intensity: 1.2, castShadow: false }), (0, jsx_runtime_1.jsx)("group", { position: placement?.position ?? CAMERA_TARGET, scale: placement?.scale ?? 1, children: (0, jsx_runtime_1.jsx)("primitive", { object: scene }) }), accessories?.map((acc) => ((0, jsx_runtime_1.jsx)(AccessoryMesh, { accessory: acc, avatarScene: scene }, acc.id)))] }));
|
|
161
318
|
}
|
|
162
319
|
// ---------------------------------------------------------------------------
|
|
163
320
|
// Error boundary
|
|
164
321
|
// ---------------------------------------------------------------------------
|
|
165
322
|
class GltfErrorBoundary extends react_1.default.Component {
|
|
166
|
-
constructor(props) {
|
|
323
|
+
constructor(props) {
|
|
324
|
+
super(props);
|
|
325
|
+
this.state = { hasError: false };
|
|
326
|
+
}
|
|
167
327
|
static getDerivedStateFromError() { return { hasError: true }; }
|
|
168
328
|
componentDidCatch(err) { this.props.onError(err.message); }
|
|
169
329
|
render() { if (this.state.hasError)
|
|
@@ -177,43 +337,88 @@ function applyVisemeCue(morphState, visemeKey) {
|
|
|
177
337
|
if (!aliases)
|
|
178
338
|
return;
|
|
179
339
|
const w = morphTables_1.VISEME_WEIGHTS[visemeKey] ?? morphTables_1.DEFAULT_VISEME_WEIGHT;
|
|
180
|
-
|
|
340
|
+
const now = performance.now();
|
|
341
|
+
for (const alias of aliases) {
|
|
181
342
|
morphState.visemeTarget[alias] = w;
|
|
343
|
+
morphState.visemeCueTime[alias] = now;
|
|
344
|
+
}
|
|
182
345
|
}
|
|
183
346
|
// ---------------------------------------------------------------------------
|
|
184
347
|
// Main exported component
|
|
185
348
|
// ---------------------------------------------------------------------------
|
|
186
|
-
exports.WgpuAvatar = (0, react_1.forwardRef)(({ style, avatarUrl, focalLength, mood = 'neutral', onReady, onError, onSceneReady }, ref) => {
|
|
187
|
-
const
|
|
188
|
-
|
|
349
|
+
exports.WgpuAvatar = (0, react_1.forwardRef)(({ style, avatarUrl, focalLength, mood = 'neutral', accessories, onReady, onError, onAvatarState, onSceneReady }, ref) => {
|
|
350
|
+
const isRemote = avatarUrl != null && (avatarUrl.startsWith('http://') || avatarUrl.startsWith('https://'));
|
|
351
|
+
const remoteUrl = (0, react_1.useMemo)(() => isRemote ? avatarUrl : null,
|
|
352
|
+
// eslint-disable-next-line react-hooks/exhaustive-deps
|
|
353
|
+
[avatarUrl]);
|
|
189
354
|
const fileResult = (0, useAuthedModelUri_1.useAuthedModelUri)(remoteUrl);
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
355
|
+
// stableUri: once a valid URI is resolved, keep showing it until a new one
|
|
356
|
+
// is ready — prevents Canvas unmount/remount flicker on transient url changes.
|
|
357
|
+
const stableUriRef = (0, react_1.useRef)(null);
|
|
358
|
+
const lastAvatarUrlRef = (0, react_1.useRef)(avatarUrl);
|
|
359
|
+
if (lastAvatarUrlRef.current !== avatarUrl) {
|
|
360
|
+
// avatarUrl changed — invalidate stable URI so next resolved URI takes over
|
|
361
|
+
lastAvatarUrlRef.current = avatarUrl;
|
|
362
|
+
stableUriRef.current = null;
|
|
363
|
+
}
|
|
364
|
+
// For non-remote URLs (file://, asset://, data:) use directly without download
|
|
365
|
+
const currentUri = isRemote ? (fileResult?.uri ?? null) : (avatarUrl ?? null);
|
|
366
|
+
if (currentUri && currentUri !== stableUriRef.current) {
|
|
367
|
+
stableUriRef.current = currentUri;
|
|
195
368
|
}
|
|
196
|
-
const
|
|
197
|
-
if (lockedUriRef.current === null && currentUri)
|
|
198
|
-
lockedUriRef.current = currentUri;
|
|
199
|
-
const localUri = lockedUriRef.current;
|
|
369
|
+
const localUri = stableUriRef.current;
|
|
200
370
|
const fov = focalLength
|
|
201
371
|
? Math.round(2 * Math.atan(21.634 / focalLength) * (180 / Math.PI))
|
|
202
372
|
: CAMERA_FOV_FULL;
|
|
373
|
+
const cameraProps = (0, react_1.useMemo)(() => ({ fov, position: CAMERA_POSITION, near: 0.01, far: 100 }), [fov]);
|
|
203
374
|
const morphStateRef = (0, react_1.useRef)({
|
|
204
|
-
current: {}, visemeTarget: {}, moodBase: {}, alpha: 0.18,
|
|
375
|
+
current: {}, visemeTarget: {}, visemeCueTime: {}, moodBase: {}, alpha: 0.18, idleTime: 0,
|
|
205
376
|
});
|
|
377
|
+
const motionStateRef = (0, react_1.useRef)((0, motionState_1.createMotionState)());
|
|
378
|
+
const [animationRequest, setAnimationRequest] = (0, react_1.useState)(null);
|
|
379
|
+
const animationNonceRef = (0, react_1.useRef)(0);
|
|
380
|
+
const onAvatarStateRef = (0, react_1.useRef)(onAvatarState);
|
|
381
|
+
onAvatarStateRef.current = onAvatarState;
|
|
206
382
|
(0, react_1.useEffect)(() => {
|
|
207
383
|
morphStateRef.current.moodBase = { ...(morphTables_1.MOOD_MORPHS[mood] ?? {}) };
|
|
208
384
|
}, [mood]);
|
|
209
385
|
const scheduleRef = (0, react_1.useRef)(null);
|
|
210
386
|
const scheduleTimersRef = (0, react_1.useRef)([]);
|
|
387
|
+
const blinkTimersRef = (0, react_1.useRef)([]);
|
|
211
388
|
const [isReady, setIsReady] = (0, react_1.useState)(false);
|
|
212
389
|
const clearScheduleTimers = (0, react_1.useCallback)(() => {
|
|
213
390
|
for (const t of scheduleTimersRef.current)
|
|
214
391
|
clearTimeout(t);
|
|
215
392
|
scheduleTimersRef.current = [];
|
|
216
393
|
}, []);
|
|
394
|
+
(0, react_1.useEffect)(() => {
|
|
395
|
+
(0, motionState_1.resetMotionStateForAvatarSwap)(motionStateRef.current);
|
|
396
|
+
}, [localUri]);
|
|
397
|
+
const stopMotion = (0, react_1.useCallback)(() => {
|
|
398
|
+
(0, motionState_1.stopMotionState)(motionStateRef.current);
|
|
399
|
+
}, []);
|
|
400
|
+
const playMotionKey = (0, react_1.useCallback)((name, autoReturnOverrideMs) => {
|
|
401
|
+
if (!(0, motion_1.isMotionKey)(name)) {
|
|
402
|
+
onAvatarStateRef.current?.(`motion_error:unknown:${name}`);
|
|
403
|
+
return false;
|
|
404
|
+
}
|
|
405
|
+
const motion = motionStateRef.current;
|
|
406
|
+
if (motion.autoReturnTimer) {
|
|
407
|
+
clearTimeout(motion.autoReturnTimer);
|
|
408
|
+
motion.autoReturnTimer = null;
|
|
409
|
+
}
|
|
410
|
+
motion.key = name;
|
|
411
|
+
motion.startTime = performance.now();
|
|
412
|
+
onAvatarStateRef.current?.(`motion:${name}`);
|
|
413
|
+
const autoReturnMs = autoReturnOverrideMs ?? motion_1.MOTION_DEFS[name].autoReturnMs;
|
|
414
|
+
if (autoReturnMs && autoReturnMs > 0) {
|
|
415
|
+
motion.autoReturnTimer = setTimeout(() => {
|
|
416
|
+
if (motionStateRef.current.key === name)
|
|
417
|
+
stopMotion();
|
|
418
|
+
}, autoReturnMs);
|
|
419
|
+
}
|
|
420
|
+
return true;
|
|
421
|
+
}, [stopMotion]);
|
|
217
422
|
const applySchedule = (0, react_1.useCallback)((schedule) => {
|
|
218
423
|
clearScheduleTimers();
|
|
219
424
|
const now = Date.now();
|
|
@@ -245,6 +450,12 @@ exports.WgpuAvatar = (0, react_1.forwardRef)(({ style, avatarUrl, focalLength, m
|
|
|
245
450
|
onError?.(msg);
|
|
246
451
|
}, [onError]);
|
|
247
452
|
(0, react_1.useEffect)(() => () => clearScheduleTimers(), [clearScheduleTimers]);
|
|
453
|
+
(0, react_1.useEffect)(() => () => { for (const t of blinkTimersRef.current)
|
|
454
|
+
clearTimeout(t); }, []);
|
|
455
|
+
(0, react_1.useEffect)(() => () => (0, motionState_1.resetMotionStateForAvatarSwap)(motionStateRef.current), []);
|
|
456
|
+
const handleCanvasCreated = (0, react_1.useCallback)((state) => {
|
|
457
|
+
state.scene.background = new THREE.Color('#1a1a2e');
|
|
458
|
+
}, []);
|
|
248
459
|
(0, react_1.useImperativeHandle)(ref, () => ({
|
|
249
460
|
setMood: (m) => { morphStateRef.current.moodBase = { ...(morphTables_1.MOOD_MORPHS[m] ?? {}) }; },
|
|
250
461
|
sendAmplitude: () => { },
|
|
@@ -253,10 +464,55 @@ exports.WgpuAvatar = (0, react_1.forwardRef)(({ style, avatarUrl, focalLength, m
|
|
|
253
464
|
if (!aliases)
|
|
254
465
|
return;
|
|
255
466
|
const w = weight ?? morphTables_1.VISEME_WEIGHTS[viseme] ?? morphTables_1.DEFAULT_VISEME_WEIGHT;
|
|
256
|
-
|
|
467
|
+
const now = performance.now();
|
|
468
|
+
for (const alias of aliases) {
|
|
257
469
|
morphStateRef.current.visemeTarget[alias] = w;
|
|
470
|
+
morphStateRef.current.visemeCueTime[alias] = now;
|
|
471
|
+
}
|
|
472
|
+
},
|
|
473
|
+
clearVisemes: () => { clearScheduleTimers(); morphStateRef.current.visemeTarget = {}; morphStateRef.current.visemeCueTime = {}; },
|
|
474
|
+
playMotion: (name) => {
|
|
475
|
+
playMotionKey(name);
|
|
476
|
+
},
|
|
477
|
+
stopMotion,
|
|
478
|
+
playGesture: (name, opts) => {
|
|
479
|
+
playMotionKey(name, opts?.ms ?? 1000);
|
|
480
|
+
},
|
|
481
|
+
stopGesture: stopMotion,
|
|
482
|
+
playPose: (name) => {
|
|
483
|
+
playMotionKey(name);
|
|
484
|
+
},
|
|
485
|
+
stopPose: stopMotion,
|
|
486
|
+
playAnimation: (url, opts) => {
|
|
487
|
+
animationNonceRef.current += 1;
|
|
488
|
+
setAnimationRequest({
|
|
489
|
+
url,
|
|
490
|
+
index: opts?.index ?? 0,
|
|
491
|
+
dur: opts?.dur,
|
|
492
|
+
nonce: animationNonceRef.current,
|
|
493
|
+
});
|
|
494
|
+
},
|
|
495
|
+
stopAnimation: () => {
|
|
496
|
+
setAnimationRequest(null);
|
|
497
|
+
onAvatarStateRef.current?.('animation:stopped');
|
|
498
|
+
},
|
|
499
|
+
triggerBlink: () => {
|
|
500
|
+
// Blink: close eyes over 80ms, hold 40ms, open over 80ms
|
|
501
|
+
const BLINK_MORPHS = [
|
|
502
|
+
'eyeBlinkLeft', 'eyeBlink_L', 'eyesClosed',
|
|
503
|
+
'eyeBlinkRight', 'eyeBlink_R',
|
|
504
|
+
];
|
|
505
|
+
const now = performance.now();
|
|
506
|
+
const set = (w) => {
|
|
507
|
+
for (const name of BLINK_MORPHS) {
|
|
508
|
+
morphStateRef.current.visemeTarget[name] = w;
|
|
509
|
+
morphStateRef.current.visemeCueTime[name] = now;
|
|
510
|
+
}
|
|
511
|
+
};
|
|
512
|
+
set(1);
|
|
513
|
+
const t1 = setTimeout(() => set(0), 120);
|
|
514
|
+
blinkTimersRef.current.push(t1);
|
|
258
515
|
},
|
|
259
|
-
clearVisemes: () => { clearScheduleTimers(); morphStateRef.current.visemeTarget = {}; },
|
|
260
516
|
scheduleVisemes: (schedule) => {
|
|
261
517
|
if (!isReady) {
|
|
262
518
|
scheduleRef.current = schedule;
|
|
@@ -264,13 +520,14 @@ exports.WgpuAvatar = (0, react_1.forwardRef)(({ style, avatarUrl, focalLength, m
|
|
|
264
520
|
}
|
|
265
521
|
applySchedule(schedule);
|
|
266
522
|
},
|
|
267
|
-
}), [isReady, applySchedule, clearScheduleTimers]);
|
|
523
|
+
}), [isReady, applySchedule, clearScheduleTimers, playMotionKey, stopMotion]);
|
|
268
524
|
if (!localUri)
|
|
269
525
|
return (0, jsx_runtime_1.jsx)(react_native_1.View, { style: [styles.placeholder, style] });
|
|
270
|
-
return ((0, jsx_runtime_1.jsx)(react_native_1.View, { style: [styles.container, style], children: (0, jsx_runtime_1.jsx)(
|
|
526
|
+
return ((0, jsx_runtime_1.jsx)(react_native_1.View, { style: [styles.container, style], children: (0, jsx_runtime_1.jsx)(R3FWebGpuCanvas_1.R3FWebGpuCanvas, { style: styles.canvas, camera: cameraProps, clearColor: "#1a1a2e", onCreated: handleCanvasCreated, onError: handleError, children: (0, jsx_runtime_1.jsx)(GltfErrorBoundary, { onError: handleError, children: (0, jsx_runtime_1.jsx)(AvatarScene, { uri: localUri, morphStateRef: morphStateRef, motionStateRef: motionStateRef, fov: fov, accessories: accessories, animationRequest: animationRequest, onReady: handleReady, onError: handleError, onAvatarState: onAvatarState, onSceneReady: onSceneReady }) }) }) }));
|
|
271
527
|
});
|
|
272
528
|
exports.WgpuAvatar.displayName = 'WgpuAvatar';
|
|
273
529
|
const styles = react_native_1.StyleSheet.create({
|
|
274
530
|
container: { overflow: 'hidden', backgroundColor: '#1a1a2e' },
|
|
531
|
+
canvas: { flex: 1 },
|
|
275
532
|
placeholder: { backgroundColor: '#1a1a2e' },
|
|
276
533
|
});
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Canonical snap-to-bone defaults per accessory category.
|
|
3
|
+
* Applied immediately when the user equips an item that has no saved placement,
|
|
4
|
+
* so the "good enough" position is free on tap for ~90% of use-cases.
|
|
5
|
+
*
|
|
6
|
+
* Offsets are in the bone's local space (metres).
|
|
7
|
+
* Rotation is Euler XYZ in radians.
|
|
8
|
+
*/
|
|
9
|
+
import type { AssetPlacement } from '../wardrobe/wardrobeStore';
|
|
10
|
+
export declare const BONE_SNAP_DEFAULTS: Record<string, AssetPlacement>;
|
|
11
|
+
/** Returns the snap default for a category, or null if the category is unknown. */
|
|
12
|
+
export declare function snapDefaultForCategory(category: string): AssetPlacement | null;
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.BONE_SNAP_DEFAULTS = void 0;
|
|
4
|
+
exports.snapDefaultForCategory = snapDefaultForCategory;
|
|
5
|
+
exports.BONE_SNAP_DEFAULTS = {
|
|
6
|
+
hat: { bone: 'Head', position: [0, 0.12, 0.00], rotation: [0, 0, 0], scale: 1.0 },
|
|
7
|
+
cap: { bone: 'Head', position: [0, 0.10, 0.02], rotation: [0, 0, 0], scale: 1.0 },
|
|
8
|
+
glasses: { bone: 'Head', position: [0, 0.02, 0.07], rotation: [0, 0, 0], scale: 1.0 },
|
|
9
|
+
sunglasses: { bone: 'Head', position: [0, 0.02, 0.07], rotation: [0, 0, 0], scale: 1.0 },
|
|
10
|
+
earring: { bone: 'Head', position: [0.085, 0.01, 0.00], rotation: [0, 0, 0], scale: 0.8 },
|
|
11
|
+
earrings: { bone: 'Head', position: [0.085, 0.01, 0.00], rotation: [0, 0, 0], scale: 0.8 },
|
|
12
|
+
necklace: { bone: 'Neck', position: [0, -0.04, 0.03], rotation: [0, 0, 0], scale: 1.0 },
|
|
13
|
+
mask: { bone: 'Head', position: [0, 0.00, 0.08], rotation: [0, 0, 0], scale: 1.0 },
|
|
14
|
+
};
|
|
15
|
+
/** Returns the snap default for a category, or null if the category is unknown. */
|
|
16
|
+
function snapDefaultForCategory(category) {
|
|
17
|
+
const key = category.toLowerCase().trim();
|
|
18
|
+
return exports.BONE_SNAP_DEFAULTS[key] ?? null;
|
|
19
|
+
}
|