talking-head-studio 0.4.10 → 0.4.12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (178) hide show
  1. package/README.md +299 -337
  2. package/dist/TalkingHead.d.ts +44 -28
  3. package/dist/TalkingHead.js +21 -2
  4. package/dist/TalkingHead.web.d.ts +37 -4
  5. package/dist/TalkingHead.web.js +28 -8
  6. package/dist/TalkingHeadVisualization.d.ts +22 -0
  7. package/dist/TalkingHeadVisualization.js +30 -10
  8. package/dist/api/studioApi.d.ts +12 -1
  9. package/dist/api/studioApi.js +41 -28
  10. package/dist/appearance/apply.js +2 -3
  11. package/dist/appearance/matchers.js +1 -2
  12. package/dist/appearance/schema.js +1 -2
  13. package/dist/contract.d.ts +14 -0
  14. package/dist/contract.js +30 -0
  15. package/dist/core/avatar/avatarCapabilities.d.ts +60 -0
  16. package/dist/core/avatar/avatarCapabilities.js +100 -0
  17. package/dist/core/avatar/backend.d.ts +130 -0
  18. package/dist/core/avatar/backend.js +4 -0
  19. package/dist/core/avatar/backends/gaussian.d.ts +49 -0
  20. package/dist/core/avatar/backends/gaussian.js +293 -0
  21. package/dist/core/avatar/backends/index.d.ts +3 -0
  22. package/dist/core/avatar/backends/index.js +7 -0
  23. package/dist/core/avatar/backends/morphTarget.d.ts +39 -0
  24. package/dist/core/avatar/backends/morphTarget.js +179 -0
  25. package/dist/core/avatar/faceControls.d.ts +40 -0
  26. package/dist/core/avatar/faceControls.js +138 -0
  27. package/dist/core/avatar/motion.d.ts +1713 -0
  28. package/dist/core/avatar/motion.js +550 -0
  29. package/dist/core/avatar/motionRuntime.d.ts +46 -0
  30. package/dist/core/avatar/motionRuntime.js +84 -0
  31. package/dist/core/avatar/schema.d.ts +78 -0
  32. package/dist/core/avatar/schema.js +134 -0
  33. package/dist/core/avatar/visemes.d.ts +47 -1
  34. package/dist/core/avatar/visemes.js +114 -1
  35. package/dist/editor/AvatarCanvas.js +93 -3
  36. package/dist/editor/AvatarEditor.native.js +19 -9
  37. package/dist/editor/AvatarModel.js +2 -2
  38. package/dist/editor/FaceSqueezeEditor.d.ts +3 -1
  39. package/dist/editor/FaceSqueezeEditor.js +195 -121
  40. package/dist/editor/FaceSqueezeEditor.web.d.ts +3 -1
  41. package/dist/editor/FaceSqueezeEditor.web.js +32 -30
  42. package/dist/editor/RigidAccessory.js +18 -4
  43. package/dist/editor/SkinnedClothing.js +19 -9
  44. package/dist/editor/boneLockedDrag.d.ts +11 -0
  45. package/dist/editor/boneLockedDrag.js +68 -0
  46. package/dist/editor/boneSnap.js +22 -12
  47. package/dist/editor/boneSnap.web.d.ts +27 -0
  48. package/dist/editor/boneSnap.web.js +99 -0
  49. package/dist/editor/index.web.d.ts +10 -0
  50. package/dist/editor/index.web.js +26 -0
  51. package/dist/editor/sounds/haha.wav +0 -0
  52. package/dist/editor/sounds/owie.wav +0 -0
  53. package/dist/editor/sounds/stop.wav +0 -0
  54. package/dist/editor/studioTheme.d.ts +14 -14
  55. package/dist/editor/studioTheme.js +19 -16
  56. package/dist/editor/types.d.ts +1 -0
  57. package/dist/html/accessories.d.ts +7 -0
  58. package/dist/html/accessories.js +149 -0
  59. package/dist/html/motion.d.ts +1 -0
  60. package/dist/html/motion.js +189 -0
  61. package/dist/html/visemes.d.ts +7 -0
  62. package/dist/html/visemes.js +348 -0
  63. package/dist/html.d.ts +1 -1
  64. package/dist/html.js +56 -734
  65. package/dist/index.d.ts +19 -1
  66. package/dist/index.js +44 -5
  67. package/dist/index.web.d.ts +18 -1
  68. package/dist/index.web.js +36 -3
  69. package/dist/platform/api/types.d.ts +10 -0
  70. package/dist/platform/api/types.js +2 -0
  71. package/dist/platform/marketplace/types.d.ts +32 -0
  72. package/dist/platform/marketplace/types.js +2 -0
  73. package/dist/platform/sdk/unity.d.ts +27 -0
  74. package/dist/platform/sdk/unity.js +2 -0
  75. package/dist/platform/sdk/unreal.d.ts +23 -0
  76. package/dist/platform/sdk/unreal.js +2 -0
  77. package/dist/platform/sdk/web.d.ts +16 -0
  78. package/dist/platform/sdk/web.js +2 -0
  79. package/dist/sketchfab/api.js +5 -5
  80. package/dist/sketchfab/glbInspect.d.ts +22 -0
  81. package/dist/sketchfab/glbInspect.js +58 -0
  82. package/dist/sketchfab/index.d.ts +3 -0
  83. package/dist/sketchfab/index.js +8 -1
  84. package/dist/sketchfab/inspectRemote.d.ts +13 -0
  85. package/dist/sketchfab/inspectRemote.js +77 -0
  86. package/dist/sketchfab/types.d.ts +10 -0
  87. package/dist/sketchfab/useSketchfabSearch.js +1 -2
  88. package/dist/studio/AccessoryBrowserScreen.d.ts +6 -0
  89. package/dist/studio/AccessoryBrowserScreen.js +626 -0
  90. package/dist/studio/AccessoryPanel.d.ts +10 -0
  91. package/dist/studio/AccessoryPanel.js +396 -0
  92. package/dist/studio/AppearancePanel.d.ts +9 -0
  93. package/dist/studio/AppearancePanel.js +77 -0
  94. package/dist/studio/AvatarCreatorScreen.d.ts +5 -0
  95. package/dist/studio/AvatarCreatorScreen.js +806 -0
  96. package/dist/studio/AvatarEditorScreen.d.ts +14 -0
  97. package/dist/studio/AvatarEditorScreen.js +510 -0
  98. package/dist/studio/AvatarGrid.d.ts +23 -0
  99. package/dist/studio/AvatarGrid.js +257 -0
  100. package/dist/studio/ColorSwatch.d.ts +8 -0
  101. package/dist/studio/ColorSwatch.js +100 -0
  102. package/dist/studio/CreateVoiceProfileSheet.d.ts +8 -0
  103. package/dist/studio/CreateVoiceProfileSheet.js +242 -0
  104. package/dist/studio/DetailsPanel.d.ts +15 -0
  105. package/dist/studio/DetailsPanel.js +239 -0
  106. package/dist/studio/FilamentEditor.d.ts +2 -0
  107. package/dist/studio/FilamentEditor.js +6 -0
  108. package/dist/studio/PrecisionPanel.d.ts +2 -0
  109. package/dist/studio/PrecisionPanel.js +7 -0
  110. package/dist/studio/PublicGalleryScreen.d.ts +5 -0
  111. package/dist/studio/PublicGalleryScreen.js +358 -0
  112. package/dist/studio/SketchfabModelCard.d.ts +20 -0
  113. package/dist/studio/SketchfabModelCard.js +104 -0
  114. package/dist/studio/StudioBrowseHeader.d.ts +9 -0
  115. package/dist/studio/StudioBrowseHeader.js +28 -0
  116. package/dist/studio/StudioEmptyState.d.ts +8 -0
  117. package/dist/studio/StudioEmptyState.js +29 -0
  118. package/dist/studio/StudioFloatingAction.d.ts +13 -0
  119. package/dist/studio/StudioFloatingAction.js +42 -0
  120. package/dist/studio/StudioSectionHeader.d.ts +7 -0
  121. package/dist/studio/StudioSectionHeader.js +27 -0
  122. package/dist/studio/StudioSurfaceCard.d.ts +8 -0
  123. package/dist/studio/StudioSurfaceCard.js +20 -0
  124. package/dist/studio/VoicePanel.d.ts +15 -0
  125. package/dist/studio/VoicePanel.js +305 -0
  126. package/dist/studio/constants.d.ts +3 -0
  127. package/dist/studio/constants.js +6 -0
  128. package/dist/studio/index.d.ts +29 -0
  129. package/dist/studio/index.js +54 -0
  130. package/dist/studio/useSketchfabCapabilities.d.ts +31 -0
  131. package/dist/studio/useSketchfabCapabilities.js +82 -0
  132. package/dist/tts/useDirectVisemeStream.d.ts +2 -6
  133. package/dist/tts/useDirectVisemeStream.js +16 -12
  134. package/dist/tts/useMotionMarkers.d.ts +0 -1
  135. package/dist/tts/useMotionMarkers.js +1 -2
  136. package/dist/utils/avatarUtils.js +94 -8
  137. package/dist/utils/faceLandmarkerToShapeWeights.js +21 -14
  138. package/dist/voice/convertToWav.js +1 -2
  139. package/dist/voice/index.d.ts +3 -0
  140. package/dist/voice/index.js +6 -1
  141. package/dist/voice/useAudioPlayer.js +18 -6
  142. package/dist/voice/useAudioRecording.js +1 -2
  143. package/dist/voice/useFaceControls.d.ts +14 -0
  144. package/dist/voice/useFaceControls.js +81 -0
  145. package/dist/voice/useVoicePreview.d.ts +7 -0
  146. package/dist/voice/useVoicePreview.js +83 -0
  147. package/dist/wardrobe/index.d.ts +3 -0
  148. package/dist/wardrobe/index.js +8 -1
  149. package/dist/wardrobe/useAccessoryGestures.d.ts +20 -0
  150. package/dist/wardrobe/useAccessoryGestures.js +94 -0
  151. package/dist/wardrobe/useAvatarWardrobeHydration.js +9 -4
  152. package/dist/wardrobe/useStudioAvatar.d.ts +29 -0
  153. package/dist/wardrobe/useStudioAvatar.js +186 -0
  154. package/dist/wardrobe/wardrobeStore.d.ts +2 -0
  155. package/dist/wardrobe/wardrobeStore.js +12 -2
  156. package/dist/wgpu/R3FWebGpuCanvas.d.ts +15 -0
  157. package/dist/wgpu/R3FWebGpuCanvas.js +176 -0
  158. package/dist/wgpu/WgpuAvatar.d.ts +26 -2
  159. package/dist/wgpu/WgpuAvatar.js +313 -46
  160. package/dist/wgpu/accessoryDefaults.d.ts +12 -0
  161. package/dist/wgpu/accessoryDefaults.js +19 -0
  162. package/dist/wgpu/blobShim.d.ts +2 -0
  163. package/dist/wgpu/blobShim.js +191 -0
  164. package/dist/wgpu/index.d.ts +1 -0
  165. package/dist/wgpu/index.js +4 -1
  166. package/dist/wgpu/loadGLTFFromUri.d.ts +2 -0
  167. package/dist/wgpu/loadGLTFFromUri.js +75 -0
  168. package/dist/wgpu/morphTables.js +21 -10
  169. package/dist/wgpu/motionState.d.ts +20 -0
  170. package/dist/wgpu/motionState.js +31 -0
  171. package/dist/wgpu/patchThreeForRN.d.ts +28 -0
  172. package/dist/wgpu/patchThreeForRN.js +292 -0
  173. package/dist/wgpu/scenePlacement.d.ts +5 -0
  174. package/dist/wgpu/scenePlacement.js +50 -0
  175. package/dist/wgpu/useAuthedModelUri.js +22 -11
  176. package/dist/wgpu/useNativeGLTF.d.ts +7 -0
  177. package/dist/wgpu/useNativeGLTF.js +36 -0
  178. package/package.json +102 -32
@@ -15,36 +15,60 @@ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (
15
15
  }) : function(o, v) {
16
16
  o["default"] = v;
17
17
  });
18
- var __importStar = (this && this.__importStar) || function (mod) {
19
- if (mod && mod.__esModule) return mod;
20
- var result = {};
21
- if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
22
- __setModuleDefault(result, mod);
23
- return result;
24
- };
18
+ var __importStar = (this && this.__importStar) || (function () {
19
+ var ownKeys = function(o) {
20
+ ownKeys = Object.getOwnPropertyNames || function (o) {
21
+ var ar = [];
22
+ for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
23
+ return ar;
24
+ };
25
+ return ownKeys(o);
26
+ };
27
+ return function (mod) {
28
+ if (mod && mod.__esModule) return mod;
29
+ var result = {};
30
+ if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
31
+ __setModuleDefault(result, mod);
32
+ return result;
33
+ };
34
+ })();
25
35
  Object.defineProperty(exports, "__esModule", { value: true });
26
36
  exports.WgpuAvatar = void 0;
27
37
  const jsx_runtime_1 = require("react/jsx-runtime");
28
38
  /**
29
39
  * WgpuAvatar — drop-in replacement for FilamentAvatar using react-three-fiber
30
- * + react-native-wgpu (or expo-gl fallback). Exposes the same ref interface as
40
+ * + react-native-wgpu. Exposes the same ref interface as
31
41
  * FilamentAvatarRef so siteclaw can swap renderers without changing call sites.
32
42
  *
33
43
  * No SurfaceTexture, no choreographer, no JNI surface lifecycle bugs.
34
44
  *
35
45
  * Peer deps required by the host app:
36
- * react-native-wgpu (or expo-gl for the expo-gl fallback canvas)
46
+ * react-native-wgpu
37
47
  * @react-three/fiber >= 8
38
48
  * @react-three/drei >= 9
39
49
  * three >= 0.170
40
50
  */
51
+ // IMPORTANT: blobShim MUST be the very first import — before any three/R3F
52
+ // imports — so the Blob+createObjectURL patch is in place before R3F's
53
+ // polyfills() smoke-tests createObjectURL at module evaluation time.
54
+ require("./blobShim");
41
55
  const react_1 = __importStar(require("react"));
42
56
  const react_native_1 = require("react-native");
43
57
  const THREE = __importStar(require("three"));
44
- const native_1 = require("@react-three/fiber/native");
45
- const native_2 = require("@react-three/drei/native");
58
+ const fiber_1 = require("@react-three/fiber");
46
59
  const morphTables_1 = require("./morphTables");
47
60
  const useAuthedModelUri_1 = require("./useAuthedModelUri");
61
+ const patchThreeForRN_1 = require("./patchThreeForRN");
62
+ const R3FWebGpuCanvas_1 = require("./R3FWebGpuCanvas");
63
+ const useNativeGLTF_1 = require("./useNativeGLTF");
64
+ const scenePlacement_1 = require("./scenePlacement");
65
+ const motion_1 = require("../core/avatar/motion");
66
+ const motionRuntime_1 = require("../core/avatar/motionRuntime");
67
+ const motionState_1 = require("./motionState");
68
+ // Patch Three.js loaders before any GLTF/texture loading happens.
69
+ // This fixes "Cannot create URL for blob!" on React Native — URL.createObjectURL
70
+ // doesn't exist in Hermes/JSC, but GLTFLoader calls it for embedded textures.
71
+ (0, patchThreeForRN_1.installThreeRNPatches)(false);
48
72
  // ---------------------------------------------------------------------------
49
73
  // Camera defaults — match FilamentAvatar constants
50
74
  // ---------------------------------------------------------------------------
@@ -55,7 +79,7 @@ const CAMERA_FOV_FULL = 38;
55
79
  // Scene/camera ref capture — runs inside Canvas so it can access R3F context
56
80
  // ---------------------------------------------------------------------------
57
81
  function SceneCapture({ onSceneReady }) {
58
- const { scene, camera } = (0, native_1.useThree)();
82
+ const { scene, camera } = (0, fiber_1.useThree)();
59
83
  (0, react_1.useEffect)(() => {
60
84
  onSceneReady(scene, camera);
61
85
  // Only fire once on mount — scene/camera identity is stable
@@ -63,6 +87,13 @@ function SceneCapture({ onSceneReady }) {
63
87
  }, []);
64
88
  return null;
65
89
  }
90
+ /** Walks up to the topmost ancestor so a bone scan covers the whole skeleton. */
91
+ function findSkeletonRoot(node) {
92
+ let root = node;
93
+ while (root.parent)
94
+ root = root.parent;
95
+ return root;
96
+ }
66
97
  function buildMorphIndex(mesh) {
67
98
  const map = new Map();
68
99
  const dict = mesh.morphTargetDictionary;
@@ -82,12 +113,39 @@ function resolveIndex(morphIndex, aliases) {
82
113
  }
83
114
  return undefined;
84
115
  }
85
- function AvatarScene({ uri, morphStateRef, fov, onReady, onError, onSceneReady }) {
86
- const { camera } = (0, native_1.useThree)();
87
- const gltf = (0, native_2.useGLTF)(uri);
116
+ // ---------------------------------------------------------------------------
117
+ // Accessory mesh loads a GLB and parents it to a named bone on the avatar
118
+ // ---------------------------------------------------------------------------
119
+ function AccessoryMesh({ accessory, avatarScene, }) {
120
+ const { gltf } = (0, useNativeGLTF_1.useNativeGLTF)(accessory.url);
121
+ (0, react_1.useEffect)(() => {
122
+ if (!gltf?.scene)
123
+ return;
124
+ const bone = avatarScene.getObjectByName(accessory.bone ?? 'Head');
125
+ const target = bone ?? avatarScene;
126
+ const clone = gltf.scene.clone(true);
127
+ if (accessory.position)
128
+ clone.position.set(...accessory.position);
129
+ if (accessory.rotation)
130
+ clone.rotation.set(...accessory.rotation);
131
+ if (accessory.scale != null)
132
+ clone.scale.setScalar(accessory.scale);
133
+ target.add(clone);
134
+ return () => { target.remove(clone); };
135
+ }, [gltf, avatarScene, accessory]);
136
+ return null;
137
+ }
138
+ function AvatarScene({ uri, morphStateRef, motionStateRef, fov, accessories, animationRequest, onReady, onError, onAvatarState, onSceneReady, }) {
139
+ const { camera } = (0, fiber_1.useThree)();
140
+ const { gltf, error } = (0, useNativeGLTF_1.useNativeGLTF)(uri);
141
+ const { gltf: animationGltf, error: animationError } = (0, useNativeGLTF_1.useNativeGLTF)(animationRequest?.url ?? null);
142
+ const scene = gltf?.scene ?? null;
143
+ const placement = (0, react_1.useMemo)(() => (scene ? (0, scenePlacement_1.computeObjectPlacement)(scene, CAMERA_TARGET) : null), [scene]);
88
144
  const headMeshRef = (0, react_1.useRef)(null);
89
145
  const morphIndexRef = (0, react_1.useRef)(new Map());
90
146
  const readyFiredRef = (0, react_1.useRef)(false);
147
+ const animationMixerRef = (0, react_1.useRef)(null);
148
+ const animationStopTimerRef = (0, react_1.useRef)(null);
91
149
  (0, react_1.useEffect)(() => {
92
150
  if (!(camera instanceof THREE.PerspectiveCamera))
93
151
  return;
@@ -97,13 +155,25 @@ function AvatarScene({ uri, morphStateRef, fov, onReady, onError, onSceneReady }
97
155
  camera.updateProjectionMatrix();
98
156
  }, [camera, fov]);
99
157
  (0, react_1.useEffect)(() => {
100
- if (!gltf?.scene)
158
+ if (!error)
159
+ return;
160
+ onError(error.message);
161
+ }, [error, onError]);
162
+ (0, react_1.useEffect)(() => {
163
+ if (!animationError || !animationRequest)
164
+ return;
165
+ onAvatarState?.(`animation_error:${animationRequest.url}`);
166
+ onError(animationError.message);
167
+ }, [animationError, animationRequest, onAvatarState, onError]);
168
+ (0, react_1.useEffect)(() => {
169
+ if (!scene)
101
170
  return;
102
171
  let bestMesh = null;
103
172
  let bestCount = 0;
104
- gltf.scene.traverse((node) => {
173
+ scene.traverse((node) => {
105
174
  if (!(node instanceof THREE.Mesh))
106
175
  return;
176
+ node.frustumCulled = false;
107
177
  const count = Object.keys(node.morphTargetDictionary ?? {}).length;
108
178
  if (count > bestCount) {
109
179
  bestCount = count;
@@ -111,8 +181,16 @@ function AvatarScene({ uri, morphStateRef, fov, onReady, onError, onSceneReady }
111
181
  }
112
182
  });
113
183
  if (!bestMesh) {
114
- onError('No mesh with morph targets found in GLB');
115
- return;
184
+ // Fallback: no morph targets find any mesh and use amplitude-only jaw drive
185
+ scene.traverse((node) => {
186
+ if (node instanceof THREE.Mesh && !bestMesh)
187
+ bestMesh = node;
188
+ });
189
+ if (!bestMesh) {
190
+ onError('No mesh found in GLB');
191
+ return;
192
+ }
193
+ console.warn('[WgpuAvatar] No morph targets found — using amplitude-only mode');
116
194
  }
117
195
  headMeshRef.current = bestMesh;
118
196
  morphIndexRef.current = buildMorphIndex(bestMesh);
@@ -120,14 +198,64 @@ function AvatarScene({ uri, morphStateRef, fov, onReady, onError, onSceneReady }
120
198
  readyFiredRef.current = true;
121
199
  onReady();
122
200
  }
123
- }, [gltf, onReady, onError]);
124
- (0, native_1.useFrame)((_, delta) => {
201
+ }, [scene, onReady, onError]);
202
+ (0, react_1.useEffect)(() => {
203
+ if (animationStopTimerRef.current) {
204
+ clearTimeout(animationStopTimerRef.current);
205
+ animationStopTimerRef.current = null;
206
+ }
207
+ animationMixerRef.current?.stopAllAction();
208
+ animationMixerRef.current = null;
209
+ if (!scene || !animationRequest)
210
+ return;
211
+ const source = animationGltf ?? (animationRequest.url === uri ? gltf : null);
212
+ if (!source)
213
+ return;
214
+ const clip = source.animations[animationRequest.index] ?? source.animations[0];
215
+ if (!clip) {
216
+ onAvatarState?.(`animation_error:no_clip:${animationRequest.url}`);
217
+ return;
218
+ }
219
+ const mixer = new THREE.AnimationMixer(scene);
220
+ const action = mixer.clipAction(clip);
221
+ action.reset().play();
222
+ animationMixerRef.current = mixer;
223
+ onAvatarState?.(`animation:${animationRequest.url}`);
224
+ if (animationRequest.dur && animationRequest.dur > 0) {
225
+ animationStopTimerRef.current = setTimeout(() => {
226
+ mixer.stopAllAction();
227
+ if (animationMixerRef.current === mixer)
228
+ animationMixerRef.current = null;
229
+ onAvatarState?.('animation:stopped');
230
+ }, animationRequest.dur * 1000);
231
+ }
232
+ return () => {
233
+ if (animationStopTimerRef.current) {
234
+ clearTimeout(animationStopTimerRef.current);
235
+ animationStopTimerRef.current = null;
236
+ }
237
+ mixer.stopAllAction();
238
+ if (animationMixerRef.current === mixer)
239
+ animationMixerRef.current = null;
240
+ };
241
+ }, [animationGltf, animationRequest, gltf, onAvatarState, scene, uri]);
242
+ (0, fiber_1.useFrame)((_, delta) => {
125
243
  const mesh = headMeshRef.current;
126
244
  if (!mesh?.morphTargetInfluences)
127
245
  return;
128
246
  const state = morphStateRef.current;
129
- const alpha = Math.min(1, state.alpha * delta * 60);
130
- const combined = { ...state.moodBase };
247
+ // Frame-rate independent lerp: same convergence speed at 30 Hz and 120 Hz
248
+ const alpha = 1 - Math.exp(-state.alpha * delta * 60);
249
+ // Idle animation — subtle breathing / brow life at ~0.3 Hz
250
+ state.idleTime += delta;
251
+ const breathe = Math.sin(state.idleTime * 2 * Math.PI * 0.3) * 0.04;
252
+ const idleIdle = {
253
+ browRaiserL: Math.max(0, breathe),
254
+ browRaiserR: Math.max(0, breathe),
255
+ browRaiser_L: Math.max(0, breathe),
256
+ browRaiser_R: Math.max(0, breathe),
257
+ };
258
+ const combined = { ...state.moodBase, ...idleIdle };
131
259
  for (const [name, w] of Object.entries(state.visemeTarget)) {
132
260
  combined[name] = Math.max(combined[name] ?? 0, w);
133
261
  }
@@ -141,19 +269,61 @@ function AvatarScene({ uri, morphStateRef, fov, onReady, onError, onSceneReady }
141
269
  if (idx !== undefined)
142
270
  mesh.morphTargetInfluences[idx] = next;
143
271
  }
272
+ // Viseme decay: hold peak for 80 ms then decay (avoids eating fast cues)
273
+ const now = performance.now();
144
274
  for (const name of Object.keys(state.visemeTarget)) {
145
- state.visemeTarget[name] *= Math.pow(0.1, delta);
146
- if (state.visemeTarget[name] < 0.001)
147
- delete state.visemeTarget[name];
275
+ const cueAge = (now - (state.visemeCueTime[name] ?? now)) / 1000;
276
+ if (cueAge > 0.08) {
277
+ state.visemeTarget[name] *= Math.pow(0.05, delta);
278
+ if (state.visemeTarget[name] < 0.001) {
279
+ delete state.visemeTarget[name];
280
+ delete state.visemeCueTime[name];
281
+ }
282
+ }
283
+ }
284
+ // Procedural body motion — drives the skeleton bones (independent of the
285
+ // face morphs above). Bones are scanned lazily on first active motion.
286
+ const motion = motionStateRef.current;
287
+ if (motion.key) {
288
+ if (!motion.bones) {
289
+ const root = headMeshRef.current?.parent ?? mesh;
290
+ const skeletonRoot = root ? findSkeletonRoot(root) : null;
291
+ if (skeletonRoot) {
292
+ const scanned = (0, motionRuntime_1.scanMotionBones)((visit) => {
293
+ skeletonRoot.traverse((node) => {
294
+ if (node.isBone) {
295
+ visit(node);
296
+ }
297
+ });
298
+ });
299
+ motion.bones = scanned.bones;
300
+ motion.rest = scanned.rest;
301
+ }
302
+ else {
303
+ // No skeleton — nothing to drive. Clear so we don't rescan every frame.
304
+ motion.key = null;
305
+ }
306
+ }
307
+ if (motion.key && motion.bones && motion.rest) {
308
+ const tSec = (now - motion.startTime) / 1000;
309
+ (0, motionRuntime_1.applyMotionFrame)(motion.key, tSec, motion.bones, motion.rest);
310
+ }
148
311
  }
312
+ animationMixerRef.current?.update(delta);
149
313
  });
150
- return ((0, jsx_runtime_1.jsxs)(jsx_runtime_1.Fragment, { children: [onSceneReady && (0, jsx_runtime_1.jsx)(SceneCapture, { onSceneReady: onSceneReady }), (0, jsx_runtime_1.jsx)("ambientLight", { intensity: 0.6 }), (0, jsx_runtime_1.jsx)("directionalLight", { position: [1, 3, 2], intensity: 1.2, castShadow: false }), (0, jsx_runtime_1.jsx)("primitive", { object: gltf.scene })] }));
314
+ if (!scene) {
315
+ return ((0, jsx_runtime_1.jsxs)(jsx_runtime_1.Fragment, { children: [onSceneReady && (0, jsx_runtime_1.jsx)(SceneCapture, { onSceneReady: onSceneReady }), (0, jsx_runtime_1.jsx)("ambientLight", { intensity: 0.6 }), (0, jsx_runtime_1.jsx)("directionalLight", { position: [1, 3, 2], intensity: 1.2, castShadow: false })] }));
316
+ }
317
+ return ((0, jsx_runtime_1.jsxs)(jsx_runtime_1.Fragment, { children: [onSceneReady && (0, jsx_runtime_1.jsx)(SceneCapture, { onSceneReady: onSceneReady }), (0, jsx_runtime_1.jsx)("ambientLight", { intensity: 0.6 }), (0, jsx_runtime_1.jsx)("directionalLight", { position: [1, 3, 2], intensity: 1.2, castShadow: false }), (0, jsx_runtime_1.jsx)("group", { position: placement?.position ?? CAMERA_TARGET, scale: placement?.scale ?? 1, children: (0, jsx_runtime_1.jsx)("primitive", { object: scene }) }), accessories?.map((acc) => ((0, jsx_runtime_1.jsx)(AccessoryMesh, { accessory: acc, avatarScene: scene }, acc.id)))] }));
151
318
  }
152
319
  // ---------------------------------------------------------------------------
153
320
  // Error boundary
154
321
  // ---------------------------------------------------------------------------
155
322
  class GltfErrorBoundary extends react_1.default.Component {
156
- constructor(props) { super(props); this.state = { hasError: false }; }
323
+ constructor(props) {
324
+ super(props);
325
+ this.state = { hasError: false };
326
+ }
157
327
  static getDerivedStateFromError() { return { hasError: true }; }
158
328
  componentDidCatch(err) { this.props.onError(err.message); }
159
329
  render() { if (this.state.hasError)
@@ -167,43 +337,88 @@ function applyVisemeCue(morphState, visemeKey) {
167
337
  if (!aliases)
168
338
  return;
169
339
  const w = morphTables_1.VISEME_WEIGHTS[visemeKey] ?? morphTables_1.DEFAULT_VISEME_WEIGHT;
170
- for (const alias of aliases)
340
+ const now = performance.now();
341
+ for (const alias of aliases) {
171
342
  morphState.visemeTarget[alias] = w;
343
+ morphState.visemeCueTime[alias] = now;
344
+ }
172
345
  }
173
346
  // ---------------------------------------------------------------------------
174
347
  // Main exported component
175
348
  // ---------------------------------------------------------------------------
176
- exports.WgpuAvatar = (0, react_1.forwardRef)(({ style, avatarUrl, focalLength, mood = 'neutral', onReady, onError, onSceneReady }, ref) => {
177
- const remoteUrl = avatarUrl && (avatarUrl.startsWith('http://') || avatarUrl.startsWith('https://'))
178
- ? avatarUrl : null;
349
+ exports.WgpuAvatar = (0, react_1.forwardRef)(({ style, avatarUrl, focalLength, mood = 'neutral', accessories, onReady, onError, onAvatarState, onSceneReady }, ref) => {
350
+ const isRemote = avatarUrl != null && (avatarUrl.startsWith('http://') || avatarUrl.startsWith('https://'));
351
+ const remoteUrl = (0, react_1.useMemo)(() => isRemote ? avatarUrl : null,
352
+ // eslint-disable-next-line react-hooks/exhaustive-deps
353
+ [avatarUrl]);
179
354
  const fileResult = (0, useAuthedModelUri_1.useAuthedModelUri)(remoteUrl);
180
- const lockedUriRef = (0, react_1.useRef)(null);
181
- const lastUrlRef = (0, react_1.useRef)(avatarUrl);
182
- if (lastUrlRef.current !== avatarUrl) {
183
- lastUrlRef.current = avatarUrl;
184
- lockedUriRef.current = null;
355
+ // stableUri: once a valid URI is resolved, keep showing it until a new one
356
+ // is ready prevents Canvas unmount/remount flicker on transient url changes.
357
+ const stableUriRef = (0, react_1.useRef)(null);
358
+ const lastAvatarUrlRef = (0, react_1.useRef)(avatarUrl);
359
+ if (lastAvatarUrlRef.current !== avatarUrl) {
360
+ // avatarUrl changed — invalidate stable URI so next resolved URI takes over
361
+ lastAvatarUrlRef.current = avatarUrl;
362
+ stableUriRef.current = null;
185
363
  }
186
- const currentUri = fileResult?.uri ?? null;
187
- if (lockedUriRef.current === null && currentUri)
188
- lockedUriRef.current = currentUri;
189
- const localUri = lockedUriRef.current;
364
+ // For non-remote URLs (file://, asset://, data:) use directly without download
365
+ const currentUri = isRemote ? (fileResult?.uri ?? null) : (avatarUrl ?? null);
366
+ if (currentUri && currentUri !== stableUriRef.current) {
367
+ stableUriRef.current = currentUri;
368
+ }
369
+ const localUri = stableUriRef.current;
190
370
  const fov = focalLength
191
371
  ? Math.round(2 * Math.atan(21.634 / focalLength) * (180 / Math.PI))
192
372
  : CAMERA_FOV_FULL;
373
+ const cameraProps = (0, react_1.useMemo)(() => ({ fov, position: CAMERA_POSITION, near: 0.01, far: 100 }), [fov]);
193
374
  const morphStateRef = (0, react_1.useRef)({
194
- current: {}, visemeTarget: {}, moodBase: {}, alpha: 0.18,
375
+ current: {}, visemeTarget: {}, visemeCueTime: {}, moodBase: {}, alpha: 0.18, idleTime: 0,
195
376
  });
377
+ const motionStateRef = (0, react_1.useRef)((0, motionState_1.createMotionState)());
378
+ const [animationRequest, setAnimationRequest] = (0, react_1.useState)(null);
379
+ const animationNonceRef = (0, react_1.useRef)(0);
380
+ const onAvatarStateRef = (0, react_1.useRef)(onAvatarState);
381
+ onAvatarStateRef.current = onAvatarState;
196
382
  (0, react_1.useEffect)(() => {
197
383
  morphStateRef.current.moodBase = { ...(morphTables_1.MOOD_MORPHS[mood] ?? {}) };
198
384
  }, [mood]);
199
385
  const scheduleRef = (0, react_1.useRef)(null);
200
386
  const scheduleTimersRef = (0, react_1.useRef)([]);
387
+ const blinkTimersRef = (0, react_1.useRef)([]);
201
388
  const [isReady, setIsReady] = (0, react_1.useState)(false);
202
389
  const clearScheduleTimers = (0, react_1.useCallback)(() => {
203
390
  for (const t of scheduleTimersRef.current)
204
391
  clearTimeout(t);
205
392
  scheduleTimersRef.current = [];
206
393
  }, []);
394
+ (0, react_1.useEffect)(() => {
395
+ (0, motionState_1.resetMotionStateForAvatarSwap)(motionStateRef.current);
396
+ }, [localUri]);
397
+ const stopMotion = (0, react_1.useCallback)(() => {
398
+ (0, motionState_1.stopMotionState)(motionStateRef.current);
399
+ }, []);
400
+ const playMotionKey = (0, react_1.useCallback)((name, autoReturnOverrideMs) => {
401
+ if (!(0, motion_1.isMotionKey)(name)) {
402
+ onAvatarStateRef.current?.(`motion_error:unknown:${name}`);
403
+ return false;
404
+ }
405
+ const motion = motionStateRef.current;
406
+ if (motion.autoReturnTimer) {
407
+ clearTimeout(motion.autoReturnTimer);
408
+ motion.autoReturnTimer = null;
409
+ }
410
+ motion.key = name;
411
+ motion.startTime = performance.now();
412
+ onAvatarStateRef.current?.(`motion:${name}`);
413
+ const autoReturnMs = autoReturnOverrideMs ?? motion_1.MOTION_DEFS[name].autoReturnMs;
414
+ if (autoReturnMs && autoReturnMs > 0) {
415
+ motion.autoReturnTimer = setTimeout(() => {
416
+ if (motionStateRef.current.key === name)
417
+ stopMotion();
418
+ }, autoReturnMs);
419
+ }
420
+ return true;
421
+ }, [stopMotion]);
207
422
  const applySchedule = (0, react_1.useCallback)((schedule) => {
208
423
  clearScheduleTimers();
209
424
  const now = Date.now();
@@ -235,6 +450,12 @@ exports.WgpuAvatar = (0, react_1.forwardRef)(({ style, avatarUrl, focalLength, m
235
450
  onError?.(msg);
236
451
  }, [onError]);
237
452
  (0, react_1.useEffect)(() => () => clearScheduleTimers(), [clearScheduleTimers]);
453
+ (0, react_1.useEffect)(() => () => { for (const t of blinkTimersRef.current)
454
+ clearTimeout(t); }, []);
455
+ (0, react_1.useEffect)(() => () => (0, motionState_1.resetMotionStateForAvatarSwap)(motionStateRef.current), []);
456
+ const handleCanvasCreated = (0, react_1.useCallback)((state) => {
457
+ state.scene.background = new THREE.Color('#1a1a2e');
458
+ }, []);
238
459
  (0, react_1.useImperativeHandle)(ref, () => ({
239
460
  setMood: (m) => { morphStateRef.current.moodBase = { ...(morphTables_1.MOOD_MORPHS[m] ?? {}) }; },
240
461
  sendAmplitude: () => { },
@@ -243,10 +464,55 @@ exports.WgpuAvatar = (0, react_1.forwardRef)(({ style, avatarUrl, focalLength, m
243
464
  if (!aliases)
244
465
  return;
245
466
  const w = weight ?? morphTables_1.VISEME_WEIGHTS[viseme] ?? morphTables_1.DEFAULT_VISEME_WEIGHT;
246
- for (const alias of aliases)
467
+ const now = performance.now();
468
+ for (const alias of aliases) {
247
469
  morphStateRef.current.visemeTarget[alias] = w;
470
+ morphStateRef.current.visemeCueTime[alias] = now;
471
+ }
472
+ },
473
+ clearVisemes: () => { clearScheduleTimers(); morphStateRef.current.visemeTarget = {}; morphStateRef.current.visemeCueTime = {}; },
474
+ playMotion: (name) => {
475
+ playMotionKey(name);
476
+ },
477
+ stopMotion,
478
+ playGesture: (name, opts) => {
479
+ playMotionKey(name, opts?.ms ?? 1000);
480
+ },
481
+ stopGesture: stopMotion,
482
+ playPose: (name) => {
483
+ playMotionKey(name);
484
+ },
485
+ stopPose: stopMotion,
486
+ playAnimation: (url, opts) => {
487
+ animationNonceRef.current += 1;
488
+ setAnimationRequest({
489
+ url,
490
+ index: opts?.index ?? 0,
491
+ dur: opts?.dur,
492
+ nonce: animationNonceRef.current,
493
+ });
494
+ },
495
+ stopAnimation: () => {
496
+ setAnimationRequest(null);
497
+ onAvatarStateRef.current?.('animation:stopped');
498
+ },
499
+ triggerBlink: () => {
500
+ // Blink: close eyes over 80ms, hold 40ms, open over 80ms
501
+ const BLINK_MORPHS = [
502
+ 'eyeBlinkLeft', 'eyeBlink_L', 'eyesClosed',
503
+ 'eyeBlinkRight', 'eyeBlink_R',
504
+ ];
505
+ const now = performance.now();
506
+ const set = (w) => {
507
+ for (const name of BLINK_MORPHS) {
508
+ morphStateRef.current.visemeTarget[name] = w;
509
+ morphStateRef.current.visemeCueTime[name] = now;
510
+ }
511
+ };
512
+ set(1);
513
+ const t1 = setTimeout(() => set(0), 120);
514
+ blinkTimersRef.current.push(t1);
248
515
  },
249
- clearVisemes: () => { clearScheduleTimers(); morphStateRef.current.visemeTarget = {}; },
250
516
  scheduleVisemes: (schedule) => {
251
517
  if (!isReady) {
252
518
  scheduleRef.current = schedule;
@@ -254,13 +520,14 @@ exports.WgpuAvatar = (0, react_1.forwardRef)(({ style, avatarUrl, focalLength, m
254
520
  }
255
521
  applySchedule(schedule);
256
522
  },
257
- }), [isReady, applySchedule, clearScheduleTimers]);
523
+ }), [isReady, applySchedule, clearScheduleTimers, playMotionKey, stopMotion]);
258
524
  if (!localUri)
259
525
  return (0, jsx_runtime_1.jsx)(react_native_1.View, { style: [styles.placeholder, style] });
260
- return ((0, jsx_runtime_1.jsx)(react_native_1.View, { style: [styles.container, style], children: (0, jsx_runtime_1.jsx)(native_1.Canvas, { style: react_native_1.StyleSheet.absoluteFill, camera: { fov, position: CAMERA_POSITION, near: 0.01, far: 100 }, gl: { antialias: true, alpha: false }, onCreated: ({ gl }) => { gl.setClearColor(new THREE.Color('#1a1a2e')); }, children: (0, jsx_runtime_1.jsx)(GltfErrorBoundary, { onError: handleError, children: (0, jsx_runtime_1.jsx)(AvatarScene, { uri: localUri, morphStateRef: morphStateRef, fov: fov, onReady: handleReady, onError: handleError, onSceneReady: onSceneReady }) }) }) }));
526
+ return ((0, jsx_runtime_1.jsx)(react_native_1.View, { style: [styles.container, style], children: (0, jsx_runtime_1.jsx)(R3FWebGpuCanvas_1.R3FWebGpuCanvas, { style: styles.canvas, camera: cameraProps, clearColor: "#1a1a2e", onCreated: handleCanvasCreated, onError: handleError, children: (0, jsx_runtime_1.jsx)(GltfErrorBoundary, { onError: handleError, children: (0, jsx_runtime_1.jsx)(AvatarScene, { uri: localUri, morphStateRef: morphStateRef, motionStateRef: motionStateRef, fov: fov, accessories: accessories, animationRequest: animationRequest, onReady: handleReady, onError: handleError, onAvatarState: onAvatarState, onSceneReady: onSceneReady }) }) }) }));
261
527
  });
262
528
  exports.WgpuAvatar.displayName = 'WgpuAvatar';
263
529
  const styles = react_native_1.StyleSheet.create({
264
530
  container: { overflow: 'hidden', backgroundColor: '#1a1a2e' },
531
+ canvas: { flex: 1 },
265
532
  placeholder: { backgroundColor: '#1a1a2e' },
266
533
  });
@@ -0,0 +1,12 @@
1
+ /**
2
+ * Canonical snap-to-bone defaults per accessory category.
3
+ * Applied immediately when the user equips an item that has no saved placement,
4
+ * so the "good enough" position is free on tap for ~90% of use-cases.
5
+ *
6
+ * Offsets are in the bone's local space (metres).
7
+ * Rotation is Euler XYZ in radians.
8
+ */
9
+ import type { AssetPlacement } from '../wardrobe/wardrobeStore';
10
+ export declare const BONE_SNAP_DEFAULTS: Record<string, AssetPlacement>;
11
+ /** Returns the snap default for a category, or null if the category is unknown. */
12
+ export declare function snapDefaultForCategory(category: string): AssetPlacement | null;
@@ -0,0 +1,19 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.BONE_SNAP_DEFAULTS = void 0;
4
+ exports.snapDefaultForCategory = snapDefaultForCategory;
5
+ exports.BONE_SNAP_DEFAULTS = {
6
+ hat: { bone: 'Head', position: [0, 0.12, 0.00], rotation: [0, 0, 0], scale: 1.0 },
7
+ cap: { bone: 'Head', position: [0, 0.10, 0.02], rotation: [0, 0, 0], scale: 1.0 },
8
+ glasses: { bone: 'Head', position: [0, 0.02, 0.07], rotation: [0, 0, 0], scale: 1.0 },
9
+ sunglasses: { bone: 'Head', position: [0, 0.02, 0.07], rotation: [0, 0, 0], scale: 1.0 },
10
+ earring: { bone: 'Head', position: [0.085, 0.01, 0.00], rotation: [0, 0, 0], scale: 0.8 },
11
+ earrings: { bone: 'Head', position: [0.085, 0.01, 0.00], rotation: [0, 0, 0], scale: 0.8 },
12
+ necklace: { bone: 'Neck', position: [0, -0.04, 0.03], rotation: [0, 0, 0], scale: 1.0 },
13
+ mask: { bone: 'Head', position: [0, 0.00, 0.08], rotation: [0, 0, 0], scale: 1.0 },
14
+ };
15
+ /** Returns the snap default for a category, or null if the category is unknown. */
16
+ function snapDefaultForCategory(category) {
17
+ const key = category.toLowerCase().trim();
18
+ return exports.BONE_SNAP_DEFAULTS[key] ?? null;
19
+ }
@@ -0,0 +1,2 @@
1
+ /** Exported so patchThreeForRN can use the same store when re-patching URL.createObjectURL. */
2
+ export declare const blobBytesStore: WeakMap<object, Uint8Array<ArrayBufferLike>>;