talking-head-studio 0.4.10 → 0.4.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +299 -337
- package/dist/TalkingHead.d.ts +44 -28
- package/dist/TalkingHead.js +21 -2
- package/dist/TalkingHead.web.d.ts +37 -4
- package/dist/TalkingHead.web.js +28 -8
- package/dist/TalkingHeadVisualization.d.ts +22 -0
- package/dist/TalkingHeadVisualization.js +30 -10
- package/dist/api/studioApi.d.ts +12 -1
- package/dist/api/studioApi.js +41 -28
- package/dist/appearance/apply.js +2 -3
- package/dist/appearance/matchers.js +1 -2
- package/dist/appearance/schema.js +1 -2
- package/dist/contract.d.ts +14 -0
- package/dist/contract.js +30 -0
- package/dist/core/avatar/avatarCapabilities.d.ts +60 -0
- package/dist/core/avatar/avatarCapabilities.js +100 -0
- package/dist/core/avatar/backend.d.ts +130 -0
- package/dist/core/avatar/backend.js +4 -0
- package/dist/core/avatar/backends/gaussian.d.ts +49 -0
- package/dist/core/avatar/backends/gaussian.js +293 -0
- package/dist/core/avatar/backends/index.d.ts +3 -0
- package/dist/core/avatar/backends/index.js +7 -0
- package/dist/core/avatar/backends/morphTarget.d.ts +39 -0
- package/dist/core/avatar/backends/morphTarget.js +179 -0
- package/dist/core/avatar/faceControls.d.ts +40 -0
- package/dist/core/avatar/faceControls.js +138 -0
- package/dist/core/avatar/motion.d.ts +1713 -0
- package/dist/core/avatar/motion.js +550 -0
- package/dist/core/avatar/motionRuntime.d.ts +46 -0
- package/dist/core/avatar/motionRuntime.js +84 -0
- package/dist/core/avatar/schema.d.ts +78 -0
- package/dist/core/avatar/schema.js +134 -0
- package/dist/core/avatar/visemes.d.ts +47 -1
- package/dist/core/avatar/visemes.js +114 -1
- package/dist/editor/AvatarCanvas.js +93 -3
- package/dist/editor/AvatarEditor.native.js +19 -9
- package/dist/editor/AvatarModel.js +2 -2
- package/dist/editor/FaceSqueezeEditor.d.ts +3 -1
- package/dist/editor/FaceSqueezeEditor.js +195 -121
- package/dist/editor/FaceSqueezeEditor.web.d.ts +3 -1
- package/dist/editor/FaceSqueezeEditor.web.js +32 -30
- package/dist/editor/RigidAccessory.js +18 -4
- package/dist/editor/SkinnedClothing.js +19 -9
- package/dist/editor/boneLockedDrag.d.ts +11 -0
- package/dist/editor/boneLockedDrag.js +68 -0
- package/dist/editor/boneSnap.js +22 -12
- package/dist/editor/boneSnap.web.d.ts +27 -0
- package/dist/editor/boneSnap.web.js +99 -0
- package/dist/editor/index.web.d.ts +10 -0
- package/dist/editor/index.web.js +26 -0
- package/dist/editor/sounds/haha.wav +0 -0
- package/dist/editor/sounds/owie.wav +0 -0
- package/dist/editor/sounds/stop.wav +0 -0
- package/dist/editor/studioTheme.d.ts +14 -14
- package/dist/editor/studioTheme.js +19 -16
- package/dist/editor/types.d.ts +1 -0
- package/dist/html/accessories.d.ts +7 -0
- package/dist/html/accessories.js +149 -0
- package/dist/html/motion.d.ts +1 -0
- package/dist/html/motion.js +189 -0
- package/dist/html/visemes.d.ts +7 -0
- package/dist/html/visemes.js +348 -0
- package/dist/html.d.ts +1 -1
- package/dist/html.js +56 -734
- package/dist/index.d.ts +19 -1
- package/dist/index.js +44 -5
- package/dist/index.web.d.ts +18 -1
- package/dist/index.web.js +36 -3
- package/dist/platform/api/types.d.ts +10 -0
- package/dist/platform/api/types.js +2 -0
- package/dist/platform/marketplace/types.d.ts +32 -0
- package/dist/platform/marketplace/types.js +2 -0
- package/dist/platform/sdk/unity.d.ts +27 -0
- package/dist/platform/sdk/unity.js +2 -0
- package/dist/platform/sdk/unreal.d.ts +23 -0
- package/dist/platform/sdk/unreal.js +2 -0
- package/dist/platform/sdk/web.d.ts +16 -0
- package/dist/platform/sdk/web.js +2 -0
- package/dist/sketchfab/api.js +5 -5
- package/dist/sketchfab/glbInspect.d.ts +22 -0
- package/dist/sketchfab/glbInspect.js +58 -0
- package/dist/sketchfab/index.d.ts +3 -0
- package/dist/sketchfab/index.js +8 -1
- package/dist/sketchfab/inspectRemote.d.ts +13 -0
- package/dist/sketchfab/inspectRemote.js +77 -0
- package/dist/sketchfab/types.d.ts +10 -0
- package/dist/sketchfab/useSketchfabSearch.js +1 -2
- package/dist/studio/AccessoryBrowserScreen.d.ts +6 -0
- package/dist/studio/AccessoryBrowserScreen.js +626 -0
- package/dist/studio/AccessoryPanel.d.ts +10 -0
- package/dist/studio/AccessoryPanel.js +396 -0
- package/dist/studio/AppearancePanel.d.ts +9 -0
- package/dist/studio/AppearancePanel.js +77 -0
- package/dist/studio/AvatarCreatorScreen.d.ts +5 -0
- package/dist/studio/AvatarCreatorScreen.js +806 -0
- package/dist/studio/AvatarEditorScreen.d.ts +14 -0
- package/dist/studio/AvatarEditorScreen.js +510 -0
- package/dist/studio/AvatarGrid.d.ts +23 -0
- package/dist/studio/AvatarGrid.js +257 -0
- package/dist/studio/ColorSwatch.d.ts +8 -0
- package/dist/studio/ColorSwatch.js +100 -0
- package/dist/studio/CreateVoiceProfileSheet.d.ts +8 -0
- package/dist/studio/CreateVoiceProfileSheet.js +242 -0
- package/dist/studio/DetailsPanel.d.ts +15 -0
- package/dist/studio/DetailsPanel.js +239 -0
- package/dist/studio/FilamentEditor.d.ts +2 -0
- package/dist/studio/FilamentEditor.js +6 -0
- package/dist/studio/PrecisionPanel.d.ts +2 -0
- package/dist/studio/PrecisionPanel.js +7 -0
- package/dist/studio/PublicGalleryScreen.d.ts +5 -0
- package/dist/studio/PublicGalleryScreen.js +358 -0
- package/dist/studio/SketchfabModelCard.d.ts +20 -0
- package/dist/studio/SketchfabModelCard.js +104 -0
- package/dist/studio/StudioBrowseHeader.d.ts +9 -0
- package/dist/studio/StudioBrowseHeader.js +28 -0
- package/dist/studio/StudioEmptyState.d.ts +8 -0
- package/dist/studio/StudioEmptyState.js +29 -0
- package/dist/studio/StudioFloatingAction.d.ts +13 -0
- package/dist/studio/StudioFloatingAction.js +42 -0
- package/dist/studio/StudioSectionHeader.d.ts +7 -0
- package/dist/studio/StudioSectionHeader.js +27 -0
- package/dist/studio/StudioSurfaceCard.d.ts +8 -0
- package/dist/studio/StudioSurfaceCard.js +20 -0
- package/dist/studio/VoicePanel.d.ts +15 -0
- package/dist/studio/VoicePanel.js +305 -0
- package/dist/studio/constants.d.ts +3 -0
- package/dist/studio/constants.js +6 -0
- package/dist/studio/index.d.ts +29 -0
- package/dist/studio/index.js +54 -0
- package/dist/studio/useSketchfabCapabilities.d.ts +31 -0
- package/dist/studio/useSketchfabCapabilities.js +82 -0
- package/dist/tts/useDirectVisemeStream.d.ts +2 -6
- package/dist/tts/useDirectVisemeStream.js +16 -12
- package/dist/tts/useMotionMarkers.d.ts +0 -1
- package/dist/tts/useMotionMarkers.js +1 -2
- package/dist/utils/avatarUtils.js +94 -8
- package/dist/utils/faceLandmarkerToShapeWeights.js +21 -14
- package/dist/voice/convertToWav.js +1 -2
- package/dist/voice/index.d.ts +3 -0
- package/dist/voice/index.js +6 -1
- package/dist/voice/useAudioPlayer.js +18 -6
- package/dist/voice/useAudioRecording.js +1 -2
- package/dist/voice/useFaceControls.d.ts +14 -0
- package/dist/voice/useFaceControls.js +81 -0
- package/dist/voice/useVoicePreview.d.ts +7 -0
- package/dist/voice/useVoicePreview.js +83 -0
- package/dist/wardrobe/index.d.ts +3 -0
- package/dist/wardrobe/index.js +8 -1
- package/dist/wardrobe/useAccessoryGestures.d.ts +20 -0
- package/dist/wardrobe/useAccessoryGestures.js +94 -0
- package/dist/wardrobe/useAvatarWardrobeHydration.js +9 -4
- package/dist/wardrobe/useStudioAvatar.d.ts +29 -0
- package/dist/wardrobe/useStudioAvatar.js +186 -0
- package/dist/wardrobe/wardrobeStore.d.ts +2 -0
- package/dist/wardrobe/wardrobeStore.js +12 -2
- package/dist/wgpu/R3FWebGpuCanvas.d.ts +15 -0
- package/dist/wgpu/R3FWebGpuCanvas.js +176 -0
- package/dist/wgpu/WgpuAvatar.d.ts +26 -2
- package/dist/wgpu/WgpuAvatar.js +313 -46
- package/dist/wgpu/accessoryDefaults.d.ts +12 -0
- package/dist/wgpu/accessoryDefaults.js +19 -0
- package/dist/wgpu/blobShim.d.ts +2 -0
- package/dist/wgpu/blobShim.js +191 -0
- package/dist/wgpu/index.d.ts +1 -0
- package/dist/wgpu/index.js +4 -1
- package/dist/wgpu/loadGLTFFromUri.d.ts +2 -0
- package/dist/wgpu/loadGLTFFromUri.js +75 -0
- package/dist/wgpu/morphTables.js +21 -10
- package/dist/wgpu/motionState.d.ts +20 -0
- package/dist/wgpu/motionState.js +31 -0
- package/dist/wgpu/patchThreeForRN.d.ts +28 -0
- package/dist/wgpu/patchThreeForRN.js +292 -0
- package/dist/wgpu/scenePlacement.d.ts +5 -0
- package/dist/wgpu/scenePlacement.js +50 -0
- package/dist/wgpu/useAuthedModelUri.js +22 -11
- package/dist/wgpu/useNativeGLTF.d.ts +7 -0
- package/dist/wgpu/useNativeGLTF.js +36 -0
- package/package.json +102 -32
package/dist/wgpu/WgpuAvatar.js
CHANGED
|
@@ -15,36 +15,60 @@ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (
|
|
|
15
15
|
}) : function(o, v) {
|
|
16
16
|
o["default"] = v;
|
|
17
17
|
});
|
|
18
|
-
var __importStar = (this && this.__importStar) || function (
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
};
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
25
35
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
26
36
|
exports.WgpuAvatar = void 0;
|
|
27
37
|
const jsx_runtime_1 = require("react/jsx-runtime");
|
|
28
38
|
/**
|
|
29
39
|
* WgpuAvatar — drop-in replacement for FilamentAvatar using react-three-fiber
|
|
30
|
-
* + react-native-wgpu
|
|
40
|
+
* + react-native-wgpu. Exposes the same ref interface as
|
|
31
41
|
* FilamentAvatarRef so siteclaw can swap renderers without changing call sites.
|
|
32
42
|
*
|
|
33
43
|
* No SurfaceTexture, no choreographer, no JNI surface lifecycle bugs.
|
|
34
44
|
*
|
|
35
45
|
* Peer deps required by the host app:
|
|
36
|
-
* react-native-wgpu
|
|
46
|
+
* react-native-wgpu
|
|
37
47
|
* @react-three/fiber >= 8
|
|
38
48
|
* @react-three/drei >= 9
|
|
39
49
|
* three >= 0.170
|
|
40
50
|
*/
|
|
51
|
+
// IMPORTANT: blobShim MUST be the very first import — before any three/R3F
|
|
52
|
+
// imports — so the Blob+createObjectURL patch is in place before R3F's
|
|
53
|
+
// polyfills() smoke-tests createObjectURL at module evaluation time.
|
|
54
|
+
require("./blobShim");
|
|
41
55
|
const react_1 = __importStar(require("react"));
|
|
42
56
|
const react_native_1 = require("react-native");
|
|
43
57
|
const THREE = __importStar(require("three"));
|
|
44
|
-
const
|
|
45
|
-
const native_2 = require("@react-three/drei/native");
|
|
58
|
+
const fiber_1 = require("@react-three/fiber");
|
|
46
59
|
const morphTables_1 = require("./morphTables");
|
|
47
60
|
const useAuthedModelUri_1 = require("./useAuthedModelUri");
|
|
61
|
+
const patchThreeForRN_1 = require("./patchThreeForRN");
|
|
62
|
+
const R3FWebGpuCanvas_1 = require("./R3FWebGpuCanvas");
|
|
63
|
+
const useNativeGLTF_1 = require("./useNativeGLTF");
|
|
64
|
+
const scenePlacement_1 = require("./scenePlacement");
|
|
65
|
+
const motion_1 = require("../core/avatar/motion");
|
|
66
|
+
const motionRuntime_1 = require("../core/avatar/motionRuntime");
|
|
67
|
+
const motionState_1 = require("./motionState");
|
|
68
|
+
// Patch Three.js loaders before any GLTF/texture loading happens.
|
|
69
|
+
// This fixes "Cannot create URL for blob!" on React Native — URL.createObjectURL
|
|
70
|
+
// doesn't exist in Hermes/JSC, but GLTFLoader calls it for embedded textures.
|
|
71
|
+
(0, patchThreeForRN_1.installThreeRNPatches)(false);
|
|
48
72
|
// ---------------------------------------------------------------------------
|
|
49
73
|
// Camera defaults — match FilamentAvatar constants
|
|
50
74
|
// ---------------------------------------------------------------------------
|
|
@@ -55,7 +79,7 @@ const CAMERA_FOV_FULL = 38;
|
|
|
55
79
|
// Scene/camera ref capture — runs inside Canvas so it can access R3F context
|
|
56
80
|
// ---------------------------------------------------------------------------
|
|
57
81
|
function SceneCapture({ onSceneReady }) {
|
|
58
|
-
const { scene, camera } = (0,
|
|
82
|
+
const { scene, camera } = (0, fiber_1.useThree)();
|
|
59
83
|
(0, react_1.useEffect)(() => {
|
|
60
84
|
onSceneReady(scene, camera);
|
|
61
85
|
// Only fire once on mount — scene/camera identity is stable
|
|
@@ -63,6 +87,13 @@ function SceneCapture({ onSceneReady }) {
|
|
|
63
87
|
}, []);
|
|
64
88
|
return null;
|
|
65
89
|
}
|
|
90
|
+
/** Walks up to the topmost ancestor so a bone scan covers the whole skeleton. */
|
|
91
|
+
function findSkeletonRoot(node) {
|
|
92
|
+
let root = node;
|
|
93
|
+
while (root.parent)
|
|
94
|
+
root = root.parent;
|
|
95
|
+
return root;
|
|
96
|
+
}
|
|
66
97
|
function buildMorphIndex(mesh) {
|
|
67
98
|
const map = new Map();
|
|
68
99
|
const dict = mesh.morphTargetDictionary;
|
|
@@ -82,12 +113,39 @@ function resolveIndex(morphIndex, aliases) {
|
|
|
82
113
|
}
|
|
83
114
|
return undefined;
|
|
84
115
|
}
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
116
|
+
// ---------------------------------------------------------------------------
|
|
117
|
+
// Accessory mesh — loads a GLB and parents it to a named bone on the avatar
|
|
118
|
+
// ---------------------------------------------------------------------------
|
|
119
|
+
function AccessoryMesh({ accessory, avatarScene, }) {
|
|
120
|
+
const { gltf } = (0, useNativeGLTF_1.useNativeGLTF)(accessory.url);
|
|
121
|
+
(0, react_1.useEffect)(() => {
|
|
122
|
+
if (!gltf?.scene)
|
|
123
|
+
return;
|
|
124
|
+
const bone = avatarScene.getObjectByName(accessory.bone ?? 'Head');
|
|
125
|
+
const target = bone ?? avatarScene;
|
|
126
|
+
const clone = gltf.scene.clone(true);
|
|
127
|
+
if (accessory.position)
|
|
128
|
+
clone.position.set(...accessory.position);
|
|
129
|
+
if (accessory.rotation)
|
|
130
|
+
clone.rotation.set(...accessory.rotation);
|
|
131
|
+
if (accessory.scale != null)
|
|
132
|
+
clone.scale.setScalar(accessory.scale);
|
|
133
|
+
target.add(clone);
|
|
134
|
+
return () => { target.remove(clone); };
|
|
135
|
+
}, [gltf, avatarScene, accessory]);
|
|
136
|
+
return null;
|
|
137
|
+
}
|
|
138
|
+
function AvatarScene({ uri, morphStateRef, motionStateRef, fov, accessories, animationRequest, onReady, onError, onAvatarState, onSceneReady, }) {
|
|
139
|
+
const { camera } = (0, fiber_1.useThree)();
|
|
140
|
+
const { gltf, error } = (0, useNativeGLTF_1.useNativeGLTF)(uri);
|
|
141
|
+
const { gltf: animationGltf, error: animationError } = (0, useNativeGLTF_1.useNativeGLTF)(animationRequest?.url ?? null);
|
|
142
|
+
const scene = gltf?.scene ?? null;
|
|
143
|
+
const placement = (0, react_1.useMemo)(() => (scene ? (0, scenePlacement_1.computeObjectPlacement)(scene, CAMERA_TARGET) : null), [scene]);
|
|
88
144
|
const headMeshRef = (0, react_1.useRef)(null);
|
|
89
145
|
const morphIndexRef = (0, react_1.useRef)(new Map());
|
|
90
146
|
const readyFiredRef = (0, react_1.useRef)(false);
|
|
147
|
+
const animationMixerRef = (0, react_1.useRef)(null);
|
|
148
|
+
const animationStopTimerRef = (0, react_1.useRef)(null);
|
|
91
149
|
(0, react_1.useEffect)(() => {
|
|
92
150
|
if (!(camera instanceof THREE.PerspectiveCamera))
|
|
93
151
|
return;
|
|
@@ -97,13 +155,25 @@ function AvatarScene({ uri, morphStateRef, fov, onReady, onError, onSceneReady }
|
|
|
97
155
|
camera.updateProjectionMatrix();
|
|
98
156
|
}, [camera, fov]);
|
|
99
157
|
(0, react_1.useEffect)(() => {
|
|
100
|
-
if (!
|
|
158
|
+
if (!error)
|
|
159
|
+
return;
|
|
160
|
+
onError(error.message);
|
|
161
|
+
}, [error, onError]);
|
|
162
|
+
(0, react_1.useEffect)(() => {
|
|
163
|
+
if (!animationError || !animationRequest)
|
|
164
|
+
return;
|
|
165
|
+
onAvatarState?.(`animation_error:${animationRequest.url}`);
|
|
166
|
+
onError(animationError.message);
|
|
167
|
+
}, [animationError, animationRequest, onAvatarState, onError]);
|
|
168
|
+
(0, react_1.useEffect)(() => {
|
|
169
|
+
if (!scene)
|
|
101
170
|
return;
|
|
102
171
|
let bestMesh = null;
|
|
103
172
|
let bestCount = 0;
|
|
104
|
-
|
|
173
|
+
scene.traverse((node) => {
|
|
105
174
|
if (!(node instanceof THREE.Mesh))
|
|
106
175
|
return;
|
|
176
|
+
node.frustumCulled = false;
|
|
107
177
|
const count = Object.keys(node.morphTargetDictionary ?? {}).length;
|
|
108
178
|
if (count > bestCount) {
|
|
109
179
|
bestCount = count;
|
|
@@ -111,8 +181,16 @@ function AvatarScene({ uri, morphStateRef, fov, onReady, onError, onSceneReady }
|
|
|
111
181
|
}
|
|
112
182
|
});
|
|
113
183
|
if (!bestMesh) {
|
|
114
|
-
|
|
115
|
-
|
|
184
|
+
// Fallback: no morph targets — find any mesh and use amplitude-only jaw drive
|
|
185
|
+
scene.traverse((node) => {
|
|
186
|
+
if (node instanceof THREE.Mesh && !bestMesh)
|
|
187
|
+
bestMesh = node;
|
|
188
|
+
});
|
|
189
|
+
if (!bestMesh) {
|
|
190
|
+
onError('No mesh found in GLB');
|
|
191
|
+
return;
|
|
192
|
+
}
|
|
193
|
+
console.warn('[WgpuAvatar] No morph targets found — using amplitude-only mode');
|
|
116
194
|
}
|
|
117
195
|
headMeshRef.current = bestMesh;
|
|
118
196
|
morphIndexRef.current = buildMorphIndex(bestMesh);
|
|
@@ -120,14 +198,64 @@ function AvatarScene({ uri, morphStateRef, fov, onReady, onError, onSceneReady }
|
|
|
120
198
|
readyFiredRef.current = true;
|
|
121
199
|
onReady();
|
|
122
200
|
}
|
|
123
|
-
}, [
|
|
124
|
-
(0,
|
|
201
|
+
}, [scene, onReady, onError]);
|
|
202
|
+
(0, react_1.useEffect)(() => {
|
|
203
|
+
if (animationStopTimerRef.current) {
|
|
204
|
+
clearTimeout(animationStopTimerRef.current);
|
|
205
|
+
animationStopTimerRef.current = null;
|
|
206
|
+
}
|
|
207
|
+
animationMixerRef.current?.stopAllAction();
|
|
208
|
+
animationMixerRef.current = null;
|
|
209
|
+
if (!scene || !animationRequest)
|
|
210
|
+
return;
|
|
211
|
+
const source = animationGltf ?? (animationRequest.url === uri ? gltf : null);
|
|
212
|
+
if (!source)
|
|
213
|
+
return;
|
|
214
|
+
const clip = source.animations[animationRequest.index] ?? source.animations[0];
|
|
215
|
+
if (!clip) {
|
|
216
|
+
onAvatarState?.(`animation_error:no_clip:${animationRequest.url}`);
|
|
217
|
+
return;
|
|
218
|
+
}
|
|
219
|
+
const mixer = new THREE.AnimationMixer(scene);
|
|
220
|
+
const action = mixer.clipAction(clip);
|
|
221
|
+
action.reset().play();
|
|
222
|
+
animationMixerRef.current = mixer;
|
|
223
|
+
onAvatarState?.(`animation:${animationRequest.url}`);
|
|
224
|
+
if (animationRequest.dur && animationRequest.dur > 0) {
|
|
225
|
+
animationStopTimerRef.current = setTimeout(() => {
|
|
226
|
+
mixer.stopAllAction();
|
|
227
|
+
if (animationMixerRef.current === mixer)
|
|
228
|
+
animationMixerRef.current = null;
|
|
229
|
+
onAvatarState?.('animation:stopped');
|
|
230
|
+
}, animationRequest.dur * 1000);
|
|
231
|
+
}
|
|
232
|
+
return () => {
|
|
233
|
+
if (animationStopTimerRef.current) {
|
|
234
|
+
clearTimeout(animationStopTimerRef.current);
|
|
235
|
+
animationStopTimerRef.current = null;
|
|
236
|
+
}
|
|
237
|
+
mixer.stopAllAction();
|
|
238
|
+
if (animationMixerRef.current === mixer)
|
|
239
|
+
animationMixerRef.current = null;
|
|
240
|
+
};
|
|
241
|
+
}, [animationGltf, animationRequest, gltf, onAvatarState, scene, uri]);
|
|
242
|
+
(0, fiber_1.useFrame)((_, delta) => {
|
|
125
243
|
const mesh = headMeshRef.current;
|
|
126
244
|
if (!mesh?.morphTargetInfluences)
|
|
127
245
|
return;
|
|
128
246
|
const state = morphStateRef.current;
|
|
129
|
-
|
|
130
|
-
const
|
|
247
|
+
// Frame-rate independent lerp: same convergence speed at 30 Hz and 120 Hz
|
|
248
|
+
const alpha = 1 - Math.exp(-state.alpha * delta * 60);
|
|
249
|
+
// Idle animation — subtle breathing / brow life at ~0.3 Hz
|
|
250
|
+
state.idleTime += delta;
|
|
251
|
+
const breathe = Math.sin(state.idleTime * 2 * Math.PI * 0.3) * 0.04;
|
|
252
|
+
const idleIdle = {
|
|
253
|
+
browRaiserL: Math.max(0, breathe),
|
|
254
|
+
browRaiserR: Math.max(0, breathe),
|
|
255
|
+
browRaiser_L: Math.max(0, breathe),
|
|
256
|
+
browRaiser_R: Math.max(0, breathe),
|
|
257
|
+
};
|
|
258
|
+
const combined = { ...state.moodBase, ...idleIdle };
|
|
131
259
|
for (const [name, w] of Object.entries(state.visemeTarget)) {
|
|
132
260
|
combined[name] = Math.max(combined[name] ?? 0, w);
|
|
133
261
|
}
|
|
@@ -141,19 +269,61 @@ function AvatarScene({ uri, morphStateRef, fov, onReady, onError, onSceneReady }
|
|
|
141
269
|
if (idx !== undefined)
|
|
142
270
|
mesh.morphTargetInfluences[idx] = next;
|
|
143
271
|
}
|
|
272
|
+
// Viseme decay: hold peak for 80 ms then decay (avoids eating fast cues)
|
|
273
|
+
const now = performance.now();
|
|
144
274
|
for (const name of Object.keys(state.visemeTarget)) {
|
|
145
|
-
state.
|
|
146
|
-
if (
|
|
147
|
-
|
|
275
|
+
const cueAge = (now - (state.visemeCueTime[name] ?? now)) / 1000;
|
|
276
|
+
if (cueAge > 0.08) {
|
|
277
|
+
state.visemeTarget[name] *= Math.pow(0.05, delta);
|
|
278
|
+
if (state.visemeTarget[name] < 0.001) {
|
|
279
|
+
delete state.visemeTarget[name];
|
|
280
|
+
delete state.visemeCueTime[name];
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
// Procedural body motion — drives the skeleton bones (independent of the
|
|
285
|
+
// face morphs above). Bones are scanned lazily on first active motion.
|
|
286
|
+
const motion = motionStateRef.current;
|
|
287
|
+
if (motion.key) {
|
|
288
|
+
if (!motion.bones) {
|
|
289
|
+
const root = headMeshRef.current?.parent ?? mesh;
|
|
290
|
+
const skeletonRoot = root ? findSkeletonRoot(root) : null;
|
|
291
|
+
if (skeletonRoot) {
|
|
292
|
+
const scanned = (0, motionRuntime_1.scanMotionBones)((visit) => {
|
|
293
|
+
skeletonRoot.traverse((node) => {
|
|
294
|
+
if (node.isBone) {
|
|
295
|
+
visit(node);
|
|
296
|
+
}
|
|
297
|
+
});
|
|
298
|
+
});
|
|
299
|
+
motion.bones = scanned.bones;
|
|
300
|
+
motion.rest = scanned.rest;
|
|
301
|
+
}
|
|
302
|
+
else {
|
|
303
|
+
// No skeleton — nothing to drive. Clear so we don't rescan every frame.
|
|
304
|
+
motion.key = null;
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
if (motion.key && motion.bones && motion.rest) {
|
|
308
|
+
const tSec = (now - motion.startTime) / 1000;
|
|
309
|
+
(0, motionRuntime_1.applyMotionFrame)(motion.key, tSec, motion.bones, motion.rest);
|
|
310
|
+
}
|
|
148
311
|
}
|
|
312
|
+
animationMixerRef.current?.update(delta);
|
|
149
313
|
});
|
|
150
|
-
|
|
314
|
+
if (!scene) {
|
|
315
|
+
return ((0, jsx_runtime_1.jsxs)(jsx_runtime_1.Fragment, { children: [onSceneReady && (0, jsx_runtime_1.jsx)(SceneCapture, { onSceneReady: onSceneReady }), (0, jsx_runtime_1.jsx)("ambientLight", { intensity: 0.6 }), (0, jsx_runtime_1.jsx)("directionalLight", { position: [1, 3, 2], intensity: 1.2, castShadow: false })] }));
|
|
316
|
+
}
|
|
317
|
+
return ((0, jsx_runtime_1.jsxs)(jsx_runtime_1.Fragment, { children: [onSceneReady && (0, jsx_runtime_1.jsx)(SceneCapture, { onSceneReady: onSceneReady }), (0, jsx_runtime_1.jsx)("ambientLight", { intensity: 0.6 }), (0, jsx_runtime_1.jsx)("directionalLight", { position: [1, 3, 2], intensity: 1.2, castShadow: false }), (0, jsx_runtime_1.jsx)("group", { position: placement?.position ?? CAMERA_TARGET, scale: placement?.scale ?? 1, children: (0, jsx_runtime_1.jsx)("primitive", { object: scene }) }), accessories?.map((acc) => ((0, jsx_runtime_1.jsx)(AccessoryMesh, { accessory: acc, avatarScene: scene }, acc.id)))] }));
|
|
151
318
|
}
|
|
152
319
|
// ---------------------------------------------------------------------------
|
|
153
320
|
// Error boundary
|
|
154
321
|
// ---------------------------------------------------------------------------
|
|
155
322
|
class GltfErrorBoundary extends react_1.default.Component {
|
|
156
|
-
constructor(props) {
|
|
323
|
+
constructor(props) {
|
|
324
|
+
super(props);
|
|
325
|
+
this.state = { hasError: false };
|
|
326
|
+
}
|
|
157
327
|
static getDerivedStateFromError() { return { hasError: true }; }
|
|
158
328
|
componentDidCatch(err) { this.props.onError(err.message); }
|
|
159
329
|
render() { if (this.state.hasError)
|
|
@@ -167,43 +337,88 @@ function applyVisemeCue(morphState, visemeKey) {
|
|
|
167
337
|
if (!aliases)
|
|
168
338
|
return;
|
|
169
339
|
const w = morphTables_1.VISEME_WEIGHTS[visemeKey] ?? morphTables_1.DEFAULT_VISEME_WEIGHT;
|
|
170
|
-
|
|
340
|
+
const now = performance.now();
|
|
341
|
+
for (const alias of aliases) {
|
|
171
342
|
morphState.visemeTarget[alias] = w;
|
|
343
|
+
morphState.visemeCueTime[alias] = now;
|
|
344
|
+
}
|
|
172
345
|
}
|
|
173
346
|
// ---------------------------------------------------------------------------
|
|
174
347
|
// Main exported component
|
|
175
348
|
// ---------------------------------------------------------------------------
|
|
176
|
-
exports.WgpuAvatar = (0, react_1.forwardRef)(({ style, avatarUrl, focalLength, mood = 'neutral', onReady, onError, onSceneReady }, ref) => {
|
|
177
|
-
const
|
|
178
|
-
|
|
349
|
+
exports.WgpuAvatar = (0, react_1.forwardRef)(({ style, avatarUrl, focalLength, mood = 'neutral', accessories, onReady, onError, onAvatarState, onSceneReady }, ref) => {
|
|
350
|
+
const isRemote = avatarUrl != null && (avatarUrl.startsWith('http://') || avatarUrl.startsWith('https://'));
|
|
351
|
+
const remoteUrl = (0, react_1.useMemo)(() => isRemote ? avatarUrl : null,
|
|
352
|
+
// eslint-disable-next-line react-hooks/exhaustive-deps
|
|
353
|
+
[avatarUrl]);
|
|
179
354
|
const fileResult = (0, useAuthedModelUri_1.useAuthedModelUri)(remoteUrl);
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
355
|
+
// stableUri: once a valid URI is resolved, keep showing it until a new one
|
|
356
|
+
// is ready — prevents Canvas unmount/remount flicker on transient url changes.
|
|
357
|
+
const stableUriRef = (0, react_1.useRef)(null);
|
|
358
|
+
const lastAvatarUrlRef = (0, react_1.useRef)(avatarUrl);
|
|
359
|
+
if (lastAvatarUrlRef.current !== avatarUrl) {
|
|
360
|
+
// avatarUrl changed — invalidate stable URI so next resolved URI takes over
|
|
361
|
+
lastAvatarUrlRef.current = avatarUrl;
|
|
362
|
+
stableUriRef.current = null;
|
|
185
363
|
}
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
364
|
+
// For non-remote URLs (file://, asset://, data:) use directly without download
|
|
365
|
+
const currentUri = isRemote ? (fileResult?.uri ?? null) : (avatarUrl ?? null);
|
|
366
|
+
if (currentUri && currentUri !== stableUriRef.current) {
|
|
367
|
+
stableUriRef.current = currentUri;
|
|
368
|
+
}
|
|
369
|
+
const localUri = stableUriRef.current;
|
|
190
370
|
const fov = focalLength
|
|
191
371
|
? Math.round(2 * Math.atan(21.634 / focalLength) * (180 / Math.PI))
|
|
192
372
|
: CAMERA_FOV_FULL;
|
|
373
|
+
const cameraProps = (0, react_1.useMemo)(() => ({ fov, position: CAMERA_POSITION, near: 0.01, far: 100 }), [fov]);
|
|
193
374
|
const morphStateRef = (0, react_1.useRef)({
|
|
194
|
-
current: {}, visemeTarget: {}, moodBase: {}, alpha: 0.18,
|
|
375
|
+
current: {}, visemeTarget: {}, visemeCueTime: {}, moodBase: {}, alpha: 0.18, idleTime: 0,
|
|
195
376
|
});
|
|
377
|
+
const motionStateRef = (0, react_1.useRef)((0, motionState_1.createMotionState)());
|
|
378
|
+
const [animationRequest, setAnimationRequest] = (0, react_1.useState)(null);
|
|
379
|
+
const animationNonceRef = (0, react_1.useRef)(0);
|
|
380
|
+
const onAvatarStateRef = (0, react_1.useRef)(onAvatarState);
|
|
381
|
+
onAvatarStateRef.current = onAvatarState;
|
|
196
382
|
(0, react_1.useEffect)(() => {
|
|
197
383
|
morphStateRef.current.moodBase = { ...(morphTables_1.MOOD_MORPHS[mood] ?? {}) };
|
|
198
384
|
}, [mood]);
|
|
199
385
|
const scheduleRef = (0, react_1.useRef)(null);
|
|
200
386
|
const scheduleTimersRef = (0, react_1.useRef)([]);
|
|
387
|
+
const blinkTimersRef = (0, react_1.useRef)([]);
|
|
201
388
|
const [isReady, setIsReady] = (0, react_1.useState)(false);
|
|
202
389
|
const clearScheduleTimers = (0, react_1.useCallback)(() => {
|
|
203
390
|
for (const t of scheduleTimersRef.current)
|
|
204
391
|
clearTimeout(t);
|
|
205
392
|
scheduleTimersRef.current = [];
|
|
206
393
|
}, []);
|
|
394
|
+
(0, react_1.useEffect)(() => {
|
|
395
|
+
(0, motionState_1.resetMotionStateForAvatarSwap)(motionStateRef.current);
|
|
396
|
+
}, [localUri]);
|
|
397
|
+
const stopMotion = (0, react_1.useCallback)(() => {
|
|
398
|
+
(0, motionState_1.stopMotionState)(motionStateRef.current);
|
|
399
|
+
}, []);
|
|
400
|
+
const playMotionKey = (0, react_1.useCallback)((name, autoReturnOverrideMs) => {
|
|
401
|
+
if (!(0, motion_1.isMotionKey)(name)) {
|
|
402
|
+
onAvatarStateRef.current?.(`motion_error:unknown:${name}`);
|
|
403
|
+
return false;
|
|
404
|
+
}
|
|
405
|
+
const motion = motionStateRef.current;
|
|
406
|
+
if (motion.autoReturnTimer) {
|
|
407
|
+
clearTimeout(motion.autoReturnTimer);
|
|
408
|
+
motion.autoReturnTimer = null;
|
|
409
|
+
}
|
|
410
|
+
motion.key = name;
|
|
411
|
+
motion.startTime = performance.now();
|
|
412
|
+
onAvatarStateRef.current?.(`motion:${name}`);
|
|
413
|
+
const autoReturnMs = autoReturnOverrideMs ?? motion_1.MOTION_DEFS[name].autoReturnMs;
|
|
414
|
+
if (autoReturnMs && autoReturnMs > 0) {
|
|
415
|
+
motion.autoReturnTimer = setTimeout(() => {
|
|
416
|
+
if (motionStateRef.current.key === name)
|
|
417
|
+
stopMotion();
|
|
418
|
+
}, autoReturnMs);
|
|
419
|
+
}
|
|
420
|
+
return true;
|
|
421
|
+
}, [stopMotion]);
|
|
207
422
|
const applySchedule = (0, react_1.useCallback)((schedule) => {
|
|
208
423
|
clearScheduleTimers();
|
|
209
424
|
const now = Date.now();
|
|
@@ -235,6 +450,12 @@ exports.WgpuAvatar = (0, react_1.forwardRef)(({ style, avatarUrl, focalLength, m
|
|
|
235
450
|
onError?.(msg);
|
|
236
451
|
}, [onError]);
|
|
237
452
|
(0, react_1.useEffect)(() => () => clearScheduleTimers(), [clearScheduleTimers]);
|
|
453
|
+
(0, react_1.useEffect)(() => () => { for (const t of blinkTimersRef.current)
|
|
454
|
+
clearTimeout(t); }, []);
|
|
455
|
+
(0, react_1.useEffect)(() => () => (0, motionState_1.resetMotionStateForAvatarSwap)(motionStateRef.current), []);
|
|
456
|
+
const handleCanvasCreated = (0, react_1.useCallback)((state) => {
|
|
457
|
+
state.scene.background = new THREE.Color('#1a1a2e');
|
|
458
|
+
}, []);
|
|
238
459
|
(0, react_1.useImperativeHandle)(ref, () => ({
|
|
239
460
|
setMood: (m) => { morphStateRef.current.moodBase = { ...(morphTables_1.MOOD_MORPHS[m] ?? {}) }; },
|
|
240
461
|
sendAmplitude: () => { },
|
|
@@ -243,10 +464,55 @@ exports.WgpuAvatar = (0, react_1.forwardRef)(({ style, avatarUrl, focalLength, m
|
|
|
243
464
|
if (!aliases)
|
|
244
465
|
return;
|
|
245
466
|
const w = weight ?? morphTables_1.VISEME_WEIGHTS[viseme] ?? morphTables_1.DEFAULT_VISEME_WEIGHT;
|
|
246
|
-
|
|
467
|
+
const now = performance.now();
|
|
468
|
+
for (const alias of aliases) {
|
|
247
469
|
morphStateRef.current.visemeTarget[alias] = w;
|
|
470
|
+
morphStateRef.current.visemeCueTime[alias] = now;
|
|
471
|
+
}
|
|
472
|
+
},
|
|
473
|
+
clearVisemes: () => { clearScheduleTimers(); morphStateRef.current.visemeTarget = {}; morphStateRef.current.visemeCueTime = {}; },
|
|
474
|
+
playMotion: (name) => {
|
|
475
|
+
playMotionKey(name);
|
|
476
|
+
},
|
|
477
|
+
stopMotion,
|
|
478
|
+
playGesture: (name, opts) => {
|
|
479
|
+
playMotionKey(name, opts?.ms ?? 1000);
|
|
480
|
+
},
|
|
481
|
+
stopGesture: stopMotion,
|
|
482
|
+
playPose: (name) => {
|
|
483
|
+
playMotionKey(name);
|
|
484
|
+
},
|
|
485
|
+
stopPose: stopMotion,
|
|
486
|
+
playAnimation: (url, opts) => {
|
|
487
|
+
animationNonceRef.current += 1;
|
|
488
|
+
setAnimationRequest({
|
|
489
|
+
url,
|
|
490
|
+
index: opts?.index ?? 0,
|
|
491
|
+
dur: opts?.dur,
|
|
492
|
+
nonce: animationNonceRef.current,
|
|
493
|
+
});
|
|
494
|
+
},
|
|
495
|
+
stopAnimation: () => {
|
|
496
|
+
setAnimationRequest(null);
|
|
497
|
+
onAvatarStateRef.current?.('animation:stopped');
|
|
498
|
+
},
|
|
499
|
+
triggerBlink: () => {
|
|
500
|
+
// Blink: close eyes over 80ms, hold 40ms, open over 80ms
|
|
501
|
+
const BLINK_MORPHS = [
|
|
502
|
+
'eyeBlinkLeft', 'eyeBlink_L', 'eyesClosed',
|
|
503
|
+
'eyeBlinkRight', 'eyeBlink_R',
|
|
504
|
+
];
|
|
505
|
+
const now = performance.now();
|
|
506
|
+
const set = (w) => {
|
|
507
|
+
for (const name of BLINK_MORPHS) {
|
|
508
|
+
morphStateRef.current.visemeTarget[name] = w;
|
|
509
|
+
morphStateRef.current.visemeCueTime[name] = now;
|
|
510
|
+
}
|
|
511
|
+
};
|
|
512
|
+
set(1);
|
|
513
|
+
const t1 = setTimeout(() => set(0), 120);
|
|
514
|
+
blinkTimersRef.current.push(t1);
|
|
248
515
|
},
|
|
249
|
-
clearVisemes: () => { clearScheduleTimers(); morphStateRef.current.visemeTarget = {}; },
|
|
250
516
|
scheduleVisemes: (schedule) => {
|
|
251
517
|
if (!isReady) {
|
|
252
518
|
scheduleRef.current = schedule;
|
|
@@ -254,13 +520,14 @@ exports.WgpuAvatar = (0, react_1.forwardRef)(({ style, avatarUrl, focalLength, m
|
|
|
254
520
|
}
|
|
255
521
|
applySchedule(schedule);
|
|
256
522
|
},
|
|
257
|
-
}), [isReady, applySchedule, clearScheduleTimers]);
|
|
523
|
+
}), [isReady, applySchedule, clearScheduleTimers, playMotionKey, stopMotion]);
|
|
258
524
|
if (!localUri)
|
|
259
525
|
return (0, jsx_runtime_1.jsx)(react_native_1.View, { style: [styles.placeholder, style] });
|
|
260
|
-
return ((0, jsx_runtime_1.jsx)(react_native_1.View, { style: [styles.container, style], children: (0, jsx_runtime_1.jsx)(
|
|
526
|
+
return ((0, jsx_runtime_1.jsx)(react_native_1.View, { style: [styles.container, style], children: (0, jsx_runtime_1.jsx)(R3FWebGpuCanvas_1.R3FWebGpuCanvas, { style: styles.canvas, camera: cameraProps, clearColor: "#1a1a2e", onCreated: handleCanvasCreated, onError: handleError, children: (0, jsx_runtime_1.jsx)(GltfErrorBoundary, { onError: handleError, children: (0, jsx_runtime_1.jsx)(AvatarScene, { uri: localUri, morphStateRef: morphStateRef, motionStateRef: motionStateRef, fov: fov, accessories: accessories, animationRequest: animationRequest, onReady: handleReady, onError: handleError, onAvatarState: onAvatarState, onSceneReady: onSceneReady }) }) }) }));
|
|
261
527
|
});
|
|
262
528
|
exports.WgpuAvatar.displayName = 'WgpuAvatar';
|
|
263
529
|
const styles = react_native_1.StyleSheet.create({
|
|
264
530
|
container: { overflow: 'hidden', backgroundColor: '#1a1a2e' },
|
|
531
|
+
canvas: { flex: 1 },
|
|
265
532
|
placeholder: { backgroundColor: '#1a1a2e' },
|
|
266
533
|
});
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Canonical snap-to-bone defaults per accessory category.
|
|
3
|
+
* Applied immediately when the user equips an item that has no saved placement,
|
|
4
|
+
* so the "good enough" position is free on tap for ~90% of use-cases.
|
|
5
|
+
*
|
|
6
|
+
* Offsets are in the bone's local space (metres).
|
|
7
|
+
* Rotation is Euler XYZ in radians.
|
|
8
|
+
*/
|
|
9
|
+
import type { AssetPlacement } from '../wardrobe/wardrobeStore';
|
|
10
|
+
export declare const BONE_SNAP_DEFAULTS: Record<string, AssetPlacement>;
|
|
11
|
+
/** Returns the snap default for a category, or null if the category is unknown. */
|
|
12
|
+
export declare function snapDefaultForCategory(category: string): AssetPlacement | null;
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.BONE_SNAP_DEFAULTS = void 0;
|
|
4
|
+
exports.snapDefaultForCategory = snapDefaultForCategory;
|
|
5
|
+
exports.BONE_SNAP_DEFAULTS = {
|
|
6
|
+
hat: { bone: 'Head', position: [0, 0.12, 0.00], rotation: [0, 0, 0], scale: 1.0 },
|
|
7
|
+
cap: { bone: 'Head', position: [0, 0.10, 0.02], rotation: [0, 0, 0], scale: 1.0 },
|
|
8
|
+
glasses: { bone: 'Head', position: [0, 0.02, 0.07], rotation: [0, 0, 0], scale: 1.0 },
|
|
9
|
+
sunglasses: { bone: 'Head', position: [0, 0.02, 0.07], rotation: [0, 0, 0], scale: 1.0 },
|
|
10
|
+
earring: { bone: 'Head', position: [0.085, 0.01, 0.00], rotation: [0, 0, 0], scale: 0.8 },
|
|
11
|
+
earrings: { bone: 'Head', position: [0.085, 0.01, 0.00], rotation: [0, 0, 0], scale: 0.8 },
|
|
12
|
+
necklace: { bone: 'Neck', position: [0, -0.04, 0.03], rotation: [0, 0, 0], scale: 1.0 },
|
|
13
|
+
mask: { bone: 'Head', position: [0, 0.00, 0.08], rotation: [0, 0, 0], scale: 1.0 },
|
|
14
|
+
};
|
|
15
|
+
/** Returns the snap default for a category, or null if the category is unknown. */
|
|
16
|
+
function snapDefaultForCategory(category) {
|
|
17
|
+
const key = category.toLowerCase().trim();
|
|
18
|
+
return exports.BONE_SNAP_DEFAULTS[key] ?? null;
|
|
19
|
+
}
|