talking-head-studio 0.4.10 → 0.4.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +227 -351
- package/dist/TalkingHead.d.ts +16 -25
- package/dist/TalkingHead.web.d.ts +6 -0
- package/dist/TalkingHead.web.js +17 -7
- package/dist/api/studioApi.js +25 -26
- package/dist/appearance/apply.js +2 -3
- package/dist/appearance/matchers.js +1 -2
- package/dist/appearance/schema.js +1 -2
- package/dist/core/avatar/backend.d.ts +130 -0
- package/dist/core/avatar/backend.js +4 -0
- package/dist/core/avatar/backends/gaussian.d.ts +49 -0
- package/dist/core/avatar/backends/gaussian.js +291 -0
- package/dist/core/avatar/backends/index.d.ts +3 -0
- package/dist/core/avatar/backends/index.js +7 -0
- package/dist/core/avatar/backends/morphTarget.d.ts +39 -0
- package/dist/core/avatar/backends/morphTarget.js +179 -0
- package/dist/core/avatar/faceControls.d.ts +40 -0
- package/dist/core/avatar/faceControls.js +138 -0
- package/dist/core/avatar/schema.d.ts +50 -0
- package/dist/core/avatar/schema.js +134 -0
- package/dist/core/avatar/visemes.d.ts +31 -0
- package/dist/core/avatar/visemes.js +67 -1
- package/dist/editor/AvatarCanvas.js +1 -2
- package/dist/editor/AvatarEditor.native.js +18 -9
- package/dist/editor/AvatarModel.js +1 -2
- package/dist/editor/FaceSqueezeEditor.js +19 -9
- package/dist/editor/FaceSqueezeEditor.web.js +2 -2
- package/dist/editor/RigidAccessory.js +1 -2
- package/dist/editor/SkinnedClothing.js +18 -9
- package/dist/editor/boneSnap.js +22 -12
- package/dist/editor/studioTheme.js +2 -2
- package/dist/html.js +1 -2
- package/dist/index.d.ts +15 -1
- package/dist/index.js +28 -5
- package/dist/platform/api/types.d.ts +10 -0
- package/dist/platform/api/types.js +2 -0
- package/dist/platform/marketplace/types.d.ts +32 -0
- package/dist/platform/marketplace/types.js +2 -0
- package/dist/platform/sdk/unity.d.ts +27 -0
- package/dist/platform/sdk/unity.js +2 -0
- package/dist/platform/sdk/unreal.d.ts +23 -0
- package/dist/platform/sdk/unreal.js +2 -0
- package/dist/platform/sdk/web.d.ts +16 -0
- package/dist/platform/sdk/web.js +2 -0
- package/dist/sketchfab/api.js +4 -5
- package/dist/sketchfab/useSketchfabSearch.js +1 -2
- package/dist/tts/useDirectVisemeStream.d.ts +2 -6
- package/dist/tts/useDirectVisemeStream.js +1 -2
- package/dist/tts/useMotionMarkers.d.ts +0 -1
- package/dist/tts/useMotionMarkers.js +1 -2
- package/dist/utils/avatarUtils.js +2 -3
- package/dist/utils/faceLandmarkerToShapeWeights.js +19 -10
- package/dist/voice/convertToWav.js +1 -2
- package/dist/voice/index.d.ts +3 -0
- package/dist/voice/index.js +6 -1
- package/dist/voice/useAudioPlayer.js +1 -2
- package/dist/voice/useAudioRecording.js +1 -2
- package/dist/voice/useFaceControls.d.ts +14 -0
- package/dist/voice/useFaceControls.js +81 -0
- package/dist/voice/useVoicePreview.d.ts +7 -0
- package/dist/voice/useVoicePreview.js +81 -0
- package/dist/wardrobe/index.d.ts +2 -0
- package/dist/wardrobe/index.js +3 -1
- package/dist/wardrobe/useAvatarWardrobeHydration.js +1 -2
- package/dist/wardrobe/useStudioAvatar.d.ts +29 -0
- package/dist/wardrobe/useStudioAvatar.js +177 -0
- package/dist/wgpu/WgpuAvatar.js +17 -7
- package/dist/wgpu/useAuthedModelUri.js +18 -9
- package/package.json +8 -4
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// src/core/avatar/faceControls.ts
|
|
3
|
+
// Canonical facial control space shared by all avatar backends.
|
|
4
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
5
|
+
exports.createNeutralExpression = createNeutralExpression;
|
|
6
|
+
exports.visemeToExpression = visemeToExpression;
|
|
7
|
+
exports.applyVisemeToExpression = applyVisemeToExpression;
|
|
8
|
+
function createNeutralExpression() {
|
|
9
|
+
return {
|
|
10
|
+
jawOpen: 0,
|
|
11
|
+
mouthSmile: 0,
|
|
12
|
+
mouthFunnel: 0,
|
|
13
|
+
mouthPucker: 0,
|
|
14
|
+
mouthWide: 0,
|
|
15
|
+
upperLipRaise: 0,
|
|
16
|
+
lowerLipDepress: 0,
|
|
17
|
+
cheekRaise: 0,
|
|
18
|
+
blinkLeft: 0,
|
|
19
|
+
blinkRight: 0,
|
|
20
|
+
browInnerUp: 0,
|
|
21
|
+
browDownLeft: 0,
|
|
22
|
+
browDownRight: 0,
|
|
23
|
+
eyeGazeLeft: { x: 0, y: 0 },
|
|
24
|
+
eyeGazeRight: { x: 0, y: 0 },
|
|
25
|
+
};
|
|
26
|
+
}
|
|
27
|
+
function clamp01(v) {
|
|
28
|
+
return v < 0 ? 0 : v > 1 ? 1 : v;
|
|
29
|
+
}
|
|
30
|
+
function scale(v, w) {
|
|
31
|
+
return clamp01(v * w);
|
|
32
|
+
}
|
|
33
|
+
// Map an Oculus viseme + weight into a sparse ExpressionState delta.
|
|
34
|
+
// These coefficients are intentionally simple and can be tuned per-avatar
|
|
35
|
+
// via a calibration layer.
|
|
36
|
+
function visemeToExpression(viseme, weight = 1) {
|
|
37
|
+
const w = clamp01(weight);
|
|
38
|
+
switch (viseme) {
|
|
39
|
+
case 'sil':
|
|
40
|
+
return { jawOpen: 0, mouthPucker: 0, mouthWide: 0 };
|
|
41
|
+
case 'PP':
|
|
42
|
+
return {
|
|
43
|
+
jawOpen: scale(0.05, w),
|
|
44
|
+
mouthPucker: scale(0.7, w),
|
|
45
|
+
};
|
|
46
|
+
case 'FF':
|
|
47
|
+
return {
|
|
48
|
+
jawOpen: scale(0.15, w),
|
|
49
|
+
upperLipRaise: scale(0.2, w),
|
|
50
|
+
lowerLipDepress: scale(0.25, w),
|
|
51
|
+
};
|
|
52
|
+
case 'TH':
|
|
53
|
+
return {
|
|
54
|
+
jawOpen: scale(0.2, w),
|
|
55
|
+
mouthWide: scale(0.15, w),
|
|
56
|
+
};
|
|
57
|
+
case 'DD':
|
|
58
|
+
return {
|
|
59
|
+
jawOpen: scale(0.25, w),
|
|
60
|
+
mouthWide: scale(0.1, w),
|
|
61
|
+
};
|
|
62
|
+
case 'kk':
|
|
63
|
+
return {
|
|
64
|
+
jawOpen: scale(0.3, w),
|
|
65
|
+
};
|
|
66
|
+
case 'CH':
|
|
67
|
+
return {
|
|
68
|
+
jawOpen: scale(0.35, w),
|
|
69
|
+
mouthPucker: scale(0.15, w),
|
|
70
|
+
mouthWide: scale(0.1, w),
|
|
71
|
+
};
|
|
72
|
+
case 'SS':
|
|
73
|
+
return {
|
|
74
|
+
jawOpen: scale(0.1, w),
|
|
75
|
+
mouthWide: scale(0.35, w),
|
|
76
|
+
};
|
|
77
|
+
case 'nn':
|
|
78
|
+
return {
|
|
79
|
+
jawOpen: scale(0.18, w),
|
|
80
|
+
};
|
|
81
|
+
case 'RR':
|
|
82
|
+
return {
|
|
83
|
+
jawOpen: scale(0.22, w),
|
|
84
|
+
mouthPucker: scale(0.2, w),
|
|
85
|
+
};
|
|
86
|
+
case 'aa':
|
|
87
|
+
return {
|
|
88
|
+
jawOpen: scale(0.95, w),
|
|
89
|
+
mouthWide: scale(0.1, w),
|
|
90
|
+
};
|
|
91
|
+
case 'ee':
|
|
92
|
+
return {
|
|
93
|
+
jawOpen: scale(0.45, w),
|
|
94
|
+
mouthWide: scale(0.8, w),
|
|
95
|
+
mouthSmile: scale(0.15, w),
|
|
96
|
+
};
|
|
97
|
+
case 'ih':
|
|
98
|
+
return {
|
|
99
|
+
jawOpen: scale(0.35, w),
|
|
100
|
+
mouthWide: scale(0.55, w),
|
|
101
|
+
};
|
|
102
|
+
case 'oh':
|
|
103
|
+
return {
|
|
104
|
+
jawOpen: scale(0.55, w),
|
|
105
|
+
mouthFunnel: scale(0.6, w),
|
|
106
|
+
mouthPucker: scale(0.2, w),
|
|
107
|
+
};
|
|
108
|
+
case 'ou':
|
|
109
|
+
return {
|
|
110
|
+
jawOpen: scale(0.25, w),
|
|
111
|
+
mouthFunnel: scale(0.85, w),
|
|
112
|
+
mouthPucker: scale(0.45, w),
|
|
113
|
+
};
|
|
114
|
+
default:
|
|
115
|
+
return {};
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
// Utility to accumulate a viseme delta into an existing ExpressionState.
|
|
119
|
+
function applyVisemeToExpression(base, viseme, weight = 1) {
|
|
120
|
+
const delta = visemeToExpression(viseme, weight);
|
|
121
|
+
return {
|
|
122
|
+
jawOpen: clamp01(base.jawOpen + (delta.jawOpen ?? 0)),
|
|
123
|
+
mouthSmile: clamp01(base.mouthSmile + (delta.mouthSmile ?? 0)),
|
|
124
|
+
mouthFunnel: clamp01(base.mouthFunnel + (delta.mouthFunnel ?? 0)),
|
|
125
|
+
mouthPucker: clamp01(base.mouthPucker + (delta.mouthPucker ?? 0)),
|
|
126
|
+
mouthWide: clamp01(base.mouthWide + (delta.mouthWide ?? 0)),
|
|
127
|
+
upperLipRaise: clamp01(base.upperLipRaise + (delta.upperLipRaise ?? 0)),
|
|
128
|
+
lowerLipDepress: clamp01(base.lowerLipDepress + (delta.lowerLipDepress ?? 0)),
|
|
129
|
+
cheekRaise: clamp01(base.cheekRaise + (delta.cheekRaise ?? 0)),
|
|
130
|
+
blinkLeft: clamp01(base.blinkLeft + (delta.blinkLeft ?? 0)),
|
|
131
|
+
blinkRight: clamp01(base.blinkRight + (delta.blinkRight ?? 0)),
|
|
132
|
+
browInnerUp: clamp01(base.browInnerUp + (delta.browInnerUp ?? 0)),
|
|
133
|
+
browDownLeft: clamp01(base.browDownLeft + (delta.browDownLeft ?? 0)),
|
|
134
|
+
browDownRight: clamp01(base.browDownRight + (delta.browDownRight ?? 0)),
|
|
135
|
+
eyeGazeLeft: base.eyeGazeLeft,
|
|
136
|
+
eyeGazeRight: base.eyeGazeRight,
|
|
137
|
+
};
|
|
138
|
+
}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import type { OculusViseme } from './visemes';
|
|
2
|
+
export type VísemeTier = 'oculus' | 'arkit' | 'minimal' | 'none';
|
|
3
|
+
export interface MorphCoverage {
|
|
4
|
+
/** All morph target names found across all meshes in the scene. */
|
|
5
|
+
all: string[];
|
|
6
|
+
/** Oculus viseme morph targets present (subset of OCULUS_VISEMES). */
|
|
7
|
+
oculusVisemes: OculusViseme[];
|
|
8
|
+
/** Whether the full Oculus set (all 15) is present. */
|
|
9
|
+
hasFullOculusSet: boolean;
|
|
10
|
+
/** ARKit blend shape names found (jawOpen, mouthFunnel, etc.). */
|
|
11
|
+
arkitShapes: string[];
|
|
12
|
+
/** Whether a meaningful ARKit set is present (≥10 mouth shapes). */
|
|
13
|
+
hasArkitSet: boolean;
|
|
14
|
+
/** Best lip-sync strategy available for this model. */
|
|
15
|
+
visemeTier: VísemeTier;
|
|
16
|
+
}
|
|
17
|
+
export interface SkeletonInfo {
|
|
18
|
+
/** Whether a skeleton (SkinnedMesh) was found. */
|
|
19
|
+
hasRig: boolean;
|
|
20
|
+
/** All bone names found in the scene. */
|
|
21
|
+
bones: string[];
|
|
22
|
+
/** Whether the standard humanoid bones are present (hips → spine → head). */
|
|
23
|
+
isHumanoid: boolean;
|
|
24
|
+
/** Missing standard humanoid bones (if any). */
|
|
25
|
+
missingHumanoidBones: string[];
|
|
26
|
+
}
|
|
27
|
+
export interface MeshStats {
|
|
28
|
+
/** Total triangle count across all meshes. */
|
|
29
|
+
triangles: number;
|
|
30
|
+
/** Number of distinct mesh primitives. */
|
|
31
|
+
meshCount: number;
|
|
32
|
+
/** Estimated LOD tier based on triangle count. */
|
|
33
|
+
lodTier: 'high' | 'medium' | 'low' | 'very-low';
|
|
34
|
+
}
|
|
35
|
+
export interface AvatarSchemaReport {
|
|
36
|
+
morphs: MorphCoverage;
|
|
37
|
+
skeleton: SkeletonInfo;
|
|
38
|
+
mesh: MeshStats;
|
|
39
|
+
/** Whether this model is ready for full-quality talking-head rendering. */
|
|
40
|
+
isReadyForLipSync: boolean;
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* Scan a loaded Three.js Object3D scene and return a structured compatibility
|
|
44
|
+
* report for lip-sync, skeleton, and mesh quality.
|
|
45
|
+
*
|
|
46
|
+
* @param scene - The root Object3D from a GLTFLoader result (gltf.scene)
|
|
47
|
+
*/
|
|
48
|
+
export declare function walkAvatarSchema(scene: {
|
|
49
|
+
traverse: (cb: (obj: any) => void) => void;
|
|
50
|
+
}): AvatarSchemaReport;
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// src/core/avatar/schema.ts
|
|
3
|
+
// GLB schema walker — scan any loaded Three.js scene and report morph target
|
|
4
|
+
// coverage, skeleton bones, viseme tier, and LOD stats.
|
|
5
|
+
//
|
|
6
|
+
// Use this to gate rendering decisions ("does this model need the ARKit remap?"),
|
|
7
|
+
// power the avatar creator's "compatibility report", and feed the Unity/Unreal
|
|
8
|
+
// SDK manifests.
|
|
9
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
10
|
+
exports.walkAvatarSchema = walkAvatarSchema;
|
|
11
|
+
const visemes_1 = require("./visemes");
|
|
12
|
+
// ─── Constants ───────────────────────────────────────────────────────────────
|
|
13
|
+
// Minimum humanoid bones expected on any rigged avatar.
|
|
14
|
+
const REQUIRED_HUMANOID_BONES = [
|
|
15
|
+
'Hips', 'Spine', 'Head', 'Neck',
|
|
16
|
+
'LeftUpperArm', 'RightUpperArm',
|
|
17
|
+
'LeftUpperLeg', 'RightUpperLeg',
|
|
18
|
+
];
|
|
19
|
+
// ARKit mouth shapes that indicate a real ARKit rig (not just jawOpen).
|
|
20
|
+
const ARKIT_MOUTH_SHAPES = [
|
|
21
|
+
'jawOpen', 'mouthFunnel', 'mouthPucker', 'mouthSmileLeft', 'mouthSmileRight',
|
|
22
|
+
'mouthFrownLeft', 'mouthFrownRight', 'mouthStretchLeft', 'mouthStretchRight',
|
|
23
|
+
'mouthRollLower', 'mouthRollUpper', 'mouthShrugLower', 'mouthShrugUpper',
|
|
24
|
+
'mouthLowerDownLeft', 'mouthLowerDownRight', 'mouthUpperUpLeft', 'mouthUpperUpRight',
|
|
25
|
+
'mouthClose', 'mouthDimpleLeft', 'mouthDimpleRight', 'mouthPressLeft', 'mouthPressRight',
|
|
26
|
+
];
|
|
27
|
+
// ─── Walker ──────────────────────────────────────────────────────────────────
|
|
28
|
+
/**
|
|
29
|
+
* Scan a loaded Three.js Object3D scene and return a structured compatibility
|
|
30
|
+
* report for lip-sync, skeleton, and mesh quality.
|
|
31
|
+
*
|
|
32
|
+
* @param scene - The root Object3D from a GLTFLoader result (gltf.scene)
|
|
33
|
+
*/
|
|
34
|
+
function walkAvatarSchema(scene) {
|
|
35
|
+
const allMorphNames = new Set();
|
|
36
|
+
const boneNames = new Set();
|
|
37
|
+
let hasRig = false;
|
|
38
|
+
let totalTris = 0;
|
|
39
|
+
let meshCount = 0;
|
|
40
|
+
scene.traverse((obj) => {
|
|
41
|
+
// Collect bone names from SkinnedMesh skeletons
|
|
42
|
+
if (obj.isSkinnedMesh) {
|
|
43
|
+
hasRig = true;
|
|
44
|
+
if (obj.skeleton?.bones) {
|
|
45
|
+
for (const bone of obj.skeleton.bones) {
|
|
46
|
+
if (bone.name)
|
|
47
|
+
boneNames.add(bone.name);
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
// Also collect from Bone objects directly
|
|
52
|
+
if (obj.isBone && obj.name) {
|
|
53
|
+
boneNames.add(obj.name);
|
|
54
|
+
}
|
|
55
|
+
// Collect morph targets and mesh stats
|
|
56
|
+
if (obj.isMesh && obj.morphTargetDictionary) {
|
|
57
|
+
for (const name of Object.keys(obj.morphTargetDictionary)) {
|
|
58
|
+
allMorphNames.add(name);
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
if (obj.isMesh && obj.geometry) {
|
|
62
|
+
meshCount++;
|
|
63
|
+
const geo = obj.geometry;
|
|
64
|
+
if (geo.index) {
|
|
65
|
+
totalTris += geo.index.count / 3;
|
|
66
|
+
}
|
|
67
|
+
else if (geo.attributes?.position) {
|
|
68
|
+
totalTris += geo.attributes.position.count / 3;
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
});
|
|
72
|
+
// ── Morph analysis ───────────────────────────────────────────────────────
|
|
73
|
+
const allMorphList = Array.from(allMorphNames);
|
|
74
|
+
const allLower = allMorphList.map(n => n.toLowerCase());
|
|
75
|
+
// Oculus visemes: look for viseme_PP, viseme_aa, etc.
|
|
76
|
+
const oculusFound = [];
|
|
77
|
+
for (const v of visemes_1.OCULUS_VISEMES) {
|
|
78
|
+
const target = `viseme_${v}`.toLowerCase();
|
|
79
|
+
if (allLower.some(n => n === target || n === v.toLowerCase())) {
|
|
80
|
+
oculusFound.push(v);
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
const hasFullOculusSet = oculusFound.length === visemes_1.OCULUS_VISEMES.length;
|
|
84
|
+
// ARKit shapes
|
|
85
|
+
const arkitFound = ARKIT_MOUTH_SHAPES.filter(shape => allLower.some(n => n === shape.toLowerCase()));
|
|
86
|
+
const hasArkitSet = arkitFound.length >= 10;
|
|
87
|
+
// Minimal: just jawOpen or mouthOpen
|
|
88
|
+
const hasMinimal = allLower.some(n => n === 'jawopen' || n === 'mouthopen' || n === 'viseme_aa');
|
|
89
|
+
let visemeTier;
|
|
90
|
+
if (hasFullOculusSet)
|
|
91
|
+
visemeTier = 'oculus';
|
|
92
|
+
else if (hasArkitSet)
|
|
93
|
+
visemeTier = 'arkit';
|
|
94
|
+
else if (hasMinimal)
|
|
95
|
+
visemeTier = 'minimal';
|
|
96
|
+
else
|
|
97
|
+
visemeTier = 'none';
|
|
98
|
+
// ── Skeleton analysis ────────────────────────────────────────────────────
|
|
99
|
+
const bonesLower = Array.from(boneNames).map(b => b.toLowerCase());
|
|
100
|
+
const missingHumanoidBones = REQUIRED_HUMANOID_BONES.filter(required => !bonesLower.some(b => b.includes(required.toLowerCase())));
|
|
101
|
+
const isHumanoid = hasRig && missingHumanoidBones.length === 0;
|
|
102
|
+
// ── Mesh stats ───────────────────────────────────────────────────────────
|
|
103
|
+
let lodTier;
|
|
104
|
+
if (totalTris > 70000)
|
|
105
|
+
lodTier = 'high';
|
|
106
|
+
else if (totalTris > 30000)
|
|
107
|
+
lodTier = 'medium';
|
|
108
|
+
else if (totalTris > 8000)
|
|
109
|
+
lodTier = 'low';
|
|
110
|
+
else
|
|
111
|
+
lodTier = 'very-low';
|
|
112
|
+
return {
|
|
113
|
+
morphs: {
|
|
114
|
+
all: allMorphList,
|
|
115
|
+
oculusVisemes: oculusFound,
|
|
116
|
+
hasFullOculusSet,
|
|
117
|
+
arkitShapes: arkitFound,
|
|
118
|
+
hasArkitSet,
|
|
119
|
+
visemeTier,
|
|
120
|
+
},
|
|
121
|
+
skeleton: {
|
|
122
|
+
hasRig,
|
|
123
|
+
bones: Array.from(boneNames),
|
|
124
|
+
isHumanoid,
|
|
125
|
+
missingHumanoidBones,
|
|
126
|
+
},
|
|
127
|
+
mesh: {
|
|
128
|
+
triangles: Math.round(totalTris),
|
|
129
|
+
meshCount,
|
|
130
|
+
lodTier,
|
|
131
|
+
},
|
|
132
|
+
isReadyForLipSync: visemeTier === 'oculus' || visemeTier === 'arkit',
|
|
133
|
+
};
|
|
134
|
+
}
|
|
@@ -31,3 +31,34 @@ export interface AgentVisemePayload {
|
|
|
31
31
|
cues: VisemeCue[];
|
|
32
32
|
}
|
|
33
33
|
export declare const OCULUS_VISEMES: OculusViseme[];
|
|
34
|
+
export type ArkitBlendShape = 'jawOpen' | 'jawForward' | 'jawLeft' | 'jawRight' | 'mouthClose' | 'mouthFunnel' | 'mouthPucker' | 'mouthLeft' | 'mouthRight' | 'mouthSmileLeft' | 'mouthSmileRight' | 'mouthFrownLeft' | 'mouthFrownRight' | 'mouthDimpleLeft' | 'mouthDimpleRight' | 'mouthStretchLeft' | 'mouthStretchRight' | 'mouthRollLower' | 'mouthRollUpper' | 'mouthShrugLower' | 'mouthShrugUpper' | 'mouthPressLeft' | 'mouthPressRight' | 'mouthLowerDownLeft' | 'mouthLowerDownRight' | 'mouthUpperUpLeft' | 'mouthUpperUpRight' | 'tongueOut';
|
|
35
|
+
/**
|
|
36
|
+
* Each Oculus viseme expressed as a weighted combination of ARKit blend shapes.
|
|
37
|
+
* Weights are in 0–1 range. Apply by multiplying each ARKit shape's influence.
|
|
38
|
+
*
|
|
39
|
+
* Sources:
|
|
40
|
+
* - OVR LipSync viseme reference (phonetic articulation targets)
|
|
41
|
+
* - Empirical tuning against RPM and face-squeeze avatars
|
|
42
|
+
*/
|
|
43
|
+
export declare const ARKIT_TO_OCULUS: Record<OculusViseme, Partial<Record<ArkitBlendShape, number>>>;
|
|
44
|
+
/**
|
|
45
|
+
* Remap a set of ARKit blend shape weights to Oculus viseme weights.
|
|
46
|
+
*
|
|
47
|
+
* Given current ARKit influences (e.g. from face tracking or morph target state),
|
|
48
|
+
* returns the equivalent Oculus viseme weights. Each viseme's output weight is the
|
|
49
|
+
* weighted average of its constituent ARKit shapes, clamped to 0–1.
|
|
50
|
+
*
|
|
51
|
+
* Use cases:
|
|
52
|
+
* - Runtime: drive Oculus viseme morphs from ARKit face tracking data
|
|
53
|
+
* - Bake-time: compute Oculus morph target deltas from ARKit shape keys in a GLB
|
|
54
|
+
*/
|
|
55
|
+
export declare function remapArkitToOculus(arkitWeights: Partial<Record<ArkitBlendShape, number>>): Record<OculusViseme, number>;
|
|
56
|
+
/**
|
|
57
|
+
* For a given Oculus viseme, return the ARKit blend shape weights that produce it.
|
|
58
|
+
* This is the "forward" direction — useful when baking Oculus morph targets into
|
|
59
|
+
* a GLB that only has ARKit shapes.
|
|
60
|
+
*
|
|
61
|
+
* Returns the raw recipe weights (not normalized). To bake into a morph target,
|
|
62
|
+
* set each ARKit shape to the returned weight and snapshot the mesh delta.
|
|
63
|
+
*/
|
|
64
|
+
export declare function getArkitWeightsForViseme(viseme: OculusViseme): Partial<Record<ArkitBlendShape, number>>;
|
|
@@ -1,6 +1,72 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.OCULUS_VISEMES = void 0;
|
|
3
|
+
exports.ARKIT_TO_OCULUS = exports.OCULUS_VISEMES = void 0;
|
|
4
|
+
exports.remapArkitToOculus = remapArkitToOculus;
|
|
5
|
+
exports.getArkitWeightsForViseme = getArkitWeightsForViseme;
|
|
4
6
|
exports.OCULUS_VISEMES = [
|
|
5
7
|
'sil', 'PP', 'FF', 'TH', 'DD', 'kk', 'CH', 'SS', 'nn', 'RR', 'aa', 'ee', 'ih', 'oh', 'ou',
|
|
6
8
|
];
|
|
9
|
+
/**
|
|
10
|
+
* Each Oculus viseme expressed as a weighted combination of ARKit blend shapes.
|
|
11
|
+
* Weights are in 0–1 range. Apply by multiplying each ARKit shape's influence.
|
|
12
|
+
*
|
|
13
|
+
* Sources:
|
|
14
|
+
* - OVR LipSync viseme reference (phonetic articulation targets)
|
|
15
|
+
* - Empirical tuning against RPM and face-squeeze avatars
|
|
16
|
+
*/
|
|
17
|
+
exports.ARKIT_TO_OCULUS = {
|
|
18
|
+
sil: { mouthClose: 0.1 },
|
|
19
|
+
PP: { mouthClose: 0.8, mouthPucker: 0.3 },
|
|
20
|
+
FF: { mouthLowerDownLeft: 0.5, mouthLowerDownRight: 0.5, mouthRollLower: 0.4 },
|
|
21
|
+
TH: { tongueOut: 0.6, jawOpen: 0.15, mouthLowerDownLeft: 0.2, mouthLowerDownRight: 0.2 },
|
|
22
|
+
DD: { mouthShrugUpper: 0.5, jawOpen: 0.2, mouthUpperUpLeft: 0.3, mouthUpperUpRight: 0.3 },
|
|
23
|
+
kk: { jawOpen: 0.25, mouthStretchLeft: 0.4, mouthStretchRight: 0.4, mouthShrugUpper: 0.2 },
|
|
24
|
+
CH: { mouthStretchLeft: 0.5, mouthStretchRight: 0.5, jawOpen: 0.3, mouthFunnel: 0.2 },
|
|
25
|
+
SS: { mouthStretchLeft: 0.35, mouthStretchRight: 0.35, mouthClose: 0.3 },
|
|
26
|
+
nn: { jawOpen: 0.15, mouthDimpleLeft: 0.4, mouthDimpleRight: 0.4, mouthShrugLower: 0.2 },
|
|
27
|
+
RR: { mouthPucker: 0.4, mouthFunnel: 0.3, jawOpen: 0.15, mouthRollLower: 0.2 },
|
|
28
|
+
aa: { jawOpen: 0.7, mouthLowerDownLeft: 0.4, mouthLowerDownRight: 0.4 },
|
|
29
|
+
ee: { mouthSmileLeft: 0.6, mouthSmileRight: 0.6, jawOpen: 0.2 },
|
|
30
|
+
ih: { mouthSmileLeft: 0.4, mouthSmileRight: 0.4, jawOpen: 0.1 },
|
|
31
|
+
oh: { jawOpen: 0.4, mouthFunnel: 0.8 },
|
|
32
|
+
ou: { mouthPucker: 0.9, mouthRollLower: 0.3 },
|
|
33
|
+
};
|
|
34
|
+
/**
|
|
35
|
+
* Remap a set of ARKit blend shape weights to Oculus viseme weights.
|
|
36
|
+
*
|
|
37
|
+
* Given current ARKit influences (e.g. from face tracking or morph target state),
|
|
38
|
+
* returns the equivalent Oculus viseme weights. Each viseme's output weight is the
|
|
39
|
+
* weighted average of its constituent ARKit shapes, clamped to 0–1.
|
|
40
|
+
*
|
|
41
|
+
* Use cases:
|
|
42
|
+
* - Runtime: drive Oculus viseme morphs from ARKit face tracking data
|
|
43
|
+
* - Bake-time: compute Oculus morph target deltas from ARKit shape keys in a GLB
|
|
44
|
+
*/
|
|
45
|
+
function remapArkitToOculus(arkitWeights) {
|
|
46
|
+
const result = {};
|
|
47
|
+
for (const viseme of exports.OCULUS_VISEMES) {
|
|
48
|
+
const recipe = exports.ARKIT_TO_OCULUS[viseme];
|
|
49
|
+
let sum = 0;
|
|
50
|
+
let coeffSum = 0;
|
|
51
|
+
for (const [shape, coeff] of Object.entries(recipe)) {
|
|
52
|
+
const input = arkitWeights[shape] ?? 0;
|
|
53
|
+
sum += input * coeff;
|
|
54
|
+
coeffSum += coeff;
|
|
55
|
+
}
|
|
56
|
+
// Normalize by total coefficient weight so recipes with more shapes
|
|
57
|
+
// don't produce disproportionately high output
|
|
58
|
+
result[viseme] = coeffSum > 0 ? Math.min(1, Math.max(0, sum / coeffSum)) : 0;
|
|
59
|
+
}
|
|
60
|
+
return result;
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* For a given Oculus viseme, return the ARKit blend shape weights that produce it.
|
|
64
|
+
* This is the "forward" direction — useful when baking Oculus morph targets into
|
|
65
|
+
* a GLB that only has ARKit shapes.
|
|
66
|
+
*
|
|
67
|
+
* Returns the raw recipe weights (not normalized). To bake into a morph target,
|
|
68
|
+
* set each ARKit shape to the returned weight and snapshot the mesh delta.
|
|
69
|
+
*/
|
|
70
|
+
function getArkitWeightsForViseme(viseme) {
|
|
71
|
+
return { ...exports.ARKIT_TO_OCULUS[viseme] };
|
|
72
|
+
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.AvatarCanvas =
|
|
3
|
+
exports.AvatarCanvas = AvatarCanvas;
|
|
4
4
|
const jsx_runtime_1 = require("react/jsx-runtime");
|
|
5
5
|
// @ts-nocheck
|
|
6
6
|
const drei_1 = require("@react-three/drei");
|
|
@@ -68,4 +68,3 @@ function AvatarCanvas({ avatarUrl, equipped = [], placements = {}, editingAssetI
|
|
|
68
68
|
return null;
|
|
69
69
|
})] }) }) })) : ((0, jsx_runtime_1.jsx)("div", { className: "flex items-center justify-center h-full text-muted-foreground", children: (0, jsx_runtime_1.jsx)("p", { children: "Select a base avatar to get started" }) }))] }));
|
|
70
70
|
}
|
|
71
|
-
exports.AvatarCanvas = AvatarCanvas;
|
|
@@ -15,15 +15,25 @@ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (
|
|
|
15
15
|
}) : function(o, v) {
|
|
16
16
|
o["default"] = v;
|
|
17
17
|
});
|
|
18
|
-
var __importStar = (this && this.__importStar) || function (
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
};
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
25
35
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
26
|
-
exports.AvatarEditor =
|
|
36
|
+
exports.AvatarEditor = AvatarEditor;
|
|
27
37
|
const jsx_runtime_1 = require("react/jsx-runtime");
|
|
28
38
|
/**
|
|
29
39
|
* Native avatar placement editor.
|
|
@@ -86,7 +96,6 @@ function AvatarEditor({ avatarUrl, activeAssetId = null, onPlacementChange, styl
|
|
|
86
96
|
}, [activeAssetId, avatarScene, onPlacementChange]);
|
|
87
97
|
return ((0, jsx_runtime_1.jsxs)(react_native_1.View, { style: [styles.container, style], onLayout: (e) => { layoutRef.current = e.nativeEvent.layout; }, children: [(0, jsx_runtime_1.jsx)(wgpu_1.WgpuAvatar, { avatarUrl: avatarUrl, style: react_native_1.StyleSheet.absoluteFill, onSceneReady: handleSceneReady }), (0, jsx_runtime_1.jsx)(react_native_1.View, { style: react_native_1.StyleSheet.absoluteFill, onTouchEnd: handleTap, pointerEvents: activeAssetId ? 'auto' : 'none' })] }));
|
|
88
98
|
}
|
|
89
|
-
exports.AvatarEditor = AvatarEditor;
|
|
90
99
|
exports.default = AvatarEditor;
|
|
91
100
|
const styles = react_native_1.StyleSheet.create({
|
|
92
101
|
container: { flex: 1, overflow: 'hidden' },
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.AvatarModel =
|
|
3
|
+
exports.AvatarModel = AvatarModel;
|
|
4
4
|
const jsx_runtime_1 = require("react/jsx-runtime");
|
|
5
5
|
// @ts-nocheck
|
|
6
6
|
const drei_1 = require("@react-three/drei");
|
|
@@ -40,4 +40,3 @@ function AvatarModel({ url, scale = 1, onSkeletonReady, onBoundsReady, }) {
|
|
|
40
40
|
});
|
|
41
41
|
return (0, jsx_runtime_1.jsx)("primitive", { ref: groupRef, object: scene, scale: scale });
|
|
42
42
|
}
|
|
43
|
-
exports.AvatarModel = AvatarModel;
|
|
@@ -15,15 +15,26 @@ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (
|
|
|
15
15
|
}) : function(o, v) {
|
|
16
16
|
o["default"] = v;
|
|
17
17
|
});
|
|
18
|
-
var __importStar = (this && this.__importStar) || function (
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
};
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
25
35
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
26
|
-
exports.
|
|
36
|
+
exports.FACE_SQUEEZE_LOCAL_MODULE = void 0;
|
|
37
|
+
exports.FaceSqueezeEditor = FaceSqueezeEditor;
|
|
27
38
|
const jsx_runtime_1 = require("react/jsx-runtime");
|
|
28
39
|
/**
|
|
29
40
|
* WgpuFaceSqueezeEditor — R3F/Three.js port of FaceSqueezeEditor.
|
|
@@ -534,7 +545,6 @@ function FaceSqueezeEditor({ onClose }) {
|
|
|
534
545
|
const handleReady = (0, react_1.useCallback)(() => setIsReady(true), []);
|
|
535
546
|
return ((0, jsx_runtime_1.jsx)(react_native_gesture_handler_1.GestureHandlerRootView, { style: { flex: 1 }, children: (0, jsx_runtime_1.jsxs)(react_native_1.View, { style: styles.container, children: [(0, jsx_runtime_1.jsx)(react_native_1.View, { style: styles.scene, children: localUri ? ((0, jsx_runtime_1.jsx)(native_1.Canvas, { style: react_native_1.StyleSheet.absoluteFill, camera: { fov: 34, position: [0, 1.55, 1.05], near: 0.01, far: 50 }, gl: { antialias: true, alpha: false }, onCreated: ({ gl }) => gl.setClearColor(new THREE.Color('#0a0a0a')), children: (0, jsx_runtime_1.jsx)(AvatarScene, { uri: localUri, morphState: morphState, onReady: handleReady }) })) : ((0, jsx_runtime_1.jsx)(react_native_1.View, { style: styles.loading, children: (0, jsx_runtime_1.jsx)(react_native_1.Text, { style: styles.loadingText, children: "Loading face\u2026" }) })) }), (0, jsx_runtime_1.jsx)(react_native_gesture_handler_1.GestureDetector, { gesture: composed, children: (0, jsx_runtime_1.jsx)(react_native_1.View, { style: react_native_1.StyleSheet.absoluteFill, onLayout: onLayout, collapsable: false }) }), (0, jsx_runtime_1.jsx)(react_native_1.Pressable, { style: styles.closeBtn, onPress: onClose, children: (0, jsx_runtime_1.jsx)(react_native_1.Text, { style: styles.closeBtnText, children: "\u2715 Done squeezing" }) }), (0, jsx_runtime_1.jsx)(react_native_1.Text, { style: styles.hint, children: "Drag to stretch \u00B7 Pinch lips \u00B7 Poke an eye \uD83D\uDC41" })] }) }));
|
|
536
547
|
}
|
|
537
|
-
exports.FaceSqueezeEditor = FaceSqueezeEditor;
|
|
538
548
|
const styles = react_native_1.StyleSheet.create({
|
|
539
549
|
container: { flex: 1, backgroundColor: '#0a0a0a' },
|
|
540
550
|
scene: { flex: 1 },
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.
|
|
3
|
+
exports.FACE_SQUEEZE_LOCAL_MODULE = void 0;
|
|
4
|
+
exports.FaceSqueezeEditor = FaceSqueezeEditor;
|
|
4
5
|
const jsx_runtime_1 = require("react/jsx-runtime");
|
|
5
6
|
const react_native_1 = require("react-native");
|
|
6
7
|
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
|
@@ -8,7 +9,6 @@ exports.FACE_SQUEEZE_LOCAL_MODULE = require('../assets/face-squeeze-local.glb');
|
|
|
8
9
|
function FaceSqueezeEditor({ onClose }) {
|
|
9
10
|
return ((0, jsx_runtime_1.jsx)(react_native_1.View, { style: styles.container, children: (0, jsx_runtime_1.jsxs)(react_native_1.View, { style: styles.card, children: [(0, jsx_runtime_1.jsx)(react_native_1.Text, { style: styles.title, children: "Face squeeze editor is mobile-only." }), (0, jsx_runtime_1.jsx)(react_native_1.Text, { style: styles.subtitle, children: "This placeholder keeps web builds safe while preserving native functionality." }), (0, jsx_runtime_1.jsx)(react_native_1.Pressable, { style: styles.closeButton, onPress: onClose, children: (0, jsx_runtime_1.jsx)(react_native_1.Text, { style: styles.closeButtonText, children: "Close" }) })] }) }));
|
|
10
11
|
}
|
|
11
|
-
exports.FaceSqueezeEditor = FaceSqueezeEditor;
|
|
12
12
|
const styles = react_native_1.StyleSheet.create({
|
|
13
13
|
container: {
|
|
14
14
|
flex: 1,
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.RigidAccessory =
|
|
3
|
+
exports.RigidAccessory = RigidAccessory;
|
|
4
4
|
const jsx_runtime_1 = require("react/jsx-runtime");
|
|
5
5
|
// @ts-nocheck
|
|
6
6
|
const drei_1 = require("@react-three/drei");
|
|
@@ -76,4 +76,3 @@ function RigidAccessory({ assetId, url, avatarScene, attachBone, offsetPosition,
|
|
|
76
76
|
}
|
|
77
77
|
return (0, jsx_runtime_1.jsx)("primitive", { object: clone });
|
|
78
78
|
}
|
|
79
|
-
exports.RigidAccessory = RigidAccessory;
|
|
@@ -15,15 +15,25 @@ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (
|
|
|
15
15
|
}) : function(o, v) {
|
|
16
16
|
o["default"] = v;
|
|
17
17
|
});
|
|
18
|
-
var __importStar = (this && this.__importStar) || function (
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
};
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
25
35
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
26
|
-
exports.SkinnedClothing =
|
|
36
|
+
exports.SkinnedClothing = SkinnedClothing;
|
|
27
37
|
const jsx_runtime_1 = require("react/jsx-runtime");
|
|
28
38
|
// @ts-nocheck
|
|
29
39
|
const drei_1 = require("@react-three/drei");
|
|
@@ -113,4 +123,3 @@ function SkinnedClothing({ url, avatarSkeleton }) {
|
|
|
113
123
|
}, [scene, avatarSkeleton]);
|
|
114
124
|
return (0, jsx_runtime_1.jsx)("group", { ref: groupRef });
|
|
115
125
|
}
|
|
116
|
-
exports.SkinnedClothing = SkinnedClothing;
|