talking-head-studio 0.3.4 → 0.3.6

package/dist/html.js

@@ -202,6 +202,7 @@ async function loadStaticFallback(loadedAvatarUrl) {
       staticModel.position.sub(scaledCenter);
 
       applyAccessories(pendingAccessoriesList);
+      motionInitFromRoot(staticModel);
       emitLoading('ready', 100);
       window.ReactNativeWebView?.postMessage(JSON.stringify({ type: 'ready' }));
 
@@ -215,6 +216,7 @@ async function loadStaticFallback(loadedAvatarUrl) {
         const delta = clock.getDelta();
         if (staticMixer) staticMixer.update(delta);
         tickVisemeDecay();
+        applyMotionBones();
         controls.update();
         renderer.render(scene, camera);
       });
@@ -320,25 +322,22 @@ async function init() {
             }
           };
           const headaudioUpdate = headaudio.update.bind(headaudio);
-          head.opt.update = (dt) => { headaudioUpdate(dt); tickVisemeDecay(); };
+          head.opt.update = (dt) => { headaudioUpdate(dt); tickVisemeDecay(); applyMotionBones(); };
           log('HeadAudio ready (phoneme lip sync)');
         } else {
           log('HeadAudio skipped: AudioWorklet not supported in this WebView. Use sendViseme() from native TTS callbacks.');
-          head.opt.update = () => tickVisemeDecay();
+          head.opt.update = () => { tickVisemeDecay(); applyMotionBones(); };
         }
       } catch (err) {
         log('HeadAudio unavailable, viseme/amplitude fallback active: ' + err.message);
-        head.opt.update = () => tickVisemeDecay();
+        head.opt.update = () => { tickVisemeDecay(); applyMotionBones(); };
       }
 
       startAudioInterception();
       log('[ACC] init() complete, calling applyAccessories with ' + pendingAccessoriesList.length + ' pending items');
       applyAccessories(pendingAccessoriesList);
+      if (head && head.armature) motionInitFromRoot(head.armature);
       emitLoading('ready', 100);
-      // Register motionEngine so dispatchMotion('wave_right') calls head.playGesture
-      (window as any).motionEngine = {
-        play: async (name: string) => { if (head) await (head as any).playGesture(name); },
-      };
       window.ReactNativeWebView?.postMessage(JSON.stringify({ type: 'ready' }));
     }
   } catch (err) {
@@ -818,6 +817,263 @@ async function applyAccessories(accessoriesList) {
   }
 }
 
+/* ── MotionEngine ────────────────────────────────────────────────────────── */
+/* Quaternion math helpers (no Three.js dep required) */
+const QM = {
+  fromAxisAngle(ax, ay, az, angle) {
+    const s = Math.sin(angle / 2);
+    return { x: ax*s, y: ay*s, z: az*s, w: Math.cos(angle/2) };
+  },
+  multiply(a, b) {
+    return {
+      x:  a.w*b.x + a.x*b.w + a.y*b.z - a.z*b.y,
+      y:  a.w*b.y - a.x*b.z + a.y*b.w + a.z*b.x,
+      z:  a.w*b.z + a.x*b.y - a.y*b.x + a.z*b.w,
+      w:  a.w*b.w - a.x*b.x - a.y*b.y - a.z*b.z,
+    };
+  },
+  applyTo(bone, q) {
+    bone.quaternion.set(q.x, q.y, q.z, q.w);
+  },
+  copyFrom(bone) {
+    const q = bone.quaternion;
+    return { x: q.x, y: q.y, z: q.z, w: q.w };
+  },
+};
+
+const MOTION_DEFS = {
+  groove: {
+    bpm: 120,
+    smile: 0.55,
+    label: 'Groove',
+    bones: {
+      hips:      [{ ax:0, ay:0, az:1, amp:0.10, freq:1.0, phase:0       },
+                  { ax:0, ay:1, az:0, amp:0.05, freq:0.5, phase:0       }],
+      spine:     [{ ax:0, ay:0, az:1, amp:0.08, freq:1.0, phase:0.3     },
+                  { ax:1, ay:0, az:0, amp:0.03, freq:2.0, phase:0       }],
+      spine2:    [{ ax:0, ay:0, az:1, amp:0.06, freq:1.0, phase:0.6     }],
+      neck:      [{ ax:0, ay:0, az:1, amp:0.05, freq:1.0, phase:Math.PI }],
+      head:      [{ ax:1, ay:0, az:0, amp:0.06, freq:2.0, phase:0       },
+                  { ax:0, ay:0, az:1, amp:0.04, freq:1.0, phase:Math.PI }],
+      leftArm:   [{ ax:0, ay:0, az:1, amp:0.22, freq:1.0, phase:Math.PI/2  }],
+      rightArm:  [{ ax:0, ay:0, az:1, amp:0.22, freq:1.0, phase:-Math.PI/2 }],
+    },
+  },
+  wave: {
+    bpm: 90,
+    smile: 0.75,
+    label: 'Wave',
+    bones: {
+      hips:      [{ ax:0, ay:0, az:1, amp:0.04, freq:0.5, phase:0 }],
+      spine:     [{ ax:0, ay:0, az:1, amp:0.04, freq:0.5, phase:0.2 }],
+      head:      [{ ax:0, ay:1, az:0, amp:0.12, freq:0.5, phase:0   },
+                  { ax:0, ay:0, az:1, amp:0.04, freq:0.5, phase:0.5 }],
+      rightShoulder:[{ ax:0, ay:0, az:1, amp:-0.6, freq:0,   phase:0 }],
+      rightArm:  [{ ax:0, ay:0, az:1, amp:-0.55, freq:0,  phase:0 },
+                  { ax:1, ay:0, az:0, amp:0.35,  freq:1.5, phase:0 }],
+      rightForeArm:[{ ax:1, ay:0, az:0, amp:0.4, freq:1.5, phase:Math.PI/3 }],
+    },
+  },
+  nod: {
+    bpm: 100,
+    smile: 0.4,
+    label: 'Head Bop',
+    bones: {
+      hips:   [{ ax:0, ay:0, az:1, amp:0.04, freq:0.83, phase:0 }],
+      spine:  [{ ax:1, ay:0, az:0, amp:0.04, freq:0.83, phase:0.2 },
+               { ax:0, ay:0, az:1, amp:0.03, freq:0.83, phase:0.4 }],
+      neck:   [{ ax:1, ay:0, az:0, amp:0.12, freq:1.67, phase:0 }],
+      head:   [{ ax:1, ay:0, az:0, amp:0.16, freq:1.67, phase:Math.PI/6 },
+               { ax:0, ay:1, az:0, amp:0.05, freq:0.83, phase:0 }],
+      leftArm:[{ ax:0, ay:0, az:1, amp:0.12, freq:0.83, phase:0 }],
+      rightArm:[{ ax:0, ay:0, az:1, amp:0.12, freq:0.83, phase:Math.PI }],
+    },
+  },
+  idle: {
+    bpm: 60,
+    smile: 0.15,
+    label: 'Idle Sway',
+    bones: {
+      hips:   [{ ax:0, ay:0, az:1, amp:0.035, freq:0.5, phase:0 }],
+      spine:  [{ ax:0, ay:0, az:1, amp:0.028, freq:0.5, phase:0.3 },
+               { ax:1, ay:0, az:0, amp:0.012, freq:0.25, phase:0 }],
+      spine2: [{ ax:0, ay:0, az:1, amp:0.018, freq:0.5, phase:0.5 }],
+      head:   [{ ax:1, ay:0, az:0, amp:0.025, freq:0.25, phase:0  },
+               { ax:0, ay:1, az:0, amp:0.015, freq:0.5,  phase:0.8}],
+    },
+  },
+  celebrate: {
+    bpm: 140,
+    smile: 0.95,
+    label: 'Celebrate',
+    bones: {
+      hips:    [{ ax:0, ay:0, az:1, amp:0.14, freq:1.17, phase:0       },
+                { ax:1, ay:0, az:0, amp:0.06, freq:2.33, phase:0       }],
+      spine:   [{ ax:0, ay:0, az:1, amp:0.10, freq:1.17, phase:0.4     },
+                { ax:1, ay:0, az:0, amp:0.05, freq:2.33, phase:0.2     }],
+      spine2:  [{ ax:0, ay:0, az:1, amp:0.07, freq:1.17, phase:0.7     }],
+      neck:    [{ ax:1, ay:0, az:0, amp:0.06, freq:2.33, phase:0       }],
+      head:    [{ ax:1, ay:0, az:0, amp:0.10, freq:2.33, phase:0.15    },
+                { ax:0, ay:0, az:1, amp:0.06, freq:1.17, phase:Math.PI }],
+      leftShoulder: [{ ax:0, ay:0, az:1, amp: 0.5, freq:0, phase:0    }],
+      rightShoulder:[{ ax:0, ay:0, az:1, amp:-0.5, freq:0, phase:0    }],
+      leftArm: [{ ax:0, ay:0, az:1, amp: 0.55, freq:0,    phase:0     },
+                { ax:1, ay:0, az:0, amp:0.30,  freq:2.33, phase:0     }],
+      rightArm:[{ ax:0, ay:0, az:1, amp:-0.55, freq:0,    phase:0     },
+                { ax:1, ay:0, az:0, amp:0.30,  freq:2.33, phase:Math.PI }],
+      leftForeArm: [{ ax:1, ay:0, az:0, amp:0.25, freq:2.33, phase:0.4 }],
+      rightForeArm:[{ ax:1, ay:0, az:0, amp:0.25, freq:2.33, phase:0.4 }],
+    },
+  },
+};
+
+/* Runtime state */
+let motionActive = false;
+let motionKey = null;
+let motionStartTime = null;
+let motionBones = {};
+let motionRestQuats = {};
+let motionSmileTargets = [];
+let motionBeatTimer = null;
+let motionBeatIndex = 0;
+
+/* Bone name map: logical key -> keywords to match in armature.
+   norm() strips punctuation and lowercases, so RPM names like
+   LeftShoulder, RightArm, LeftForeArm etc. all match correctly. */
+const BONE_SEARCH = {
+  hips:          ['hips','pelvis','hip','root'],
+  spine:         ['spine','spine0','spine_0','spine_01','spine1_'],
+  spine2:        ['spine2','spine_02','upperchest','chest','spine1'],
+  neck:          ['neck'],
+  head:          ['head'],
+  leftShoulder:  ['leftshoulder','l_shoulder','shoulderleft','leftclavicle','l_clavicle'],
+  rightShoulder: ['rightshoulder','r_shoulder','shoulderright','rightclavicle','r_clavicle'],
+  leftArm:       ['leftarm','l_arm','armleft','leftupperarm','l_upperarm'],
+  rightArm:      ['rightarm','r_arm','armright','rightupperarm','r_upperarm'],
+  leftForeArm:   ['leftforearm','l_forearm','forearmleft','leftlowerarm'],
+  rightForeArm:  ['rightforearm','r_forearm','forearmright','rightlowerarm'],
+};
+
+function motionScanBones(root) {
+  motionBones = {}; motionRestQuats = {};
+  if (!root) return;
+  const norm = s => s.toLowerCase().replace(/[_\\s\\-\\.]/g, '');
+  root.traverse(child => {
+    if (!child.isBone) return;
+    const lk = norm(child.name);
+    for (const [key, kws] of Object.entries(BONE_SEARCH)) {
+      if (motionBones[key]) continue;
+      if (kws.some(kw => lk.includes(norm(kw)))) {
+        motionBones[key] = child;
+        motionRestQuats[key] = QM.copyFrom(child);
+      }
+    }
+  });
+  const found = Object.keys(motionBones);
+  log('MotionEngine: ' + (found.length === 0
+    ? 'No rigged bones detected — motion engine needs a humanoid skeleton.'
+    : found.length + ' bones mapped: ' + found.join(', ') + '.'));
+}
+
+function motionScanSmile(root) {
+  motionSmileTargets = [];
+  if (!root) return;
+  root.traverse(child => {
+    if (!child.isMesh || !child.morphTargetDictionary) return;
+    Object.keys(child.morphTargetDictionary).forEach(name => {
+      const lk = name.toLowerCase();
+      if (lk.includes('smile') || lk.includes('happy') || lk.includes('joy') || lk.includes('mouthsmile')) {
+        motionSmileTargets.push({ mesh: child, index: child.morphTargetDictionary[name] });
+      }
+    });
+  });
+}
+
+function motionInitFromRoot(root) {
+  motionScanBones(root);
+  motionScanSmile(root);
+}
+
+function motionSetSmile(v) {
+  motionSmileTargets.forEach(t => { t.mesh.morphTargetInfluences[t.index] = v; });
+}
+
+/* BPM beat indicator — no-ops in WebView (no beat UI) */
+function motionStartBeat(bpm) {
+  clearInterval(motionBeatTimer);
+  motionBeatIndex = 0;
+  /* beat UI elements not present in WebView — intentional no-op */
+}
+
+function motionStopBeat() {
+  clearInterval(motionBeatTimer);
+  motionBeatTimer = null;
+}
+
+/* ── applyMotionBones ────────────────────────────────────────
+   Called every frame from inside the render loop
+   (fallback: before renderer.render; TH: inside head.opt.update).
+   Runs AFTER TH has written its own bones so our values are
+   the last thing written before the draw call.
+ ─────────────────────────────────────────────────────────── */
+function applyMotionBones() {
+  if (!motionActive) return;
+  const def = MOTION_DEFS[motionKey];
+  if (!def) return;
+  const tSec = (performance.now() - motionStartTime) / 1000;
+
+  for (const [boneName, oscillators] of Object.entries(def.bones)) {
+    const bone = motionBones[boneName];
+    if (!bone) continue;
+    const rest = motionRestQuats[boneName];
+    let q = { ...rest };
+
+    for (const osc of oscillators) {
+      const angle = osc.freq === 0
+        ? osc.amp
+        : osc.amp * Math.sin(2 * Math.PI * osc.freq * tSec + osc.phase);
+      q = QM.multiply(q, QM.fromAxisAngle(osc.ax, osc.ay, osc.az, angle));
+    }
+
+    bone.quaternion.set(q.x, q.y, q.z, q.w);
+  }
+}
+
+window.playMotion = function(key) {
+  const def = MOTION_DEFS[key];
+  if (!def) { log('MotionEngine: unknown motion key: ' + key); return; }
+  const root = (head && head.armature) ? head.armature : staticModel;
+  if (!root) { log('MotionEngine: no model loaded'); return; }
+
+  if (Object.keys(motionBones).length === 0) motionInitFromRoot(root);
+
+  window.stopMotion(false);
+
+  motionKey = key;
+  motionActive = true;
+  motionStartTime = performance.now();
+
+  motionStartBeat(def.bpm);
+  motionSetSmile(def.smile);
+  log('MotionEngine: playing ' + def.label + ' @ ' + def.bpm + ' BPM');
+};
+
+window.stopMotion = function(restore) {
+  if (restore === undefined) restore = true;
+  motionActive = false;
+  motionStopBeat();
+
+  if (restore) {
+    for (const [key, bone] of Object.entries(motionBones)) {
+      const rest = motionRestQuats[key];
+      if (rest) bone.quaternion.set(rest.x, rest.y, rest.z, rest.w);
+    }
+    motionSetSmile(0);
+    log('MotionEngine: stopped');
+  }
+};
+/* ── End MotionEngine ─────────────────────────────────────── */
+
 function onIncomingMessage(event) {
   try {
     const msg = JSON.parse(event.data);
@@ -861,8 +1117,8 @@ function onIncomingMessage(event) {
     } else if (msg.type === 'set_accessories') {
       applyAccessories(msg.accessories || []);
     } else if (msg.type === 'motion' && typeof msg.name === 'string') {
-      if ((window as any).motionEngine) {
-        (window as any).motionEngine.play(msg.name).catch(() => {});
+      if (typeof window.playMotion === 'function') {
+        window.playMotion(msg.name);
       }
       window.ReactNativeWebView?.postMessage(JSON.stringify({ type: 'avatarState', state: 'motion:' + msg.name }));
       log('motion dispatched: ' + msg.name);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "talking-head-studio",
-  "version": "0.3.4",
+  "version": "0.3.6",
   "description": "Cross-platform 3D avatar component for React Native & web — lip-sync, gestures, accessories, and LLM integration. Powered by TalkingHead + Three.js.",
   "main": "dist/index.web.js",
   "browser": "dist/index.web.js",
@@ -41,6 +41,11 @@
       "react-native": "./dist/filament/index.js",
       "types": "./dist/filament/index.d.ts",
       "default": "./dist/filament/index.js"
+    },
+    "./filament/editor": {
+      "react-native": "./dist/filament/editor/index.js",
+      "types": "./dist/filament/editor/index.d.ts",
+      "default": "./dist/filament/editor/index.js"
     }
   },
   "files": [
@@ -139,6 +144,7 @@
     "@babel/core": "^7.29.0",
     "@babel/preset-env": "^7.29.0",
     "@babel/preset-typescript": "^7.28.5",
+    "@expo/vector-icons": "^15.1.1",
     "@react-native/babel-preset": "^0.84.1",
     "@testing-library/react-native": "^13.3.3",
     "@types/jest": "^30.0.0",
@@ -152,12 +158,16 @@
     "eslint-plugin-react": "^7.37.5",
     "eslint-plugin-react-hooks": "^7.0.1",
     "eslint-plugin-react-native": "^5.0.0",
+    "expo-haptics": "^55.0.9",
     "express": "^5.2.1",
     "jest": "^30.2.0",
     "jest-environment-jsdom": "^30.2.0",
     "metro-react-native-babel-preset": "^0.77.0",
+    "moti": "^0.30.0",
     "multer": "^2.1.0",
     "prettier": "^3.8.1",
+    "react-native-gesture-handler": "^2.30.0",
+    "react-native-reanimated": "^4.2.3",
     "react-native-webview": "^13.16.0",
     "react-test-renderer": "^19.2.4",
     "ts-jest": "^29.4.6"