talking-head-studio 0.3.9 → 0.4.1

package/dist/TalkingHeadVisualization.d.ts

@@ -28,7 +28,7 @@ export interface TalkingHeadVisualizationRef {
 /**
  * TalkingHeadVisualization — optimized component for rendering the 3D avatar.
  *
- * On native: uses FilamentAvatar (direct morph writes, no WebView bridge).
+ * On native: uses WgpuAvatar (direct morph writes, no WebView bridge).
  * On web: uses TalkingHead WebView renderer.
  */
 export declare const TalkingHeadVisualization: React.ForwardRefExoticComponent<TalkingHeadVisualizationProps & React.RefAttributes<TalkingHeadVisualizationRef>>;

package/dist/TalkingHeadVisualization.js

@@ -6,9 +6,9 @@ const jsx_runtime_1 = require("react/jsx-runtime");
 const react_1 = require("react");
 const react_native_1 = require("react-native");
 const TalkingHead_1 = require("./TalkingHead");
-const FilamentAvatar_1 = require("./filament/FilamentAvatar");
+const WgpuAvatar_1 = require("./wgpu/WgpuAvatar");
 const avatarUtils_1 = require("./utils/avatarUtils");
-const faceSqueezeAssets_1 = require("./filament/faceSqueezeAssets");
+const faceSqueezeAssets_1 = require("./wgpu/faceSqueezeAssets");
 // Cached fallback data URI — resolved once, reused across all instances
 let _fallbackDataUri = null;
 let _fallbackPromise = null;
@@ -41,16 +41,16 @@ function getLoadingLabel(stage) {
 /**
  * TalkingHeadVisualization — optimized component for rendering the 3D avatar.
  *
- * On native: uses FilamentAvatar (direct morph writes, no WebView bridge).
+ * On native: uses WgpuAvatar (direct morph writes, no WebView bridge).
  * On web: uses TalkingHead WebView renderer.
  */
 exports.TalkingHeadVisualization = (0, react_1.forwardRef)(({ style, avatarUrl, authToken, cameraView = 'head', cameraDistance = 0.2, accessories, mood: initialMood = 'neutral', aspect, focalLength, visemeSchedule, onVisemeScheduleApplied, vendorBaseUrl }, ref) => {
     const avatarRef = (0, react_1.useRef)(null);
     // On native, Filament ref is wired via callback ref — store it here so
     // scheduleVisemes / sendAmplitude can route to it.
-    const filamentRef = (0, react_1.useRef)(null);
+    const wgpuRef = (0, react_1.useRef)(null);
     // Unified accessor — Filament on native, WebView on web
-    const activeAvatar = (0, react_1.useCallback)(() => (filamentRef.current ?? avatarRef.current), []);
+    const activeAvatar = (0, react_1.useCallback)(() => (wgpuRef.current ?? avatarRef.current), []);
     // Fallback local GLB data URI — resolved once on mount
     const [fallbackUrl, setFallbackUrl] = (0, react_1.useState)(_fallbackDataUri);
     (0, react_1.useEffect)(() => {
@@ -116,7 +116,7 @@ exports.TalkingHeadVisualization = (0, react_1.forwardRef)(({ style, avatarUrl,
             const av = activeAvatar();
             // Filament buffers pending schedules internally — no ready gate needed.
             // WebView (avatarRef) still needs the ready gate.
-            if (!av || (!filamentRef.current && !isAvatarReady)) {
+            if (!av || (!wgpuRef.current && !isAvatarReady)) {
                 pendingVisemeScheduleRef.current = schedule;
                 return;
             }
@@ -166,7 +166,7 @@ exports.TalkingHeadVisualization = (0, react_1.forwardRef)(({ style, avatarUrl,
             ready: isAvatarReady,
         });
         const av = activeAvatar();
-        if (!av || (!filamentRef.current && !isAvatarReady)) {
+        if (!av || (!wgpuRef.current && !isAvatarReady)) {
             pendingVisemeScheduleRef.current = visemeSchedule;
             return;
         }
@@ -183,8 +183,8 @@ exports.TalkingHeadVisualization = (0, react_1.forwardRef)(({ style, avatarUrl,
     }, [isAvatarReady, onVisemeScheduleApplied, visemeSchedule, activeAvatar]);
     // On native use Filament — direct morph writes, no WebView bridge.
     if (react_native_1.Platform.OS !== 'web') {
-        return ((0, jsx_runtime_1.jsx)(FilamentAvatar_1.FilamentAvatar, { focalLength: focalLength, ref: (fr) => {
-                filamentRef.current = fr;
+        return ((0, jsx_runtime_1.jsx)(WgpuAvatar_1.WgpuAvatar, { focalLength: focalLength, ref: (fr) => {
+                wgpuRef.current = fr;
                 if (typeof ref === 'function')
                     ref(fr);
                 else if (ref)

package/dist/wgpu/WgpuAvatar.d.ts

@@ -0,0 +1,36 @@
+/**
+ * WgpuAvatar — drop-in replacement for FilamentAvatar using react-three-fiber
+ * + react-native-wgpu (or expo-gl fallback). Exposes the same ref interface as
+ * FilamentAvatarRef so siteclaw can swap renderers without changing call sites.
+ *
+ * No SurfaceTexture, no choreographer, no JNI surface lifecycle bugs.
+ *
+ * Peer deps required by the host app:
+ *   react-native-wgpu  (or expo-gl for the expo-gl fallback canvas)
+ *   @react-three/fiber >= 8
+ *   @react-three/drei  >= 9
+ *   three              >= 0.170
+ */
+import React from 'react';
+import { type StyleProp, type ViewStyle } from 'react-native';
+import type { TalkingHeadMood, TalkingHeadAccessory, TalkingHeadVisemeSchedule } from '../index';
+export interface WgpuAvatarRef {
+    setMood: (mood: TalkingHeadMood) => void;
+    sendAmplitude: (amplitude: number) => void;
+    sendViseme: (viseme: string, weight?: number) => void;
+    scheduleVisemes: (schedule: TalkingHeadVisemeSchedule) => void;
+    clearVisemes: () => void;
+}
+interface WgpuAvatarProps {
+    style?: StyleProp<ViewStyle>;
+    avatarUrl: string | null;
+    authToken?: string;
+    aspect?: number;
+    focalLength?: number;
+    mood?: TalkingHeadMood;
+    accessories?: TalkingHeadAccessory[];
+    onReady?: () => void;
+    onError?: (message: string) => void;
+}
+export declare const WgpuAvatar: React.ForwardRefExoticComponent<WgpuAvatarProps & React.RefAttributes<WgpuAvatarRef>>;
+export {};

package/dist/wgpu/WgpuAvatar.js

@@ -0,0 +1,293 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.WgpuAvatar = void 0;
+const jsx_runtime_1 = require("react/jsx-runtime");
+/**
+ * WgpuAvatar — drop-in replacement for FilamentAvatar using react-three-fiber
+ * + react-native-wgpu (or expo-gl fallback). Exposes the same ref interface as
+ * FilamentAvatarRef so siteclaw can swap renderers without changing call sites.
+ *
+ * No SurfaceTexture, no choreographer, no JNI surface lifecycle bugs.
+ *
+ * Peer deps required by the host app:
+ *   react-native-wgpu  (or expo-gl for the expo-gl fallback canvas)
+ *   @react-three/fiber >= 8
+ *   @react-three/drei  >= 9
+ *   three              >= 0.170
+ */
+const react_1 = __importStar(require("react"));
+const react_native_1 = require("react-native");
+const THREE = __importStar(require("three"));
+const native_1 = require("@react-three/fiber/native");
+const native_2 = require("@react-three/drei/native");
+const morphTables_1 = require("./morphTables");
+const useAuthedFilamentUri_1 = require("./useAuthedFilamentUri");
+// ---------------------------------------------------------------------------
+// Camera defaults — match FilamentAvatar constants
+// ---------------------------------------------------------------------------
+const CAMERA_POSITION = [0, 1.52, 0.85];
+const CAMERA_TARGET = new THREE.Vector3(0, 1.52, 0);
+const CAMERA_FOV_FULL = 38; // rough Three.js FOV equiv to 50mm focal length
+function buildMorphIndex(mesh) {
+    const map = new Map();
+    const dict = mesh.morphTargetDictionary;
+    if (!dict)
+        return map;
+    for (const [name, idx] of Object.entries(dict)) {
+        map.set(name.toLowerCase(), idx);
+        map.set(name, idx); // keep original casing too
+    }
+    return map;
+}
+function resolveIndex(morphIndex, aliases) {
+    for (const alias of aliases) {
+        const idx = morphIndex.get(alias) ?? morphIndex.get(alias.toLowerCase());
+        if (idx !== undefined)
+            return idx;
+    }
+    return undefined;
+}
+function AvatarScene({ uri, morphStateRef, fov, onReady, onError }) {
+    const { camera } = (0, native_1.useThree)();
+    const gltf = (0, native_2.useGLTF)(uri);
+    const headMeshRef = (0, react_1.useRef)(null);
+    const morphIndexRef = (0, react_1.useRef)(new Map());
+    const readyFiredRef = (0, react_1.useRef)(false);
+    // Set up camera on mount
+    (0, react_1.useEffect)(() => {
+        if (!(camera instanceof THREE.PerspectiveCamera))
+            return;
+        camera.fov = fov;
+        camera.position.set(...CAMERA_POSITION);
+        camera.lookAt(CAMERA_TARGET);
+        camera.updateProjectionMatrix();
+    }, [camera, fov]);
+    // Find the head/face mesh with morph targets after GLTF loads
+    (0, react_1.useEffect)(() => {
+        if (!gltf?.scene)
+            return;
+        let bestMesh = null;
+        let bestCount = 0;
+        gltf.scene.traverse((node) => {
+            if (!(node instanceof THREE.Mesh))
+                return;
+            const count = Object.keys(node.morphTargetDictionary ?? {}).length;
+            if (count > bestCount) {
+                bestCount = count;
+                bestMesh = node;
+            }
+        });
+        if (!bestMesh) {
+            onError('No mesh with morph targets found in GLB');
+            return;
+        }
+        headMeshRef.current = bestMesh;
+        morphIndexRef.current = buildMorphIndex(bestMesh);
+        if (!readyFiredRef.current) {
+            readyFiredRef.current = true;
+            onReady();
+        }
+    }, [gltf, onReady, onError]);
+    // Per-frame morph weight application
+    (0, native_1.useFrame)((_, delta) => {
+        const mesh = headMeshRef.current;
+        if (!mesh?.morphTargetInfluences)
+            return;
+        const state = morphStateRef.current;
+        const alpha = Math.min(1, state.alpha * delta * 60); // normalize to 60fps
+        // Merge mood base + viseme target
+        const combined = { ...state.moodBase };
+        for (const [name, w] of Object.entries(state.visemeTarget)) {
+            combined[name] = Math.max(combined[name] ?? 0, w);
+        }
+        // Smooth current toward combined
+        const allNames = new Set([
+            ...Object.keys(state.current),
+            ...Object.keys(combined),
+        ]);
+        for (const name of allNames) {
+            const target = combined[name] ?? 0;
+            const cur = state.current[name] ?? 0;
+            const next = cur + (target - cur) * alpha;
+            state.current[name] = next;
+            const idx = resolveIndex(morphIndexRef.current, [name]);
+            if (idx !== undefined) {
+                mesh.morphTargetInfluences[idx] = next;
+            }
+        }
+        // Decay viseme layer
+        for (const name of Object.keys(state.visemeTarget)) {
+            state.visemeTarget[name] *= Math.pow(0.1, delta); // ~10x decay/s
+            if (state.visemeTarget[name] < 0.001) {
+                delete state.visemeTarget[name];
+            }
+        }
+    });
+    return ((0, jsx_runtime_1.jsxs)(jsx_runtime_1.Fragment, { children: [(0, jsx_runtime_1.jsx)("ambientLight", { intensity: 0.6 }), (0, jsx_runtime_1.jsx)("directionalLight", { position: [1, 3, 2], intensity: 1.2, castShadow: false }), (0, jsx_runtime_1.jsx)("primitive", { object: gltf.scene })] }));
+}
+// ---------------------------------------------------------------------------
+// Error boundary so GLTF load failures surface to onError
+// ---------------------------------------------------------------------------
+class GltfErrorBoundary extends react_1.default.Component {
+    constructor(props) {
+        super(props);
+        this.state = { hasError: false };
+    }
+    static getDerivedStateFromError() { return { hasError: true }; }
+    componentDidCatch(err) { this.props.onError(err.message); }
+    render() {
+        if (this.state.hasError)
+            return null;
+        return this.props.children;
+    }
+}
+// ---------------------------------------------------------------------------
+// Viseme schedule player
+// ---------------------------------------------------------------------------
+function applyVisemeCue(morphState, visemeKey) {
+    const aliases = morphTables_1.VISEME_MORPH_ALIASES[visemeKey];
+    if (!aliases)
+        return;
+    const w = morphTables_1.VISEME_WEIGHTS[visemeKey] ?? morphTables_1.DEFAULT_VISEME_WEIGHT;
+    for (const alias of aliases) {
+        morphState.visemeTarget[alias] = w;
+    }
+}
+// ---------------------------------------------------------------------------
+// Main exported component
+// ---------------------------------------------------------------------------
+exports.WgpuAvatar = (0, react_1.forwardRef)(({ style, avatarUrl, focalLength, mood = 'neutral', onReady, onError, }, ref) => {
+    // Resolve authenticated file URI (same logic as FilamentAvatar)
+    const remoteUrl = avatarUrl && (avatarUrl.startsWith('http://') || avatarUrl.startsWith('https://'))
+        ? avatarUrl
+        : null;
+    const fileResult = (0, useAuthedFilamentUri_1.useAuthedFilamentUri)(remoteUrl);
+    // Lock-in URI on first resolve — never change mid-session
+    const lockedUriRef = (0, react_1.useRef)(null);
+    const currentUri = fileResult?.uri ?? null;
+    if (lockedUriRef.current === null && currentUri) {
+        lockedUriRef.current = currentUri;
+    }
+    const localUri = lockedUriRef.current;
+    const fov = focalLength
+        ? Math.round(2 * Math.atan(21.634 / focalLength) * (180 / Math.PI))
+        : CAMERA_FOV_FULL;
+    // Shared morph state — mutated directly in useFrame, never causes re-renders
+    const morphStateRef = (0, react_1.useRef)({
+        current: {},
+        visemeTarget: {},
+        moodBase: {},
+        alpha: 0.18,
+    });
+    // Update mood baseline when mood prop changes
+    (0, react_1.useEffect)(() => {
+        morphStateRef.current.moodBase = { ...(morphTables_1.MOOD_MORPHS[mood] ?? {}) };
+    }, [mood]);
+    // Pending viseme schedule
+    const scheduleRef = (0, react_1.useRef)(null);
+    const scheduleTimersRef = (0, react_1.useRef)([]);
+    const [isReady, setIsReady] = (0, react_1.useState)(false);
+    const clearScheduleTimers = (0, react_1.useCallback)(() => {
+        for (const t of scheduleTimersRef.current)
+            clearTimeout(t);
+        scheduleTimersRef.current = [];
+    }, []);
+    const handleReady = (0, react_1.useCallback)(() => {
+        setIsReady(true);
+        onReady?.();
+        // Flush any pending schedule
+        if (scheduleRef.current) {
+            const s = scheduleRef.current;
+            scheduleRef.current = null;
+            applySchedule(s);
+        }
+        // eslint-disable-next-line react-hooks/exhaustive-deps
+    }, [onReady]);
+    const handleError = (0, react_1.useCallback)((msg) => {
+        console.warn('[WgpuAvatar] GLTF error:', msg);
+        onError?.(msg);
+    }, [onError]);
+    const applySchedule = (0, react_1.useCallback)((schedule) => {
+        clearScheduleTimers();
+        const now = Date.now();
+        const startedAt = schedule.startedAtMs ?? now;
+        const offset = now - startedAt;
+        for (const cue of schedule.cues) {
+            const delay = cue.startMs - offset;
+            if (delay < -200)
+                continue; // already expired
+            const rhubarbKey = cue.viseme;
+            const visemeKey = morphTables_1.RHUBARB_TO_VISEME[rhubarbKey] ?? rhubarbKey;
+            const t = setTimeout(() => {
+                applyVisemeCue(morphStateRef.current, visemeKey);
+            }, Math.max(0, delay));
+            scheduleTimersRef.current.push(t);
+        }
+    }, [clearScheduleTimers]);
+    // Cleanup on unmount
+    (0, react_1.useEffect)(() => () => clearScheduleTimers(), [clearScheduleTimers]);
+    (0, react_1.useImperativeHandle)(ref, () => ({
+        setMood: (m) => {
+            morphStateRef.current.moodBase = { ...(morphTables_1.MOOD_MORPHS[m] ?? {}) };
+        },
+        // eslint-disable-next-line @typescript-eslint/no-unused-vars
+        sendAmplitude: (_amplitude) => {
+            // Optional: could drive jaw open with amplitude
+            // morphStateRef.current.visemeTarget['jawOpen'] = _amplitude * 0.3;
+        },
+        sendViseme: (viseme, weight) => {
+            const aliases = morphTables_1.VISEME_MORPH_ALIASES[viseme];
+            if (!aliases)
+                return;
+            const w = weight ?? morphTables_1.VISEME_WEIGHTS[viseme] ?? morphTables_1.DEFAULT_VISEME_WEIGHT;
+            for (const alias of aliases) {
+                morphStateRef.current.visemeTarget[alias] = w;
+            }
+        },
+        clearVisemes: () => {
+            clearScheduleTimers();
+            morphStateRef.current.visemeTarget = {};
+        },
+        scheduleVisemes: (schedule) => {
+            if (!isReady) {
+                scheduleRef.current = schedule;
+                return;
+            }
+            applySchedule(schedule);
+        },
+    }), [isReady, applySchedule, clearScheduleTimers]);
+    if (!localUri) {
+        return (0, jsx_runtime_1.jsx)(react_native_1.View, { style: [styles.placeholder, style] });
+    }
+    return ((0, jsx_runtime_1.jsx)(react_native_1.View, { style: [styles.container, style], children: (0, jsx_runtime_1.jsx)(native_1.Canvas, { style: react_native_1.StyleSheet.absoluteFill, camera: { fov, position: CAMERA_POSITION, near: 0.01, far: 100 }, gl: { antialias: true, alpha: false }, onCreated: ({ gl }) => {
+                gl.setClearColor(new THREE.Color('#1a1a2e'));
+            }, children: (0, jsx_runtime_1.jsx)(GltfErrorBoundary, { onError: handleError, children: (0, jsx_runtime_1.jsx)(AvatarScene, { uri: localUri, morphStateRef: morphStateRef, fov: fov, onReady: handleReady, onError: handleError }) }) }) }));
+});
+exports.WgpuAvatar.displayName = 'WgpuAvatar';
+const styles = react_native_1.StyleSheet.create({
+    container: { overflow: 'hidden', backgroundColor: '#1a1a2e' },
+    placeholder: { backgroundColor: '#1a1a2e' },
+});

package/dist/wgpu/index.d.ts

@@ -0,0 +1,2 @@
+export { WgpuAvatar } from './WgpuAvatar';
+export type { WgpuAvatarRef } from './WgpuAvatar';

package/dist/wgpu/index.js

@@ -0,0 +1,5 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.WgpuAvatar = void 0;
+var WgpuAvatar_1 = require("./WgpuAvatar");
+Object.defineProperty(exports, "WgpuAvatar", { enumerable: true, get: function () { return WgpuAvatar_1.WgpuAvatar; } });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "talking-head-studio",
-  "version": "0.3.9",
+  "version": "0.4.1",
   "description": "Cross-platform 3D avatar component for React Native & web — lip-sync, gestures, accessories, and LLM integration. Powered by TalkingHead + Three.js.",
   "main": "dist/index.web.js",
   "browser": "dist/index.web.js",
@@ -37,15 +37,10 @@
       "types": "./dist/wardrobe/index.d.ts",
       "default": "./dist/wardrobe/index.js"
     },
-    "./filament": {
-      "react-native": "./dist/filament/index.js",
-      "types": "./dist/filament/index.d.ts",
-      "default": "./dist/filament/index.js"
-    },
-    "./filament/editor": {
-      "react-native": "./dist/filament/editor/index.js",
-      "types": "./dist/filament/editor/index.d.ts",
-      "default": "./dist/filament/editor/index.js"
+    "./wgpu": {
+      "react-native": "./dist/wgpu/index.js",
+      "types": "./dist/wgpu/index.d.ts",
+      "default": "./dist/wgpu/index.js"
     }
   },
   "files": [
@@ -109,8 +104,8 @@
     "expo-file-system": ">=17",
     "react": ">=18",
     "react-native": ">=0.73",
-    "react-native-filament": ">=1",
     "react-native-webview": ">=13",
+    "react-native-wgpu": ">=0.1",
     "three": ">=0.170"
   },
   "peerDependenciesMeta": {
@@ -120,7 +115,7 @@
     "react-native-webview": {
       "optional": true
     },
-    "react-native-filament": {
+    "react-native-wgpu": {
       "optional": true
     },
     "expo": {
@@ -178,7 +173,7 @@
     "react-native-gesture-handler": "^2.30.0",
     "react-native-reanimated": "^4.2.3",
     "react-native-webview": "^13.16.0",
-    "react-test-renderer": "^19.2.4",
+    "react-test-renderer": "^18.3.1",
     "three": "^0.180.0",
     "ts-jest": "^29.4.6",
     "typescript": "^5.3.3",

package/dist/filament/FilamentAvatar.d.ts

@@ -1,41 +0,0 @@
-/**
- * FilamentAvatar — native Filament-based avatar renderer.
- *
- * Replaces the TalkingHead WebView on iOS/Android with direct Filament
- * morph target writes — no WebView bridge, no JS→native→JS roundtrip,
- * visemes applied at 60fps in React Native JS.
- *
- * Implements the full TalkingHeadRef interface so it can be swapped in
- * transparently for the WebView renderer.
- */
-import React from 'react';
-import { ViewStyle, StyleProp } from 'react-native';
-import type { TalkingHeadViseme, TalkingHeadVisemeSchedule, TalkingHeadAccessory, TalkingHeadMood } from '../TalkingHead';
-export declare const CAMERA_FOCAL_FULL = 28;
-export declare const CAMERA_FOCAL_PIP = 50;
-export interface FilamentAvatarRef {
-    sendViseme: (viseme: TalkingHeadViseme, weight?: number) => void;
-    scheduleVisemes: (schedule: TalkingHeadVisemeSchedule) => void;
-    clearVisemes: () => void;
-    sendAmplitude: (amplitude: number) => void;
-    setMood: (mood: TalkingHeadMood) => void;
-    setHairColor: (color: string) => void;
-    setSkinColor: (color: string) => void;
-    setEyeColor: (color: string) => void;
-    setAccessories: (accessories: TalkingHeadAccessory[]) => void;
-}
-interface FilamentAvatarProps {
-    style?: StyleProp<ViewStyle>;
-    avatarUrl: string | null;
-    aspect?: number;
-    focalLength?: number;
-    mood?: TalkingHeadMood;
-    hairColor?: string;
-    skinColor?: string;
-    eyeColor?: string;
-    accessories?: TalkingHeadAccessory[];
-    onReady?: () => void;
-    onError?: (message: string) => void;
-}
-export declare const FilamentAvatar: React.ForwardRefExoticComponent<FilamentAvatarProps & React.RefAttributes<FilamentAvatarRef>>;
-export {};