talking-head-studio 0.4.0 → 0.4.2

package/dist/wgpu/WgpuAvatar.js

@@ -43,14 +43,26 @@ const react_native_1 = require("react-native");
 const THREE = __importStar(require("three"));
 const native_1 = require("@react-three/fiber/native");
 const native_2 = require("@react-three/drei/native");
-const morphTables_1 = require("../filament/morphTables");
-const useAuthedFilamentUri_1 = require("../filament/useAuthedFilamentUri");
+const morphTables_1 = require("./morphTables");
+const useAuthedModelUri_1 = require("./useAuthedModelUri");
 // ---------------------------------------------------------------------------
 // Camera defaults — match FilamentAvatar constants
 // ---------------------------------------------------------------------------
 const CAMERA_POSITION = [0, 1.52, 0.85];
 const CAMERA_TARGET = new THREE.Vector3(0, 1.52, 0);
-const CAMERA_FOV_FULL = 38; // rough Three.js FOV equiv to 50mm focal length
+const CAMERA_FOV_FULL = 38;
+// ---------------------------------------------------------------------------
+// Scene/camera ref capture — runs inside Canvas so it can access R3F context
+// ---------------------------------------------------------------------------
+function SceneCapture({ onSceneReady }) {
+    const { scene, camera } = (0, native_1.useThree)();
+    (0, react_1.useEffect)(() => {
+        onSceneReady(scene, camera);
+        // Only fire once on mount — scene/camera identity is stable
+        // eslint-disable-next-line react-hooks/exhaustive-deps
+    }, []);
+    return null;
+}
 function buildMorphIndex(mesh) {
     const map = new Map();
     const dict = mesh.morphTargetDictionary;
@@ -58,7 +70,7 @@ function buildMorphIndex(mesh) {
         return map;
     for (const [name, idx] of Object.entries(dict)) {
         map.set(name.toLowerCase(), idx);
-        map.set(name, idx); // keep original casing too
+        map.set(name, idx);
     }
     return map;
 }
@@ -70,13 +82,12 @@ function resolveIndex(morphIndex, aliases) {
     }
     return undefined;
 }
-function AvatarScene({ uri, morphStateRef, fov, onReady, onError }) {
+function AvatarScene({ uri, morphStateRef, fov, onReady, onError, onSceneReady }) {
     const { camera } = (0, native_1.useThree)();
     const gltf = (0, native_2.useGLTF)(uri);
     const headMeshRef = (0, react_1.useRef)(null);
     const morphIndexRef = (0, react_1.useRef)(new Map());
     const readyFiredRef = (0, react_1.useRef)(false);
-    // Set up camera on mount
     (0, react_1.useEffect)(() => {
         if (!(camera instanceof THREE.PerspectiveCamera))
             return;
@@ -85,7 +96,6 @@ function AvatarScene({ uri, morphStateRef, fov, onReady, onError }) {
         camera.lookAt(CAMERA_TARGET);
         camera.updateProjectionMatrix();
     }, [camera, fov]);
-    // Find the head/face mesh with morph targets after GLTF loads
     (0, react_1.useEffect)(() => {
         if (!gltf?.scene)
             return;
@@ -111,58 +121,43 @@ function AvatarScene({ uri, morphStateRef, fov, onReady, onError }) {
             onReady();
         }
     }, [gltf, onReady, onError]);
-    // Per-frame morph weight application
     (0, native_1.useFrame)((_, delta) => {
         const mesh = headMeshRef.current;
         if (!mesh?.morphTargetInfluences)
             return;
         const state = morphStateRef.current;
-        const alpha = Math.min(1, state.alpha * delta * 60); // normalize to 60fps
-        // Merge mood base + viseme target
+        const alpha = Math.min(1, state.alpha * delta * 60);
         const combined = { ...state.moodBase };
         for (const [name, w] of Object.entries(state.visemeTarget)) {
             combined[name] = Math.max(combined[name] ?? 0, w);
         }
-        // Smooth current toward combined
-        const allNames = new Set([
-            ...Object.keys(state.current),
-            ...Object.keys(combined),
-        ]);
+        const allNames = new Set([...Object.keys(state.current), ...Object.keys(combined)]);
         for (const name of allNames) {
             const target = combined[name] ?? 0;
             const cur = state.current[name] ?? 0;
             const next = cur + (target - cur) * alpha;
             state.current[name] = next;
             const idx = resolveIndex(morphIndexRef.current, [name]);
-            if (idx !== undefined) {
+            if (idx !== undefined)
                 mesh.morphTargetInfluences[idx] = next;
-            }
         }
-        // Decay viseme layer
         for (const name of Object.keys(state.visemeTarget)) {
-            state.visemeTarget[name] *= Math.pow(0.1, delta); // ~10x decay/s
-            if (state.visemeTarget[name] < 0.001) {
+            state.visemeTarget[name] *= Math.pow(0.1, delta);
+            if (state.visemeTarget[name] < 0.001)
                 delete state.visemeTarget[name];
-            }
         }
     });
-    return ((0, jsx_runtime_1.jsxs)(jsx_runtime_1.Fragment, { children: [(0, jsx_runtime_1.jsx)("ambientLight", { intensity: 0.6 }), (0, jsx_runtime_1.jsx)("directionalLight", { position: [1, 3, 2], intensity: 1.2, castShadow: false }), (0, jsx_runtime_1.jsx)("primitive", { object: gltf.scene })] }));
+    return ((0, jsx_runtime_1.jsxs)(jsx_runtime_1.Fragment, { children: [onSceneReady && (0, jsx_runtime_1.jsx)(SceneCapture, { onSceneReady: onSceneReady }), (0, jsx_runtime_1.jsx)("ambientLight", { intensity: 0.6 }), (0, jsx_runtime_1.jsx)("directionalLight", { position: [1, 3, 2], intensity: 1.2, castShadow: false }), (0, jsx_runtime_1.jsx)("primitive", { object: gltf.scene })] }));
 }
 // ---------------------------------------------------------------------------
-// Error boundary so GLTF load failures surface to onError
+// Error boundary
 // ---------------------------------------------------------------------------
 class GltfErrorBoundary extends react_1.default.Component {
-    constructor(props) {
-        super(props);
-        this.state = { hasError: false };
-    }
+    constructor(props) { super(props); this.state = { hasError: false }; }
     static getDerivedStateFromError() { return { hasError: true }; }
     componentDidCatch(err) { this.props.onError(err.message); }
-    render() {
-        if (this.state.hasError)
-            return null;
-        return this.props.children;
-    }
+    render() { if (this.state.hasError)
+        return null; return this.props.children; }
 }
 // ---------------------------------------------------------------------------
 // Viseme schedule player
@@ -172,41 +167,30 @@ function applyVisemeCue(morphState, visemeKey) {
     if (!aliases)
         return;
     const w = morphTables_1.VISEME_WEIGHTS[visemeKey] ?? morphTables_1.DEFAULT_VISEME_WEIGHT;
-    for (const alias of aliases) {
+    for (const alias of aliases)
         morphState.visemeTarget[alias] = w;
-    }
 }
 // ---------------------------------------------------------------------------
 // Main exported component
 // ---------------------------------------------------------------------------
-exports.WgpuAvatar = (0, react_1.forwardRef)(({ style, avatarUrl, focalLength, mood = 'neutral', onReady, onError, }, ref) => {
-    // Resolve authenticated file URI (same logic as FilamentAvatar)
+exports.WgpuAvatar = (0, react_1.forwardRef)(({ style, avatarUrl, focalLength, mood = 'neutral', onReady, onError, onSceneReady }, ref) => {
     const remoteUrl = avatarUrl && (avatarUrl.startsWith('http://') || avatarUrl.startsWith('https://'))
-        ? avatarUrl
-        : null;
-    const fileResult = (0, useAuthedFilamentUri_1.useAuthedFilamentUri)(remoteUrl);
-    // Lock-in URI on first resolve — never change mid-session
+        ? avatarUrl : null;
+    const fileResult = (0, useAuthedModelUri_1.useAuthedModelUri)(remoteUrl);
     const lockedUriRef = (0, react_1.useRef)(null);
     const currentUri = fileResult?.uri ?? null;
-    if (lockedUriRef.current === null && currentUri) {
+    if (lockedUriRef.current === null && currentUri)
         lockedUriRef.current = currentUri;
-    }
     const localUri = lockedUriRef.current;
     const fov = focalLength
         ? Math.round(2 * Math.atan(21.634 / focalLength) * (180 / Math.PI))
         : CAMERA_FOV_FULL;
-    // Shared morph state — mutated directly in useFrame, never causes re-renders
     const morphStateRef = (0, react_1.useRef)({
-        current: {},
-        visemeTarget: {},
-        moodBase: {},
-        alpha: 0.18,
+        current: {}, visemeTarget: {}, moodBase: {}, alpha: 0.18,
     });
-    // Update mood baseline when mood prop changes
     (0, react_1.useEffect)(() => {
         morphStateRef.current.moodBase = { ...(morphTables_1.MOOD_MORPHS[mood] ?? {}) };
     }, [mood]);
-    // Pending viseme schedule
     const scheduleRef = (0, react_1.useRef)(null);
     const scheduleTimersRef = (0, react_1.useRef)([]);
     const [isReady, setIsReady] = (0, react_1.useState)(false);
@@ -215,21 +199,6 @@ exports.WgpuAvatar = (0, react_1.forwardRef)(({ style, avatarUrl, focalLength, m
             clearTimeout(t);
         scheduleTimersRef.current = [];
     }, []);
-    const handleReady = (0, react_1.useCallback)(() => {
-        setIsReady(true);
-        onReady?.();
-        // Flush any pending schedule
-        if (scheduleRef.current) {
-            const s = scheduleRef.current;
-            scheduleRef.current = null;
-            applySchedule(s);
-        }
-        // eslint-disable-next-line react-hooks/exhaustive-deps
-    }, [onReady]);
-    const handleError = (0, react_1.useCallback)((msg) => {
-        console.warn('[WgpuAvatar] GLTF error:', msg);
-        onError?.(msg);
-    }, [onError]);
     const applySchedule = (0, react_1.useCallback)((schedule) => {
         clearScheduleTimers();
         const now = Date.now();
@@ -238,39 +207,41 @@ exports.WgpuAvatar = (0, react_1.forwardRef)(({ style, avatarUrl, focalLength, m
         for (const cue of schedule.cues) {
             const delay = cue.startMs - offset;
             if (delay < -200)
-                continue; // already expired
-            const rhubarbKey = cue.viseme;
-            const visemeKey = morphTables_1.RHUBARB_TO_VISEME[rhubarbKey] ?? rhubarbKey;
+                continue;
+            const visemeKey = morphTables_1.RHUBARB_TO_VISEME[cue.viseme] ?? cue.viseme;
             const t = setTimeout(() => {
                 applyVisemeCue(morphStateRef.current, visemeKey);
             }, Math.max(0, delay));
             scheduleTimersRef.current.push(t);
         }
     }, [clearScheduleTimers]);
-    // Cleanup on unmount
+    const handleReady = (0, react_1.useCallback)(() => {
+        setIsReady(true);
+        onReady?.();
+        if (scheduleRef.current) {
+            const s = scheduleRef.current;
+            scheduleRef.current = null;
+            applySchedule(s);
+        }
+        // eslint-disable-next-line react-hooks/exhaustive-deps
+    }, [onReady]);
+    const handleError = (0, react_1.useCallback)((msg) => {
+        console.warn('[WgpuAvatar] GLTF error:', msg);
+        onError?.(msg);
+    }, [onError]);
     (0, react_1.useEffect)(() => () => clearScheduleTimers(), [clearScheduleTimers]);
     (0, react_1.useImperativeHandle)(ref, () => ({
-        setMood: (m) => {
-            morphStateRef.current.moodBase = { ...(morphTables_1.MOOD_MORPHS[m] ?? {}) };
-        },
-        // eslint-disable-next-line @typescript-eslint/no-unused-vars
-        sendAmplitude: (_amplitude) => {
-            // Optional: could drive jaw open with amplitude
-            // morphStateRef.current.visemeTarget['jawOpen'] = _amplitude * 0.3;
-        },
+        setMood: (m) => { morphStateRef.current.moodBase = { ...(morphTables_1.MOOD_MORPHS[m] ?? {}) }; },
+        sendAmplitude: (_amplitude) => { },
         sendViseme: (viseme, weight) => {
             const aliases = morphTables_1.VISEME_MORPH_ALIASES[viseme];
             if (!aliases)
                 return;
             const w = weight ?? morphTables_1.VISEME_WEIGHTS[viseme] ?? morphTables_1.DEFAULT_VISEME_WEIGHT;
-            for (const alias of aliases) {
+            for (const alias of aliases)
                 morphStateRef.current.visemeTarget[alias] = w;
-            }
-        },
-        clearVisemes: () => {
-            clearScheduleTimers();
-            morphStateRef.current.visemeTarget = {};
         },
+        clearVisemes: () => { clearScheduleTimers(); morphStateRef.current.visemeTarget = {}; },
         scheduleVisemes: (schedule) => {
             if (!isReady) {
                 scheduleRef.current = schedule;
@@ -279,12 +250,9 @@ exports.WgpuAvatar = (0, react_1.forwardRef)(({ style, avatarUrl, focalLength, m
             applySchedule(schedule);
         },
     }), [isReady, applySchedule, clearScheduleTimers]);
-    if (!localUri) {
+    if (!localUri)
         return (0, jsx_runtime_1.jsx)(react_native_1.View, { style: [styles.placeholder, style] });
-    }
-    return ((0, jsx_runtime_1.jsx)(react_native_1.View, { style: [styles.container, style], children: (0, jsx_runtime_1.jsx)(native_1.Canvas, { style: react_native_1.StyleSheet.absoluteFill, camera: { fov, position: CAMERA_POSITION, near: 0.01, far: 100 }, gl: { antialias: true, alpha: false }, onCreated: ({ gl }) => {
-                gl.setClearColor(new THREE.Color('#1a1a2e'));
-            }, children: (0, jsx_runtime_1.jsx)(GltfErrorBoundary, { onError: handleError, children: (0, jsx_runtime_1.jsx)(AvatarScene, { uri: localUri, morphStateRef: morphStateRef, fov: fov, onReady: handleReady, onError: handleError }) }) }) }));
+    return ((0, jsx_runtime_1.jsx)(react_native_1.View, { style: [styles.container, style], children: (0, jsx_runtime_1.jsx)(native_1.Canvas, { style: react_native_1.StyleSheet.absoluteFill, camera: { fov, position: CAMERA_POSITION, near: 0.01, far: 100 }, gl: { antialias: true, alpha: false }, onCreated: ({ gl }) => { gl.setClearColor(new THREE.Color('#1a1a2e')); }, children: (0, jsx_runtime_1.jsx)(GltfErrorBoundary, { onError: handleError, children: (0, jsx_runtime_1.jsx)(AvatarScene, { uri: localUri, morphStateRef: morphStateRef, fov: fov, onReady: handleReady, onError: handleError, onSceneReady: onSceneReady }) }) }) }));
 });
 exports.WgpuAvatar.displayName = 'WgpuAvatar';
 const styles = react_native_1.StyleSheet.create({

package/dist/{filament → wgpu}/morphTables.js

@@ -1,6 +1,6 @@
 "use strict";
 // ---------------------------------------------------------------------------
-// Morph target tables for Filament avatar viseme/mood rendering
+// Morph target tables for avatar viseme/mood rendering
 // ---------------------------------------------------------------------------
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.MOOD_MORPHS = exports.DEFAULT_VISEME_WEIGHT = exports.VISEME_WEIGHTS = exports.VISEME_MORPH_ALIASES = exports.RHUBARB_TO_VISEME = void 0;

package/dist/wgpu/useAuthedModelUri.d.ts

@@ -0,0 +1,11 @@
+export type AuthedFileResult = {
+    uri: string;
+    size: number;
+} | null;
+/**
+ * Downloads a remote URL (with Bearer auth) to the local cache and returns a
+ * `file://` URI suitable for WgpuAvatar model source.
+ * The native model fetcher doesn't send auth headers, so we pre-fetch here.
+ * Also returns the file size in bytes for GPU memory budgeting.
+ */
+export declare function useAuthedModelUri(remoteUrl: string | null): AuthedFileResult;

package/dist/{filament/useAuthedFilamentUri.js → wgpu/useAuthedModelUri.js}

@@ -23,7 +23,7 @@ var __importStar = (this && this.__importStar) || function (mod) {
     return result;
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.useAuthedFilamentUri = void 0;
+exports.useAuthedModelUri = void 0;
 const react_1 = require("react");
 const FileSystem = __importStar(require("expo-file-system/legacy"));
 const studioApi_1 = require("../api/studioApi");
@@ -31,11 +31,11 @@ const studioApi_1 = require("../api/studioApi");
 const failedUrls = new Set();
 /**
  * Downloads a remote URL (with Bearer auth) to the local cache and returns a
- * `file://` URI suitable for react-native-filament's <Model> source prop.
- * The native Filament fetcher doesn't send auth headers, so we pre-fetch here.
+ * `file://` URI suitable for WgpuAvatar model source.
+ * The native model fetcher doesn't send auth headers, so we pre-fetch here.
  * Also returns the file size in bytes for GPU memory budgeting.
  */
-function useAuthedFilamentUri(remoteUrl) {
+function useAuthedModelUri(remoteUrl) {
     const [result, setResult] = (0, react_1.useState)(null);
     (0, react_1.useEffect)(() => {
         if (!remoteUrl) {
@@ -57,7 +57,7 @@ function useAuthedFilamentUri(remoteUrl) {
                 // Strip query params (access_token changes per session) so the same GLB is cached across sessions
                 const urlWithoutQuery = remoteUrl.split('?')[0];
                 const key = urlWithoutQuery.replace(/[^a-zA-Z0-9]/g, '_').slice(-100);
-                const dir = `${FileSystem.cacheDirectory}filament/`;
+                const dir = `${FileSystem.cacheDirectory}model/`;
                 // Append .glb so native loaders can identify the format
                 const localPath = `${dir}${key}.glb`;
                 await FileSystem.makeDirectoryAsync(dir, { intermediates: true });
@@ -85,7 +85,7 @@ function useAuthedFilamentUri(remoteUrl) {
                 const downloaded = await FileSystem.getInfoAsync(dlResult.uri);
                 const downloadedSize = downloaded.size;
                 if (!downloaded.exists || !downloadedSize || downloadedSize < 1000) {
-                    console.error('[useAuthedFilamentUri] Download too small (' + (downloadedSize ?? 0) + 'B), likely an error response');
+                    console.error('[useAuthedModelUri] Download too small (' + (downloadedSize ?? 0) + 'B), likely an error response');
                     failedUrls.add(remoteUrl);
                     await FileSystem.deleteAsync(dlResult.uri);
                     return;
@@ -96,7 +96,7 @@ function useAuthedFilamentUri(remoteUrl) {
                     position: 0,
                 });
                 if (dlHeader !== 'Z2xURg==') {
-                    console.error('[useAuthedFilamentUri] Downloaded file is not a valid GLB (bad magic bytes)');
+                    console.error('[useAuthedModelUri] Downloaded file is not a valid GLB (bad magic bytes)');
                     failedUrls.add(remoteUrl);
                     await FileSystem.deleteAsync(dlResult.uri);
                     return;
@@ -105,7 +105,7 @@ function useAuthedFilamentUri(remoteUrl) {
                     setResult({ uri: dlResult.uri, size: downloadedSize });
             }
             catch (e) {
-                console.error('[useAuthedFilamentUri] Failed to download:', remoteUrl.slice(-60), e);
+                console.error('[useAuthedModelUri] Failed to download:', remoteUrl.slice(-60), e);
             }
         })();
         return () => {
@@ -114,4 +114,4 @@ function useAuthedFilamentUri(remoteUrl) {
     }, [remoteUrl]);
     return result;
 }
-exports.useAuthedFilamentUri = useAuthedFilamentUri;
+exports.useAuthedModelUri = useAuthedModelUri;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "talking-head-studio",
-  "version": "0.4.0",
+  "version": "0.4.2",
   "description": "Cross-platform 3D avatar component for React Native & web — lip-sync, gestures, accessories, and LLM integration. Powered by TalkingHead + Three.js.",
   "main": "dist/index.web.js",
   "browser": "dist/index.web.js",
@@ -37,16 +37,6 @@
       "types": "./dist/wardrobe/index.d.ts",
       "default": "./dist/wardrobe/index.js"
     },
-    "./filament": {
-      "react-native": "./dist/filament/index.js",
-      "types": "./dist/filament/index.d.ts",
-      "default": "./dist/filament/index.js"
-    },
-    "./filament/editor": {
-      "react-native": "./dist/filament/editor/index.js",
-      "types": "./dist/filament/editor/index.d.ts",
-      "default": "./dist/filament/editor/index.js"
-    },
     "./wgpu": {
       "react-native": "./dist/wgpu/index.js",
       "types": "./dist/wgpu/index.d.ts",
@@ -104,6 +94,7 @@
   "sideEffects": false,
   "dependencies": {
     "@mediapipe/tasks-vision": "^0.10.34",
+    "expo-gl": "^55.0.10",
     "zustand": "^5.0.12"
   },
   "peerDependencies": {
@@ -114,7 +105,6 @@
     "expo-file-system": ">=17",
     "react": ">=18",
     "react-native": ">=0.73",
-    "react-native-filament": ">=1",
     "react-native-webview": ">=13",
     "react-native-wgpu": ">=0.1",
     "three": ">=0.170"
@@ -126,9 +116,6 @@
     "react-native-webview": {
       "optional": true
     },
-    "react-native-filament": {
-      "optional": true
-    },
     "react-native-wgpu": {
       "optional": true
     },

package/dist/filament/FilamentAvatar.d.ts

@@ -1,41 +0,0 @@
-/**
- * FilamentAvatar — native Filament-based avatar renderer.
- *
- * Replaces the TalkingHead WebView on iOS/Android with direct Filament
- * morph target writes — no WebView bridge, no JS→native→JS roundtrip,
- * visemes applied at 60fps in React Native JS.
- *
- * Implements the full TalkingHeadRef interface so it can be swapped in
- * transparently for the WebView renderer.
- */
-import React from 'react';
-import { ViewStyle, StyleProp } from 'react-native';
-import type { TalkingHeadViseme, TalkingHeadVisemeSchedule, TalkingHeadAccessory, TalkingHeadMood } from '../TalkingHead';
-export declare const CAMERA_FOCAL_FULL = 50;
-export declare const CAMERA_FOCAL_PIP = 85;
-export interface FilamentAvatarRef {
-    sendViseme: (viseme: TalkingHeadViseme, weight?: number) => void;
-    scheduleVisemes: (schedule: TalkingHeadVisemeSchedule) => void;
-    clearVisemes: () => void;
-    sendAmplitude: (amplitude: number) => void;
-    setMood: (mood: TalkingHeadMood) => void;
-    setHairColor: (color: string) => void;
-    setSkinColor: (color: string) => void;
-    setEyeColor: (color: string) => void;
-    setAccessories: (accessories: TalkingHeadAccessory[]) => void;
-}
-interface FilamentAvatarProps {
-    style?: StyleProp<ViewStyle>;
-    avatarUrl: string | null;
-    aspect?: number;
-    focalLength?: number;
-    mood?: TalkingHeadMood;
-    hairColor?: string;
-    skinColor?: string;
-    eyeColor?: string;
-    accessories?: TalkingHeadAccessory[];
-    onReady?: () => void;
-    onError?: (message: string) => void;
-}
-export declare const FilamentAvatar: React.ForwardRefExoticComponent<FilamentAvatarProps & React.RefAttributes<FilamentAvatarRef>>;
-export {};