talking-head-studio 0.4.10 → 0.4.12

package/dist/wgpu/patchThreeForRN.js

@@ -0,0 +1,292 @@
+"use strict";
+/**
+ * patchThreeForRN — patches Three.js loaders for React Native compatibility.
+ *
+ * Problem: GLTFLoader (three-stdlib) calls URL.createObjectURL(blob) when
+ * extracting embedded textures from a GLB's buffer views. React Native's
+ * Hermes / JSC runtime does not implement URL.createObjectURL, so every
+ * texture inside the GLB fails with "Cannot create URL for blob!".
+ *
+ * Fix strategy (3 layers):
+ *
+ *  Layer 1 — URL.createObjectURL polyfill
+ *    Instead of throwing, store the Blob in a module-level Map under a
+ *    synthetic "rnblob://<id>" key and return that key as the "URL".
+ *    URL.revokeObjectURL deletes the entry.
+ *
+ *  Layer 2 — THREE.ImageLoader patch
+ *    When ImageLoader.load() receives an "rnblob://" URL, retrieve the
+ *    stored Blob, read it as base64 via FileReader, build a "data:" URI,
+ *    then set it as img.src so expo-gl can decode it natively.
+ *
+ *  Layer 3 — THREE.TextureLoader / THREE.FileLoader safety net
+ *    Same interception for any direct TextureLoader usage and for
+ *    FileLoader which is also used internally by GLTFLoader.
+ *
+ * Call installThreeRNPatches() once at module load time, before any
+ * useGLTF / GLTFLoader usage.  Pass debug=true for verbose logging.
+ */
+/* eslint-disable @typescript-eslint/no-explicit-any -- Three.js loader monkey patches must touch dynamic private fields across RN runtimes. */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.installThreeRNPatches = installThreeRNPatches;
+const react_native_1 = require("react-native");
+const base64_js_1 = require("base64-js");
+const THREE = __importStar(require("three"));
+const blobShim_1 = require("./blobShim");
+let installed = false;
+// blobStore is only used if something still produces rnblob:// URLs (shouldn't happen
+// after blobShim returns data: URIs directly, but kept as safety net)
+const blobStore = new Map();
+/** Re-patch URL.createObjectURL on the current globalThis.URL.
+ *  Must run every time installThreeRNPatches() is called because libraries like
+ *  @livekit/react-native call setupURLPolyfill() which replaces globalThis.URL
+ *  with a new whatwg-url object whose createObjectURL always throws
+ *  "Cannot create URL for blob!". */
+function repatchUrlCreateObjectURL() {
+    const URLObj = globalThis.URL;
+    if (!URLObj)
+        return;
+    if (typeof URLObj.createObjectURL === 'function' &&
+        URLObj.createObjectURL.__blobShim) {
+        return; // already our patch — no-op
+    }
+    const patchedCreateObjectURL = (blob) => {
+        const bytes = blobShim_1.blobBytesStore.get(blob);
+        if (bytes) {
+            const mime = blob.type || 'application/octet-stream';
+            const b64 = (0, base64_js_1.fromByteArray)(bytes);
+            return `data:${mime};base64,${b64}`;
+        }
+        throw new Error('[BlobShim] createObjectURL: no bytes stashed for this Blob');
+    };
+    patchedCreateObjectURL.__blobShim = true;
+    URLObj.createObjectURL = patchedCreateObjectURL;
+    URLObj.revokeObjectURL = (_url) => { };
+}
+function isInlineImageUrl(url) {
+    return url.startsWith('rnblob://') || url.startsWith('data:image/');
+}
+function getMimeTypeFromDataUri(dataUri) {
+    const mimeMatch = dataUri.match(/^data:([^;,]+)[;,]/i);
+    return mimeMatch?.[1] ?? 'image/png';
+}
+function decodeDataUriBytes(dataUri) {
+    const marker = ';base64,';
+    const markerIndex = dataUri.indexOf(marker);
+    if (markerIndex < 0) {
+        throw new Error('[ThreeRNPatch] Unsupported data URI encoding');
+    }
+    const base64 = dataUri.slice(markerIndex + marker.length);
+    return (0, base64_js_1.toByteArray)(base64);
+}
+async function bytesFromBlob(blob) {
+    if (typeof blob.arrayBuffer === 'function') {
+        return new Uint8Array(await blob.arrayBuffer());
+    }
+    const dataUri = await new Promise((resolve, reject) => {
+        const reader = new FileReader();
+        reader.onload = () => resolve(String(reader.result));
+        reader.onerror = () => reject(new Error('[ThreeRNPatch] FileReader failed while decoding blob'));
+        reader.readAsDataURL(blob);
+    });
+    return decodeDataUriBytes(dataUri);
+}
+function installThreeRNPatches(debug = false) {
+    if (react_native_1.Platform.OS === 'web')
+        return;
+    const log = (...args) => {
+        if (debug)
+            console.log('[ThreeRNPatch]', ...args);
+    };
+    // Layer 1: Re-patch URL.createObjectURL on the *current* globalThis.URL.
+    // This must run every call (not guarded by `installed`) because libraries like
+    // @livekit/react-native call setupURLPolyfill() which replaces globalThis.URL
+    // with a new whatwg-url object whose createObjectURL always throws.
+    // We re-apply our override so GLTFLoader's `const URL = self.URL` gets our patch.
+    repatchUrlCreateObjectURL();
+    log('URL.createObjectURL re-patched on current globalThis.URL');
+    if (installed)
+        return;
+    installed = true;
+    log('Installing Three.js RN patches…');
+    // -----------------------------------------------------------------------
+    // Layer 2: THREE.ImageLoader patch — patch prototype to avoid ESM live-binding restrictions
+    // three.webgpu.js is an ESM file; its named exports are getter-only live bindings
+    // from the consumer's perspective, so assigning THREE.ImageLoader = X throws.
+    // Patching the prototype achieves the same effect without touching the namespace.
+    // -----------------------------------------------------------------------
+    const origImageLoaderLoad = THREE.ImageLoader.prototype.load;
+    THREE.ImageLoader.prototype.load = function patchedImageLoad(url, onLoad, onProgress, onError) {
+        if (!url || !isInlineImageUrl(url)) {
+            return origImageLoaderLoad.call(this, url, onLoad, onProgress, onError);
+        }
+        if (url.startsWith('data:image/')) {
+            const img = _makeImageFromDataUri(url, getMimeTypeFromDataUri(url));
+            onLoad?.(img);
+            return img;
+        }
+        const blob = blobStore.get(url);
+        if (!blob) {
+            log(`ImageLoader: no blob found for ${url} — returning placeholder`);
+            const placeholder = _makePlaceholder();
+            onLoad?.(placeholder);
+            return placeholder;
+        }
+        log(`ImageLoader: reading blob ${url} size=${blob.size} type=${blob.type}`);
+        const reader = new FileReader();
+        reader.onload = () => {
+            const dataUri = reader.result;
+            log(`ImageLoader: blob → data URI (${dataUri.length} chars)`);
+            const img = _makeImageFromDataUri(dataUri, blob.type || 'image/png');
+            onLoad?.(img);
+        };
+        reader.onerror = () => {
+            const err = new Error(`[ThreeRNPatch] FileReader failed for ${url}`);
+            log(`ImageLoader: FileReader error for ${url}`);
+            onError?.(err);
+        };
+        reader.readAsDataURL(blob);
+        return _makePlaceholder();
+    };
+    log('Patched THREE.ImageLoader.prototype.load');
+    // -----------------------------------------------------------------------
+    // Layer 3: THREE.TextureLoader safety net — same prototype-patch approach
+    // -----------------------------------------------------------------------
+    const origTextureLoaderLoad = THREE.TextureLoader.prototype.load;
+    THREE.TextureLoader.prototype.load = function patchedTextureLoad(url, onLoad, onProgress, onError) {
+        if (!url || !isInlineImageUrl(url)) {
+            return origTextureLoaderLoad.call(this, url, onLoad, onProgress, onError);
+        }
+        if (url.startsWith('data:image/')) {
+            const tex = new THREE.Texture();
+            tex.image = _makeImageFromDataUri(url, getMimeTypeFromDataUri(url));
+            tex.needsUpdate = true;
+            onLoad?.(tex);
+            return tex;
+        }
+        const blob = blobStore.get(url);
+        if (!blob) {
+            log(`TextureLoader: no blob for ${url} — returning empty texture`);
+            const tex = new THREE.Texture();
+            setTimeout(() => onLoad?.(tex), 0);
+            return tex;
+        }
+        log(`TextureLoader: reading blob ${url} size=${blob.size}`);
+        const tex = new THREE.Texture();
+        const reader = new FileReader();
+        reader.onload = () => {
+            const dataUri = reader.result;
+            const img = _makeImageFromDataUri(dataUri, blob.type || 'image/png');
+            tex.image = img;
+            tex.needsUpdate = true;
+            onLoad?.(tex);
+        };
+        reader.onerror = () => onError?.(new ErrorEvent('error'));
+        reader.readAsDataURL(blob);
+        return tex;
+    };
+    log('Patched THREE.TextureLoader.prototype.load');
+    const ImageBitmapLoader = THREE.ImageBitmapLoader;
+    const origImageBitmapLoad = ImageBitmapLoader?.prototype?.load;
+    if (origImageBitmapLoad) {
+        ImageBitmapLoader.prototype.load = function patchedImageBitmapLoad(url, onLoad, onProgress, onError) {
+            if (!url || !isInlineImageUrl(url)) {
+                return origImageBitmapLoad.call(this, url, onLoad, onProgress, onError);
+            }
+            const loadInlineBitmap = async () => {
+                try {
+                    const bytes = url.startsWith('data:image/')
+                        ? decodeDataUriBytes(url)
+                        : await (async () => {
+                            const blob = blobStore.get(url);
+                            if (!blob)
+                                return null;
+                            return await bytesFromBlob(blob);
+                        })();
+                    if (!bytes) {
+                        throw new Error(`[ThreeRNPatch] No blob found for ${url}`);
+                    }
+                    if (typeof createImageBitmap !== 'function') {
+                        throw new Error('[ThreeRNPatch] createImageBitmap is unavailable');
+                    }
+                    const imageBitmap = await createImageBitmap(bytes);
+                    onLoad?.(imageBitmap);
+                }
+                catch (error) {
+                    onError?.(error instanceof Error ? error : new Error(String(error)));
+                }
+            };
+            void loadInlineBitmap();
+            return undefined;
+        };
+        log('Patched THREE.ImageBitmapLoader.prototype.load');
+    }
+    log('Three.js RN patches installed — embedded GLB textures will now load via FileReader+dataURI');
+}
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+/**
+ * Build a minimal HTMLImageElement-like object from a data URI.
+ * expo-gl / react-three-fiber / @react-three/fiber/native accept this shape
+ * for texture.image on native (it gets handed to gl.texImage2D via Expo's NativeImageLoader).
+ */
+function _makeImageFromDataUri(dataUri, _mimeType) {
+    return {
+        src: dataUri,
+        width: 0, // Three.js reads these after decoding; 0 is fine pre-decode
+        height: 0,
+        complete: false,
+        naturalWidth: 0,
+        naturalHeight: 0,
+        crossOrigin: '',
+        // expo-gl NativeImageLoader checks for 'uri' as well
+        uri: dataUri,
+    };
+}
+/** 1×1 transparent pixel stand-in (used when we have no blob data at all). */
+function _makePlaceholder() {
+    return {
+        src: '',
+        width: 1,
+        height: 1,
+        complete: true,
+        naturalWidth: 1,
+        naturalHeight: 1,
+        crossOrigin: '',
+        uri: '',
+    };
+}

package/dist/wgpu/scenePlacement.d.ts

@@ -0,0 +1,5 @@
+import * as THREE from 'three';
+export declare function computeObjectPlacement(object: THREE.Object3D, target: THREE.Vector3, desiredMaxDimension?: number): {
+    position: THREE.Vector3;
+    scale: number;
+} | null;

package/dist/wgpu/scenePlacement.js

@@ -0,0 +1,50 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.computeObjectPlacement = computeObjectPlacement;
+const THREE = __importStar(require("three"));
+function computeObjectPlacement(object, target, desiredMaxDimension = 1.5) {
+    const box = new THREE.Box3().setFromObject(object);
+    if (box.isEmpty())
+        return null;
+    const size = box.getSize(new THREE.Vector3());
+    const maxDimension = Math.max(size.x, size.y, size.z);
+    if (!Number.isFinite(maxDimension) || maxDimension <= 0)
+        return null;
+    const scale = desiredMaxDimension / maxDimension;
+    const center = box.getCenter(new THREE.Vector3());
+    const position = target.clone().sub(center.multiplyScalar(scale));
+    return { position, scale };
+}

package/dist/wgpu/useAuthedModelUri.js

@@ -15,15 +15,25 @@ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (
 }) : function(o, v) {
     o["default"] = v;
 });
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-};
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.useAuthedModelUri = void 0;
+exports.useAuthedModelUri = useAuthedModelUri;
 const react_1 = require("react");
 const FileSystem = __importStar(require("expo-file-system/legacy"));
 const studioApi_1 = require("../api/studioApi");
@@ -51,7 +61,7 @@ function useAuthedModelUri(remoteUrl) {
             return;
         }
         let cancelled = false;
-        (async () => {
+        void (async () => {
             try {
                 const token = await (0, studioApi_1.getToken)();
                 // Strip query params (access_token changes per session) so the same GLB is cached across sessions
@@ -83,7 +93,9 @@ function useAuthedModelUri(remoteUrl) {
                 });
                 // Verify we got a valid GLB — check size and magic bytes ("glTF")
                 const downloaded = await FileSystem.getInfoAsync(dlResult.uri);
-                const downloadedSize = downloaded.size;
+                const downloadedSize = 'size' in downloaded && typeof downloaded.size === 'number'
+                    ? downloaded.size
+                    : undefined;
                 if (!downloaded.exists || !downloadedSize || downloadedSize < 1000) {
                     console.error('[useAuthedModelUri] Download too small (' + (downloadedSize ?? 0) + 'B), likely an error response');
                     failedUrls.add(remoteUrl);
@@ -114,4 +126,3 @@ function useAuthedModelUri(remoteUrl) {
     }, [remoteUrl]);
     return result;
 }
-exports.useAuthedModelUri = useAuthedModelUri;

package/dist/wgpu/useNativeGLTF.d.ts

@@ -0,0 +1,7 @@
+import type { GLTF } from 'three-stdlib';
+type NativeGLTFState = {
+    gltf: GLTF | null;
+    error: Error | null;
+};
+export declare function useNativeGLTF(uri: string | null): NativeGLTFState;
+export {};

package/dist/wgpu/useNativeGLTF.js

@@ -0,0 +1,36 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.useNativeGLTF = useNativeGLTF;
+const react_1 = require("react");
+const loadGLTFFromUri_1 = require("./loadGLTFFromUri");
+const gltfPromiseCache = new Map();
+function getOrLoad(uri) {
+    const cached = gltfPromiseCache.get(uri);
+    if (cached)
+        return cached;
+    const promise = (0, loadGLTFFromUri_1.loadGLTFFromUri)(uri);
+    gltfPromiseCache.set(uri, promise);
+    return promise;
+}
+function useNativeGLTF(uri) {
+    const [state, setState] = (0, react_1.useState)({ gltf: null, error: null });
+    (0, react_1.useEffect)(() => {
+        if (!uri) {
+            setState({ gltf: null, error: null });
+            return;
+        }
+        let cancelled = false;
+        setState({ gltf: null, error: null });
+        getOrLoad(uri).then((gltf) => {
+            if (!cancelled)
+                setState({ gltf, error: null });
+        }, (error) => {
+            if (!cancelled)
+                setState({ gltf: null, error });
+        });
+        return () => {
+            cancelled = true;
+        };
+    }, [uri]);
+    return state;
+}