talking-head-studio 0.3.6 → 0.3.8

package/dist/utils/faceLandmarkerToShapeWeights.js

@@ -0,0 +1,112 @@
+"use strict";
+/**
+ * faceLandmarkerToShapeWeights
+ *
+ * Runs MediaPipe FaceLandmarker on a photo and returns a
+ * faceShapeWeights Record<string, number> ready to store in
+ * AvatarAppearance. Works with any GLB that has ARKit morph
+ * target names — mesh-agnostic, pure named floats.
+ *
+ * Lazy-loads @mediapipe/tasks-vision on first call (~3MB WASM).
+ * Subsequent calls reuse the same FaceLandmarker instance.
+ *
+ * Apache 2.0 — no API key, no license restrictions.
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.prefetchFaceLandmarker = exports.faceLandmarkerToShapeWeights = void 0;
+const MEDIAPIPE_CDN = 'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@latest/wasm';
+// ---------------------------------------------------------------------------
+// Singleton — built once, reused across calls
+// ---------------------------------------------------------------------------
+let _landmarkerPromise = null;
+async function getLandmarker() {
+    if (_landmarkerPromise)
+        return _landmarkerPromise;
+    _landmarkerPromise = (async () => {
+        // Dynamic import so this module is tree-shaken when unused
+        // @mediapipe/tasks-vision is loaded at runtime from CDN — not in node_modules.
+        // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+        // @ts-ignore
+        const vision = await Promise.resolve().then(() => __importStar(require(
+        /* webpackChunkName: "mediapipe-tasks-vision" */
+        '@mediapipe/tasks-vision')));
+        const { FaceLandmarker, FilesetResolver } = vision;
+        const filesetResolver = await FilesetResolver.forVisionTasks(MEDIAPIPE_CDN);
+        const landmarker = await FaceLandmarker.createFromOptions(filesetResolver, {
+            baseOptions: {
+                modelAssetPath: 'https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task',
+                delegate: 'GPU',
+            },
+            outputFaceBlendshapes: true,
+            runningMode: 'IMAGE',
+            numFaces: 1,
+        });
+        return landmarker;
+    })();
+    // Reset on failure so next call retries
+    _landmarkerPromise.catch(() => {
+        _landmarkerPromise = null;
+    });
+    return _landmarkerPromise;
+}
+/**
+ * Detect face blendshapes from a photo and return a faceShapeWeights
+ * dict compatible with AvatarAppearance.
+ *
+ * @param source  Any image/video/canvas/bitmap element
+ * @param options Optional thresholds
+ * @returns faceShapeWeights Record<string, number>, or null if no face detected
+ */
+async function faceLandmarkerToShapeWeights(source, options = {}) {
+    const { minScore = 0.01, includeNeutral = false } = options;
+    const landmarker = await getLandmarker();
+    const result = landmarker.detect(source);
+    const blendshapes = result.faceBlendshapes?.[0]?.categories;
+    if (!blendshapes || blendshapes.length === 0) {
+        return null; // No face detected
+    }
+    const weights = {};
+    for (const { categoryName, score } of blendshapes) {
+        // MediaPipe includes '_neutral' as a catch-all — skip unless requested
+        if (!includeNeutral && categoryName === '_neutral')
+            continue;
+        if (score < minScore)
+            continue;
+        weights[categoryName] = Math.round(score * 1000) / 1000; // 3 dp is plenty
+    }
+    return weights;
+}
+exports.faceLandmarkerToShapeWeights = faceLandmarkerToShapeWeights;
+/**
+ * Load the FaceLandmarker WASM bundle eagerly — call this on app start
+ * to avoid a cold-start delay when the user first uploads a photo.
+ */
+function prefetchFaceLandmarker() {
+    getLandmarker().catch(() => {
+        // Silently ignore prefetch failures — will retry on actual use
+    });
+}
+exports.prefetchFaceLandmarker = prefetchFaceLandmarker;

package/dist/voice/convertToWav.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.convertToWav = convertToWav;
+exports.convertToWav = void 0;
 /**
  * Convert any audio blob to WAV format using the Web Audio API.
  * Handles WebM/opus output from MediaRecorder without requiring ffmpeg.
@@ -19,6 +19,7 @@ async function convertToWav(audioBlob) {
         await audioContext.close();
     }
 }
+exports.convertToWav = convertToWav;
 /**
  * Encode an AudioBuffer as a 16-bit PCM WAV blob.
  */

package/dist/voice/useAudioPlayer.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.useAudioPlayer = useAudioPlayer;
+exports.useAudioPlayer = void 0;
 const react_1 = require("react");
 function useAudioPlayer({ onError, } = {}) {
     const [isPlaying, setIsPlaying] = (0, react_1.useState)(false);
@@ -62,3 +62,4 @@ function useAudioPlayer({ onError, } = {}) {
         cleanup,
     };
 }
+exports.useAudioPlayer = useAudioPlayer;

package/dist/voice/useAudioRecording.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.useAudioRecording = useAudioRecording;
+exports.useAudioRecording = void 0;
 const react_1 = require("react");
 const convertToWav_1 = require("./convertToWav");
 function useAudioRecording({ maxDurationSeconds = 29, onRecordingComplete, } = {}) {
@@ -163,3 +163,4 @@ function useAudioRecording({ maxDurationSeconds = 29, onRecordingComplete, } = {
         cancelRecording,
     };
 }
+exports.useAudioRecording = useAudioRecording;

package/dist/wardrobe/useAvatarWardrobeHydration.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.useAvatarWardrobeHydration = useAvatarWardrobeHydration;
+exports.useAvatarWardrobeHydration = void 0;
 const react_1 = require("react");
 const studioApi_1 = require("../api/studioApi");
 const wardrobeStore_1 = require("./wardrobeStore");
@@ -32,3 +32,4 @@ function useAvatarWardrobeHydration({ avatarId, accessories, }) {
         };
     }, [accessorySignature, avatarId, hydrateFromApi]);
 }
+exports.useAvatarWardrobeHydration = useAvatarWardrobeHydration;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "talking-head-studio",
-  "version": "0.3.6",
+  "version": "0.3.8",
   "description": "Cross-platform 3D avatar component for React Native & web — lip-sync, gestures, accessories, and LLM integration. Powered by TalkingHead + Three.js.",
   "main": "dist/index.web.js",
   "browser": "dist/index.web.js",
@@ -58,7 +58,8 @@
     "lint": "eslint 'src/**/*.{ts,tsx}'",
     "format": "prettier --write 'src/**/*.{ts,tsx}'",
     "test": "jest",
-    "prepublishOnly": "npm run lint && npm run typecheck && npm test -- --runInBand && npm run build"
+    "prepublishOnly": "npm run lint && npm run typecheck && npm test -- --runInBand && npm run build",
+    "build:creator": "vite build --config creator-app/vite.config.ts"
   },
   "keywords": [
     "react-native",
@@ -97,6 +98,7 @@
   },
   "sideEffects": false,
   "dependencies": {
+    "@mediapipe/tasks-vision": "^0.10.34",
     "zustand": "^5.0.12"
   },
   "peerDependencies": {
@@ -146,12 +148,17 @@
     "@babel/preset-typescript": "^7.28.5",
     "@expo/vector-icons": "^15.1.1",
     "@react-native/babel-preset": "^0.84.1",
+    "@react-three/drei": "^9.122.0",
+    "@react-three/fiber": "^8.18.0",
     "@testing-library/react-native": "^13.3.3",
     "@types/jest": "^30.0.0",
     "@types/react": "^19.2.14",
+    "@types/react-dom": "^18.3.7",
     "@types/react-native": "^0.73.0",
+    "@types/three": "^0.180.0",
     "@typescript-eslint/eslint-plugin": "^8.56.1",
     "@typescript-eslint/parser": "^8.56.1",
+    "@vitejs/plugin-react": "^4.7.0",
     "babel-jest": "^30.2.0",
     "eslint": "^9.39.3",
     "eslint-config-prettier": "^10.1.8",
@@ -166,10 +173,15 @@
     "moti": "^0.30.0",
     "multer": "^2.1.0",
     "prettier": "^3.8.1",
+    "react": "^18.3.1",
+    "react-dom": "^18.3.1",
     "react-native-gesture-handler": "^2.30.0",
     "react-native-reanimated": "^4.2.3",
     "react-native-webview": "^13.16.0",
     "react-test-renderer": "^19.2.4",
-    "ts-jest": "^29.4.6"
+    "three": "^0.180.0",
+    "ts-jest": "^29.4.6",
+    "typescript": "^5.3.3",
+    "vite": "^5.4.21"
   }
 }