talking-head-studio 0.4.11 → 0.4.12

package/dist/core/avatar/schema.d.ts

@@ -1,5 +1,7 @@
 import type { OculusViseme } from './visemes';
-export type VísemeTier = 'oculus' | 'arkit' | 'minimal' | 'none';
+export type VisemeTier = 'oculus' | 'arkit' | 'minimal' | 'none';
+/** @deprecated Use VisemeTier. */
+export type VísemeTier = VisemeTier;
 export interface MorphCoverage {
     /** All morph target names found across all meshes in the scene. */
     all: string[];
@@ -12,7 +14,7 @@ export interface MorphCoverage {
     /** Whether a meaningful ARKit set is present (≥10 mouth shapes). */
     hasArkitSet: boolean;
     /** Best lip-sync strategy available for this model. */
-    visemeTier: VísemeTier;
+    visemeTier: VisemeTier;
 }
 export interface SkeletonInfo {
     /** Whether a skeleton (SkinnedMesh) was found. */
@@ -39,12 +41,38 @@ export interface AvatarSchemaReport {
     /** Whether this model is ready for full-quality talking-head rendering. */
     isReadyForLipSync: boolean;
 }
+/** Minimal scene shape needed by the GLB schema walker. */
+interface AvatarSchemaScene {
+    traverse: (cb: (obj: AvatarSchemaObject) => void) => void;
+}
+/** Minimal traversed object shape used to avoid depending on runtime Three.js classes. */
+interface AvatarSchemaObject {
+    isSkinnedMesh?: boolean;
+    isBone?: boolean;
+    isMesh?: boolean;
+    name?: string;
+    skeleton?: {
+        bones?: Array<{
+            name?: string;
+        }>;
+    };
+    morphTargetDictionary?: Record<string, unknown>;
+    geometry?: {
+        index?: {
+            count: number;
+        };
+        attributes?: {
+            position?: {
+                count: number;
+            };
+        };
+    };
+}
 /**
  * Scan a loaded Three.js Object3D scene and return a structured compatibility
  * report for lip-sync, skeleton, and mesh quality.
  *
  * @param scene - The root Object3D from a GLTFLoader result (gltf.scene)
  */
-export declare function walkAvatarSchema(scene: {
-    traverse: (cb: (obj: any) => void) => void;
-}): AvatarSchemaReport;
+export declare function walkAvatarSchema(scene: AvatarSchemaScene): AvatarSchemaReport;
+export {};

package/dist/core/avatar/visemes.d.ts

@@ -2,11 +2,12 @@
  * Standard viseme keys supported by the avatar (Oculus set).
  */
 export type OculusViseme = 'sil' | 'PP' | 'FF' | 'TH' | 'DD' | 'kk' | 'CH' | 'SS' | 'nn' | 'RR' | 'aa' | 'ee' | 'ih' | 'oh' | 'ou';
+export type RhubarbViseme = 'A' | 'B' | 'C' | 'D' | 'E' | 'F' | 'G' | 'H' | 'X';
 /** Rhubarb mouth shape cue (Preston Blair set: A-H, X) */
 export interface VisemeCue {
     startMs: number;
     endMs: number;
-    viseme: 'A' | 'B' | 'C' | 'D' | 'E' | 'F' | 'G' | 'H' | 'X';
+    viseme: RhubarbViseme;
     weight?: number;
 }
 /**
@@ -31,6 +32,10 @@ export interface AgentVisemePayload {
     cues: VisemeCue[];
 }
 export declare const OCULUS_VISEMES: OculusViseme[];
+export declare const RHUBARB_VISEMES: RhubarbViseme[];
+export declare const OCULUS_MORPH_TARGET_NAMES: Record<OculusViseme, string>;
+export type OculusVisemeWeights = Partial<Record<OculusViseme, number>>;
+export declare const RHUBARB_TO_OCULUS: Record<RhubarbViseme, OculusVisemeWeights>;
 export type ArkitBlendShape = 'jawOpen' | 'jawForward' | 'jawLeft' | 'jawRight' | 'mouthClose' | 'mouthFunnel' | 'mouthPucker' | 'mouthLeft' | 'mouthRight' | 'mouthSmileLeft' | 'mouthSmileRight' | 'mouthFrownLeft' | 'mouthFrownRight' | 'mouthDimpleLeft' | 'mouthDimpleRight' | 'mouthStretchLeft' | 'mouthStretchRight' | 'mouthRollLower' | 'mouthRollUpper' | 'mouthShrugLower' | 'mouthShrugUpper' | 'mouthPressLeft' | 'mouthPressRight' | 'mouthLowerDownLeft' | 'mouthLowerDownRight' | 'mouthUpperUpLeft' | 'mouthUpperUpRight' | 'tongueOut';
 /**
  * Each Oculus viseme expressed as a weighted combination of ARKit blend shapes.
@@ -41,6 +46,16 @@ export type ArkitBlendShape = 'jawOpen' | 'jawForward' | 'jawLeft' | 'jawRight'
  * - Empirical tuning against RPM and face-squeeze avatars
  */
 export declare const ARKIT_TO_OCULUS: Record<OculusViseme, Partial<Record<ArkitBlendShape, number>>>;
+export type EngineVisemeMappingArtifact = {
+    schemaVersion: 1;
+    rhubarbVisemes: RhubarbViseme[];
+    oculusVisemes: OculusViseme[];
+    oculusMorphTargetNames: Record<OculusViseme, string>;
+    rhubarbToOculus: Record<RhubarbViseme, OculusVisemeWeights>;
+    oculusToArkit: Record<OculusViseme, Partial<Record<ArkitBlendShape, number>>>;
+};
+export declare const ENGINE_VISEME_MAPPING: EngineVisemeMappingArtifact;
+export declare function oculusWeightsForRhubarbCue(cue: Pick<VisemeCue, 'viseme' | 'weight'>): Record<OculusViseme, number>;
 /**
  * Remap a set of ARKit blend shape weights to Oculus viseme weights.
  *

package/dist/core/avatar/visemes.js

@@ -1,11 +1,41 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ARKIT_TO_OCULUS = exports.OCULUS_VISEMES = void 0;
+exports.ENGINE_VISEME_MAPPING = exports.ARKIT_TO_OCULUS = exports.RHUBARB_TO_OCULUS = exports.OCULUS_MORPH_TARGET_NAMES = exports.RHUBARB_VISEMES = exports.OCULUS_VISEMES = void 0;
+exports.oculusWeightsForRhubarbCue = oculusWeightsForRhubarbCue;
 exports.remapArkitToOculus = remapArkitToOculus;
 exports.getArkitWeightsForViseme = getArkitWeightsForViseme;
 exports.OCULUS_VISEMES = [
     'sil', 'PP', 'FF', 'TH', 'DD', 'kk', 'CH', 'SS', 'nn', 'RR', 'aa', 'ee', 'ih', 'oh', 'ou',
 ];
+exports.RHUBARB_VISEMES = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'X'];
+exports.OCULUS_MORPH_TARGET_NAMES = {
+    sil: 'viseme_sil',
+    PP: 'viseme_PP',
+    FF: 'viseme_FF',
+    TH: 'viseme_TH',
+    DD: 'viseme_DD',
+    kk: 'viseme_kk',
+    CH: 'viseme_CH',
+    SS: 'viseme_SS',
+    nn: 'viseme_nn',
+    RR: 'viseme_RR',
+    aa: 'viseme_aa',
+    ee: 'viseme_ee',
+    ih: 'viseme_ih',
+    oh: 'viseme_oh',
+    ou: 'viseme_ou',
+};
+exports.RHUBARB_TO_OCULUS = {
+    A: { aa: 0.72 },
+    B: { PP: 0.85 },
+    C: { ih: 0.68 },
+    D: { FF: 0.78 },
+    E: { ee: 0.72 },
+    F: { oh: 0.62 },
+    G: { ou: 0.58 },
+    H: { nn: 0.72 },
+    X: { sil: 1 },
+};
 /**
  * Each Oculus viseme expressed as a weighted combination of ARKit blend shapes.
  * Weights are in 0–1 range. Apply by multiplying each ARKit shape's influence.
@@ -31,6 +61,23 @@ exports.ARKIT_TO_OCULUS = {
     oh: { jawOpen: 0.4, mouthFunnel: 0.8 },
     ou: { mouthPucker: 0.9, mouthRollLower: 0.3 },
 };
+exports.ENGINE_VISEME_MAPPING = {
+    schemaVersion: 1,
+    rhubarbVisemes: exports.RHUBARB_VISEMES,
+    oculusVisemes: exports.OCULUS_VISEMES,
+    oculusMorphTargetNames: exports.OCULUS_MORPH_TARGET_NAMES,
+    rhubarbToOculus: exports.RHUBARB_TO_OCULUS,
+    oculusToArkit: exports.ARKIT_TO_OCULUS,
+};
+function oculusWeightsForRhubarbCue(cue) {
+    const weight = Math.max(0, Math.min(1, cue.weight ?? 1));
+    const result = Object.fromEntries(exports.OCULUS_VISEMES.map((viseme) => [viseme, 0]));
+    const mapping = exports.RHUBARB_TO_OCULUS[cue.viseme];
+    for (const [viseme, value] of Object.entries(mapping)) {
+        result[viseme] = Math.max(0, Math.min(1, value * weight));
+    }
+    return result;
+}
 /**
  * Remap a set of ARKit blend shape weights to Oculus viseme weights.
  *

package/dist/editor/AvatarCanvas.js

@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.AvatarCanvas = AvatarCanvas;
 const jsx_runtime_1 = require("react/jsx-runtime");
 // @ts-nocheck
+/* eslint-disable @typescript-eslint/no-explicit-any -- R3F pointer events and Three.js scene internals are intentionally dynamic in this editor bridge. */
 const drei_1 = require("@react-three/drei");
 const fiber_1 = require("@react-three/fiber");
 const react_1 = require("react");
@@ -10,6 +11,96 @@ const AvatarCanvasErrorBoundary_1 = require("./AvatarCanvasErrorBoundary");
 const AvatarModel_1 = require("./AvatarModel");
 const RigidAccessory_1 = require("./RigidAccessory");
 const SkinnedClothing_1 = require("./SkinnedClothing");
+function DirectTransform({ activeAssetId, placements, onPlacementChange }) {
+    const { gl } = (0, fiber_1.useThree)();
+    (0, react_1.useEffect)(() => {
+        const canvas = gl.domElement;
+        let dragging = false;
+        let lastX = 0;
+        let lastY = 0;
+        function isModified(e) {
+            return e.shiftKey || e.ctrlKey || e.metaKey;
+        }
+        function onPointerDown(e) {
+            if (!activeAssetId || !isModified(e))
+                return;
+            e.stopPropagation();
+            canvas.setPointerCapture(e.pointerId);
+            dragging = true;
+            lastX = e.clientX;
+            lastY = e.clientY;
+        }
+        function onPointerMove(e) {
+            if (!dragging || !activeAssetId || !onPlacementChange)
+                return;
+            const rect = canvas.getBoundingClientRect();
+            // Normalise deltas to canvas size so sensitivity is viewport-independent
+            const dx = (e.clientX - lastX) / rect.width;
+            const dy = (e.clientY - lastY) / rect.height;
+            lastX = e.clientX;
+            lastY = e.clientY;
+            const p = placements[activeAssetId];
+            if (!p)
+                return;
+            const TRANSLATE_SPEED = 2.0;
+            const ROTATE_SPEED = Math.PI * 2;
+            if (e.shiftKey && (e.altKey || e.metaKey)) {
+                // Shift+Alt/Meta drag → Y axis (lift/lower)
+                onPlacementChange(activeAssetId, {
+                    ...p,
+                    position: [p.position[0], p.position[1] - dy * TRANSLATE_SPEED, p.position[2]],
+                });
+            }
+            else if (e.shiftKey) {
+                // Shift drag → XZ plane (slide left/right/forward/back)
+                onPlacementChange(activeAssetId, {
+                    ...p,
+                    position: [
+                        p.position[0] + dx * TRANSLATE_SPEED,
+                        p.position[1],
+                        p.position[2] + dy * TRANSLATE_SPEED,
+                    ],
+                });
+            }
+            else if (e.ctrlKey || e.metaKey) {
+                // Ctrl/Cmd drag → Y-axis rotation (yaw)
+                onPlacementChange(activeAssetId, {
+                    ...p,
+                    rotation: [p.rotation[0], p.rotation[1] + dx * ROTATE_SPEED, p.rotation[2]],
+                });
+            }
+        }
+        function onPointerUp(e) {
+            if (dragging)
+                canvas.releasePointerCapture(e.pointerId);
+            dragging = false;
+        }
+        function onWheel(e) {
+            if (!e.shiftKey || !activeAssetId || !onPlacementChange)
+                return;
+            e.preventDefault();
+            const p = placements[activeAssetId];
+            if (!p)
+                return;
+            const factor = e.deltaY > 0 ? 0.95 : 1.05;
+            onPlacementChange(activeAssetId, {
+                ...p,
+                scale: Math.max(0.01, (p.scale ?? 1) * factor),
+            });
+        }
+        canvas.addEventListener('pointerdown', onPointerDown);
+        canvas.addEventListener('pointermove', onPointerMove);
+        canvas.addEventListener('pointerup', onPointerUp);
+        canvas.addEventListener('wheel', onWheel, { passive: false });
+        return () => {
+            canvas.removeEventListener('pointerdown', onPointerDown);
+            canvas.removeEventListener('pointermove', onPointerMove);
+            canvas.removeEventListener('pointerup', onPointerUp);
+            canvas.removeEventListener('wheel', onWheel);
+        };
+    }, [activeAssetId, placements, onPlacementChange, gl]);
+    return null;
+}
 /** Captures the R3F scene object (which contains everything) and passes it up. */
 function SceneRefCapture({ onSceneRef }) {
     const { scene } = (0, fiber_1.useThree)();
@@ -45,7 +136,7 @@ function AvatarCanvas({ avatarUrl, equipped = [], placements = {}, editingAssetI
     ]
         .filter(Boolean)
         .join(' ');
-    return ((0, jsx_runtime_1.jsxs)("div", { className: wrapperClass, style: style, children: [(0, jsx_runtime_1.jsx)("div", { className: "absolute inset-0 opacity-[0.04] pointer-events-none", style: { backgroundImage: 'linear-gradient(rgba(255,255,255,0.3) 1px,transparent 1px),linear-gradient(90deg,rgba(255,255,255,0.3) 1px,transparent 1px)', backgroundSize: '60px 60px' } }), avatarUrl ? ((0, jsx_runtime_1.jsx)(AvatarCanvasErrorBoundary_1.AvatarCanvasErrorBoundary, { children: (0, jsx_runtime_1.jsx)(fiber_1.Canvas, { ref: canvasRef, camera: { position: cameraPosition, fov: 45 }, style: { width: '100%', height: '100%' }, children: (0, jsx_runtime_1.jsxs)(react_1.Suspense, { fallback: null, children: [onSceneRef && (0, jsx_runtime_1.jsx)(SceneRefCapture, { onSceneRef: onSceneRef }), (0, jsx_runtime_1.jsx)("ambientLight", { intensity: 0.6 }), (0, jsx_runtime_1.jsx)("directionalLight", { position: [5, 5, 5], intensity: 0.8 }), (0, jsx_runtime_1.jsx)(drei_1.Environment, { preset: "studio" }), (0, jsx_runtime_1.jsx)(drei_1.OrbitControls, { target: cameraTarget, minDistance: 1, maxDistance: 10, enablePan: false, makeDefault: true, enabled: !editingId }), (0, jsx_runtime_1.jsx)(AvatarModel_1.AvatarModel, { url: avatarUrl, scale: 1, onSkeletonReady: handleSkeletonReady, onBoundsReady: handleBoundsReady }), equippedItems.map((asset) => {
+    return ((0, jsx_runtime_1.jsxs)("div", { className: wrapperClass, style: style, children: [(0, jsx_runtime_1.jsx)("div", { className: "absolute inset-0 opacity-[0.04] pointer-events-none", style: { backgroundImage: 'linear-gradient(rgba(255,255,255,0.3) 1px,transparent 1px),linear-gradient(90deg,rgba(255,255,255,0.3) 1px,transparent 1px)', backgroundSize: '60px 60px' } }), avatarUrl ? ((0, jsx_runtime_1.jsx)(AvatarCanvasErrorBoundary_1.AvatarCanvasErrorBoundary, { children: (0, jsx_runtime_1.jsx)(fiber_1.Canvas, { ref: canvasRef, camera: { position: cameraPosition, fov: 45 }, style: { width: '100%', height: '100%' }, children: (0, jsx_runtime_1.jsxs)(react_1.Suspense, { fallback: null, children: [onSceneRef && (0, jsx_runtime_1.jsx)(SceneRefCapture, { onSceneRef: onSceneRef }), (0, jsx_runtime_1.jsx)("ambientLight", { intensity: 0.6 }), (0, jsx_runtime_1.jsx)("directionalLight", { position: [5, 5, 5], intensity: 0.8 }), (0, jsx_runtime_1.jsx)(drei_1.Environment, { preset: "studio" }), (0, jsx_runtime_1.jsx)(drei_1.OrbitControls, { target: cameraTarget, minDistance: 1, maxDistance: 10, enablePan: false, makeDefault: true }), (0, jsx_runtime_1.jsx)(drei_1.GizmoHelper, { alignment: "bottom-right", margin: [60, 60], children: (0, jsx_runtime_1.jsx)(drei_1.GizmoViewport, { axisColors: ['#e24444', '#44cc44', '#4488ff'], labelColor: "white" }) }), (0, jsx_runtime_1.jsx)(DirectTransform, { activeAssetId: editingId, placements: placements, onPlacementChange: onPlacementChange }), (0, jsx_runtime_1.jsx)(AvatarModel_1.AvatarModel, { url: avatarUrl, scale: 1, onSkeletonReady: handleSkeletonReady, onBoundsReady: handleBoundsReady }), equippedItems.map((asset) => {
                                 if (!asset)
                                     return null;
                                 const llmPlacement = placements[asset.id];

package/dist/editor/AvatarEditor.native.js

@@ -40,6 +40,7 @@ const jsx_runtime_1 = require("react/jsx-runtime");
  * Uses WgpuAvatar (R3F + expo-gl) as the render surface and Three.js raycasting
  * for gesture-driven accessory placement. Replaces the old FilamentEditor.
  */
+/* eslint-disable @typescript-eslint/no-explicit-any -- React Native gesture events and Three.js raycast payloads do not expose complete structural types here. */
 const react_1 = require("react");
 const react_native_1 = require("react-native");
 const THREE = __importStar(require("three"));

package/dist/editor/AvatarModel.js

@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.AvatarModel = AvatarModel;
 const jsx_runtime_1 = require("react/jsx-runtime");
 // @ts-nocheck
+/* eslint-disable @typescript-eslint/no-explicit-any -- R3F GLTF skeleton data is passed through from untyped loader internals. */
 const drei_1 = require("@react-three/drei");
 const fiber_1 = require("@react-three/fiber");
 const react_1 = require("react");

package/dist/editor/FaceSqueezeEditor.d.ts

@@ -10,8 +10,10 @@
  * Drop-in replacement: same FaceSqueezeEditorProps interface.
  * The .web.tsx stub is shared — import from FaceSqueezeEditor.web.tsx.
  */
+import '../wgpu/blobShim';
 export declare const FACE_SQUEEZE_LOCAL_MODULE: number;
 export interface FaceSqueezeEditorProps {
     onClose: () => void;
+    avatarModule?: unknown;
 }
-export declare function FaceSqueezeEditor({ onClose }: FaceSqueezeEditorProps): import("react/jsx-runtime").JSX.Element;
+export declare function FaceSqueezeEditor({ onClose, avatarModule, }: FaceSqueezeEditorProps): import("react/jsx-runtime").JSX.Element;