talking-head-studio 0.4.9 → 0.4.11

package/dist/TalkingHead.d.ts

@@ -1,6 +1,7 @@
 import React from 'react';
 import { type StyleProp, type ViewStyle } from 'react-native';
 export type TalkingHeadMood = 'neutral' | 'happy' | 'sad' | 'angry' | 'excited' | 'thinking' | 'concerned' | 'surprised';
+import type { AgentVisemePayload, OculusViseme, VisemeCue } from './core/avatar/visemes';
 export type TalkingHeadLoadingStage = 'booting' | 'fetching_model' | 'loading_avatar' | 'loading_fallback' | 'ready';
 export interface TalkingHeadLoadingState {
     stage: TalkingHeadLoadingStage;
@@ -9,36 +10,20 @@ export interface TalkingHeadLoadingState {
 /**
  * Standard viseme keys supported by the avatar.
  * Use with sendViseme() from your TTS viseme callbacks.
+ * @deprecated Use OculusViseme from @yourplatform/avatar/core/avatar/visemes
  */
-export type TalkingHeadViseme = 'sil' | 'PP' | 'FF' | 'TH' | 'DD' | 'kk' | 'CH' | 'SS' | 'nn' | 'RR' | 'aa' | 'ee' | 'ih' | 'oh' | 'ou';
-/** Rhubarb mouth shape cue (Preston Blair set: A-H, X) */
-export interface TalkingHeadVisemeCue {
-    startMs: number;
-    endMs: number;
-    viseme: 'A' | 'B' | 'C' | 'D' | 'E' | 'F' | 'G' | 'H' | 'X';
-}
+export type TalkingHeadViseme = OculusViseme;
+/**
+ * Rhubarb mouth shape cue (Preston Blair set: A-H, X)
+ * @deprecated Use VisemeCue from @yourplatform/avatar/core/avatar/visemes
+ */
+export type TalkingHeadVisemeCue = VisemeCue;
 /**
  * A full viseme schedule from the Rhubarb sidecar endpoint.
  * Pass to scheduleVisemes() when agent_visemes arrives on the data channel.
+ * @deprecated Use AgentVisemePayload from @yourplatform/avatar/core/avatar/visemes
  */
-export interface TalkingHeadVisemeSchedule {
-    /** Matches X-TTS-Request-Id / agent_visemes.requestId */
-    requestId?: string;
-    /**
-     * Wall-clock ms at which the TTS request was fired (agent side).
-     * Used as the scheduling anchor plus AUDIO_PIPELINE_DELAY_MS.
-     */
-    startedAtMs?: number;
-    /**
-     * Wall-clock ms at which audio actually began playing in the speaker.
-     * When present, used directly as the scheduling anchor with no additional
-     * pipeline offset — more accurate than startedAtMs on fast connections.
-     * Stamp this from the LiveKit onAudioPlaybackStarted callback if available.
-     */
-    audioStartedAtMs?: number;
-    durationMs?: number;
-    cues: TalkingHeadVisemeCue[];
-}
+export type TalkingHeadVisemeSchedule = AgentVisemePayload;
 export interface TalkingHeadAccessory {
     id: string;
     url: string;
@@ -66,6 +51,9 @@ export interface TalkingHeadProps {
     /** Base URL for vendored assets. When set, replaces all cdn.jsdelivr.net references. */
     vendorBaseUrl?: string | null;
 }
+/** @deprecated Use AvatarPlayerProps */
+export type TalkingHeadPropsAlias = TalkingHeadProps;
+export type AvatarPlayerProps = TalkingHeadProps;
 export interface TalkingHeadRef {
     sendAmplitude: (amplitude: number) => void;
     /**
@@ -103,4 +91,7 @@ export interface TalkingHeadRef {
     /** Dispatch a named motion/gesture to the avatar (e.g. 'wave_right', 'dance_idle'). */
     dispatchMotion: (name: string) => void;
 }
+/** @deprecated Use AvatarPlayerRef */
+export type TalkingHeadRefAlias = TalkingHeadRef;
+export type AvatarPlayerRef = TalkingHeadRef;
 export declare const TalkingHead: React.ForwardRefExoticComponent<TalkingHeadProps & React.RefAttributes<TalkingHeadRef>>;

package/dist/TalkingHead.web.d.ts

@@ -26,6 +26,9 @@ export interface TalkingHeadProps {
     onAvatarState?: (state: string) => void;
     style?: React.CSSProperties;
 }
+/** @deprecated Use AvatarPlayerProps */
+export type TalkingHeadPropsAlias = TalkingHeadProps;
+export type AvatarPlayerProps = TalkingHeadProps;
 export interface TalkingHeadRef {
     sendAmplitude: (amplitude: number) => void;
     scheduleVisemes: (schedule: TalkingHeadVisemeSchedule) => void;
@@ -37,4 +40,7 @@ export interface TalkingHeadRef {
     setAccessories: (accessories: TalkingHeadAccessory[]) => void;
     dispatchMotion: (name: string) => void;
 }
+/** @deprecated Use AvatarPlayerRef */
+export type TalkingHeadRefAlias = TalkingHeadRef;
+export type AvatarPlayerRef = TalkingHeadRef;
 export declare const TalkingHead: React.ForwardRefExoticComponent<TalkingHeadProps & React.RefAttributes<TalkingHeadRef>>;

package/dist/TalkingHead.web.js

@@ -15,13 +15,23 @@ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (
 }) : function(o, v) {
     o["default"] = v;
 });
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-};
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.TalkingHead = void 0;
 const jsx_runtime_1 = require("react/jsx-runtime");
@@ -203,6 +213,6 @@ exports.TalkingHead = (0, react_1.forwardRef)(({ avatarUrl, authToken, mood = 'n
         window.addEventListener('message', onMessage);
         return () => window.removeEventListener('message', onMessage);
     }, [post]);
-    return ((0, jsx_runtime_1.jsx)("div", { style: { ...containerStyle, ...style }, children: (0, jsx_runtime_1.jsx)("iframe", { ref: iframeRef, srcDoc: srcdoc, style: iframeStyle, sandbox: "allow-scripts allow-same-origin", title: "TalkingHead Avatar" }) }));
+    return ((0, jsx_runtime_1.jsx)("div", { style: { ...containerStyle, ...style }, children: (0, jsx_runtime_1.jsx)("iframe", { ref: iframeRef, srcDoc: srcdoc, style: iframeStyle, title: "TalkingHead Avatar" }) }));
 });
 exports.TalkingHead.displayName = 'TalkingHead';

package/dist/api/studioApi.js

@@ -1,6 +1,30 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.uploadVoiceSample = exports.createVoiceProfile = exports.assetFileUrl = exports.suggestPlacement = exports.deleteAsset = exports.uploadAsset = exports.getAsset = exports.listAssets = exports.thumbnailHeaders = exports.avatarAnimatedThumbnailUrl = exports.avatarThumbnailUrl = exports.avatarFileUrl = exports.removeDefaultVoice = exports.setDefaultVoice = exports.getVoiceProfiles = exports.getVoiceProfileSamples = exports.getPublicAvatars = exports.createAvatar = exports.deleteAvatar = exports.updateAvatar = exports.getAvatar = exports.getMyAvatars = exports.studioFetch = exports.getToken = exports.configureAvatarApi = void 0;
+exports.configureAvatarApi = configureAvatarApi;
+exports.getToken = getToken;
+exports.studioFetch = studioFetch;
+exports.getMyAvatars = getMyAvatars;
+exports.getAvatar = getAvatar;
+exports.updateAvatar = updateAvatar;
+exports.deleteAvatar = deleteAvatar;
+exports.createAvatar = createAvatar;
+exports.getPublicAvatars = getPublicAvatars;
+exports.getVoiceProfileSamples = getVoiceProfileSamples;
+exports.getVoiceProfiles = getVoiceProfiles;
+exports.setDefaultVoice = setDefaultVoice;
+exports.removeDefaultVoice = removeDefaultVoice;
+exports.avatarFileUrl = avatarFileUrl;
+exports.avatarThumbnailUrl = avatarThumbnailUrl;
+exports.avatarAnimatedThumbnailUrl = avatarAnimatedThumbnailUrl;
+exports.thumbnailHeaders = thumbnailHeaders;
+exports.listAssets = listAssets;
+exports.getAsset = getAsset;
+exports.uploadAsset = uploadAsset;
+exports.deleteAsset = deleteAsset;
+exports.suggestPlacement = suggestPlacement;
+exports.assetFileUrl = assetFileUrl;
+exports.createVoiceProfile = createVoiceProfile;
+exports.uploadVoiceSample = uploadVoiceSample;
 // ---------------------------------------------------------------------------
 // Injectable configuration
 // ---------------------------------------------------------------------------
@@ -11,11 +35,9 @@ function configureAvatarApi(opts) {
         _baseUrl = opts.baseUrl;
     _getToken = opts.getToken;
 }
-exports.configureAvatarApi = configureAvatarApi;
 async function getToken() {
     return _getToken ? _getToken() : null;
 }
-exports.getToken = getToken;
 // ---------------------------------------------------------------------------
 // Core fetch wrappers
 // ---------------------------------------------------------------------------
@@ -48,7 +70,6 @@ async function studioFetch(path, options) {
     }
     return res.json();
 }
-exports.studioFetch = studioFetch;
 /** POST/PUT with FormData, sharing auth token logic. */
 async function studioFetchForm(path, method, formData) {
     const token = await getToken();
@@ -76,11 +97,9 @@ async function studioFetchForm(path, method, formData) {
 function getMyAvatars() {
     return studioFetch('/v1/avatars');
 }
-exports.getMyAvatars = getMyAvatars;
 function getAvatar(id) {
     return studioFetch(`/v1/avatars/${id}`);
 }
-exports.getAvatar = getAvatar;
 function updateAvatar(id, data) {
     return studioFetch(`/v1/avatars/${id}`, {
         method: 'PATCH',
@@ -88,11 +107,9 @@ function updateAvatar(id, data) {
         body: JSON.stringify(data),
     });
 }
-exports.updateAvatar = updateAvatar;
 function deleteAvatar(id) {
     return studioFetch(`/v1/avatars/${id}`, { method: 'DELETE' });
 }
-exports.deleteAvatar = deleteAvatar;
 async function createAvatar(fileUri, name, description) {
     const formData = new FormData();
     formData.append('file', {
@@ -106,22 +123,18 @@ async function createAvatar(fileUri, name, description) {
     }
     return studioFetchForm('/v1/avatars', 'POST', formData);
 }
-exports.createAvatar = createAvatar;
 function getPublicAvatars() {
     return studioFetch('/v1/avatars/public');
 }
-exports.getPublicAvatars = getPublicAvatars;
 // ---------------------------------------------------------------------------
 // Voice profile endpoints
 // ---------------------------------------------------------------------------
 function getVoiceProfileSamples(profileId) {
     return studioFetch(`/profiles/${profileId}/samples`);
 }
-exports.getVoiceProfileSamples = getVoiceProfileSamples;
 function getVoiceProfiles() {
     return studioFetch('/profiles');
 }
-exports.getVoiceProfiles = getVoiceProfiles;
 function setDefaultVoice(avatarId, profileId) {
     return studioFetch(`/v1/avatars/${avatarId}/default-voice`, {
         method: 'PUT',
@@ -129,13 +142,11 @@ function setDefaultVoice(avatarId, profileId) {
         body: JSON.stringify({ profile_id: profileId }),
     });
 }
-exports.setDefaultVoice = setDefaultVoice;
 function removeDefaultVoice(avatarId) {
     return studioFetch(`/v1/avatars/${avatarId}/default-voice`, {
         method: 'DELETE',
     });
 }
-exports.removeDefaultVoice = removeDefaultVoice;
 // ---------------------------------------------------------------------------
 // URL helpers
 // ---------------------------------------------------------------------------
@@ -143,27 +154,23 @@ function avatarFileUrl(avatar) {
     const base = `${_baseUrl}${avatar.url}`;
     return avatar.updated_at ? `${base}?v=${encodeURIComponent(avatar.updated_at)}` : base;
 }
-exports.avatarFileUrl = avatarFileUrl;
 function avatarThumbnailUrl(avatar) {
     const url = avatar.thumbnail_url || avatar.animated_thumbnail_url;
     if (!url)
         return null;
     return `${_baseUrl}${url}`;
 }
-exports.avatarThumbnailUrl = avatarThumbnailUrl;
 function avatarAnimatedThumbnailUrl(avatar) {
     if (!avatar.animated_thumbnail_url)
         return null;
     return `${_baseUrl}${avatar.animated_thumbnail_url}`;
 }
-exports.avatarAnimatedThumbnailUrl = avatarAnimatedThumbnailUrl;
 async function thumbnailHeaders() {
     const token = await getToken();
     if (!token)
         return {};
     return { Authorization: `Bearer ${token}` };
 }
-exports.thumbnailHeaders = thumbnailHeaders;
 // ---------------------------------------------------------------------------
 // Wearable Asset endpoints
 // ---------------------------------------------------------------------------
@@ -171,11 +178,9 @@ function listAssets(category) {
     const query = category ? `?category=${encodeURIComponent(category)}` : '';
     return studioFetch(`/v1/assets${query}`);
 }
-exports.listAssets = listAssets;
 function getAsset(id) {
     return studioFetch(`/v1/assets/${id}`);
 }
-exports.getAsset = getAsset;
 async function uploadAsset(fileUri, meta) {
     const formData = new FormData();
     formData.append('file', {
@@ -198,20 +203,16 @@ async function uploadAsset(fileUri, meta) {
         formData.append('hides_body_parts', JSON.stringify(meta.hides_body_parts));
     return studioFetchForm('/v1/assets', 'POST', formData);
 }
-exports.uploadAsset = uploadAsset;
 function deleteAsset(id) {
     return studioFetch(`/v1/assets/${id}`, { method: 'DELETE' });
 }
-exports.deleteAsset = deleteAsset;
 function suggestPlacement(assetId, avatarId) {
     return studioFetch(`/v1/assets/${assetId}/suggest-placement?avatar_id=${encodeURIComponent(avatarId)}`, { method: 'POST' });
 }
-exports.suggestPlacement = suggestPlacement;
 function assetFileUrl(asset) {
     const base = `${_baseUrl}${asset.url}`;
     return asset.updated_at ? `${base}?v=${encodeURIComponent(asset.updated_at)}` : base;
 }
-exports.assetFileUrl = assetFileUrl;
 async function createVoiceProfile(name, language = 'en') {
     return studioFetch('/profiles', {
         method: 'POST',
@@ -221,7 +222,6 @@ async function createVoiceProfile(name, language = 'en') {
         body: JSON.stringify({ name, language }),
     });
 }
-exports.createVoiceProfile = createVoiceProfile;
 async function uploadVoiceSample(profileId, fileUri, fileName, referenceText) {
     const formData = new FormData();
     formData.append('reference_text', referenceText);
@@ -233,4 +233,3 @@ async function uploadVoiceSample(profileId, fileUri, fileName, referenceText) {
     });
     return studioFetchForm(`/profiles/${profileId}/samples`, 'POST', formData);
 }
-exports.uploadVoiceSample = uploadVoiceSample;

package/dist/appearance/apply.js

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.applyAppearanceToObject3D = exports.invalidateMorphCache = void 0;
+exports.invalidateMorphCache = invalidateMorphCache;
+exports.applyAppearanceToObject3D = applyAppearanceToObject3D;
 const matchers_1 = require("./matchers");
 const schema_1 = require("./schema");
 function asMaterialArray(material) {
@@ -60,7 +61,6 @@ function getMorphCache(object3d) {
 function invalidateMorphCache(object3d) {
     morphCacheByScene.delete(object3d);
 }
-exports.invalidateMorphCache = invalidateMorphCache;
 // ---------------------------------------------------------------------------
 // Main apply function
 // ---------------------------------------------------------------------------
@@ -121,4 +121,3 @@ function applyAppearanceToObject3D(object3d, appearance) {
         }
     }
 }
-exports.applyAppearanceToObject3D = applyAppearanceToObject3D;

package/dist/appearance/matchers.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.pickTargetForMaterialName = void 0;
+exports.pickTargetForMaterialName = pickTargetForMaterialName;
 function tokenizeMaterialName(name) {
     return name
         .replace(/([a-z\d])([A-Z])/g, '$1 $2')
@@ -33,4 +33,3 @@ function pickTargetForMaterialName(name) {
     }
     return null;
 }
-exports.pickTargetForMaterialName = pickTargetForMaterialName;

package/dist/appearance/schema.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.normalizeAppearance = void 0;
+exports.normalizeAppearance = normalizeAppearance;
 const HEX_COLOR_PATTERN = /^#(?:[\dA-Fa-f]{3}|[\dA-Fa-f]{6})$/;
 function normalizeHexColor(value) {
     if (!HEX_COLOR_PATTERN.test(value)) {
@@ -22,4 +22,3 @@ function normalizeAppearance(input) {
         faceShapeWeights: input.faceShapeWeights,
     };
 }
-exports.normalizeAppearance = normalizeAppearance;

package/dist/core/avatar/backend.d.ts

@@ -0,0 +1,130 @@
+import type { FaceControl } from './faceControls';
+/**
+ * A render target is intentionally abstract here; concrete backends can
+ * accept whatever they need (WebGL/WebGPU context, React Native surface,
+ * WebView, etc.) via their own config layer.
+ */
+export type AvatarRenderTarget = unknown;
+/**
+ * Canonical interface that all avatar renderers implement.
+ *
+ * The idea is:
+ *   - talking-head-studio produces a stream of FaceControl values
+ *     (pose + ExpressionState) from TTS visemes, emotions, and motions.
+ *   - Each AvatarBackend implementation adapts those controls to its
+ *     own rendering mechanism (morph targets, FLAME coefficients,
+ *     Gaussian splats, 2D sprites, etc.).
+ */
+export interface AvatarBackend {
+    /** Optional one-time initialization step (e.g. load meshes, shaders, splats). */
+    initialize?(): Promise<void> | void;
+    /** Configure the backend with a render target (canvas, surface, WebView, etc.). */
+    attach(target: AvatarRenderTarget): Promise<void> | void;
+    /**
+     * Update the current pose + expression state.
+     * This should be a cheap, per-frame call.
+     */
+    setControl(control: FaceControl): void;
+    /** Render the current control state into the attached target. */
+    renderFrame(): void;
+    /** Clean up graphics resources when the avatar is no longer needed. */
+    dispose?(): void;
+}
+/**
+ * Per-avatar calibration profile, allowing each asset to respond to the
+ * canonical control space in its own way.
+ *
+ * This is intentionally lightweight and focused on scalar ranges and
+ * asymmetries instead of model-specific knobs. Backends can use these
+ * values to remap 0..1 sliders into whatever ranges their rigs expect.
+ */
+export type CalibrationProfile = {
+    /** Neutral pose + expression for this avatar. */
+    neutral: FaceControl;
+    /** Per-channel scalar ranges (0..1 logical → model space). */
+    ranges?: {
+        jawOpen?: {
+            min: number;
+            max: number;
+        };
+        mouthSmile?: {
+            min: number;
+            max: number;
+        };
+        mouthFunnel?: {
+            min: number;
+            max: number;
+        };
+        mouthPucker?: {
+            min: number;
+            max: number;
+        };
+        mouthWide?: {
+            min: number;
+            max: number;
+        };
+        upperLipRaise?: {
+            min: number;
+            max: number;
+        };
+        lowerLipDepress?: {
+            min: number;
+            max: number;
+        };
+        cheekRaise?: {
+            min: number;
+            max: number;
+        };
+        blinkLeft?: {
+            min: number;
+            max: number;
+        };
+        blinkRight?: {
+            min: number;
+            max: number;
+        };
+        browInnerUp?: {
+            min: number;
+            max: number;
+        };
+        browDownLeft?: {
+            min: number;
+            max: number;
+        };
+        browDownRight?: {
+            min: number;
+            max: number;
+        };
+    };
+    /** Simple asymmetry controls (e.g. left/right smile strength). */
+    asymmetry?: {
+        smileLeftBias?: number;
+        blinkLeftBias?: number;
+    };
+    /** Eye gaze range limits (default: full -1..1 on both axes). */
+    gazeLimits?: {
+        x?: {
+            min: number;
+            max: number;
+        };
+        y?: {
+            min: number;
+            max: number;
+        };
+    };
+    /** Optional head pose limits to keep motion comfortable. */
+    poseLimits?: {
+        yaw?: {
+            min: number;
+            max: number;
+        };
+        pitch?: {
+            min: number;
+            max: number;
+        };
+        roll?: {
+            min: number;
+            max: number;
+        };
+    };
+};

package/dist/core/avatar/backend.js

@@ -0,0 +1,4 @@
+"use strict";
+// src/core/avatar/backend.ts
+// Backend-agnostic avatar driver contracts built on top of FaceControl.
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/core/avatar/backends/gaussian.d.ts

@@ -0,0 +1,49 @@
+import type * as THREE from 'three';
+import type { AvatarBackend, AvatarRenderTarget, CalibrationProfile } from '../backend';
+import type { FaceControl } from '../faceControls';
+import type { AvatarSchemaReport } from '../schema';
+export interface GaussianBackendConfig {
+    /** Three.js scene root (gltf.scene). */
+    scene: THREE.Object3D;
+    /** Schema report from walkAvatarSchema — used to log coverage and skip morphs. */
+    schemaReport: AvatarSchemaReport;
+    /** Optional calibration profile for gaze limits and expression scaling. */
+    calibration?: CalibrationProfile;
+    /**
+     * 'fast' — only jaw bone driven (best for real-time on low-end devices).
+     * 'full' — all FLAME bone coefficients applied (default).
+     */
+    patchQuality?: 'fast' | 'full';
+}
+export declare class GaussianBackend implements AvatarBackend {
+    private scene;
+    private schemaReport;
+    private calibration;
+    private patchQuality;
+    /** Resolved skeleton bone map: canonical name → Three.js Bone */
+    private boneMap;
+    /** Pre-computed deformation patches, one per Oculus viseme. */
+    private patches;
+    /** Snapshot of bone rotations at attach() — restored on dispose(). */
+    private basePose;
+    /** Last control applied — used in renderFrame() to avoid redundant writes. */
+    private lastControl;
+    constructor(config: GaussianBackendConfig);
+    initialize(): void;
+    attach(_target: AvatarRenderTarget): void;
+    setControl(control: FaceControl): void;
+    renderFrame(): void;
+    dispose(): void;
+    private _buildBoneMap;
+    private _captureBasePose;
+    private _restoreBasePose;
+    private _buildPatches;
+    private _applyControl;
+    private _applyGaze;
+    /** Canonical bones resolved from the skeleton. Useful for debugging rig coverage. */
+    get resolvedBones(): string[];
+    /** Number of viseme patches with at least one resolved bone. */
+    get patchCoverage(): number;
+    /** The schema report this backend was initialized with. */
+    get schema(): AvatarSchemaReport;
+}