talking-head-studio 0.4.10 → 0.4.12

package/dist/appearance/apply.js

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.applyAppearanceToObject3D = exports.invalidateMorphCache = void 0;
+exports.invalidateMorphCache = invalidateMorphCache;
+exports.applyAppearanceToObject3D = applyAppearanceToObject3D;
 const matchers_1 = require("./matchers");
 const schema_1 = require("./schema");
 function asMaterialArray(material) {
@@ -60,7 +61,6 @@ function getMorphCache(object3d) {
 function invalidateMorphCache(object3d) {
     morphCacheByScene.delete(object3d);
 }
-exports.invalidateMorphCache = invalidateMorphCache;
 // ---------------------------------------------------------------------------
 // Main apply function
 // ---------------------------------------------------------------------------
@@ -121,4 +121,3 @@ function applyAppearanceToObject3D(object3d, appearance) {
         }
     }
 }
-exports.applyAppearanceToObject3D = applyAppearanceToObject3D;

package/dist/appearance/matchers.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.pickTargetForMaterialName = void 0;
+exports.pickTargetForMaterialName = pickTargetForMaterialName;
 function tokenizeMaterialName(name) {
     return name
         .replace(/([a-z\d])([A-Z])/g, '$1 $2')
@@ -33,4 +33,3 @@ function pickTargetForMaterialName(name) {
     }
     return null;
 }
-exports.pickTargetForMaterialName = pickTargetForMaterialName;

package/dist/appearance/schema.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.normalizeAppearance = void 0;
+exports.normalizeAppearance = normalizeAppearance;
 const HEX_COLOR_PATTERN = /^#(?:[\dA-Fa-f]{3}|[\dA-Fa-f]{6})$/;
 function normalizeHexColor(value) {
     if (!HEX_COLOR_PATTERN.test(value)) {
@@ -22,4 +22,3 @@ function normalizeAppearance(input) {
         faceShapeWeights: input.faceShapeWeights,
     };
 }
-exports.normalizeAppearance = normalizeAppearance;

package/dist/contract.d.ts

@@ -0,0 +1,14 @@
+/**
+ * Stable avatar contract entrypoint for SiteBay clients.
+ *
+ * Runtime packages should import avatar control, viseme, and backend surface
+ * types from `talking-head-studio/contract` instead of reaching into internal
+ * `src/core/avatar/*` paths.
+ */
+export type { AgentVisemePayload, ArkitBlendShape, EngineVisemeMappingArtifact, OculusViseme, OculusVisemeWeights, RhubarbViseme, VisemeCue, } from './core/avatar/visemes';
+export { ARKIT_TO_OCULUS, ENGINE_VISEME_MAPPING, OCULUS_MORPH_TARGET_NAMES, OCULUS_VISEMES, RHUBARB_TO_OCULUS, RHUBARB_VISEMES, getArkitWeightsForViseme, oculusWeightsForRhubarbCue, remapArkitToOculus, } from './core/avatar/visemes';
+export type { EyeGaze, ExpressionState, FaceControl, HeadPose, Viseme, } from './core/avatar/faceControls';
+export { applyVisemeToExpression, createNeutralExpression, visemeToExpression, } from './core/avatar/faceControls';
+export type { AvatarBackend, AvatarRenderTarget, CalibrationProfile, } from './core/avatar/backend';
+export { MOTION_DEFS, MOTION_KEYS, isMotionKey, TALKINGHEAD_GESTURES, TALKINGHEAD_POSES, } from './core/avatar/motion';
+export type { MotionKey, MotionDef, MotionDefWithBones, MotionBoneKey, MotionOscillator, TalkingHeadGesture, TalkingHeadPose, } from './core/avatar/motion';

package/dist/contract.js

@@ -0,0 +1,30 @@
+"use strict";
+/**
+ * Stable avatar contract entrypoint for SiteBay clients.
+ *
+ * Runtime packages should import avatar control, viseme, and backend surface
+ * types from `talking-head-studio/contract` instead of reaching into internal
+ * `src/core/avatar/*` paths.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TALKINGHEAD_POSES = exports.TALKINGHEAD_GESTURES = exports.isMotionKey = exports.MOTION_KEYS = exports.MOTION_DEFS = exports.visemeToExpression = exports.createNeutralExpression = exports.applyVisemeToExpression = exports.remapArkitToOculus = exports.oculusWeightsForRhubarbCue = exports.getArkitWeightsForViseme = exports.RHUBARB_VISEMES = exports.RHUBARB_TO_OCULUS = exports.OCULUS_VISEMES = exports.OCULUS_MORPH_TARGET_NAMES = exports.ENGINE_VISEME_MAPPING = exports.ARKIT_TO_OCULUS = void 0;
+var visemes_1 = require("./core/avatar/visemes");
+Object.defineProperty(exports, "ARKIT_TO_OCULUS", { enumerable: true, get: function () { return visemes_1.ARKIT_TO_OCULUS; } });
+Object.defineProperty(exports, "ENGINE_VISEME_MAPPING", { enumerable: true, get: function () { return visemes_1.ENGINE_VISEME_MAPPING; } });
+Object.defineProperty(exports, "OCULUS_MORPH_TARGET_NAMES", { enumerable: true, get: function () { return visemes_1.OCULUS_MORPH_TARGET_NAMES; } });
+Object.defineProperty(exports, "OCULUS_VISEMES", { enumerable: true, get: function () { return visemes_1.OCULUS_VISEMES; } });
+Object.defineProperty(exports, "RHUBARB_TO_OCULUS", { enumerable: true, get: function () { return visemes_1.RHUBARB_TO_OCULUS; } });
+Object.defineProperty(exports, "RHUBARB_VISEMES", { enumerable: true, get: function () { return visemes_1.RHUBARB_VISEMES; } });
+Object.defineProperty(exports, "getArkitWeightsForViseme", { enumerable: true, get: function () { return visemes_1.getArkitWeightsForViseme; } });
+Object.defineProperty(exports, "oculusWeightsForRhubarbCue", { enumerable: true, get: function () { return visemes_1.oculusWeightsForRhubarbCue; } });
+Object.defineProperty(exports, "remapArkitToOculus", { enumerable: true, get: function () { return visemes_1.remapArkitToOculus; } });
+var faceControls_1 = require("./core/avatar/faceControls");
+Object.defineProperty(exports, "applyVisemeToExpression", { enumerable: true, get: function () { return faceControls_1.applyVisemeToExpression; } });
+Object.defineProperty(exports, "createNeutralExpression", { enumerable: true, get: function () { return faceControls_1.createNeutralExpression; } });
+Object.defineProperty(exports, "visemeToExpression", { enumerable: true, get: function () { return faceControls_1.visemeToExpression; } });
+var motion_1 = require("./core/avatar/motion");
+Object.defineProperty(exports, "MOTION_DEFS", { enumerable: true, get: function () { return motion_1.MOTION_DEFS; } });
+Object.defineProperty(exports, "MOTION_KEYS", { enumerable: true, get: function () { return motion_1.MOTION_KEYS; } });
+Object.defineProperty(exports, "isMotionKey", { enumerable: true, get: function () { return motion_1.isMotionKey; } });
+Object.defineProperty(exports, "TALKINGHEAD_GESTURES", { enumerable: true, get: function () { return motion_1.TALKINGHEAD_GESTURES; } });
+Object.defineProperty(exports, "TALKINGHEAD_POSES", { enumerable: true, get: function () { return motion_1.TALKINGHEAD_POSES; } });

package/dist/core/avatar/avatarCapabilities.d.ts

@@ -0,0 +1,60 @@
+import { type AvatarSchemaReport, type VisemeTier } from './schema';
+import { type MotionBoneKey } from './motion';
+export interface MotionCoverage {
+    /** Motion-engine bone keys this rig can drive (head, spine, arms, …). */
+    matched: MotionBoneKey[];
+    /** Motion-engine bone keys with no matching bone. */
+    missing: MotionBoneKey[];
+    /** Enough of the body is rigged for the procedural motion engine to read as
+     *  "moving" (spine + head + at least one arm). */
+    ready: boolean;
+}
+export interface AvatarCapabilities {
+    /** Full schema report (visemes, skeleton, mesh) from walkAvatarSchema. */
+    schema: AvatarSchemaReport;
+    /** Best lip-sync strategy: oculus / arkit / minimal / none. */
+    lipSyncTier: VisemeTier;
+    /** Can this avatar lip-sync from a viseme schedule (morph-driven)? */
+    canLipSync: boolean;
+    /** Can the talking-head motion engine give this avatar body motion/gestures? */
+    canMove: boolean;
+    /** Procedural-motion bone coverage detail. */
+    motion: MotionCoverage;
+    /** Baked animation clip count (informational; the motion engine is procedural). */
+    animations: number;
+}
+type GltfJson = {
+    nodes?: {
+        name?: string;
+    }[];
+    meshes?: {
+        primitives?: {
+            indices?: number;
+            attributes?: Record<string, number>;
+            targets?: unknown[];
+        }[];
+        extras?: {
+            targetNames?: string[];
+        };
+    }[];
+    skins?: {
+        joints?: number[];
+    }[];
+    accessors?: {
+        count?: number;
+    }[];
+    animations?: unknown[];
+};
+/** Decode a .glb (binary) or .gltf (JSON) ArrayBuffer/Uint8Array into its glTF JSON. */
+export declare function parseGlbJson(data: ArrayBuffer | Uint8Array): GltfJson;
+/** Capability report from glTF JSON (the unit-testable core). */
+export declare function inspectAvatarGltf(json: GltfJson): AvatarCapabilities;
+/** Capability report from raw GLB/glTF bytes (browser: fetch(url).arrayBuffer()). */
+export declare function inspectAvatarCapabilities(data: ArrayBuffer | Uint8Array): AvatarCapabilities;
+/**
+ * Capability report from a GLB's raw JSON-chunk bytes — for the streaming
+ * import path that reads only the header + JSON chunk off disk (mirrors
+ * sketchfab's inspectGlbJsonChunk, but for full avatar capabilities).
+ */
+export declare function inspectAvatarJsonChunk(jsonBytes: Uint8Array): AvatarCapabilities;
+export {};

package/dist/core/avatar/avatarCapabilities.js

@@ -0,0 +1,100 @@
+"use strict";
+// src/core/avatar/glbInspect.ts
+// Pre-flight avatar capability check from raw GLB/glTF bytes — no Three.js, no
+// WebGL load required. Reuses walkAvatarSchema() (the single source of truth for
+// viseme tier + humanoid rig) by adapting the glTF JSON into the minimal scene
+// that walker expects, then layers on motion-engine bone coverage.
+//
+// The avatar creator runs this when a user picks/uploads/URLs an avatar, so it
+// can soft-warn ("this model can't lip-sync / won't gesture") before committing.
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseGlbJson = parseGlbJson;
+exports.inspectAvatarGltf = inspectAvatarGltf;
+exports.inspectAvatarCapabilities = inspectAvatarCapabilities;
+exports.inspectAvatarJsonChunk = inspectAvatarJsonChunk;
+const schema_1 = require("./schema");
+const motion_1 = require("./motion");
+/** Decode a .glb (binary) or .gltf (JSON) ArrayBuffer/Uint8Array into its glTF JSON. */
+function parseGlbJson(data) {
+    const bytes = data instanceof Uint8Array ? data : new Uint8Array(data);
+    // GLB: 'glTF' magic (0x46546C67), then 12-byte header + JSON chunk.
+    if (bytes.length >= 12 && bytes[0] === 0x67 && bytes[1] === 0x6c && bytes[2] === 0x54 && bytes[3] === 0x46) {
+        const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+        const jsonLen = view.getUint32(12, true); // first chunk length (JSON)
+        const jsonBytes = bytes.subarray(20, 20 + jsonLen);
+        return JSON.parse(new TextDecoder().decode(jsonBytes));
+    }
+    // Otherwise assume .gltf JSON text.
+    return JSON.parse(new TextDecoder().decode(bytes));
+}
+const norm = (s) => s.toLowerCase().replace(/[_\s\-.]/g, '');
+/** Capability report from glTF JSON (the unit-testable core). */
+function inspectAvatarGltf(json) {
+    const nodes = json.nodes ?? [];
+    const meshes = json.meshes ?? [];
+    const skins = json.skins ?? [];
+    const accessors = json.accessors ?? [];
+    // Adapt the glTF JSON into the minimal { traverse } scene walkAvatarSchema wants.
+    const objects = [];
+    for (const skin of skins) {
+        const bones = (skin.joints ?? []).map((j) => ({ name: nodes[j]?.name }));
+        objects.push({ isSkinnedMesh: true, skeleton: { bones } });
+        for (const j of skin.joints ?? []) {
+            const nm = nodes[j]?.name;
+            if (nm)
+                objects.push({ isBone: true, name: nm });
+        }
+    }
+    for (const mesh of meshes) {
+        const targetNames = mesh.extras?.targetNames ?? [];
+        const dict = targetNames.length ? Object.fromEntries(targetNames.map((n, i) => [String(n), i])) : undefined;
+        let idxCount = 0;
+        let posCount = 0;
+        let hasIdx = false;
+        for (const prim of mesh.primitives ?? []) {
+            if (prim.indices != null && accessors[prim.indices]) {
+                idxCount += accessors[prim.indices].count ?? 0;
+                hasIdx = true;
+            }
+            const posIdx = prim.attributes?.POSITION;
+            if (posIdx != null && accessors[posIdx])
+                posCount += accessors[posIdx].count ?? 0;
+        }
+        objects.push({
+            isMesh: true,
+            ...(dict ? { morphTargetDictionary: dict } : {}),
+            geometry: hasIdx ? { index: { count: idxCount } } : { attributes: { position: { count: posCount } } },
+        });
+    }
+    const schema = (0, schema_1.walkAvatarSchema)({ traverse: (cb) => objects.forEach((o) => cb(o)) });
+    // Motion-engine coverage: which logical bones the procedural engine can drive.
+    const bonesNorm = schema.skeleton.bones.map(norm);
+    const matched = [];
+    const missing = [];
+    for (const [key, kws] of Object.entries(motion_1.MOTION_BONE_SEARCH)) {
+        const hit = kws.some((kw) => bonesNorm.some((b) => b.includes(norm(kw))));
+        (hit ? matched : missing).push(key);
+    }
+    const has = (k) => matched.includes(k);
+    const ready = schema.skeleton.hasRig && has('spine') && has('head') && (has('leftArm') || has('rightArm'));
+    return {
+        schema,
+        lipSyncTier: schema.morphs.visemeTier,
+        canLipSync: schema.morphs.visemeTier !== 'none',
+        canMove: ready,
+        motion: { matched, missing, ready },
+        animations: (json.animations ?? []).length,
+    };
+}
+/** Capability report from raw GLB/glTF bytes (browser: fetch(url).arrayBuffer()). */
+function inspectAvatarCapabilities(data) {
+    return inspectAvatarGltf(parseGlbJson(data));
+}
+/**
+ * Capability report from a GLB's raw JSON-chunk bytes — for the streaming
+ * import path that reads only the header + JSON chunk off disk (mirrors
+ * sketchfab's inspectGlbJsonChunk, but for full avatar capabilities).
+ */
+function inspectAvatarJsonChunk(jsonBytes) {
+    return inspectAvatarGltf(JSON.parse(new TextDecoder().decode(jsonBytes)));
+}

package/dist/core/avatar/backend.d.ts

@@ -0,0 +1,130 @@
+import type { FaceControl } from './faceControls';
+/**
+ * A render target is intentionally abstract here; concrete backends can
+ * accept whatever they need (WebGL/WebGPU context, React Native surface,
+ * WebView, etc.) via their own config layer.
+ */
+export type AvatarRenderTarget = unknown;
+/**
+ * Canonical interface that all avatar renderers implement.
+ *
+ * The idea is:
+ *   - talking-head-studio produces a stream of FaceControl values
+ *     (pose + ExpressionState) from TTS visemes, emotions, and motions.
+ *   - Each AvatarBackend implementation adapts those controls to its
+ *     own rendering mechanism (morph targets, FLAME coefficients,
+ *     Gaussian splats, 2D sprites, etc.).
+ */
+export interface AvatarBackend {
+    /** Optional one-time initialization step (e.g. load meshes, shaders, splats). */
+    initialize?(): Promise<void> | void;
+    /** Configure the backend with a render target (canvas, surface, WebView, etc.). */
+    attach(target: AvatarRenderTarget): Promise<void> | void;
+    /**
+     * Update the current pose + expression state.
+     * This should be a cheap, per-frame call.
+     */
+    setControl(control: FaceControl): void;
+    /** Render the current control state into the attached target. */
+    renderFrame(): void;
+    /** Clean up graphics resources when the avatar is no longer needed. */
+    dispose?(): void;
+}
+/**
+ * Per-avatar calibration profile, allowing each asset to respond to the
+ * canonical control space in its own way.
+ *
+ * This is intentionally lightweight and focused on scalar ranges and
+ * asymmetries instead of model-specific knobs. Backends can use these
+ * values to remap 0..1 sliders into whatever ranges their rigs expect.
+ */
+export type CalibrationProfile = {
+    /** Neutral pose + expression for this avatar. */
+    neutral: FaceControl;
+    /** Per-channel scalar ranges (0..1 logical → model space). */
+    ranges?: {
+        jawOpen?: {
+            min: number;
+            max: number;
+        };
+        mouthSmile?: {
+            min: number;
+            max: number;
+        };
+        mouthFunnel?: {
+            min: number;
+            max: number;
+        };
+        mouthPucker?: {
+            min: number;
+            max: number;
+        };
+        mouthWide?: {
+            min: number;
+            max: number;
+        };
+        upperLipRaise?: {
+            min: number;
+            max: number;
+        };
+        lowerLipDepress?: {
+            min: number;
+            max: number;
+        };
+        cheekRaise?: {
+            min: number;
+            max: number;
+        };
+        blinkLeft?: {
+            min: number;
+            max: number;
+        };
+        blinkRight?: {
+            min: number;
+            max: number;
+        };
+        browInnerUp?: {
+            min: number;
+            max: number;
+        };
+        browDownLeft?: {
+            min: number;
+            max: number;
+        };
+        browDownRight?: {
+            min: number;
+            max: number;
+        };
+    };
+    /** Simple asymmetry controls (e.g. left/right smile strength). */
+    asymmetry?: {
+        smileLeftBias?: number;
+        blinkLeftBias?: number;
+    };
+    /** Eye gaze range limits (default: full -1..1 on both axes). */
+    gazeLimits?: {
+        x?: {
+            min: number;
+            max: number;
+        };
+        y?: {
+            min: number;
+            max: number;
+        };
+    };
+    /** Optional head pose limits to keep motion comfortable. */
+    poseLimits?: {
+        yaw?: {
+            min: number;
+            max: number;
+        };
+        pitch?: {
+            min: number;
+            max: number;
+        };
+        roll?: {
+            min: number;
+            max: number;
+        };
+    };
+};

package/dist/core/avatar/backend.js

@@ -0,0 +1,4 @@
+"use strict";
+// src/core/avatar/backend.ts
+// Backend-agnostic avatar driver contracts built on top of FaceControl.
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/core/avatar/backends/gaussian.d.ts

@@ -0,0 +1,49 @@
+import type * as THREE from 'three';
+import type { AvatarBackend, AvatarRenderTarget, CalibrationProfile } from '../backend';
+import type { FaceControl } from '../faceControls';
+import type { AvatarSchemaReport } from '../schema';
+export interface GaussianBackendConfig {
+    /** Three.js scene root (gltf.scene). */
+    scene: THREE.Object3D;
+    /** Schema report from walkAvatarSchema — used to log coverage and skip morphs. */
+    schemaReport: AvatarSchemaReport;
+    /** Optional calibration profile for gaze limits and expression scaling. */
+    calibration?: CalibrationProfile;
+    /**
+     * 'fast' — only jaw bone driven (best for real-time on low-end devices).
+     * 'full' — all FLAME bone coefficients applied (default).
+     */
+    patchQuality?: 'fast' | 'full';
+}
+export declare class GaussianBackend implements AvatarBackend {
+    private scene;
+    private schemaReport;
+    private calibration;
+    private patchQuality;
+    /** Resolved skeleton bone map: canonical name → Three.js Bone */
+    private boneMap;
+    /** Pre-computed deformation patches, one per Oculus viseme. */
+    private patches;
+    /** Snapshot of bone rotations at attach() — restored on dispose(). */
+    private basePose;
+    /** Last control applied — used in renderFrame() to avoid redundant writes. */
+    private lastControl;
+    constructor(config: GaussianBackendConfig);
+    initialize(): void;
+    attach(_target: AvatarRenderTarget): void;
+    setControl(control: FaceControl): void;
+    renderFrame(): void;
+    dispose(): void;
+    private _buildBoneMap;
+    private _captureBasePose;
+    private _restoreBasePose;
+    private _buildPatches;
+    private _applyControl;
+    private _applyGaze;
+    /** Canonical bones resolved from the skeleton. Useful for debugging rig coverage. */
+    get resolvedBones(): string[];
+    /** Number of viseme patches with at least one resolved bone. */
+    get patchCoverage(): number;
+    /** The schema report this backend was initialized with. */
+    get schema(): AvatarSchemaReport;
+}