talking-head-studio 0.3.1 → 0.3.3

package/dist/TalkingHead.d.ts

@@ -54,6 +54,7 @@ export interface TalkingHeadProps {
     onLoadingChange?: (state: TalkingHeadLoadingState) => void;
     onReady?: () => void;
     onError?: (message: string) => void;
+    onAvatarState?: (state: string) => void;
     style?: StyleProp<ViewStyle>;
     /** Base URL for vendored assets. When set, replaces all cdn.jsdelivr.net references. */
     vendorBaseUrl?: string | null;
@@ -92,5 +93,7 @@ export interface TalkingHeadRef {
     setSkinColor: (color: string) => void;
     setEyeColor: (color: string) => void;
     setAccessories: (accessories: TalkingHeadAccessory[]) => void;
+    /** Dispatch a named motion/gesture to the avatar (e.g. 'wave_right', 'dance_idle'). */
+    dispatchMotion: (name: string) => void;
 }
 export declare const TalkingHead: React.ForwardRefExoticComponent<TalkingHeadProps & React.RefAttributes<TalkingHeadRef>>;

package/dist/TalkingHead.js

@@ -6,7 +6,7 @@ const react_1 = require("react");
 const react_native_1 = require("react-native");
 const react_native_webview_1 = require("react-native-webview");
 const html_1 = require("./html");
-exports.TalkingHead = (0, react_1.forwardRef)(({ avatarUrl, authToken, mood = 'neutral', cameraView = 'upper', cameraDistance = -0.5, hairColor, skinColor, eyeColor, accessories, onLoadingChange, onReady, onError, style, vendorBaseUrl, }, ref) => {
+exports.TalkingHead = (0, react_1.forwardRef)(({ avatarUrl, authToken, mood = 'neutral', cameraView = 'upper', cameraDistance = -0.5, hairColor, skinColor, eyeColor, accessories, onLoadingChange, onReady, onError, onAvatarState, style, vendorBaseUrl, }, ref) => {
     const webViewRef = (0, react_1.useRef)(null);
     const readyRef = (0, react_1.useRef)(false);
     const firstMessageSeenRef = (0, react_1.useRef)(false);
@@ -23,6 +23,8 @@ exports.TalkingHead = (0, react_1.forwardRef)(({ avatarUrl, authToken, mood = 'n
     onReadyRef.current = onReady;
     const onErrorRef = (0, react_1.useRef)(onError);
     onErrorRef.current = onError;
+    const onAvatarStateRef = (0, react_1.useRef)(onAvatarState);
+    onAvatarStateRef.current = onAvatarState;
     // The WebView HTML is built once from stable initial values.
     // avatarUrl + authToken changing causes a controlled key-based remount.
     // All other prop changes (mood, colors, accessories) go via postMessage.
@@ -124,6 +126,7 @@ exports.TalkingHead = (0, react_1.forwardRef)(({ avatarUrl, authToken, mood = 'n
                 post({ type: 'set_accessories', accessories: newAccessories });
             }
         },
+        dispatchMotion: (name) => post({ type: 'motion', name }),
     }), [post]);
     // Sync mood via postMessage only — never causes a WebView reload
     (0, react_1.useEffect)(() => {
@@ -220,6 +223,9 @@ exports.TalkingHead = (0, react_1.forwardRef)(({ avatarUrl, authToken, mood = 'n
             else if (msg.type === 'error') {
                 onErrorRef.current?.(msg.message);
             }
+            else if (msg.type === 'avatarState') {
+                onAvatarStateRef.current?.(msg.state);
+            }
             else if (msg.type === 'log') {
                 console.log('[TalkingHead]', msg.message);
             }

package/dist/TalkingHead.web.d.ts

@@ -23,6 +23,7 @@ export interface TalkingHeadProps {
     onLoadingChange?: (state: TalkingHeadLoadingState) => void;
     onReady?: () => void;
     onError?: (message: string) => void;
+    onAvatarState?: (state: string) => void;
     style?: React.CSSProperties;
 }
 export interface TalkingHeadRef {
@@ -34,5 +35,6 @@ export interface TalkingHeadRef {
     setSkinColor: (color: string) => void;
     setEyeColor: (color: string) => void;
     setAccessories: (accessories: TalkingHeadAccessory[]) => void;
+    dispatchMotion: (name: string) => void;
 }
 export declare const TalkingHead: React.ForwardRefExoticComponent<TalkingHeadProps & React.RefAttributes<TalkingHeadRef>>;

package/dist/TalkingHead.web.js

@@ -48,7 +48,7 @@ const iframeStyle = {
     border: 'none',
     backgroundColor: 'transparent',
 };
-exports.TalkingHead = (0, react_1.forwardRef)(({ avatarUrl, authToken, mood = 'neutral', cameraView = 'upper', cameraDistance = -0.5, hairColor, skinColor, eyeColor, accessories, onLoadingChange, onReady, onError, style, }, ref) => {
+exports.TalkingHead = (0, react_1.forwardRef)(({ avatarUrl, authToken, mood = 'neutral', cameraView = 'upper', cameraDistance = -0.5, hairColor, skinColor, eyeColor, accessories, onLoadingChange, onReady, onError, onAvatarState, style, }, ref) => {
     const iframeRef = (0, react_1.useRef)(null);
     const readyRef = (0, react_1.useRef)(false);
     const pendingMoodRef = (0, react_1.useRef)(mood);
@@ -97,6 +97,7 @@ exports.TalkingHead = (0, react_1.forwardRef)(({ avatarUrl, authToken, mood = 'n
                 post({ type: 'set_accessories', accessories: newAccessories });
             }
         },
+        dispatchMotion: (name) => post({ type: 'motion', name }),
     }), [post]);
     (0, react_1.useEffect)(() => {
         pendingMoodRef.current = mood;
@@ -188,6 +189,9 @@ exports.TalkingHead = (0, react_1.forwardRef)(({ avatarUrl, authToken, mood = 'n
                 else if (msg.type === 'error') {
                     onError?.(msg.message);
                 }
+                else if (msg.type === 'avatarState') {
+                    onAvatarState?.(msg.state);
+                }
                 else if (msg.type === 'log') {
                     console.log('[TalkingHead]', msg.message);
                 }

package/dist/html.js

@@ -856,6 +856,12 @@ function onIncomingMessage(event) {
       EYE_COLOR = msg.value; applyColorOverrides();
     } else if (msg.type === 'set_accessories') {
       applyAccessories(msg.accessories || []);
+    } else if (msg.type === 'motion' && typeof msg.name === 'string') {
+      if ((window as any).motionEngine) {
+        (window as any).motionEngine.play(msg.name).catch(() => {});
+      }
+      window.ReactNativeWebView?.postMessage(JSON.stringify({ type: 'avatarState', state: 'motion:' + msg.name }));
+      log('motion dispatched: ' + msg.name);
     }
   } catch (err) {
     log('Message parse error: ' + err);

package/dist/index.d.ts

@@ -10,3 +10,4 @@ export { TalkingHeadVisualization } from './TalkingHeadVisualization';
 export type { TalkingHeadVisualizationRef } from './TalkingHeadVisualization';
 export { useDirectVisemeStream } from './tts/useDirectVisemeStream';
 export type { VisemeStreamPayload } from './tts/useDirectVisemeStream';
+export { useMotionMarkers } from './tts/useMotionMarkers';

package/dist/index.js

@@ -14,7 +14,7 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.useDirectVisemeStream = exports.TalkingHeadVisualization = exports.normalizeAppearance = exports.pickTargetForMaterialName = exports.applyAppearanceToObject3D = exports.TalkingHead = void 0;
+exports.useMotionMarkers = exports.useDirectVisemeStream = exports.TalkingHeadVisualization = exports.normalizeAppearance = exports.pickTargetForMaterialName = exports.applyAppearanceToObject3D = exports.TalkingHead = void 0;
 var TalkingHead_1 = require("./TalkingHead");
 Object.defineProperty(exports, "TalkingHead", { enumerable: true, get: function () { return TalkingHead_1.TalkingHead; } });
 // Export appearance utilities, but exclude AvatarAppearance — the canonical
@@ -31,3 +31,5 @@ var TalkingHeadVisualization_1 = require("./TalkingHeadVisualization");
 Object.defineProperty(exports, "TalkingHeadVisualization", { enumerable: true, get: function () { return TalkingHeadVisualization_1.TalkingHeadVisualization; } });
 var useDirectVisemeStream_1 = require("./tts/useDirectVisemeStream");
 Object.defineProperty(exports, "useDirectVisemeStream", { enumerable: true, get: function () { return useDirectVisemeStream_1.useDirectVisemeStream; } });
+var useMotionMarkers_1 = require("./tts/useMotionMarkers");
+Object.defineProperty(exports, "useMotionMarkers", { enumerable: true, get: function () { return useMotionMarkers_1.useMotionMarkers; } });

package/dist/tts/useDirectVisemeStream.d.ts

@@ -14,6 +14,10 @@ type OpenStreamOptions = {
  *
  * Uses fetch() with streaming response body instead of EventSource because
  * React Native does not have a reliable EventSource polyfill.
+ *
+ * Retries on transient failures (network blip, 503) with exponential backoff
+ * up to STREAM_RETRY_BUDGET_MS. Aborts cleanly when a new requestId arrives
+ * or the component unmounts.
  */
 export declare function useDirectVisemeStream(onVisemes: (payload: VisemeStreamPayload) => void): {
     openStream: ({ requestId, ttsBaseUrl }: OpenStreamOptions) => void;

package/dist/tts/useDirectVisemeStream.js

@@ -3,108 +3,91 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.useDirectVisemeStream = useDirectVisemeStream;
 const react_1 = require("react");
 const fetch_1 = require("expo/fetch");
+// How long to keep retrying a stream before giving up (ms).
+const STREAM_RETRY_BUDGET_MS = 3000;
+// Initial retry delay; doubles each attempt up to MAX_RETRY_DELAY_MS.
+const INITIAL_RETRY_DELAY_MS = 150;
+const MAX_RETRY_DELAY_MS = 1000;
 /**
  * Opens a direct SSE connection to the TTS server to receive viseme data,
  * bypassing the agent data channel relay.
  *
  * Uses fetch() with streaming response body instead of EventSource because
  * React Native does not have a reliable EventSource polyfill.
+ *
+ * Retries on transient failures (network blip, 503) with exponential backoff
+ * up to STREAM_RETRY_BUDGET_MS. Aborts cleanly when a new requestId arrives
+ * or the component unmounts.
  */
 function useDirectVisemeStream(onVisemes) {
-    // Track the current abort controller keyed by requestId so we can detect
-    // when a new requestId arrives and tear down the previous stream.
     const abortControllerRef = (0, react_1.useRef)(null);
-    const activeRequestIdRef = (0, react_1.useRef)(null);
     const onVisemesRef = (0, react_1.useRef)(onVisemes);
     // Keep callback ref up to date without requiring it in openStream's dep array
     (0, react_1.useEffect)(() => {
         onVisemesRef.current = onVisemes;
     });
     const openStream = (0, react_1.useCallback)(({ requestId, ttsBaseUrl }) => {
-        // Abort any existing stream — whether for the same or a different requestId
+        // Abort any existing stream for a previous request
         if (abortControllerRef.current) {
             abortControllerRef.current.abort();
             abortControllerRef.current = null;
         }
-        activeRequestIdRef.current = requestId;
         const controller = new AbortController();
         abortControllerRef.current = controller;
         const { signal } = controller;
         // Strip trailing /v1 if present so we don't double it
-        const base = ttsBaseUrl.replace(/\/v1\/?$/, '');
+        const base = ttsBaseUrl.replace(/\/v1\/?$/, "");
         const url = `${base}/v1/audio/visemes/${encodeURIComponent(requestId)}/stream`;
         (async () => {
-            try {
-                const response = await (0, fetch_1.fetch)(url, {
-                    headers: { Accept: "text/event-stream" },
-                    signal,
-                });
-                if (!response.ok) {
-                    console.warn("[VisemeSSE] Non-OK response", { requestId, status: response.status });
-                    return;
-                }
-                const reader = response.body?.getReader();
-                if (!reader) {
-                    console.warn("[VisemeSSE] No response body reader", { requestId });
-                    return;
-                }
-                const decoder = new TextDecoder();
-                let buffer = "";
-                let pendingEvent = null;
-                while (true) {
-                    const { done, value } = await reader.read();
-                    if (done)
-                        break;
-                    buffer += decoder.decode(value, { stream: true });
-                    // Split on newlines, keeping the remainder (incomplete line) in buffer
-                    const lines = buffer.split("\n");
-                    buffer = lines.pop() ?? "";
-                    for (const rawLine of lines) {
-                        const line = rawLine.trimEnd();
-                        if (line.startsWith("event:")) {
-                            pendingEvent = line.slice("event:".length).trim();
-                        }
-                        else if (line.startsWith("data:")) {
-                            if (pendingEvent === "visemes") {
-                                const jsonText = line.slice("data:".length).trim();
-                                try {
-                                    const payload = JSON.parse(jsonText);
-                                    console.log("[VisemeSSE] received", {
-                                        requestId,
-                                        cues: Array.isArray(payload.cues) ? payload.cues.length : 0,
-                                        durationMs: payload.durationMs ?? null,
-                                    });
-                                    onVisemesRef.current(payload);
-                                }
-                                catch (parseErr) {
-                                    console.warn("[VisemeSSE] JSON parse error", parseErr);
-                                }
-                            }
-                            // Reset pending event after consuming the data line
-                            pendingEvent = null;
-                        }
-                        else if (line === "") {
-                            // Empty line = end of SSE message block; reset pending event
-                            pendingEvent = null;
+            const startedAt = Date.now();
+            let retryDelay = INITIAL_RETRY_DELAY_MS;
+            while (!signal.aborted) {
+                try {
+                    const response = await (0, fetch_1.fetch)(url, {
+                        headers: { Accept: "text/event-stream" },
+                        signal,
+                    });
+                    if (!response.ok) {
+                        const retryable = response.status === 503 || response.status === 502 || response.status === 429;
+                        if (!retryable || Date.now() - startedAt >= STREAM_RETRY_BUDGET_MS) {
+                            console.warn("[VisemeSSE] Non-OK response, giving up", { requestId, status: response.status });
+                            return;
                         }
+                        console.warn("[VisemeSSE] Retryable error, backing off", { requestId, status: response.status, retryDelay });
+                        await sleep(retryDelay, signal);
+                        retryDelay = Math.min(retryDelay * 2, MAX_RETRY_DELAY_MS);
+                        continue;
                     }
-                }
-                console.log("[VisemeSSE] stream ended", { requestId });
-            }
-            catch (err) {
-                if (err?.name === "AbortError") {
-                    // Expected — stream was intentionally cancelled
+                    const reader = response.body?.getReader();
+                    if (!reader) {
+                        console.warn("[VisemeSSE] No response body reader", { requestId });
+                        return;
+                    }
+                    await readSseStream(reader, signal, (payload) => {
+                        onVisemesRef.current(payload);
+                    });
+                    // Stream ended cleanly — done.
+                    console.log("[VisemeSSE] stream ended", { requestId });
                     return;
                 }
-                console.warn("[VisemeSSE] stream error", { requestId, err });
-            }
-            finally {
-                // Only clear the ref if it still points to our controller
-                if (abortControllerRef.current === controller) {
-                    abortControllerRef.current = null;
+                catch (err) {
+                    if (err?.name === "AbortError" || signal.aborted)
+                        return;
+                    const elapsed = Date.now() - startedAt;
+                    if (elapsed >= STREAM_RETRY_BUDGET_MS) {
+                        console.warn("[VisemeSSE] stream error, retry budget exhausted", { requestId, err });
+                        return;
+                    }
+                    console.warn("[VisemeSSE] stream error, retrying", { requestId, retryDelay, err });
+                    await sleep(retryDelay, signal);
+                    retryDelay = Math.min(retryDelay * 2, MAX_RETRY_DELAY_MS);
                 }
             }
-        })();
+        })().finally(() => {
+            if (abortControllerRef.current === controller) {
+                abortControllerRef.current = null;
+            }
+        });
     }, []);
     // Clean up on unmount
     (0, react_1.useEffect)(() => {
@@ -117,3 +100,86 @@ function useDirectVisemeStream(onVisemes) {
     }, []);
     return { openStream };
 }
+// ─── SSE parser ──────────────────────────────────────────────────────────────
+/**
+ * Reads an SSE stream to completion, dispatching `event: visemes` messages.
+ *
+ * Follows the SSE spec: fields accumulate per-message block; an empty line
+ * dispatches the block. Handles streams that end without a trailing newline
+ * by flushing the remaining buffer on EOF.
+ */
+async function readSseStream(reader, signal, onVisemes) {
+    const decoder = new TextDecoder();
+    let buffer = "";
+    // Per-message accumulator (reset on empty-line dispatch)
+    let eventType = null;
+    let dataLines = [];
+    const dispatchBlock = () => {
+        if (eventType === "visemes" && dataLines.length > 0) {
+            const jsonText = dataLines.join("\n");
+            try {
+                const payload = JSON.parse(jsonText);
+                console.log("[VisemeSSE] received", {
+                    requestId: payload.requestId,
+                    cues: Array.isArray(payload.cues) ? payload.cues.length : 0,
+                    durationMs: payload.durationMs ?? null,
+                });
+                onVisemes(payload);
+            }
+            catch (parseErr) {
+                console.warn("[VisemeSSE] JSON parse error", parseErr);
+            }
+        }
+        eventType = null;
+        dataLines = [];
+    };
+    const processLines = (chunk) => {
+        const lines = chunk.split("\n");
+        for (const rawLine of lines) {
+            const line = rawLine.trimEnd();
+            if (line === "") {
+                // Empty line = end of SSE message block → dispatch
+                dispatchBlock();
+            }
+            else if (line.startsWith("event:")) {
+                eventType = line.slice("event:".length).trim();
+            }
+            else if (line.startsWith("data:")) {
+                dataLines.push(line.slice("data:".length).trimStart());
+            }
+            // Ignore comment lines (":"), id:, retry: fields
+        }
+    };
+    while (!signal.aborted) {
+        const { done, value } = await reader.read();
+        if (done) {
+            // Flush any remaining buffered content without a trailing newline
+            if (buffer) {
+                processLines(buffer + "\n");
+                buffer = "";
+            }
+            break;
+        }
+        const chunk = decoder.decode(value, { stream: true });
+        // Combine with any previous incomplete line, then split on newlines.
+        // Keep the last (potentially incomplete) segment in the buffer.
+        const combined = buffer + chunk;
+        const lastNewline = combined.lastIndexOf("\n");
+        if (lastNewline === -1) {
+            buffer = combined;
+            continue;
+        }
+        buffer = combined.slice(lastNewline + 1);
+        processLines(combined.slice(0, lastNewline + 1));
+    }
+}
+// ─── Helpers ─────────────────────────────────────────────────────────────────
+function sleep(ms, signal) {
+    return new Promise((resolve, reject) => {
+        const id = setTimeout(resolve, ms);
+        signal.addEventListener("abort", () => {
+            clearTimeout(id);
+            reject(Object.assign(new Error("AbortError"), { name: "AbortError" }));
+        }, { once: true });
+    });
+}

package/dist/tts/useMotionMarkers.d.ts

@@ -0,0 +1,2 @@
+import type { TalkingHeadRef } from '../TalkingHead';
+export declare function useMotionMarkers(ref: React.RefObject<TalkingHeadRef | null>): (text: string) => string;

package/dist/tts/useMotionMarkers.js

@@ -0,0 +1,24 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.useMotionMarkers = useMotionMarkers;
+const react_1 = require("react");
+/**
+ * Parses ::marker_name:: tokens out of LLM transcript text, fires the
+ * corresponding motion on the avatar ref, and returns the cleaned text
+ * (markers stripped) ready to pass to TTS.
+ *
+ * Supports optional arguments: ::wave_right(slow):: — the arg string is
+ * forwarded as the second parameter of dispatchMotion for future use.
+ *
+ * @example
+ * const parseMarkers = useMotionMarkers(avatarRef);
+ * const cleanText = parseMarkers(rawLlmText); // fires motions, strips markers
+ * tts.speak(cleanText);
+ */
+const MARKER_RE = /::([a-z][a-z0-9_]*)(?:\(([^)]*)\))?::/g;
+function useMotionMarkers(ref) {
+    return (0, react_1.useCallback)((text) => text.replace(MARKER_RE, (_, name) => {
+        ref.current?.dispatchMotion(name);
+        return '';
+    }), [ref]);
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "talking-head-studio",
-  "version": "0.3.1",
+  "version": "0.3.3",
   "description": "Cross-platform 3D avatar component for React Native & web — lip-sync, gestures, accessories, and LLM integration. Powered by TalkingHead + Three.js.",
   "main": "dist/index.web.js",
   "browser": "dist/index.web.js",