talking-head-studio 0.3.0 → 0.3.2

package/dist/filament/index.d.ts

@@ -3,3 +3,4 @@ export type { FilamentAvatarRef } from './FilamentAvatar';
 export { useAuthedFilamentUri } from './useAuthedFilamentUri';
 export type { AuthedFileResult } from './useAuthedFilamentUri';
 export * from './morphTables';
+export { FACE_SQUEEZE_LOCAL_MODULE } from './faceSqueezeAssets';

package/dist/filament/index.js

@@ -14,9 +14,11 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.useAuthedFilamentUri = exports.FilamentAvatar = void 0;
+exports.FACE_SQUEEZE_LOCAL_MODULE = exports.useAuthedFilamentUri = exports.FilamentAvatar = void 0;
 var FilamentAvatar_1 = require("./FilamentAvatar");
 Object.defineProperty(exports, "FilamentAvatar", { enumerable: true, get: function () { return FilamentAvatar_1.FilamentAvatar; } });
 var useAuthedFilamentUri_1 = require("./useAuthedFilamentUri");
 Object.defineProperty(exports, "useAuthedFilamentUri", { enumerable: true, get: function () { return useAuthedFilamentUri_1.useAuthedFilamentUri; } });
 __exportStar(require("./morphTables"), exports);
+var faceSqueezeAssets_1 = require("./faceSqueezeAssets");
+Object.defineProperty(exports, "FACE_SQUEEZE_LOCAL_MODULE", { enumerable: true, get: function () { return faceSqueezeAssets_1.FACE_SQUEEZE_LOCAL_MODULE; } });

package/dist/index.d.ts

@@ -8,3 +8,5 @@ export * from './api';
 export * from './wardrobe';
 export { TalkingHeadVisualization } from './TalkingHeadVisualization';
 export type { TalkingHeadVisualizationRef } from './TalkingHeadVisualization';
+export { useDirectVisemeStream } from './tts/useDirectVisemeStream';
+export type { VisemeStreamPayload } from './tts/useDirectVisemeStream';

package/dist/index.js

@@ -14,7 +14,7 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.TalkingHeadVisualization = exports.normalizeAppearance = exports.pickTargetForMaterialName = exports.applyAppearanceToObject3D = exports.TalkingHead = void 0;
+exports.useDirectVisemeStream = exports.TalkingHeadVisualization = exports.normalizeAppearance = exports.pickTargetForMaterialName = exports.applyAppearanceToObject3D = exports.TalkingHead = void 0;
 var TalkingHead_1 = require("./TalkingHead");
 Object.defineProperty(exports, "TalkingHead", { enumerable: true, get: function () { return TalkingHead_1.TalkingHead; } });
 // Export appearance utilities, but exclude AvatarAppearance — the canonical
@@ -29,3 +29,5 @@ __exportStar(require("./api"), exports);
 __exportStar(require("./wardrobe"), exports);
 var TalkingHeadVisualization_1 = require("./TalkingHeadVisualization");
 Object.defineProperty(exports, "TalkingHeadVisualization", { enumerable: true, get: function () { return TalkingHeadVisualization_1.TalkingHeadVisualization; } });
+var useDirectVisemeStream_1 = require("./tts/useDirectVisemeStream");
+Object.defineProperty(exports, "useDirectVisemeStream", { enumerable: true, get: function () { return useDirectVisemeStream_1.useDirectVisemeStream; } });

package/dist/tts/useDirectVisemeStream.d.ts

@@ -0,0 +1,25 @@
+import type { TalkingHeadVisemeSchedule } from "../TalkingHead";
+export type VisemeStreamPayload = {
+    requestId?: string;
+    durationMs?: number;
+    cues?: TalkingHeadVisemeSchedule["cues"];
+};
+type OpenStreamOptions = {
+    requestId: string;
+    ttsBaseUrl: string;
+};
+/**
+ * Opens a direct SSE connection to the TTS server to receive viseme data,
+ * bypassing the agent data channel relay.
+ *
+ * Uses fetch() with streaming response body instead of EventSource because
+ * React Native does not have a reliable EventSource polyfill.
+ *
+ * Retries on transient failures (network blip, 503) with exponential backoff
+ * up to STREAM_RETRY_BUDGET_MS. Aborts cleanly when a new requestId arrives
+ * or the component unmounts.
+ */
+export declare function useDirectVisemeStream(onVisemes: (payload: VisemeStreamPayload) => void): {
+    openStream: ({ requestId, ttsBaseUrl }: OpenStreamOptions) => void;
+};
+export {};

package/dist/tts/useDirectVisemeStream.js

@@ -0,0 +1,185 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.useDirectVisemeStream = useDirectVisemeStream;
+const react_1 = require("react");
+const fetch_1 = require("expo/fetch");
+// How long to keep retrying a stream before giving up (ms).
+const STREAM_RETRY_BUDGET_MS = 3000;
+// Initial retry delay; doubles each attempt up to MAX_RETRY_DELAY_MS.
+const INITIAL_RETRY_DELAY_MS = 150;
+const MAX_RETRY_DELAY_MS = 1000;
+/**
+ * Opens a direct SSE connection to the TTS server to receive viseme data,
+ * bypassing the agent data channel relay.
+ *
+ * Uses fetch() with streaming response body instead of EventSource because
+ * React Native does not have a reliable EventSource polyfill.
+ *
+ * Retries on transient failures (network blip, 503) with exponential backoff
+ * up to STREAM_RETRY_BUDGET_MS. Aborts cleanly when a new requestId arrives
+ * or the component unmounts.
+ */
+function useDirectVisemeStream(onVisemes) {
+    const abortControllerRef = (0, react_1.useRef)(null);
+    const onVisemesRef = (0, react_1.useRef)(onVisemes);
+    // Keep callback ref up to date without requiring it in openStream's dep array
+    (0, react_1.useEffect)(() => {
+        onVisemesRef.current = onVisemes;
+    });
+    const openStream = (0, react_1.useCallback)(({ requestId, ttsBaseUrl }) => {
+        // Abort any existing stream for a previous request
+        if (abortControllerRef.current) {
+            abortControllerRef.current.abort();
+            abortControllerRef.current = null;
+        }
+        const controller = new AbortController();
+        abortControllerRef.current = controller;
+        const { signal } = controller;
+        // Strip trailing /v1 if present so we don't double it
+        const base = ttsBaseUrl.replace(/\/v1\/?$/, "");
+        const url = `${base}/v1/audio/visemes/${encodeURIComponent(requestId)}/stream`;
+        (async () => {
+            const startedAt = Date.now();
+            let retryDelay = INITIAL_RETRY_DELAY_MS;
+            while (!signal.aborted) {
+                try {
+                    const response = await (0, fetch_1.fetch)(url, {
+                        headers: { Accept: "text/event-stream" },
+                        signal,
+                    });
+                    if (!response.ok) {
+                        const retryable = response.status === 503 || response.status === 502 || response.status === 429;
+                        if (!retryable || Date.now() - startedAt >= STREAM_RETRY_BUDGET_MS) {
+                            console.warn("[VisemeSSE] Non-OK response, giving up", { requestId, status: response.status });
+                            return;
+                        }
+                        console.warn("[VisemeSSE] Retryable error, backing off", { requestId, status: response.status, retryDelay });
+                        await sleep(retryDelay, signal);
+                        retryDelay = Math.min(retryDelay * 2, MAX_RETRY_DELAY_MS);
+                        continue;
+                    }
+                    const reader = response.body?.getReader();
+                    if (!reader) {
+                        console.warn("[VisemeSSE] No response body reader", { requestId });
+                        return;
+                    }
+                    await readSseStream(reader, signal, (payload) => {
+                        onVisemesRef.current(payload);
+                    });
+                    // Stream ended cleanly — done.
+                    console.log("[VisemeSSE] stream ended", { requestId });
+                    return;
+                }
+                catch (err) {
+                    if (err?.name === "AbortError" || signal.aborted)
+                        return;
+                    const elapsed = Date.now() - startedAt;
+                    if (elapsed >= STREAM_RETRY_BUDGET_MS) {
+                        console.warn("[VisemeSSE] stream error, retry budget exhausted", { requestId, err });
+                        return;
+                    }
+                    console.warn("[VisemeSSE] stream error, retrying", { requestId, retryDelay, err });
+                    await sleep(retryDelay, signal);
+                    retryDelay = Math.min(retryDelay * 2, MAX_RETRY_DELAY_MS);
+                }
+            }
+        })().finally(() => {
+            if (abortControllerRef.current === controller) {
+                abortControllerRef.current = null;
+            }
+        });
+    }, []);
+    // Clean up on unmount
+    (0, react_1.useEffect)(() => {
+        return () => {
+            if (abortControllerRef.current) {
+                abortControllerRef.current.abort();
+                abortControllerRef.current = null;
+            }
+        };
+    }, []);
+    return { openStream };
+}
+// ─── SSE parser ──────────────────────────────────────────────────────────────
+/**
+ * Reads an SSE stream to completion, dispatching `event: visemes` messages.
+ *
+ * Follows the SSE spec: fields accumulate per-message block; an empty line
+ * dispatches the block. Handles streams that end without a trailing newline
+ * by flushing the remaining buffer on EOF.
+ */
+async function readSseStream(reader, signal, onVisemes) {
+    const decoder = new TextDecoder();
+    let buffer = "";
+    // Per-message accumulator (reset on empty-line dispatch)
+    let eventType = null;
+    let dataLines = [];
+    const dispatchBlock = () => {
+        if (eventType === "visemes" && dataLines.length > 0) {
+            const jsonText = dataLines.join("\n");
+            try {
+                const payload = JSON.parse(jsonText);
+                console.log("[VisemeSSE] received", {
+                    requestId: payload.requestId,
+                    cues: Array.isArray(payload.cues) ? payload.cues.length : 0,
+                    durationMs: payload.durationMs ?? null,
+                });
+                onVisemes(payload);
+            }
+            catch (parseErr) {
+                console.warn("[VisemeSSE] JSON parse error", parseErr);
+            }
+        }
+        eventType = null;
+        dataLines = [];
+    };
+    const processLines = (chunk) => {
+        const lines = chunk.split("\n");
+        for (const rawLine of lines) {
+            const line = rawLine.trimEnd();
+            if (line === "") {
+                // Empty line = end of SSE message block → dispatch
+                dispatchBlock();
+            }
+            else if (line.startsWith("event:")) {
+                eventType = line.slice("event:".length).trim();
+            }
+            else if (line.startsWith("data:")) {
+                dataLines.push(line.slice("data:".length).trimStart());
+            }
+            // Ignore comment lines (":"), id:, retry: fields
+        }
+    };
+    while (!signal.aborted) {
+        const { done, value } = await reader.read();
+        if (done) {
+            // Flush any remaining buffered content without a trailing newline
+            if (buffer) {
+                processLines(buffer + "\n");
+                buffer = "";
+            }
+            break;
+        }
+        const chunk = decoder.decode(value, { stream: true });
+        // Combine with any previous incomplete line, then split on newlines.
+        // Keep the last (potentially incomplete) segment in the buffer.
+        const combined = buffer + chunk;
+        const lastNewline = combined.lastIndexOf("\n");
+        if (lastNewline === -1) {
+            buffer = combined;
+            continue;
+        }
+        buffer = combined.slice(lastNewline + 1);
+        processLines(combined.slice(0, lastNewline + 1));
+    }
+}
+// ─── Helpers ─────────────────────────────────────────────────────────────────
+function sleep(ms, signal) {
+    return new Promise((resolve, reject) => {
+        const id = setTimeout(resolve, ms);
+        signal.addEventListener("abort", () => {
+            clearTimeout(id);
+            reject(Object.assign(new Error("AbortError"), { name: "AbortError" }));
+        }, { once: true });
+    });
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "talking-head-studio",
-  "version": "0.3.0",
+  "version": "0.3.2",
   "description": "Cross-platform 3D avatar component for React Native & web — lip-sync, gestures, accessories, and LLM integration. Powered by TalkingHead + Three.js.",
   "main": "dist/index.web.js",
   "browser": "dist/index.web.js",
@@ -97,6 +97,7 @@
   "peerDependencies": {
     "@react-three/drei": ">=9",
     "@react-three/fiber": ">=8",
+    "expo": ">=51",
     "expo-asset": ">=10",
     "expo-file-system": ">=17",
     "react": ">=18",
@@ -115,6 +116,9 @@
     "react-native-filament": {
       "optional": true
     },
+    "expo": {
+      "optional": true
+    },
     "expo-asset": {
       "optional": true
     },