talking-head-studio 0.2.9 → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. package/dist/TalkingHead.js +30 -2
  2. package/dist/TalkingHeadVisualization.d.ts +35 -0
  3. package/dist/TalkingHeadVisualization.js +277 -0
  4. package/dist/api/index.d.ts +2 -0
  5. package/dist/api/index.js +18 -0
  6. package/dist/api/studioApi.d.ts +38 -0
  7. package/dist/api/studioApi.js +235 -0
  8. package/dist/api/types.d.ts +87 -0
  9. package/dist/api/types.js +5 -0
  10. package/dist/assets/face-squeeze-local.glb +0 -0
  11. package/dist/filament/FilamentAvatar.d.ts +41 -0
  12. package/dist/filament/FilamentAvatar.js +737 -0
  13. package/dist/filament/faceSqueezeAssets.d.ts +1 -0
  14. package/dist/filament/faceSqueezeAssets.js +5 -0
  15. package/dist/filament/index.d.ts +6 -0
  16. package/dist/filament/index.js +24 -0
  17. package/dist/filament/morphTables.d.ts +5 -0
  18. package/dist/filament/morphTables.js +93 -0
  19. package/dist/filament/useAuthedFilamentUri.d.ts +11 -0
  20. package/dist/filament/useAuthedFilamentUri.js +126 -0
  21. package/dist/index.d.ts +10 -1
  22. package/dist/index.js +15 -2
  23. package/dist/index.web.d.ts +5 -1
  24. package/dist/index.web.js +10 -2
  25. package/dist/tts/useDirectVisemeStream.d.ts +21 -0
  26. package/dist/tts/useDirectVisemeStream.js +119 -0
  27. package/dist/utils/avatarUtils.d.ts +13 -0
  28. package/dist/utils/avatarUtils.js +56 -0
  29. package/dist/wardrobe/index.d.ts +2 -0
  30. package/dist/wardrobe/index.js +20 -0
  31. package/dist/wardrobe/useAvatarWardrobeHydration.d.ts +7 -0
  32. package/dist/wardrobe/useAvatarWardrobeHydration.js +34 -0
  33. package/dist/wardrobe/wardrobeStore.d.ts +30 -0
  34. package/dist/wardrobe/wardrobeStore.js +106 -0
  35. package/package.json +34 -2
@@ -0,0 +1 @@
1
+ export declare const FACE_SQUEEZE_LOCAL_MODULE: number;
@@ -0,0 +1,5 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.FACE_SQUEEZE_LOCAL_MODULE = void 0;
4
+ // eslint-disable-next-line @typescript-eslint/no-require-imports
5
+ exports.FACE_SQUEEZE_LOCAL_MODULE = require('../assets/face-squeeze-local.glb');
@@ -0,0 +1,6 @@
1
+ export { FilamentAvatar } from './FilamentAvatar';
2
+ export type { FilamentAvatarRef } from './FilamentAvatar';
3
+ export { useAuthedFilamentUri } from './useAuthedFilamentUri';
4
+ export type { AuthedFileResult } from './useAuthedFilamentUri';
5
+ export * from './morphTables';
6
+ export { FACE_SQUEEZE_LOCAL_MODULE } from './faceSqueezeAssets';
@@ -0,0 +1,24 @@
1
+ "use strict";
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ var desc = Object.getOwnPropertyDescriptor(m, k);
5
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
+ desc = { enumerable: true, get: function() { return m[k]; } };
7
+ }
8
+ Object.defineProperty(o, k2, desc);
9
+ }) : (function(o, m, k, k2) {
10
+ if (k2 === undefined) k2 = k;
11
+ o[k2] = m[k];
12
+ }));
13
+ var __exportStar = (this && this.__exportStar) || function(m, exports) {
14
+ for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
15
+ };
16
+ Object.defineProperty(exports, "__esModule", { value: true });
17
+ exports.FACE_SQUEEZE_LOCAL_MODULE = exports.useAuthedFilamentUri = exports.FilamentAvatar = void 0;
18
+ var FilamentAvatar_1 = require("./FilamentAvatar");
19
+ Object.defineProperty(exports, "FilamentAvatar", { enumerable: true, get: function () { return FilamentAvatar_1.FilamentAvatar; } });
20
+ var useAuthedFilamentUri_1 = require("./useAuthedFilamentUri");
21
+ Object.defineProperty(exports, "useAuthedFilamentUri", { enumerable: true, get: function () { return useAuthedFilamentUri_1.useAuthedFilamentUri; } });
22
+ __exportStar(require("./morphTables"), exports);
23
+ var faceSqueezeAssets_1 = require("./faceSqueezeAssets");
24
+ Object.defineProperty(exports, "FACE_SQUEEZE_LOCAL_MODULE", { enumerable: true, get: function () { return faceSqueezeAssets_1.FACE_SQUEEZE_LOCAL_MODULE; } });
@@ -0,0 +1,5 @@
1
+ export declare const RHUBARB_TO_VISEME: Record<string, string>;
2
+ export declare const VISEME_MORPH_ALIASES: Record<string, string[]>;
3
+ export declare const VISEME_WEIGHTS: Record<string, number>;
4
+ export declare const DEFAULT_VISEME_WEIGHT = 0.35;
5
+ export declare const MOOD_MORPHS: Record<string, Record<string, number>>;
@@ -0,0 +1,93 @@
1
+ "use strict";
2
+ // ---------------------------------------------------------------------------
3
+ // Morph target tables for Filament avatar viseme/mood rendering
4
+ // ---------------------------------------------------------------------------
5
+ Object.defineProperty(exports, "__esModule", { value: true });
6
+ exports.MOOD_MORPHS = exports.DEFAULT_VISEME_WEIGHT = exports.VISEME_WEIGHTS = exports.VISEME_MORPH_ALIASES = exports.RHUBARB_TO_VISEME = void 0;
7
+ // Rhubarb → viseme key mapping (mirrors html.ts)
8
+ exports.RHUBARB_TO_VISEME = {
9
+ A: 'aa',
10
+ B: 'PP',
11
+ C: 'ih',
12
+ D: 'FF',
13
+ E: 'ee',
14
+ F: 'oh',
15
+ G: 'ou',
16
+ H: 'nn',
17
+ X: 'sil',
18
+ };
19
+ // Oculus viseme morph name aliases per viseme key
20
+ exports.VISEME_MORPH_ALIASES = {
21
+ sil: ['viseme_sil', 'sil', 'mouthClose'],
22
+ PP: ['viseme_PP', 'viseme_pp', 'mouthPucker'],
23
+ FF: ['viseme_FF', 'viseme_ff', 'mouthRollLower', 'mouthShrugLower'],
24
+ TH: ['viseme_TH', 'viseme_th', 'tongueOut'],
25
+ DD: ['viseme_DD', 'viseme_dd', 'mouthShrugUpper'],
26
+ kk: ['viseme_kk', 'viseme_k', 'mouthStretchLeft'],
27
+ CH: ['viseme_CH', 'viseme_ch', 'mouthSmileLeft', 'mouthSmile'],
28
+ SS: ['viseme_SS', 'viseme_ss', 'mouthStretchRight'],
29
+ nn: ['viseme_nn', 'viseme_n', 'mouthDimpleLeft'],
30
+ RR: ['viseme_RR', 'viseme_r', 'mouthDimpleRight'],
31
+ aa: ['viseme_aa', 'viseme_AA', 'jawOpen', 'mouthOpen'],
32
+ ee: ['viseme_E', 'viseme_ee', 'mouthSmileLeft'],
33
+ ih: ['viseme_I', 'viseme_ih', 'mouthSmileRight'],
34
+ oh: ['viseme_O', 'viseme_oh', 'mouthFunnel'],
35
+ ou: ['viseme_U', 'viseme_ou', 'mouthRollLower'],
36
+ };
37
+ exports.VISEME_WEIGHTS = {
38
+ PP: 0.45, FF: 0.40, ee: 0.38, ih: 0.35,
39
+ oh: 0.35, ou: 0.32, aa: 0.40,
40
+ };
41
+ exports.DEFAULT_VISEME_WEIGHT = 0.35;
42
+ // Mood → baseline morph weights (persistent low-level expression layer)
43
+ exports.MOOD_MORPHS = {
44
+ neutral: {},
45
+ happy: {
46
+ mouthSmileLeft: 0.35, mouthSmileRight: 0.35,
47
+ cheekSquintLeft: 0.2, cheekSquintRight: 0.2,
48
+ },
49
+ sad: {
50
+ browInnerUp: 0.5, mouthFrownLeft: 0.4, mouthFrownRight: 0.4,
51
+ eyeLookDownLeft: 0.2, eyeLookDownRight: 0.2,
52
+ },
53
+ angry: {
54
+ browDownLeft: 0.5, browDownRight: 0.5,
55
+ noseSneerLeft: 0.3, noseSneerRight: 0.3,
56
+ eyeSquintLeft: 0.2, eyeSquintRight: 0.2,
57
+ },
58
+ surprised: {
59
+ eyeWideLeft: 0.5, eyeWideRight: 0.5,
60
+ browInnerUp: 0.5, browOuterUpLeft: 0.4, browOuterUpRight: 0.4,
61
+ jawOpen: 0.1,
62
+ },
63
+ excited: {
64
+ mouthSmileLeft: 0.55, mouthSmileRight: 0.55,
65
+ eyeWideLeft: 0.3, eyeWideRight: 0.3,
66
+ cheekSquintLeft: 0.3, cheekSquintRight: 0.3,
67
+ browOuterUpLeft: 0.3, browOuterUpRight: 0.3,
68
+ },
69
+ thinking: {
70
+ browInnerUp: 0.3,
71
+ eyeLookUpLeft: 0.2, eyeLookUpRight: 0.2,
72
+ mouthPucker: 0.15,
73
+ },
74
+ concerned: {
75
+ browInnerUp: 0.45, browDownLeft: 0.25, browDownRight: 0.25,
76
+ mouthFrownLeft: 0.2, mouthFrownRight: 0.2,
77
+ },
78
+ disgust: {
79
+ noseSneerLeft: 0.6, noseSneerRight: 0.6,
80
+ browDownLeft: 0.4, browDownRight: 0.4,
81
+ mouthShrugUpper: 0.3,
82
+ },
83
+ fear: {
84
+ eyeWideLeft: 0.6, eyeWideRight: 0.6,
85
+ browInnerUp: 0.7, browOuterUpLeft: 0.3, browOuterUpRight: 0.3,
86
+ mouthStretchLeft: 0.2, mouthStretchRight: 0.2,
87
+ },
88
+ exhausted: {
89
+ eyeBlinkLeft: 0.4, eyeBlinkRight: 0.4,
90
+ browDownLeft: 0.3, browDownRight: 0.3,
91
+ mouthFrownLeft: 0.2, mouthFrownRight: 0.2,
92
+ },
93
+ };
@@ -0,0 +1,11 @@
1
+ export type AuthedFileResult = {
2
+ uri: string;
3
+ size: number;
4
+ } | null;
5
+ /**
6
+ * Downloads a remote URL (with Bearer auth) to the local cache and returns a
7
+ * `file://` URI suitable for react-native-filament's <Model> source prop.
8
+ * The native Filament fetcher doesn't send auth headers, so we pre-fetch here.
9
+ * Also returns the file size in bytes for GPU memory budgeting.
10
+ */
11
+ export declare function useAuthedFilamentUri(remoteUrl: string | null): AuthedFileResult;
@@ -0,0 +1,126 @@
1
+ "use strict";
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ var desc = Object.getOwnPropertyDescriptor(m, k);
5
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
+ desc = { enumerable: true, get: function() { return m[k]; } };
7
+ }
8
+ Object.defineProperty(o, k2, desc);
9
+ }) : (function(o, m, k, k2) {
10
+ if (k2 === undefined) k2 = k;
11
+ o[k2] = m[k];
12
+ }));
13
+ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
+ Object.defineProperty(o, "default", { enumerable: true, value: v });
15
+ }) : function(o, v) {
16
+ o["default"] = v;
17
+ });
18
+ var __importStar = (this && this.__importStar) || (function () {
19
+ var ownKeys = function(o) {
20
+ ownKeys = Object.getOwnPropertyNames || function (o) {
21
+ var ar = [];
22
+ for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
23
+ return ar;
24
+ };
25
+ return ownKeys(o);
26
+ };
27
+ return function (mod) {
28
+ if (mod && mod.__esModule) return mod;
29
+ var result = {};
30
+ if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
31
+ __setModuleDefault(result, mod);
32
+ return result;
33
+ };
34
+ })();
35
+ Object.defineProperty(exports, "__esModule", { value: true });
36
+ exports.useAuthedFilamentUri = useAuthedFilamentUri;
37
+ const react_1 = require("react");
38
+ const FileSystem = __importStar(require("expo-file-system/legacy"));
39
+ const studioApi_1 = require("../api/studioApi");
40
+ // Module-level set of URLs that returned non-GLB responses — skip retrying them.
41
+ const failedUrls = new Set();
42
+ /**
43
+ * Downloads a remote URL (with Bearer auth) to the local cache and returns a
44
+ * `file://` URI suitable for react-native-filament's <Model> source prop.
45
+ * The native Filament fetcher doesn't send auth headers, so we pre-fetch here.
46
+ * Also returns the file size in bytes for GPU memory budgeting.
47
+ */
48
+ function useAuthedFilamentUri(remoteUrl) {
49
+ const [result, setResult] = (0, react_1.useState)(null);
50
+ (0, react_1.useEffect)(() => {
51
+ if (!remoteUrl) {
52
+ setResult(null);
53
+ return;
54
+ }
55
+ // Skip URLs that previously returned a non-GLB response (e.g. 404)
56
+ if (failedUrls.has(remoteUrl))
57
+ return;
58
+ // Pass through non-HTTP URLs (file://, data:, asset://) without downloading
59
+ if (!remoteUrl.startsWith('http://') && !remoteUrl.startsWith('https://')) {
60
+ setResult({ uri: remoteUrl, size: 0 });
61
+ return;
62
+ }
63
+ let cancelled = false;
64
+ (async () => {
65
+ try {
66
+ const token = await (0, studioApi_1.getToken)();
67
+ // Strip query params (access_token changes per session) so the same GLB is cached across sessions
68
+ const urlWithoutQuery = remoteUrl.split('?')[0];
69
+ const key = urlWithoutQuery.replace(/[^a-zA-Z0-9]/g, '_').slice(-100);
70
+ const dir = `${FileSystem.cacheDirectory}filament/`;
71
+ // Append .glb so native loaders can identify the format
72
+ const localPath = `${dir}${key}.glb`;
73
+ await FileSystem.makeDirectoryAsync(dir, { intermediates: true });
74
+ const info = await FileSystem.getInfoAsync(localPath);
75
+ if (info.exists && info.size && info.size > 1000) {
76
+ // Verify magic bytes — GLB files start with 0x676C5446 ("glTF")
77
+ const header = await FileSystem.readAsStringAsync(localPath, {
78
+ encoding: FileSystem.EncodingType.Base64,
79
+ length: 4,
80
+ position: 0,
81
+ });
82
+ // base64("glTF") == "Z2xURg=="
83
+ if (header === 'Z2xURg==') {
84
+ if (!cancelled)
85
+ setResult({ uri: localPath, size: info.size });
86
+ return;
87
+ }
88
+ // Cached file isn't a valid GLB — delete and re-download
89
+ await FileSystem.deleteAsync(localPath);
90
+ }
91
+ const dlResult = await FileSystem.downloadAsync(remoteUrl, localPath, {
92
+ headers: token ? { Authorization: `Bearer ${token}` } : {},
93
+ });
94
+ // Verify we got a valid GLB — check size and magic bytes ("glTF")
95
+ const downloaded = await FileSystem.getInfoAsync(dlResult.uri);
96
+ const downloadedSize = downloaded.size;
97
+ if (!downloaded.exists || !downloadedSize || downloadedSize < 1000) {
98
+ console.error('[useAuthedFilamentUri] Download too small (' + (downloadedSize ?? 0) + 'B), likely an error response');
99
+ failedUrls.add(remoteUrl);
100
+ await FileSystem.deleteAsync(dlResult.uri);
101
+ return;
102
+ }
103
+ const dlHeader = await FileSystem.readAsStringAsync(dlResult.uri, {
104
+ encoding: FileSystem.EncodingType.Base64,
105
+ length: 4,
106
+ position: 0,
107
+ });
108
+ if (dlHeader !== 'Z2xURg==') {
109
+ console.error('[useAuthedFilamentUri] Downloaded file is not a valid GLB (bad magic bytes)');
110
+ failedUrls.add(remoteUrl);
111
+ await FileSystem.deleteAsync(dlResult.uri);
112
+ return;
113
+ }
114
+ if (!cancelled)
115
+ setResult({ uri: dlResult.uri, size: downloadedSize });
116
+ }
117
+ catch (e) {
118
+ console.error('[useAuthedFilamentUri] Failed to download:', remoteUrl.slice(-60), e);
119
+ }
120
+ })();
121
+ return () => {
122
+ cancelled = true;
123
+ };
124
+ }, [remoteUrl]);
125
+ return result;
126
+ }
package/dist/index.d.ts CHANGED
@@ -1,3 +1,12 @@
1
1
  export type { TalkingHeadAccessory, TalkingHeadLoadingState, TalkingHeadLoadingStage, TalkingHeadMood, TalkingHeadProps, TalkingHeadRef, TalkingHeadViseme, TalkingHeadVisemeCue, TalkingHeadVisemeSchedule, } from './TalkingHead';
2
2
  export { TalkingHead } from './TalkingHead';
3
- export * from './appearance';
3
+ export { applyAppearanceToObject3D } from './appearance/apply';
4
+ export { pickTargetForMaterialName } from './appearance/matchers';
5
+ export { normalizeAppearance } from './appearance/schema';
6
+ export type { AppearanceTarget } from './appearance/matchers';
7
+ export * from './api';
8
+ export * from './wardrobe';
9
+ export { TalkingHeadVisualization } from './TalkingHeadVisualization';
10
+ export type { TalkingHeadVisualizationRef } from './TalkingHeadVisualization';
11
+ export { useDirectVisemeStream } from './tts/useDirectVisemeStream';
12
+ export type { VisemeStreamPayload } from './tts/useDirectVisemeStream';
package/dist/index.js CHANGED
@@ -14,7 +14,20 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
14
14
  for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
15
15
  };
16
16
  Object.defineProperty(exports, "__esModule", { value: true });
17
- exports.TalkingHead = void 0;
17
+ exports.useDirectVisemeStream = exports.TalkingHeadVisualization = exports.normalizeAppearance = exports.pickTargetForMaterialName = exports.applyAppearanceToObject3D = exports.TalkingHead = void 0;
18
18
  var TalkingHead_1 = require("./TalkingHead");
19
19
  Object.defineProperty(exports, "TalkingHead", { enumerable: true, get: function () { return TalkingHead_1.TalkingHead; } });
20
- __exportStar(require("./appearance"), exports);
20
+ // Export appearance utilities, but exclude AvatarAppearance — the canonical
21
+ // AvatarAppearance (with equippedAccessories) comes from ./api below.
22
+ var apply_1 = require("./appearance/apply");
23
+ Object.defineProperty(exports, "applyAppearanceToObject3D", { enumerable: true, get: function () { return apply_1.applyAppearanceToObject3D; } });
24
+ var matchers_1 = require("./appearance/matchers");
25
+ Object.defineProperty(exports, "pickTargetForMaterialName", { enumerable: true, get: function () { return matchers_1.pickTargetForMaterialName; } });
26
+ var schema_1 = require("./appearance/schema");
27
+ Object.defineProperty(exports, "normalizeAppearance", { enumerable: true, get: function () { return schema_1.normalizeAppearance; } });
28
+ __exportStar(require("./api"), exports);
29
+ __exportStar(require("./wardrobe"), exports);
30
+ var TalkingHeadVisualization_1 = require("./TalkingHeadVisualization");
31
+ Object.defineProperty(exports, "TalkingHeadVisualization", { enumerable: true, get: function () { return TalkingHeadVisualization_1.TalkingHeadVisualization; } });
32
+ var useDirectVisemeStream_1 = require("./tts/useDirectVisemeStream");
33
+ Object.defineProperty(exports, "useDirectVisemeStream", { enumerable: true, get: function () { return useDirectVisemeStream_1.useDirectVisemeStream; } });
@@ -1,3 +1,7 @@
1
1
  export type { TalkingHeadAccessory, TalkingHeadLoadingState, TalkingHeadMood, TalkingHeadProps, TalkingHeadRef, } from './TalkingHead.web';
2
2
  export { TalkingHead } from './TalkingHead.web';
3
- export * from './appearance';
3
+ export { applyAppearanceToObject3D } from './appearance/apply';
4
+ export { pickTargetForMaterialName } from './appearance/matchers';
5
+ export { normalizeAppearance } from './appearance/schema';
6
+ export type { AppearanceTarget } from './appearance/matchers';
7
+ export * from './api';
package/dist/index.web.js CHANGED
@@ -14,7 +14,15 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
14
14
  for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
15
15
  };
16
16
  Object.defineProperty(exports, "__esModule", { value: true });
17
- exports.TalkingHead = void 0;
17
+ exports.normalizeAppearance = exports.pickTargetForMaterialName = exports.applyAppearanceToObject3D = exports.TalkingHead = void 0;
18
18
  var TalkingHead_web_1 = require("./TalkingHead.web");
19
19
  Object.defineProperty(exports, "TalkingHead", { enumerable: true, get: function () { return TalkingHead_web_1.TalkingHead; } });
20
- __exportStar(require("./appearance"), exports);
20
+ // Export appearance utilities, but exclude AvatarAppearance — the canonical
21
+ // AvatarAppearance (with equippedAccessories) comes from ./api below.
22
+ var apply_1 = require("./appearance/apply");
23
+ Object.defineProperty(exports, "applyAppearanceToObject3D", { enumerable: true, get: function () { return apply_1.applyAppearanceToObject3D; } });
24
+ var matchers_1 = require("./appearance/matchers");
25
+ Object.defineProperty(exports, "pickTargetForMaterialName", { enumerable: true, get: function () { return matchers_1.pickTargetForMaterialName; } });
26
+ var schema_1 = require("./appearance/schema");
27
+ Object.defineProperty(exports, "normalizeAppearance", { enumerable: true, get: function () { return schema_1.normalizeAppearance; } });
28
+ __exportStar(require("./api"), exports);
@@ -0,0 +1,21 @@
1
+ import type { TalkingHeadVisemeSchedule } from "../TalkingHead";
2
+ export type VisemeStreamPayload = {
3
+ requestId?: string;
4
+ durationMs?: number;
5
+ cues?: TalkingHeadVisemeSchedule["cues"];
6
+ };
7
+ type OpenStreamOptions = {
8
+ requestId: string;
9
+ ttsBaseUrl: string;
10
+ };
11
+ /**
12
+ * Opens a direct SSE connection to the TTS server to receive viseme data,
13
+ * bypassing the agent data channel relay.
14
+ *
15
+ * Uses fetch() with streaming response body instead of EventSource because
16
+ * React Native does not have a reliable EventSource polyfill.
17
+ */
18
+ export declare function useDirectVisemeStream(onVisemes: (payload: VisemeStreamPayload) => void): {
19
+ openStream: ({ requestId, ttsBaseUrl }: OpenStreamOptions) => void;
20
+ };
21
+ export {};
@@ -0,0 +1,119 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.useDirectVisemeStream = useDirectVisemeStream;
4
+ const react_1 = require("react");
5
+ const fetch_1 = require("expo/fetch");
6
+ /**
7
+ * Opens a direct SSE connection to the TTS server to receive viseme data,
8
+ * bypassing the agent data channel relay.
9
+ *
10
+ * Uses fetch() with streaming response body instead of EventSource because
11
+ * React Native does not have a reliable EventSource polyfill.
12
+ */
13
+ function useDirectVisemeStream(onVisemes) {
14
+ // Track the current abort controller keyed by requestId so we can detect
15
+ // when a new requestId arrives and tear down the previous stream.
16
+ const abortControllerRef = (0, react_1.useRef)(null);
17
+ const activeRequestIdRef = (0, react_1.useRef)(null);
18
+ const onVisemesRef = (0, react_1.useRef)(onVisemes);
19
+ // Keep callback ref up to date without requiring it in openStream's dep array
20
+ (0, react_1.useEffect)(() => {
21
+ onVisemesRef.current = onVisemes;
22
+ });
23
+ const openStream = (0, react_1.useCallback)(({ requestId, ttsBaseUrl }) => {
24
+ // Abort any existing stream — whether for the same or a different requestId
25
+ if (abortControllerRef.current) {
26
+ abortControllerRef.current.abort();
27
+ abortControllerRef.current = null;
28
+ }
29
+ activeRequestIdRef.current = requestId;
30
+ const controller = new AbortController();
31
+ abortControllerRef.current = controller;
32
+ const { signal } = controller;
33
+ // Strip trailing /v1 if present so we don't double it
34
+ const base = ttsBaseUrl.replace(/\/v1\/?$/, '');
35
+ const url = `${base}/v1/audio/visemes/${encodeURIComponent(requestId)}/stream`;
36
+ (async () => {
37
+ try {
38
+ const response = await (0, fetch_1.fetch)(url, {
39
+ headers: { Accept: "text/event-stream" },
40
+ signal,
41
+ });
42
+ if (!response.ok) {
43
+ console.warn("[VisemeSSE] Non-OK response", { requestId, status: response.status });
44
+ return;
45
+ }
46
+ const reader = response.body?.getReader();
47
+ if (!reader) {
48
+ console.warn("[VisemeSSE] No response body reader", { requestId });
49
+ return;
50
+ }
51
+ const decoder = new TextDecoder();
52
+ let buffer = "";
53
+ let pendingEvent = null;
54
+ while (true) {
55
+ const { done, value } = await reader.read();
56
+ if (done)
57
+ break;
58
+ buffer += decoder.decode(value, { stream: true });
59
+ // Split on newlines, keeping the remainder (incomplete line) in buffer
60
+ const lines = buffer.split("\n");
61
+ buffer = lines.pop() ?? "";
62
+ for (const rawLine of lines) {
63
+ const line = rawLine.trimEnd();
64
+ if (line.startsWith("event:")) {
65
+ pendingEvent = line.slice("event:".length).trim();
66
+ }
67
+ else if (line.startsWith("data:")) {
68
+ if (pendingEvent === "visemes") {
69
+ const jsonText = line.slice("data:".length).trim();
70
+ try {
71
+ const payload = JSON.parse(jsonText);
72
+ console.log("[VisemeSSE] received", {
73
+ requestId,
74
+ cues: Array.isArray(payload.cues) ? payload.cues.length : 0,
75
+ durationMs: payload.durationMs ?? null,
76
+ });
77
+ onVisemesRef.current(payload);
78
+ }
79
+ catch (parseErr) {
80
+ console.warn("[VisemeSSE] JSON parse error", parseErr);
81
+ }
82
+ }
83
+ // Reset pending event after consuming the data line
84
+ pendingEvent = null;
85
+ }
86
+ else if (line === "") {
87
+ // Empty line = end of SSE message block; reset pending event
88
+ pendingEvent = null;
89
+ }
90
+ }
91
+ }
92
+ console.log("[VisemeSSE] stream ended", { requestId });
93
+ }
94
+ catch (err) {
95
+ if (err?.name === "AbortError") {
96
+ // Expected — stream was intentionally cancelled
97
+ return;
98
+ }
99
+ console.warn("[VisemeSSE] stream error", { requestId, err });
100
+ }
101
+ finally {
102
+ // Only clear the ref if it still points to our controller
103
+ if (abortControllerRef.current === controller) {
104
+ abortControllerRef.current = null;
105
+ }
106
+ }
107
+ })();
108
+ }, []);
109
+ // Clean up on unmount
110
+ (0, react_1.useEffect)(() => {
111
+ return () => {
112
+ if (abortControllerRef.current) {
113
+ abortControllerRef.current.abort();
114
+ abortControllerRef.current = null;
115
+ }
116
+ };
117
+ }, []);
118
+ return { openStream };
119
+ }
@@ -0,0 +1,13 @@
1
+ /**
2
+ * Resolves a local Expo asset module into a file:// URI for use with
3
+ * react-native-filament's useModel(). Never base64-encodes — Filament reads
4
+ * file:// directly on both iOS and Android.
5
+ */
6
+ export declare function resolveFilamentAssetUri(module: unknown): Promise<string | null>;
7
+ /**
8
+ * Resolves a local Expo asset module (from require()) into a usable URL string.
9
+ * On web, returns an absolute URL.
10
+ * On Android/iOS, converts to a base64 data URI so the WebView can load it
11
+ * without a cross-origin file:// fetch (which is blocked on Android).
12
+ */
13
+ export declare function resolveLocalAssetUrl(module: unknown): Promise<string | null>;
@@ -0,0 +1,56 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.resolveFilamentAssetUri = resolveFilamentAssetUri;
4
+ exports.resolveLocalAssetUrl = resolveLocalAssetUrl;
5
+ const expo_asset_1 = require("expo-asset");
6
+ const react_native_1 = require("react-native");
7
+ const expo_file_system_1 = require("expo-file-system");
8
+ /**
9
+ * Resolves a local Expo asset module into a file:// URI for use with
10
+ * react-native-filament's useModel(). Never base64-encodes — Filament reads
11
+ * file:// directly on both iOS and Android.
12
+ */
13
+ async function resolveFilamentAssetUri(module) {
14
+ try {
15
+ const asset = expo_asset_1.Asset.fromModule(module);
16
+ await asset.downloadAsync();
17
+ const uri = asset.localUri ?? asset.uri;
18
+ return uri ?? null;
19
+ }
20
+ catch (e) {
21
+ console.error('[AssetUtils] Failed to resolve Filament asset:', e);
22
+ return null;
23
+ }
24
+ }
25
+ /**
26
+ * Resolves a local Expo asset module (from require()) into a usable URL string.
27
+ * On web, returns an absolute URL.
28
+ * On Android/iOS, converts to a base64 data URI so the WebView can load it
29
+ * without a cross-origin file:// fetch (which is blocked on Android).
30
+ */
31
+ async function resolveLocalAssetUrl(module) {
32
+ try {
33
+ const asset = expo_asset_1.Asset.fromModule(module);
34
+ await asset.downloadAsync();
35
+ const uri = asset.localUri ?? asset.uri;
36
+ if (!uri)
37
+ return null;
38
+ if (react_native_1.Platform.OS === 'web') {
39
+ if (uri.startsWith('/')) {
40
+ return window.location.origin + uri;
41
+ }
42
+ return uri;
43
+ }
44
+ // On Android the WebView cannot fetch file:// URIs — convert to data URI.
45
+ if (react_native_1.Platform.OS === 'android' && uri.startsWith('file://')) {
46
+ const file = new expo_file_system_1.File(uri);
47
+ const base64 = await file.base64();
48
+ return `data:model/gltf-binary;base64,${base64}`;
49
+ }
50
+ return uri;
51
+ }
52
+ catch (e) {
53
+ console.error('[AssetUtils] Failed to resolve local asset:', e);
54
+ return null;
55
+ }
56
+ }
@@ -0,0 +1,2 @@
1
+ export * from './wardrobeStore';
2
+ export { useAvatarWardrobeHydration } from './useAvatarWardrobeHydration';
@@ -0,0 +1,20 @@
1
+ "use strict";
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ var desc = Object.getOwnPropertyDescriptor(m, k);
5
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
+ desc = { enumerable: true, get: function() { return m[k]; } };
7
+ }
8
+ Object.defineProperty(o, k2, desc);
9
+ }) : (function(o, m, k, k2) {
10
+ if (k2 === undefined) k2 = k;
11
+ o[k2] = m[k];
12
+ }));
13
+ var __exportStar = (this && this.__exportStar) || function(m, exports) {
14
+ for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
15
+ };
16
+ Object.defineProperty(exports, "__esModule", { value: true });
17
+ exports.useAvatarWardrobeHydration = void 0;
18
+ __exportStar(require("./wardrobeStore"), exports);
19
+ var useAvatarWardrobeHydration_1 = require("./useAvatarWardrobeHydration");
20
+ Object.defineProperty(exports, "useAvatarWardrobeHydration", { enumerable: true, get: function () { return useAvatarWardrobeHydration_1.useAvatarWardrobeHydration; } });
@@ -0,0 +1,7 @@
1
+ import type { EquippedAccessory } from '../api/types';
2
+ type UseAvatarWardrobeHydrationArgs = {
3
+ avatarId: string | null | undefined;
4
+ accessories: EquippedAccessory[] | null | undefined;
5
+ };
6
+ export declare function useAvatarWardrobeHydration({ avatarId, accessories, }: UseAvatarWardrobeHydrationArgs): void;
7
+ export {};
@@ -0,0 +1,34 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.useAvatarWardrobeHydration = useAvatarWardrobeHydration;
4
+ const react_1 = require("react");
5
+ const studioApi_1 = require("../api/studioApi");
6
+ const wardrobeStore_1 = require("./wardrobeStore");
7
+ function useAvatarWardrobeHydration({ avatarId, accessories, }) {
8
+ const hydrateFromApi = (0, wardrobeStore_1.useWardrobeStore)((s) => s.hydrateFromApi);
9
+ const accessorySignature = (0, react_1.useMemo)(() => JSON.stringify(accessories ?? []), [accessories]);
10
+ (0, react_1.useEffect)(() => {
11
+ const nextAccessories = accessories ?? [];
12
+ if (!avatarId || nextAccessories.length === 0) {
13
+ hydrateFromApi([], {});
14
+ return;
15
+ }
16
+ let cancelled = false;
17
+ void (0, studioApi_1.listAssets)()
18
+ .then((allAssets) => {
19
+ if (cancelled)
20
+ return;
21
+ const lookup = {};
22
+ for (const asset of allAssets) {
23
+ lookup[asset.id] = asset;
24
+ }
25
+ hydrateFromApi(nextAccessories, lookup);
26
+ })
27
+ .catch((err) => {
28
+ console.error('[useAvatarWardrobeHydration] Failed to hydrate wardrobe:', err);
29
+ });
30
+ return () => {
31
+ cancelled = true;
32
+ };
33
+ }, [accessorySignature, avatarId, hydrateFromApi]);
34
+ }