talking-head-studio 0.4.10 → 0.4.12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (178) hide show
  1. package/README.md +299 -337
  2. package/dist/TalkingHead.d.ts +44 -28
  3. package/dist/TalkingHead.js +21 -2
  4. package/dist/TalkingHead.web.d.ts +37 -4
  5. package/dist/TalkingHead.web.js +28 -8
  6. package/dist/TalkingHeadVisualization.d.ts +22 -0
  7. package/dist/TalkingHeadVisualization.js +30 -10
  8. package/dist/api/studioApi.d.ts +12 -1
  9. package/dist/api/studioApi.js +41 -28
  10. package/dist/appearance/apply.js +2 -3
  11. package/dist/appearance/matchers.js +1 -2
  12. package/dist/appearance/schema.js +1 -2
  13. package/dist/contract.d.ts +14 -0
  14. package/dist/contract.js +30 -0
  15. package/dist/core/avatar/avatarCapabilities.d.ts +60 -0
  16. package/dist/core/avatar/avatarCapabilities.js +100 -0
  17. package/dist/core/avatar/backend.d.ts +130 -0
  18. package/dist/core/avatar/backend.js +4 -0
  19. package/dist/core/avatar/backends/gaussian.d.ts +49 -0
  20. package/dist/core/avatar/backends/gaussian.js +293 -0
  21. package/dist/core/avatar/backends/index.d.ts +3 -0
  22. package/dist/core/avatar/backends/index.js +7 -0
  23. package/dist/core/avatar/backends/morphTarget.d.ts +39 -0
  24. package/dist/core/avatar/backends/morphTarget.js +179 -0
  25. package/dist/core/avatar/faceControls.d.ts +40 -0
  26. package/dist/core/avatar/faceControls.js +138 -0
  27. package/dist/core/avatar/motion.d.ts +1713 -0
  28. package/dist/core/avatar/motion.js +550 -0
  29. package/dist/core/avatar/motionRuntime.d.ts +46 -0
  30. package/dist/core/avatar/motionRuntime.js +84 -0
  31. package/dist/core/avatar/schema.d.ts +78 -0
  32. package/dist/core/avatar/schema.js +134 -0
  33. package/dist/core/avatar/visemes.d.ts +47 -1
  34. package/dist/core/avatar/visemes.js +114 -1
  35. package/dist/editor/AvatarCanvas.js +93 -3
  36. package/dist/editor/AvatarEditor.native.js +19 -9
  37. package/dist/editor/AvatarModel.js +2 -2
  38. package/dist/editor/FaceSqueezeEditor.d.ts +3 -1
  39. package/dist/editor/FaceSqueezeEditor.js +195 -121
  40. package/dist/editor/FaceSqueezeEditor.web.d.ts +3 -1
  41. package/dist/editor/FaceSqueezeEditor.web.js +32 -30
  42. package/dist/editor/RigidAccessory.js +18 -4
  43. package/dist/editor/SkinnedClothing.js +19 -9
  44. package/dist/editor/boneLockedDrag.d.ts +11 -0
  45. package/dist/editor/boneLockedDrag.js +68 -0
  46. package/dist/editor/boneSnap.js +22 -12
  47. package/dist/editor/boneSnap.web.d.ts +27 -0
  48. package/dist/editor/boneSnap.web.js +99 -0
  49. package/dist/editor/index.web.d.ts +10 -0
  50. package/dist/editor/index.web.js +26 -0
  51. package/dist/editor/sounds/haha.wav +0 -0
  52. package/dist/editor/sounds/owie.wav +0 -0
  53. package/dist/editor/sounds/stop.wav +0 -0
  54. package/dist/editor/studioTheme.d.ts +14 -14
  55. package/dist/editor/studioTheme.js +19 -16
  56. package/dist/editor/types.d.ts +1 -0
  57. package/dist/html/accessories.d.ts +7 -0
  58. package/dist/html/accessories.js +149 -0
  59. package/dist/html/motion.d.ts +1 -0
  60. package/dist/html/motion.js +189 -0
  61. package/dist/html/visemes.d.ts +7 -0
  62. package/dist/html/visemes.js +348 -0
  63. package/dist/html.d.ts +1 -1
  64. package/dist/html.js +56 -734
  65. package/dist/index.d.ts +19 -1
  66. package/dist/index.js +44 -5
  67. package/dist/index.web.d.ts +18 -1
  68. package/dist/index.web.js +36 -3
  69. package/dist/platform/api/types.d.ts +10 -0
  70. package/dist/platform/api/types.js +2 -0
  71. package/dist/platform/marketplace/types.d.ts +32 -0
  72. package/dist/platform/marketplace/types.js +2 -0
  73. package/dist/platform/sdk/unity.d.ts +27 -0
  74. package/dist/platform/sdk/unity.js +2 -0
  75. package/dist/platform/sdk/unreal.d.ts +23 -0
  76. package/dist/platform/sdk/unreal.js +2 -0
  77. package/dist/platform/sdk/web.d.ts +16 -0
  78. package/dist/platform/sdk/web.js +2 -0
  79. package/dist/sketchfab/api.js +5 -5
  80. package/dist/sketchfab/glbInspect.d.ts +22 -0
  81. package/dist/sketchfab/glbInspect.js +58 -0
  82. package/dist/sketchfab/index.d.ts +3 -0
  83. package/dist/sketchfab/index.js +8 -1
  84. package/dist/sketchfab/inspectRemote.d.ts +13 -0
  85. package/dist/sketchfab/inspectRemote.js +77 -0
  86. package/dist/sketchfab/types.d.ts +10 -0
  87. package/dist/sketchfab/useSketchfabSearch.js +1 -2
  88. package/dist/studio/AccessoryBrowserScreen.d.ts +6 -0
  89. package/dist/studio/AccessoryBrowserScreen.js +626 -0
  90. package/dist/studio/AccessoryPanel.d.ts +10 -0
  91. package/dist/studio/AccessoryPanel.js +396 -0
  92. package/dist/studio/AppearancePanel.d.ts +9 -0
  93. package/dist/studio/AppearancePanel.js +77 -0
  94. package/dist/studio/AvatarCreatorScreen.d.ts +5 -0
  95. package/dist/studio/AvatarCreatorScreen.js +806 -0
  96. package/dist/studio/AvatarEditorScreen.d.ts +14 -0
  97. package/dist/studio/AvatarEditorScreen.js +510 -0
  98. package/dist/studio/AvatarGrid.d.ts +23 -0
  99. package/dist/studio/AvatarGrid.js +257 -0
  100. package/dist/studio/ColorSwatch.d.ts +8 -0
  101. package/dist/studio/ColorSwatch.js +100 -0
  102. package/dist/studio/CreateVoiceProfileSheet.d.ts +8 -0
  103. package/dist/studio/CreateVoiceProfileSheet.js +242 -0
  104. package/dist/studio/DetailsPanel.d.ts +15 -0
  105. package/dist/studio/DetailsPanel.js +239 -0
  106. package/dist/studio/FilamentEditor.d.ts +2 -0
  107. package/dist/studio/FilamentEditor.js +6 -0
  108. package/dist/studio/PrecisionPanel.d.ts +2 -0
  109. package/dist/studio/PrecisionPanel.js +7 -0
  110. package/dist/studio/PublicGalleryScreen.d.ts +5 -0
  111. package/dist/studio/PublicGalleryScreen.js +358 -0
  112. package/dist/studio/SketchfabModelCard.d.ts +20 -0
  113. package/dist/studio/SketchfabModelCard.js +104 -0
  114. package/dist/studio/StudioBrowseHeader.d.ts +9 -0
  115. package/dist/studio/StudioBrowseHeader.js +28 -0
  116. package/dist/studio/StudioEmptyState.d.ts +8 -0
  117. package/dist/studio/StudioEmptyState.js +29 -0
  118. package/dist/studio/StudioFloatingAction.d.ts +13 -0
  119. package/dist/studio/StudioFloatingAction.js +42 -0
  120. package/dist/studio/StudioSectionHeader.d.ts +7 -0
  121. package/dist/studio/StudioSectionHeader.js +27 -0
  122. package/dist/studio/StudioSurfaceCard.d.ts +8 -0
  123. package/dist/studio/StudioSurfaceCard.js +20 -0
  124. package/dist/studio/VoicePanel.d.ts +15 -0
  125. package/dist/studio/VoicePanel.js +305 -0
  126. package/dist/studio/constants.d.ts +3 -0
  127. package/dist/studio/constants.js +6 -0
  128. package/dist/studio/index.d.ts +29 -0
  129. package/dist/studio/index.js +54 -0
  130. package/dist/studio/useSketchfabCapabilities.d.ts +31 -0
  131. package/dist/studio/useSketchfabCapabilities.js +82 -0
  132. package/dist/tts/useDirectVisemeStream.d.ts +2 -6
  133. package/dist/tts/useDirectVisemeStream.js +16 -12
  134. package/dist/tts/useMotionMarkers.d.ts +0 -1
  135. package/dist/tts/useMotionMarkers.js +1 -2
  136. package/dist/utils/avatarUtils.js +94 -8
  137. package/dist/utils/faceLandmarkerToShapeWeights.js +21 -14
  138. package/dist/voice/convertToWav.js +1 -2
  139. package/dist/voice/index.d.ts +3 -0
  140. package/dist/voice/index.js +6 -1
  141. package/dist/voice/useAudioPlayer.js +18 -6
  142. package/dist/voice/useAudioRecording.js +1 -2
  143. package/dist/voice/useFaceControls.d.ts +14 -0
  144. package/dist/voice/useFaceControls.js +81 -0
  145. package/dist/voice/useVoicePreview.d.ts +7 -0
  146. package/dist/voice/useVoicePreview.js +83 -0
  147. package/dist/wardrobe/index.d.ts +3 -0
  148. package/dist/wardrobe/index.js +8 -1
  149. package/dist/wardrobe/useAccessoryGestures.d.ts +20 -0
  150. package/dist/wardrobe/useAccessoryGestures.js +94 -0
  151. package/dist/wardrobe/useAvatarWardrobeHydration.js +9 -4
  152. package/dist/wardrobe/useStudioAvatar.d.ts +29 -0
  153. package/dist/wardrobe/useStudioAvatar.js +186 -0
  154. package/dist/wardrobe/wardrobeStore.d.ts +2 -0
  155. package/dist/wardrobe/wardrobeStore.js +12 -2
  156. package/dist/wgpu/R3FWebGpuCanvas.d.ts +15 -0
  157. package/dist/wgpu/R3FWebGpuCanvas.js +176 -0
  158. package/dist/wgpu/WgpuAvatar.d.ts +26 -2
  159. package/dist/wgpu/WgpuAvatar.js +313 -46
  160. package/dist/wgpu/accessoryDefaults.d.ts +12 -0
  161. package/dist/wgpu/accessoryDefaults.js +19 -0
  162. package/dist/wgpu/blobShim.d.ts +2 -0
  163. package/dist/wgpu/blobShim.js +191 -0
  164. package/dist/wgpu/index.d.ts +1 -0
  165. package/dist/wgpu/index.js +4 -1
  166. package/dist/wgpu/loadGLTFFromUri.d.ts +2 -0
  167. package/dist/wgpu/loadGLTFFromUri.js +75 -0
  168. package/dist/wgpu/morphTables.js +21 -10
  169. package/dist/wgpu/motionState.d.ts +20 -0
  170. package/dist/wgpu/motionState.js +31 -0
  171. package/dist/wgpu/patchThreeForRN.d.ts +28 -0
  172. package/dist/wgpu/patchThreeForRN.js +292 -0
  173. package/dist/wgpu/scenePlacement.d.ts +5 -0
  174. package/dist/wgpu/scenePlacement.js +50 -0
  175. package/dist/wgpu/useAuthedModelUri.js +22 -11
  176. package/dist/wgpu/useNativeGLTF.d.ts +7 -0
  177. package/dist/wgpu/useNativeGLTF.js +36 -0
  178. package/package.json +102 -32
@@ -0,0 +1,29 @@
1
+ export { default as ColorSwatch } from './ColorSwatch';
2
+ export type { ColorSwatchProps } from './ColorSwatch';
3
+ export { StudioSurfaceCard, default as StudioSurfaceCardDefault } from './StudioSurfaceCard';
4
+ export { StudioEmptyState } from './StudioEmptyState';
5
+ export { StudioSectionHeader } from './StudioSectionHeader';
6
+ export { StudioFloatingAction } from './StudioFloatingAction';
7
+ export { StudioBrowseHeader } from './StudioBrowseHeader';
8
+ export { default as AppearancePanel } from './AppearancePanel';
9
+ export type { AppearancePanelProps } from './AppearancePanel';
10
+ export { default as AvatarGrid } from './AvatarGrid';
11
+ export type { AvatarGridItem, AvatarGridProps } from './AvatarGrid';
12
+ export { DetailsPanel } from './DetailsPanel';
13
+ export { default as AccessoryPanel } from './AccessoryPanel';
14
+ export type { AccessoryPanelProps } from './AccessoryPanel';
15
+ export { default as VoicePanel } from './VoicePanel';
16
+ export { default as CreateVoiceProfileSheet } from './CreateVoiceProfileSheet';
17
+ export { FilamentEditor } from './FilamentEditor';
18
+ export type { FilamentEditorProps } from './FilamentEditor';
19
+ export { PrecisionPanel } from './PrecisionPanel';
20
+ export { SketchfabModelCard } from './SketchfabModelCard';
21
+ export { AccessoryBrowserScreen } from './AccessoryBrowserScreen';
22
+ export type { AccessoryBrowserScreenProps } from './AccessoryBrowserScreen';
23
+ export { AvatarCreatorScreen } from './AvatarCreatorScreen';
24
+ export type { AvatarCreatorScreenProps } from './AvatarCreatorScreen';
25
+ export { AvatarEditorScreen } from './AvatarEditorScreen';
26
+ export type { AvatarEditorScreenProps } from './AvatarEditorScreen';
27
+ export { PublicGalleryScreen } from './PublicGalleryScreen';
28
+ export type { PublicGalleryScreenProps } from './PublicGalleryScreen';
29
+ export { BLURHASH, MAX_ASSET_SIZE, RESULTS_PER_PAGE } from './constants';
@@ -0,0 +1,54 @@
1
+ "use strict";
2
+ var __importDefault = (this && this.__importDefault) || function (mod) {
3
+ return (mod && mod.__esModule) ? mod : { "default": mod };
4
+ };
5
+ Object.defineProperty(exports, "__esModule", { value: true });
6
+ exports.RESULTS_PER_PAGE = exports.MAX_ASSET_SIZE = exports.BLURHASH = exports.PublicGalleryScreen = exports.AvatarEditorScreen = exports.AvatarCreatorScreen = exports.AccessoryBrowserScreen = exports.SketchfabModelCard = exports.PrecisionPanel = exports.FilamentEditor = exports.CreateVoiceProfileSheet = exports.VoicePanel = exports.AccessoryPanel = exports.DetailsPanel = exports.AvatarGrid = exports.AppearancePanel = exports.StudioBrowseHeader = exports.StudioFloatingAction = exports.StudioSectionHeader = exports.StudioEmptyState = exports.StudioSurfaceCardDefault = exports.StudioSurfaceCard = exports.ColorSwatch = void 0;
7
+ // UI primitives
8
+ var ColorSwatch_1 = require("./ColorSwatch");
9
+ Object.defineProperty(exports, "ColorSwatch", { enumerable: true, get: function () { return __importDefault(ColorSwatch_1).default; } });
10
+ var StudioSurfaceCard_1 = require("./StudioSurfaceCard");
11
+ Object.defineProperty(exports, "StudioSurfaceCard", { enumerable: true, get: function () { return StudioSurfaceCard_1.StudioSurfaceCard; } });
12
+ Object.defineProperty(exports, "StudioSurfaceCardDefault", { enumerable: true, get: function () { return __importDefault(StudioSurfaceCard_1).default; } });
13
+ var StudioEmptyState_1 = require("./StudioEmptyState");
14
+ Object.defineProperty(exports, "StudioEmptyState", { enumerable: true, get: function () { return StudioEmptyState_1.StudioEmptyState; } });
15
+ var StudioSectionHeader_1 = require("./StudioSectionHeader");
16
+ Object.defineProperty(exports, "StudioSectionHeader", { enumerable: true, get: function () { return StudioSectionHeader_1.StudioSectionHeader; } });
17
+ var StudioFloatingAction_1 = require("./StudioFloatingAction");
18
+ Object.defineProperty(exports, "StudioFloatingAction", { enumerable: true, get: function () { return StudioFloatingAction_1.StudioFloatingAction; } });
19
+ var StudioBrowseHeader_1 = require("./StudioBrowseHeader");
20
+ Object.defineProperty(exports, "StudioBrowseHeader", { enumerable: true, get: function () { return StudioBrowseHeader_1.StudioBrowseHeader; } });
21
+ // Editor panels
22
+ var AppearancePanel_1 = require("./AppearancePanel");
23
+ Object.defineProperty(exports, "AppearancePanel", { enumerable: true, get: function () { return __importDefault(AppearancePanel_1).default; } });
24
+ var AvatarGrid_1 = require("./AvatarGrid");
25
+ Object.defineProperty(exports, "AvatarGrid", { enumerable: true, get: function () { return __importDefault(AvatarGrid_1).default; } });
26
+ var DetailsPanel_1 = require("./DetailsPanel");
27
+ Object.defineProperty(exports, "DetailsPanel", { enumerable: true, get: function () { return DetailsPanel_1.DetailsPanel; } });
28
+ var AccessoryPanel_1 = require("./AccessoryPanel");
29
+ Object.defineProperty(exports, "AccessoryPanel", { enumerable: true, get: function () { return __importDefault(AccessoryPanel_1).default; } });
30
+ var VoicePanel_1 = require("./VoicePanel");
31
+ Object.defineProperty(exports, "VoicePanel", { enumerable: true, get: function () { return __importDefault(VoicePanel_1).default; } });
32
+ var CreateVoiceProfileSheet_1 = require("./CreateVoiceProfileSheet");
33
+ Object.defineProperty(exports, "CreateVoiceProfileSheet", { enumerable: true, get: function () { return __importDefault(CreateVoiceProfileSheet_1).default; } });
34
+ var FilamentEditor_1 = require("./FilamentEditor");
35
+ Object.defineProperty(exports, "FilamentEditor", { enumerable: true, get: function () { return FilamentEditor_1.FilamentEditor; } });
36
+ var PrecisionPanel_1 = require("./PrecisionPanel");
37
+ Object.defineProperty(exports, "PrecisionPanel", { enumerable: true, get: function () { return PrecisionPanel_1.PrecisionPanel; } });
38
+ // Sketchfab shared
39
+ var SketchfabModelCard_1 = require("./SketchfabModelCard");
40
+ Object.defineProperty(exports, "SketchfabModelCard", { enumerable: true, get: function () { return SketchfabModelCard_1.SketchfabModelCard; } });
41
+ // Screen-level components (navigation via props/callbacks)
42
+ var AccessoryBrowserScreen_1 = require("./AccessoryBrowserScreen");
43
+ Object.defineProperty(exports, "AccessoryBrowserScreen", { enumerable: true, get: function () { return AccessoryBrowserScreen_1.AccessoryBrowserScreen; } });
44
+ var AvatarCreatorScreen_1 = require("./AvatarCreatorScreen");
45
+ Object.defineProperty(exports, "AvatarCreatorScreen", { enumerable: true, get: function () { return AvatarCreatorScreen_1.AvatarCreatorScreen; } });
46
+ var AvatarEditorScreen_1 = require("./AvatarEditorScreen");
47
+ Object.defineProperty(exports, "AvatarEditorScreen", { enumerable: true, get: function () { return AvatarEditorScreen_1.AvatarEditorScreen; } });
48
+ var PublicGalleryScreen_1 = require("./PublicGalleryScreen");
49
+ Object.defineProperty(exports, "PublicGalleryScreen", { enumerable: true, get: function () { return PublicGalleryScreen_1.PublicGalleryScreen; } });
50
+ // Constants
51
+ var constants_1 = require("./constants");
52
+ Object.defineProperty(exports, "BLURHASH", { enumerable: true, get: function () { return constants_1.BLURHASH; } });
53
+ Object.defineProperty(exports, "MAX_ASSET_SIZE", { enumerable: true, get: function () { return constants_1.MAX_ASSET_SIZE; } });
54
+ Object.defineProperty(exports, "RESULTS_PER_PAGE", { enumerable: true, get: function () { return constants_1.RESULTS_PER_PAGE; } });
@@ -0,0 +1,31 @@
1
+ import type { SketchfabModel } from '../sketchfab';
2
+ import type { AvatarCapabilities } from '../core/avatar/avatarCapabilities';
3
+ /** Per-model inspection state. A model's capabilities never change, so once a
4
+ * uid is 'done'/'unknown' it stays cached for the whole session. */
5
+ export type CapabilityEntry = {
6
+ status: 'pending';
7
+ } | {
8
+ status: 'unknown';
9
+ } | {
10
+ status: 'done';
11
+ cap: AvatarCapabilities;
12
+ };
13
+ export interface UseSketchfabCapabilitiesOptions {
14
+ models: SketchfabModel[];
15
+ apiKey: string;
16
+ /** How many leading results to pre-inspect. */
17
+ topN?: number;
18
+ /** Max concurrent range-inspections. */
19
+ concurrency?: number;
20
+ enabled?: boolean;
21
+ }
22
+ export declare const DEFAULT_CAPABILITY_TOP_N = 12;
23
+ /**
24
+ * Background-inspects the top-N search results (range-reads each GLB's JSON
25
+ * chunk) so the picker can rank/badge known-good avatars before import.
26
+ *
27
+ * Results are cached per uid for the session and survive query changes. The
28
+ * effect keys on the *set* of leading uids — appending pages (load-more) does
29
+ * not re-run it, so in-flight inspections are never thrashed.
30
+ */
31
+ export declare function useSketchfabCapabilities({ models, apiKey, topN, concurrency, enabled, }: UseSketchfabCapabilitiesOptions): Map<string, CapabilityEntry>;
@@ -0,0 +1,82 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.DEFAULT_CAPABILITY_TOP_N = void 0;
4
+ exports.useSketchfabCapabilities = useSketchfabCapabilities;
5
+ const react_1 = require("react");
6
+ const sketchfab_1 = require("../sketchfab");
7
+ exports.DEFAULT_CAPABILITY_TOP_N = 12;
8
+ const DEFAULT_CONCURRENCY = 3;
9
+ /**
10
+ * Background-inspects the top-N search results (range-reads each GLB's JSON
11
+ * chunk) so the picker can rank/badge known-good avatars before import.
12
+ *
13
+ * Results are cached per uid for the session and survive query changes. The
14
+ * effect keys on the *set* of leading uids — appending pages (load-more) does
15
+ * not re-run it, so in-flight inspections are never thrashed.
16
+ */
17
+ function useSketchfabCapabilities({ models, apiKey, topN = exports.DEFAULT_CAPABILITY_TOP_N, concurrency = DEFAULT_CONCURRENCY, enabled = true, }) {
18
+ // Durable per-uid cache; the snapshot is a cheap clone published on mutation
19
+ // so consumers re-render.
20
+ const cacheRef = (0, react_1.useRef)(new Map());
21
+ const [snapshot, setSnapshot] = (0, react_1.useState)(() => new Map());
22
+ const publish = (0, react_1.useCallback)(() => {
23
+ setSnapshot(new Map(cacheRef.current));
24
+ }, []);
25
+ // Mirror the latest props into a ref (in an effect, never during render) so
26
+ // the inspection effect can read them without re-running on every list
27
+ // change — it should only re-run when the leading uid set changes.
28
+ const latest = (0, react_1.useRef)({ models, apiKey, topN, concurrency });
29
+ (0, react_1.useEffect)(() => {
30
+ latest.current = { models, apiKey, topN, concurrency };
31
+ });
32
+ const topKey = enabled && apiKey
33
+ ? models.slice(0, topN).map((m) => m.uid).join(',')
34
+ : '';
35
+ (0, react_1.useEffect)(() => {
36
+ if (!topKey)
37
+ return;
38
+ const { models, apiKey, topN, concurrency } = latest.current;
39
+ // The cache Map identity is stable for the hook's lifetime; capture it so
40
+ // the cleanup closure doesn't read a (lint-flagged) ref in cleanup.
41
+ const cache = cacheRef.current;
42
+ const targets = models
43
+ .slice(0, topN)
44
+ .map((m) => m.uid)
45
+ .filter((uid) => !cache.has(uid));
46
+ if (targets.length === 0)
47
+ return;
48
+ const ac = new AbortController();
49
+ let cancelled = false;
50
+ for (const uid of targets)
51
+ cache.set(uid, { status: 'pending' });
52
+ publish();
53
+ const queue = [...targets];
54
+ const worker = async () => {
55
+ while (!cancelled && queue.length > 0) {
56
+ const uid = queue.shift();
57
+ const cap = await (0, sketchfab_1.fetchSketchfabCapabilities)(uid, apiKey, ac.signal);
58
+ if (cancelled)
59
+ return;
60
+ cache.set(uid, cap ? { status: 'done', cap } : { status: 'unknown' });
61
+ publish();
62
+ }
63
+ };
64
+ const lanes = Math.min(Math.max(1, concurrency), queue.length);
65
+ void Promise.all(Array.from({ length: lanes }, () => worker()));
66
+ return () => {
67
+ cancelled = true;
68
+ ac.abort();
69
+ // Drop entries still pending so a later run (or re-appearance) retries.
70
+ let changed = false;
71
+ for (const uid of targets) {
72
+ if (cache.get(uid)?.status === 'pending') {
73
+ cache.delete(uid);
74
+ changed = true;
75
+ }
76
+ }
77
+ if (changed)
78
+ publish();
79
+ };
80
+ }, [topKey, publish]);
81
+ return snapshot;
82
+ }
@@ -1,9 +1,5 @@
1
- import type { TalkingHeadVisemeSchedule } from "../TalkingHead";
2
- export type VisemeStreamPayload = {
3
- requestId?: string;
4
- durationMs?: number;
5
- cues?: TalkingHeadVisemeSchedule["cues"];
6
- };
1
+ import type { AgentVisemePayload } from "../core/avatar/visemes";
2
+ export type VisemeStreamPayload = AgentVisemePayload;
7
3
  type OpenStreamOptions = {
8
4
  requestId: string;
9
5
  ttsBaseUrl: string;
@@ -1,13 +1,13 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.useDirectVisemeStream = void 0;
3
+ exports.useDirectVisemeStream = useDirectVisemeStream;
4
4
  const react_1 = require("react");
5
- const fetch_1 = require("expo/fetch");
6
5
  // How long to keep retrying a stream before giving up (ms).
7
6
  const STREAM_RETRY_BUDGET_MS = 3000;
8
7
  // Initial retry delay; doubles each attempt up to MAX_RETRY_DELAY_MS.
9
8
  const INITIAL_RETRY_DELAY_MS = 150;
10
9
  const MAX_RETRY_DELAY_MS = 1000;
10
+ const shouldLogDebugMessages = typeof process !== "undefined" && process.env.NODE_ENV !== "production";
11
11
  /**
12
12
  * Opens a direct SSE connection to the TTS server to receive viseme data,
13
13
  * bypassing the agent data channel relay.
@@ -38,12 +38,12 @@ function useDirectVisemeStream(onVisemes) {
38
38
  // Strip trailing /v1 if present so we don't double it
39
39
  const base = ttsBaseUrl.replace(/\/v1\/?$/, "");
40
40
  const url = `${base}/v1/audio/visemes/${encodeURIComponent(requestId)}/stream`;
41
- (async () => {
41
+ void (async () => {
42
42
  const startedAt = Date.now();
43
43
  let retryDelay = INITIAL_RETRY_DELAY_MS;
44
44
  while (!signal.aborted) {
45
45
  try {
46
- const response = await (0, fetch_1.fetch)(url, {
46
+ const response = await fetch(url, {
47
47
  headers: { Accept: "text/event-stream" },
48
48
  signal,
49
49
  });
@@ -67,11 +67,14 @@ function useDirectVisemeStream(onVisemes) {
67
67
  onVisemesRef.current(payload);
68
68
  });
69
69
  // Stream ended cleanly — done.
70
- console.log("[VisemeSSE] stream ended", { requestId });
70
+ if (shouldLogDebugMessages) {
71
+ console.log("[VisemeSSE] stream ended", { requestId });
72
+ }
71
73
  return;
72
74
  }
73
75
  catch (err) {
74
- if (err?.name === "AbortError" || signal.aborted)
76
+ const isAbortError = err instanceof Error && err.name === "AbortError";
77
+ if (isAbortError || signal.aborted)
75
78
  return;
76
79
  const elapsed = Date.now() - startedAt;
77
80
  if (elapsed >= STREAM_RETRY_BUDGET_MS) {
@@ -100,7 +103,6 @@ function useDirectVisemeStream(onVisemes) {
100
103
  }, []);
101
104
  return { openStream };
102
105
  }
103
- exports.useDirectVisemeStream = useDirectVisemeStream;
104
106
  // ─── SSE parser ──────────────────────────────────────────────────────────────
105
107
  /**
106
108
  * Reads an SSE stream to completion, dispatching `event: visemes` messages.
@@ -120,11 +122,13 @@ async function readSseStream(reader, signal, onVisemes) {
120
122
  const jsonText = dataLines.join("\n");
121
123
  try {
122
124
  const payload = JSON.parse(jsonText);
123
- console.log("[VisemeSSE] received", {
124
- requestId: payload.requestId,
125
- cues: Array.isArray(payload.cues) ? payload.cues.length : 0,
126
- durationMs: payload.durationMs ?? null,
127
- });
125
+ if (shouldLogDebugMessages) {
126
+ console.log("[VisemeSSE] received", {
127
+ requestId: payload.requestId,
128
+ cues: Array.isArray(payload.cues) ? payload.cues.length : 0,
129
+ durationMs: payload.durationMs ?? null,
130
+ });
131
+ }
128
132
  onVisemes(payload);
129
133
  }
130
134
  catch (parseErr) {
@@ -1,3 +1,2 @@
1
- /// <reference types="react" />
2
1
  import type { TalkingHeadRef } from '../TalkingHead';
3
2
  export declare function useMotionMarkers(ref: React.RefObject<TalkingHeadRef | null>): (text: string) => string;
@@ -1,6 +1,6 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.useMotionMarkers = void 0;
3
+ exports.useMotionMarkers = useMotionMarkers;
4
4
  const react_1 = require("react");
5
5
  /**
6
6
  * Parses ::marker_name:: tokens out of LLM transcript text, fires the
@@ -22,4 +22,3 @@ function useMotionMarkers(ref) {
22
22
  return '';
23
23
  }), [ref]);
24
24
  }
25
- exports.useMotionMarkers = useMotionMarkers;
@@ -1,26 +1,111 @@
1
1
  "use strict";
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ var desc = Object.getOwnPropertyDescriptor(m, k);
5
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
+ desc = { enumerable: true, get: function() { return m[k]; } };
7
+ }
8
+ Object.defineProperty(o, k2, desc);
9
+ }) : (function(o, m, k, k2) {
10
+ if (k2 === undefined) k2 = k;
11
+ o[k2] = m[k];
12
+ }));
13
+ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
+ Object.defineProperty(o, "default", { enumerable: true, value: v });
15
+ }) : function(o, v) {
16
+ o["default"] = v;
17
+ });
18
+ var __importStar = (this && this.__importStar) || (function () {
19
+ var ownKeys = function(o) {
20
+ ownKeys = Object.getOwnPropertyNames || function (o) {
21
+ var ar = [];
22
+ for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
23
+ return ar;
24
+ };
25
+ return ownKeys(o);
26
+ };
27
+ return function (mod) {
28
+ if (mod && mod.__esModule) return mod;
29
+ var result = {};
30
+ if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
31
+ __setModuleDefault(result, mod);
32
+ return result;
33
+ };
34
+ })();
2
35
  Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.resolveLocalAssetUrl = exports.resolveLocalAssetUri = void 0;
36
+ exports.resolveLocalAssetUri = resolveLocalAssetUri;
37
+ exports.resolveLocalAssetUrl = resolveLocalAssetUrl;
4
38
  const expo_asset_1 = require("expo-asset");
5
39
  const react_native_1 = require("react-native");
6
40
  const expo_file_system_1 = require("expo-file-system");
41
+ const LegacyFileSystem = __importStar(require("expo-file-system/legacy"));
42
+ /**
43
+ * Narrows dynamic require() results to the Expo asset module shapes accepted by Asset.fromModule.
44
+ */
45
+ function isExpoAssetModule(module) {
46
+ if (typeof module === 'number' || typeof module === 'string') {
47
+ return true;
48
+ }
49
+ return (typeof module === 'object' &&
50
+ module !== null &&
51
+ 'uri' in module &&
52
+ typeof module.uri === 'string' &&
53
+ 'width' in module &&
54
+ typeof module.width === 'number' &&
55
+ 'height' in module &&
56
+ typeof module.height === 'number');
57
+ }
58
+ function inferAssetFileName(uri) {
59
+ try {
60
+ const parsed = new URL(uri);
61
+ const unstablePath = parsed.searchParams.get('unstable_path') ?? '';
62
+ const hash = parsed.searchParams.get('hash');
63
+ const sourcePath = unstablePath || parsed.pathname;
64
+ const extMatch = sourcePath.match(/\.([a-z0-9]+)$/i);
65
+ const ext = extMatch?.[1] ?? 'bin';
66
+ return `${hash ?? 'asset'}.${ext}`;
67
+ }
68
+ catch {
69
+ return 'asset.bin';
70
+ }
71
+ }
72
+ async function resolveAssetUriWithFallback(asset) {
73
+ try {
74
+ await asset.downloadAsync();
75
+ return asset.localUri ?? asset.uri ?? null;
76
+ }
77
+ catch (error) {
78
+ const remoteUri = asset.uri;
79
+ if (!remoteUri || (!remoteUri.startsWith('http://') && !remoteUri.startsWith('https://'))) {
80
+ throw error;
81
+ }
82
+ if (!LegacyFileSystem.cacheDirectory) {
83
+ throw error;
84
+ }
85
+ const dir = `${LegacyFileSystem.cacheDirectory}expo-asset-fallback/`;
86
+ const fileUri = `${dir}${inferAssetFileName(remoteUri)}`;
87
+ await LegacyFileSystem.makeDirectoryAsync(dir, { intermediates: true });
88
+ const downloaded = await LegacyFileSystem.downloadAsync(remoteUri, fileUri);
89
+ return downloaded.uri;
90
+ }
91
+ }
7
92
  /**
8
93
  * Resolves a local Expo asset module into a file:// URI for use with
9
94
  * WgpuAvatar model loading.
10
95
  */
11
96
  async function resolveLocalAssetUri(module) {
12
97
  try {
98
+ if (!isExpoAssetModule(module)) {
99
+ throw new TypeError('Expected an Expo asset module id, URI, or asset descriptor.');
100
+ }
13
101
  const asset = expo_asset_1.Asset.fromModule(module);
14
- await asset.downloadAsync();
15
- const uri = asset.localUri ?? asset.uri;
16
- return uri ?? null;
102
+ return await resolveAssetUriWithFallback(asset);
17
103
  }
18
104
  catch (e) {
19
105
  console.error('[AssetUtils] Failed to resolve asset:', e);
20
106
  return null;
21
107
  }
22
108
  }
23
- exports.resolveLocalAssetUri = resolveLocalAssetUri;
24
109
  /**
25
110
  * Resolves a local Expo asset module (from require()) into a usable URL string.
26
111
  * On web, returns an absolute URL.
@@ -29,9 +114,11 @@ exports.resolveLocalAssetUri = resolveLocalAssetUri;
29
114
  */
30
115
  async function resolveLocalAssetUrl(module) {
31
116
  try {
117
+ if (!isExpoAssetModule(module)) {
118
+ throw new TypeError('Expected an Expo asset module id, URI, or asset descriptor.');
119
+ }
32
120
  const asset = expo_asset_1.Asset.fromModule(module);
33
- await asset.downloadAsync();
34
- const uri = asset.localUri ?? asset.uri;
121
+ const uri = await resolveAssetUriWithFallback(asset);
35
122
  if (!uri)
36
123
  return null;
37
124
  if (react_native_1.Platform.OS === 'web') {
@@ -53,4 +140,3 @@ async function resolveLocalAssetUrl(module) {
53
140
  return null;
54
141
  }
55
142
  }
56
- exports.resolveLocalAssetUrl = resolveLocalAssetUrl;
@@ -28,15 +28,26 @@ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (
28
28
  }) : function(o, v) {
29
29
  o["default"] = v;
30
30
  });
31
- var __importStar = (this && this.__importStar) || function (mod) {
32
- if (mod && mod.__esModule) return mod;
33
- var result = {};
34
- if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
35
- __setModuleDefault(result, mod);
36
- return result;
37
- };
31
+ var __importStar = (this && this.__importStar) || (function () {
32
+ var ownKeys = function(o) {
33
+ ownKeys = Object.getOwnPropertyNames || function (o) {
34
+ var ar = [];
35
+ for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
36
+ return ar;
37
+ };
38
+ return ownKeys(o);
39
+ };
40
+ return function (mod) {
41
+ if (mod && mod.__esModule) return mod;
42
+ var result = {};
43
+ if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
44
+ __setModuleDefault(result, mod);
45
+ return result;
46
+ };
47
+ })();
38
48
  Object.defineProperty(exports, "__esModule", { value: true });
39
- exports.prefetchFaceLandmarker = exports.faceLandmarkerToShapeWeights = void 0;
49
+ exports.faceLandmarkerToShapeWeights = faceLandmarkerToShapeWeights;
50
+ exports.prefetchFaceLandmarker = prefetchFaceLandmarker;
40
51
  const MEDIAPIPE_CDN = 'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@latest/wasm';
41
52
  // ---------------------------------------------------------------------------
42
53
  // Singleton — built once, reused across calls
@@ -46,10 +57,8 @@ async function getLandmarker() {
46
57
  if (_landmarkerPromise)
47
58
  return _landmarkerPromise;
48
59
  _landmarkerPromise = (async () => {
49
- // Dynamic import so this module is tree-shaken when unused
50
- // @mediapipe/tasks-vision is loaded at runtime from CDN — not in node_modules.
51
- // eslint-disable-next-line @typescript-eslint/ban-ts-comment
52
- // @ts-ignore
60
+ // Dynamic import keeps the MediaPipe WASM/runtime out of bundles that never
61
+ // call this utility.
53
62
  const vision = await Promise.resolve().then(() => __importStar(require(
54
63
  /* webpackChunkName: "mediapipe-tasks-vision" */
55
64
  '@mediapipe/tasks-vision')));
@@ -99,7 +108,6 @@ async function faceLandmarkerToShapeWeights(source, options = {}) {
99
108
  }
100
109
  return weights;
101
110
  }
102
- exports.faceLandmarkerToShapeWeights = faceLandmarkerToShapeWeights;
103
111
  /**
104
112
  * Load the FaceLandmarker WASM bundle eagerly — call this on app start
105
113
  * to avoid a cold-start delay when the user first uploads a photo.
@@ -109,4 +117,3 @@ function prefetchFaceLandmarker() {
109
117
  // Silently ignore prefetch failures — will retry on actual use
110
118
  });
111
119
  }
112
- exports.prefetchFaceLandmarker = prefetchFaceLandmarker;
@@ -1,6 +1,6 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.convertToWav = void 0;
3
+ exports.convertToWav = convertToWav;
4
4
  /**
5
5
  * Convert any audio blob to WAV format using the Web Audio API.
6
6
  * Handles WebM/opus output from MediaRecorder without requiring ffmpeg.
@@ -19,7 +19,6 @@ async function convertToWav(audioBlob) {
19
19
  await audioContext.close();
20
20
  }
21
21
  }
22
- exports.convertToWav = convertToWav;
23
22
  /**
24
23
  * Encode an AudioBuffer as a 16-bit PCM WAV blob.
25
24
  */
@@ -3,3 +3,6 @@ export type { UseAudioRecordingOptions, UseAudioRecordingReturn } from './useAud
3
3
  export { useAudioPlayer } from './useAudioPlayer';
4
4
  export type { UseAudioPlayerOptions, UseAudioPlayerReturn } from './useAudioPlayer';
5
5
  export { convertToWav } from './convertToWav';
6
+ export { useVoicePreview, configureVoicePreviewBaseUrl } from './useVoicePreview';
7
+ export { useFaceControlsFromVisemes } from './useFaceControls';
8
+ export type { FaceControl } from '../core/avatar/faceControls';
@@ -1,9 +1,14 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.convertToWav = exports.useAudioPlayer = exports.useAudioRecording = void 0;
3
+ exports.useFaceControlsFromVisemes = exports.configureVoicePreviewBaseUrl = exports.useVoicePreview = exports.convertToWav = exports.useAudioPlayer = exports.useAudioRecording = void 0;
4
4
  var useAudioRecording_1 = require("./useAudioRecording");
5
5
  Object.defineProperty(exports, "useAudioRecording", { enumerable: true, get: function () { return useAudioRecording_1.useAudioRecording; } });
6
6
  var useAudioPlayer_1 = require("./useAudioPlayer");
7
7
  Object.defineProperty(exports, "useAudioPlayer", { enumerable: true, get: function () { return useAudioPlayer_1.useAudioPlayer; } });
8
8
  var convertToWav_1 = require("./convertToWav");
9
9
  Object.defineProperty(exports, "convertToWav", { enumerable: true, get: function () { return convertToWav_1.convertToWav; } });
10
+ var useVoicePreview_1 = require("./useVoicePreview");
11
+ Object.defineProperty(exports, "useVoicePreview", { enumerable: true, get: function () { return useVoicePreview_1.useVoicePreview; } });
12
+ Object.defineProperty(exports, "configureVoicePreviewBaseUrl", { enumerable: true, get: function () { return useVoicePreview_1.configureVoicePreviewBaseUrl; } });
13
+ var useFaceControls_1 = require("./useFaceControls");
14
+ Object.defineProperty(exports, "useFaceControlsFromVisemes", { enumerable: true, get: function () { return useFaceControls_1.useFaceControlsFromVisemes; } });
@@ -1,6 +1,6 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.useAudioPlayer = void 0;
3
+ exports.useAudioPlayer = useAudioPlayer;
4
4
  const react_1 = require("react");
5
5
  function useAudioPlayer({ onError, } = {}) {
6
6
  const [isPlaying, setIsPlaying] = (0, react_1.useState)(false);
@@ -14,8 +14,12 @@ function useAudioPlayer({ onError, } = {}) {
14
14
  setIsPlaying(false);
15
15
  }
16
16
  else {
17
- audioRef.current.play();
18
- setIsPlaying(true);
17
+ void audioRef.current.play()
18
+ .then(() => setIsPlaying(true))
19
+ .catch((error) => {
20
+ const err = error instanceof Error ? error : new Error('Failed to play audio file');
21
+ onError?.(err);
22
+ });
19
23
  }
20
24
  }
21
25
  else {
@@ -42,8 +46,17 @@ function useAudioPlayer({ onError, } = {}) {
42
46
  }
43
47
  audioRef.current = null;
44
48
  });
45
- audio.play();
46
- setIsPlaying(true);
49
+ void audio.play()
50
+ .then(() => setIsPlaying(true))
51
+ .catch((error) => {
52
+ const err = error instanceof Error ? error : new Error('Failed to play audio file');
53
+ if (onError) {
54
+ onError(err);
55
+ }
56
+ else {
57
+ console.error('[useAudioPlayer] Playback error:', err.message);
58
+ }
59
+ });
47
60
  }
48
61
  };
49
62
  const cleanup = () => {
@@ -62,4 +75,3 @@ function useAudioPlayer({ onError, } = {}) {
62
75
  cleanup,
63
76
  };
64
77
  }
65
- exports.useAudioPlayer = useAudioPlayer;
@@ -1,6 +1,6 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.useAudioRecording = void 0;
3
+ exports.useAudioRecording = useAudioRecording;
4
4
  const react_1 = require("react");
5
5
  const convertToWav_1 = require("./convertToWav");
6
6
  function useAudioRecording({ maxDurationSeconds = 29, onRecordingComplete, } = {}) {
@@ -163,4 +163,3 @@ function useAudioRecording({ maxDurationSeconds = 29, onRecordingComplete, } = {
163
163
  cancelRecording,
164
164
  };
165
165
  }
166
- exports.useAudioRecording = useAudioRecording;
@@ -0,0 +1,14 @@
1
+ import type { TalkingHeadVisemeSchedule } from '../TalkingHead';
2
+ import { type FaceControl } from '../core/avatar/faceControls';
3
+ /**
4
+ * useFaceControlsFromVisemes
5
+ *
6
+ * Given a TalkingHead viseme schedule (Rhubarb shape cues + startedAtMs),
7
+ * produce a time-varying FaceControl object (neutral head pose + ExpressionState).
8
+ *
9
+ * This hook is intentionally simple: it drives expression purely from the
10
+ * viseme cues and keeps head pose neutral (0 yaw/pitch/roll). Downstream
11
+ * backends can layer motion / camera moves on top.
12
+ */
13
+ export declare function useFaceControlsFromVisemes(schedule: TalkingHeadVisemeSchedule | null): FaceControl;
14
+ export type { FaceControl } from '../core/avatar/faceControls';