ugly-app 0.1.562 → 0.1.563

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (18) hide show
  1. package/dist/cli/version.d.ts +1 -1
  2. package/dist/cli/version.js +1 -1
  3. package/dist/client/audio/AudioPlayer.d.ts +8 -0
  4. package/dist/client/audio/AudioPlayer.d.ts.map +1 -1
  5. package/dist/client/audio/AudioPlayer.js +54 -1
  6. package/dist/client/audio/AudioPlayer.js.map +1 -1
  7. package/dist/client/audio/useTTS.d.ts +18 -0
  8. package/dist/client/audio/useTTS.d.ts.map +1 -1
  9. package/dist/client/audio/useTTS.js +86 -11
  10. package/dist/client/audio/useTTS.js.map +1 -1
  11. package/dist/client/index.d.ts +1 -0
  12. package/dist/client/index.d.ts.map +1 -1
  13. package/dist/client/index.js.map +1 -1
  14. package/package.json +1 -1
  15. package/src/cli/version.ts +1 -1
  16. package/src/client/audio/AudioPlayer.ts +57 -1
  17. package/src/client/audio/useTTS.ts +106 -11
  18. package/src/client/index.ts +1 -0
package/dist/cli/version.d.ts CHANGED
@@ -1,2 +1,2 @@
1
- export declare const CLI_VERSION = "0.1.562";
1
+ export declare const CLI_VERSION = "0.1.563";
2
2
  //# sourceMappingURL=version.d.ts.map
package/dist/cli/version.js CHANGED
@@ -1,3 +1,3 @@
1
1
  // Auto-generated by prebuild — do not edit manually
2
- export const CLI_VERSION = "0.1.562";
2
+ export const CLI_VERSION = "0.1.563";
3
3
  //# sourceMappingURL=version.js.map
package/dist/client/audio/AudioPlayer.d.ts CHANGED
@@ -3,12 +3,20 @@ export declare class AudioPlayer {
3
3
  private workletNode;
4
4
  private analyser;
5
5
  private _playing;
6
+ private playedSamples;
7
+ private _drained;
6
8
  get playing(): boolean;
7
9
  /**
8
10
  * Real-time amplitude analyser tapped off the playback graph. Available
9
11
  * after `init()`. Used to drive talking-avatar lip-sync (jaw open ∝ RMS).
10
12
  */
11
13
  getAnalyser(): AnalyserNode | null;
14
+ /** Milliseconds of audio actually played so far (the lip-sync/caption clock). */
15
+ getPlaybackTimeMs(): number;
16
+ /** True once the queue has drained after playing (stream finished). */
17
+ get drained(): boolean;
18
+ /** Reset the playback clock for a new stream. */
19
+ reset(): void;
12
20
  init(): Promise<void>;
13
21
  /** Resume the AudioContext if the browser autoplay policy suspended it. */
14
22
  resume(): Promise<void>;
package/dist/client/audio/AudioPlayer.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"AudioPlayer.d.ts","sourceRoot":"","sources":["../../../src/client/audio/AudioPlayer.ts"],"names":[],"mappings":"AAsDA,qBAAa,WAAW;IACtB,OAAO,CAAC,GAAG,CAA6B;IACxC,OAAO,CAAC,WAAW,CAAiC;IACpD,OAAO,CAAC,QAAQ,CAA6B;IAC7C,OAAO,CAAC,QAAQ,CAAS;IAEzB,IAAI,OAAO,IAAI,OAAO,CAErB;IAED;;;OAGG;IACH,WAAW,IAAI,YAAY,GAAG,IAAI;IAI5B,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAiB3B,2EAA2E;IACrE,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;IAUvB,SAAS,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAgBnD,IAAI,IAAI,IAAI;IAKN,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAO7B"}
1
+ {"version":3,"file":"AudioPlayer.d.ts","sourceRoot":"","sources":["../../../src/client/audio/AudioPlayer.ts"],"names":[],"mappings":"AA4EA,qBAAa,WAAW;IACtB,OAAO,CAAC,GAAG,CAA6B;IACxC,OAAO,CAAC,WAAW,CAAiC;IACpD,OAAO,CAAC,QAAQ,CAA6B;IAC7C,OAAO,CAAC,QAAQ,CAAS;IAGzB,OAAO,CAAC,aAAa,CAAK;IAC1B,OAAO,CAAC,QAAQ,CAAS;IAEzB,IAAI,OAAO,IAAI,OAAO,CAErB;IAED;;;OAGG;IACH,WAAW,IAAI,YAAY,GAAG,IAAI;IAIlC,iFAAiF;IACjF,iBAAiB,IAAI,MAAM;IAI3B,uEAAuE;IACvE,IAAI,OAAO,IAAI,OAAO,CAErB;IAED,iDAAiD;IACjD,KAAK,IAAI,IAAI;IAOP,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IA4B3B,2EAA2E;IACrE,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;IAUvB,SAAS,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAiBnD,IAAI,IAAI,IAAI;IAKN,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAO7B"}
package/dist/client/audio/AudioPlayer.js CHANGED
@@ -6,12 +6,16 @@ class PCM16Player extends AudioWorkletProcessor {
6
6
  this.queue = [];
7
7
  this.buffer = new Float32Array(0);
8
8
  this.offset = 0;
9
+ this.played = 0; // total real samples emitted (the playback clock)
10
+ this.tick = 0;
11
+ this.draining = false;
9
12
  this.port.onmessage = (event) => this.handleMessage(event);
10
13
  }
11
14
  process(_inputs, outputs) {
12
15
  const output = outputs[0]?.[0];
13
16
  if (!output) return true;
14
17
  let writePos = 0;
18
+ let real = 0;
15
19
  while (writePos < output.length) {
16
20
  if (this.offset >= this.buffer.length) {
17
21
  const next = this.queue.shift();
@@ -25,6 +29,19 @@ class PCM16Player extends AudioWorkletProcessor {
25
29
  output.set(this.buffer.subarray(this.offset, this.offset + toCopy), writePos);
26
30
  this.offset += toCopy;
27
31
  writePos += toCopy;
32
+ real += toCopy;
33
+ }
34
+ if (real > 0) {
35
+ this.played += real;
36
+ this.draining = true;
37
+ // Report the playback clock ~40×/s so the main thread can schedule
38
+ // visemes/captions against actual audio position.
39
+ if (++this.tick % 6 === 0) this.port.postMessage({ type: 'time', played: this.played });
40
+ } else if (this.draining) {
41
+ // Queue ran dry after playing — playback finished.
42
+ this.draining = false;
43
+ this.port.postMessage({ type: 'time', played: this.played });
44
+ this.port.postMessage({ type: 'drained' });
28
45
  }
29
46
  return true;
30
47
  }
@@ -35,11 +52,15 @@ class PCM16Player extends AudioWorkletProcessor {
35
52
  this.queue = [];
36
53
  this.buffer = new Float32Array(0);
37
54
  this.offset = 0;
55
+ this.played = 0;
56
+ this.draining = false;
38
57
  }
39
58
  }
40
59
  }
41
60
  registerProcessor('pcm16-player', PCM16Player);
42
61
  `;
62
+ /** TTS audio is always mono PCM16 @ 24 kHz (see TTSChunkMsg). */
63
+ const TTS_SAMPLE_RATE = 24000;
43
64
  const WORKLET_BLOB = new Blob([WORKLET_CODE], { type: 'application/javascript' });
44
65
  const WORKLET_URL = URL.createObjectURL(WORKLET_BLOB);
45
66
  function pcm16ToFloat32(buffer) {
@@ -55,6 +76,10 @@ export class AudioPlayer {
55
76
  workletNode = null;
56
77
  analyser = null;
57
78
  _playing = false;
79
+ // Playback clock fed by the worklet (samples actually emitted). Used to
80
+ // schedule visemes + caption indices against real audio position.
81
+ playedSamples = 0;
82
+ _drained = false;
58
83
  get playing() {
59
84
  return this._playing;
60
85
  }
@@ -65,12 +90,39 @@ export class AudioPlayer {
65
90
  getAnalyser() {
66
91
  return this.analyser;
67
92
  }
93
+ /** Milliseconds of audio actually played so far (the lip-sync/caption clock). */
94
+ getPlaybackTimeMs() {
95
+ return (this.playedSamples / TTS_SAMPLE_RATE) * 1000;
96
+ }
97
+ /** True once the queue has drained after playing (stream finished). */
98
+ get drained() {
99
+ return this._drained;
100
+ }
101
+ /** Reset the playback clock for a new stream. */
102
+ reset() {
103
+ this.workletNode?.port.postMessage({ type: 'stop' });
104
+ this.playedSamples = 0;
105
+ this._drained = false;
106
+ this._playing = false;
107
+ }
68
108
  async init() {
69
109
  if (this.workletNode)
70
110
  return;
71
- this.ctx = new AudioContext();
111
+ // Match the TTS sample rate so PCM16 plays at correct pitch (the worklet
112
+ // outputs 1:1, no resampling) and the playback clock is accurate.
113
+ this.ctx = new AudioContext({ sampleRate: TTS_SAMPLE_RATE });
72
114
  await this.ctx.audioWorklet.addModule(WORKLET_URL);
73
115
  this.workletNode = new AudioWorkletNode(this.ctx, 'pcm16-player');
116
+ this.workletNode.port.onmessage = (e) => {
117
+ const d = e.data;
118
+ if (d?.type === 'time' && typeof d.played === 'number') {
119
+ this.playedSamples = d.played;
120
+ }
121
+ else if (d?.type === 'drained') {
122
+ this._drained = true;
123
+ this._playing = false;
124
+ }
125
+ };
74
126
  this.analyser = this.ctx.createAnalyser();
75
127
  this.analyser.fftSize = 256;
76
128
  this.analyser.smoothingTimeConstant = 0.6;
@@ -105,6 +157,7 @@ export class AudioPlayer {
105
157
  bytes[i] = binary.charCodeAt(i);
106
158
  const samples = pcm16ToFloat32(bytes.buffer);
107
159
  this._playing = true;
160
+ this._drained = false;
108
161
  this.workletNode.port.postMessage({ type: 'chunk', samples }, [
109
162
  samples.buffer,
110
163
  ]);
package/dist/client/audio/AudioPlayer.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"AudioPlayer.js","sourceRoot":"","sources":["../../../src/client/audio/AudioPlayer.ts"],"names":[],"mappings":"AAAA,kCAAkC;AAClC,MAAM,YAAY,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAwCpB,CAAC;AACF,MAAM,YAAY,GAAG,IAAI,IAAI,CAAC,CAAC,YAAY,CAAC,EAAE,EAAE,IAAI,EAAE,wBAAwB,EAAE,CAAC,CAAC;AAClF,MAAM,WAAW,GAAG,GAAG,CAAC,eAAe,CAAC,YAAY,CAAC,CAAC;AAEtD,SAAS,cAAc,CAAC,MAAmB;IACzC,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC,MAAM,CAAC,CAAC;IACrC,MAAM,OAAO,GAAG,IAAI,YAAY,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAC/C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,KAAK,CAAC;IACvC,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,OAAO,WAAW;IACd,GAAG,GAAwB,IAAI,CAAC;IAChC,WAAW,GAA4B,IAAI,CAAC;IAC5C,QAAQ,GAAwB,IAAI,CAAC;IACrC,QAAQ,GAAG,KAAK,CAAC;IAEzB,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC;IAED;;;OAGG;IACH,WAAW;QACT,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC;IAED,KAAK,CAAC,IAAI;QACR,IAAI,IAAI,CAAC,WAAW;YAAE,OAAO;QAC7B,IAAI,CAAC,GAAG,GAAG,IAAI,YAAY,EAAE,CAAC;QAC9B,MAAM,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;QACnD,IAAI,CAAC,WAAW,GAAG,IAAI,gBAAgB,CAAC,IAAI,CAAC,GAAG,EAAE,cAAc,CAAC,CAAC;QAClE,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,cAAc,EAAE,CAAC;QAC1C,IAAI,CAAC,QAAQ,CAAC,OAAO,GAAG,GAAG,CAAC;QAC5B,IAAI,CAAC,QAAQ,CAAC,qBAAqB,GAAG,GAAG,CAAC;QAC1C,qEAAqE;QACrE,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACxC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QAC5C,0EAA0E;QAC1E,qEAAqE;QACrE,oDAAoD;QACpD,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC;IACtB,CAAC;IAED,2EAA2E;IAC3E,KAAK,CAAC,MAAM;QACV,IAAI,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,KAAK,KAAK,WAAW,EAAE,CAAC;YAC/C,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC;YAC1B,CAAC;YAAC,MAAM,CAAC;gBACP,kCAAkC;YACpC,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,WAAmB;QACjC,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAClB,IAAI,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC,IAAI,CAAC,GAAG;YAAE,OAAO;QAC3C,IAAI,IAAI,CAAC,GAAG,CAAC,KAAK,KAAK,WAAW;YAAE,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC;QAExD,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC;QACjC,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAC5C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE;YAAE,KAAK,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC,CAAE,CAAC;QAEzE,MAAM,OAAO,GAAG,cAAc,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAC7C,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACrB,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,EAAE;YAC5D,OAAO,CAAC,MAAM;SACf,CAAC,CAAC;IACL,CAAC;IAED,IAAI;QACF,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;QACrD,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;IACxB,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,IAAI,EAAE,CAAC;QACZ,MAAM,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,CAAC;QACxB,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC;QAChB,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QACxB,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;IACvB,CAAC;CACF"}
1
+ {"version":3,"file":"AudioPlayer.js","sourceRoot":"","sources":["../../../src/client/audio/AudioPlayer.ts"],"names":[],"mappings":"AAAA,kCAAkC;AAClC,MAAM,YAAY,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA2DpB,CAAC;AAEF,iEAAiE;AACjE,MAAM,eAAe,GAAG,KAAK,CAAC;AAC9B,MAAM,YAAY,GAAG,IAAI,IAAI,CAAC,CAAC,YAAY,CAAC,EAAE,EAAE,IAAI,EAAE,wBAAwB,EAAE,CAAC,CAAC;AAClF,MAAM,WAAW,GAAG,GAAG,CAAC,eAAe,CAAC,YAAY,CAAC,CAAC;AAEtD,SAAS,cAAc,CAAC,MAAmB;IACzC,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC,MAAM,CAAC,CAAC;IACrC,MAAM,OAAO,GAAG,IAAI,YAAY,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAC/C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,KAAK,CAAC;IACvC,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,OAAO,WAAW;IACd,GAAG,GAAwB,IAAI,CAAC;IAChC,WAAW,GAA4B,IAAI,CAAC;IAC5C,QAAQ,GAAwB,IAAI,CAAC;IACrC,QAAQ,GAAG,KAAK,CAAC;IACzB,wEAAwE;IACxE,kEAAkE;IAC1D,aAAa,GAAG,CAAC,CAAC;IAClB,QAAQ,GAAG,KAAK,CAAC;IAEzB,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC;IAED;;;OAGG;IACH,WAAW;QACT,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC;IAED,iFAAiF;IACjF,iBAAiB;QACf,OAAO,CAAC,IAAI,CAAC,aAAa,GAAG,eAAe,CAAC,GAAG,IAAI,CAAC;IACvD,CAAC;IAED,uEAAuE;IACvE,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC;IAED,iDAAiD;IACjD,KAAK;QACH,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;QACrD,IAAI,CAAC,aAAa,GAAG,CAAC,CAAC;QACvB,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;QACtB,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;IACxB,CAAC;IAED,KAAK,CAAC,IAAI;QACR,IAAI,IAAI,CAAC,WAAW;YAAE,OAAO;QAC7B,yEAAyE;QACzE,kEAAkE;QAClE,IAAI,CAAC,GAAG,GAAG,IAAI,YAAY,CAAC,EAAE,UAAU,EAAE,eAAe,EAAE,CAAC,CAAC;QAC7D,MAAM,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;QACnD,IAAI,CAAC,WAAW,GAAG,IAAI,gBAAgB,CAAC,IAAI,CAAC,GAAG,EAAE,cAAc,CAAC,CAAC;QAClE,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,SAAS,GAAG,CAAC,CAAe,EAAE,EAAE;YACpD,MAAM,CAAC,GAAG,CAAC,CAAC,IAA0C,CAAC;YACvD,IAAI,CAAC,EAAE,IAAI,KAAK,MAAM,IAAI,OAAO,CAAC,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;gBACvD,IAAI,CAAC,aAAa,GAAG,CAAC,CAAC,MAAM,CAAC;YAChC,CAAC;iBAAM,IAAI,CAAC,EAAE,IAAI,KAAK,SAAS,EAAE,CAAC;gBACjC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;gBACrB,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;YACxB,CAAC;QACH,CAAC,CAAC;QACF,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,cAAc,EAAE,CAAC;QAC1C,IAAI,CAAC,QAAQ,CAAC,OAAO,GAAG,GAAG,CAAC;QAC5B,IAAI,CAAC,QAAQ,CAAC,qBAAqB,GAAG,GAAG,CAAC;QAC1C,qEAAqE;QACrE,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACxC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QAC5C,0EAA0E;QAC1E,qEAAqE;QACrE,oDAAoD;QACpD,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC;IACtB,CAAC;IAED,2EAA2E;IAC3E,KAAK,CAAC,MAAM;QACV,IAAI,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,KAAK,KAAK,WAAW,EAAE,CAAC;YAC/C,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC;YAC1B,CAAC;YAAC,MAAM,CAAC;gBACP,kCAAkC;YACpC,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,WAAmB;QACjC,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAClB,IAAI,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC,IAAI,CAAC,GAAG;YAAE,OAAO;QAC3C,IAAI,IAAI,CAAC,GAAG,CAAC,KAAK,KAAK,WAAW;YAAE,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC;QAExD,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC;QACjC,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAC5C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE;YAAE,KAAK,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC,CAAE,CAAC;QAEzE,MAAM,OAAO,GAAG,cAAc,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAC7C,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACrB,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;QACtB,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,EAAE;YAC5D,OAAO,CAAC,MAAM;SACf,CAAC,CAAC;IACL,CAAC;IAED,IAAI;QACF,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;QACrD,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;IACxB,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,IAAI,EAAE,CAAC;QACZ,MAAM,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,CAAC;QACxB,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC;QAChB,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QACxB,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;IACvB,CAAC;CACF"}
package/dist/client/audio/useTTS.d.ts CHANGED
@@ -1,6 +1,24 @@
1
+ import type { TTSVisemeName } from '../../shared/Audio.js';
1
2
  import type { UglyBotSocket } from '../uglyBotSocket.js';
3
+ /** A viseme delivered to the avatar (timing stripped — driven by the audio clock). */
4
+ export interface TTSVisemeEvent {
5
+ name: TTSVisemeName;
6
+ durationMs: number;
7
+ intensity: number;
8
+ }
2
9
  interface TTSOptions {
3
10
  voice?: string;
11
+ /**
12
+ * Fired as each viseme reaches the current audio-playback position. Wire to
13
+ * `AvatarModel.showViseme(...)` for true InWorld lip-sync.
14
+ */
15
+ onViseme?: (v: TTSVisemeEvent) => void;
16
+ /**
17
+ * Fired as each spoken word reaches the playback position — value is the
18
+ * character offset of the word's end in the original text. Reveal the caption
19
+ * up to this index for karaoke-style subtitles.
20
+ */
21
+ onSubtitleIndex?: (charIndex: number) => void;
4
22
  }
5
23
  export declare function useTTS(socket: UglyBotSocket): {
6
24
  play: (text: string, options?: TTSOptions) => Promise<void>;
package/dist/client/audio/useTTS.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"useTTS.d.ts","sourceRoot":"","sources":["../../../src/client/audio/useTTS.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAGzD,UAAU,UAAU;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAID,wBAAgB,MAAM,CAAC,MAAM,EAAE,aAAa;iBA4B3B,MAAM,YAAY,UAAU;;;;;;;EAuE5C"}
1
+ {"version":3,"file":"useTTS.d.ts","sourceRoot":"","sources":["../../../src/client/audio/useTTS.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAa,aAAa,EAAE,MAAM,uBAAuB,CAAC;AACtE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAGzD,sFAAsF;AACtF,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,aAAa,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,UAAU,UAAU;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,QAAQ,CAAC,EAAE,CAAC,CAAC,EAAE,cAAc,KAAK,IAAI,CAAC;IACvC;;;;OAIG;IACH,eAAe,CAAC,EAAE,CAAC,SAAS,EAAE,MAAM,KAAK,IAAI,CAAC;CAC/C;AAID,wBAAgB,MAAM,CAAC,MAAM,EAAE,aAAa;iBAqC3B,MAAM,YAAY,UAAU;;;;;;;EA0I5C"}
package/dist/client/audio/useTTS.js CHANGED
@@ -6,16 +6,24 @@ export function useTTS(socket) {
6
6
  const [playing, setPlaying] = useState(false);
7
7
  const [currentWord, setCurrentWord] = useState('');
8
8
  const [error, setError] = useState(null);
9
- // Real-time amplitude analyser of the TTS playback — drives avatar lip-sync.
9
+ // Real-time amplitude analyser of the TTS playback — a fallback lip-sync
10
+ // signal when viseme data isn't requested/available.
10
11
  const [analyser, setAnalyser] = useState(null);
11
12
  const playerRef = useRef(null);
12
13
  const streamIdRef = useRef(null);
13
14
  const unsubsRef = useRef([]);
15
+ const rafRef = useRef(null);
14
16
  function cleanupListeners() {
15
17
  for (const unsub of unsubsRef.current)
16
18
  unsub();
17
19
  unsubsRef.current = [];
18
20
  }
21
+ function stopRaf() {
22
+ if (rafRef.current !== null) {
23
+ cancelAnimationFrame(rafRef.current);
24
+ rafRef.current = null;
25
+ }
26
+ }
19
27
  // Create + resume the AudioContext from inside a user gesture (e.g. the
20
28
  // user's send-click). Browsers (esp. Safari) only unlock audio when resume()
21
29
  // runs within a gesture; the TTS reply itself plays later, async, so without
@@ -32,46 +40,112 @@ export function useTTS(socket) {
32
40
  streamIdRef.current = streamId;
33
41
  if (!playerRef.current)
34
42
  playerRef.current = new AudioPlayer();
35
- await playerRef.current.init();
36
- setAnalyser(playerRef.current.getAnalyser());
43
+ const player = playerRef.current;
44
+ await player.init();
45
+ player.reset();
46
+ setAnalyser(player.getAnalyser());
37
47
  setPlaying(true);
38
48
  setCurrentWord('');
39
49
  setError(null);
50
+ // Collect viseme/word timing as it streams; an rAF loop dispatches each
51
+ // against the real audio-playback clock (collect-then-schedule).
52
+ const visemes = [];
53
+ const words = [];
54
+ let vi = 0;
55
+ let wi = 0;
56
+ let serverDone = false;
57
+ let lastEndMs = 0;
58
+ const wantViseme = !!options?.onViseme;
59
+ const finish = () => {
60
+ if (streamIdRef.current !== streamId)
61
+ return;
62
+ streamIdRef.current = null;
63
+ stopRaf();
64
+ // Flush any remaining caption indices so the text fully reveals.
65
+ while (wi < words.length)
66
+ options?.onSubtitleIndex?.(words[wi++].charEndIndex);
67
+ setPlaying(false);
68
+ setCurrentWord('');
69
+ cleanupListeners();
70
+ socket.release();
71
+ };
72
+ const tick = () => {
73
+ if (streamIdRef.current !== streamId)
74
+ return;
75
+ const t = player.getPlaybackTimeMs();
76
+ while (vi < visemes.length && visemes[vi].startMs <= t) {
77
+ const v = visemes[vi++];
78
+ options?.onViseme?.({
79
+ name: v.name,
80
+ durationMs: v.durationMs,
81
+ intensity: v.intensity,
82
+ });
83
+ }
84
+ while (wi < words.length && words[wi].startMs <= t) {
85
+ options?.onSubtitleIndex?.(words[wi++].charEndIndex);
86
+ }
87
+ // Done when the server finished, the queue drained, and everything
88
+ // scheduled has been dispatched (or playback passed the last event).
89
+ const dispatched = vi >= visemes.length && wi >= words.length;
90
+ if (serverDone && (player.drained || t >= lastEndMs) && dispatched) {
91
+ finish();
92
+ return;
93
+ }
94
+ rafRef.current = requestAnimationFrame(tick);
95
+ };
40
96
  await socket.acquire();
41
97
  cleanupListeners();
98
+ stopRaf();
42
99
  unsubsRef.current.push(socket.on('tts:chunk', (data) => {
43
100
  if (data.streamId !== streamId)
44
101
  return;
45
- void playerRef.current?.feedChunk(data.audio);
102
+ void player.feedChunk(data.audio);
103
+ if (data.visemes) {
104
+ for (const v of data.visemes) {
105
+ visemes.push(v);
106
+ lastEndMs = Math.max(lastEndMs, v.startMs + v.durationMs);
107
+ }
108
+ }
109
+ }), socket.on('tts:viseme', (data) => {
110
+ if (data.streamId !== streamId)
111
+ return;
112
+ for (const v of data.visemes) {
113
+ visemes.push(v);
114
+ lastEndMs = Math.max(lastEndMs, v.startMs + v.durationMs);
115
+ }
46
116
  }), socket.on('tts:word', (data) => {
47
117
  if (data.streamId !== streamId)
48
118
  return;
49
119
  setCurrentWord(data.word);
120
+ const ce = data.charEndIndex;
121
+ if (typeof ce === 'number') {
122
+ words.push({ charEndIndex: ce, startMs: data.startMs });
123
+ lastEndMs = Math.max(lastEndMs, data.startMs + data.durationMs);
124
+ }
50
125
  }), socket.on('tts:done', (data) => {
51
126
  if (data.streamId !== streamId)
52
127
  return;
53
- setPlaying(false);
54
- setCurrentWord('');
55
- cleanupListeners();
56
- socket.release();
128
+ serverDone = true;
57
129
  }), socket.on('tts:error', (data) => {
58
130
  if (data.streamId !== streamId)
59
131
  return;
60
- setPlaying(false);
61
132
  setError(data.message);
62
- cleanupListeners();
63
- socket.release();
133
+ finish();
64
134
  }));
135
+ rafRef.current = requestAnimationFrame(tick);
65
136
  socket.send('tts:start', {
66
137
  streamId,
67
138
  text,
68
139
  voice: options?.voice,
140
+ requestVisemes: wantViseme,
69
141
  });
70
142
  }, [socket]);
71
143
  const stop = useCallback(() => {
72
144
  if (streamIdRef.current) {
73
145
  socket.send('tts:cancel', { streamId: streamIdRef.current });
74
146
  }
147
+ streamIdRef.current = null;
148
+ stopRaf();
75
149
  playerRef.current?.stop();
76
150
  setPlaying(false);
77
151
  cleanupListeners();
@@ -82,6 +156,7 @@ export function useTTS(socket) {
82
156
  if (streamIdRef.current) {
83
157
  socket.send('tts:cancel', { streamId: streamIdRef.current });
84
158
  }
159
+ stopRaf();
85
160
  cleanupListeners();
86
161
  socket.release();
87
162
  playerRef.current?.close().catch(() => { });
package/dist/client/audio/useTTS.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"useTTS.js","sourceRoot":"","sources":["../../../src/client/audio/useTTS.ts"],"names":[],"mappings":"AAAA,6BAA6B;AAC7B,OAAO,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,OAAO,CAAC;AAEjE,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAM/C,IAAI,aAAa,GAAG,CAAC,CAAC;AAEtB,MAAM,UAAU,MAAM,CAAC,MAAqB;IAC1C,MAAM,CAAC,OAAO,EAAE,UAAU,CAAC,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;IAC9C,MAAM,CAAC,WAAW,EAAE,cAAc,CAAC,GAAG,QAAQ,CAAC,EAAE,CAAC,CAAC;IACnD,MAAM,CAAC,KAAK,EAAE,QAAQ,CAAC,GAAG,QAAQ,CAAgB,IAAI,CAAC,CAAC;IACxD,6EAA6E;IAC7E,MAAM,CAAC,QAAQ,EAAE,WAAW,CAAC,GAAG,QAAQ,CAAsB,IAAI,CAAC,CAAC;IAEpE,MAAM,SAAS,GAAG,MAAM,CAAqB,IAAI,CAAC,CAAC;IACnD,MAAM,WAAW,GAAG,MAAM,CAAgB,IAAI,CAAC,CAAC;IAChD,MAAM,SAAS,GAAG,MAAM,CAAiB,EAAE,CAAC,CAAC;IAE7C,SAAS,gBAAgB;QACvB,KAAK,MAAM,KAAK,IAAI,SAAS,CAAC,OAAO;YAAE,KAAK,EAAE,CAAC;QAC/C,SAAS,CAAC,OAAO,GAAG,EAAE,CAAC;IACzB,CAAC;IAED,wEAAwE;IACxE,6EAA6E;IAC7E,6EAA6E;IAC7E,gEAAgE;IAChE,MAAM,MAAM,GAAG,WAAW,CAAC,KAAK,IAAI,EAAE;QACpC,IAAI,CAAC,SAAS,CAAC,OAAO;YAAE,SAAS,CAAC,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC;QAC9D,MAAM,SAAS,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QAC/B,MAAM,SAAS,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;QACjC,WAAW,CAAC,SAAS,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;IAC/C,CAAC,EAAE,EAAE,CAAC,CAAC;IAEP,MAAM,IAAI,GAAG,WAAW,CACtB,KAAK,EAAE,IAAY,EAAE,OAAoB,EAAE,EAAE;QAC3C,MAAM,QAAQ,GAAG,OAAO,EAAE,aAAa,EAAE,CAAC;QAC1C,WAAW,CAAC,OAAO,GAAG,QAAQ,CAAC;QAE/B,IAAI,CAAC,SAAS,CAAC,OAAO;YAAE,SAAS,CAAC,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC;QAC9D,MAAM,SAAS,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QAC/B,WAAW,CAAC,SAAS,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;QAE7C,UAAU,CAAC,IAAI,CAAC,CAAC;QACjB,cAAc,CAAC,EAAE,CAAC,CAAC;QACnB,QAAQ,CAAC,IAAI,CAAC,CAAC;QAEf,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;QAEvB,gBAAgB,EAAE,CAAC;QACnB,SAAS,CAAC,OAAO,CAAC,IAAI,CACpB,MAAM,CAAC,EAAE,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,EAAE;YAC9B,IAAI,IAAI,CAAC,QAAQ,KAAK,QAAQ;gBAAE,OAAO;YACvC,KAAK,SAAS,CAAC,OAAO,EAAE,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAChD,CAAC,CAAC,EACF,MAAM,CAAC,EAAE,CAAC,UAAU,EAAE,CAAC,IAAI,EAAE,EAAE;YAC7B,IAAI,IAAI,CAAC,QAAQ,KAAK,QAAQ;gBAAE,OAAO;YACvC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC5B,CAAC,CAAC,EACF,MAAM,CAAC,EAAE,CAAC,UAAU,EAAE,CAAC,IAAI,EAAE,EAAE;YAC7B,IAAI,IAAI,CAAC,QAAQ,KAAK,QAAQ;gBAAE,OAAO;YACvC,UAAU,CAAC,KAAK,CAAC,CAAC;YAClB,cAAc,CAAC,EAAE,CAAC,CAAC;YACnB,gBAAgB,EAAE,CAAC;YACnB,MAAM,CAAC,OAAO,EAAE,CAAC;QACnB,CAAC,CAAC,EACF,MAAM,CAAC,EAAE,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,EAAE;YAC9B,IAAI,IAAI,CAAC,QAAQ,KAAK,QAAQ;gBAAE,OAAO;YACvC,UAAU,CAAC,KAAK,CAAC,CAAC;YAClB,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACvB,gBAAgB,EAAE,CAAC;YACnB,MAAM,CAAC,OAAO,EAAE,CAAC;QACnB,CAAC,CAAC,CACH,CAAC;QAEF,MAAM,CAAC,IAAI,CAAC,WAAW,EAAE;YACvB,QAAQ;YACR,IAAI;YACJ,KAAK,EAAE,OAAO,EAAE,KAAK;SACtB,CAAC,CAAC;IACL,CAAC,EACD,CAAC,MAAM,CAAC,CACT,CAAC;IAEF,MAAM,IAAI,GAAG,WAAW,CAAC,GAAG,EAAE;QAC5B,IAAI,WAAW,CAAC,OAAO,EAAE,CAAC;YACxB,MAAM,CAAC,IAAI,CAAC,YAAY,EAAE,EAAE,QAAQ,EAAE,WAAW,CAAC,OAAO,EAAE,CAAC,CAAC;QAC/D,CAAC;QACD,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC;QAC1B,UAAU,CAAC,KAAK,CAAC,CAAC;QAClB,gBAAgB,EAAE,CAAC;QACnB,MAAM,CAAC,OAAO,EAAE,CAAC;IACnB,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC;IAEb,SAAS,CAAC,GAAG,EAAE;QACb,OAAO,GAAG,EAAE;YACV,IAAI,WAAW,CAAC,OAAO,EAAE,CAAC;gBACxB,MAAM,CAAC,IAAI,CAAC,YAAY,EAAE,EAAE,QAAQ,EAAE,WAAW,CAAC,OAAO,EAAE,CAAC,CAAC;YAC/D,CAAC;YACD,gBAAgB,EAAE,CAAC;YACnB,MAAM,CAAC,OAAO,EAAE,CAAC;YACjB,SAAS,CAAC,OAAO,EAAE,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAC7C,CAAC,CAAC;IACJ,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC;IAEb,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;AACvE,CAAC"}
1
+ {"version":3,"file":"useTTS.js","sourceRoot":"","sources":["../../../src/client/audio/useTTS.ts"],"names":[],"mappings":"AAAA,6BAA6B;AAC7B,OAAO,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,OAAO,CAAC;AAGjE,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAwB/C,IAAI,aAAa,GAAG,CAAC,CAAC;AAEtB,MAAM,UAAU,MAAM,CAAC,MAAqB;IAC1C,MAAM,CAAC,OAAO,EAAE,UAAU,CAAC,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;IAC9C,MAAM,CAAC,WAAW,EAAE,cAAc,CAAC,GAAG,QAAQ,CAAC,EAAE,CAAC,CAAC;IACnD,MAAM,CAAC,KAAK,EAAE,QAAQ,CAAC,GAAG,QAAQ,CAAgB,IAAI,CAAC,CAAC;IACxD,yEAAyE;IACzE,qDAAqD;IACrD,MAAM,CAAC,QAAQ,EAAE,WAAW,CAAC,GAAG,QAAQ,CAAsB,IAAI,CAAC,CAAC;IAEpE,MAAM,SAAS,GAAG,MAAM,CAAqB,IAAI,CAAC,CAAC;IACnD,MAAM,WAAW,GAAG,MAAM,CAAgB,IAAI,CAAC,CAAC;IAChD,MAAM,SAAS,GAAG,MAAM,CAAiB,EAAE,CAAC,CAAC;IAC7C,MAAM,MAAM,GAAG,MAAM,CAAgB,IAAI,CAAC,CAAC;IAE3C,SAAS,gBAAgB;QACvB,KAAK,MAAM,KAAK,IAAI,SAAS,CAAC,OAAO;YAAE,KAAK,EAAE,CAAC;QAC/C,SAAS,CAAC,OAAO,GAAG,EAAE,CAAC;IACzB,CAAC;IAED,SAAS,OAAO;QACd,IAAI,MAAM,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;YAC5B,oBAAoB,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YACrC,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC;QACxB,CAAC;IACH,CAAC;IAED,wEAAwE;IACxE,6EAA6E;IAC7E,6EAA6E;IAC7E,gEAAgE;IAChE,MAAM,MAAM,GAAG,WAAW,CAAC,KAAK,IAAI,EAAE;QACpC,IAAI,CAAC,SAAS,CAAC,OAAO;YAAE,SAAS,CAAC,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC;QAC9D,MAAM,SAAS,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QAC/B,MAAM,SAAS,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;QACjC,WAAW,CAAC,SAAS,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;IAC/C,CAAC,EAAE,EAAE,CAAC,CAAC;IAEP,MAAM,IAAI,GAAG,WAAW,CACtB,KAAK,EAAE,IAAY,EAAE,OAAoB,EAAE,EAAE;QAC3C,MAAM,QAAQ,GAAG,OAAO,EAAE,aAAa,EAAE,CAAC;QAC1C,WAAW,CAAC,OAAO,GAAG,QAAQ,CAAC;QAE/B,IAAI,CAAC,SAAS,CAAC,OAAO;YAAE,SAAS,CAAC,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC;QAC9D,MAAM,MAAM,GAAG,SAAS,CAAC,OAAO,CAAC;QACjC,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;QACpB,MAAM,CAAC,KAAK,EAAE,CAAC;QACf,WAAW,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC;QAElC,UAAU,CAAC,IAAI,CAAC,CAAC;QACjB,cAAc,CAAC,EAAE,CAAC,CAAC;QACnB,QAAQ,CAAC,IAAI,CAAC,CAAC;QAEf,wEAAwE;QACxE,iEAAiE;QACjE,MAAM,OAAO,GAAgB,EAAE,CAAC;QAChC,MAAM,KAAK,GAAgD,EAAE,CAAC;QAC9D,IAAI,EAAE,GAAG,CAAC,CAAC;QACX,IAAI,EAAE,GAAG,CAAC,CAAC;QACX,IAAI,UAAU,GAAG,KAAK,CAAC;QACvB,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,MAAM,UAAU,GAAG,CAAC,CAAC,OAAO,EAAE,QAAQ,CAAC;QAEvC,MAAM,MAAM,GAAG,GAAG,EAAE;YAClB,IAAI,WAAW,CAAC,OAAO,KAAK,QAAQ;gBAAE,OAAO;YAC7C,WAAW,CAAC,OAAO,GAAG,IAAI,CAAC;YAC3B,OAAO,EAAE,CAAC;YACV,iEAAiE;YACjE,OAAO,EAAE,GAAG,KAAK,CAAC,MAAM;gBAAE,OAAO,EAAE,eAAe,EAAE,CAAC,KAAK,CAAC,EAAE,EAAE,CAAE,CAAC,YAAY,CAAC,CAAC;YAChF,UAAU,CAAC,KAAK,CAAC,CAAC;YAClB,cAAc,CAAC,EAAE,CAAC,CAAC;YACnB,gBAAgB,EAAE,CAAC;YACnB,MAAM,CAAC,OAAO,EAAE,CAAC;QACnB,CAAC,CAAC;QAEF,MAAM,IAAI,GAAG,GAAG,EAAE;YAChB,IAAI,WAAW,CAAC,OAAO,KAAK,QAAQ;gBAAE,OAAO;YAC7C,MAAM,CAAC,GAAG,MAAM,CAAC,iBAAiB,EAAE,CAAC;YACrC,OAAO,EAAE,GAAG,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,EAAE,CAAE,CAAC,OAAO,IAAI,CAAC,EAAE,CAAC;gBACxD,MAAM,CAAC,GAAG,OAAO,CAAC,EAAE,EAAE,CAAE,CAAC;gBACzB,OAAO,EAAE,QAAQ,EAAE,CAAC;oBAClB,IAAI,EAAE,CAAC,CAAC,IAAI;oBACZ,UAAU,EAAE,CAAC,CAAC,UAAU;oBACxB,SAAS,EAAE,CAAC,CAAC,SAAS;iBACvB,CAAC,CAAC;YACL,CAAC;YACD,OAAO,EAAE,GAAG,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,EAAE,CAAE,CAAC,OAAO,IAAI,CAAC,EAAE,CAAC;gBACpD,OAAO,EAAE,eAAe,EAAE,CAAC,KAAK,CAAC,EAAE,EAAE,CAAE,CAAC,YAAY,CAAC,CAAC;YACxD,CAAC;YACD,mEAAmE;YACnE,qEAAqE;YACrE,MAAM,UAAU,GAAG,EAAE,IAAI,OAAO,CAAC,MAAM,IAAI,EAAE,IAAI,KAAK,CAAC,MAAM,CAAC;YAC9D,IAAI,UAAU,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,CAAC,IAAI,SAAS,CAAC,IAAI,UAAU,EAAE,CAAC;gBACnE,MAAM,EAAE,CAAC;gBACT,OAAO;YACT,CAAC;YACD,MAAM,CAAC,OAAO,GAAG,qBAAqB,CAAC,IAAI,CAAC,CAAC;QAC/C,CAAC,CAAC;QAEF,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;QAEvB,gBAAgB,EAAE,CAAC;QACnB,OAAO,EAAE,CAAC;QACV,SAAS,CAAC,OAAO,CAAC,IAAI,CACpB,MAAM,CAAC,EAAE,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,EAAE;YAC9B,IAAI,IAAI,CAAC,QAAQ,KAAK,QAAQ;gBAAE,OAAO;YACvC,KAAK,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAClC,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;gBACjB,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;oBAC7B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;oBAChB,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC;gBAC5D,CAAC;YACH,CAAC;QACH,CAAC,CAAC,EACF,MAAM,CAAC,EAAE,CAAC,YAAY,EAAE,CAAC,IAAI,EAAE,EAAE;YAC/B,IAAI,IAAI,CAAC,QAAQ,KAAK,QAAQ;gBAAE,OAAO;YACvC,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;gBAC7B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBAChB,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC;YAC5D,CAAC;QACH,CAAC,CAAC,EACF,MAAM,CAAC,EAAE,CAAC,UAAU,EAAE,CAAC,IAAI,EAAE,EAAE;YAC7B,IAAI,IAAI,CAAC,QAAQ,KAAK,QAAQ;gBAAE,OAAO;YACvC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC1B,MAAM,EAAE,GAAG,IAAI,CAAC,YAAY,CAAC;YAC7B,IAAI,OAAO,EAAE,KAAK,QAAQ,EAAE,CAAC;gBAC3B,KAAK,CAAC,IAAI,CAAC,EAAE,YAAY,EAAE,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;gBACxD,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC;YAClE,CAAC;QACH,CAAC,CAAC,EACF,MAAM,CAAC,EAAE,CAAC,UAAU,EAAE,CAAC,IAAI,EAAE,EAAE;YAC7B,IAAI,IAAI,CAAC,QAAQ,KAAK,QAAQ;gBAAE,OAAO;YACvC,UAAU,GAAG,IAAI,CAAC;QACpB,CAAC,CAAC,EACF,MAAM,CAAC,EAAE,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,EAAE;YAC9B,IAAI,IAAI,CAAC,QAAQ,KAAK,QAAQ;gBAAE,OAAO;YACvC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACvB,MAAM,EAAE,CAAC;QACX,CAAC,CAAC,CACH,CAAC;QACF,MAAM,CAAC,OAAO,GAAG,qBAAqB,CAAC,IAAI,CAAC,CAAC;QAE7C,MAAM,CAAC,IAAI,CAAC,WAAW,EAAE;YACvB,QAAQ;YACR,IAAI;YACJ,KAAK,EAAE,OAAO,EAAE,KAAK;YACrB,cAAc,EAAE,UAAU;SAC3B,CAAC,CAAC;IACL,CAAC,EACD,CAAC,MAAM,CAAC,CACT,CAAC;IAEF,MAAM,IAAI,GAAG,WAAW,CAAC,GAAG,EAAE;QAC5B,IAAI,WAAW,CAAC,OAAO,EAAE,CAAC;YACxB,MAAM,CAAC,IAAI,CAAC,YAAY,EAAE,EAAE,QAAQ,EAAE,WAAW,CAAC,OAAO,EAAE,CAAC,CAAC;QAC/D,CAAC;QACD,WAAW,CAAC,OAAO,GAAG,IAAI,CAAC;QAC3B,OAAO,EAAE,CAAC;QACV,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC;QAC1B,UAAU,CAAC,KAAK,CAAC,CAAC;QAClB,gBAAgB,EAAE,CAAC;QACnB,MAAM,CAAC,OAAO,EAAE,CAAC;IACnB,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC;IAEb,SAAS,CAAC,GAAG,EAAE;QACb,OAAO,GAAG,EAAE;YACV,IAAI,WAAW,CAAC,OAAO,EAAE,CAAC;gBACxB,MAAM,CAAC,IAAI,CAAC,YAAY,EAAE,EAAE,QAAQ,EAAE,WAAW,CAAC,OAAO,EAAE,CAAC,CAAC;YAC/D,CAAC;YACD,OAAO,EAAE,CAAC;YACV,gBAAgB,EAAE,CAAC;YACnB,MAAM,CAAC,OAAO,EAAE,CAAC;YACjB,SAAS,CAAC,OAAO,EAAE,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAC7C,CAAC,CAAC;IACJ,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC;IAEb,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;AACvE,CAAC"}
package/dist/client/index.d.ts CHANGED
@@ -27,6 +27,7 @@ export { BlobVisualizerCore } from './audio/BlobVisualizerCore.js';
27
27
  export { MicVisualizer } from './audio/MicVisualizer.js';
28
28
  export { useSTT } from './audio/useSTT.js';
29
29
  export { useTTS } from './audio/useTTS.js';
30
+ export type { TTSVisemeEvent } from './audio/useTTS.js';
30
31
  export { WaveVisualizer } from './audio/WaveVisualizer.js';
31
32
  export { callImageGen, callJsonGen, callTextGen } from './callAI.js';
32
33
  export { uploadBlob, promoteBlob, downscaleImage } from './uploadBlob.js';
package/dist/client/index.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/client/index.ts"],"names":[],"mappings":"AAAA,YAAY,EACV,WAAW,EACX,mBAAmB,EACnB,aAAa,EACb,YAAY,EACZ,OAAO,EACP,kBAAkB,EAClB,YAAY,GACb,MAAM,oBAAoB,CAAC;AAC5B,YAAY,EACV,UAAU,EACV,aAAa,EACb,SAAS,EACT,OAAO,EACP,UAAU,EACV,YAAY,EACZ,SAAS,EACT,cAAc,EACd,WAAW,EACX,cAAc,EACd,cAAc,GACf,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,QAAQ,EAAE,MAAM,yBAAyB,CAAC;AACnD,YAAY,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAC;AAC7D,OAAO,EACL,mBAAmB,EACnB,eAAe,EACf,gBAAgB,EAChB,uBAAuB,GACxB,MAAM,8BAA8B,CAAC;AACtC,YAAY,EACV,gBAAgB,EAChB,cAAc,EACd,gBAAgB,GACjB,MAAM,8BAA8B,CAAC;AACtC,OAAO,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,qBAAqB,EAAE,MAAM,sCAAsC,CAAC;AACzH,YAAY,EAAE,eAAe,EAAE,cAAc,EAAE,gBAAgB,EAAE,MAAM,sCAAsC,CAAC;AAC9G,OAAO,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,MAAM,mCAAmC,CAAC;AAC5F,YAAY,EAAE,qBAAqB,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,MAAM,mCAAmC,CAAC;AAC1H,OAAO,EAAE,MAAM,EAAE,MAAM,wBAAwB,CAAC;AAChD,OAAO,EAAE,eAAe,EAAE,MAAM,iCAAiC,CAAC;AAClE,OAAO,EAAE,cAAc,EAAE,MAAM,gCAAgC,CAAC;AAChE,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AACrF,YAAY,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,EACL,eAAe,EACf,iBAAiB,EACjB,OAAO,EACP,UAAU,GACX,MAAM,sBAAsB,CAAC;AAC9B,YAAY,EACV,mBAAmB,EACnB,qBAAqB,GACtB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AACrD,OAAO,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AACzD,YAAY,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAC;AAClE,OAAO,EAAE,iBAAiB,EAAE,MAAM,8BAA8B,CAAC;AACjE,OAAO,EACL,gBAAgB,EAChB,iBAAiB,EACjB,0BAA0B,EAC1B,WAAW,EACX,QAAQ,GACT,MAAM,sBAAsB,CAAC;AAC9B,YAAY,EACV,UAAU,EACV,eAAe,EACf,kBAAkB,EAClB,WAAW,EACX,SAAS,EACT,aAAa,GACd,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,YAAY,EACV,uBAAuB,EACvB,mBAAmB,GACpB,MAAM,2BAA2B,CAAC;AACnC,OAAO,EAAE,kBAAkB,EAAE,MAAM,+BAA+B,CAAC;AACnE,OAAO,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AACzD,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AACrE,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAC1E,YAAY,EAAE,YAAY,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AACvE,OAAO,EAAE,cAAc,EAAE,MAAM,gCAAgC,CAAC;AAChE,cAAc,uBAAuB,CAAC;AACtC,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,YAAY,EACV,aAAa,EACb,uBAAuB,GACxB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACjD,YAAY,EAAE,SAAS,EAAE,eAAe,EAAE,mBAAmB,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACzG,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AACzD,YAAY,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACxD,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACjD,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,EACL,oBAAoB,EACpB,kBAAkB,EAClB,kBAAkB,GACnB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AACjE,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AAC3E,YAAY,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AAC9D,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC3E,YAAY,EACV,WAAW,EACX,eAAe,EACf,YAAY,EACZ,kBAAkB,GACnB,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AACvE,YAAY,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AACnE,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,YAAY,EACV,cAAc,EACd,QAAQ,EACR,YAAY,EACZ,WAAW,EACX,aAAa,EACb,YAAY,EACZ,UAAU,EACV,eAAe,EACf,iBAAiB,EACjB,iBAAiB,EACjB,iBAAiB,GAClB,MAAM,oBAAoB,CAAC;AAC5B,YAAY,EAAE,qBAAqB,EAAE,MAAM,kBAAkB,CAAC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/client/index.ts"],"names":[],"mappings":"AAAA,YAAY,EACV,WAAW,EACX,mBAAmB,EACnB,aAAa,EACb,YAAY,EACZ,OAAO,EACP,kBAAkB,EAClB,YAAY,GACb,MAAM,oBAAoB,CAAC;AAC5B,YAAY,EACV,UAAU,EACV,aAAa,EACb,SAAS,EACT,OAAO,EACP,UAAU,EACV,YAAY,EACZ,SAAS,EACT,cAAc,EACd,WAAW,EACX,cAAc,EACd,cAAc,GACf,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,QAAQ,EAAE,MAAM,yBAAyB,CAAC;AACnD,YAAY,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAC;AAC7D,OAAO,EACL,mBAAmB,EACnB,eAAe,EACf,gBAAgB,EAChB,uBAAuB,GACxB,MAAM,8BAA8B,CAAC;AACtC,YAAY,EACV,gBAAgB,EAChB,cAAc,EACd,gBAAgB,GACjB,MAAM,8BAA8B,CAAC;AACtC,OAAO,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,qBAAqB,EAAE,MAAM,sCAAsC,CAAC;AACzH,YAAY,EAAE,eAAe,EAAE,cAAc,EAAE,gBAAgB,EAAE,MAAM,sCAAsC,CAAC;AAC9G,OAAO,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,MAAM,mCAAmC,CAAC;AAC5F,YAAY,EAAE,qBAAqB,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,MAAM,mCAAmC,CAAC;AAC1H,OAAO,EAAE,MAAM,EAAE,MAAM,wBAAwB,CAAC;AAChD,OAAO,EAAE,eAAe,EAAE,MAAM,iCAAiC,CAAC;AAClE,OAAO,EAAE,cAAc,EAAE,MAAM,gCAAgC,CAAC;AAChE,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AACrF,YAAY,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,EACL,eAAe,EACf,iBAAiB,EACjB,OAAO,EACP,UAAU,GACX,MAAM,sBAAsB,CAAC;AAC9B,YAAY,EACV,mBAAmB,EACnB,qBAAqB,GACtB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AACrD,OAAO,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AACzD,YAAY,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAC;AAClE,OAAO,EAAE,iBAAiB,EAAE,MAAM,8BAA8B,CAAC;AACjE,OAAO,EACL,gBAAgB,EAChB,iBAAiB,EACjB,0BAA0B,EAC1B,WAAW,EACX,QAAQ,GACT,MAAM,sBAAsB,CAAC;AAC9B,YAAY,EACV,UAAU,EACV,eAAe,EACf,kBAAkB,EAClB,WAAW,EACX,SAAS,EACT,aAAa,GACd,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,YAAY,EACV,uBAAuB,EACvB,mBAAmB,GACpB,MAAM,2BAA2B,CAAC;AACnC,OAAO,EAAE,kBAAkB,EAAE,MAAM,+BAA+B,CAAC;AACnE,OAAO,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AACzD,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,YAAY,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACxD,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AACrE,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAC1E,YAAY,EAAE,YAAY,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AACvE,OAAO,EAAE,cAAc,EAAE,MAAM,gCAAgC,CAAC;AAChE,cAAc,uBAAuB,CAAC;AACtC,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,YAAY,EACV,aAAa,EACb,uBAAuB,GACxB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACjD,YAAY,EAAE,SAAS,EAAE,eAAe,EAAE,mBAAmB,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACzG,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AACzD,YAAY,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACxD,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACjD,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,EACL,oBAAoB,EACpB,kBAAkB,EAClB,kBAAkB,GACnB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AACjE,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AAC3E,YAAY,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AAC9D,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC3E,YAAY,EACV,WAAW,EACX,eAAe,EACf,YAAY,EACZ,kBAAkB,GACnB,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AACvE,YAAY,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AACnE,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,YAAY,EACV,cAAc,EACd,QAAQ,EACR,YAAY,EACZ,WAAW,EACX,aAAa,EACb,YAAY,EACZ,UAAU,EACV,eAAe,EACf,iBAAiB,EACjB,iBAAiB,EACjB,iBAAiB,GAClB,MAAM,oBAAoB,CAAC;AAC5B,YAAY,EAAE,qBAAqB,EAAE,MAAM,kBAAkB,CAAC"}
package/dist/client/index.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/client/index.ts"],"names":[],"mappings":"AAsBA,OAAO,EAAE,QAAQ,EAAE,MAAM,yBAAyB,CAAC;AAEnD,OAAO,EACL,mBAAmB,EACnB,eAAe,EACf,gBAAgB,EAChB,uBAAuB,GACxB,MAAM,8BAA8B,CAAC;AAMtC,OAAO,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,qBAAqB,EAAE,MAAM,sCAAsC,CAAC;AAEzH,OAAO,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,MAAM,mCAAmC,CAAC;AAE5F,OAAO,EAAE,MAAM,EAAE,MAAM,wBAAwB,CAAC;AAChD,OAAO,EAAE,eAAe,EAAE,MAAM,iCAAiC,CAAC;AAClE,OAAO,EAAE,cAAc,EAAE,MAAM,gCAAgC,CAAC;AAChE,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAErF,OAAO,EACL,eAAe,EACf,iBAAiB,EACjB,OAAO,EACP,UAAU,GACX,MAAM,sBAAsB,CAAC;AAK9B,OAAO,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AACrD,OAAO,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AAEzD,OAAO,EAAE,iBAAiB,EAAE,MAAM,8BAA8B,CAAC;AACjE,OAAO,EACL,gBAAgB,EAChB,iBAAiB,EACjB,0BAA0B,EAC1B,WAAW,EACX,QAAQ,GACT,MAAM,sBAAsB,CAAC;AAS9B,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAK3D,OAAO,EAAE,kBAAkB,EAAE,MAAM,+BAA+B,CAAC;AACnE,OAAO,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AACzD,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AACrE,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAE1E,OAAO,EAAE,cAAc,EAAE,MAAM,gCAAgC,CAAC;AAChE,cAAc,uBAAuB,CAAC;AACtC,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAKzD,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAEjD,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAEzD,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACjD,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,EACL,oBAAoB,EACpB,kBAAkB,EAClB,kBAAkB,GACnB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AACjE,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AAE3E,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAO3E,OAAO,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AAEvE,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC"}
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/client/index.ts"],"names":[],"mappings":"AAsBA,OAAO,EAAE,QAAQ,EAAE,MAAM,yBAAyB,CAAC;AAEnD,OAAO,EACL,mBAAmB,EACnB,eAAe,EACf,gBAAgB,EAChB,uBAAuB,GACxB,MAAM,8BAA8B,CAAC;AAMtC,OAAO,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,qBAAqB,EAAE,MAAM,sCAAsC,CAAC;AAEzH,OAAO,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,MAAM,mCAAmC,CAAC;AAE5F,OAAO,EAAE,MAAM,EAAE,MAAM,wBAAwB,CAAC;AAChD,OAAO,EAAE,eAAe,EAAE,MAAM,iCAAiC,CAAC;AAClE,OAAO,EAAE,cAAc,EAAE,MAAM,gCAAgC,CAAC;AAChE,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAErF,OAAO,EACL,eAAe,EACf,iBAAiB,EACjB,OAAO,EACP,UAAU,GACX,MAAM,sBAAsB,CAAC;AAK9B,OAAO,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AACrD,OAAO,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AAEzD,OAAO,EAAE,iBAAiB,EAAE,MAAM,8BAA8B,CAAC;AACjE,OAAO,EACL,gBAAgB,EAChB,iBAAiB,EACjB,0BAA0B,EAC1B,WAAW,EACX,QAAQ,GACT,MAAM,sBAAsB,CAAC;AAS9B,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAK3D,OAAO,EAAE,kBAAkB,EAAE,MAAM,+BAA+B,CAAC;AACnE,OAAO,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AACzD,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAE3C,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AACrE,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAE1E,OAAO,EAAE,cAAc,EAAE,MAAM,gCAAgC,CAAC;AAChE,cAAc,uBAAuB,CAAC;AACtC,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAKzD,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAEjD,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAEzD,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACjD,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,EACL,oBAAoB,EACpB,kBAAkB,EAClB,kBAAkB,GACnB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AACjE,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AAE3E,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAO3E,OAAO,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AAEvE,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "ugly-app",
3
- "version": "0.1.562",
3
+ "version": "0.1.563",
4
4
  "type": "module",
5
5
  "main": "./dist/server/index.js",
6
6
  "exports": {
package/src/cli/version.ts CHANGED
@@ -1,2 +1,2 @@
1
1
  // Auto-generated by prebuild — do not edit manually
2
- export const CLI_VERSION = "0.1.562";
2
+ export const CLI_VERSION = "0.1.563";
package/src/client/audio/AudioPlayer.ts CHANGED
@@ -6,12 +6,16 @@ class PCM16Player extends AudioWorkletProcessor {
6
6
  this.queue = [];
7
7
  this.buffer = new Float32Array(0);
8
8
  this.offset = 0;
9
+ this.played = 0; // total real samples emitted (the playback clock)
10
+ this.tick = 0;
11
+ this.draining = false;
9
12
  this.port.onmessage = (event) => this.handleMessage(event);
10
13
  }
11
14
  process(_inputs, outputs) {
12
15
  const output = outputs[0]?.[0];
13
16
  if (!output) return true;
14
17
  let writePos = 0;
18
+ let real = 0;
15
19
  while (writePos < output.length) {
16
20
  if (this.offset >= this.buffer.length) {
17
21
  const next = this.queue.shift();
@@ -25,6 +29,19 @@ class PCM16Player extends AudioWorkletProcessor {
25
29
  output.set(this.buffer.subarray(this.offset, this.offset + toCopy), writePos);
26
30
  this.offset += toCopy;
27
31
  writePos += toCopy;
32
+ real += toCopy;
33
+ }
34
+ if (real > 0) {
35
+ this.played += real;
36
+ this.draining = true;
37
+ // Report the playback clock ~40×/s so the main thread can schedule
38
+ // visemes/captions against actual audio position.
39
+ if (++this.tick % 6 === 0) this.port.postMessage({ type: 'time', played: this.played });
40
+ } else if (this.draining) {
41
+ // Queue ran dry after playing — playback finished.
42
+ this.draining = false;
43
+ this.port.postMessage({ type: 'time', played: this.played });
44
+ this.port.postMessage({ type: 'drained' });
28
45
  }
29
46
  return true;
30
47
  }
@@ -35,11 +52,16 @@ class PCM16Player extends AudioWorkletProcessor {
35
52
  this.queue = [];
36
53
  this.buffer = new Float32Array(0);
37
54
  this.offset = 0;
55
+ this.played = 0;
56
+ this.draining = false;
38
57
  }
39
58
  }
40
59
  }
41
60
  registerProcessor('pcm16-player', PCM16Player);
42
61
  `;
62
+
63
+ /** TTS audio is always mono PCM16 @ 24 kHz (see TTSChunkMsg). */
64
+ const TTS_SAMPLE_RATE = 24000;
43
65
  const WORKLET_BLOB = new Blob([WORKLET_CODE], { type: 'application/javascript' });
44
66
  const WORKLET_URL = URL.createObjectURL(WORKLET_BLOB);
45
67
 
@@ -57,6 +79,10 @@ export class AudioPlayer {
57
79
  private workletNode: AudioWorkletNode | null = null;
58
80
  private analyser: AnalyserNode | null = null;
59
81
  private _playing = false;
82
+ // Playback clock fed by the worklet (samples actually emitted). Used to
83
+ // schedule visemes + caption indices against real audio position.
84
+ private playedSamples = 0;
85
+ private _drained = false;
60
86
 
61
87
  get playing(): boolean {
62
88
  return this._playing;
@@ -70,11 +96,40 @@ export class AudioPlayer {
70
96
  return this.analyser;
71
97
  }
72
98
 
99
+ /** Milliseconds of audio actually played so far (the lip-sync/caption clock). */
100
+ getPlaybackTimeMs(): number {
101
+ return (this.playedSamples / TTS_SAMPLE_RATE) * 1000;
102
+ }
103
+
104
+ /** True once the queue has drained after playing (stream finished). */
105
+ get drained(): boolean {
106
+ return this._drained;
107
+ }
108
+
109
+ /** Reset the playback clock for a new stream. */
110
+ reset(): void {
111
+ this.workletNode?.port.postMessage({ type: 'stop' });
112
+ this.playedSamples = 0;
113
+ this._drained = false;
114
+ this._playing = false;
115
+ }
116
+
73
117
  async init(): Promise<void> {
74
118
  if (this.workletNode) return;
75
- this.ctx = new AudioContext();
119
+ // Match the TTS sample rate so PCM16 plays at correct pitch (the worklet
120
+ // outputs 1:1, no resampling) and the playback clock is accurate.
121
+ this.ctx = new AudioContext({ sampleRate: TTS_SAMPLE_RATE });
76
122
  await this.ctx.audioWorklet.addModule(WORKLET_URL);
77
123
  this.workletNode = new AudioWorkletNode(this.ctx, 'pcm16-player');
124
+ this.workletNode.port.onmessage = (e: MessageEvent) => {
125
+ const d = e.data as { type?: string; played?: number };
126
+ if (d?.type === 'time' && typeof d.played === 'number') {
127
+ this.playedSamples = d.played;
128
+ } else if (d?.type === 'drained') {
129
+ this._drained = true;
130
+ this._playing = false;
131
+ }
132
+ };
78
133
  this.analyser = this.ctx.createAnalyser();
79
134
  this.analyser.fftSize = 256;
80
135
  this.analyser.smoothingTimeConstant = 0.6;
@@ -109,6 +164,7 @@ export class AudioPlayer {
109
164
 
110
165
  const samples = pcm16ToFloat32(bytes.buffer);
111
166
  this._playing = true;
167
+ this._drained = false;
112
168
  this.workletNode.port.postMessage({ type: 'chunk', samples }, [
113
169
  samples.buffer,
114
170
  ]);
package/src/client/audio/useTTS.ts CHANGED
@@ -1,10 +1,29 @@
1
1
  // src/client/audio/useTTS.ts
2
2
  import { useCallback, useEffect, useRef, useState } from 'react';
3
+ import type { TTSViseme, TTSVisemeName } from '../../shared/Audio.js';
3
4
  import type { UglyBotSocket } from '../uglyBotSocket.js';
4
5
  import { AudioPlayer } from './AudioPlayer.js';
5
6
 
7
+ /** A viseme delivered to the avatar (timing stripped — driven by the audio clock). */
8
+ export interface TTSVisemeEvent {
9
+ name: TTSVisemeName;
10
+ durationMs: number;
11
+ intensity: number;
12
+ }
13
+
6
14
  interface TTSOptions {
7
15
  voice?: string;
16
+ /**
17
+ * Fired as each viseme reaches the current audio-playback position. Wire to
18
+ * `AvatarModel.showViseme(...)` for true InWorld lip-sync.
19
+ */
20
+ onViseme?: (v: TTSVisemeEvent) => void;
21
+ /**
22
+ * Fired as each spoken word reaches the playback position — value is the
23
+ * character offset of the word's end in the original text. Reveal the caption
24
+ * up to this index for karaoke-style subtitles.
25
+ */
26
+ onSubtitleIndex?: (charIndex: number) => void;
8
27
  }
9
28
 
10
29
  let streamCounter = 0;
@@ -13,18 +32,27 @@ export function useTTS(socket: UglyBotSocket) {
13
32
  const [playing, setPlaying] = useState(false);
14
33
  const [currentWord, setCurrentWord] = useState('');
15
34
  const [error, setError] = useState<string | null>(null);
16
- // Real-time amplitude analyser of the TTS playback — drives avatar lip-sync.
35
+ // Real-time amplitude analyser of the TTS playback — a fallback lip-sync
36
+ // signal when viseme data isn't requested/available.
17
37
  const [analyser, setAnalyser] = useState<AnalyserNode | null>(null);
18
38
 
19
39
  const playerRef = useRef<AudioPlayer | null>(null);
20
40
  const streamIdRef = useRef<string | null>(null);
21
41
  const unsubsRef = useRef<(() => void)[]>([]);
42
+ const rafRef = useRef<number | null>(null);
22
43
 
23
44
  function cleanupListeners() {
24
45
  for (const unsub of unsubsRef.current) unsub();
25
46
  unsubsRef.current = [];
26
47
  }
27
48
 
49
+ function stopRaf() {
50
+ if (rafRef.current !== null) {
51
+ cancelAnimationFrame(rafRef.current);
52
+ rafRef.current = null;
53
+ }
54
+ }
55
+
28
56
  // Create + resume the AudioContext from inside a user gesture (e.g. the
29
57
  // user's send-click). Browsers (esp. Safari) only unlock audio when resume()
30
58
  // runs within a gesture; the TTS reply itself plays later, async, so without
@@ -42,45 +70,109 @@ export function useTTS(socket: UglyBotSocket) {
42
70
  streamIdRef.current = streamId;
43
71
 
44
72
  if (!playerRef.current) playerRef.current = new AudioPlayer();
45
- await playerRef.current.init();
46
- setAnalyser(playerRef.current.getAnalyser());
73
+ const player = playerRef.current;
74
+ await player.init();
75
+ player.reset();
76
+ setAnalyser(player.getAnalyser());
47
77
 
48
78
  setPlaying(true);
49
79
  setCurrentWord('');
50
80
  setError(null);
51
81
 
82
+ // Collect viseme/word timing as it streams; an rAF loop dispatches each
83
+ // against the real audio-playback clock (collect-then-schedule).
84
+ const visemes: TTSViseme[] = [];
85
+ const words: { charEndIndex: number; startMs: number }[] = [];
86
+ let vi = 0;
87
+ let wi = 0;
88
+ let serverDone = false;
89
+ let lastEndMs = 0;
90
+ const wantViseme = !!options?.onViseme;
91
+
92
+ const finish = () => {
93
+ if (streamIdRef.current !== streamId) return;
94
+ streamIdRef.current = null;
95
+ stopRaf();
96
+ // Flush any remaining caption indices so the text fully reveals.
97
+ while (wi < words.length) options?.onSubtitleIndex?.(words[wi++]!.charEndIndex);
98
+ setPlaying(false);
99
+ setCurrentWord('');
100
+ cleanupListeners();
101
+ socket.release();
102
+ };
103
+
104
+ const tick = () => {
105
+ if (streamIdRef.current !== streamId) return;
106
+ const t = player.getPlaybackTimeMs();
107
+ while (vi < visemes.length && visemes[vi]!.startMs <= t) {
108
+ const v = visemes[vi++]!;
109
+ options?.onViseme?.({
110
+ name: v.name,
111
+ durationMs: v.durationMs,
112
+ intensity: v.intensity,
113
+ });
114
+ }
115
+ while (wi < words.length && words[wi]!.startMs <= t) {
116
+ options?.onSubtitleIndex?.(words[wi++]!.charEndIndex);
117
+ }
118
+ // Done when the server finished, the queue drained, and everything
119
+ // scheduled has been dispatched (or playback passed the last event).
120
+ const dispatched = vi >= visemes.length && wi >= words.length;
121
+ if (serverDone && (player.drained || t >= lastEndMs) && dispatched) {
122
+ finish();
123
+ return;
124
+ }
125
+ rafRef.current = requestAnimationFrame(tick);
126
+ };
127
+
52
128
  await socket.acquire();
53
129
 
54
130
  cleanupListeners();
131
+ stopRaf();
55
132
  unsubsRef.current.push(
56
133
  socket.on('tts:chunk', (data) => {
57
134
  if (data.streamId !== streamId) return;
58
- void playerRef.current?.feedChunk(data.audio);
135
+ void player.feedChunk(data.audio);
136
+ if (data.visemes) {
137
+ for (const v of data.visemes) {
138
+ visemes.push(v);
139
+ lastEndMs = Math.max(lastEndMs, v.startMs + v.durationMs);
140
+ }
141
+ }
142
+ }),
143
+ socket.on('tts:viseme', (data) => {
144
+ if (data.streamId !== streamId) return;
145
+ for (const v of data.visemes) {
146
+ visemes.push(v);
147
+ lastEndMs = Math.max(lastEndMs, v.startMs + v.durationMs);
148
+ }
59
149
  }),
60
150
  socket.on('tts:word', (data) => {
61
151
  if (data.streamId !== streamId) return;
62
152
  setCurrentWord(data.word);
153
+ const ce = data.charEndIndex;
154
+ if (typeof ce === 'number') {
155
+ words.push({ charEndIndex: ce, startMs: data.startMs });
156
+ lastEndMs = Math.max(lastEndMs, data.startMs + data.durationMs);
157
+ }
63
158
  }),
64
159
  socket.on('tts:done', (data) => {
65
160
  if (data.streamId !== streamId) return;
66
- setPlaying(false);
67
- setCurrentWord('');
68
- cleanupListeners();
69
- socket.release();
161
+ serverDone = true;
70
162
  }),
71
163
  socket.on('tts:error', (data) => {
72
164
  if (data.streamId !== streamId) return;
73
- setPlaying(false);
74
165
  setError(data.message);
75
- cleanupListeners();
76
- socket.release();
166
+ finish();
77
167
  }),
78
168
  );
169
+ rafRef.current = requestAnimationFrame(tick);
79
170
 
80
171
  socket.send('tts:start', {
81
172
  streamId,
82
173
  text,
83
174
  voice: options?.voice,
175
+ requestVisemes: wantViseme,
84
176
  });
85
177
  },
86
178
  [socket],
@@ -90,6 +182,8 @@ export function useTTS(socket: UglyBotSocket) {
90
182
  if (streamIdRef.current) {
91
183
  socket.send('tts:cancel', { streamId: streamIdRef.current });
92
184
  }
185
+ streamIdRef.current = null;
186
+ stopRaf();
93
187
  playerRef.current?.stop();
94
188
  setPlaying(false);
95
189
  cleanupListeners();
@@ -101,6 +195,7 @@ export function useTTS(socket: UglyBotSocket) {
101
195
  if (streamIdRef.current) {
102
196
  socket.send('tts:cancel', { streamId: streamIdRef.current });
103
197
  }
198
+ stopRaf();
104
199
  cleanupListeners();
105
200
  socket.release();
106
201
  playerRef.current?.close().catch(() => {});
package/src/client/index.ts CHANGED
@@ -80,6 +80,7 @@ export { BlobVisualizerCore } from './audio/BlobVisualizerCore.js';
80
80
  export { MicVisualizer } from './audio/MicVisualizer.js';
81
81
  export { useSTT } from './audio/useSTT.js';
82
82
  export { useTTS } from './audio/useTTS.js';
83
+ export type { TTSVisemeEvent } from './audio/useTTS.js';
83
84
  export { WaveVisualizer } from './audio/WaveVisualizer.js';
84
85
  export { callImageGen, callJsonGen, callTextGen } from './callAI.js';
85
86
  export { uploadBlob, promoteBlob, downscaleImage } from './uploadBlob.js';