ugly-app 0.1.561 → 0.1.563
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/version.d.ts +1 -1
- package/dist/cli/version.js +1 -1
- package/dist/client/audio/AudioPlayer.d.ts +10 -0
- package/dist/client/audio/AudioPlayer.d.ts.map +1 -1
- package/dist/client/audio/AudioPlayer.js +71 -1
- package/dist/client/audio/AudioPlayer.js.map +1 -1
- package/dist/client/audio/useTTS.d.ts +19 -0
- package/dist/client/audio/useTTS.d.ts.map +1 -1
- package/dist/client/audio/useTTS.js +98 -12
- package/dist/client/audio/useTTS.js.map +1 -1
- package/dist/client/index.d.ts +1 -0
- package/dist/client/index.d.ts.map +1 -1
- package/dist/client/index.js.map +1 -1
- package/package.json +1 -1
- package/src/cli/version.ts +1 -1
- package/src/client/audio/AudioPlayer.ts +73 -1
- package/src/client/audio/useTTS.ts +118 -12
- package/src/client/index.ts +1 -0
package/dist/cli/version.d.ts
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
export declare const CLI_VERSION = "0.1.
|
|
1
|
+
export declare const CLI_VERSION = "0.1.563";
|
|
2
2
|
//# sourceMappingURL=version.d.ts.map
|
package/dist/cli/version.js
CHANGED
|
@@ -3,13 +3,23 @@ export declare class AudioPlayer {
|
|
|
3
3
|
private workletNode;
|
|
4
4
|
private analyser;
|
|
5
5
|
private _playing;
|
|
6
|
+
private playedSamples;
|
|
7
|
+
private _drained;
|
|
6
8
|
get playing(): boolean;
|
|
7
9
|
/**
|
|
8
10
|
* Real-time amplitude analyser tapped off the playback graph. Available
|
|
9
11
|
* after `init()`. Used to drive talking-avatar lip-sync (jaw open ∝ RMS).
|
|
10
12
|
*/
|
|
11
13
|
getAnalyser(): AnalyserNode | null;
|
|
14
|
+
/** Milliseconds of audio actually played so far (the lip-sync/caption clock). */
|
|
15
|
+
getPlaybackTimeMs(): number;
|
|
16
|
+
/** True once the queue has drained after playing (stream finished). */
|
|
17
|
+
get drained(): boolean;
|
|
18
|
+
/** Reset the playback clock for a new stream. */
|
|
19
|
+
reset(): void;
|
|
12
20
|
init(): Promise<void>;
|
|
21
|
+
/** Resume the AudioContext if the browser autoplay policy suspended it. */
|
|
22
|
+
resume(): Promise<void>;
|
|
13
23
|
feedChunk(base64Audio: string): Promise<void>;
|
|
14
24
|
stop(): void;
|
|
15
25
|
close(): Promise<void>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"AudioPlayer.d.ts","sourceRoot":"","sources":["../../../src/client/audio/AudioPlayer.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"AudioPlayer.d.ts","sourceRoot":"","sources":["../../../src/client/audio/AudioPlayer.ts"],"names":[],"mappings":"AA4EA,qBAAa,WAAW;IACtB,OAAO,CAAC,GAAG,CAA6B;IACxC,OAAO,CAAC,WAAW,CAAiC;IACpD,OAAO,CAAC,QAAQ,CAA6B;IAC7C,OAAO,CAAC,QAAQ,CAAS;IAGzB,OAAO,CAAC,aAAa,CAAK;IAC1B,OAAO,CAAC,QAAQ,CAAS;IAEzB,IAAI,OAAO,IAAI,OAAO,CAErB;IAED;;;OAGG;IACH,WAAW,IAAI,YAAY,GAAG,IAAI;IAIlC,iFAAiF;IACjF,iBAAiB,IAAI,MAAM;IAI3B,uEAAuE;IACvE,IAAI,OAAO,IAAI,OAAO,CAErB;IAED,iDAAiD;IACjD,KAAK,IAAI,IAAI;IAOP,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IA4B3B,2EAA2E;IACrE,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;IAUvB,SAAS,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAiBnD,IAAI,IAAI,IAAI;IAKN,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAO7B"}
|
|
@@ -6,12 +6,16 @@ class PCM16Player extends AudioWorkletProcessor {
|
|
|
6
6
|
this.queue = [];
|
|
7
7
|
this.buffer = new Float32Array(0);
|
|
8
8
|
this.offset = 0;
|
|
9
|
+
this.played = 0; // total real samples emitted (the playback clock)
|
|
10
|
+
this.tick = 0;
|
|
11
|
+
this.draining = false;
|
|
9
12
|
this.port.onmessage = (event) => this.handleMessage(event);
|
|
10
13
|
}
|
|
11
14
|
process(_inputs, outputs) {
|
|
12
15
|
const output = outputs[0]?.[0];
|
|
13
16
|
if (!output) return true;
|
|
14
17
|
let writePos = 0;
|
|
18
|
+
let real = 0;
|
|
15
19
|
while (writePos < output.length) {
|
|
16
20
|
if (this.offset >= this.buffer.length) {
|
|
17
21
|
const next = this.queue.shift();
|
|
@@ -25,6 +29,19 @@ class PCM16Player extends AudioWorkletProcessor {
|
|
|
25
29
|
output.set(this.buffer.subarray(this.offset, this.offset + toCopy), writePos);
|
|
26
30
|
this.offset += toCopy;
|
|
27
31
|
writePos += toCopy;
|
|
32
|
+
real += toCopy;
|
|
33
|
+
}
|
|
34
|
+
if (real > 0) {
|
|
35
|
+
this.played += real;
|
|
36
|
+
this.draining = true;
|
|
37
|
+
// Report the playback clock ~40×/s so the main thread can schedule
|
|
38
|
+
// visemes/captions against actual audio position.
|
|
39
|
+
if (++this.tick % 6 === 0) this.port.postMessage({ type: 'time', played: this.played });
|
|
40
|
+
} else if (this.draining) {
|
|
41
|
+
// Queue ran dry after playing — playback finished.
|
|
42
|
+
this.draining = false;
|
|
43
|
+
this.port.postMessage({ type: 'time', played: this.played });
|
|
44
|
+
this.port.postMessage({ type: 'drained' });
|
|
28
45
|
}
|
|
29
46
|
return true;
|
|
30
47
|
}
|
|
@@ -35,11 +52,15 @@ class PCM16Player extends AudioWorkletProcessor {
|
|
|
35
52
|
this.queue = [];
|
|
36
53
|
this.buffer = new Float32Array(0);
|
|
37
54
|
this.offset = 0;
|
|
55
|
+
this.played = 0;
|
|
56
|
+
this.draining = false;
|
|
38
57
|
}
|
|
39
58
|
}
|
|
40
59
|
}
|
|
41
60
|
registerProcessor('pcm16-player', PCM16Player);
|
|
42
61
|
`;
|
|
62
|
+
/** TTS audio is always mono PCM16 @ 24 kHz (see TTSChunkMsg). */
|
|
63
|
+
const TTS_SAMPLE_RATE = 24000;
|
|
43
64
|
const WORKLET_BLOB = new Blob([WORKLET_CODE], { type: 'application/javascript' });
|
|
44
65
|
const WORKLET_URL = URL.createObjectURL(WORKLET_BLOB);
|
|
45
66
|
function pcm16ToFloat32(buffer) {
|
|
@@ -55,6 +76,10 @@ export class AudioPlayer {
|
|
|
55
76
|
workletNode = null;
|
|
56
77
|
analyser = null;
|
|
57
78
|
_playing = false;
|
|
79
|
+
// Playback clock fed by the worklet (samples actually emitted). Used to
|
|
80
|
+
// schedule visemes + caption indices against real audio position.
|
|
81
|
+
playedSamples = 0;
|
|
82
|
+
_drained = false;
|
|
58
83
|
get playing() {
|
|
59
84
|
return this._playing;
|
|
60
85
|
}
|
|
@@ -65,29 +90,74 @@ export class AudioPlayer {
|
|
|
65
90
|
getAnalyser() {
|
|
66
91
|
return this.analyser;
|
|
67
92
|
}
|
|
93
|
+
/** Milliseconds of audio actually played so far (the lip-sync/caption clock). */
|
|
94
|
+
getPlaybackTimeMs() {
|
|
95
|
+
return (this.playedSamples / TTS_SAMPLE_RATE) * 1000;
|
|
96
|
+
}
|
|
97
|
+
/** True once the queue has drained after playing (stream finished). */
|
|
98
|
+
get drained() {
|
|
99
|
+
return this._drained;
|
|
100
|
+
}
|
|
101
|
+
/** Reset the playback clock for a new stream. */
|
|
102
|
+
reset() {
|
|
103
|
+
this.workletNode?.port.postMessage({ type: 'stop' });
|
|
104
|
+
this.playedSamples = 0;
|
|
105
|
+
this._drained = false;
|
|
106
|
+
this._playing = false;
|
|
107
|
+
}
|
|
68
108
|
async init() {
|
|
69
109
|
if (this.workletNode)
|
|
70
110
|
return;
|
|
71
|
-
|
|
111
|
+
// Match the TTS sample rate so PCM16 plays at correct pitch (the worklet
|
|
112
|
+
// outputs 1:1, no resampling) and the playback clock is accurate.
|
|
113
|
+
this.ctx = new AudioContext({ sampleRate: TTS_SAMPLE_RATE });
|
|
72
114
|
await this.ctx.audioWorklet.addModule(WORKLET_URL);
|
|
73
115
|
this.workletNode = new AudioWorkletNode(this.ctx, 'pcm16-player');
|
|
116
|
+
this.workletNode.port.onmessage = (e) => {
|
|
117
|
+
const d = e.data;
|
|
118
|
+
if (d?.type === 'time' && typeof d.played === 'number') {
|
|
119
|
+
this.playedSamples = d.played;
|
|
120
|
+
}
|
|
121
|
+
else if (d?.type === 'drained') {
|
|
122
|
+
this._drained = true;
|
|
123
|
+
this._playing = false;
|
|
124
|
+
}
|
|
125
|
+
};
|
|
74
126
|
this.analyser = this.ctx.createAnalyser();
|
|
75
127
|
this.analyser.fftSize = 256;
|
|
76
128
|
this.analyser.smoothingTimeConstant = 0.6;
|
|
77
129
|
// worklet → analyser → destination (analyser is a pass-through tap).
|
|
78
130
|
this.workletNode.connect(this.analyser);
|
|
79
131
|
this.analyser.connect(this.ctx.destination);
|
|
132
|
+
// A context created outside a user gesture starts suspended → no audio is
|
|
133
|
+
// ever heard and the analyser reads silence. Resume it (the page has
|
|
134
|
+
// already had user gestures by the time TTS plays).
|
|
135
|
+
await this.resume();
|
|
136
|
+
}
|
|
137
|
+
/** Resume the AudioContext if the browser autoplay policy suspended it. */
|
|
138
|
+
async resume() {
|
|
139
|
+
if (this.ctx && this.ctx.state === 'suspended') {
|
|
140
|
+
try {
|
|
141
|
+
await this.ctx.resume();
|
|
142
|
+
}
|
|
143
|
+
catch {
|
|
144
|
+
/* will retry on the next chunk */
|
|
145
|
+
}
|
|
146
|
+
}
|
|
80
147
|
}
|
|
81
148
|
async feedChunk(base64Audio) {
|
|
82
149
|
await this.init();
|
|
83
150
|
if (!this.workletNode || !this.ctx)
|
|
84
151
|
return;
|
|
152
|
+
if (this.ctx.state === 'suspended')
|
|
153
|
+
await this.resume();
|
|
85
154
|
const binary = atob(base64Audio);
|
|
86
155
|
const bytes = new Uint8Array(binary.length);
|
|
87
156
|
for (let i = 0; i < binary.length; i++)
|
|
88
157
|
bytes[i] = binary.charCodeAt(i);
|
|
89
158
|
const samples = pcm16ToFloat32(bytes.buffer);
|
|
90
159
|
this._playing = true;
|
|
160
|
+
this._drained = false;
|
|
91
161
|
this.workletNode.port.postMessage({ type: 'chunk', samples }, [
|
|
92
162
|
samples.buffer,
|
|
93
163
|
]);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"AudioPlayer.js","sourceRoot":"","sources":["../../../src/client/audio/AudioPlayer.ts"],"names":[],"mappings":"AAAA,kCAAkC;AAClC,MAAM,YAAY,GAAG
|
|
1
|
+
{"version":3,"file":"AudioPlayer.js","sourceRoot":"","sources":["../../../src/client/audio/AudioPlayer.ts"],"names":[],"mappings":"AAAA,kCAAkC;AAClC,MAAM,YAAY,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA2DpB,CAAC;AAEF,iEAAiE;AACjE,MAAM,eAAe,GAAG,KAAK,CAAC;AAC9B,MAAM,YAAY,GAAG,IAAI,IAAI,CAAC,CAAC,YAAY,CAAC,EAAE,EAAE,IAAI,EAAE,wBAAwB,EAAE,CAAC,CAAC;AAClF,MAAM,WAAW,GAAG,GAAG,CAAC,eAAe,CAAC,YAAY,CAAC,CAAC;AAEtD,SAAS,cAAc,CAAC,MAAmB;IACzC,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC,MAAM,CAAC,CAAC;IACrC,MAAM,OAAO,GAAG,IAAI,YAAY,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAC/C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,KAAK,CAAC;IACvC,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,OAAO,WAAW;IACd,GAAG,GAAwB,IAAI,CAAC;IAChC,WAAW,GAA4B,IAAI,CAAC;IAC5C,QAAQ,GAAwB,IAAI,CAAC;IACrC,QAAQ,GAAG,KAAK,CAAC;IACzB,wEAAwE;IACxE,kEAAkE;IAC1D,aAAa,GAAG,CAAC,CAAC;IAClB,QAAQ,GAAG,KAAK,CAAC;IAEzB,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC;IAED;;;OAGG;IACH,WAAW;QACT,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC;IAED,iFAAiF;IACjF,iBAAiB;QACf,OAAO,CAAC,IAAI,CAAC,aAAa,GAAG,eAAe,CAAC,GAAG,IAAI,CAAC;IACvD,CAAC;IAED,uEAAuE;IACvE,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC;IAED,iDAAiD;IACjD,KAAK;QACH,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;QACrD,IAAI,CAAC,aAAa,GAAG,CAAC,CAAC;QACvB,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;QACtB,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;IACxB,CAAC;IAED,KAAK,CAAC,IAAI;QACR,IAAI,IAAI,CAAC,WAAW;YAAE,OAAO;QAC7B,yEAAyE;QACzE,kEAAkE;QAClE,IAAI,CAAC,GAAG,GAAG,IAAI,YAAY,CAAC,EAAE,UAAU,EAAE,eAAe,EAAE,CAAC,CAAC;QAC7D,MAAM,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;QACnD,IAAI,CAAC,WAAW,GAAG,IAAI,gBAAgB,CAAC,IAAI,CAAC,GAAG,EAAE,cAAc,CAAC,CAAC;QAClE,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,SAAS,GAAG,CAAC,CAAe,EAAE,EAAE;YACpD,MAAM,CAAC,GAAG,CAAC,CAAC,IAA0C,CAAC;YACvD,IAAI,CAAC,EAAE,IAAI,KAAK,MAAM,IAAI,OAAO,CAAC,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;gBACvD,IAAI,CAAC,aAAa,GAAG,CAAC,CAAC,MAAM,CAAC;YAChC,CAAC;iBAAM,IAAI,CAAC,EAAE,IAAI,KAAK,SAAS,EAAE,CAAC;gBACjC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;gBACrB,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;YACxB,CAAC;QACH,CAAC,CAAC;QACF,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,cAAc,EAAE,CAAC;QAC1C,IAAI,CAAC,QAAQ,CAAC,OAAO,GAAG,GAAG,CAAC;QAC5B,IAAI,CAAC,QAAQ,CAAC,qBAAqB,GAAG,GAAG,CAAC;QAC1C,qEAAqE;QACrE,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACxC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QAC5C,0EAA0E;QAC1E,qEAAqE;QACrE,oDAAoD;QACpD,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC;IACtB,CAAC;IAED,2EAA2E;IAC3E,KAAK,CAAC,MAAM;QACV,IAAI,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,KAAK,KAAK,WAAW,EAAE,CAAC;YAC/C,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC;YAC1B,CAAC;YAAC,MAAM,CAAC;gBACP,kCAAkC;YACpC,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,WAAmB;QACjC,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAClB,IAAI,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC,IAAI,CAAC,GAAG;YAAE,OAAO;QAC3C,IAAI,IAAI,CAAC,GAAG,CAAC,KAAK,KAAK,WAAW;YAAE,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC;QAExD,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC;QACjC,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAC5C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE;YAAE,KAAK,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC,CAAE,CAAC;QAEzE,MAAM,OAAO,GAAG,cAAc,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAC7C,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACrB,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;QACtB,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,EAAE;YAC5D,OAAO,CAAC,MAAM;SACf,CAAC,CAAC;IACL,CAAC;IAED,IAAI;QACF,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;QACrD,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;IACxB,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,IAAI,EAAE,CAAC;QACZ,MAAM,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,CAAC;QACxB,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC;QAChB,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QACxB,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;IACvB,CAAC;CACF"}
|
|
@@ -1,10 +1,29 @@
|
|
|
1
|
+
import type { TTSVisemeName } from '../../shared/Audio.js';
|
|
1
2
|
import type { UglyBotSocket } from '../uglyBotSocket.js';
|
|
3
|
+
/** A viseme delivered to the avatar (timing stripped — driven by the audio clock). */
|
|
4
|
+
export interface TTSVisemeEvent {
|
|
5
|
+
name: TTSVisemeName;
|
|
6
|
+
durationMs: number;
|
|
7
|
+
intensity: number;
|
|
8
|
+
}
|
|
2
9
|
interface TTSOptions {
|
|
3
10
|
voice?: string;
|
|
11
|
+
/**
|
|
12
|
+
* Fired as each viseme reaches the current audio-playback position. Wire to
|
|
13
|
+
* `AvatarModel.showViseme(...)` for true InWorld lip-sync.
|
|
14
|
+
*/
|
|
15
|
+
onViseme?: (v: TTSVisemeEvent) => void;
|
|
16
|
+
/**
|
|
17
|
+
* Fired as each spoken word reaches the playback position — value is the
|
|
18
|
+
* character offset of the word's end in the original text. Reveal the caption
|
|
19
|
+
* up to this index for karaoke-style subtitles.
|
|
20
|
+
*/
|
|
21
|
+
onSubtitleIndex?: (charIndex: number) => void;
|
|
4
22
|
}
|
|
5
23
|
export declare function useTTS(socket: UglyBotSocket): {
|
|
6
24
|
play: (text: string, options?: TTSOptions) => Promise<void>;
|
|
7
25
|
stop: () => void;
|
|
26
|
+
warmup: () => Promise<void>;
|
|
8
27
|
playing: boolean;
|
|
9
28
|
currentWord: string;
|
|
10
29
|
error: string | null;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"useTTS.d.ts","sourceRoot":"","sources":["../../../src/client/audio/useTTS.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAGzD,UAAU,UAAU;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"useTTS.d.ts","sourceRoot":"","sources":["../../../src/client/audio/useTTS.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAa,aAAa,EAAE,MAAM,uBAAuB,CAAC;AACtE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAGzD,sFAAsF;AACtF,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,aAAa,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,UAAU,UAAU;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,QAAQ,CAAC,EAAE,CAAC,CAAC,EAAE,cAAc,KAAK,IAAI,CAAC;IACvC;;;;OAIG;IACH,eAAe,CAAC,EAAE,CAAC,SAAS,EAAE,MAAM,KAAK,IAAI,CAAC;CAC/C;AAID,wBAAgB,MAAM,CAAC,MAAM,EAAE,aAAa;iBAqC3B,MAAM,YAAY,UAAU;;;;;;;EA0I5C"}
|
|
@@ -6,61 +6,146 @@ export function useTTS(socket) {
|
|
|
6
6
|
const [playing, setPlaying] = useState(false);
|
|
7
7
|
const [currentWord, setCurrentWord] = useState('');
|
|
8
8
|
const [error, setError] = useState(null);
|
|
9
|
-
// Real-time amplitude analyser of the TTS playback —
|
|
9
|
+
// Real-time amplitude analyser of the TTS playback — a fallback lip-sync
|
|
10
|
+
// signal when viseme data isn't requested/available.
|
|
10
11
|
const [analyser, setAnalyser] = useState(null);
|
|
11
12
|
const playerRef = useRef(null);
|
|
12
13
|
const streamIdRef = useRef(null);
|
|
13
14
|
const unsubsRef = useRef([]);
|
|
15
|
+
const rafRef = useRef(null);
|
|
14
16
|
function cleanupListeners() {
|
|
15
17
|
for (const unsub of unsubsRef.current)
|
|
16
18
|
unsub();
|
|
17
19
|
unsubsRef.current = [];
|
|
18
20
|
}
|
|
21
|
+
function stopRaf() {
|
|
22
|
+
if (rafRef.current !== null) {
|
|
23
|
+
cancelAnimationFrame(rafRef.current);
|
|
24
|
+
rafRef.current = null;
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
// Create + resume the AudioContext from inside a user gesture (e.g. the
|
|
28
|
+
// user's send-click). Browsers (esp. Safari) only unlock audio when resume()
|
|
29
|
+
// runs within a gesture; the TTS reply itself plays later, async, so without
|
|
30
|
+
// this warmup the context stays suspended and nothing is heard.
|
|
31
|
+
const warmup = useCallback(async () => {
|
|
32
|
+
if (!playerRef.current)
|
|
33
|
+
playerRef.current = new AudioPlayer();
|
|
34
|
+
await playerRef.current.init();
|
|
35
|
+
await playerRef.current.resume();
|
|
36
|
+
setAnalyser(playerRef.current.getAnalyser());
|
|
37
|
+
}, []);
|
|
19
38
|
const play = useCallback(async (text, options) => {
|
|
20
39
|
const streamId = `tts-${++streamCounter}`;
|
|
21
40
|
streamIdRef.current = streamId;
|
|
22
41
|
if (!playerRef.current)
|
|
23
42
|
playerRef.current = new AudioPlayer();
|
|
24
|
-
|
|
25
|
-
|
|
43
|
+
const player = playerRef.current;
|
|
44
|
+
await player.init();
|
|
45
|
+
player.reset();
|
|
46
|
+
setAnalyser(player.getAnalyser());
|
|
26
47
|
setPlaying(true);
|
|
27
48
|
setCurrentWord('');
|
|
28
49
|
setError(null);
|
|
50
|
+
// Collect viseme/word timing as it streams; an rAF loop dispatches each
|
|
51
|
+
// against the real audio-playback clock (collect-then-schedule).
|
|
52
|
+
const visemes = [];
|
|
53
|
+
const words = [];
|
|
54
|
+
let vi = 0;
|
|
55
|
+
let wi = 0;
|
|
56
|
+
let serverDone = false;
|
|
57
|
+
let lastEndMs = 0;
|
|
58
|
+
const wantViseme = !!options?.onViseme;
|
|
59
|
+
const finish = () => {
|
|
60
|
+
if (streamIdRef.current !== streamId)
|
|
61
|
+
return;
|
|
62
|
+
streamIdRef.current = null;
|
|
63
|
+
stopRaf();
|
|
64
|
+
// Flush any remaining caption indices so the text fully reveals.
|
|
65
|
+
while (wi < words.length)
|
|
66
|
+
options?.onSubtitleIndex?.(words[wi++].charEndIndex);
|
|
67
|
+
setPlaying(false);
|
|
68
|
+
setCurrentWord('');
|
|
69
|
+
cleanupListeners();
|
|
70
|
+
socket.release();
|
|
71
|
+
};
|
|
72
|
+
const tick = () => {
|
|
73
|
+
if (streamIdRef.current !== streamId)
|
|
74
|
+
return;
|
|
75
|
+
const t = player.getPlaybackTimeMs();
|
|
76
|
+
while (vi < visemes.length && visemes[vi].startMs <= t) {
|
|
77
|
+
const v = visemes[vi++];
|
|
78
|
+
options?.onViseme?.({
|
|
79
|
+
name: v.name,
|
|
80
|
+
durationMs: v.durationMs,
|
|
81
|
+
intensity: v.intensity,
|
|
82
|
+
});
|
|
83
|
+
}
|
|
84
|
+
while (wi < words.length && words[wi].startMs <= t) {
|
|
85
|
+
options?.onSubtitleIndex?.(words[wi++].charEndIndex);
|
|
86
|
+
}
|
|
87
|
+
// Done when the server finished, the queue drained, and everything
|
|
88
|
+
// scheduled has been dispatched (or playback passed the last event).
|
|
89
|
+
const dispatched = vi >= visemes.length && wi >= words.length;
|
|
90
|
+
if (serverDone && (player.drained || t >= lastEndMs) && dispatched) {
|
|
91
|
+
finish();
|
|
92
|
+
return;
|
|
93
|
+
}
|
|
94
|
+
rafRef.current = requestAnimationFrame(tick);
|
|
95
|
+
};
|
|
29
96
|
await socket.acquire();
|
|
30
97
|
cleanupListeners();
|
|
98
|
+
stopRaf();
|
|
31
99
|
unsubsRef.current.push(socket.on('tts:chunk', (data) => {
|
|
32
100
|
if (data.streamId !== streamId)
|
|
33
101
|
return;
|
|
34
|
-
void
|
|
102
|
+
void player.feedChunk(data.audio);
|
|
103
|
+
if (data.visemes) {
|
|
104
|
+
for (const v of data.visemes) {
|
|
105
|
+
visemes.push(v);
|
|
106
|
+
lastEndMs = Math.max(lastEndMs, v.startMs + v.durationMs);
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
}), socket.on('tts:viseme', (data) => {
|
|
110
|
+
if (data.streamId !== streamId)
|
|
111
|
+
return;
|
|
112
|
+
for (const v of data.visemes) {
|
|
113
|
+
visemes.push(v);
|
|
114
|
+
lastEndMs = Math.max(lastEndMs, v.startMs + v.durationMs);
|
|
115
|
+
}
|
|
35
116
|
}), socket.on('tts:word', (data) => {
|
|
36
117
|
if (data.streamId !== streamId)
|
|
37
118
|
return;
|
|
38
119
|
setCurrentWord(data.word);
|
|
120
|
+
const ce = data.charEndIndex;
|
|
121
|
+
if (typeof ce === 'number') {
|
|
122
|
+
words.push({ charEndIndex: ce, startMs: data.startMs });
|
|
123
|
+
lastEndMs = Math.max(lastEndMs, data.startMs + data.durationMs);
|
|
124
|
+
}
|
|
39
125
|
}), socket.on('tts:done', (data) => {
|
|
40
126
|
if (data.streamId !== streamId)
|
|
41
127
|
return;
|
|
42
|
-
|
|
43
|
-
setCurrentWord('');
|
|
44
|
-
cleanupListeners();
|
|
45
|
-
socket.release();
|
|
128
|
+
serverDone = true;
|
|
46
129
|
}), socket.on('tts:error', (data) => {
|
|
47
130
|
if (data.streamId !== streamId)
|
|
48
131
|
return;
|
|
49
|
-
setPlaying(false);
|
|
50
132
|
setError(data.message);
|
|
51
|
-
|
|
52
|
-
socket.release();
|
|
133
|
+
finish();
|
|
53
134
|
}));
|
|
135
|
+
rafRef.current = requestAnimationFrame(tick);
|
|
54
136
|
socket.send('tts:start', {
|
|
55
137
|
streamId,
|
|
56
138
|
text,
|
|
57
139
|
voice: options?.voice,
|
|
140
|
+
requestVisemes: wantViseme,
|
|
58
141
|
});
|
|
59
142
|
}, [socket]);
|
|
60
143
|
const stop = useCallback(() => {
|
|
61
144
|
if (streamIdRef.current) {
|
|
62
145
|
socket.send('tts:cancel', { streamId: streamIdRef.current });
|
|
63
146
|
}
|
|
147
|
+
streamIdRef.current = null;
|
|
148
|
+
stopRaf();
|
|
64
149
|
playerRef.current?.stop();
|
|
65
150
|
setPlaying(false);
|
|
66
151
|
cleanupListeners();
|
|
@@ -71,11 +156,12 @@ export function useTTS(socket) {
|
|
|
71
156
|
if (streamIdRef.current) {
|
|
72
157
|
socket.send('tts:cancel', { streamId: streamIdRef.current });
|
|
73
158
|
}
|
|
159
|
+
stopRaf();
|
|
74
160
|
cleanupListeners();
|
|
75
161
|
socket.release();
|
|
76
162
|
playerRef.current?.close().catch(() => { });
|
|
77
163
|
};
|
|
78
164
|
}, [socket]);
|
|
79
|
-
return { play, stop, playing, currentWord, error, analyser };
|
|
165
|
+
return { play, stop, warmup, playing, currentWord, error, analyser };
|
|
80
166
|
}
|
|
81
167
|
//# sourceMappingURL=useTTS.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"useTTS.js","sourceRoot":"","sources":["../../../src/client/audio/useTTS.ts"],"names":[],"mappings":"AAAA,6BAA6B;AAC7B,OAAO,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,OAAO,CAAC;
|
|
1
|
+
{"version":3,"file":"useTTS.js","sourceRoot":"","sources":["../../../src/client/audio/useTTS.ts"],"names":[],"mappings":"AAAA,6BAA6B;AAC7B,OAAO,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,OAAO,CAAC;AAGjE,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAwB/C,IAAI,aAAa,GAAG,CAAC,CAAC;AAEtB,MAAM,UAAU,MAAM,CAAC,MAAqB;IAC1C,MAAM,CAAC,OAAO,EAAE,UAAU,CAAC,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;IAC9C,MAAM,CAAC,WAAW,EAAE,cAAc,CAAC,GAAG,QAAQ,CAAC,EAAE,CAAC,CAAC;IACnD,MAAM,CAAC,KAAK,EAAE,QAAQ,CAAC,GAAG,QAAQ,CAAgB,IAAI,CAAC,CAAC;IACxD,yEAAyE;IACzE,qDAAqD;IACrD,MAAM,CAAC,QAAQ,EAAE,WAAW,CAAC,GAAG,QAAQ,CAAsB,IAAI,CAAC,CAAC;IAEpE,MAAM,SAAS,GAAG,MAAM,CAAqB,IAAI,CAAC,CAAC;IACnD,MAAM,WAAW,GAAG,MAAM,CAAgB,IAAI,CAAC,CAAC;IAChD,MAAM,SAAS,GAAG,MAAM,CAAiB,EAAE,CAAC,CAAC;IAC7C,MAAM,MAAM,GAAG,MAAM,CAAgB,IAAI,CAAC,CAAC;IAE3C,SAAS,gBAAgB;QACvB,KAAK,MAAM,KAAK,IAAI,SAAS,CAAC,OAAO;YAAE,KAAK,EAAE,CAAC;QAC/C,SAAS,CAAC,OAAO,GAAG,EAAE,CAAC;IACzB,CAAC;IAED,SAAS,OAAO;QACd,IAAI,MAAM,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;YAC5B,oBAAoB,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YACrC,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC;QACxB,CAAC;IACH,CAAC;IAED,wEAAwE;IACxE,6EAA6E;IAC7E,6EAA6E;IAC7E,gEAAgE;IAChE,MAAM,MAAM,GAAG,WAAW,CAAC,KAAK,IAAI,EAAE;QACpC,IAAI,CAAC,SAAS,CAAC,OAAO;YAAE,SAAS,CAAC,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC;QAC9D,MAAM,SAAS,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QAC/B,MAAM,SAAS,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;QACjC,WAAW,CAAC,SAAS,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;IAC/C,CAAC,EAAE,EAAE,CAAC,CAAC;IAEP,MAAM,IAAI,GAAG,WAAW,CACtB,KAAK,EAAE,IAAY,EAAE,OAAoB,EAAE,EAAE;QAC3C,MAAM,QAAQ,GAAG,OAAO,EAAE,aAAa,EAAE,CAAC;QAC1C,WAAW,CAAC,OAAO,GAAG,QAAQ,CAAC;QAE/B,IAAI,CAAC,SAAS,CAAC,OAAO;YAAE,SAAS,CAAC,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC;QAC9D,MAAM,MAAM,GAAG,SAAS,CAAC,OAAO,CAAC;QACjC,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;QACpB,MAAM,CAAC,KAAK,EAAE,CAAC;QACf,WAAW,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC;QAElC,UAAU,CAAC,IAAI,CAAC,CAAC;QACjB,cAAc,CAAC,EAAE,CAAC,CAAC;QACnB,QAAQ,CAAC,IAAI,CAAC,CAAC;QAEf,wEAAwE;QACxE,iEAAiE;QACjE,MAAM,OAAO,GAAgB,EAAE,CAAC;QAChC,MAAM,KAAK,GAAgD,EAAE,CAAC;QAC9D,IAAI,EAAE,GAAG,CAAC,CAAC;QACX,IAAI,EAAE,GAAG,CAAC,CAAC;QACX,IAAI,UAAU,GAAG,KAAK,CAAC;QACvB,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,MAAM,UAAU,GAAG,CAAC,CAAC,OAAO,EAAE,QAAQ,CAAC;QAEvC,MAAM,MAAM,GAAG,GAAG,EAAE;YAClB,IAAI,WAAW,CAAC,OAAO,KAAK,QAAQ;gBAAE,OAAO;YAC7C,WAAW,CAAC,OAAO,GAAG,IAAI,CAAC;YAC3B,OAAO,EAAE,CAAC;YACV,iEAAiE;YACjE,OAAO,EAAE,GAAG,KAAK,CAAC,MAAM;gBAAE,OAAO,EAAE,eAAe,EAAE,CAAC,KAAK,CAAC,EAAE,EAAE,CAAE,CAAC,YAAY,CAAC,CAAC;YAChF,UAAU,CAAC,KAAK,CAAC,CAAC;YAClB,cAAc,CAAC,EAAE,CAAC,CAAC;YACnB,gBAAgB,EAAE,CAAC;YACnB,MAAM,CAAC,OAAO,EAAE,CAAC;QACnB,CAAC,CAAC;QAEF,MAAM,IAAI,GAAG,GAAG,EAAE;YAChB,IAAI,WAAW,CAAC,OAAO,KAAK,QAAQ;gBAAE,OAAO;YAC7C,MAAM,CAAC,GAAG,MAAM,CAAC,iBAAiB,EAAE,CAAC;YACrC,OAAO,EAAE,GAAG,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,EAAE,CAAE,CAAC,OAAO,IAAI,CAAC,EAAE,CAAC;gBACxD,MAAM,CAAC,GAAG,OAAO,CAAC,EAAE,EAAE,CAAE,CAAC;gBACzB,OAAO,EAAE,QAAQ,EAAE,CAAC;oBAClB,IAAI,EAAE,CAAC,CAAC,IAAI;oBACZ,UAAU,EAAE,CAAC,CAAC,UAAU;oBACxB,SAAS,EAAE,CAAC,CAAC,SAAS;iBACvB,CAAC,CAAC;YACL,CAAC;YACD,OAAO,EAAE,GAAG,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,EAAE,CAAE,CAAC,OAAO,IAAI,CAAC,EAAE,CAAC;gBACpD,OAAO,EAAE,eAAe,EAAE,CAAC,KAAK,CAAC,EAAE,EAAE,CAAE,CAAC,YAAY,CAAC,CAAC;YACxD,CAAC;YACD,mEAAmE;YACnE,qEAAqE;YACrE,MAAM,UAAU,GAAG,EAAE,IAAI,OAAO,CAAC,MAAM,IAAI,EAAE,IAAI,KAAK,CAAC,MAAM,CAAC;YAC9D,IAAI,UAAU,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,CAAC,IAAI,SAAS,CAAC,IAAI,UAAU,EAAE,CAAC;gBACnE,MAAM,EAAE,CAAC;gBACT,OAAO;YACT,CAAC;YACD,MAAM,CAAC,OAAO,GAAG,qBAAqB,CAAC,IAAI,CAAC,CAAC;QAC/C,CAAC,CAAC;QAEF,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;QAEvB,gBAAgB,EAAE,CAAC;QACnB,OAAO,EAAE,CAAC;QACV,SAAS,CAAC,OAAO,CAAC,IAAI,CACpB,MAAM,CAAC,EAAE,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,EAAE;YAC9B,IAAI,IAAI,CAAC,QAAQ,KAAK,QAAQ;gBAAE,OAAO;YACvC,KAAK,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAClC,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;gBACjB,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;oBAC7B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;oBAChB,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC;gBAC5D,CAAC;YACH,CAAC;QACH,CAAC,CAAC,EACF,MAAM,CAAC,EAAE,CAAC,YAAY,EAAE,CAAC,IAAI,EAAE,EAAE;YAC/B,IAAI,IAAI,CAAC,QAAQ,KAAK,QAAQ;gBAAE,OAAO;YACvC,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;gBAC7B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBAChB,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC;YAC5D,CAAC;QACH,CAAC,CAAC,EACF,MAAM,CAAC,EAAE,CAAC,UAAU,EAAE,CAAC,IAAI,EAAE,EAAE;YAC7B,IAAI,IAAI,CAAC,QAAQ,KAAK,QAAQ;gBAAE,OAAO;YACvC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC1B,MAAM,EAAE,GAAG,IAAI,CAAC,YAAY,CAAC;YAC7B,IAAI,OAAO,EAAE,KAAK,QAAQ,EAAE,CAAC;gBAC3B,KAAK,CAAC,IAAI,CAAC,EAAE,YAAY,EAAE,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;gBACxD,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC;YAClE,CAAC;QACH,CAAC,CAAC,EACF,MAAM,CAAC,EAAE,CAAC,UAAU,EAAE,CAAC,IAAI,EAAE,EAAE;YAC7B,IAAI,IAAI,CAAC,QAAQ,KAAK,QAAQ;gBAAE,OAAO;YACvC,UAAU,GAAG,IAAI,CAAC;QACpB,CAAC,CAAC,EACF,MAAM,CAAC,EAAE,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,EAAE;YAC9B,IAAI,IAAI,CAAC,QAAQ,KAAK,QAAQ;gBAAE,OAAO;YACvC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACvB,MAAM,EAAE,CAAC;QACX,CAAC,CAAC,CACH,CAAC;QACF,MAAM,CAAC,OAAO,GAAG,qBAAqB,CAAC,IAAI,CAAC,CAAC;QAE7C,MAAM,CAAC,IAAI,CAAC,WAAW,EAAE;YACvB,QAAQ;YACR,IAAI;YACJ,KAAK,EAAE,OAAO,EAAE,KAAK;YACrB,cAAc,EAAE,UAAU;SAC3B,CAAC,CAAC;IACL,CAAC,EACD,CAAC,MAAM,CAAC,CACT,CAAC;IAEF,MAAM,IAAI,GAAG,WAAW,CAAC,GAAG,EAAE;QAC5B,IAAI,WAAW,CAAC,OAAO,EAAE,CAAC;YACxB,MAAM,CAAC,IAAI,CAAC,YAAY,EAAE,EAAE,QAAQ,EAAE,WAAW,CAAC,OAAO,EAAE,CAAC,CAAC;QAC/D,CAAC;QACD,WAAW,CAAC,OAAO,GAAG,IAAI,CAAC;QAC3B,OAAO,EAAE,CAAC;QACV,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC;QAC1B,UAAU,CAAC,KAAK,CAAC,CAAC;QAClB,gBAAgB,EAAE,CAAC;QACnB,MAAM,CAAC,OAAO,EAAE,CAAC;IACnB,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC;IAEb,SAAS,CAAC,GAAG,EAAE;QACb,OAAO,GAAG,EAAE;YACV,IAAI,WAAW,CAAC,OAAO,EAAE,CAAC;gBACxB,MAAM,CAAC,IAAI,CAAC,YAAY,EAAE,EAAE,QAAQ,EAAE,WAAW,CAAC,OAAO,EAAE,CAAC,CAAC;YAC/D,CAAC;YACD,OAAO,EAAE,CAAC;YACV,gBAAgB,EAAE,CAAC;YACnB,MAAM,CAAC,OAAO,EAAE,CAAC;YACjB,SAAS,CAAC,OAAO,EAAE,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAC7C,CAAC,CAAC;IACJ,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC;IAEb,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;AACvE,CAAC"}
|
package/dist/client/index.d.ts
CHANGED
|
@@ -27,6 +27,7 @@ export { BlobVisualizerCore } from './audio/BlobVisualizerCore.js';
|
|
|
27
27
|
export { MicVisualizer } from './audio/MicVisualizer.js';
|
|
28
28
|
export { useSTT } from './audio/useSTT.js';
|
|
29
29
|
export { useTTS } from './audio/useTTS.js';
|
|
30
|
+
export type { TTSVisemeEvent } from './audio/useTTS.js';
|
|
30
31
|
export { WaveVisualizer } from './audio/WaveVisualizer.js';
|
|
31
32
|
export { callImageGen, callJsonGen, callTextGen } from './callAI.js';
|
|
32
33
|
export { uploadBlob, promoteBlob, downscaleImage } from './uploadBlob.js';
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/client/index.ts"],"names":[],"mappings":"AAAA,YAAY,EACV,WAAW,EACX,mBAAmB,EACnB,aAAa,EACb,YAAY,EACZ,OAAO,EACP,kBAAkB,EAClB,YAAY,GACb,MAAM,oBAAoB,CAAC;AAC5B,YAAY,EACV,UAAU,EACV,aAAa,EACb,SAAS,EACT,OAAO,EACP,UAAU,EACV,YAAY,EACZ,SAAS,EACT,cAAc,EACd,WAAW,EACX,cAAc,EACd,cAAc,GACf,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,QAAQ,EAAE,MAAM,yBAAyB,CAAC;AACnD,YAAY,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAC;AAC7D,OAAO,EACL,mBAAmB,EACnB,eAAe,EACf,gBAAgB,EAChB,uBAAuB,GACxB,MAAM,8BAA8B,CAAC;AACtC,YAAY,EACV,gBAAgB,EAChB,cAAc,EACd,gBAAgB,GACjB,MAAM,8BAA8B,CAAC;AACtC,OAAO,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,qBAAqB,EAAE,MAAM,sCAAsC,CAAC;AACzH,YAAY,EAAE,eAAe,EAAE,cAAc,EAAE,gBAAgB,EAAE,MAAM,sCAAsC,CAAC;AAC9G,OAAO,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,MAAM,mCAAmC,CAAC;AAC5F,YAAY,EAAE,qBAAqB,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,MAAM,mCAAmC,CAAC;AAC1H,OAAO,EAAE,MAAM,EAAE,MAAM,wBAAwB,CAAC;AAChD,OAAO,EAAE,eAAe,EAAE,MAAM,iCAAiC,CAAC;AAClE,OAAO,EAAE,cAAc,EAAE,MAAM,gCAAgC,CAAC;AAChE,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AACrF,YAAY,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,EACL,eAAe,EACf,iBAAiB,EACjB,OAAO,EACP,UAAU,GACX,MAAM,sBAAsB,CAAC;AAC9B,YAAY,EACV,mBAAmB,EACnB,qBAAqB,GACtB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AACrD,OAAO,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AACzD,YAAY,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAC;AAClE,OAAO,EAAE,iBAAiB,EAAE,MAAM,8BAA8B,CAAC;AACjE,OAAO,EACL,gBAAgB,EAChB,iBAAiB,EACjB,0BAA0B,EAC1B,WAAW,EACX,QAAQ,GACT,MAAM,sBAAsB,CAAC;AAC9B,YAAY,EACV,UAAU,EACV,eAAe,EACf,kBAAkB,EAClB,WAAW,EACX,SAAS,EACT,aAAa,GACd,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,YAAY,EACV,uBAAuB,EACvB,mBAAmB,GACpB,MAAM,2BAA2B,CAAC;AACnC,OAAO,EAAE,kBAAkB,EAAE,MAAM,+BAA+B,CAAC;AACnE,OAAO,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AACzD,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AACrE,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAC1E,YAAY,EAAE,YAAY,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AACvE,OAAO,EAAE,cAAc,EAAE,MAAM,gCAAgC,CAAC;AAChE,cAAc,uBAAuB,CAAC;AACtC,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,YAAY,EACV,aAAa,EACb,uBAAuB,GACxB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACjD,YAAY,EAAE,SAAS,EAAE,eAAe,EAAE,mBAAmB,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACzG,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AACzD,YAAY,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACxD,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACjD,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,EACL,oBAAoB,EACpB,kBAAkB,EAClB,kBAAkB,GACnB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AACjE,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AAC3E,YAAY,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AAC9D,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC3E,YAAY,EACV,WAAW,EACX,eAAe,EACf,YAAY,EACZ,kBAAkB,GACnB,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AACvE,YAAY,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AACnE,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,YAAY,EACV,cAAc,EACd,QAAQ,EACR,YAAY,EACZ,WAAW,EACX,aAAa,EACb,YAAY,EACZ,UAAU,EACV,eAAe,EACf,iBAAiB,EACjB,iBAAiB,EACjB,iBAAiB,GAClB,MAAM,oBAAoB,CAAC;AAC5B,YAAY,EAAE,qBAAqB,EAAE,MAAM,kBAAkB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/client/index.ts"],"names":[],"mappings":"AAAA,YAAY,EACV,WAAW,EACX,mBAAmB,EACnB,aAAa,EACb,YAAY,EACZ,OAAO,EACP,kBAAkB,EAClB,YAAY,GACb,MAAM,oBAAoB,CAAC;AAC5B,YAAY,EACV,UAAU,EACV,aAAa,EACb,SAAS,EACT,OAAO,EACP,UAAU,EACV,YAAY,EACZ,SAAS,EACT,cAAc,EACd,WAAW,EACX,cAAc,EACd,cAAc,GACf,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,QAAQ,EAAE,MAAM,yBAAyB,CAAC;AACnD,YAAY,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAC;AAC7D,OAAO,EACL,mBAAmB,EACnB,eAAe,EACf,gBAAgB,EAChB,uBAAuB,GACxB,MAAM,8BAA8B,CAAC;AACtC,YAAY,EACV,gBAAgB,EAChB,cAAc,EACd,gBAAgB,GACjB,MAAM,8BAA8B,CAAC;AACtC,OAAO,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,qBAAqB,EAAE,MAAM,sCAAsC,CAAC;AACzH,YAAY,EAAE,eAAe,EAAE,cAAc,EAAE,gBAAgB,EAAE,MAAM,sCAAsC,CAAC;AAC9G,OAAO,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,MAAM,mCAAmC,CAAC;AAC5F,YAAY,EAAE,qBAAqB,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,MAAM,mCAAmC,CAAC;AAC1H,OAAO,EAAE,MAAM,EAAE,MAAM,wBAAwB,CAAC;AAChD,OAAO,EAAE,eAAe,EAAE,MAAM,iCAAiC,CAAC;AAClE,OAAO,EAAE,cAAc,EAAE,MAAM,gCAAgC,CAAC;AAChE,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AACrF,YAAY,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,EACL,eAAe,EACf,iBAAiB,EACjB,OAAO,EACP,UAAU,GACX,MAAM,sBAAsB,CAAC;AAC9B,YAAY,EACV,mBAAmB,EACnB,qBAAqB,GACtB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AACrD,OAAO,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AACzD,YAAY,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAC;AAClE,OAAO,EAAE,iBAAiB,EAAE,MAAM,8BAA8B,CAAC;AACjE,OAAO,EACL,gBAAgB,EAChB,iBAAiB,EACjB,0BAA0B,EAC1B,WAAW,EACX,QAAQ,GACT,MAAM,sBAAsB,CAAC;AAC9B,YAAY,EACV,UAAU,EACV,eAAe,EACf,kBAAkB,EAClB,WAAW,EACX,SAAS,EACT,aAAa,GACd,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,YAAY,EACV,uBAAuB,EACvB,mBAAmB,GACpB,MAAM,2BAA2B,CAAC;AACnC,OAAO,EAAE,kBAAkB,EAAE,MAAM,+BAA+B,CAAC;AACnE,OAAO,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AACzD,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,YAAY,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACxD,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AACrE,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAC1E,YAAY,EAAE,YAAY,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AACvE,OAAO,EAAE,cAAc,EAAE,MAAM,gCAAgC,CAAC;AAChE,cAAc,uBAAuB,CAAC;AACtC,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,YAAY,EACV,aAAa,EACb,uBAAuB,GACxB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACjD,YAAY,EAAE,SAAS,EAAE,eAAe,EAAE,mBAAmB,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACzG,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AACzD,YAAY,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACxD,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACjD,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,EACL,oBAAoB,EACpB,kBAAkB,EAClB,kBAAkB,GACnB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AACjE,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AAC3E,YAAY,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AAC9D,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC3E,YAAY,EACV,WAAW,EACX,eAAe,EACf,YAAY,EACZ,kBAAkB,GACnB,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AACvE,YAAY,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AACnE,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,YAAY,EACV,cAAc,EACd,QAAQ,EACR,YAAY,EACZ,WAAW,EACX,aAAa,EACb,YAAY,EACZ,UAAU,EACV,eAAe,EACf,iBAAiB,EACjB,iBAAiB,EACjB,iBAAiB,GAClB,MAAM,oBAAoB,CAAC;AAC5B,YAAY,EAAE,qBAAqB,EAAE,MAAM,kBAAkB,CAAC"}
|
package/dist/client/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/client/index.ts"],"names":[],"mappings":"AAsBA,OAAO,EAAE,QAAQ,EAAE,MAAM,yBAAyB,CAAC;AAEnD,OAAO,EACL,mBAAmB,EACnB,eAAe,EACf,gBAAgB,EAChB,uBAAuB,GACxB,MAAM,8BAA8B,CAAC;AAMtC,OAAO,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,qBAAqB,EAAE,MAAM,sCAAsC,CAAC;AAEzH,OAAO,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,MAAM,mCAAmC,CAAC;AAE5F,OAAO,EAAE,MAAM,EAAE,MAAM,wBAAwB,CAAC;AAChD,OAAO,EAAE,eAAe,EAAE,MAAM,iCAAiC,CAAC;AAClE,OAAO,EAAE,cAAc,EAAE,MAAM,gCAAgC,CAAC;AAChE,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAErF,OAAO,EACL,eAAe,EACf,iBAAiB,EACjB,OAAO,EACP,UAAU,GACX,MAAM,sBAAsB,CAAC;AAK9B,OAAO,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AACrD,OAAO,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AAEzD,OAAO,EAAE,iBAAiB,EAAE,MAAM,8BAA8B,CAAC;AACjE,OAAO,EACL,gBAAgB,EAChB,iBAAiB,EACjB,0BAA0B,EAC1B,WAAW,EACX,QAAQ,GACT,MAAM,sBAAsB,CAAC;AAS9B,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAK3D,OAAO,EAAE,kBAAkB,EAAE,MAAM,+BAA+B,CAAC;AACnE,OAAO,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AACzD,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/client/index.ts"],"names":[],"mappings":"AAsBA,OAAO,EAAE,QAAQ,EAAE,MAAM,yBAAyB,CAAC;AAEnD,OAAO,EACL,mBAAmB,EACnB,eAAe,EACf,gBAAgB,EAChB,uBAAuB,GACxB,MAAM,8BAA8B,CAAC;AAMtC,OAAO,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,qBAAqB,EAAE,MAAM,sCAAsC,CAAC;AAEzH,OAAO,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,MAAM,mCAAmC,CAAC;AAE5F,OAAO,EAAE,MAAM,EAAE,MAAM,wBAAwB,CAAC;AAChD,OAAO,EAAE,eAAe,EAAE,MAAM,iCAAiC,CAAC;AAClE,OAAO,EAAE,cAAc,EAAE,MAAM,gCAAgC,CAAC;AAChE,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAErF,OAAO,EACL,eAAe,EACf,iBAAiB,EACjB,OAAO,EACP,UAAU,GACX,MAAM,sBAAsB,CAAC;AAK9B,OAAO,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AACrD,OAAO,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AAEzD,OAAO,EAAE,iBAAiB,EAAE,MAAM,8BAA8B,CAAC;AACjE,OAAO,EACL,gBAAgB,EAChB,iBAAiB,EACjB,0BAA0B,EAC1B,WAAW,EACX,QAAQ,GACT,MAAM,sBAAsB,CAAC;AAS9B,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAK3D,OAAO,EAAE,kBAAkB,EAAE,MAAM,+BAA+B,CAAC;AACnE,OAAO,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AACzD,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAE3C,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AACrE,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAE1E,OAAO,EAAE,cAAc,EAAE,MAAM,gCAAgC,CAAC;AAChE,cAAc,uBAAuB,CAAC;AACtC,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAKzD,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAEjD,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAEzD,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACjD,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,EACL,oBAAoB,EACpB,kBAAkB,EAClB,kBAAkB,GACnB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AACjE,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AAE3E,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAO3E,OAAO,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AAEvE,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC"}
|
package/package.json
CHANGED
package/src/cli/version.ts
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
// Auto-generated by prebuild — do not edit manually
|
|
2
|
-
export const CLI_VERSION = "0.1.
|
|
2
|
+
export const CLI_VERSION = "0.1.563";
|
|
@@ -6,12 +6,16 @@ class PCM16Player extends AudioWorkletProcessor {
|
|
|
6
6
|
this.queue = [];
|
|
7
7
|
this.buffer = new Float32Array(0);
|
|
8
8
|
this.offset = 0;
|
|
9
|
+
this.played = 0; // total real samples emitted (the playback clock)
|
|
10
|
+
this.tick = 0;
|
|
11
|
+
this.draining = false;
|
|
9
12
|
this.port.onmessage = (event) => this.handleMessage(event);
|
|
10
13
|
}
|
|
11
14
|
process(_inputs, outputs) {
|
|
12
15
|
const output = outputs[0]?.[0];
|
|
13
16
|
if (!output) return true;
|
|
14
17
|
let writePos = 0;
|
|
18
|
+
let real = 0;
|
|
15
19
|
while (writePos < output.length) {
|
|
16
20
|
if (this.offset >= this.buffer.length) {
|
|
17
21
|
const next = this.queue.shift();
|
|
@@ -25,6 +29,19 @@ class PCM16Player extends AudioWorkletProcessor {
|
|
|
25
29
|
output.set(this.buffer.subarray(this.offset, this.offset + toCopy), writePos);
|
|
26
30
|
this.offset += toCopy;
|
|
27
31
|
writePos += toCopy;
|
|
32
|
+
real += toCopy;
|
|
33
|
+
}
|
|
34
|
+
if (real > 0) {
|
|
35
|
+
this.played += real;
|
|
36
|
+
this.draining = true;
|
|
37
|
+
// Report the playback clock ~40×/s so the main thread can schedule
|
|
38
|
+
// visemes/captions against actual audio position.
|
|
39
|
+
if (++this.tick % 6 === 0) this.port.postMessage({ type: 'time', played: this.played });
|
|
40
|
+
} else if (this.draining) {
|
|
41
|
+
// Queue ran dry after playing — playback finished.
|
|
42
|
+
this.draining = false;
|
|
43
|
+
this.port.postMessage({ type: 'time', played: this.played });
|
|
44
|
+
this.port.postMessage({ type: 'drained' });
|
|
28
45
|
}
|
|
29
46
|
return true;
|
|
30
47
|
}
|
|
@@ -35,11 +52,16 @@ class PCM16Player extends AudioWorkletProcessor {
|
|
|
35
52
|
this.queue = [];
|
|
36
53
|
this.buffer = new Float32Array(0);
|
|
37
54
|
this.offset = 0;
|
|
55
|
+
this.played = 0;
|
|
56
|
+
this.draining = false;
|
|
38
57
|
}
|
|
39
58
|
}
|
|
40
59
|
}
|
|
41
60
|
registerProcessor('pcm16-player', PCM16Player);
|
|
42
61
|
`;
|
|
62
|
+
|
|
63
|
+
/** TTS audio is always mono PCM16 @ 24 kHz (see TTSChunkMsg). */
|
|
64
|
+
const TTS_SAMPLE_RATE = 24000;
|
|
43
65
|
const WORKLET_BLOB = new Blob([WORKLET_CODE], { type: 'application/javascript' });
|
|
44
66
|
const WORKLET_URL = URL.createObjectURL(WORKLET_BLOB);
|
|
45
67
|
|
|
@@ -57,6 +79,10 @@ export class AudioPlayer {
|
|
|
57
79
|
private workletNode: AudioWorkletNode | null = null;
|
|
58
80
|
private analyser: AnalyserNode | null = null;
|
|
59
81
|
private _playing = false;
|
|
82
|
+
// Playback clock fed by the worklet (samples actually emitted). Used to
|
|
83
|
+
// schedule visemes + caption indices against real audio position.
|
|
84
|
+
private playedSamples = 0;
|
|
85
|
+
private _drained = false;
|
|
60
86
|
|
|
61
87
|
get playing(): boolean {
|
|
62
88
|
return this._playing;
|
|
@@ -70,22 +96,67 @@ export class AudioPlayer {
|
|
|
70
96
|
return this.analyser;
|
|
71
97
|
}
|
|
72
98
|
|
|
99
|
+
/** Milliseconds of audio actually played so far (the lip-sync/caption clock). */
|
|
100
|
+
getPlaybackTimeMs(): number {
|
|
101
|
+
return (this.playedSamples / TTS_SAMPLE_RATE) * 1000;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/** True once the queue has drained after playing (stream finished). */
|
|
105
|
+
get drained(): boolean {
|
|
106
|
+
return this._drained;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/** Reset the playback clock for a new stream. */
|
|
110
|
+
reset(): void {
|
|
111
|
+
this.workletNode?.port.postMessage({ type: 'stop' });
|
|
112
|
+
this.playedSamples = 0;
|
|
113
|
+
this._drained = false;
|
|
114
|
+
this._playing = false;
|
|
115
|
+
}
|
|
116
|
+
|
|
73
117
|
async init(): Promise<void> {
|
|
74
118
|
if (this.workletNode) return;
|
|
75
|
-
|
|
119
|
+
// Match the TTS sample rate so PCM16 plays at correct pitch (the worklet
|
|
120
|
+
// outputs 1:1, no resampling) and the playback clock is accurate.
|
|
121
|
+
this.ctx = new AudioContext({ sampleRate: TTS_SAMPLE_RATE });
|
|
76
122
|
await this.ctx.audioWorklet.addModule(WORKLET_URL);
|
|
77
123
|
this.workletNode = new AudioWorkletNode(this.ctx, 'pcm16-player');
|
|
124
|
+
this.workletNode.port.onmessage = (e: MessageEvent) => {
|
|
125
|
+
const d = e.data as { type?: string; played?: number };
|
|
126
|
+
if (d?.type === 'time' && typeof d.played === 'number') {
|
|
127
|
+
this.playedSamples = d.played;
|
|
128
|
+
} else if (d?.type === 'drained') {
|
|
129
|
+
this._drained = true;
|
|
130
|
+
this._playing = false;
|
|
131
|
+
}
|
|
132
|
+
};
|
|
78
133
|
this.analyser = this.ctx.createAnalyser();
|
|
79
134
|
this.analyser.fftSize = 256;
|
|
80
135
|
this.analyser.smoothingTimeConstant = 0.6;
|
|
81
136
|
// worklet → analyser → destination (analyser is a pass-through tap).
|
|
82
137
|
this.workletNode.connect(this.analyser);
|
|
83
138
|
this.analyser.connect(this.ctx.destination);
|
|
139
|
+
// A context created outside a user gesture starts suspended → no audio is
|
|
140
|
+
// ever heard and the analyser reads silence. Resume it (the page has
|
|
141
|
+
// already had user gestures by the time TTS plays).
|
|
142
|
+
await this.resume();
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
/** Resume the AudioContext if the browser autoplay policy suspended it. */
|
|
146
|
+
async resume(): Promise<void> {
|
|
147
|
+
if (this.ctx && this.ctx.state === 'suspended') {
|
|
148
|
+
try {
|
|
149
|
+
await this.ctx.resume();
|
|
150
|
+
} catch {
|
|
151
|
+
/* will retry on the next chunk */
|
|
152
|
+
}
|
|
153
|
+
}
|
|
84
154
|
}
|
|
85
155
|
|
|
86
156
|
async feedChunk(base64Audio: string): Promise<void> {
|
|
87
157
|
await this.init();
|
|
88
158
|
if (!this.workletNode || !this.ctx) return;
|
|
159
|
+
if (this.ctx.state === 'suspended') await this.resume();
|
|
89
160
|
|
|
90
161
|
const binary = atob(base64Audio);
|
|
91
162
|
const bytes = new Uint8Array(binary.length);
|
|
@@ -93,6 +164,7 @@ export class AudioPlayer {
|
|
|
93
164
|
|
|
94
165
|
const samples = pcm16ToFloat32(bytes.buffer);
|
|
95
166
|
this._playing = true;
|
|
167
|
+
this._drained = false;
|
|
96
168
|
this.workletNode.port.postMessage({ type: 'chunk', samples }, [
|
|
97
169
|
samples.buffer,
|
|
98
170
|
]);
|
|
@@ -1,10 +1,29 @@
|
|
|
1
1
|
// src/client/audio/useTTS.ts
|
|
2
2
|
import { useCallback, useEffect, useRef, useState } from 'react';
|
|
3
|
+
import type { TTSViseme, TTSVisemeName } from '../../shared/Audio.js';
|
|
3
4
|
import type { UglyBotSocket } from '../uglyBotSocket.js';
|
|
4
5
|
import { AudioPlayer } from './AudioPlayer.js';
|
|
5
6
|
|
|
7
|
+
/** A viseme delivered to the avatar (timing stripped — driven by the audio clock). */
|
|
8
|
+
export interface TTSVisemeEvent {
|
|
9
|
+
name: TTSVisemeName;
|
|
10
|
+
durationMs: number;
|
|
11
|
+
intensity: number;
|
|
12
|
+
}
|
|
13
|
+
|
|
6
14
|
interface TTSOptions {
|
|
7
15
|
voice?: string;
|
|
16
|
+
/**
|
|
17
|
+
* Fired as each viseme reaches the current audio-playback position. Wire to
|
|
18
|
+
* `AvatarModel.showViseme(...)` for true InWorld lip-sync.
|
|
19
|
+
*/
|
|
20
|
+
onViseme?: (v: TTSVisemeEvent) => void;
|
|
21
|
+
/**
|
|
22
|
+
* Fired as each spoken word reaches the playback position — value is the
|
|
23
|
+
* character offset of the word's end in the original text. Reveal the caption
|
|
24
|
+
* up to this index for karaoke-style subtitles.
|
|
25
|
+
*/
|
|
26
|
+
onSubtitleIndex?: (charIndex: number) => void;
|
|
8
27
|
}
|
|
9
28
|
|
|
10
29
|
let streamCounter = 0;
|
|
@@ -13,63 +32,147 @@ export function useTTS(socket: UglyBotSocket) {
|
|
|
13
32
|
const [playing, setPlaying] = useState(false);
|
|
14
33
|
const [currentWord, setCurrentWord] = useState('');
|
|
15
34
|
const [error, setError] = useState<string | null>(null);
|
|
16
|
-
// Real-time amplitude analyser of the TTS playback —
|
|
35
|
+
// Real-time amplitude analyser of the TTS playback — a fallback lip-sync
|
|
36
|
+
// signal when viseme data isn't requested/available.
|
|
17
37
|
const [analyser, setAnalyser] = useState<AnalyserNode | null>(null);
|
|
18
38
|
|
|
19
39
|
const playerRef = useRef<AudioPlayer | null>(null);
|
|
20
40
|
const streamIdRef = useRef<string | null>(null);
|
|
21
41
|
const unsubsRef = useRef<(() => void)[]>([]);
|
|
42
|
+
const rafRef = useRef<number | null>(null);
|
|
22
43
|
|
|
23
44
|
function cleanupListeners() {
|
|
24
45
|
for (const unsub of unsubsRef.current) unsub();
|
|
25
46
|
unsubsRef.current = [];
|
|
26
47
|
}
|
|
27
48
|
|
|
49
|
+
function stopRaf() {
|
|
50
|
+
if (rafRef.current !== null) {
|
|
51
|
+
cancelAnimationFrame(rafRef.current);
|
|
52
|
+
rafRef.current = null;
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
// Create + resume the AudioContext from inside a user gesture (e.g. the
|
|
57
|
+
// user's send-click). Browsers (esp. Safari) only unlock audio when resume()
|
|
58
|
+
// runs within a gesture; the TTS reply itself plays later, async, so without
|
|
59
|
+
// this warmup the context stays suspended and nothing is heard.
|
|
60
|
+
const warmup = useCallback(async () => {
|
|
61
|
+
if (!playerRef.current) playerRef.current = new AudioPlayer();
|
|
62
|
+
await playerRef.current.init();
|
|
63
|
+
await playerRef.current.resume();
|
|
64
|
+
setAnalyser(playerRef.current.getAnalyser());
|
|
65
|
+
}, []);
|
|
66
|
+
|
|
28
67
|
const play = useCallback(
|
|
29
68
|
async (text: string, options?: TTSOptions) => {
|
|
30
69
|
const streamId = `tts-${++streamCounter}`;
|
|
31
70
|
streamIdRef.current = streamId;
|
|
32
71
|
|
|
33
72
|
if (!playerRef.current) playerRef.current = new AudioPlayer();
|
|
34
|
-
|
|
35
|
-
|
|
73
|
+
const player = playerRef.current;
|
|
74
|
+
await player.init();
|
|
75
|
+
player.reset();
|
|
76
|
+
setAnalyser(player.getAnalyser());
|
|
36
77
|
|
|
37
78
|
setPlaying(true);
|
|
38
79
|
setCurrentWord('');
|
|
39
80
|
setError(null);
|
|
40
81
|
|
|
82
|
+
// Collect viseme/word timing as it streams; an rAF loop dispatches each
|
|
83
|
+
// against the real audio-playback clock (collect-then-schedule).
|
|
84
|
+
const visemes: TTSViseme[] = [];
|
|
85
|
+
const words: { charEndIndex: number; startMs: number }[] = [];
|
|
86
|
+
let vi = 0;
|
|
87
|
+
let wi = 0;
|
|
88
|
+
let serverDone = false;
|
|
89
|
+
let lastEndMs = 0;
|
|
90
|
+
const wantViseme = !!options?.onViseme;
|
|
91
|
+
|
|
92
|
+
const finish = () => {
|
|
93
|
+
if (streamIdRef.current !== streamId) return;
|
|
94
|
+
streamIdRef.current = null;
|
|
95
|
+
stopRaf();
|
|
96
|
+
// Flush any remaining caption indices so the text fully reveals.
|
|
97
|
+
while (wi < words.length) options?.onSubtitleIndex?.(words[wi++]!.charEndIndex);
|
|
98
|
+
setPlaying(false);
|
|
99
|
+
setCurrentWord('');
|
|
100
|
+
cleanupListeners();
|
|
101
|
+
socket.release();
|
|
102
|
+
};
|
|
103
|
+
|
|
104
|
+
const tick = () => {
|
|
105
|
+
if (streamIdRef.current !== streamId) return;
|
|
106
|
+
const t = player.getPlaybackTimeMs();
|
|
107
|
+
while (vi < visemes.length && visemes[vi]!.startMs <= t) {
|
|
108
|
+
const v = visemes[vi++]!;
|
|
109
|
+
options?.onViseme?.({
|
|
110
|
+
name: v.name,
|
|
111
|
+
durationMs: v.durationMs,
|
|
112
|
+
intensity: v.intensity,
|
|
113
|
+
});
|
|
114
|
+
}
|
|
115
|
+
while (wi < words.length && words[wi]!.startMs <= t) {
|
|
116
|
+
options?.onSubtitleIndex?.(words[wi++]!.charEndIndex);
|
|
117
|
+
}
|
|
118
|
+
// Done when the server finished, the queue drained, and everything
|
|
119
|
+
// scheduled has been dispatched (or playback passed the last event).
|
|
120
|
+
const dispatched = vi >= visemes.length && wi >= words.length;
|
|
121
|
+
if (serverDone && (player.drained || t >= lastEndMs) && dispatched) {
|
|
122
|
+
finish();
|
|
123
|
+
return;
|
|
124
|
+
}
|
|
125
|
+
rafRef.current = requestAnimationFrame(tick);
|
|
126
|
+
};
|
|
127
|
+
|
|
41
128
|
await socket.acquire();
|
|
42
129
|
|
|
43
130
|
cleanupListeners();
|
|
131
|
+
stopRaf();
|
|
44
132
|
unsubsRef.current.push(
|
|
45
133
|
socket.on('tts:chunk', (data) => {
|
|
46
134
|
if (data.streamId !== streamId) return;
|
|
47
|
-
void
|
|
135
|
+
void player.feedChunk(data.audio);
|
|
136
|
+
if (data.visemes) {
|
|
137
|
+
for (const v of data.visemes) {
|
|
138
|
+
visemes.push(v);
|
|
139
|
+
lastEndMs = Math.max(lastEndMs, v.startMs + v.durationMs);
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
}),
|
|
143
|
+
socket.on('tts:viseme', (data) => {
|
|
144
|
+
if (data.streamId !== streamId) return;
|
|
145
|
+
for (const v of data.visemes) {
|
|
146
|
+
visemes.push(v);
|
|
147
|
+
lastEndMs = Math.max(lastEndMs, v.startMs + v.durationMs);
|
|
148
|
+
}
|
|
48
149
|
}),
|
|
49
150
|
socket.on('tts:word', (data) => {
|
|
50
151
|
if (data.streamId !== streamId) return;
|
|
51
152
|
setCurrentWord(data.word);
|
|
153
|
+
const ce = data.charEndIndex;
|
|
154
|
+
if (typeof ce === 'number') {
|
|
155
|
+
words.push({ charEndIndex: ce, startMs: data.startMs });
|
|
156
|
+
lastEndMs = Math.max(lastEndMs, data.startMs + data.durationMs);
|
|
157
|
+
}
|
|
52
158
|
}),
|
|
53
159
|
socket.on('tts:done', (data) => {
|
|
54
160
|
if (data.streamId !== streamId) return;
|
|
55
|
-
|
|
56
|
-
setCurrentWord('');
|
|
57
|
-
cleanupListeners();
|
|
58
|
-
socket.release();
|
|
161
|
+
serverDone = true;
|
|
59
162
|
}),
|
|
60
163
|
socket.on('tts:error', (data) => {
|
|
61
164
|
if (data.streamId !== streamId) return;
|
|
62
|
-
setPlaying(false);
|
|
63
165
|
setError(data.message);
|
|
64
|
-
|
|
65
|
-
socket.release();
|
|
166
|
+
finish();
|
|
66
167
|
}),
|
|
67
168
|
);
|
|
169
|
+
rafRef.current = requestAnimationFrame(tick);
|
|
68
170
|
|
|
69
171
|
socket.send('tts:start', {
|
|
70
172
|
streamId,
|
|
71
173
|
text,
|
|
72
174
|
voice: options?.voice,
|
|
175
|
+
requestVisemes: wantViseme,
|
|
73
176
|
});
|
|
74
177
|
},
|
|
75
178
|
[socket],
|
|
@@ -79,6 +182,8 @@ export function useTTS(socket: UglyBotSocket) {
|
|
|
79
182
|
if (streamIdRef.current) {
|
|
80
183
|
socket.send('tts:cancel', { streamId: streamIdRef.current });
|
|
81
184
|
}
|
|
185
|
+
streamIdRef.current = null;
|
|
186
|
+
stopRaf();
|
|
82
187
|
playerRef.current?.stop();
|
|
83
188
|
setPlaying(false);
|
|
84
189
|
cleanupListeners();
|
|
@@ -90,11 +195,12 @@ export function useTTS(socket: UglyBotSocket) {
|
|
|
90
195
|
if (streamIdRef.current) {
|
|
91
196
|
socket.send('tts:cancel', { streamId: streamIdRef.current });
|
|
92
197
|
}
|
|
198
|
+
stopRaf();
|
|
93
199
|
cleanupListeners();
|
|
94
200
|
socket.release();
|
|
95
201
|
playerRef.current?.close().catch(() => {});
|
|
96
202
|
};
|
|
97
203
|
}, [socket]);
|
|
98
204
|
|
|
99
|
-
return { play, stop, playing, currentWord, error, analyser };
|
|
205
|
+
return { play, stop, warmup, playing, currentWord, error, analyser };
|
|
100
206
|
}
|
package/src/client/index.ts
CHANGED
|
@@ -80,6 +80,7 @@ export { BlobVisualizerCore } from './audio/BlobVisualizerCore.js';
|
|
|
80
80
|
export { MicVisualizer } from './audio/MicVisualizer.js';
|
|
81
81
|
export { useSTT } from './audio/useSTT.js';
|
|
82
82
|
export { useTTS } from './audio/useTTS.js';
|
|
83
|
+
export type { TTSVisemeEvent } from './audio/useTTS.js';
|
|
83
84
|
export { WaveVisualizer } from './audio/WaveVisualizer.js';
|
|
84
85
|
export { callImageGen, callJsonGen, callTextGen } from './callAI.js';
|
|
85
86
|
export { uploadBlob, promoteBlob, downscaleImage } from './uploadBlob.js';
|