realtime-avatar 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +176 -0
- package/dist/browser.d.ts +203 -0
- package/dist/browser.js +172 -0
- package/dist/express.d.ts +28 -0
- package/dist/express.js +510 -0
- package/dist/hono.d.ts +19 -0
- package/dist/hono.js +498 -0
- package/dist/index.d.ts +192 -0
- package/dist/index.js +453 -0
- package/dist/nextjs.d.ts +18 -0
- package/dist/nextjs.js +498 -0
- package/dist/proxy-client-cZyX50-O.d.ts +1508 -0
- package/dist/react-native.d.ts +103 -0
- package/dist/react-native.js +2305 -0
- package/dist/react.d.ts +107 -0
- package/dist/react.js +2610 -0
- package/dist/server-only-guard.d.ts +2 -0
- package/dist/server-only-guard.js +4 -0
- package/dist/server.d.ts +2 -0
- package/dist/server.js +453 -0
- package/dist/tanstack-start.d.ts +24 -0
- package/dist/tanstack-start.js +498 -0
- package/dist/tools.d.ts +111 -0
- package/dist/tools.js +121 -0
- package/dist/types-C_EMPwN7.d.ts +831 -0
- package/dist/types-E8SrD6sv.d.ts +48 -0
- package/package.json +150 -0
package/dist/react.js
ADDED
|
@@ -0,0 +1,2610 @@
|
|
|
1
|
+
"use client";
|
|
2
|
+
import { useRef, useState, useEffect, useCallback, useMemo, createElement, Fragment } from 'react';
|
|
3
|
+
import { useTranscriptions, useLocalParticipant, RoomAudioRenderer, LiveKitRoom, useMaybeRoomContext, useVoiceAssistant, useConnectionState, VideoTrack, useRoomContext, useChat } from '@livekit/components-react';
|
|
4
|
+
import { ConnectionQuality, RoomEvent, ConnectionState, TrackEvent, Track, VideoQuality, ConnectionError, ConnectionErrorReason, DisconnectReason, Room } from 'livekit-client';
|
|
5
|
+
import { z } from 'zod';
|
|
6
|
+
|
|
7
|
+
// ../client/src/react/avatar-call.ts
|
|
8
|
+
var holder = null;
|
|
9
|
+
var queue = [];
|
|
10
|
+
var pendingEnded = null;
|
|
11
|
+
function acquireMicLease(token) {
|
|
12
|
+
if (holder === null || holder === token) {
|
|
13
|
+
holder = token;
|
|
14
|
+
if (pendingEnded === token) pendingEnded = null;
|
|
15
|
+
return Promise.resolve();
|
|
16
|
+
}
|
|
17
|
+
const existing = queue.find((w) => w.token === token);
|
|
18
|
+
if (existing) {
|
|
19
|
+
return new Promise((resolve) => {
|
|
20
|
+
const prior = existing.resolve;
|
|
21
|
+
existing.resolve = () => {
|
|
22
|
+
prior();
|
|
23
|
+
resolve();
|
|
24
|
+
};
|
|
25
|
+
});
|
|
26
|
+
}
|
|
27
|
+
return new Promise((resolve) => {
|
|
28
|
+
queue.push({ token, resolve });
|
|
29
|
+
});
|
|
30
|
+
}
|
|
31
|
+
function releaseMicLease(token) {
|
|
32
|
+
if (holder !== token) {
|
|
33
|
+
const i = queue.findIndex((w) => w.token === token);
|
|
34
|
+
if (i !== -1) queue.splice(i, 1);
|
|
35
|
+
return;
|
|
36
|
+
}
|
|
37
|
+
const next = queue.shift();
|
|
38
|
+
if (next) {
|
|
39
|
+
holder = next.token;
|
|
40
|
+
next.resolve();
|
|
41
|
+
} else {
|
|
42
|
+
holder = null;
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
function releaseMicLeaseWhenEnded(token, timeoutMs, setTimer) {
|
|
46
|
+
if (holder !== token) {
|
|
47
|
+
releaseMicLease(token);
|
|
48
|
+
return () => {
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
pendingEnded = token;
|
|
52
|
+
const cancelTimer = setTimer(() => {
|
|
53
|
+
if (pendingEnded === token) finishPendingRelease(token);
|
|
54
|
+
}, timeoutMs);
|
|
55
|
+
return cancelTimer;
|
|
56
|
+
}
|
|
57
|
+
function confirmMicTrackEnded(token) {
|
|
58
|
+
if (pendingEnded === token) finishPendingRelease(token);
|
|
59
|
+
}
|
|
60
|
+
function finishPendingRelease(token) {
|
|
61
|
+
pendingEnded = null;
|
|
62
|
+
releaseMicLease(token);
|
|
63
|
+
}
|
|
64
|
+
var MIC_LEASE_ENDED_TIMEOUT_MS = 1500;
|
|
65
|
+
function useMicLease(want) {
|
|
66
|
+
const tokenRef = useRef(null);
|
|
67
|
+
if (tokenRef.current === null) tokenRef.current = /* @__PURE__ */ Symbol("mic-lease");
|
|
68
|
+
const token = tokenRef.current;
|
|
69
|
+
const [held, setHeld] = useState(false);
|
|
70
|
+
useEffect(() => {
|
|
71
|
+
if (!want) {
|
|
72
|
+
releaseMicLeaseWhenEnded(token, MIC_LEASE_ENDED_TIMEOUT_MS, timeoutSetter);
|
|
73
|
+
setHeld(false);
|
|
74
|
+
return;
|
|
75
|
+
}
|
|
76
|
+
let alive = true;
|
|
77
|
+
void acquireMicLease(token).then(() => {
|
|
78
|
+
if (alive) setHeld(true);
|
|
79
|
+
});
|
|
80
|
+
return () => {
|
|
81
|
+
alive = false;
|
|
82
|
+
releaseMicLeaseWhenEnded(token, MIC_LEASE_ENDED_TIMEOUT_MS, timeoutSetter);
|
|
83
|
+
};
|
|
84
|
+
}, [want, token]);
|
|
85
|
+
return { held, token };
|
|
86
|
+
}
|
|
87
|
+
function timeoutSetter(fn, ms) {
|
|
88
|
+
const id = setTimeout(fn, ms);
|
|
89
|
+
return () => clearTimeout(id);
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
// ../client/src/errors.ts
|
|
93
|
+
var RealtimeAvatarCapacityError = class extends Error {
|
|
94
|
+
constructor(message, busy) {
|
|
95
|
+
super(message);
|
|
96
|
+
this.busy = busy;
|
|
97
|
+
this.name = "RealtimeAvatarCapacityError";
|
|
98
|
+
}
|
|
99
|
+
busy;
|
|
100
|
+
get queueSize() {
|
|
101
|
+
return this.busy.queue_size;
|
|
102
|
+
}
|
|
103
|
+
get queuePosition() {
|
|
104
|
+
return this.busy.queue_position;
|
|
105
|
+
}
|
|
106
|
+
get queueTicketId() {
|
|
107
|
+
return this.busy.queue_ticket_id;
|
|
108
|
+
}
|
|
109
|
+
get recommendedRetryMs() {
|
|
110
|
+
return this.busy.recommended_retry_ms;
|
|
111
|
+
}
|
|
112
|
+
};
|
|
113
|
+
function splitCallTranscript(all, localIdentity) {
|
|
114
|
+
if (!localIdentity) return { user: [], agent: all };
|
|
115
|
+
const user = [];
|
|
116
|
+
const agent = [];
|
|
117
|
+
for (const segment of all) {
|
|
118
|
+
if (segment.participantInfo.identity === localIdentity) user.push(segment);
|
|
119
|
+
else agent.push(segment);
|
|
120
|
+
}
|
|
121
|
+
return { user, agent };
|
|
122
|
+
}
|
|
123
|
+
function useCallTranscript() {
|
|
124
|
+
const all = useTranscriptions();
|
|
125
|
+
const { localParticipant } = useLocalParticipant();
|
|
126
|
+
return splitCallTranscript(all, localParticipant?.identity);
|
|
127
|
+
}
|
|
128
|
+
function useReleaseMicLeaseOnTrackEnded(token) {
|
|
129
|
+
const { localParticipant } = useLocalParticipant();
|
|
130
|
+
useEffect(() => {
|
|
131
|
+
if (!localParticipant) return;
|
|
132
|
+
const cleanups = [];
|
|
133
|
+
const watch = (raw) => {
|
|
134
|
+
if (!raw) return;
|
|
135
|
+
if (raw.readyState === "ended") {
|
|
136
|
+
confirmMicTrackEnded(token);
|
|
137
|
+
return;
|
|
138
|
+
}
|
|
139
|
+
const onEnded = () => confirmMicTrackEnded(token);
|
|
140
|
+
raw.addEventListener("ended", onEnded);
|
|
141
|
+
cleanups.push(() => raw.removeEventListener("ended", onEnded));
|
|
142
|
+
};
|
|
143
|
+
for (const pub of localParticipant.audioTrackPublications.values()) {
|
|
144
|
+
watch(pub.track?.mediaStreamTrack);
|
|
145
|
+
}
|
|
146
|
+
return () => {
|
|
147
|
+
for (const c of cleanups) c();
|
|
148
|
+
};
|
|
149
|
+
}, [localParticipant, token, localParticipant?.audioTrackPublications.size]);
|
|
150
|
+
}
|
|
151
|
+
var DEFAULT_AVATAR_PLAYOUT_DELAY_SECONDS = 0.5;
|
|
152
|
+
function useAvatarPlayoutDelay(videoTrack, audioTrack, delaySeconds = DEFAULT_AVATAR_PLAYOUT_DELAY_SECONDS) {
|
|
153
|
+
const videoMediaTrack = videoTrack?.publication?.track;
|
|
154
|
+
const audioMediaTrack = audioTrack?.publication?.track;
|
|
155
|
+
useEffect(() => {
|
|
156
|
+
applyAvatarPlayoutDelay(videoMediaTrack, audioMediaTrack, delaySeconds);
|
|
157
|
+
}, [videoMediaTrack, audioMediaTrack, delaySeconds]);
|
|
158
|
+
}
|
|
159
|
+
function applyAvatarPlayoutDelay(videoTrack, audioTrack, delaySeconds = DEFAULT_AVATAR_PLAYOUT_DELAY_SECONDS) {
|
|
160
|
+
const delay = Math.max(0, delaySeconds);
|
|
161
|
+
playoutDelayTarget(videoTrack)?.setPlayoutDelay(delay);
|
|
162
|
+
playoutDelayTarget(audioTrack)?.setPlayoutDelay(delay);
|
|
163
|
+
}
|
|
164
|
+
function playoutDelayTarget(track) {
|
|
165
|
+
const candidate = track;
|
|
166
|
+
return typeof candidate?.setPlayoutDelay === "function" ? candidate : void 0;
|
|
167
|
+
}
|
|
168
|
+
function resolveReleaseTarget(heldSessionId, queueTicketId) {
|
|
169
|
+
if (heldSessionId) return { kind: "session", sessionId: heldSessionId };
|
|
170
|
+
if (queueTicketId) return { kind: "ticket", queueTicketId };
|
|
171
|
+
return { kind: "none" };
|
|
172
|
+
}
|
|
173
|
+
function capacityStateFromGrant(state) {
|
|
174
|
+
switch (state.status) {
|
|
175
|
+
case "busy":
|
|
176
|
+
if (state.busy) return { kind: "queued", busy: state.busy };
|
|
177
|
+
return { kind: "connecting" };
|
|
178
|
+
case "failed":
|
|
179
|
+
return { kind: "error", error: state.error ?? new Error("Realtime session request failed") };
|
|
180
|
+
case "ready":
|
|
181
|
+
return state.grant ? { kind: "active", grant: state.grant } : { kind: "connecting" };
|
|
182
|
+
case "requesting":
|
|
183
|
+
return { kind: "connecting" };
|
|
184
|
+
case "idle":
|
|
185
|
+
default:
|
|
186
|
+
return { kind: "idle" };
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
var cachedClientId = null;
|
|
190
|
+
function randomClientId() {
|
|
191
|
+
return globalThis.crypto?.randomUUID?.() ?? `c_${Date.now().toString(36)}_${Math.random().toString(36).slice(2)}`;
|
|
192
|
+
}
|
|
193
|
+
function stableClientId() {
|
|
194
|
+
if (cachedClientId) return cachedClientId;
|
|
195
|
+
try {
|
|
196
|
+
const key = "rta.client_id";
|
|
197
|
+
const existing = globalThis.localStorage?.getItem(key) ?? null;
|
|
198
|
+
if (existing) return cachedClientId = existing;
|
|
199
|
+
const fresh = randomClientId();
|
|
200
|
+
globalThis.localStorage?.setItem(key, fresh);
|
|
201
|
+
return cachedClientId = fresh;
|
|
202
|
+
} catch {
|
|
203
|
+
return cachedClientId = randomClientId();
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
function createConnectionWarmer(prepare) {
|
|
207
|
+
const warmed = /* @__PURE__ */ new Set();
|
|
208
|
+
return (url) => {
|
|
209
|
+
const valid = validWarmUrl(url);
|
|
210
|
+
if (!valid || warmed.has(valid)) return;
|
|
211
|
+
warmed.add(valid);
|
|
212
|
+
prepare(valid);
|
|
213
|
+
};
|
|
214
|
+
}
|
|
215
|
+
function validWarmUrl(value) {
|
|
216
|
+
if (!value) return null;
|
|
217
|
+
try {
|
|
218
|
+
const { protocol } = new URL(value);
|
|
219
|
+
return protocol === "wss:" || protocol === "ws:" || protocol === "https:" || protocol === "http:" ? value : null;
|
|
220
|
+
} catch {
|
|
221
|
+
return null;
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
var warmRoom = null;
|
|
225
|
+
var warmLiveKitHost = createConnectionWarmer((url) => {
|
|
226
|
+
try {
|
|
227
|
+
warmRoom ??= new Room();
|
|
228
|
+
void warmRoom.prepareConnection(url);
|
|
229
|
+
} catch {
|
|
230
|
+
}
|
|
231
|
+
});
|
|
232
|
+
var LIVEKIT_URL_HINT_KEY = "rta.livekit_url_hint";
|
|
233
|
+
var cachedUrlHint = null;
|
|
234
|
+
function readLiveKitUrlHint() {
|
|
235
|
+
try {
|
|
236
|
+
return validWarmUrl(globalThis.localStorage?.getItem(LIVEKIT_URL_HINT_KEY)) ?? cachedUrlHint;
|
|
237
|
+
} catch {
|
|
238
|
+
return cachedUrlHint;
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
function writeLiveKitUrlHint(url) {
|
|
242
|
+
if (!validWarmUrl(url)) return;
|
|
243
|
+
cachedUrlHint = url;
|
|
244
|
+
try {
|
|
245
|
+
globalThis.localStorage?.setItem(LIVEKIT_URL_HINT_KEY, url);
|
|
246
|
+
} catch {
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
function useLiveKitAvatarGrant(input) {
|
|
250
|
+
const { client, session, active = true, autoRetryBusy = false, requestOptions, serverUrlHint } = input;
|
|
251
|
+
const [version, setVersion] = useState(0);
|
|
252
|
+
const queueTicketRef = useRef(null);
|
|
253
|
+
const clientIdRef = useRef(stableClientId());
|
|
254
|
+
const heldSessionRef = useRef(null);
|
|
255
|
+
const releaseHeld = useCallback(
|
|
256
|
+
(reason, viaBeacon) => {
|
|
257
|
+
const target = resolveReleaseTarget(heldSessionRef.current, queueTicketRef.current);
|
|
258
|
+
if (target.kind === "none") return;
|
|
259
|
+
if (target.kind === "session") {
|
|
260
|
+
heldSessionRef.current = null;
|
|
261
|
+
if (viaBeacon && client.releaseLiveKitSessionBeacon(target.sessionId, reason)) return;
|
|
262
|
+
void client.releaseLiveKitSession(target.sessionId, reason);
|
|
263
|
+
return;
|
|
264
|
+
}
|
|
265
|
+
queueTicketRef.current = null;
|
|
266
|
+
if (viaBeacon && client.releaseLiveKitQueueTicketBeacon(target.queueTicketId, reason)) return;
|
|
267
|
+
void client.releaseLiveKitQueueTicket(target.queueTicketId, reason);
|
|
268
|
+
},
|
|
269
|
+
[client]
|
|
270
|
+
);
|
|
271
|
+
const sessionKey = useMemo(() => session ? stableStringify(session) : null, [session]);
|
|
272
|
+
const sessionRef = useRef(session);
|
|
273
|
+
sessionRef.current = session;
|
|
274
|
+
const serverUrlHintRef = useRef(serverUrlHint);
|
|
275
|
+
serverUrlHintRef.current = serverUrlHint;
|
|
276
|
+
const prevKeyRef = useRef(null);
|
|
277
|
+
const [state, setState] = useState({
|
|
278
|
+
status: "idle",
|
|
279
|
+
grant: null,
|
|
280
|
+
busy: null,
|
|
281
|
+
error: null
|
|
282
|
+
});
|
|
283
|
+
const stateRef = useRef(state);
|
|
284
|
+
stateRef.current = state;
|
|
285
|
+
const [tabVisible, setTabVisible] = useState(
|
|
286
|
+
() => typeof document === "undefined" || document.visibilityState === "visible"
|
|
287
|
+
);
|
|
288
|
+
useEffect(() => {
|
|
289
|
+
if (typeof document === "undefined") return;
|
|
290
|
+
const onVisibility = () => setTabVisible(document.visibilityState === "visible");
|
|
291
|
+
document.addEventListener("visibilitychange", onVisibility);
|
|
292
|
+
return () => document.removeEventListener("visibilitychange", onVisibility);
|
|
293
|
+
}, []);
|
|
294
|
+
useEffect(() => {
|
|
295
|
+
const current = sessionRef.current;
|
|
296
|
+
if (!active || !current || !sessionKey) {
|
|
297
|
+
releaseHeld("unmount", false);
|
|
298
|
+
queueTicketRef.current = null;
|
|
299
|
+
prevKeyRef.current = sessionKey;
|
|
300
|
+
setState({ status: "idle", grant: null, busy: null, error: null });
|
|
301
|
+
return;
|
|
302
|
+
}
|
|
303
|
+
if (prevKeyRef.current !== sessionKey) {
|
|
304
|
+
releaseHeld("superseded", false);
|
|
305
|
+
queueTicketRef.current = current.queueTicketId ?? null;
|
|
306
|
+
prevKeyRef.current = sessionKey;
|
|
307
|
+
}
|
|
308
|
+
const request = {
|
|
309
|
+
...current,
|
|
310
|
+
queueTicketId: current.queueTicketId ?? queueTicketRef.current ?? clientIdRef.current
|
|
311
|
+
};
|
|
312
|
+
let cancelled = false;
|
|
313
|
+
const prior = stateRef.current;
|
|
314
|
+
const retryingQueued = prior.status === "busy" && prior.busy !== null;
|
|
315
|
+
if (!retryingQueued) {
|
|
316
|
+
warmLiveKitHost(serverUrlHintRef.current ?? readLiveKitUrlHint());
|
|
317
|
+
setState({ status: "requesting", grant: null, busy: null, error: null });
|
|
318
|
+
}
|
|
319
|
+
void client.createLiveKitSessionOrBusy(request, requestOptions).then((result) => {
|
|
320
|
+
if (cancelled) return;
|
|
321
|
+
if (result.status === "busy") {
|
|
322
|
+
queueTicketRef.current = result.busy.queue_ticket_id ?? queueTicketRef.current;
|
|
323
|
+
setState({ status: "busy", grant: null, busy: result.busy, error: null });
|
|
324
|
+
return;
|
|
325
|
+
}
|
|
326
|
+
queueTicketRef.current = null;
|
|
327
|
+
if (heldSessionRef.current && heldSessionRef.current !== result.grant.session_id) {
|
|
328
|
+
releaseHeld("superseded", false);
|
|
329
|
+
}
|
|
330
|
+
heldSessionRef.current = result.grant.session_id;
|
|
331
|
+
writeLiveKitUrlHint(result.grant.livekit_url);
|
|
332
|
+
setState({ status: "ready", grant: result.grant, busy: null, error: null });
|
|
333
|
+
}).catch((error) => {
|
|
334
|
+
if (cancelled) return;
|
|
335
|
+
setState({
|
|
336
|
+
status: "failed",
|
|
337
|
+
grant: null,
|
|
338
|
+
busy: null,
|
|
339
|
+
error: error instanceof Error ? error : new Error(String(error))
|
|
340
|
+
});
|
|
341
|
+
});
|
|
342
|
+
return () => {
|
|
343
|
+
cancelled = true;
|
|
344
|
+
};
|
|
345
|
+
}, [active, client, sessionKey, requestOptions, version, releaseHeld]);
|
|
346
|
+
useEffect(() => {
|
|
347
|
+
const canListen = typeof window !== "undefined" && typeof window.addEventListener === "function";
|
|
348
|
+
const onPageHide = () => releaseHeld("page_hide", true);
|
|
349
|
+
if (canListen) window.addEventListener("pagehide", onPageHide);
|
|
350
|
+
return () => {
|
|
351
|
+
if (canListen) window.removeEventListener("pagehide", onPageHide);
|
|
352
|
+
releaseHeld("unmount", false);
|
|
353
|
+
};
|
|
354
|
+
}, [releaseHeld]);
|
|
355
|
+
const refresh = useCallback(() => setVersion((current) => current + 1), []);
|
|
356
|
+
const clear = useCallback(() => {
|
|
357
|
+
releaseHeld("manual", false);
|
|
358
|
+
queueTicketRef.current = null;
|
|
359
|
+
setState({ status: "idle", grant: null, busy: null, error: null });
|
|
360
|
+
}, [releaseHeld]);
|
|
361
|
+
const release = useCallback(
|
|
362
|
+
(reason = "disconnected") => releaseHeld(reason, false),
|
|
363
|
+
[releaseHeld]
|
|
364
|
+
);
|
|
365
|
+
useEffect(() => {
|
|
366
|
+
if (!autoRetryBusy || !active || !tabVisible || state.status !== "busy" || !state.busy) return;
|
|
367
|
+
const retryMs = Math.max(state.busy.recommended_retry_ms, 250);
|
|
368
|
+
const timer = window.setTimeout(refresh, retryMs);
|
|
369
|
+
return () => window.clearTimeout(timer);
|
|
370
|
+
}, [active, autoRetryBusy, tabVisible, refresh, state.busy, state.status]);
|
|
371
|
+
const capacityRef = useRef(null);
|
|
372
|
+
const capacity = useMemo(() => {
|
|
373
|
+
const next = capacityStateFromGrant(state);
|
|
374
|
+
const prev = capacityRef.current;
|
|
375
|
+
if (prev && sameCapacityState(prev, next)) return prev;
|
|
376
|
+
capacityRef.current = next;
|
|
377
|
+
return next;
|
|
378
|
+
}, [state]);
|
|
379
|
+
return useMemo(
|
|
380
|
+
() => ({ ...state, capacity, refresh, clear, release }),
|
|
381
|
+
[state, capacity, refresh, clear, release]
|
|
382
|
+
);
|
|
383
|
+
}
|
|
384
|
+
function sameCapacityState(a, b) {
|
|
385
|
+
if (a.kind !== b.kind) return false;
|
|
386
|
+
switch (a.kind) {
|
|
387
|
+
case "queued": {
|
|
388
|
+
const next = b.busy;
|
|
389
|
+
return a.busy.queue_position === next.queue_position && a.busy.queue_size === next.queue_size && a.busy.recommended_retry_ms === next.recommended_retry_ms;
|
|
390
|
+
}
|
|
391
|
+
case "active":
|
|
392
|
+
return a.grant === b.grant;
|
|
393
|
+
case "error":
|
|
394
|
+
return a.error === b.error;
|
|
395
|
+
default:
|
|
396
|
+
return true;
|
|
397
|
+
}
|
|
398
|
+
}
|
|
399
|
+
function RealtimeAvatarLiveKitRoom(props) {
|
|
400
|
+
const {
|
|
401
|
+
grant,
|
|
402
|
+
connect = true,
|
|
403
|
+
audio,
|
|
404
|
+
video = false,
|
|
405
|
+
options,
|
|
406
|
+
renderRoomAudio = true,
|
|
407
|
+
children,
|
|
408
|
+
...roomProps
|
|
409
|
+
} = props;
|
|
410
|
+
const shouldConnect = Boolean(grant && connect);
|
|
411
|
+
const roomAudio = renderRoomAudio ? createElement(RoomAudioRenderer, { key: "room-audio" }) : null;
|
|
412
|
+
const roomOptions = { adaptiveStream: false, dynacast: true, ...options };
|
|
413
|
+
return createElement(
|
|
414
|
+
LiveKitRoom,
|
|
415
|
+
{
|
|
416
|
+
...roomProps,
|
|
417
|
+
serverUrl: grant?.livekit_url,
|
|
418
|
+
token: grant?.participant_token,
|
|
419
|
+
connect: shouldConnect,
|
|
420
|
+
audio: audio ?? grant?.stt_mode === "server",
|
|
421
|
+
video,
|
|
422
|
+
options: roomOptions
|
|
423
|
+
},
|
|
424
|
+
createElement(Fragment, null, children, roomAudio)
|
|
425
|
+
);
|
|
426
|
+
}
|
|
427
|
+
var capacityErrorFromBusy = (busy) => {
|
|
428
|
+
return new RealtimeAvatarCapacityError(busy.message, busy);
|
|
429
|
+
};
|
|
430
|
+
function stableStringify(value) {
|
|
431
|
+
if (value === null || typeof value !== "object") return JSON.stringify(value) ?? "null";
|
|
432
|
+
if (Array.isArray(value)) return `[${value.map(stableStringify).join(",")}]`;
|
|
433
|
+
const entries = Object.entries(value).filter(([, entry]) => entry !== void 0).sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0);
|
|
434
|
+
return `{${entries.map(([key, entry]) => `${JSON.stringify(key)}:${stableStringify(entry)}`).join(",")}}`;
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
// ../client/src/react/quality-governor.ts
|
|
438
|
+
var DEFAULT_GOVERNOR_CONFIG = {
|
|
439
|
+
openingCap: "low",
|
|
440
|
+
downgradeFreezeMs: 150,
|
|
441
|
+
probationFreezeMs: 100,
|
|
442
|
+
openingDwellMs: 2e3,
|
|
443
|
+
dwellBaseMs: 8e3,
|
|
444
|
+
dwellMaxMs: 12e4,
|
|
445
|
+
cleanMs: 3e3,
|
|
446
|
+
probeMs: 1e4,
|
|
447
|
+
healthyResetMs: 12e4
|
|
448
|
+
};
|
|
449
|
+
var JITTER_BUFFER_RISE_THRESHOLD_MS = 25;
|
|
450
|
+
function stepJitterBufferTrend(previous, current, riseThresholdMs = JITTER_BUFFER_RISE_THRESHOLD_MS) {
|
|
451
|
+
const baseline = { ...current, intervalAverageMs: null };
|
|
452
|
+
if (previous === null || !Number.isFinite(current.delaySeconds) || !Number.isFinite(current.emittedCount) || current.delaySeconds < previous.delaySeconds || current.emittedCount <= previous.emittedCount) {
|
|
453
|
+
return { state: baseline, rising: false };
|
|
454
|
+
}
|
|
455
|
+
const intervalAverageMs = (current.delaySeconds - previous.delaySeconds) * 1e3 / (current.emittedCount - previous.emittedCount);
|
|
456
|
+
const threshold = Number.isFinite(riseThresholdMs) ? Math.max(0, riseThresholdMs) : 0;
|
|
457
|
+
const rising = previous.intervalAverageMs !== null && intervalAverageMs > previous.intervalAverageMs + threshold;
|
|
458
|
+
return {
|
|
459
|
+
state: { ...current, intervalAverageMs },
|
|
460
|
+
rising
|
|
461
|
+
};
|
|
462
|
+
}
|
|
463
|
+
var initGovernor = (nowMs, openingCap = "low") => ({
|
|
464
|
+
state: openingCap === "high" ? "opening_high" : "opening",
|
|
465
|
+
cap: openingCap,
|
|
466
|
+
failures: 0,
|
|
467
|
+
enteredAtMs: nowMs,
|
|
468
|
+
healthySinceMs: null
|
|
469
|
+
});
|
|
470
|
+
var isDowngrade = (s, cfg) => s.paused || s.freezeMsInWindow >= cfg.downgradeFreezeMs || s.jitterRising && s.freezeMsInWindow > 0;
|
|
471
|
+
var isProbationFail = (s, cfg) => s.paused || s.freezeMsInWindow >= cfg.probationFreezeMs;
|
|
472
|
+
var isHealthy = (s) => !s.paused && s.freezeMsInWindow === 0 && !s.jitterRising && s.connectionQuality !== "poor" && s.connectionQuality !== "lost";
|
|
473
|
+
var dwellMs = (failures, cfg) => Math.min(cfg.dwellBaseMs * 2 ** failures, cfg.dwellMaxMs);
|
|
474
|
+
var enter = (g, state, cap, nowMs) => ({
|
|
475
|
+
...g,
|
|
476
|
+
state,
|
|
477
|
+
cap,
|
|
478
|
+
enteredAtMs: nowMs,
|
|
479
|
+
healthySinceMs: null
|
|
480
|
+
});
|
|
481
|
+
var step = (g, s, nowMs, cfg = DEFAULT_GOVERNOR_CONFIG) => {
|
|
482
|
+
if (s.inhibited) {
|
|
483
|
+
return { governor: g };
|
|
484
|
+
}
|
|
485
|
+
const onProbation = g.state === "probing_up" || g.state === "opening_high";
|
|
486
|
+
const downgradeNow = onProbation ? isProbationFail(s, cfg) : isDowngrade(s, cfg);
|
|
487
|
+
if (g.cap === "high" && downgradeNow) {
|
|
488
|
+
const failed = onProbation;
|
|
489
|
+
return {
|
|
490
|
+
governor: {
|
|
491
|
+
...enter(g, "cap_low_sticky", "low", nowMs),
|
|
492
|
+
failures: failed ? g.failures + 1 : g.failures
|
|
493
|
+
},
|
|
494
|
+
action: { setCap: "low" }
|
|
495
|
+
};
|
|
496
|
+
}
|
|
497
|
+
switch (g.state) {
|
|
498
|
+
case "opening_high": {
|
|
499
|
+
if (nowMs - g.enteredAtMs >= cfg.probeMs && isHealthy(s)) {
|
|
500
|
+
return { governor: enter(g, "cap_high_stable", "high", nowMs) };
|
|
501
|
+
}
|
|
502
|
+
return { governor: g };
|
|
503
|
+
}
|
|
504
|
+
case "opening": {
|
|
505
|
+
if (nowMs - g.enteredAtMs >= cfg.openingDwellMs && isHealthy(s)) {
|
|
506
|
+
return { governor: { ...enter(g, "cap_low_eligible", "low", nowMs), healthySinceMs: nowMs } };
|
|
507
|
+
}
|
|
508
|
+
return { governor: g };
|
|
509
|
+
}
|
|
510
|
+
case "cap_low_sticky": {
|
|
511
|
+
const healthy = isHealthy(s);
|
|
512
|
+
const healthySince = healthy ? g.healthySinceMs ?? nowMs : null;
|
|
513
|
+
const resetFailures = healthy && healthySince !== null && nowMs - healthySince >= cfg.healthyResetMs;
|
|
514
|
+
const dwellDone = nowMs - g.enteredAtMs >= dwellMs(g.failures, cfg);
|
|
515
|
+
if (dwellDone && healthy) {
|
|
516
|
+
return {
|
|
517
|
+
governor: {
|
|
518
|
+
...enter(g, "cap_low_eligible", "low", nowMs),
|
|
519
|
+
failures: resetFailures ? 0 : g.failures,
|
|
520
|
+
healthySinceMs: nowMs
|
|
521
|
+
}
|
|
522
|
+
};
|
|
523
|
+
}
|
|
524
|
+
return {
|
|
525
|
+
governor: {
|
|
526
|
+
...g,
|
|
527
|
+
healthySinceMs: healthySince,
|
|
528
|
+
failures: resetFailures ? 0 : g.failures
|
|
529
|
+
}
|
|
530
|
+
};
|
|
531
|
+
}
|
|
532
|
+
case "cap_low_eligible": {
|
|
533
|
+
if (!isHealthy(s)) {
|
|
534
|
+
return { governor: { ...g, healthySinceMs: null } };
|
|
535
|
+
}
|
|
536
|
+
const cleanSince = g.healthySinceMs ?? nowMs;
|
|
537
|
+
if (nowMs - cleanSince >= cfg.cleanMs) {
|
|
538
|
+
return {
|
|
539
|
+
governor: enter(g, "probing_up", "high", nowMs),
|
|
540
|
+
action: { setCap: "high" }
|
|
541
|
+
};
|
|
542
|
+
}
|
|
543
|
+
return { governor: { ...g, healthySinceMs: cleanSince } };
|
|
544
|
+
}
|
|
545
|
+
case "probing_up": {
|
|
546
|
+
if (nowMs - g.enteredAtMs >= cfg.probeMs) {
|
|
547
|
+
return { governor: { ...enter(g, "cap_high_stable", "high", nowMs), failures: 0 } };
|
|
548
|
+
}
|
|
549
|
+
return { governor: g };
|
|
550
|
+
}
|
|
551
|
+
case "cap_high_stable":
|
|
552
|
+
return { governor: g };
|
|
553
|
+
default:
|
|
554
|
+
return { governor: g };
|
|
555
|
+
}
|
|
556
|
+
};
|
|
557
|
+
var resolveLowCapQuality = (declaredLayerQualities) => {
|
|
558
|
+
const sorted = [...declaredLayerQualities].sort((a, b) => a - b);
|
|
559
|
+
return sorted.length >= 2 ? sorted[sorted.length - 2] : 1;
|
|
560
|
+
};
|
|
561
|
+
|
|
562
|
+
// ../client/src/react/use-quality-governor.ts
|
|
563
|
+
var qualityToSignal = (q) => {
|
|
564
|
+
switch (q) {
|
|
565
|
+
case ConnectionQuality.Excellent:
|
|
566
|
+
return "excellent";
|
|
567
|
+
case ConnectionQuality.Good:
|
|
568
|
+
return "good";
|
|
569
|
+
case ConnectionQuality.Poor:
|
|
570
|
+
return "poor";
|
|
571
|
+
case ConnectionQuality.Lost:
|
|
572
|
+
return "lost";
|
|
573
|
+
default:
|
|
574
|
+
return "unknown";
|
|
575
|
+
}
|
|
576
|
+
};
|
|
577
|
+
function useAvatarQualityGovernor(input) {
|
|
578
|
+
const { enabled, freezeReading, config = DEFAULT_GOVERNOR_CONFIG, tickMs = 1e3 } = input;
|
|
579
|
+
const room = useMaybeRoomContext();
|
|
580
|
+
const { videoTrack } = useVoiceAssistant();
|
|
581
|
+
const pausedSinceTick = useRef(false);
|
|
582
|
+
const connQuality = useRef("unknown");
|
|
583
|
+
const lastFreezeStat = useRef(null);
|
|
584
|
+
const jitterTrend = useRef(null);
|
|
585
|
+
useEffect(() => {
|
|
586
|
+
if (!enabled || !room) return;
|
|
587
|
+
const targetPublication = videoTrack?.publication;
|
|
588
|
+
const targetParticipant = videoTrack?.participant;
|
|
589
|
+
pausedSinceTick.current = false;
|
|
590
|
+
lastFreezeStat.current = null;
|
|
591
|
+
jitterTrend.current = null;
|
|
592
|
+
connQuality.current = qualityToSignal(
|
|
593
|
+
targetParticipant?.connectionQuality ?? ConnectionQuality.Unknown
|
|
594
|
+
);
|
|
595
|
+
const onStreamState = (pub, streamState) => {
|
|
596
|
+
if (targetPublication && pub !== targetPublication) return;
|
|
597
|
+
if (streamState === Track.StreamState.Paused) pausedSinceTick.current = true;
|
|
598
|
+
};
|
|
599
|
+
const onQuality = (q, participant) => {
|
|
600
|
+
if (targetParticipant && participant.sid !== targetParticipant.sid) return;
|
|
601
|
+
connQuality.current = qualityToSignal(q);
|
|
602
|
+
};
|
|
603
|
+
try {
|
|
604
|
+
room.on(RoomEvent.TrackStreamStateChanged, onStreamState);
|
|
605
|
+
room.on(RoomEvent.ConnectionQualityChanged, onQuality);
|
|
606
|
+
} catch {
|
|
607
|
+
return;
|
|
608
|
+
}
|
|
609
|
+
let gov = initGovernor(Date.now());
|
|
610
|
+
const readGetStatsSignals = async () => {
|
|
611
|
+
try {
|
|
612
|
+
const track = targetPublication?.track;
|
|
613
|
+
const stats = await track?.getRTCStatsReport?.();
|
|
614
|
+
if (!stats) return { freezeMs: 0, jitterRising: false };
|
|
615
|
+
let frozenTotalMs = 0;
|
|
616
|
+
let jitterDelaySeconds = 0;
|
|
617
|
+
let jitterEmittedCount = 0;
|
|
618
|
+
stats.forEach((r) => {
|
|
619
|
+
if (r.type === "inbound-rtp" && typeof r.totalFreezesDuration === "number") {
|
|
620
|
+
frozenTotalMs = r.totalFreezesDuration * 1e3;
|
|
621
|
+
}
|
|
622
|
+
if (r.type === "inbound-rtp" && typeof r.jitterBufferDelay === "number" && typeof r.jitterBufferEmittedCount === "number") {
|
|
623
|
+
jitterDelaySeconds += r.jitterBufferDelay;
|
|
624
|
+
jitterEmittedCount += r.jitterBufferEmittedCount;
|
|
625
|
+
}
|
|
626
|
+
});
|
|
627
|
+
const now = Date.now();
|
|
628
|
+
const prev = lastFreezeStat.current;
|
|
629
|
+
lastFreezeStat.current = { frozen: frozenTotalMs, ts: now };
|
|
630
|
+
const trend = stepJitterBufferTrend(jitterTrend.current, {
|
|
631
|
+
delaySeconds: jitterDelaySeconds,
|
|
632
|
+
emittedCount: jitterEmittedCount
|
|
633
|
+
});
|
|
634
|
+
jitterTrend.current = trend.state;
|
|
635
|
+
return {
|
|
636
|
+
freezeMs: prev ? Math.max(0, frozenTotalMs - prev.frozen) : 0,
|
|
637
|
+
jitterRising: trend.rising
|
|
638
|
+
};
|
|
639
|
+
} catch {
|
|
640
|
+
return { freezeMs: 0, jitterRising: false };
|
|
641
|
+
}
|
|
642
|
+
};
|
|
643
|
+
const applyCap = (cap) => {
|
|
644
|
+
try {
|
|
645
|
+
const lowCap = resolveLowCapQuality(
|
|
646
|
+
(targetPublication?.trackInfo?.layers ?? []).map((l) => l.quality)
|
|
647
|
+
);
|
|
648
|
+
targetPublication?.setVideoQuality?.(cap === "low" ? lowCap : VideoQuality.HIGH);
|
|
649
|
+
} catch {
|
|
650
|
+
}
|
|
651
|
+
};
|
|
652
|
+
let disposed = false;
|
|
653
|
+
let tickRunning = false;
|
|
654
|
+
const tick = async () => {
|
|
655
|
+
if (disposed || tickRunning) return;
|
|
656
|
+
tickRunning = true;
|
|
657
|
+
try {
|
|
658
|
+
const rvfc = freezeReading?.() ?? { freezeMsInWindow: 0, inhibited: false };
|
|
659
|
+
const statsSignals = await readGetStatsSignals();
|
|
660
|
+
if (disposed) return;
|
|
661
|
+
const signal = {
|
|
662
|
+
paused: pausedSinceTick.current,
|
|
663
|
+
freezeMsInWindow: Math.max(rvfc.freezeMsInWindow, statsSignals.freezeMs),
|
|
664
|
+
jitterRising: statsSignals.jitterRising,
|
|
665
|
+
connectionQuality: connQuality.current,
|
|
666
|
+
inhibited: rvfc.inhibited
|
|
667
|
+
};
|
|
668
|
+
pausedSinceTick.current = false;
|
|
669
|
+
const { governor, action } = step(gov, signal, Date.now(), config);
|
|
670
|
+
gov = governor;
|
|
671
|
+
if (action) applyCap(action.setCap);
|
|
672
|
+
} catch {
|
|
673
|
+
} finally {
|
|
674
|
+
tickRunning = false;
|
|
675
|
+
}
|
|
676
|
+
};
|
|
677
|
+
applyCap(gov.cap);
|
|
678
|
+
const handle = setInterval(() => void tick(), tickMs);
|
|
679
|
+
return () => {
|
|
680
|
+
disposed = true;
|
|
681
|
+
clearInterval(handle);
|
|
682
|
+
try {
|
|
683
|
+
room.off(RoomEvent.TrackStreamStateChanged, onStreamState);
|
|
684
|
+
room.off(RoomEvent.ConnectionQualityChanged, onQuality);
|
|
685
|
+
} catch {
|
|
686
|
+
}
|
|
687
|
+
};
|
|
688
|
+
}, [enabled, room, videoTrack, freezeReading, config, tickMs]);
|
|
689
|
+
}
|
|
690
|
+
|
|
691
|
+
// ../client/src/react/avatar-video-surface.ts
|
|
692
|
+
var DEFAULT_AVATAR_FRAME_STALL_MS = 800;
|
|
693
|
+
var AVATAR_FRAME_GAP_FREEZE_FLOOR_MS = 100;
|
|
694
|
+
function AvatarVideoSurface(props) {
|
|
695
|
+
const {
|
|
696
|
+
idleVideoUrl,
|
|
697
|
+
poster = null,
|
|
698
|
+
live = true,
|
|
699
|
+
adaptiveQuality = true,
|
|
700
|
+
openingCap,
|
|
701
|
+
fit = "contain",
|
|
702
|
+
aspectRatio,
|
|
703
|
+
idleReturnDelayMs = 700,
|
|
704
|
+
frameStallMs = DEFAULT_AVATAR_FRAME_STALL_MS,
|
|
705
|
+
className,
|
|
706
|
+
style,
|
|
707
|
+
children,
|
|
708
|
+
showLiveBadge = true
|
|
709
|
+
} = props;
|
|
710
|
+
const testId = props["data-testid"];
|
|
711
|
+
const { videoTrack, audioTrack } = useVoiceAssistant();
|
|
712
|
+
useAvatarPlayoutDelay(videoTrack, audioTrack);
|
|
713
|
+
const connectionState = useConnectionState();
|
|
714
|
+
const connected = connectionState === ConnectionState.Connected;
|
|
715
|
+
const trackProducing = useLiveTrackProducing(videoTrack);
|
|
716
|
+
const liveWrapRef = useRef(null);
|
|
717
|
+
const frameFlow = useLiveFrameFlow(
|
|
718
|
+
liveWrapRef,
|
|
719
|
+
trackProducing,
|
|
720
|
+
videoTrack?.publication?.track,
|
|
721
|
+
frameStallMs
|
|
722
|
+
);
|
|
723
|
+
const governorConfig = useMemo(
|
|
724
|
+
() => ({
|
|
725
|
+
...DEFAULT_GOVERNOR_CONFIG,
|
|
726
|
+
openingCap: openingCap ?? DEFAULT_GOVERNOR_CONFIG.openingCap
|
|
727
|
+
}),
|
|
728
|
+
[openingCap]
|
|
729
|
+
);
|
|
730
|
+
useAvatarQualityGovernor({
|
|
731
|
+
// Keep the governor alive across presentation-intent changes; resetting it on
|
|
732
|
+
// every turn would re-apply LOW and a normal short turn could never earn HIGH.
|
|
733
|
+
enabled: adaptiveQuality,
|
|
734
|
+
freezeReading: frameFlow.freezeReading,
|
|
735
|
+
config: governorConfig
|
|
736
|
+
});
|
|
737
|
+
const liveWanted = live && connected && trackProducing && frameFlow.flowing;
|
|
738
|
+
const networkStalled = trackProducing && frameFlow.seenFrame && !frameFlow.flowing;
|
|
739
|
+
const showLive = useDebouncedHide(
|
|
740
|
+
liveWanted,
|
|
741
|
+
connected && !networkStalled ? idleReturnDelayMs : 0
|
|
742
|
+
);
|
|
743
|
+
const liveDims = videoTrack?.publication?.dimensions ?? null;
|
|
744
|
+
const idleVideoRef = useRef(null);
|
|
745
|
+
const boxAspect = aspectRatio ?? null;
|
|
746
|
+
useIdleWhileResting(idleVideoRef, idleVideoUrl, showLive);
|
|
747
|
+
useLiveResumeOnProducing(liveWrapRef, trackProducing);
|
|
748
|
+
const fitClass = fit === "cover" ? AVATAR_VIDEO_FIT_COVER : AVATAR_VIDEO_FIT_CONTAIN;
|
|
749
|
+
const liveLabel = showLive && liveDims && liveDims.width > 0 && liveDims.height > 0 ? `${liveDims.width}\xD7${liveDims.height}` : null;
|
|
750
|
+
const layers = resolveSurfaceLayers({ idleVideoUrl, poster });
|
|
751
|
+
const backFitClass = fitClass;
|
|
752
|
+
const posterLayer = layers.showPoster ? createElement("img", {
|
|
753
|
+
key: "poster",
|
|
754
|
+
src: poster,
|
|
755
|
+
alt: "",
|
|
756
|
+
"aria-hidden": true,
|
|
757
|
+
className: joinClass(AVATAR_VIDEO_LAYER, backFitClass),
|
|
758
|
+
"data-testid": "avatar-poster"
|
|
759
|
+
}) : null;
|
|
760
|
+
const idleLayer = layers.showIdleVideo ? createElement("video", {
|
|
761
|
+
key: "idle",
|
|
762
|
+
ref: idleVideoRef,
|
|
763
|
+
src: idleVideoUrl,
|
|
764
|
+
muted: true,
|
|
765
|
+
// NATIVE loop, not a scripted boomerang. These clips are generated i2v from
|
|
766
|
+
// the portrait with a matched head and tail, so the wrap closes on its own —
|
|
767
|
+
// measured on the shipped roster, every frame of a clip's last 0.1s differs
|
|
768
|
+
// from its frame 0 on 0.0% of the picture. Boomerang existed for clips that
|
|
769
|
+
// did NOT close, and it costs what the render platform retired it for:
|
|
770
|
+
// ambient motion played BACKWARDS (a breath un-inhaling) is an uncanny tell.
|
|
771
|
+
loop: true,
|
|
772
|
+
playsInline: true,
|
|
773
|
+
autoPlay: true,
|
|
774
|
+
preload: "auto",
|
|
775
|
+
className: joinClass(AVATAR_VIDEO_LAYER, backFitClass),
|
|
776
|
+
"data-testid": "avatar-idle-video"
|
|
777
|
+
}) : null;
|
|
778
|
+
const frontLayer = createElement(
|
|
779
|
+
"div",
|
|
780
|
+
{
|
|
781
|
+
key: "live",
|
|
782
|
+
ref: liveWrapRef,
|
|
783
|
+
className: joinClass(AVATAR_VIDEO_LIVE_WRAP, showLive ? void 0 : "pointer-events-none"),
|
|
784
|
+
style: {
|
|
785
|
+
opacity: showLive ? 1 : 0,
|
|
786
|
+
transitionDuration: "0ms"
|
|
787
|
+
},
|
|
788
|
+
"data-testid": "avatar-live-layer",
|
|
789
|
+
"aria-hidden": !showLive
|
|
790
|
+
},
|
|
791
|
+
videoTrack ? createElement(VideoTrack, {
|
|
792
|
+
trackRef: videoTrack,
|
|
793
|
+
className: joinClass(AVATAR_VIDEO_LAYER, fitClass),
|
|
794
|
+
playsInline: true,
|
|
795
|
+
autoPlay: true,
|
|
796
|
+
muted: true,
|
|
797
|
+
"aria-label": "Realtime avatar video"
|
|
798
|
+
}) : null
|
|
799
|
+
);
|
|
800
|
+
const badge = showLiveBadge && showLive ? createElement(
|
|
801
|
+
"span",
|
|
802
|
+
{ key: "badge", className: AVATAR_VIDEO_BADGE },
|
|
803
|
+
createElement("span", { key: "dot", className: AVATAR_VIDEO_BADGE_DOT, "aria-hidden": true }),
|
|
804
|
+
liveLabel ? `live \xB7 ${liveLabel}` : "live"
|
|
805
|
+
) : null;
|
|
806
|
+
return createElement(
|
|
807
|
+
"div",
|
|
808
|
+
{
|
|
809
|
+
className: joinClass(AVATAR_VIDEO_BOX, className),
|
|
810
|
+
style: boxAspect ? { aspectRatio: boxAspect, ...style } : style,
|
|
811
|
+
"data-testid": testId
|
|
812
|
+
},
|
|
813
|
+
// Deepest → shallowest: poster floor (the always-present face), the idle clip
|
|
814
|
+
// over it, then the live video. The poster guarantees the surface is never a
|
|
815
|
+
// black box before/between turns; the idle clip and live render layer over it.
|
|
816
|
+
posterLayer,
|
|
817
|
+
idleLayer,
|
|
818
|
+
frontLayer,
|
|
819
|
+
badge,
|
|
820
|
+
children
|
|
821
|
+
);
|
|
822
|
+
}
|
|
823
|
+
function resolveSurfaceLayers(input) {
|
|
824
|
+
return {
|
|
825
|
+
showPoster: Boolean(input.poster),
|
|
826
|
+
showIdleVideo: Boolean(input.idleVideoUrl)
|
|
827
|
+
};
|
|
828
|
+
}
|
|
829
|
+
function isLiveTrackProducing(videoTrack) {
|
|
830
|
+
const mst = videoTrack?.publication?.track?.mediaStreamTrack;
|
|
831
|
+
return mst != null && mst.readyState === "live" && mst.enabled && !mst.muted;
|
|
832
|
+
}
|
|
833
|
+
function useLiveTrackProducing(videoTrack, isProducing = isLiveTrackProducing) {
|
|
834
|
+
const publication = videoTrack?.publication;
|
|
835
|
+
const [producing, setProducing] = useState(() => isProducing(videoTrack));
|
|
836
|
+
useEffect(() => {
|
|
837
|
+
if (!publication) {
|
|
838
|
+
setProducing(false);
|
|
839
|
+
return;
|
|
840
|
+
}
|
|
841
|
+
const sync = () => setProducing(isProducing(videoTrack));
|
|
842
|
+
sync();
|
|
843
|
+
publication.on(TrackEvent.Muted, sync);
|
|
844
|
+
publication.on(TrackEvent.Unmuted, sync);
|
|
845
|
+
publication.on(TrackEvent.Ended, sync);
|
|
846
|
+
const mst = publication.track?.mediaStreamTrack;
|
|
847
|
+
mst?.addEventListener("ended", sync);
|
|
848
|
+
mst?.addEventListener("mute", sync);
|
|
849
|
+
mst?.addEventListener("unmute", sync);
|
|
850
|
+
return () => {
|
|
851
|
+
publication.off(TrackEvent.Muted, sync);
|
|
852
|
+
publication.off(TrackEvent.Unmuted, sync);
|
|
853
|
+
publication.off(TrackEvent.Ended, sync);
|
|
854
|
+
mst?.removeEventListener("ended", sync);
|
|
855
|
+
mst?.removeEventListener("mute", sync);
|
|
856
|
+
mst?.removeEventListener("unmute", sync);
|
|
857
|
+
};
|
|
858
|
+
}, [publication, videoTrack?.publication?.track, isProducing]);
|
|
859
|
+
return producing;
|
|
860
|
+
}
|
|
861
|
+
function normalizeFrameStallMs(value) {
|
|
862
|
+
if (!Number.isFinite(value)) return DEFAULT_AVATAR_FRAME_STALL_MS;
|
|
863
|
+
return Math.max(100, Math.min(1e4, value));
|
|
864
|
+
}
|
|
865
|
+
function isFrameFlowingAt(snapshot) {
|
|
866
|
+
const stallAfterMs = normalizeFrameStallMs(snapshot.stallAfterMs);
|
|
867
|
+
return snapshot.seenFrame && snapshot.lastFrameAtMs !== null && snapshot.nowMs - snapshot.lastFrameAtMs <= stallAfterMs;
|
|
868
|
+
}
|
|
869
|
+
function freezeMsFromFrameGap(gapMs) {
|
|
870
|
+
if (!Number.isFinite(gapMs) || gapMs <= AVATAR_FRAME_GAP_FREEZE_FLOOR_MS) return 0;
|
|
871
|
+
return gapMs;
|
|
872
|
+
}
|
|
873
|
+
function isFrameFreezeInhibited(snapshot) {
|
|
874
|
+
return snapshot.hidden || snapshot.resumePending || !snapshot.trackProducing || !snapshot.seenFrame || snapshot.lastFrameAtMs === null;
|
|
875
|
+
}
|
|
876
|
+
function useLiveFrameFlow(wrapRef, trackProducing, trackIdentity, stallAfterMs) {
|
|
877
|
+
const boundedStallMs = normalizeFrameStallMs(stallAfterMs);
|
|
878
|
+
const [flowing, setFlowing] = useState(false);
|
|
879
|
+
const [seenFrame, setSeenFrame] = useState(false);
|
|
880
|
+
const flowingRef = useRef(false);
|
|
881
|
+
const sampleRef = useRef({
|
|
882
|
+
seenFrame: false,
|
|
883
|
+
lastFrameAtMs: null,
|
|
884
|
+
maxGapMs: 0,
|
|
885
|
+
resumePending: false
|
|
886
|
+
});
|
|
887
|
+
useEffect(() => {
|
|
888
|
+
sampleRef.current = {
|
|
889
|
+
seenFrame: false,
|
|
890
|
+
lastFrameAtMs: null,
|
|
891
|
+
maxGapMs: 0,
|
|
892
|
+
resumePending: document.visibilityState !== "visible"
|
|
893
|
+
};
|
|
894
|
+
flowingRef.current = false;
|
|
895
|
+
setSeenFrame(false);
|
|
896
|
+
setFlowing(false);
|
|
897
|
+
if (!trackProducing) return;
|
|
898
|
+
const wrap = wrapRef.current;
|
|
899
|
+
if (!wrap) return;
|
|
900
|
+
let video = null;
|
|
901
|
+
let frameCallbackId = null;
|
|
902
|
+
let callbackGeneration = 0;
|
|
903
|
+
let lastCurrentTime = -1;
|
|
904
|
+
const markFrame = () => {
|
|
905
|
+
const previous = sampleRef.current;
|
|
906
|
+
const now = Date.now();
|
|
907
|
+
const firstFrame = !previous.seenFrame;
|
|
908
|
+
sampleRef.current = {
|
|
909
|
+
seenFrame: true,
|
|
910
|
+
lastFrameAtMs: now,
|
|
911
|
+
// A hidden tab/bfcache resume is a local scheduling gap, not network
|
|
912
|
+
// congestion. The first fresh frame becomes the new baseline.
|
|
913
|
+
maxGapMs: previous.resumePending ? 0 : Math.max(
|
|
914
|
+
previous.maxGapMs,
|
|
915
|
+
previous.lastFrameAtMs === null ? 0 : now - previous.lastFrameAtMs
|
|
916
|
+
),
|
|
917
|
+
resumePending: false
|
|
918
|
+
};
|
|
919
|
+
if (firstFrame) setSeenFrame(true);
|
|
920
|
+
if (!flowingRef.current) {
|
|
921
|
+
flowingRef.current = true;
|
|
922
|
+
setFlowing(true);
|
|
923
|
+
}
|
|
924
|
+
};
|
|
925
|
+
const cancelFrameCallback = () => {
|
|
926
|
+
if (video && frameCallbackId !== null && video.cancelVideoFrameCallback) {
|
|
927
|
+
video.cancelVideoFrameCallback(frameCallbackId);
|
|
928
|
+
}
|
|
929
|
+
frameCallbackId = null;
|
|
930
|
+
};
|
|
931
|
+
const bind = (next) => {
|
|
932
|
+
cancelFrameCallback();
|
|
933
|
+
callbackGeneration += 1;
|
|
934
|
+
video = next;
|
|
935
|
+
lastCurrentTime = next?.currentTime ?? -1;
|
|
936
|
+
if (!next?.requestVideoFrameCallback) return;
|
|
937
|
+
const generation = callbackGeneration;
|
|
938
|
+
const onFrame = () => {
|
|
939
|
+
if (generation !== callbackGeneration || !video) return;
|
|
940
|
+
markFrame();
|
|
941
|
+
frameCallbackId = video.requestVideoFrameCallback(onFrame);
|
|
942
|
+
};
|
|
943
|
+
frameCallbackId = next.requestVideoFrameCallback(onFrame);
|
|
944
|
+
};
|
|
945
|
+
const stopObserving = observeVideoElement(wrap, bind);
|
|
946
|
+
const inhibitUntilFreshFrame = () => {
|
|
947
|
+
sampleRef.current.resumePending = true;
|
|
948
|
+
sampleRef.current.maxGapMs = 0;
|
|
949
|
+
};
|
|
950
|
+
const onVisibilityChange = () => {
|
|
951
|
+
if (document.visibilityState !== "visible") inhibitUntilFreshFrame();
|
|
952
|
+
};
|
|
953
|
+
const onPageShow = (event) => {
|
|
954
|
+
if (event.persisted) inhibitUntilFreshFrame();
|
|
955
|
+
};
|
|
956
|
+
document.addEventListener("visibilitychange", onVisibilityChange);
|
|
957
|
+
window.addEventListener("pageshow", onPageShow);
|
|
958
|
+
const pollEveryMs = Math.max(100, Math.min(250, Math.floor(boundedStallMs / 2)));
|
|
959
|
+
const interval = window.setInterval(() => {
|
|
960
|
+
if (video && video.currentTime > lastCurrentTime + 1e-3) {
|
|
961
|
+
lastCurrentTime = video.currentTime;
|
|
962
|
+
markFrame();
|
|
963
|
+
}
|
|
964
|
+
const nextFlowing = isFrameFlowingAt({
|
|
965
|
+
seenFrame: sampleRef.current.seenFrame,
|
|
966
|
+
lastFrameAtMs: sampleRef.current.lastFrameAtMs,
|
|
967
|
+
nowMs: Date.now(),
|
|
968
|
+
stallAfterMs: boundedStallMs
|
|
969
|
+
});
|
|
970
|
+
if (flowingRef.current !== nextFlowing) {
|
|
971
|
+
flowingRef.current = nextFlowing;
|
|
972
|
+
setFlowing(nextFlowing);
|
|
973
|
+
}
|
|
974
|
+
}, pollEveryMs);
|
|
975
|
+
return () => {
|
|
976
|
+
window.clearInterval(interval);
|
|
977
|
+
document.removeEventListener("visibilitychange", onVisibilityChange);
|
|
978
|
+
window.removeEventListener("pageshow", onPageShow);
|
|
979
|
+
stopObserving();
|
|
980
|
+
cancelFrameCallback();
|
|
981
|
+
};
|
|
982
|
+
}, [wrapRef, trackProducing, trackIdentity, boundedStallMs]);
|
|
983
|
+
const freezeReading = useCallback(() => {
|
|
984
|
+
const sample = sampleRef.current;
|
|
985
|
+
const lastFrameAtMs = sample.lastFrameAtMs;
|
|
986
|
+
const hidden = typeof document !== "undefined" && document.visibilityState !== "visible";
|
|
987
|
+
const inhibited = isFrameFreezeInhibited({
|
|
988
|
+
hidden,
|
|
989
|
+
resumePending: sample.resumePending,
|
|
990
|
+
trackProducing,
|
|
991
|
+
seenFrame: sample.seenFrame,
|
|
992
|
+
lastFrameAtMs
|
|
993
|
+
});
|
|
994
|
+
if (inhibited || lastFrameAtMs === null) {
|
|
995
|
+
sample.maxGapMs = 0;
|
|
996
|
+
return { freezeMsInWindow: 0, inhibited: true };
|
|
997
|
+
}
|
|
998
|
+
const ongoingGapMs = Math.max(0, Date.now() - lastFrameAtMs);
|
|
999
|
+
const freezeMsInWindow = freezeMsFromFrameGap(Math.max(sample.maxGapMs, ongoingGapMs));
|
|
1000
|
+
sample.maxGapMs = 0;
|
|
1001
|
+
return { freezeMsInWindow, inhibited: false };
|
|
1002
|
+
}, [trackProducing]);
|
|
1003
|
+
return { flowing, seenFrame, freezeReading };
|
|
1004
|
+
}
|
|
1005
|
+
function useDebouncedHide(wanted, delayMs) {
|
|
1006
|
+
const [shown, setShown] = useState(wanted);
|
|
1007
|
+
useEffect(() => {
|
|
1008
|
+
if (wanted) {
|
|
1009
|
+
setShown(true);
|
|
1010
|
+
return;
|
|
1011
|
+
}
|
|
1012
|
+
if (delayMs <= 0) {
|
|
1013
|
+
setShown(false);
|
|
1014
|
+
return;
|
|
1015
|
+
}
|
|
1016
|
+
const timer = window.setTimeout(() => setShown(false), delayMs);
|
|
1017
|
+
return () => window.clearTimeout(timer);
|
|
1018
|
+
}, [wanted, delayMs]);
|
|
1019
|
+
return shown;
|
|
1020
|
+
}
|
|
1021
|
+
function useIdleWhileResting(videoRef, idleVideoUrl, showLive) {
|
|
1022
|
+
useEffect(() => {
|
|
1023
|
+
const video = videoRef.current;
|
|
1024
|
+
if (!video || !idleVideoUrl) return;
|
|
1025
|
+
applyIdleRest(video, showLive);
|
|
1026
|
+
}, [videoRef, idleVideoUrl, showLive]);
|
|
1027
|
+
}
|
|
1028
|
+
function applyIdleRest(video, showLive) {
|
|
1029
|
+
if (showLive) {
|
|
1030
|
+
video.pause();
|
|
1031
|
+
return;
|
|
1032
|
+
}
|
|
1033
|
+
if (video.currentTime !== 0) video.currentTime = 0;
|
|
1034
|
+
if (video.paused) void Promise.resolve(video.play()).catch(() => {
|
|
1035
|
+
});
|
|
1036
|
+
}
|
|
1037
|
+
function observeVideoElement(wrap, onBind) {
|
|
1038
|
+
let current = wrap.querySelector("video");
|
|
1039
|
+
onBind(current);
|
|
1040
|
+
if (typeof MutationObserver !== "function") {
|
|
1041
|
+
return () => onBind(null);
|
|
1042
|
+
}
|
|
1043
|
+
const observer = new MutationObserver(() => {
|
|
1044
|
+
const next = wrap.querySelector("video");
|
|
1045
|
+
if (next === current) return;
|
|
1046
|
+
current = next;
|
|
1047
|
+
onBind(next);
|
|
1048
|
+
});
|
|
1049
|
+
observer.observe(wrap, { childList: true, subtree: true });
|
|
1050
|
+
return () => {
|
|
1051
|
+
observer.disconnect();
|
|
1052
|
+
onBind(null);
|
|
1053
|
+
};
|
|
1054
|
+
}
|
|
1055
|
+
var LIVE_RESUME_EVENTS = ["canplay", "playing", "loadeddata"];
|
|
1056
|
+
function createLivePlaybackKeeper() {
|
|
1057
|
+
let bound = null;
|
|
1058
|
+
let disposed = false;
|
|
1059
|
+
const resume = () => {
|
|
1060
|
+
if (disposed) return;
|
|
1061
|
+
const el = bound;
|
|
1062
|
+
if (el && el.paused) {
|
|
1063
|
+
void Promise.resolve(el.play()).catch(() => {
|
|
1064
|
+
});
|
|
1065
|
+
}
|
|
1066
|
+
};
|
|
1067
|
+
const unbindListeners = (el) => {
|
|
1068
|
+
if (!el) return;
|
|
1069
|
+
for (const type of LIVE_RESUME_EVENTS) el.removeEventListener(type, resume);
|
|
1070
|
+
};
|
|
1071
|
+
const bind = (video) => {
|
|
1072
|
+
if (disposed || video === bound) return;
|
|
1073
|
+
unbindListeners(bound);
|
|
1074
|
+
bound = video;
|
|
1075
|
+
if (!video) return;
|
|
1076
|
+
for (const type of LIVE_RESUME_EVENTS) video.addEventListener(type, resume);
|
|
1077
|
+
resume();
|
|
1078
|
+
};
|
|
1079
|
+
const dispose = () => {
|
|
1080
|
+
if (disposed) return;
|
|
1081
|
+
disposed = true;
|
|
1082
|
+
unbindListeners(bound);
|
|
1083
|
+
bound = null;
|
|
1084
|
+
};
|
|
1085
|
+
return { bind, resume, dispose };
|
|
1086
|
+
}
|
|
1087
|
+
function useLiveResumeOnProducing(wrapRef, producing) {
|
|
1088
|
+
useEffect(() => {
|
|
1089
|
+
if (!producing) return;
|
|
1090
|
+
const wrap = wrapRef.current;
|
|
1091
|
+
if (!wrap) return;
|
|
1092
|
+
const keeper = createLivePlaybackKeeper();
|
|
1093
|
+
const stopObserving = observeVideoElement(wrap, (video) => keeper.bind(video));
|
|
1094
|
+
const onVisible = () => {
|
|
1095
|
+
if (document.visibilityState === "visible") keeper.resume();
|
|
1096
|
+
};
|
|
1097
|
+
const onPageShow = () => keeper.resume();
|
|
1098
|
+
document.addEventListener("visibilitychange", onVisible);
|
|
1099
|
+
window.addEventListener("pageshow", onPageShow);
|
|
1100
|
+
return () => {
|
|
1101
|
+
document.removeEventListener("visibilitychange", onVisible);
|
|
1102
|
+
window.removeEventListener("pageshow", onPageShow);
|
|
1103
|
+
stopObserving();
|
|
1104
|
+
keeper.dispose();
|
|
1105
|
+
};
|
|
1106
|
+
}, [wrapRef, producing]);
|
|
1107
|
+
}
|
|
1108
|
+
function joinClass(...parts) {
|
|
1109
|
+
return parts.filter(Boolean).join(" ");
|
|
1110
|
+
}
|
|
1111
|
+
var AVATAR_VIDEO_BOX = "relative size-full overflow-hidden";
|
|
1112
|
+
var AVATAR_VIDEO_LAYER = "absolute inset-0 size-full transform-gpu [image-rendering:auto] [object-position:center_22%]";
|
|
1113
|
+
var AVATAR_VIDEO_FIT_CONTAIN = "object-contain";
|
|
1114
|
+
var AVATAR_VIDEO_FIT_COVER = "object-cover";
|
|
1115
|
+
var AVATAR_VIDEO_LIVE_WRAP = "absolute inset-0 z-20 size-full transition-opacity ease-out";
|
|
1116
|
+
var AVATAR_VIDEO_BADGE = "absolute top-2 right-2 z-30 inline-flex items-center gap-1.5 rounded-md bg-black/55 px-1.5 py-0.5 font-mono text-[10px] text-white/85 backdrop-blur-sm";
|
|
1117
|
+
var AVATAR_VIDEO_BADGE_DOT = "size-1.5 rounded-full bg-emerald-400";
|
|
1118
|
+
var DEFAULT_AVATAR_ID = "maria";
|
|
1119
|
+
var DEFAULT_BACKGROUND_ID = "plain_white";
|
|
1120
|
+
var sessionModeSchema = z.enum(["avatar", "voice"]);
|
|
1121
|
+
var DEFAULT_SESSION_MODE = "avatar";
|
|
1122
|
+
var DEFAULT_MAX_SESSION_SECONDS = 1800;
|
|
1123
|
+
var avatarSourceKindSchema = z.enum(["portrait", "source_video"]);
|
|
1124
|
+
var liveKitSttModeSchema = z.enum(["server", "off"]);
|
|
1125
|
+
var renderBackendSchema = z.string();
|
|
1126
|
+
var sessionLiveEditSchema = z.object({
|
|
1127
|
+
rules: z.string().min(1).max(2e3),
|
|
1128
|
+
cooldown_seconds: z.number().int().min(5).max(600).optional(),
|
|
1129
|
+
// Which machinery runs the re-edit. Absent ⇒ the server default ("editor"). A deploy
|
|
1130
|
+
// that cannot provide the requested renderer serves the editor lane and logs it,
|
|
1131
|
+
// rather than failing a call that would otherwise have connected fine.
|
|
1132
|
+
renderer: z.enum(["editor", "generative"]).optional()
|
|
1133
|
+
}).strict();
|
|
1134
|
+
var sessionSupportEditsSchema = z.object({
|
|
1135
|
+
instruction: z.string().min(1).max(1e3),
|
|
1136
|
+
reference_url: z.string().url().optional(),
|
|
1137
|
+
live_edit: sessionLiveEditSchema.optional()
|
|
1138
|
+
}).strict();
|
|
1139
|
+
var LLM_PROVIDERS = ["local", "gemini", "openai"];
|
|
1140
|
+
var llmProviderSchema = z.enum(LLM_PROVIDERS);
|
|
1141
|
+
var llmConfigSchema = z.object({
|
|
1142
|
+
backend: llmProviderSchema.optional(),
|
|
1143
|
+
model: z.string().max(200).nullable().optional()
|
|
1144
|
+
}).strict();
|
|
1145
|
+
var llmSelectionSchema = z.object({
|
|
1146
|
+
provider: llmProviderSchema,
|
|
1147
|
+
model: z.string().max(200).nullable().optional()
|
|
1148
|
+
}).strict();
|
|
1149
|
+
var liveKitInitialContextMessageSchema = z.object({
|
|
1150
|
+
role: z.enum(["system", "user", "assistant"]),
|
|
1151
|
+
content: z.string().min(1).max(4e3)
|
|
1152
|
+
}).strict();
|
|
1153
|
+
var CARTESIA_TTS_MODELS = [
|
|
1154
|
+
"cartesia/sonic-2",
|
|
1155
|
+
"cartesia/sonic-2-latest",
|
|
1156
|
+
"cartesia/sonic-3",
|
|
1157
|
+
"cartesia/sonic-3-latest",
|
|
1158
|
+
"cartesia/sonic-turbo",
|
|
1159
|
+
"cartesia/sonic-turbo-latest"
|
|
1160
|
+
];
|
|
1161
|
+
var cartesiaTtsModelSchema = z.enum(CARTESIA_TTS_MODELS);
|
|
1162
|
+
var cartesiaVoiceSpecSchema = z.object({
|
|
1163
|
+
provider: z.literal("cartesia"),
|
|
1164
|
+
model: cartesiaTtsModelSchema.default("cartesia/sonic-3"),
|
|
1165
|
+
voice_id: z.string().min(1).max(120),
|
|
1166
|
+
speed: z.number().min(0.5).max(2).nullable().optional(),
|
|
1167
|
+
emotion: z.string().min(1).max(80).nullable().optional(),
|
|
1168
|
+
language: z.string().min(2).max(16).nullable().optional()
|
|
1169
|
+
}).strict();
|
|
1170
|
+
var FISH_TTS_MODELS = ["speech-1.6", "s1", "s2-pro", "speech-1.5", "s1-mini"];
|
|
1171
|
+
var fishTtsModelSchema = z.enum(FISH_TTS_MODELS);
|
|
1172
|
+
var breezeVoiceSpecSchema = z.object({
|
|
1173
|
+
provider: z.literal("breezeblue"),
|
|
1174
|
+
model: z.string().min(1).max(80).default("bluebell-v1-en"),
|
|
1175
|
+
voice_id: z.string().min(1).max(120),
|
|
1176
|
+
guidance_scale: z.number().min(1).max(10).nullable().optional(),
|
|
1177
|
+
instructions: z.string().min(1).max(1e3).nullable().optional(),
|
|
1178
|
+
language: z.string().min(2).max(16).nullable().optional()
|
|
1179
|
+
}).strict();
|
|
1180
|
+
var fishVoiceSpecSchema = z.object({
|
|
1181
|
+
provider: z.literal("fish"),
|
|
1182
|
+
model: fishTtsModelSchema.default("speech-1.6"),
|
|
1183
|
+
voice_id: z.string().min(1).max(120),
|
|
1184
|
+
speed: z.number().min(0.5).max(2).nullable().optional(),
|
|
1185
|
+
emotion: z.string().min(1).max(80).nullable().optional(),
|
|
1186
|
+
language: z.string().min(2).max(16).nullable().optional()
|
|
1187
|
+
}).strict();
|
|
1188
|
+
var voiceSpecSchema = z.discriminatedUnion("provider", [
|
|
1189
|
+
cartesiaVoiceSpecSchema,
|
|
1190
|
+
breezeVoiceSpecSchema,
|
|
1191
|
+
fishVoiceSpecSchema
|
|
1192
|
+
]);
|
|
1193
|
+
var nullableUrlSchema = z.string().url().nullable();
|
|
1194
|
+
var clipTriggerSchema = z.enum([
|
|
1195
|
+
"idle",
|
|
1196
|
+
"listen",
|
|
1197
|
+
"think",
|
|
1198
|
+
"directive"
|
|
1199
|
+
]);
|
|
1200
|
+
var sessionClipSchema = z.object({
|
|
1201
|
+
clip_id: z.string().regex(/^[A-Za-z0-9_-]{1,64}$/, "clip_id must be a slug").refine((id) => id !== "primary", "'primary' is reserved for the avatar's source video"),
|
|
1202
|
+
source_video_url: z.string().url().optional(),
|
|
1203
|
+
video_cache_id: z.string().min(8).max(160).optional(),
|
|
1204
|
+
max_seconds: z.number().min(1).max(10).optional(),
|
|
1205
|
+
trigger: clipTriggerSchema.optional(),
|
|
1206
|
+
loop: z.boolean().optional(),
|
|
1207
|
+
weight: z.number().min(0).max(100).optional(),
|
|
1208
|
+
crossfade_ms: z.number().int().min(0).max(1e3).optional(),
|
|
1209
|
+
trim_start_ms: z.number().int().min(0).max(2e3).optional(),
|
|
1210
|
+
trim_end_ms: z.number().int().min(0).max(2e3).optional(),
|
|
1211
|
+
// The cue the character reads to decide this clip. `when` is the public name and the
|
|
1212
|
+
// one the docs use; `hint` is the name the wire first shipped under and still accepts.
|
|
1213
|
+
// Both are listed because this object is `.strict()` — omitting `when` would make the
|
|
1214
|
+
// public name a validation error. Send one, never both.
|
|
1215
|
+
when: z.string().min(1).max(120).optional(),
|
|
1216
|
+
hint: z.string().min(1).max(120).optional()
|
|
1217
|
+
}).strict().refine((clip) => clip.source_video_url || clip.video_cache_id, {
|
|
1218
|
+
message: "a clip needs source_video_url or video_cache_id"
|
|
1219
|
+
}).refine((clip) => !(clip.when && clip.hint), {
|
|
1220
|
+
message: "set `when` or `hint`, not both \u2014 they are the same field",
|
|
1221
|
+
path: ["when"]
|
|
1222
|
+
});
|
|
1223
|
+
var sessionChoreographySchema = z.object({
|
|
1224
|
+
idle_dwell_min_seconds: z.number().min(1).max(60).optional(),
|
|
1225
|
+
idle_dwell_max_seconds: z.number().min(1).max(120).optional(),
|
|
1226
|
+
special_weight: z.number().min(0).max(100).optional(),
|
|
1227
|
+
start_grace_seconds: z.number().min(0).max(60).optional(),
|
|
1228
|
+
crossfade_ms: z.number().int().min(0).max(1e3).optional(),
|
|
1229
|
+
crossfade_easing: z.enum(["linear", "smooth", "ease_out"]).optional(),
|
|
1230
|
+
wrap_crossfade_ms: z.number().int().min(0).max(1e3).optional()
|
|
1231
|
+
}).strict().refine(
|
|
1232
|
+
(c) => c.idle_dwell_min_seconds === void 0 || c.idle_dwell_max_seconds === void 0 || c.idle_dwell_min_seconds <= c.idle_dwell_max_seconds,
|
|
1233
|
+
{ message: "idle_dwell_min_seconds must be <= idle_dwell_max_seconds" }
|
|
1234
|
+
);
|
|
1235
|
+
var sessionBehaviorSchema = z.object({
|
|
1236
|
+
gestures_enabled: z.boolean().optional(),
|
|
1237
|
+
gesture_freq: z.enum(["sparse", "balanced", "lively"]).optional()
|
|
1238
|
+
}).strict();
|
|
1239
|
+
var _SCENE_CLIP_ID_RE = /^[A-Za-z0-9_-]{1,64}$/;
|
|
1240
|
+
var sceneIdSchema = z.string().regex(/^[a-z0-9_]{1,40}$/, "scene_id must be a lowercase slug");
|
|
1241
|
+
var sceneTransitionSchema = z.object({
|
|
1242
|
+
clip_id: z.string().regex(_SCENE_CLIP_ID_RE, "clip_id must be a slug"),
|
|
1243
|
+
source_video_url: z.string().url(),
|
|
1244
|
+
from_scene: sceneIdSchema,
|
|
1245
|
+
to_scene: sceneIdSchema,
|
|
1246
|
+
max_seconds: z.number().min(1).max(10).optional()
|
|
1247
|
+
}).strict().superRefine((v, ctx) => {
|
|
1248
|
+
if (v.from_scene === v.to_scene) {
|
|
1249
|
+
ctx.addIssue({ code: "custom", message: "a transition's from_scene must differ from to_scene", path: ["to_scene"] });
|
|
1250
|
+
}
|
|
1251
|
+
});
|
|
1252
|
+
var sceneClusterSchema = z.object({
|
|
1253
|
+
scene_id: sceneIdSchema,
|
|
1254
|
+
hub_clip_id: z.string().regex(_SCENE_CLIP_ID_RE, "hub_clip_id must be a slug"),
|
|
1255
|
+
clips: z.array(sessionClipSchema).min(1).max(4)
|
|
1256
|
+
}).strict();
|
|
1257
|
+
var sceneGraphSchema = z.object({
|
|
1258
|
+
scenes: z.array(sceneClusterSchema).min(1).max(4),
|
|
1259
|
+
transitions: z.array(sceneTransitionSchema).min(2).max(12)
|
|
1260
|
+
}).strict();
|
|
1261
|
+
var transcriptWebhookSchema = z.object({
|
|
1262
|
+
url: z.string().url().max(500),
|
|
1263
|
+
secret: z.string().min(16).max(200)
|
|
1264
|
+
}).strict();
|
|
1265
|
+
var clientMetadataSchema = z.record(z.string().min(1).max(64), z.string().max(200)).refine((value) => Object.keys(value).length <= 16, {
|
|
1266
|
+
message: "client_metadata carries at most 16 entries"
|
|
1267
|
+
});
|
|
1268
|
+
var MAX_SESSION_INSTRUCTIONS_CHARS = 4e3;
|
|
1269
|
+
z.object({
|
|
1270
|
+
avatar_id: z.string().min(1).max(160).default(DEFAULT_AVATAR_ID),
|
|
1271
|
+
background_id: z.string().min(1).max(160).default(DEFAULT_BACKGROUND_ID),
|
|
1272
|
+
mode: sessionModeSchema.default(DEFAULT_SESSION_MODE),
|
|
1273
|
+
create_room: z.boolean().default(true),
|
|
1274
|
+
dispatch_agent: z.boolean().default(true),
|
|
1275
|
+
instructions: z.string().min(1).max(MAX_SESSION_INSTRUCTIONS_CHARS).optional(),
|
|
1276
|
+
initial_context: z.array(liveKitInitialContextMessageSchema).max(32).default([]),
|
|
1277
|
+
initial_say: z.string().min(1).max(1e3).optional(),
|
|
1278
|
+
llm: llmConfigSchema.nullable().optional(),
|
|
1279
|
+
max_session_seconds: z.number().int().min(1).max(DEFAULT_MAX_SESSION_SECONDS).optional(),
|
|
1280
|
+
participant_identity: z.string().min(1).max(160).optional(),
|
|
1281
|
+
participant_name: z.string().max(160).optional(),
|
|
1282
|
+
queue_ticket_id: z.string().min(1).max(160).optional(),
|
|
1283
|
+
portrait_url: nullableUrlSchema.optional(),
|
|
1284
|
+
room_name: z.string().min(1).max(160).optional(),
|
|
1285
|
+
source_kind: avatarSourceKindSchema.default("portrait"),
|
|
1286
|
+
source_video_url: nullableUrlSchema.optional(),
|
|
1287
|
+
stt_mode: liveKitSttModeSchema.default("server"),
|
|
1288
|
+
video_cache_id: z.string().min(1).max(240).nullable().optional(),
|
|
1289
|
+
voice: voiceSpecSchema.nullable().optional(),
|
|
1290
|
+
voice_id: z.string().min(1).max(240).nullable().optional(),
|
|
1291
|
+
// Deliberately UNBOUNDED. This object is `.strict()`, so it rejects rather than trims:
|
|
1292
|
+
// a count cap here does not mean "use fewer clips", it means "there is no call". How
|
|
1293
|
+
// many a session actually warms is decided where the clips are loaded, and loading
|
|
1294
|
+
// fewer is always safe — so the wire must not hold a number too.
|
|
1295
|
+
clip_library: z.array(sessionClipSchema).optional(),
|
|
1296
|
+
choreography: sessionChoreographySchema.optional(),
|
|
1297
|
+
scene_graph: sceneGraphSchema.optional(),
|
|
1298
|
+
behavior: sessionBehaviorSchema.optional(),
|
|
1299
|
+
expression_profile: z.string().min(1).max(40).optional(),
|
|
1300
|
+
render_backend: renderBackendSchema.optional(),
|
|
1301
|
+
support_edits: sessionSupportEditsSchema.optional(),
|
|
1302
|
+
transcript_webhook: transcriptWebhookSchema.optional(),
|
|
1303
|
+
client_metadata: clientMetadataSchema.optional()
|
|
1304
|
+
}).strict().superRefine((value, ctx) => {
|
|
1305
|
+
if (value.support_edits && value.render_backend === "generative") {
|
|
1306
|
+
ctx.addIssue({
|
|
1307
|
+
code: "custom",
|
|
1308
|
+
message: "support_edits needs a source video to edit; it cannot be combined with render_backend='generative'",
|
|
1309
|
+
path: ["support_edits"]
|
|
1310
|
+
});
|
|
1311
|
+
}
|
|
1312
|
+
if (value.support_edits && value.mode === "voice") {
|
|
1313
|
+
ctx.addIssue({
|
|
1314
|
+
code: "custom",
|
|
1315
|
+
message: "support_edits needs a video session; it cannot be combined with mode='voice'",
|
|
1316
|
+
path: ["support_edits"]
|
|
1317
|
+
});
|
|
1318
|
+
}
|
|
1319
|
+
if (value.source_kind === "portrait") {
|
|
1320
|
+
if (value.source_video_url || value.video_cache_id) {
|
|
1321
|
+
ctx.addIssue({
|
|
1322
|
+
code: "custom",
|
|
1323
|
+
message: "source_video_url/video_cache_id require source_kind='source_video'",
|
|
1324
|
+
path: ["source_kind"]
|
|
1325
|
+
});
|
|
1326
|
+
}
|
|
1327
|
+
return;
|
|
1328
|
+
}
|
|
1329
|
+
if (value.portrait_url) {
|
|
1330
|
+
ctx.addIssue({
|
|
1331
|
+
code: "custom",
|
|
1332
|
+
message: "portrait_url cannot be combined with source_kind='source_video'",
|
|
1333
|
+
path: ["portrait_url"]
|
|
1334
|
+
});
|
|
1335
|
+
}
|
|
1336
|
+
if (!value.source_video_url && !value.video_cache_id) {
|
|
1337
|
+
ctx.addIssue({
|
|
1338
|
+
code: "custom",
|
|
1339
|
+
message: "source_kind='source_video' requires source_video_url or video_cache_id",
|
|
1340
|
+
path: ["source_video_url"]
|
|
1341
|
+
});
|
|
1342
|
+
}
|
|
1343
|
+
});
|
|
1344
|
+
z.object({
|
|
1345
|
+
avatarId: z.string().min(1).max(160),
|
|
1346
|
+
backgroundId: z.string().min(1).max(160).default(DEFAULT_BACKGROUND_ID),
|
|
1347
|
+
mode: sessionModeSchema.default(DEFAULT_SESSION_MODE),
|
|
1348
|
+
createRoom: z.boolean().default(true),
|
|
1349
|
+
dispatchAgent: z.boolean().default(true),
|
|
1350
|
+
instructions: z.string().min(1).max(MAX_SESSION_INSTRUCTIONS_CHARS).optional(),
|
|
1351
|
+
initialContext: z.array(liveKitInitialContextMessageSchema).max(32).default([]),
|
|
1352
|
+
initialSay: z.string().min(1).max(1e3).optional(),
|
|
1353
|
+
llm: llmSelectionSchema.nullable().optional(),
|
|
1354
|
+
maxSessionSeconds: z.number().int().min(1).max(DEFAULT_MAX_SESSION_SECONDS).optional(),
|
|
1355
|
+
participantIdentity: z.string().min(1).max(160).optional(),
|
|
1356
|
+
participantName: z.string().max(160).optional(),
|
|
1357
|
+
queueTicketId: z.string().min(1).max(160).optional(),
|
|
1358
|
+
roomName: z.string().min(1).max(160).optional(),
|
|
1359
|
+
sttMode: liveKitSttModeSchema.default("server"),
|
|
1360
|
+
voice: voiceSpecSchema.nullable().optional(),
|
|
1361
|
+
voiceId: z.string().min(1).max(240).nullable().optional(),
|
|
1362
|
+
// Unbounded, for the same reason as `clip_library` on the wire schema above.
|
|
1363
|
+
clipLibrary: z.array(sessionClipSchema).optional(),
|
|
1364
|
+
sceneGraph: sceneGraphSchema.optional(),
|
|
1365
|
+
behavior: sessionBehaviorSchema.optional(),
|
|
1366
|
+
renderBackend: renderBackendSchema.optional(),
|
|
1367
|
+
supportEdits: sessionSupportEditsSchema.optional(),
|
|
1368
|
+
transcriptWebhook: transcriptWebhookSchema.optional(),
|
|
1369
|
+
clientMetadata: clientMetadataSchema.optional()
|
|
1370
|
+
}).strict();
|
|
1371
|
+
z.object({
|
|
1372
|
+
status: z.literal("ready").default("ready"),
|
|
1373
|
+
session_id: z.string().min(1),
|
|
1374
|
+
room_name: z.string().min(1),
|
|
1375
|
+
livekit_url: z.string().min(1),
|
|
1376
|
+
participant_token: z.string().min(1),
|
|
1377
|
+
participant_identity: z.string().min(1),
|
|
1378
|
+
reservation_expires_at: z.string().datetime({ offset: true }),
|
|
1379
|
+
stt_mode: liveKitSttModeSchema.default("server"),
|
|
1380
|
+
room_created: z.boolean().default(false),
|
|
1381
|
+
dispatch_created: z.boolean().default(false),
|
|
1382
|
+
join_timeout_seconds: z.number().int().nonnegative().default(0),
|
|
1383
|
+
idle_timeout_seconds: z.number().int().nonnegative().default(0),
|
|
1384
|
+
max_session_seconds: z.number().int().nonnegative().default(0)
|
|
1385
|
+
}).passthrough();
|
|
1386
|
+
var RTA_LIFECYCLE_TOPIC = "rta.lifecycle";
|
|
1387
|
+
var RTA_TURN_INSTRUCTIONS_ATTR = "rta.turn_instructions";
|
|
1388
|
+
var RTA_CLOSING_TURN_ATTR = "rta.closing_turn";
|
|
1389
|
+
var RTA_TURN_ID_ATTR = "rta.turn_id";
|
|
1390
|
+
var sessionEndReasonSchema = z.enum([
|
|
1391
|
+
"user_ended",
|
|
1392
|
+
"session_cap",
|
|
1393
|
+
"idle",
|
|
1394
|
+
"disconnected",
|
|
1395
|
+
"out_of_credits",
|
|
1396
|
+
"agent_ended",
|
|
1397
|
+
"failed"
|
|
1398
|
+
]);
|
|
1399
|
+
var approachingEndReasonSchema = z.enum(["session_cap", "idle"]);
|
|
1400
|
+
var sessionClockFrameSchema = z.object({
|
|
1401
|
+
kind: z.literal("session_clock"),
|
|
1402
|
+
started_at_unix_ms: z.number().int().nonnegative(),
|
|
1403
|
+
max_session_seconds: z.number().int().nonnegative(),
|
|
1404
|
+
idle_timeout_seconds: z.number().int().nonnegative()
|
|
1405
|
+
}).strict();
|
|
1406
|
+
var endingFrameSchema = z.object({ kind: z.literal("ending"), reason: approachingEndReasonSchema }).strict();
|
|
1407
|
+
var closingTurnDoneFrameSchema = z.object({ kind: z.literal("closing_turn_done"), turn_id: z.string().min(1) }).strict();
|
|
1408
|
+
var endedFrameSchema = z.object({ kind: z.literal("ended"), reason: sessionEndReasonSchema }).strict();
|
|
1409
|
+
var knownBehaviorStates = ["idle", "listening", "thinking", "speaking"];
|
|
1410
|
+
var behaviorStateFrameSchema = z.object({
|
|
1411
|
+
kind: z.literal("behavior_state"),
|
|
1412
|
+
state: z.string().min(1).max(32),
|
|
1413
|
+
clip_id: z.string().min(1).max(64).optional(),
|
|
1414
|
+
trigger: clipTriggerSchema.optional(),
|
|
1415
|
+
loop: z.boolean().optional(),
|
|
1416
|
+
prev_clip_id: z.string().min(1).max(64).optional(),
|
|
1417
|
+
scene: z.string().min(1).max(40).optional(),
|
|
1418
|
+
scene_transition: z.object({
|
|
1419
|
+
clip_id: z.string().min(1).max(64),
|
|
1420
|
+
from_scene: z.string().min(1).max(40),
|
|
1421
|
+
to_scene: z.string().min(1).max(40)
|
|
1422
|
+
}).optional()
|
|
1423
|
+
}).strip();
|
|
1424
|
+
var clipAckFrameSchema = z.object({
|
|
1425
|
+
kind: z.literal("clip_ack"),
|
|
1426
|
+
request_id: z.string().max(64),
|
|
1427
|
+
accepted: z.boolean(),
|
|
1428
|
+
reason: z.string().max(64)
|
|
1429
|
+
}).strip();
|
|
1430
|
+
var lifecycleServerFrameSchema = z.discriminatedUnion("kind", [
|
|
1431
|
+
sessionClockFrameSchema,
|
|
1432
|
+
endingFrameSchema,
|
|
1433
|
+
closingTurnDoneFrameSchema,
|
|
1434
|
+
endedFrameSchema,
|
|
1435
|
+
behaviorStateFrameSchema,
|
|
1436
|
+
clipAckFrameSchema
|
|
1437
|
+
]);
|
|
1438
|
+
var liveKitCapacitySnapshotSchema = z.object({
|
|
1439
|
+
// Placement identity + per-worker session ceiling. These are always present on the
|
|
1440
|
+
// wire (the platform serializes them and they come back on the grant, so a customer
|
|
1441
|
+
// already sees them) but are typed OPTIONAL here on purpose: this SDK is the defensive
|
|
1442
|
+
// READER of that wire, so a consumer must tolerate a response variant that omits them
|
|
1443
|
+
// rather than hard-fail parse. `max_sessions_per_gpu` is how many sessions one waking
|
|
1444
|
+
// worker serves — a queue-depth estimate input a consumer reads off the busy response.
|
|
1445
|
+
capacity_pool: z.string().min(1).optional(),
|
|
1446
|
+
agent_name: z.string().min(1).optional(),
|
|
1447
|
+
max_sessions: z.number().int().nonnegative(),
|
|
1448
|
+
max_sessions_per_gpu: z.number().int().positive().optional(),
|
|
1449
|
+
worker_count: z.number().int().nonnegative(),
|
|
1450
|
+
active_sessions: z.number().int().nonnegative(),
|
|
1451
|
+
reserved_sessions: z.number().int().nonnegative(),
|
|
1452
|
+
observed_worker_active_sessions: z.number().int().nonnegative(),
|
|
1453
|
+
available_sessions: z.number().int().nonnegative(),
|
|
1454
|
+
queue_size: z.number().int().nonnegative(),
|
|
1455
|
+
admission_open: z.boolean(),
|
|
1456
|
+
recommended_retry_ms: z.number().int().nonnegative(),
|
|
1457
|
+
load: z.number().min(0).max(1)
|
|
1458
|
+
}).passthrough();
|
|
1459
|
+
z.object({
|
|
1460
|
+
message: z.string().min(1),
|
|
1461
|
+
capacity: liveKitCapacitySnapshotSchema,
|
|
1462
|
+
queue_size: z.number().int().nonnegative(),
|
|
1463
|
+
queue_ticket_id: z.string().min(1).optional(),
|
|
1464
|
+
queue_position: z.number().int().positive().optional(),
|
|
1465
|
+
recommended_retry_ms: z.number().int().nonnegative()
|
|
1466
|
+
}).strict();
|
|
1467
|
+
z.enum([
|
|
1468
|
+
"page_hide",
|
|
1469
|
+
"disconnected",
|
|
1470
|
+
"superseded",
|
|
1471
|
+
"unmount",
|
|
1472
|
+
"manual",
|
|
1473
|
+
"idle_timeout"
|
|
1474
|
+
]);
|
|
1475
|
+
|
|
1476
|
+
// ../client/src/react/behavior-snapshot.ts
|
|
1477
|
+
function nextBehaviorSnapshot(prev, frame) {
|
|
1478
|
+
const next = {
|
|
1479
|
+
state: frame.state,
|
|
1480
|
+
clipId: frame.clip_id ?? null,
|
|
1481
|
+
trigger: frame.trigger ?? null,
|
|
1482
|
+
loop: frame.loop ?? null,
|
|
1483
|
+
prevClipId: frame.prev_clip_id ?? null
|
|
1484
|
+
};
|
|
1485
|
+
if (prev && prev.state === next.state && prev.clipId === next.clipId && prev.trigger === next.trigger && prev.loop === next.loop && prev.prevClipId === next.prevClipId)
|
|
1486
|
+
return null;
|
|
1487
|
+
return next;
|
|
1488
|
+
}
|
|
1489
|
+
|
|
1490
|
+
// ../client/src/react/session-lifecycle.ts
|
|
1491
|
+
function phaseForReason(reason) {
|
|
1492
|
+
switch (reason) {
|
|
1493
|
+
case DisconnectReason.CLIENT_INITIATED:
|
|
1494
|
+
return "ignore";
|
|
1495
|
+
case DisconnectReason.ROOM_DELETED:
|
|
1496
|
+
case DisconnectReason.SERVER_SHUTDOWN:
|
|
1497
|
+
case DisconnectReason.DUPLICATE_IDENTITY:
|
|
1498
|
+
case DisconnectReason.PARTICIPANT_REMOVED:
|
|
1499
|
+
return "ended";
|
|
1500
|
+
default:
|
|
1501
|
+
return "reconnecting";
|
|
1502
|
+
}
|
|
1503
|
+
}
|
|
1504
|
+
function disconnectAction(reason, _everConnected) {
|
|
1505
|
+
const phase = phaseForReason(reason);
|
|
1506
|
+
if (phase === "ignore") return "reset";
|
|
1507
|
+
if (phase === "ended") return "end";
|
|
1508
|
+
return "reconnect";
|
|
1509
|
+
}
|
|
1510
|
+
function isRecoverableConnectionError(error) {
|
|
1511
|
+
if (!(error instanceof ConnectionError)) return false;
|
|
1512
|
+
switch (error.reason) {
|
|
1513
|
+
case ConnectionErrorReason.ServerUnreachable:
|
|
1514
|
+
case ConnectionErrorReason.InternalError:
|
|
1515
|
+
case ConnectionErrorReason.Timeout:
|
|
1516
|
+
case ConnectionErrorReason.WebSocket:
|
|
1517
|
+
return true;
|
|
1518
|
+
default:
|
|
1519
|
+
return false;
|
|
1520
|
+
}
|
|
1521
|
+
}
|
|
1522
|
+
function shouldReplayPendingTurn(args) {
|
|
1523
|
+
const { wasReconnecting, hasPendingTurn, sessionId, lastReplayedSessionId } = args;
|
|
1524
|
+
if (!wasReconnecting || !hasPendingTurn || !sessionId) return false;
|
|
1525
|
+
return lastReplayedSessionId !== sessionId;
|
|
1526
|
+
}
|
|
1527
|
+
var RECONNECT_BACKOFF_MS = [800, 2e3, 4e3];
|
|
1528
|
+
var MAX_RECONNECT_ATTEMPTS = 4;
|
|
1529
|
+
function resolveReconnectPolicy(override) {
|
|
1530
|
+
const backoff = override?.backoffMs?.filter((ms) => Number.isFinite(ms) && ms > 0) ?? [];
|
|
1531
|
+
const backoffMs = backoff.length > 0 ? backoff : RECONNECT_BACKOFF_MS;
|
|
1532
|
+
const max = override?.maxAttempts;
|
|
1533
|
+
const maxAttempts = typeof max === "number" && Number.isFinite(max) && max > 0 ? Math.floor(max) : MAX_RECONNECT_ATTEMPTS;
|
|
1534
|
+
return { backoffMs, maxAttempts };
|
|
1535
|
+
}
|
|
1536
|
+
function retryStep(attempt, policy = resolveReconnectPolicy()) {
|
|
1537
|
+
if (attempt >= policy.maxAttempts) return { kind: "give-up" };
|
|
1538
|
+
const delayMs = policy.backoffMs[Math.min(attempt, policy.backoffMs.length - 1)];
|
|
1539
|
+
return { kind: "retry", delayMs, attempt: attempt + 1 };
|
|
1540
|
+
}
|
|
1541
|
+
function idlePhaseFor(args) {
|
|
1542
|
+
const { connected, agentPresent, msSinceActivity, idleTimeoutMs, warnAtMs } = args;
|
|
1543
|
+
if (!connected || !agentPresent || idleTimeoutMs <= 0) return { kind: "inactive" };
|
|
1544
|
+
const remainingMs = idleTimeoutMs - msSinceActivity;
|
|
1545
|
+
if (remainingMs > warnAtMs) return { kind: "live" };
|
|
1546
|
+
const secondsRemaining = Math.max(0, Math.ceil(remainingMs / 1e3));
|
|
1547
|
+
return { kind: "idle-warning", secondsRemaining };
|
|
1548
|
+
}
|
|
1549
|
+
function idleExpired(args) {
|
|
1550
|
+
const { msSinceActivity, idleTimeoutMs } = args;
|
|
1551
|
+
if (idleTimeoutMs <= 0) return false;
|
|
1552
|
+
return msSinceActivity >= idleTimeoutMs;
|
|
1553
|
+
}
|
|
1554
|
+
function needsFreshGrant(recovery) {
|
|
1555
|
+
return recovery.kind === "reconnecting";
|
|
1556
|
+
}
|
|
1557
|
+
function lifecyclePhaseFrom(args) {
|
|
1558
|
+
const { capacity, recovery, idle, connected, agentPresent } = args;
|
|
1559
|
+
if (recovery.kind === "in-place-reconnecting") {
|
|
1560
|
+
return { kind: "reconnectable", reconnecting: true, attempt: 0, strategy: "in-place" };
|
|
1561
|
+
}
|
|
1562
|
+
if (recovery.kind === "refreshing" || recovery.kind === "reconnecting") {
|
|
1563
|
+
return {
|
|
1564
|
+
kind: "reconnectable",
|
|
1565
|
+
reconnecting: true,
|
|
1566
|
+
attempt: recovery.attempt,
|
|
1567
|
+
strategy: "fresh-grant"
|
|
1568
|
+
};
|
|
1569
|
+
}
|
|
1570
|
+
if (recovery.kind === "failed") {
|
|
1571
|
+
return {
|
|
1572
|
+
kind: "reconnectable",
|
|
1573
|
+
reconnecting: false,
|
|
1574
|
+
attempt: MAX_RECONNECT_ATTEMPTS,
|
|
1575
|
+
strategy: "fresh-grant"
|
|
1576
|
+
};
|
|
1577
|
+
}
|
|
1578
|
+
if (recovery.kind === "ended") {
|
|
1579
|
+
return recovery.reason ? { kind: "ended", reason: recovery.reason } : { kind: "ended" };
|
|
1580
|
+
}
|
|
1581
|
+
switch (capacity.kind) {
|
|
1582
|
+
case "idle":
|
|
1583
|
+
return { kind: "idle" };
|
|
1584
|
+
case "connecting":
|
|
1585
|
+
return { kind: "requesting" };
|
|
1586
|
+
case "queued":
|
|
1587
|
+
return { kind: "queued", busy: capacity.busy };
|
|
1588
|
+
case "error":
|
|
1589
|
+
return { kind: "ended", reason: "error" };
|
|
1590
|
+
case "active": {
|
|
1591
|
+
if (!connected || !agentPresent) {
|
|
1592
|
+
return { kind: "connecting", grant: capacity.grant };
|
|
1593
|
+
}
|
|
1594
|
+
if (idle.kind === "idle-warning") {
|
|
1595
|
+
return { kind: "idle-warning", secondsRemaining: idle.secondsRemaining, deadlineAt: idle.deadlineAt };
|
|
1596
|
+
}
|
|
1597
|
+
return { kind: "live" };
|
|
1598
|
+
}
|
|
1599
|
+
default:
|
|
1600
|
+
return { kind: "idle" };
|
|
1601
|
+
}
|
|
1602
|
+
}
|
|
1603
|
+
function isCallActivity(args) {
|
|
1604
|
+
const { assistantState, transcriptionCount, prevTranscriptionCount } = args;
|
|
1605
|
+
if (assistantState === "speaking" || assistantState === "thinking") return true;
|
|
1606
|
+
return transcriptionCount > prevTranscriptionCount;
|
|
1607
|
+
}
|
|
1608
|
+
var DEFAULT_IDLE_SECONDS = 120;
|
|
1609
|
+
var DEFAULT_IDLE_WARN_LEAD_SECONDS = 20;
|
|
1610
|
+
var DEFAULT_TURN_TIMEOUT_SECONDS = 20;
|
|
1611
|
+
function resolveIdleTimeoutMs(args) {
|
|
1612
|
+
const { idleSecondsOption, grantIdleSeconds } = args;
|
|
1613
|
+
if (typeof idleSecondsOption === "number" && idleSecondsOption > 0) {
|
|
1614
|
+
return Math.floor(idleSecondsOption) * 1e3;
|
|
1615
|
+
}
|
|
1616
|
+
if (grantIdleSeconds > 0) return grantIdleSeconds * 1e3;
|
|
1617
|
+
return DEFAULT_IDLE_SECONDS * 1e3;
|
|
1618
|
+
}
|
|
1619
|
+
function resolveWarnBeforeMs(idleTimeoutMs, override, idleWarnLeadSeconds) {
|
|
1620
|
+
if (idleTimeoutMs <= 0) return 0;
|
|
1621
|
+
const maxLeadMs = Math.max(0, idleTimeoutMs - 1e3);
|
|
1622
|
+
let leadMs;
|
|
1623
|
+
if (typeof idleWarnLeadSeconds === "number" && idleWarnLeadSeconds > 0) {
|
|
1624
|
+
leadMs = Math.floor(idleWarnLeadSeconds) * 1e3;
|
|
1625
|
+
} else if (typeof override === "number" && override > 0) {
|
|
1626
|
+
leadMs = override;
|
|
1627
|
+
} else {
|
|
1628
|
+
leadMs = DEFAULT_IDLE_WARN_LEAD_SECONDS * 1e3;
|
|
1629
|
+
}
|
|
1630
|
+
return Math.min(leadMs, maxLeadMs);
|
|
1631
|
+
}
|
|
1632
|
+
function useSessionLifecycle(input) {
|
|
1633
|
+
const {
|
|
1634
|
+
client,
|
|
1635
|
+
session,
|
|
1636
|
+
active = true,
|
|
1637
|
+
idleSeconds,
|
|
1638
|
+
idleWarnLeadSeconds,
|
|
1639
|
+
warnBeforeMs,
|
|
1640
|
+
reconnectBackoffMs,
|
|
1641
|
+
maxReconnectAttempts,
|
|
1642
|
+
autoRetryBusy = true,
|
|
1643
|
+
requestOptions,
|
|
1644
|
+
onBehaviorChange
|
|
1645
|
+
} = input;
|
|
1646
|
+
const reconnectPolicy = useMemo(
|
|
1647
|
+
() => resolveReconnectPolicy({ backoffMs: reconnectBackoffMs, maxAttempts: maxReconnectAttempts }),
|
|
1648
|
+
[reconnectBackoffMs, maxReconnectAttempts]
|
|
1649
|
+
);
|
|
1650
|
+
const reconnectPolicyRef = useRef(reconnectPolicy);
|
|
1651
|
+
reconnectPolicyRef.current = reconnectPolicy;
|
|
1652
|
+
const grantState = useLiveKitAvatarGrant({
|
|
1653
|
+
client,
|
|
1654
|
+
session,
|
|
1655
|
+
active,
|
|
1656
|
+
autoRetryBusy,
|
|
1657
|
+
requestOptions
|
|
1658
|
+
});
|
|
1659
|
+
const capacity = grantState.capacity;
|
|
1660
|
+
const [recovery, setRecovery] = useState({ kind: "connected" });
|
|
1661
|
+
const recoveryRef = useRef(recovery);
|
|
1662
|
+
recoveryRef.current = recovery;
|
|
1663
|
+
const [attempt, setAttempt] = useState(0);
|
|
1664
|
+
const connectedRef = useRef(false);
|
|
1665
|
+
const attemptRef = useRef(0);
|
|
1666
|
+
const timerRef = useRef(null);
|
|
1667
|
+
const manualReconnectPendingRef = useRef(false);
|
|
1668
|
+
const refreshRef = useRef(grantState.refresh);
|
|
1669
|
+
refreshRef.current = grantState.refresh;
|
|
1670
|
+
const releaseRef = useRef(grantState.release);
|
|
1671
|
+
releaseRef.current = grantState.release;
|
|
1672
|
+
const capacityRef = useRef(capacity);
|
|
1673
|
+
capacityRef.current = capacity;
|
|
1674
|
+
const refreshBaselineCapacityRef = useRef(null);
|
|
1675
|
+
const idleTimeoutMs = resolveIdleTimeoutMs({
|
|
1676
|
+
idleSecondsOption: idleSeconds,
|
|
1677
|
+
grantIdleSeconds: grantState.grant?.idle_timeout_seconds ?? 0
|
|
1678
|
+
});
|
|
1679
|
+
const warnAtMs = resolveWarnBeforeMs(idleTimeoutMs, warnBeforeMs, idleWarnLeadSeconds);
|
|
1680
|
+
const [agentPresent, setAgentPresentState] = useState(false);
|
|
1681
|
+
const [connected, setConnected] = useState(false);
|
|
1682
|
+
const lastActivityRef = useRef(Date.now());
|
|
1683
|
+
const leaveRoomRef = useRef(null);
|
|
1684
|
+
const [, setClockTick] = useState(0);
|
|
1685
|
+
const clearTimer = useCallback(() => {
|
|
1686
|
+
if (timerRef.current !== null) {
|
|
1687
|
+
window.clearTimeout(timerRef.current);
|
|
1688
|
+
timerRef.current = null;
|
|
1689
|
+
}
|
|
1690
|
+
}, []);
|
|
1691
|
+
const reset = useCallback(() => {
|
|
1692
|
+
clearTimer();
|
|
1693
|
+
connectedRef.current = false;
|
|
1694
|
+
manualReconnectPendingRef.current = false;
|
|
1695
|
+
refreshBaselineCapacityRef.current = null;
|
|
1696
|
+
attemptRef.current = 0;
|
|
1697
|
+
setAttempt(0);
|
|
1698
|
+
setRecovery({ kind: "connected" });
|
|
1699
|
+
setConnected(false);
|
|
1700
|
+
setAgentPresentState(false);
|
|
1701
|
+
lastActivityRef.current = Date.now();
|
|
1702
|
+
}, [clearTimer]);
|
|
1703
|
+
useEffect(() => {
|
|
1704
|
+
if (!active) reset();
|
|
1705
|
+
}, [active, reset]);
|
|
1706
|
+
const markActivity = useCallback(() => {
|
|
1707
|
+
lastActivityRef.current = Date.now();
|
|
1708
|
+
setClockTick((value) => value + 1);
|
|
1709
|
+
}, []);
|
|
1710
|
+
const stayConnected = useCallback(() => {
|
|
1711
|
+
if (!connectedRef.current) return;
|
|
1712
|
+
lastActivityRef.current = Date.now();
|
|
1713
|
+
setClockTick((value) => value + 1);
|
|
1714
|
+
}, []);
|
|
1715
|
+
const setAgentPresent = useCallback((present) => {
|
|
1716
|
+
setAgentPresentState((prev) => {
|
|
1717
|
+
if (present && !prev) lastActivityRef.current = Date.now();
|
|
1718
|
+
return present;
|
|
1719
|
+
});
|
|
1720
|
+
}, []);
|
|
1721
|
+
const registerLeaveRoom = useCallback((leave) => {
|
|
1722
|
+
leaveRoomRef.current = leave;
|
|
1723
|
+
}, []);
|
|
1724
|
+
const endIdle = useCallback(() => {
|
|
1725
|
+
if (!connectedRef.current) return;
|
|
1726
|
+
clearTimer();
|
|
1727
|
+
connectedRef.current = false;
|
|
1728
|
+
setConnected(false);
|
|
1729
|
+
setAgentPresentState(false);
|
|
1730
|
+
releaseRef.current("idle_timeout");
|
|
1731
|
+
setRecovery({ kind: "ended", reason: "idle" });
|
|
1732
|
+
leaveRoomRef.current?.();
|
|
1733
|
+
}, [clearTimer]);
|
|
1734
|
+
const onConnected = useCallback(() => {
|
|
1735
|
+
clearTimer();
|
|
1736
|
+
connectedRef.current = true;
|
|
1737
|
+
manualReconnectPendingRef.current = false;
|
|
1738
|
+
refreshBaselineCapacityRef.current = null;
|
|
1739
|
+
setConnected(true);
|
|
1740
|
+
attemptRef.current = 0;
|
|
1741
|
+
setAttempt(0);
|
|
1742
|
+
setRecovery({ kind: "connected" });
|
|
1743
|
+
lastActivityRef.current = Date.now();
|
|
1744
|
+
}, [clearTimer]);
|
|
1745
|
+
const onDisconnected = useCallback(
|
|
1746
|
+
(reason) => {
|
|
1747
|
+
if (!active) return;
|
|
1748
|
+
const action = disconnectAction(reason, connectedRef.current);
|
|
1749
|
+
if (action === "noop") return;
|
|
1750
|
+
if (action === "reset") {
|
|
1751
|
+
if (recoveryRef.current.kind === "refreshing") return;
|
|
1752
|
+
reset();
|
|
1753
|
+
return;
|
|
1754
|
+
}
|
|
1755
|
+
connectedRef.current = false;
|
|
1756
|
+
setConnected(false);
|
|
1757
|
+
setAgentPresentState(false);
|
|
1758
|
+
if (action === "end") {
|
|
1759
|
+
manualReconnectPendingRef.current = false;
|
|
1760
|
+
refreshBaselineCapacityRef.current = null;
|
|
1761
|
+
releaseRef.current("disconnected");
|
|
1762
|
+
setRecovery({ kind: "ended" });
|
|
1763
|
+
return;
|
|
1764
|
+
}
|
|
1765
|
+
refreshBaselineCapacityRef.current = null;
|
|
1766
|
+
setRecovery({ kind: "reconnecting", attempt: attemptRef.current });
|
|
1767
|
+
},
|
|
1768
|
+
[active, reset]
|
|
1769
|
+
);
|
|
1770
|
+
const onConnectionError = useCallback(
|
|
1771
|
+
(error) => {
|
|
1772
|
+
if (!active || !isRecoverableConnectionError(error)) return;
|
|
1773
|
+
connectedRef.current = false;
|
|
1774
|
+
setConnected(false);
|
|
1775
|
+
setAgentPresentState(false);
|
|
1776
|
+
setRecovery((prev) => {
|
|
1777
|
+
if (prev.kind === "in-place-reconnecting") return prev;
|
|
1778
|
+
if (prev.kind === "reconnecting" || prev.kind === "failed" || prev.kind === "ended") return prev;
|
|
1779
|
+
refreshBaselineCapacityRef.current = null;
|
|
1780
|
+
return { kind: "reconnecting", attempt: attemptRef.current };
|
|
1781
|
+
});
|
|
1782
|
+
},
|
|
1783
|
+
[active]
|
|
1784
|
+
);
|
|
1785
|
+
useEffect(() => {
|
|
1786
|
+
if (recovery.kind !== "refreshing" || capacity.kind !== "error") return;
|
|
1787
|
+
const baseline = refreshBaselineCapacityRef.current;
|
|
1788
|
+
if (baseline?.kind === "error" && baseline.error === capacity.error) return;
|
|
1789
|
+
refreshBaselineCapacityRef.current = null;
|
|
1790
|
+
setRecovery({ kind: "reconnecting", attempt: recovery.attempt });
|
|
1791
|
+
}, [capacity, recovery]);
|
|
1792
|
+
const onConnectionStateChange = useCallback(
|
|
1793
|
+
(state) => {
|
|
1794
|
+
if (!active) return;
|
|
1795
|
+
if (state === "connected") {
|
|
1796
|
+
onConnected();
|
|
1797
|
+
return;
|
|
1798
|
+
}
|
|
1799
|
+
if (state === "reconnecting" || state === "signalReconnecting") {
|
|
1800
|
+
if (connectedRef.current) {
|
|
1801
|
+
setConnected(false);
|
|
1802
|
+
setRecovery(
|
|
1803
|
+
(prev) => prev.kind === "connected" ? { kind: "in-place-reconnecting" } : prev
|
|
1804
|
+
);
|
|
1805
|
+
}
|
|
1806
|
+
}
|
|
1807
|
+
},
|
|
1808
|
+
[active, onConnected]
|
|
1809
|
+
);
|
|
1810
|
+
useEffect(() => {
|
|
1811
|
+
if (!active || !needsFreshGrant(recovery)) return;
|
|
1812
|
+
const step2 = retryStep(attemptRef.current, reconnectPolicyRef.current);
|
|
1813
|
+
if (step2.kind === "give-up") {
|
|
1814
|
+
manualReconnectPendingRef.current = false;
|
|
1815
|
+
refreshBaselineCapacityRef.current = null;
|
|
1816
|
+
releaseRef.current("disconnected");
|
|
1817
|
+
setRecovery({ kind: "failed" });
|
|
1818
|
+
return;
|
|
1819
|
+
}
|
|
1820
|
+
timerRef.current = window.setTimeout(() => {
|
|
1821
|
+
timerRef.current = null;
|
|
1822
|
+
attemptRef.current = step2.attempt;
|
|
1823
|
+
setAttempt(step2.attempt);
|
|
1824
|
+
refreshBaselineCapacityRef.current = capacityRef.current;
|
|
1825
|
+
setRecovery({ kind: "refreshing", attempt: step2.attempt });
|
|
1826
|
+
refreshRef.current();
|
|
1827
|
+
}, step2.delayMs);
|
|
1828
|
+
return clearTimer;
|
|
1829
|
+
}, [recovery.kind, active, attempt, clearTimer]);
|
|
1830
|
+
const reconnect = useCallback(() => {
|
|
1831
|
+
if (!active || manualReconnectPendingRef.current || recovery.kind === "in-place-reconnecting") return;
|
|
1832
|
+
manualReconnectPendingRef.current = true;
|
|
1833
|
+
clearTimer();
|
|
1834
|
+
attemptRef.current = 0;
|
|
1835
|
+
setAttempt(0);
|
|
1836
|
+
refreshBaselineCapacityRef.current = capacityRef.current;
|
|
1837
|
+
setRecovery({ kind: "refreshing", attempt: 0 });
|
|
1838
|
+
refreshRef.current();
|
|
1839
|
+
}, [active, clearTimer, recovery.kind]);
|
|
1840
|
+
const endIdleRef = useRef(endIdle);
|
|
1841
|
+
endIdleRef.current = endIdle;
|
|
1842
|
+
useEffect(() => {
|
|
1843
|
+
if (!connected || !agentPresent || idleTimeoutMs <= 0 || recovery.kind !== "connected") return;
|
|
1844
|
+
const timer = window.setInterval(() => {
|
|
1845
|
+
if (idleExpired({ msSinceActivity: Date.now() - lastActivityRef.current, idleTimeoutMs })) {
|
|
1846
|
+
endIdleRef.current();
|
|
1847
|
+
return;
|
|
1848
|
+
}
|
|
1849
|
+
setClockTick((value) => value + 1);
|
|
1850
|
+
}, 1e3);
|
|
1851
|
+
return () => window.clearInterval(timer);
|
|
1852
|
+
}, [connected, agentPresent, idleTimeoutMs, recovery.kind]);
|
|
1853
|
+
const timeToDisconnectMs = connected && agentPresent && idleTimeoutMs > 0 && recovery.kind === "connected" ? Math.max(0, idleTimeoutMs - (Date.now() - lastActivityRef.current)) : null;
|
|
1854
|
+
const idle = useMemo(() => {
|
|
1855
|
+
const base = idlePhaseFor({
|
|
1856
|
+
connected,
|
|
1857
|
+
agentPresent,
|
|
1858
|
+
msSinceActivity: Date.now() - lastActivityRef.current,
|
|
1859
|
+
idleTimeoutMs,
|
|
1860
|
+
warnAtMs
|
|
1861
|
+
});
|
|
1862
|
+
if (base.kind === "idle-warning") {
|
|
1863
|
+
return { ...base, deadlineAt: lastActivityRef.current + idleTimeoutMs };
|
|
1864
|
+
}
|
|
1865
|
+
return base;
|
|
1866
|
+
}, [connected, agentPresent, idleTimeoutMs, warnAtMs, timeToDisconnectMs]);
|
|
1867
|
+
const phase = useMemo(
|
|
1868
|
+
() => lifecyclePhaseFrom({ capacity, recovery, idle, connected, agentPresent }),
|
|
1869
|
+
[capacity, recovery, idle, connected, agentPresent]
|
|
1870
|
+
);
|
|
1871
|
+
const onBehaviorChangeRef = useRef(onBehaviorChange);
|
|
1872
|
+
onBehaviorChangeRef.current = onBehaviorChange;
|
|
1873
|
+
const lastBehaviorRef = useRef(null);
|
|
1874
|
+
const behaviorEnabled = Boolean(onBehaviorChange);
|
|
1875
|
+
const onLifecycleData = useMemo(() => {
|
|
1876
|
+
if (!behaviorEnabled) return void 0;
|
|
1877
|
+
return (frame) => {
|
|
1878
|
+
const parsed = lifecycleServerFrameSchema.safeParse(frame);
|
|
1879
|
+
if (!parsed.success || parsed.data.kind !== "behavior_state") return;
|
|
1880
|
+
const next = nextBehaviorSnapshot(lastBehaviorRef.current, parsed.data);
|
|
1881
|
+
if (!next) return;
|
|
1882
|
+
lastBehaviorRef.current = next;
|
|
1883
|
+
onBehaviorChangeRef.current?.(next);
|
|
1884
|
+
};
|
|
1885
|
+
}, [behaviorEnabled]);
|
|
1886
|
+
return useMemo(
|
|
1887
|
+
() => ({
|
|
1888
|
+
phase,
|
|
1889
|
+
grant: grantState.grant,
|
|
1890
|
+
capacity,
|
|
1891
|
+
attempt,
|
|
1892
|
+
timeToDisconnectMs,
|
|
1893
|
+
stayConnected,
|
|
1894
|
+
markActivity,
|
|
1895
|
+
reconnect,
|
|
1896
|
+
onConnected,
|
|
1897
|
+
onDisconnected,
|
|
1898
|
+
onConnectionError,
|
|
1899
|
+
onConnectionStateChange,
|
|
1900
|
+
setAgentPresent,
|
|
1901
|
+
registerLeaveRoom,
|
|
1902
|
+
reset,
|
|
1903
|
+
onLifecycleData
|
|
1904
|
+
}),
|
|
1905
|
+
[
|
|
1906
|
+
phase,
|
|
1907
|
+
grantState.grant,
|
|
1908
|
+
capacity,
|
|
1909
|
+
attempt,
|
|
1910
|
+
timeToDisconnectMs,
|
|
1911
|
+
stayConnected,
|
|
1912
|
+
markActivity,
|
|
1913
|
+
reconnect,
|
|
1914
|
+
onConnected,
|
|
1915
|
+
onDisconnected,
|
|
1916
|
+
onConnectionError,
|
|
1917
|
+
onConnectionStateChange,
|
|
1918
|
+
setAgentPresent,
|
|
1919
|
+
registerLeaveRoom,
|
|
1920
|
+
reset,
|
|
1921
|
+
onLifecycleData
|
|
1922
|
+
]
|
|
1923
|
+
);
|
|
1924
|
+
}
|
|
1925
|
+
function SessionLifecycleRoomBridge({ lifecycle }) {
|
|
1926
|
+
const {
|
|
1927
|
+
onConnectionStateChange,
|
|
1928
|
+
setAgentPresent,
|
|
1929
|
+
registerLeaveRoom,
|
|
1930
|
+
markActivity,
|
|
1931
|
+
onLifecycleData,
|
|
1932
|
+
registerDataPublisher,
|
|
1933
|
+
registerTurnSender,
|
|
1934
|
+
setTurnState,
|
|
1935
|
+
setMedia
|
|
1936
|
+
} = lifecycle;
|
|
1937
|
+
const connectionState = useConnectionState();
|
|
1938
|
+
const assistant = useVoiceAssistant();
|
|
1939
|
+
const transcriptions = useTranscriptions();
|
|
1940
|
+
const room = useRoomContext();
|
|
1941
|
+
const { send } = useChat();
|
|
1942
|
+
const agentPresent = Boolean(assistant.agent);
|
|
1943
|
+
const assistantState = assistant.state;
|
|
1944
|
+
const videoLive = Boolean(assistant.videoTrack);
|
|
1945
|
+
const audioLive = Boolean(assistant.audioTrack);
|
|
1946
|
+
const transcriptionCount = transcriptions.length;
|
|
1947
|
+
const prevTranscriptionCountRef = useRef(0);
|
|
1948
|
+
useEffect(() => {
|
|
1949
|
+
onConnectionStateChange(connectionState);
|
|
1950
|
+
}, [connectionState, onConnectionStateChange]);
|
|
1951
|
+
useEffect(() => {
|
|
1952
|
+
setAgentPresent(agentPresent);
|
|
1953
|
+
}, [agentPresent, setAgentPresent]);
|
|
1954
|
+
useEffect(() => {
|
|
1955
|
+
const active = isCallActivity({
|
|
1956
|
+
assistantState,
|
|
1957
|
+
transcriptionCount,
|
|
1958
|
+
prevTranscriptionCount: prevTranscriptionCountRef.current
|
|
1959
|
+
});
|
|
1960
|
+
prevTranscriptionCountRef.current = transcriptionCount;
|
|
1961
|
+
if (active) markActivity();
|
|
1962
|
+
}, [assistantState, transcriptionCount, markActivity]);
|
|
1963
|
+
useEffect(() => {
|
|
1964
|
+
const leave = () => {
|
|
1965
|
+
void Promise.resolve(room.disconnect()).catch(() => void 0);
|
|
1966
|
+
};
|
|
1967
|
+
registerLeaveRoom(leave);
|
|
1968
|
+
return () => registerLeaveRoom(null);
|
|
1969
|
+
}, [room, registerLeaveRoom]);
|
|
1970
|
+
useEffect(() => {
|
|
1971
|
+
if (!onLifecycleData) return;
|
|
1972
|
+
const decoder = new TextDecoder();
|
|
1973
|
+
const handler = (payload, _participant, _kind, topic) => {
|
|
1974
|
+
if (topic !== RTA_LIFECYCLE_TOPIC) return;
|
|
1975
|
+
try {
|
|
1976
|
+
onLifecycleData(JSON.parse(decoder.decode(payload)));
|
|
1977
|
+
} catch {
|
|
1978
|
+
}
|
|
1979
|
+
};
|
|
1980
|
+
room.on(RoomEvent.DataReceived, handler);
|
|
1981
|
+
return () => {
|
|
1982
|
+
room.off(RoomEvent.DataReceived, handler);
|
|
1983
|
+
};
|
|
1984
|
+
}, [room, onLifecycleData]);
|
|
1985
|
+
useEffect(() => {
|
|
1986
|
+
if (!registerDataPublisher) return;
|
|
1987
|
+
const encoder = new TextEncoder();
|
|
1988
|
+
const publish = (frame) => {
|
|
1989
|
+
try {
|
|
1990
|
+
void room.localParticipant?.publishData(encoder.encode(JSON.stringify(frame)), { reliable: true, topic: RTA_LIFECYCLE_TOPIC }).catch(() => void 0);
|
|
1991
|
+
} catch {
|
|
1992
|
+
}
|
|
1993
|
+
};
|
|
1994
|
+
registerDataPublisher(publish);
|
|
1995
|
+
return () => registerDataPublisher(null);
|
|
1996
|
+
}, [room, registerDataPublisher]);
|
|
1997
|
+
useEffect(() => {
|
|
1998
|
+
if (!registerTurnSender) return;
|
|
1999
|
+
const sender = (text, opts) => Promise.resolve(send(text, opts)).then(() => void 0);
|
|
2000
|
+
registerTurnSender(sender);
|
|
2001
|
+
return () => registerTurnSender(null);
|
|
2002
|
+
}, [send, registerTurnSender]);
|
|
2003
|
+
useEffect(() => {
|
|
2004
|
+
setTurnState?.(assistantState);
|
|
2005
|
+
}, [assistantState, setTurnState]);
|
|
2006
|
+
useEffect(() => {
|
|
2007
|
+
setMedia?.({ video: videoLive ? "live" : "connecting", audio: audioLive ? "flowing" : "silent" });
|
|
2008
|
+
}, [videoLive, audioLive, setMedia]);
|
|
2009
|
+
return null;
|
|
2010
|
+
}
|
|
2011
|
+
|
|
2012
|
+
// ../client/src/react/grace-window.ts
|
|
2013
|
+
function mapTurnState(assistantState) {
|
|
2014
|
+
switch (assistantState) {
|
|
2015
|
+
case "listening":
|
|
2016
|
+
case "thinking":
|
|
2017
|
+
case "speaking":
|
|
2018
|
+
return assistantState;
|
|
2019
|
+
default:
|
|
2020
|
+
return "quiet";
|
|
2021
|
+
}
|
|
2022
|
+
}
|
|
2023
|
+
function endsAtFrom(args) {
|
|
2024
|
+
const { serverStartedAtUnixMs, maxSessionSeconds } = args;
|
|
2025
|
+
if (serverStartedAtUnixMs == null || !maxSessionSeconds) return null;
|
|
2026
|
+
return serverStartedAtUnixMs + maxSessionSeconds * 1e3;
|
|
2027
|
+
}
|
|
2028
|
+
function sessionRemainingMsFrom(args) {
|
|
2029
|
+
const endsAt = endsAtFrom(args);
|
|
2030
|
+
if (endsAt == null) return null;
|
|
2031
|
+
return Math.max(0, endsAt - args.nowUnixMs);
|
|
2032
|
+
}
|
|
2033
|
+
function approachingEndFrom(args) {
|
|
2034
|
+
const { sessionRemainingMs, approachingEndLeadMs, alreadyFired } = args;
|
|
2035
|
+
if (alreadyFired || sessionRemainingMs == null) return null;
|
|
2036
|
+
if (sessionRemainingMs <= approachingEndLeadMs && sessionRemainingMs > 0) {
|
|
2037
|
+
return { secondsLeft: Math.ceil(sessionRemainingMs / 1e3) };
|
|
2038
|
+
}
|
|
2039
|
+
return null;
|
|
2040
|
+
}
|
|
2041
|
+
function creditsLowFrom(args) {
|
|
2042
|
+
const { creditRemainingMs, creditsLowLeadMs, alreadyFired } = args;
|
|
2043
|
+
if (alreadyFired || creditRemainingMs == null) return null;
|
|
2044
|
+
if (creditRemainingMs <= creditsLowLeadMs && creditRemainingMs > 0) {
|
|
2045
|
+
return { secondsLeft: Math.ceil(creditRemainingMs / 1e3) };
|
|
2046
|
+
}
|
|
2047
|
+
return null;
|
|
2048
|
+
}
|
|
2049
|
+
function nextGraceWindow(args) {
|
|
2050
|
+
const { prev, sessionRemainingMs, graceWindowLeadMs, endsAt, workerEnding, nowMs } = args;
|
|
2051
|
+
switch (prev.kind) {
|
|
2052
|
+
case "spent":
|
|
2053
|
+
return prev;
|
|
2054
|
+
// terminal
|
|
2055
|
+
case "closed": {
|
|
2056
|
+
const clockCrossed = sessionRemainingMs != null && sessionRemainingMs <= graceWindowLeadMs && sessionRemainingMs > 0;
|
|
2057
|
+
if ((workerEnding || clockCrossed) && endsAt != null) {
|
|
2058
|
+
return { kind: "open", reason: "session_cap", deadlineAt: endsAt, msLeft: Math.max(0, endsAt - nowMs) };
|
|
2059
|
+
}
|
|
2060
|
+
return prev;
|
|
2061
|
+
}
|
|
2062
|
+
case "open":
|
|
2063
|
+
if (nowMs >= prev.deadlineAt) return { kind: "spent", delivered: false };
|
|
2064
|
+
return { ...prev, msLeft: Math.max(0, prev.deadlineAt - nowMs) };
|
|
2065
|
+
case "delivering":
|
|
2066
|
+
if (nowMs >= prev.deadlineAt) return { kind: "spent", delivered: false };
|
|
2067
|
+
return prev;
|
|
2068
|
+
}
|
|
2069
|
+
}
|
|
2070
|
+
function resolveEndReason(workerLabel, innerReason) {
|
|
2071
|
+
if (workerLabel) return workerLabel;
|
|
2072
|
+
switch (innerReason) {
|
|
2073
|
+
case "idle":
|
|
2074
|
+
return "idle";
|
|
2075
|
+
case "error":
|
|
2076
|
+
return "failed";
|
|
2077
|
+
default:
|
|
2078
|
+
return "disconnected";
|
|
2079
|
+
}
|
|
2080
|
+
}
|
|
2081
|
+
|
|
2082
|
+
// ../client/src/react/use-realtime-session.ts
|
|
2083
|
+
var DEFAULT_APPROACHING_END_LEAD_SECONDS = 45;
|
|
2084
|
+
var DEFAULT_GRACE_WINDOW_LEAD_SECONDS = 12;
|
|
2085
|
+
var DEFAULT_GRACE_CEILING_SECONDS = 10;
|
|
2086
|
+
var DEFAULT_CREDITS_LOW_LEAD_SECONDS = 300;
|
|
2087
|
+
function positive(value, fallbackSeconds) {
|
|
2088
|
+
return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : fallbackSeconds;
|
|
2089
|
+
}
|
|
2090
|
+
function useRealtimeSession(input) {
|
|
2091
|
+
const {
|
|
2092
|
+
session,
|
|
2093
|
+
maxSessionSeconds,
|
|
2094
|
+
approachingEndLeadSeconds,
|
|
2095
|
+
graceWindowLeadSeconds,
|
|
2096
|
+
graceCeilingSeconds,
|
|
2097
|
+
creditsLowLeadSeconds,
|
|
2098
|
+
turnTimeoutSeconds,
|
|
2099
|
+
creditRemainingMs = null,
|
|
2100
|
+
autoStayConnectedDuringGrace = true,
|
|
2101
|
+
onApproachingEnd,
|
|
2102
|
+
onGraceWindowOpen,
|
|
2103
|
+
onGraceWindowClosed,
|
|
2104
|
+
onIdleWarning,
|
|
2105
|
+
onCreditsLow,
|
|
2106
|
+
onTurnTimeout,
|
|
2107
|
+
onReconnecting,
|
|
2108
|
+
onReconnected,
|
|
2109
|
+
onEnded,
|
|
2110
|
+
onBehaviorChange,
|
|
2111
|
+
onClipResult,
|
|
2112
|
+
// everything else (client/active/idleSeconds/idleWarnLeadSeconds/reconnect/requestOptions…)
|
|
2113
|
+
// flows straight into the inner SSOT hook.
|
|
2114
|
+
...inner
|
|
2115
|
+
} = input;
|
|
2116
|
+
const effectiveSession = useMemo(
|
|
2117
|
+
() => session && typeof maxSessionSeconds === "number" ? { ...session, maxSessionSeconds } : session,
|
|
2118
|
+
[session, maxSessionSeconds]
|
|
2119
|
+
);
|
|
2120
|
+
const lifecycle = useSessionLifecycle({ ...inner, session: effectiveSession });
|
|
2121
|
+
const [serverClock, setServerClock] = useState(null);
|
|
2122
|
+
const [workerEnding, setWorkerEnding] = useState(false);
|
|
2123
|
+
const [graceWindow, setGraceWindow] = useState({ kind: "closed" });
|
|
2124
|
+
const [turn, setTurn] = useState("quiet");
|
|
2125
|
+
const [media, setMediaState] = useState({ video: "connecting", audio: "silent" });
|
|
2126
|
+
const [behavior, setBehavior] = useState(null);
|
|
2127
|
+
const lastBehaviorRef = useRef(null);
|
|
2128
|
+
const lastLabeledEndReasonRef = useRef(null);
|
|
2129
|
+
const pendingClipsRef = useRef(/* @__PURE__ */ new Map());
|
|
2130
|
+
const cbRef = useRef({ onApproachingEnd, onGraceWindowOpen, onGraceWindowClosed, onIdleWarning, onCreditsLow, onTurnTimeout, onReconnecting, onReconnected, onEnded, onBehaviorChange, onClipResult });
|
|
2131
|
+
cbRef.current = { onApproachingEnd, onGraceWindowOpen, onGraceWindowClosed, onIdleWarning, onCreditsLow, onTurnTimeout, onReconnecting, onReconnected, onEnded, onBehaviorChange, onClipResult };
|
|
2132
|
+
const approachingEndLeadMs = positive(approachingEndLeadSeconds, DEFAULT_APPROACHING_END_LEAD_SECONDS) * 1e3;
|
|
2133
|
+
const graceWindowLeadMs = positive(graceWindowLeadSeconds, DEFAULT_GRACE_WINDOW_LEAD_SECONDS) * 1e3;
|
|
2134
|
+
const graceCeilingMs = positive(graceCeilingSeconds, DEFAULT_GRACE_CEILING_SECONDS) * 1e3;
|
|
2135
|
+
const creditsLowLeadMs = positive(creditsLowLeadSeconds, DEFAULT_CREDITS_LOW_LEAD_SECONDS) * 1e3;
|
|
2136
|
+
const turnTimeoutMs = positive(turnTimeoutSeconds, DEFAULT_TURN_TIMEOUT_SECONDS) * 1e3;
|
|
2137
|
+
const approachingFiredRef = useRef(false);
|
|
2138
|
+
const creditsLowFiredRef = useRef(false);
|
|
2139
|
+
const idleWarnFiredRef = useRef(false);
|
|
2140
|
+
const endedFiredRef = useRef(false);
|
|
2141
|
+
const prevPhaseKindRef = useRef("idle");
|
|
2142
|
+
const graceWindowRef = useRef(graceWindow);
|
|
2143
|
+
graceWindowRef.current = graceWindow;
|
|
2144
|
+
const turnSenderRef = useRef(null);
|
|
2145
|
+
const dataPublisherRef = useRef(null);
|
|
2146
|
+
const lastTurnRef = useRef(null);
|
|
2147
|
+
const [clockTick, setClockTick] = useState(0);
|
|
2148
|
+
const endsAt = endsAtFrom({
|
|
2149
|
+
serverStartedAtUnixMs: serverClock?.startedAtUnixMs ?? null,
|
|
2150
|
+
maxSessionSeconds: serverClock?.maxSessionSeconds ?? null
|
|
2151
|
+
});
|
|
2152
|
+
const sessionRemainingMs = sessionRemainingMsFrom({
|
|
2153
|
+
serverStartedAtUnixMs: serverClock?.startedAtUnixMs ?? null,
|
|
2154
|
+
maxSessionSeconds: serverClock?.maxSessionSeconds ?? null,
|
|
2155
|
+
nowUnixMs: Date.now()
|
|
2156
|
+
});
|
|
2157
|
+
const clocks = {
|
|
2158
|
+
sessionRemainingMs,
|
|
2159
|
+
idleRemainingMs: lifecycle.timeToDisconnectMs,
|
|
2160
|
+
creditRemainingMs
|
|
2161
|
+
};
|
|
2162
|
+
const phaseKind = lifecycle.phase.kind;
|
|
2163
|
+
const stayConnected = lifecycle.stayConnected;
|
|
2164
|
+
const newTurnId = useCallback(() => {
|
|
2165
|
+
try {
|
|
2166
|
+
if (typeof crypto !== "undefined" && typeof crypto.randomUUID === "function") return crypto.randomUUID();
|
|
2167
|
+
} catch {
|
|
2168
|
+
}
|
|
2169
|
+
return `t-${Date.now()}-${clockTick}`;
|
|
2170
|
+
}, [clockTick]);
|
|
2171
|
+
const onLifecycleData = useCallback((frame) => {
|
|
2172
|
+
const parsed = lifecycleServerFrameSchema.safeParse(frame);
|
|
2173
|
+
if (!parsed.success) return;
|
|
2174
|
+
const f = parsed.data;
|
|
2175
|
+
switch (f.kind) {
|
|
2176
|
+
case "session_clock":
|
|
2177
|
+
setServerClock({
|
|
2178
|
+
startedAtUnixMs: f.started_at_unix_ms,
|
|
2179
|
+
maxSessionSeconds: f.max_session_seconds,
|
|
2180
|
+
idleTimeoutSeconds: f.idle_timeout_seconds
|
|
2181
|
+
});
|
|
2182
|
+
break;
|
|
2183
|
+
case "ending":
|
|
2184
|
+
setWorkerEnding(true);
|
|
2185
|
+
break;
|
|
2186
|
+
case "closing_turn_done":
|
|
2187
|
+
setGraceWindow((prev) => {
|
|
2188
|
+
if (prev.kind === "delivering" && prev.turnId === f.turn_id) {
|
|
2189
|
+
cbRef.current.onGraceWindowClosed?.({ reason: "session_cap", delivered: true });
|
|
2190
|
+
return { kind: "spent", delivered: true };
|
|
2191
|
+
}
|
|
2192
|
+
return prev;
|
|
2193
|
+
});
|
|
2194
|
+
break;
|
|
2195
|
+
case "ended":
|
|
2196
|
+
lastLabeledEndReasonRef.current = f.reason;
|
|
2197
|
+
break;
|
|
2198
|
+
case "behavior_state": {
|
|
2199
|
+
const next = nextBehaviorSnapshot(lastBehaviorRef.current, f);
|
|
2200
|
+
if (next) {
|
|
2201
|
+
lastBehaviorRef.current = next;
|
|
2202
|
+
setBehavior(next);
|
|
2203
|
+
cbRef.current.onBehaviorChange?.(next);
|
|
2204
|
+
}
|
|
2205
|
+
break;
|
|
2206
|
+
}
|
|
2207
|
+
case "clip_ack": {
|
|
2208
|
+
const result = { requestId: f.request_id, accepted: f.accepted, reason: f.reason };
|
|
2209
|
+
const pending = pendingClipsRef.current.get(f.request_id);
|
|
2210
|
+
if (pending) {
|
|
2211
|
+
pendingClipsRef.current.delete(f.request_id);
|
|
2212
|
+
clearTimeout(pending.timer);
|
|
2213
|
+
pending.resolve(result);
|
|
2214
|
+
}
|
|
2215
|
+
cbRef.current.onClipResult?.(result);
|
|
2216
|
+
break;
|
|
2217
|
+
}
|
|
2218
|
+
}
|
|
2219
|
+
}, []);
|
|
2220
|
+
const registerDataPublisher = useCallback((publish) => {
|
|
2221
|
+
dataPublisherRef.current = publish;
|
|
2222
|
+
}, []);
|
|
2223
|
+
const registerTurnSender = useCallback((send) => {
|
|
2224
|
+
turnSenderRef.current = send;
|
|
2225
|
+
}, []);
|
|
2226
|
+
const setTurnState = useCallback((state) => {
|
|
2227
|
+
setTurn(mapTurnState(state));
|
|
2228
|
+
}, []);
|
|
2229
|
+
const setMedia = useCallback((next) => {
|
|
2230
|
+
setMediaState((prev) => prev.video === next.video && prev.audio === next.audio ? prev : next);
|
|
2231
|
+
}, []);
|
|
2232
|
+
useEffect(() => {
|
|
2233
|
+
if (phaseKind === "idle" || phaseKind === "ended") return;
|
|
2234
|
+
const timer = window.setInterval(() => {
|
|
2235
|
+
const now = Date.now();
|
|
2236
|
+
const remaining = sessionRemainingMsFrom({
|
|
2237
|
+
serverStartedAtUnixMs: serverClock?.startedAtUnixMs ?? null,
|
|
2238
|
+
maxSessionSeconds: serverClock?.maxSessionSeconds ?? null,
|
|
2239
|
+
nowUnixMs: now
|
|
2240
|
+
});
|
|
2241
|
+
const ends = endsAtFrom({
|
|
2242
|
+
serverStartedAtUnixMs: serverClock?.startedAtUnixMs ?? null,
|
|
2243
|
+
maxSessionSeconds: serverClock?.maxSessionSeconds ?? null
|
|
2244
|
+
});
|
|
2245
|
+
const approaching = approachingEndFrom({ sessionRemainingMs: remaining, approachingEndLeadMs, alreadyFired: approachingFiredRef.current });
|
|
2246
|
+
if (approaching) {
|
|
2247
|
+
approachingFiredRef.current = true;
|
|
2248
|
+
cbRef.current.onApproachingEnd?.({ secondsLeft: approaching.secondsLeft, reason: "session_cap", threshold: approachingEndLeadMs / 1e3 });
|
|
2249
|
+
}
|
|
2250
|
+
const low = creditsLowFrom({ creditRemainingMs, creditsLowLeadMs, alreadyFired: creditsLowFiredRef.current });
|
|
2251
|
+
if (low) {
|
|
2252
|
+
creditsLowFiredRef.current = true;
|
|
2253
|
+
cbRef.current.onCreditsLow?.({ secondsLeft: low.secondsLeft });
|
|
2254
|
+
}
|
|
2255
|
+
setGraceWindow((prev) => {
|
|
2256
|
+
const next = nextGraceWindow({ prev, sessionRemainingMs: remaining, graceWindowLeadMs, endsAt: ends, workerEnding, nowMs: now });
|
|
2257
|
+
if (prev.kind !== "open" && next.kind === "open") {
|
|
2258
|
+
cbRef.current.onGraceWindowOpen?.({ reason: next.reason, deadlineAt: next.deadlineAt, msLeft: next.msLeft });
|
|
2259
|
+
}
|
|
2260
|
+
if (prev.kind !== "spent" && next.kind === "spent" && !next.delivered) {
|
|
2261
|
+
cbRef.current.onGraceWindowClosed?.({ reason: "session_cap", delivered: false });
|
|
2262
|
+
}
|
|
2263
|
+
return next;
|
|
2264
|
+
});
|
|
2265
|
+
if (autoStayConnectedDuringGrace) {
|
|
2266
|
+
const gw = graceWindowRef.current.kind;
|
|
2267
|
+
if (gw === "open" || gw === "delivering") stayConnected();
|
|
2268
|
+
}
|
|
2269
|
+
const turnInfo = lastTurnRef.current;
|
|
2270
|
+
if (turnInfo && now - turnInfo.sentAt >= turnTimeoutMs) {
|
|
2271
|
+
lastTurnRef.current = null;
|
|
2272
|
+
cbRef.current.onTurnTimeout?.({ turnId: turnInfo.id });
|
|
2273
|
+
}
|
|
2274
|
+
setClockTick((value) => value + 1);
|
|
2275
|
+
}, 1e3);
|
|
2276
|
+
return () => window.clearInterval(timer);
|
|
2277
|
+
}, [phaseKind, serverClock, workerEnding, approachingEndLeadMs, graceWindowLeadMs, creditRemainingMs, creditsLowLeadMs, turnTimeoutMs, autoStayConnectedDuringGrace, stayConnected]);
|
|
2278
|
+
useEffect(() => {
|
|
2279
|
+
if (lifecycle.phase.kind === "idle-warning") {
|
|
2280
|
+
if (!idleWarnFiredRef.current) {
|
|
2281
|
+
idleWarnFiredRef.current = true;
|
|
2282
|
+
cbRef.current.onIdleWarning?.({ secondsLeft: lifecycle.phase.secondsRemaining });
|
|
2283
|
+
}
|
|
2284
|
+
} else {
|
|
2285
|
+
idleWarnFiredRef.current = false;
|
|
2286
|
+
}
|
|
2287
|
+
}, [lifecycle.phase]);
|
|
2288
|
+
useEffect(() => {
|
|
2289
|
+
const prev = prevPhaseKindRef.current;
|
|
2290
|
+
if (phaseKind === "reconnectable" && prev !== "reconnectable") {
|
|
2291
|
+
const attempt = lifecycle.phase.kind === "reconnectable" ? lifecycle.phase.attempt : lifecycle.attempt;
|
|
2292
|
+
cbRef.current.onReconnecting?.({ attempt });
|
|
2293
|
+
} else if (prev === "reconnectable" && (phaseKind === "live" || phaseKind === "connecting")) {
|
|
2294
|
+
cbRef.current.onReconnected?.();
|
|
2295
|
+
}
|
|
2296
|
+
prevPhaseKindRef.current = phaseKind;
|
|
2297
|
+
}, [phaseKind, lifecycle.phase, lifecycle.attempt]);
|
|
2298
|
+
useEffect(() => {
|
|
2299
|
+
if (phaseKind === "ended") {
|
|
2300
|
+
if (!endedFiredRef.current) {
|
|
2301
|
+
endedFiredRef.current = true;
|
|
2302
|
+
const inner2 = lifecycle.phase.kind === "ended" ? lifecycle.phase.reason : void 0;
|
|
2303
|
+
cbRef.current.onEnded?.({ reason: resolveEndReason(lastLabeledEndReasonRef.current, inner2) });
|
|
2304
|
+
}
|
|
2305
|
+
} else {
|
|
2306
|
+
endedFiredRef.current = false;
|
|
2307
|
+
}
|
|
2308
|
+
}, [phaseKind, lifecycle.phase]);
|
|
2309
|
+
const sendClosingTurn = useCallback((text, opts) => {
|
|
2310
|
+
const trimmed = text.trim();
|
|
2311
|
+
const gw = graceWindowRef.current;
|
|
2312
|
+
if (gw.kind === "spent") return { ok: false, reason: "already_spent" };
|
|
2313
|
+
if (gw.kind !== "open") return { ok: false, reason: "window_closed" };
|
|
2314
|
+
const sender = turnSenderRef.current;
|
|
2315
|
+
if (!sender || !trimmed) return { ok: false, reason: "not_connected" };
|
|
2316
|
+
const turnId = newTurnId();
|
|
2317
|
+
const attributes = { [RTA_CLOSING_TURN_ATTR]: "1", [RTA_TURN_ID_ATTR]: turnId };
|
|
2318
|
+
if (opts?.instructions) attributes[RTA_TURN_INSTRUCTIONS_ATTR] = opts.instructions;
|
|
2319
|
+
void sender(trimmed, { attributes }).catch(() => void 0);
|
|
2320
|
+
const deadlineAt = (endsAt ?? Date.now()) + graceCeilingMs;
|
|
2321
|
+
setGraceWindow({ kind: "delivering", turnId, deadlineAt });
|
|
2322
|
+
return { ok: true, turnId };
|
|
2323
|
+
}, [endsAt, graceCeilingMs, newTurnId]);
|
|
2324
|
+
const requestGracefulClose = useCallback(() => {
|
|
2325
|
+
dataPublisherRef.current?.({ kind: "request_graceful_close" });
|
|
2326
|
+
}, []);
|
|
2327
|
+
const extend = useCallback((req) => {
|
|
2328
|
+
const publish = dataPublisherRef.current;
|
|
2329
|
+
if (!publish || !(req.addSeconds > 0)) return { ok: false };
|
|
2330
|
+
publish({ kind: "extend", add_seconds: Math.floor(req.addSeconds), ...req.proof ? { proof: req.proof } : {} });
|
|
2331
|
+
approachingFiredRef.current = false;
|
|
2332
|
+
setWorkerEnding(false);
|
|
2333
|
+
setGraceWindow({ kind: "closed" });
|
|
2334
|
+
return { ok: true };
|
|
2335
|
+
}, []);
|
|
2336
|
+
const sendTurn = useCallback(async (text, opts) => {
|
|
2337
|
+
const trimmed = text.trim();
|
|
2338
|
+
const sender = turnSenderRef.current;
|
|
2339
|
+
if (!sender || !trimmed) return;
|
|
2340
|
+
const turnId = newTurnId();
|
|
2341
|
+
lastTurnRef.current = { text: trimmed, opts, id: turnId, sentAt: Date.now() };
|
|
2342
|
+
lifecycle.markActivity();
|
|
2343
|
+
const attributes = opts?.instructions ? { [RTA_TURN_INSTRUCTIONS_ATTR]: opts.instructions } : void 0;
|
|
2344
|
+
await sender(trimmed, attributes ? { attributes } : void 0);
|
|
2345
|
+
}, [lifecycle, newTurnId]);
|
|
2346
|
+
const retryTurn = useCallback(() => {
|
|
2347
|
+
const last = lastTurnRef.current;
|
|
2348
|
+
if (!last) return;
|
|
2349
|
+
void sendTurn(last.text, last.opts);
|
|
2350
|
+
}, [sendTurn]);
|
|
2351
|
+
const end = useCallback((reason) => {
|
|
2352
|
+
if (reason) lastLabeledEndReasonRef.current = reason;
|
|
2353
|
+
requestGracefulClose();
|
|
2354
|
+
lifecycle.reset();
|
|
2355
|
+
}, [lifecycle, requestGracefulClose]);
|
|
2356
|
+
const performClip = useCallback(
|
|
2357
|
+
(clipId, opts) => {
|
|
2358
|
+
const requestId = newTurnId();
|
|
2359
|
+
const publish = dataPublisherRef.current;
|
|
2360
|
+
const trimmed = clipId.trim();
|
|
2361
|
+
if (!publish || !trimmed) {
|
|
2362
|
+
const result = { requestId, accepted: false, reason: "not_connected" };
|
|
2363
|
+
cbRef.current.onClipResult?.(result);
|
|
2364
|
+
return Promise.resolve(result);
|
|
2365
|
+
}
|
|
2366
|
+
return new Promise((resolve) => {
|
|
2367
|
+
const timer = setTimeout(() => {
|
|
2368
|
+
pendingClipsRef.current.delete(requestId);
|
|
2369
|
+
const result = { requestId, accepted: false, reason: "timeout" };
|
|
2370
|
+
cbRef.current.onClipResult?.(result);
|
|
2371
|
+
resolve(result);
|
|
2372
|
+
}, opts?.timeoutMs ?? 5e3);
|
|
2373
|
+
pendingClipsRef.current.set(requestId, { resolve, timer });
|
|
2374
|
+
publish({
|
|
2375
|
+
kind: "clip_request",
|
|
2376
|
+
request_id: requestId,
|
|
2377
|
+
clip_id: trimmed,
|
|
2378
|
+
...typeof opts?.holdSeconds === "number" ? { hold_seconds: opts.holdSeconds } : {}
|
|
2379
|
+
});
|
|
2380
|
+
});
|
|
2381
|
+
},
|
|
2382
|
+
[newTurnId]
|
|
2383
|
+
);
|
|
2384
|
+
useEffect(
|
|
2385
|
+
() => () => {
|
|
2386
|
+
for (const [requestId, pending] of pendingClipsRef.current) {
|
|
2387
|
+
clearTimeout(pending.timer);
|
|
2388
|
+
pending.resolve({ requestId, accepted: false, reason: "unmounted" });
|
|
2389
|
+
}
|
|
2390
|
+
pendingClipsRef.current.clear();
|
|
2391
|
+
},
|
|
2392
|
+
[]
|
|
2393
|
+
);
|
|
2394
|
+
const reset = useCallback(() => {
|
|
2395
|
+
setServerClock(null);
|
|
2396
|
+
setWorkerEnding(false);
|
|
2397
|
+
setGraceWindow({ kind: "closed" });
|
|
2398
|
+
lastLabeledEndReasonRef.current = null;
|
|
2399
|
+
approachingFiredRef.current = false;
|
|
2400
|
+
creditsLowFiredRef.current = false;
|
|
2401
|
+
lastTurnRef.current = null;
|
|
2402
|
+
lifecycle.reset();
|
|
2403
|
+
}, [lifecycle]);
|
|
2404
|
+
return useMemo(
|
|
2405
|
+
() => ({
|
|
2406
|
+
// Spread the inner SSOT verbatim (phase/grant/capacity/attempt/timeToDisconnectMs/
|
|
2407
|
+
// stayConnected/reconnect/markActivity/the sinks) → a true drop-in superset, then
|
|
2408
|
+
// add the realtime-session surface. `reset` + `lifecycle` come AFTER so ours win.
|
|
2409
|
+
...lifecycle,
|
|
2410
|
+
turn,
|
|
2411
|
+
clocks,
|
|
2412
|
+
endsAt,
|
|
2413
|
+
graceWindow,
|
|
2414
|
+
media,
|
|
2415
|
+
lifecycle,
|
|
2416
|
+
sendClosingTurn,
|
|
2417
|
+
requestGracefulClose,
|
|
2418
|
+
extend,
|
|
2419
|
+
sendTurn,
|
|
2420
|
+
retryTurn,
|
|
2421
|
+
end,
|
|
2422
|
+
behavior,
|
|
2423
|
+
performClip,
|
|
2424
|
+
onLifecycleData,
|
|
2425
|
+
registerDataPublisher,
|
|
2426
|
+
registerTurnSender,
|
|
2427
|
+
setTurnState,
|
|
2428
|
+
setMedia,
|
|
2429
|
+
reset
|
|
2430
|
+
}),
|
|
2431
|
+
[
|
|
2432
|
+
lifecycle,
|
|
2433
|
+
turn,
|
|
2434
|
+
clocks,
|
|
2435
|
+
endsAt,
|
|
2436
|
+
graceWindow,
|
|
2437
|
+
media,
|
|
2438
|
+
sendClosingTurn,
|
|
2439
|
+
requestGracefulClose,
|
|
2440
|
+
extend,
|
|
2441
|
+
sendTurn,
|
|
2442
|
+
retryTurn,
|
|
2443
|
+
end,
|
|
2444
|
+
behavior,
|
|
2445
|
+
performClip,
|
|
2446
|
+
onLifecycleData,
|
|
2447
|
+
registerDataPublisher,
|
|
2448
|
+
registerTurnSender,
|
|
2449
|
+
setTurnState,
|
|
2450
|
+
setMedia,
|
|
2451
|
+
reset
|
|
2452
|
+
]
|
|
2453
|
+
);
|
|
2454
|
+
}
|
|
2455
|
+
|
|
2456
|
+
// ../client/src/react/avatar-call.ts
|
|
2457
|
+
function statusFor(session) {
|
|
2458
|
+
switch (session.phase.kind) {
|
|
2459
|
+
case "queued":
|
|
2460
|
+
return "waiting";
|
|
2461
|
+
case "live":
|
|
2462
|
+
case "idle-warning":
|
|
2463
|
+
return "live";
|
|
2464
|
+
case "reconnectable":
|
|
2465
|
+
return "recovering";
|
|
2466
|
+
case "ended":
|
|
2467
|
+
return "ended";
|
|
2468
|
+
default:
|
|
2469
|
+
return "connecting";
|
|
2470
|
+
}
|
|
2471
|
+
}
|
|
2472
|
+
function handleFor(session) {
|
|
2473
|
+
const remainingMs = session.clocks.sessionRemainingMs;
|
|
2474
|
+
return {
|
|
2475
|
+
status: statusFor(session),
|
|
2476
|
+
queuePosition: session.phase.kind === "queued" ? session.phase.busy.queue_position ?? null : null,
|
|
2477
|
+
secondsRemaining: remainingMs === null ? null : Math.max(0, Math.round(remainingMs / 1e3)),
|
|
2478
|
+
say: (text, options) => session.sendTurn(text, options?.steer ? { instructions: options.steer } : void 0),
|
|
2479
|
+
sayAndEnd: (text) => {
|
|
2480
|
+
session.sendClosingTurn(text);
|
|
2481
|
+
},
|
|
2482
|
+
keepAlive: session.stayConnected,
|
|
2483
|
+
end: () => session.end("user_ended")
|
|
2484
|
+
};
|
|
2485
|
+
}
|
|
2486
|
+
function useAvatarCall(props) {
|
|
2487
|
+
const sessionRef = useRef(null);
|
|
2488
|
+
const session = useRealtimeSession({
|
|
2489
|
+
client: props.client,
|
|
2490
|
+
session: { avatarId: props.avatarId, mode: props.mode ?? "avatar", sttMode: props.listen === false ? "off" : "server" },
|
|
2491
|
+
creditRemainingMs: props.balanceMs,
|
|
2492
|
+
onApproachingEnd: props.onEnding ? ({ secondsLeft }) => {
|
|
2493
|
+
const live = sessionRef.current;
|
|
2494
|
+
if (live) props.onEnding?.({ secondsLeft, call: handleFor(live) });
|
|
2495
|
+
} : void 0,
|
|
2496
|
+
onIdleWarning: props.onQuiet ? ({ secondsLeft }) => props.onQuiet?.({ secondsLeft }) : void 0,
|
|
2497
|
+
onCreditsLow: props.onLowBalance ? ({ secondsLeft }) => props.onLowBalance?.({ secondsLeft }) : void 0,
|
|
2498
|
+
onEnded: props.onEnded ? ({ reason }) => props.onEnded?.({ reason }) : void 0
|
|
2499
|
+
});
|
|
2500
|
+
sessionRef.current = session;
|
|
2501
|
+
const call = handleFor(session);
|
|
2502
|
+
const lastStatusRef = useRef(null);
|
|
2503
|
+
const onStatusChange = props.onStatusChange;
|
|
2504
|
+
useEffect(() => {
|
|
2505
|
+
if (lastStatusRef.current === call.status) return;
|
|
2506
|
+
lastStatusRef.current = call.status;
|
|
2507
|
+
onStatusChange?.(call.status);
|
|
2508
|
+
}, [call.status, onStatusChange]);
|
|
2509
|
+
const view = createElement(
|
|
2510
|
+
RealtimeAvatarLiveKitRoom,
|
|
2511
|
+
{
|
|
2512
|
+
grant: session.grant,
|
|
2513
|
+
audio: props.listen !== false,
|
|
2514
|
+
onConnected: session.onConnected,
|
|
2515
|
+
onDisconnected: session.onDisconnected,
|
|
2516
|
+
onError: session.onConnectionError
|
|
2517
|
+
},
|
|
2518
|
+
createElement(SessionLifecycleRoomBridge, { key: "bridge", lifecycle: session }),
|
|
2519
|
+
createElement(
|
|
2520
|
+
AvatarVideoSurface,
|
|
2521
|
+
{
|
|
2522
|
+
key: "surface",
|
|
2523
|
+
idleVideoUrl: props.idleVideoUrl ?? null,
|
|
2524
|
+
poster: props.poster ?? null,
|
|
2525
|
+
fit: props.fit ?? "cover",
|
|
2526
|
+
className: props.className
|
|
2527
|
+
},
|
|
2528
|
+
props.children ? props.children(call) : null
|
|
2529
|
+
)
|
|
2530
|
+
);
|
|
2531
|
+
return { call, view };
|
|
2532
|
+
}
|
|
2533
|
+
function AvatarCall(props) {
|
|
2534
|
+
const { call, view } = useAvatarCall(props);
|
|
2535
|
+
return view;
|
|
2536
|
+
}
|
|
2537
|
+
|
|
2538
|
+
// ../client/src/proxy-client.ts
|
|
2539
|
+
var DEFAULT_TIMEOUT_MS = 6e4;
|
|
2540
|
+
var normalize = (url) => url.replace(/\/+$/, "");
|
|
2541
|
+
function createProxyClient(options) {
|
|
2542
|
+
const base = normalize(options.proxyUrl);
|
|
2543
|
+
const doFetch = options.fetch ?? globalThis.fetch?.bind(globalThis);
|
|
2544
|
+
const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
|
2545
|
+
const deadline = (caller) => {
|
|
2546
|
+
if (!timeoutMs) return caller;
|
|
2547
|
+
const timer = AbortSignal.timeout(timeoutMs);
|
|
2548
|
+
return caller ? AbortSignal.any([caller, timer]) : timer;
|
|
2549
|
+
};
|
|
2550
|
+
const post = async (path, body, request) => {
|
|
2551
|
+
if (!doFetch) throw new Error("realtime-avatar: no fetch available \u2014 pass one via `fetch`.");
|
|
2552
|
+
return doFetch(`${base}${path}`, {
|
|
2553
|
+
method: "POST",
|
|
2554
|
+
headers: { "content-type": "application/json", ...request?.headers ?? {} },
|
|
2555
|
+
body: JSON.stringify(body),
|
|
2556
|
+
signal: deadline(request?.signal)
|
|
2557
|
+
});
|
|
2558
|
+
};
|
|
2559
|
+
const beacon = (path, body) => {
|
|
2560
|
+
const send = globalThis.navigator?.sendBeacon?.bind(globalThis.navigator);
|
|
2561
|
+
if (!send) return false;
|
|
2562
|
+
return send(`${base}${path}`, new Blob([JSON.stringify(body)], { type: "application/json" }));
|
|
2563
|
+
};
|
|
2564
|
+
return {
|
|
2565
|
+
async createLiveKitSessionOrBusy(input, requestOptions) {
|
|
2566
|
+
const response = await post(
|
|
2567
|
+
"/connect",
|
|
2568
|
+
{ avatarId: input.avatarId, mode: input.mode },
|
|
2569
|
+
requestOptions
|
|
2570
|
+
);
|
|
2571
|
+
if (response.status === 429) {
|
|
2572
|
+
const busy = await response.json().catch(() => ({}));
|
|
2573
|
+
return { status: "busy", busy };
|
|
2574
|
+
}
|
|
2575
|
+
if (!response.ok) {
|
|
2576
|
+
const detail = await response.text().catch(() => "");
|
|
2577
|
+
throw new Error(`realtime-avatar: proxy refused the call (${response.status}) ${detail}`.trim());
|
|
2578
|
+
}
|
|
2579
|
+
return { status: "ready", grant: await response.json() };
|
|
2580
|
+
},
|
|
2581
|
+
async releaseLiveKitSession(sessionId, reason, requestOptions) {
|
|
2582
|
+
if (!sessionId) return false;
|
|
2583
|
+
try {
|
|
2584
|
+
const response = await post("/end", { session_id: sessionId, reason }, requestOptions);
|
|
2585
|
+
return response.ok;
|
|
2586
|
+
} catch {
|
|
2587
|
+
return false;
|
|
2588
|
+
}
|
|
2589
|
+
},
|
|
2590
|
+
releaseLiveKitSessionBeacon(sessionId, reason) {
|
|
2591
|
+
if (!sessionId) return false;
|
|
2592
|
+
return beacon("/end", { session_id: sessionId, reason: reason ?? "page_hide" });
|
|
2593
|
+
},
|
|
2594
|
+
async releaseLiveKitQueueTicket(queueTicketId, reason, requestOptions) {
|
|
2595
|
+
if (!queueTicketId) return false;
|
|
2596
|
+
try {
|
|
2597
|
+
const response = await post("/end", { queue_ticket_id: queueTicketId, reason }, requestOptions);
|
|
2598
|
+
return response.ok;
|
|
2599
|
+
} catch {
|
|
2600
|
+
return false;
|
|
2601
|
+
}
|
|
2602
|
+
},
|
|
2603
|
+
releaseLiveKitQueueTicketBeacon(queueTicketId, reason) {
|
|
2604
|
+
if (!queueTicketId) return false;
|
|
2605
|
+
return beacon("/end", { queue_ticket_id: queueTicketId, reason: reason ?? "page_hide" });
|
|
2606
|
+
}
|
|
2607
|
+
};
|
|
2608
|
+
}
|
|
2609
|
+
|
|
2610
|
+
export { AvatarCall, AvatarVideoSurface, DEFAULT_APPROACHING_END_LEAD_SECONDS, DEFAULT_AVATAR_PLAYOUT_DELAY_SECONDS, DEFAULT_CREDITS_LOW_LEAD_SECONDS, DEFAULT_GOVERNOR_CONFIG, DEFAULT_GRACE_CEILING_SECONDS, DEFAULT_GRACE_WINDOW_LEAD_SECONDS, DEFAULT_IDLE_SECONDS, DEFAULT_IDLE_WARN_LEAD_SECONDS, DEFAULT_TURN_TIMEOUT_SECONDS, MAX_RECONNECT_ATTEMPTS, MAX_SESSION_INSTRUCTIONS_CHARS, MIC_LEASE_ENDED_TIMEOUT_MS, RECONNECT_BACKOFF_MS, RealtimeAvatarCapacityError, RealtimeAvatarLiveKitRoom, SessionLifecycleRoomBridge, capacityErrorFromBusy, capacityStateFromGrant, createProxyClient, knownBehaviorStates, mapTurnState, sessionBehaviorSchema, sessionClipSchema, shouldReplayPendingTurn, splitCallTranscript, useAvatarCall, useAvatarPlayoutDelay, useAvatarQualityGovernor, useCallTranscript, useLiveKitAvatarGrant, useMicLease, useRealtimeSession, useReleaseMicLeaseOnTrackEnded, useSessionLifecycle };
|