realtime-avatar 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,2305 @@
1
+ import { AudioSession, LiveKitRoom, VideoTrack } from '@livekit/react-native';
2
+ export { AudioSession, VideoTrack, registerGlobals } from '@livekit/react-native';
3
+ import { useEffect, createElement, useRef, useState, useCallback, useMemo } from 'react';
4
+ import { useTranscriptions, useLocalParticipant, useMaybeRoomContext, useVoiceAssistant, useConnectionState, isTrackReference, useRoomContext, useChat } from '@livekit/components-react';
5
+ export { useChat, useConnectionState, useLocalParticipant, useRoomContext, useTrackToggle, useTranscriptions, useVoiceAssistant } from '@livekit/components-react';
6
+ import { ConnectionQuality, RoomEvent, TrackEvent, ConnectionState, Track, VideoQuality, ConnectionError, ConnectionErrorReason, DisconnectReason, Room } from 'livekit-client';
7
+ export { DisconnectReason, Room, RoomEvent, Track } from 'livekit-client';
8
+ import { Platform, StyleSheet, Animated, Image, View, Text } from 'react-native';
9
+ import { z } from 'zod';
10
+
11
+ // ../client/src/react-native/index.ts
12
+ function useRealtimeAvatarAudioSession(enabled) {
13
+ useEffect(() => {
14
+ if (!enabled) return;
15
+ void AudioSession.startAudioSession();
16
+ return () => {
17
+ void AudioSession.stopAudioSession();
18
+ };
19
+ }, [enabled]);
20
+ }
21
+ function RealtimeAvatarLiveKitRoom(props) {
22
+ const {
23
+ grant,
24
+ connect = true,
25
+ audio,
26
+ video = false,
27
+ options,
28
+ manageAudioSession = true,
29
+ children,
30
+ ...roomProps
31
+ } = props;
32
+ const shouldConnect = Boolean(grant && connect);
33
+ useRealtimeAvatarAudioSession(manageAudioSession && shouldConnect);
34
+ const roomOptions = { adaptiveStream: false, dynacast: true, ...options };
35
+ return createElement(
36
+ LiveKitRoom,
37
+ {
38
+ ...roomProps,
39
+ serverUrl: grant?.livekit_url,
40
+ token: grant?.participant_token,
41
+ connect: shouldConnect,
42
+ audio: audio ?? grant?.stt_mode === "server",
43
+ video,
44
+ options: roomOptions
45
+ },
46
+ children
47
+ );
48
+ }
49
+ var holder = null;
50
+ var queue = [];
51
+ var pendingEnded = null;
52
+ function acquireMicLease(token) {
53
+ if (holder === null || holder === token) {
54
+ holder = token;
55
+ if (pendingEnded === token) pendingEnded = null;
56
+ return Promise.resolve();
57
+ }
58
+ const existing = queue.find((w) => w.token === token);
59
+ if (existing) {
60
+ return new Promise((resolve) => {
61
+ const prior = existing.resolve;
62
+ existing.resolve = () => {
63
+ prior();
64
+ resolve();
65
+ };
66
+ });
67
+ }
68
+ return new Promise((resolve) => {
69
+ queue.push({ token, resolve });
70
+ });
71
+ }
72
+ function releaseMicLease(token) {
73
+ if (holder !== token) {
74
+ const i = queue.findIndex((w) => w.token === token);
75
+ if (i !== -1) queue.splice(i, 1);
76
+ return;
77
+ }
78
+ const next = queue.shift();
79
+ if (next) {
80
+ holder = next.token;
81
+ next.resolve();
82
+ } else {
83
+ holder = null;
84
+ }
85
+ }
86
+ function releaseMicLeaseWhenEnded(token, timeoutMs, setTimer) {
87
+ if (holder !== token) {
88
+ releaseMicLease(token);
89
+ return () => {
90
+ };
91
+ }
92
+ pendingEnded = token;
93
+ const cancelTimer = setTimer(() => {
94
+ if (pendingEnded === token) finishPendingRelease(token);
95
+ }, timeoutMs);
96
+ return cancelTimer;
97
+ }
98
+ function confirmMicTrackEnded(token) {
99
+ if (pendingEnded === token) finishPendingRelease(token);
100
+ }
101
+ function finishPendingRelease(token) {
102
+ pendingEnded = null;
103
+ releaseMicLease(token);
104
+ }
105
+ var MIC_LEASE_ENDED_TIMEOUT_MS = 1500;
106
+ function useMicLease(want) {
107
+ const tokenRef = useRef(null);
108
+ if (tokenRef.current === null) tokenRef.current = /* @__PURE__ */ Symbol("mic-lease");
109
+ const token = tokenRef.current;
110
+ const [held, setHeld] = useState(false);
111
+ useEffect(() => {
112
+ if (!want) {
113
+ releaseMicLeaseWhenEnded(token, MIC_LEASE_ENDED_TIMEOUT_MS, timeoutSetter);
114
+ setHeld(false);
115
+ return;
116
+ }
117
+ let alive = true;
118
+ void acquireMicLease(token).then(() => {
119
+ if (alive) setHeld(true);
120
+ });
121
+ return () => {
122
+ alive = false;
123
+ releaseMicLeaseWhenEnded(token, MIC_LEASE_ENDED_TIMEOUT_MS, timeoutSetter);
124
+ };
125
+ }, [want, token]);
126
+ return { held, token };
127
+ }
128
+ function timeoutSetter(fn, ms) {
129
+ const id = setTimeout(fn, ms);
130
+ return () => clearTimeout(id);
131
+ }
132
+
133
+ // ../client/src/errors.ts
134
+ var RealtimeAvatarCapacityError = class extends Error {
135
+ constructor(message, busy) {
136
+ super(message);
137
+ this.busy = busy;
138
+ this.name = "RealtimeAvatarCapacityError";
139
+ }
140
+ busy;
141
+ get queueSize() {
142
+ return this.busy.queue_size;
143
+ }
144
+ get queuePosition() {
145
+ return this.busy.queue_position;
146
+ }
147
+ get queueTicketId() {
148
+ return this.busy.queue_ticket_id;
149
+ }
150
+ get recommendedRetryMs() {
151
+ return this.busy.recommended_retry_ms;
152
+ }
153
+ };
154
+ function splitCallTranscript(all, localIdentity) {
155
+ if (!localIdentity) return { user: [], agent: all };
156
+ const user = [];
157
+ const agent = [];
158
+ for (const segment of all) {
159
+ if (segment.participantInfo.identity === localIdentity) user.push(segment);
160
+ else agent.push(segment);
161
+ }
162
+ return { user, agent };
163
+ }
164
+ function useCallTranscript() {
165
+ const all = useTranscriptions();
166
+ const { localParticipant } = useLocalParticipant();
167
+ return splitCallTranscript(all, localParticipant?.identity);
168
+ }
169
+ function useReleaseMicLeaseOnTrackEnded(token) {
170
+ const { localParticipant } = useLocalParticipant();
171
+ useEffect(() => {
172
+ if (!localParticipant) return;
173
+ const cleanups = [];
174
+ const watch = (raw) => {
175
+ if (!raw) return;
176
+ if (raw.readyState === "ended") {
177
+ confirmMicTrackEnded(token);
178
+ return;
179
+ }
180
+ const onEnded = () => confirmMicTrackEnded(token);
181
+ raw.addEventListener("ended", onEnded);
182
+ cleanups.push(() => raw.removeEventListener("ended", onEnded));
183
+ };
184
+ for (const pub of localParticipant.audioTrackPublications.values()) {
185
+ watch(pub.track?.mediaStreamTrack);
186
+ }
187
+ return () => {
188
+ for (const c of cleanups) c();
189
+ };
190
+ }, [localParticipant, token, localParticipant?.audioTrackPublications.size]);
191
+ }
192
+ var DEFAULT_AVATAR_PLAYOUT_DELAY_SECONDS = 0.5;
193
+ function useAvatarPlayoutDelay(videoTrack, audioTrack, delaySeconds = DEFAULT_AVATAR_PLAYOUT_DELAY_SECONDS) {
194
+ const videoMediaTrack = videoTrack?.publication?.track;
195
+ const audioMediaTrack = audioTrack?.publication?.track;
196
+ useEffect(() => {
197
+ applyAvatarPlayoutDelay(videoMediaTrack, audioMediaTrack, delaySeconds);
198
+ }, [videoMediaTrack, audioMediaTrack, delaySeconds]);
199
+ }
200
+ function applyAvatarPlayoutDelay(videoTrack, audioTrack, delaySeconds = DEFAULT_AVATAR_PLAYOUT_DELAY_SECONDS) {
201
+ const delay = Math.max(0, delaySeconds);
202
+ playoutDelayTarget(videoTrack)?.setPlayoutDelay(delay);
203
+ playoutDelayTarget(audioTrack)?.setPlayoutDelay(delay);
204
+ }
205
+ function playoutDelayTarget(track) {
206
+ const candidate = track;
207
+ return typeof candidate?.setPlayoutDelay === "function" ? candidate : void 0;
208
+ }
209
+ function resolveReleaseTarget(heldSessionId, queueTicketId) {
210
+ if (heldSessionId) return { kind: "session", sessionId: heldSessionId };
211
+ if (queueTicketId) return { kind: "ticket", queueTicketId };
212
+ return { kind: "none" };
213
+ }
214
+ function capacityStateFromGrant(state) {
215
+ switch (state.status) {
216
+ case "busy":
217
+ if (state.busy) return { kind: "queued", busy: state.busy };
218
+ return { kind: "connecting" };
219
+ case "failed":
220
+ return { kind: "error", error: state.error ?? new Error("Realtime session request failed") };
221
+ case "ready":
222
+ return state.grant ? { kind: "active", grant: state.grant } : { kind: "connecting" };
223
+ case "requesting":
224
+ return { kind: "connecting" };
225
+ case "idle":
226
+ default:
227
+ return { kind: "idle" };
228
+ }
229
+ }
230
+ var cachedClientId = null;
231
+ function randomClientId() {
232
+ return globalThis.crypto?.randomUUID?.() ?? `c_${Date.now().toString(36)}_${Math.random().toString(36).slice(2)}`;
233
+ }
234
+ function stableClientId() {
235
+ if (cachedClientId) return cachedClientId;
236
+ try {
237
+ const key = "rta.client_id";
238
+ const existing = globalThis.localStorage?.getItem(key) ?? null;
239
+ if (existing) return cachedClientId = existing;
240
+ const fresh = randomClientId();
241
+ globalThis.localStorage?.setItem(key, fresh);
242
+ return cachedClientId = fresh;
243
+ } catch {
244
+ return cachedClientId = randomClientId();
245
+ }
246
+ }
247
+ function createConnectionWarmer(prepare) {
248
+ const warmed = /* @__PURE__ */ new Set();
249
+ return (url) => {
250
+ const valid = validWarmUrl(url);
251
+ if (!valid || warmed.has(valid)) return;
252
+ warmed.add(valid);
253
+ prepare(valid);
254
+ };
255
+ }
256
+ function validWarmUrl(value) {
257
+ if (!value) return null;
258
+ try {
259
+ const { protocol } = new URL(value);
260
+ return protocol === "wss:" || protocol === "ws:" || protocol === "https:" || protocol === "http:" ? value : null;
261
+ } catch {
262
+ return null;
263
+ }
264
+ }
265
+ var warmRoom = null;
266
+ var warmLiveKitHost = createConnectionWarmer((url) => {
267
+ try {
268
+ warmRoom ??= new Room();
269
+ void warmRoom.prepareConnection(url);
270
+ } catch {
271
+ }
272
+ });
273
+ var LIVEKIT_URL_HINT_KEY = "rta.livekit_url_hint";
274
+ var cachedUrlHint = null;
275
+ function readLiveKitUrlHint() {
276
+ try {
277
+ return validWarmUrl(globalThis.localStorage?.getItem(LIVEKIT_URL_HINT_KEY)) ?? cachedUrlHint;
278
+ } catch {
279
+ return cachedUrlHint;
280
+ }
281
+ }
282
+ function writeLiveKitUrlHint(url) {
283
+ if (!validWarmUrl(url)) return;
284
+ cachedUrlHint = url;
285
+ try {
286
+ globalThis.localStorage?.setItem(LIVEKIT_URL_HINT_KEY, url);
287
+ } catch {
288
+ }
289
+ }
290
+ function useLiveKitAvatarGrant(input) {
291
+ const { client, session, active = true, autoRetryBusy = false, requestOptions, serverUrlHint } = input;
292
+ const [version, setVersion] = useState(0);
293
+ const queueTicketRef = useRef(null);
294
+ const clientIdRef = useRef(stableClientId());
295
+ const heldSessionRef = useRef(null);
296
+ const releaseHeld = useCallback(
297
+ (reason, viaBeacon) => {
298
+ const target = resolveReleaseTarget(heldSessionRef.current, queueTicketRef.current);
299
+ if (target.kind === "none") return;
300
+ if (target.kind === "session") {
301
+ heldSessionRef.current = null;
302
+ if (viaBeacon && client.releaseLiveKitSessionBeacon(target.sessionId, reason)) return;
303
+ void client.releaseLiveKitSession(target.sessionId, reason);
304
+ return;
305
+ }
306
+ queueTicketRef.current = null;
307
+ if (viaBeacon && client.releaseLiveKitQueueTicketBeacon(target.queueTicketId, reason)) return;
308
+ void client.releaseLiveKitQueueTicket(target.queueTicketId, reason);
309
+ },
310
+ [client]
311
+ );
312
+ const sessionKey = useMemo(() => session ? stableStringify(session) : null, [session]);
313
+ const sessionRef = useRef(session);
314
+ sessionRef.current = session;
315
+ const serverUrlHintRef = useRef(serverUrlHint);
316
+ serverUrlHintRef.current = serverUrlHint;
317
+ const prevKeyRef = useRef(null);
318
+ const [state, setState] = useState({
319
+ status: "idle",
320
+ grant: null,
321
+ busy: null,
322
+ error: null
323
+ });
324
+ const stateRef = useRef(state);
325
+ stateRef.current = state;
326
+ const [tabVisible, setTabVisible] = useState(
327
+ () => typeof document === "undefined" || document.visibilityState === "visible"
328
+ );
329
+ useEffect(() => {
330
+ if (typeof document === "undefined") return;
331
+ const onVisibility = () => setTabVisible(document.visibilityState === "visible");
332
+ document.addEventListener("visibilitychange", onVisibility);
333
+ return () => document.removeEventListener("visibilitychange", onVisibility);
334
+ }, []);
335
+ useEffect(() => {
336
+ const current = sessionRef.current;
337
+ if (!active || !current || !sessionKey) {
338
+ releaseHeld("unmount", false);
339
+ queueTicketRef.current = null;
340
+ prevKeyRef.current = sessionKey;
341
+ setState({ status: "idle", grant: null, busy: null, error: null });
342
+ return;
343
+ }
344
+ if (prevKeyRef.current !== sessionKey) {
345
+ releaseHeld("superseded", false);
346
+ queueTicketRef.current = current.queueTicketId ?? null;
347
+ prevKeyRef.current = sessionKey;
348
+ }
349
+ const request = {
350
+ ...current,
351
+ queueTicketId: current.queueTicketId ?? queueTicketRef.current ?? clientIdRef.current
352
+ };
353
+ let cancelled = false;
354
+ const prior = stateRef.current;
355
+ const retryingQueued = prior.status === "busy" && prior.busy !== null;
356
+ if (!retryingQueued) {
357
+ warmLiveKitHost(serverUrlHintRef.current ?? readLiveKitUrlHint());
358
+ setState({ status: "requesting", grant: null, busy: null, error: null });
359
+ }
360
+ void client.createLiveKitSessionOrBusy(request, requestOptions).then((result) => {
361
+ if (cancelled) return;
362
+ if (result.status === "busy") {
363
+ queueTicketRef.current = result.busy.queue_ticket_id ?? queueTicketRef.current;
364
+ setState({ status: "busy", grant: null, busy: result.busy, error: null });
365
+ return;
366
+ }
367
+ queueTicketRef.current = null;
368
+ if (heldSessionRef.current && heldSessionRef.current !== result.grant.session_id) {
369
+ releaseHeld("superseded", false);
370
+ }
371
+ heldSessionRef.current = result.grant.session_id;
372
+ writeLiveKitUrlHint(result.grant.livekit_url);
373
+ setState({ status: "ready", grant: result.grant, busy: null, error: null });
374
+ }).catch((error) => {
375
+ if (cancelled) return;
376
+ setState({
377
+ status: "failed",
378
+ grant: null,
379
+ busy: null,
380
+ error: error instanceof Error ? error : new Error(String(error))
381
+ });
382
+ });
383
+ return () => {
384
+ cancelled = true;
385
+ };
386
+ }, [active, client, sessionKey, requestOptions, version, releaseHeld]);
387
+ useEffect(() => {
388
+ const canListen = typeof window !== "undefined" && typeof window.addEventListener === "function";
389
+ const onPageHide = () => releaseHeld("page_hide", true);
390
+ if (canListen) window.addEventListener("pagehide", onPageHide);
391
+ return () => {
392
+ if (canListen) window.removeEventListener("pagehide", onPageHide);
393
+ releaseHeld("unmount", false);
394
+ };
395
+ }, [releaseHeld]);
396
+ const refresh = useCallback(() => setVersion((current) => current + 1), []);
397
+ const clear = useCallback(() => {
398
+ releaseHeld("manual", false);
399
+ queueTicketRef.current = null;
400
+ setState({ status: "idle", grant: null, busy: null, error: null });
401
+ }, [releaseHeld]);
402
+ const release = useCallback(
403
+ (reason = "disconnected") => releaseHeld(reason, false),
404
+ [releaseHeld]
405
+ );
406
+ useEffect(() => {
407
+ if (!autoRetryBusy || !active || !tabVisible || state.status !== "busy" || !state.busy) return;
408
+ const retryMs = Math.max(state.busy.recommended_retry_ms, 250);
409
+ const timer = window.setTimeout(refresh, retryMs);
410
+ return () => window.clearTimeout(timer);
411
+ }, [active, autoRetryBusy, tabVisible, refresh, state.busy, state.status]);
412
+ const capacityRef = useRef(null);
413
+ const capacity = useMemo(() => {
414
+ const next = capacityStateFromGrant(state);
415
+ const prev = capacityRef.current;
416
+ if (prev && sameCapacityState(prev, next)) return prev;
417
+ capacityRef.current = next;
418
+ return next;
419
+ }, [state]);
420
+ return useMemo(
421
+ () => ({ ...state, capacity, refresh, clear, release }),
422
+ [state, capacity, refresh, clear, release]
423
+ );
424
+ }
425
+ function sameCapacityState(a, b) {
426
+ if (a.kind !== b.kind) return false;
427
+ switch (a.kind) {
428
+ case "queued": {
429
+ const next = b.busy;
430
+ return a.busy.queue_position === next.queue_position && a.busy.queue_size === next.queue_size && a.busy.recommended_retry_ms === next.recommended_retry_ms;
431
+ }
432
+ case "active":
433
+ return a.grant === b.grant;
434
+ case "error":
435
+ return a.error === b.error;
436
+ default:
437
+ return true;
438
+ }
439
+ }
440
+ var capacityErrorFromBusy = (busy) => {
441
+ return new RealtimeAvatarCapacityError(busy.message, busy);
442
+ };
443
+ function stableStringify(value) {
444
+ if (value === null || typeof value !== "object") return JSON.stringify(value) ?? "null";
445
+ if (Array.isArray(value)) return `[${value.map(stableStringify).join(",")}]`;
446
+ const entries = Object.entries(value).filter(([, entry]) => entry !== void 0).sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0);
447
+ return `{${entries.map(([key, entry]) => `${JSON.stringify(key)}:${stableStringify(entry)}`).join(",")}}`;
448
+ }
449
+
450
+ // ../client/src/react/quality-governor.ts
451
+ var DEFAULT_GOVERNOR_CONFIG = {
452
+ openingCap: "low",
453
+ downgradeFreezeMs: 150,
454
+ probationFreezeMs: 100,
455
+ openingDwellMs: 2e3,
456
+ dwellBaseMs: 8e3,
457
+ dwellMaxMs: 12e4,
458
+ cleanMs: 3e3,
459
+ probeMs: 1e4,
460
+ healthyResetMs: 12e4
461
+ };
462
+ var JITTER_BUFFER_RISE_THRESHOLD_MS = 25;
463
+ function stepJitterBufferTrend(previous, current, riseThresholdMs = JITTER_BUFFER_RISE_THRESHOLD_MS) {
464
+ const baseline = { ...current, intervalAverageMs: null };
465
+ if (previous === null || !Number.isFinite(current.delaySeconds) || !Number.isFinite(current.emittedCount) || current.delaySeconds < previous.delaySeconds || current.emittedCount <= previous.emittedCount) {
466
+ return { state: baseline, rising: false };
467
+ }
468
+ const intervalAverageMs = (current.delaySeconds - previous.delaySeconds) * 1e3 / (current.emittedCount - previous.emittedCount);
469
+ const threshold = Number.isFinite(riseThresholdMs) ? Math.max(0, riseThresholdMs) : 0;
470
+ const rising = previous.intervalAverageMs !== null && intervalAverageMs > previous.intervalAverageMs + threshold;
471
+ return {
472
+ state: { ...current, intervalAverageMs },
473
+ rising
474
+ };
475
+ }
476
+ var initGovernor = (nowMs, openingCap = "low") => ({
477
+ state: openingCap === "high" ? "opening_high" : "opening",
478
+ cap: openingCap,
479
+ failures: 0,
480
+ enteredAtMs: nowMs,
481
+ healthySinceMs: null
482
+ });
483
+ var isDowngrade = (s, cfg) => s.paused || s.freezeMsInWindow >= cfg.downgradeFreezeMs || s.jitterRising && s.freezeMsInWindow > 0;
484
+ var isProbationFail = (s, cfg) => s.paused || s.freezeMsInWindow >= cfg.probationFreezeMs;
485
+ var isHealthy = (s) => !s.paused && s.freezeMsInWindow === 0 && !s.jitterRising && s.connectionQuality !== "poor" && s.connectionQuality !== "lost";
486
+ var dwellMs = (failures, cfg) => Math.min(cfg.dwellBaseMs * 2 ** failures, cfg.dwellMaxMs);
487
+ var enter = (g, state, cap, nowMs) => ({
488
+ ...g,
489
+ state,
490
+ cap,
491
+ enteredAtMs: nowMs,
492
+ healthySinceMs: null
493
+ });
494
+ var step = (g, s, nowMs, cfg = DEFAULT_GOVERNOR_CONFIG) => {
495
+ if (s.inhibited) {
496
+ return { governor: g };
497
+ }
498
+ const onProbation = g.state === "probing_up" || g.state === "opening_high";
499
+ const downgradeNow = onProbation ? isProbationFail(s, cfg) : isDowngrade(s, cfg);
500
+ if (g.cap === "high" && downgradeNow) {
501
+ const failed = onProbation;
502
+ return {
503
+ governor: {
504
+ ...enter(g, "cap_low_sticky", "low", nowMs),
505
+ failures: failed ? g.failures + 1 : g.failures
506
+ },
507
+ action: { setCap: "low" }
508
+ };
509
+ }
510
+ switch (g.state) {
511
+ case "opening_high": {
512
+ if (nowMs - g.enteredAtMs >= cfg.probeMs && isHealthy(s)) {
513
+ return { governor: enter(g, "cap_high_stable", "high", nowMs) };
514
+ }
515
+ return { governor: g };
516
+ }
517
+ case "opening": {
518
+ if (nowMs - g.enteredAtMs >= cfg.openingDwellMs && isHealthy(s)) {
519
+ return { governor: { ...enter(g, "cap_low_eligible", "low", nowMs), healthySinceMs: nowMs } };
520
+ }
521
+ return { governor: g };
522
+ }
523
+ case "cap_low_sticky": {
524
+ const healthy = isHealthy(s);
525
+ const healthySince = healthy ? g.healthySinceMs ?? nowMs : null;
526
+ const resetFailures = healthy && healthySince !== null && nowMs - healthySince >= cfg.healthyResetMs;
527
+ const dwellDone = nowMs - g.enteredAtMs >= dwellMs(g.failures, cfg);
528
+ if (dwellDone && healthy) {
529
+ return {
530
+ governor: {
531
+ ...enter(g, "cap_low_eligible", "low", nowMs),
532
+ failures: resetFailures ? 0 : g.failures,
533
+ healthySinceMs: nowMs
534
+ }
535
+ };
536
+ }
537
+ return {
538
+ governor: {
539
+ ...g,
540
+ healthySinceMs: healthySince,
541
+ failures: resetFailures ? 0 : g.failures
542
+ }
543
+ };
544
+ }
545
+ case "cap_low_eligible": {
546
+ if (!isHealthy(s)) {
547
+ return { governor: { ...g, healthySinceMs: null } };
548
+ }
549
+ const cleanSince = g.healthySinceMs ?? nowMs;
550
+ if (nowMs - cleanSince >= cfg.cleanMs) {
551
+ return {
552
+ governor: enter(g, "probing_up", "high", nowMs),
553
+ action: { setCap: "high" }
554
+ };
555
+ }
556
+ return { governor: { ...g, healthySinceMs: cleanSince } };
557
+ }
558
+ case "probing_up": {
559
+ if (nowMs - g.enteredAtMs >= cfg.probeMs) {
560
+ return { governor: { ...enter(g, "cap_high_stable", "high", nowMs), failures: 0 } };
561
+ }
562
+ return { governor: g };
563
+ }
564
+ case "cap_high_stable":
565
+ return { governor: g };
566
+ default:
567
+ return { governor: g };
568
+ }
569
+ };
570
+ var resolveLowCapQuality = (declaredLayerQualities) => {
571
+ const sorted = [...declaredLayerQualities].sort((a, b) => a - b);
572
+ return sorted.length >= 2 ? sorted[sorted.length - 2] : 1;
573
+ };
574
+
575
+ // ../client/src/react/use-quality-governor.ts
576
+ var qualityToSignal = (q) => {
577
+ switch (q) {
578
+ case ConnectionQuality.Excellent:
579
+ return "excellent";
580
+ case ConnectionQuality.Good:
581
+ return "good";
582
+ case ConnectionQuality.Poor:
583
+ return "poor";
584
+ case ConnectionQuality.Lost:
585
+ return "lost";
586
+ default:
587
+ return "unknown";
588
+ }
589
+ };
590
+ function useAvatarQualityGovernor(input) {
591
+ const { enabled, freezeReading, config = DEFAULT_GOVERNOR_CONFIG, tickMs = 1e3 } = input;
592
+ const room = useMaybeRoomContext();
593
+ const { videoTrack } = useVoiceAssistant();
594
+ const pausedSinceTick = useRef(false);
595
+ const connQuality = useRef("unknown");
596
+ const lastFreezeStat = useRef(null);
597
+ const jitterTrend = useRef(null);
598
+ useEffect(() => {
599
+ if (!enabled || !room) return;
600
+ const targetPublication = videoTrack?.publication;
601
+ const targetParticipant = videoTrack?.participant;
602
+ pausedSinceTick.current = false;
603
+ lastFreezeStat.current = null;
604
+ jitterTrend.current = null;
605
+ connQuality.current = qualityToSignal(
606
+ targetParticipant?.connectionQuality ?? ConnectionQuality.Unknown
607
+ );
608
+ const onStreamState = (pub, streamState) => {
609
+ if (targetPublication && pub !== targetPublication) return;
610
+ if (streamState === Track.StreamState.Paused) pausedSinceTick.current = true;
611
+ };
612
+ const onQuality = (q, participant) => {
613
+ if (targetParticipant && participant.sid !== targetParticipant.sid) return;
614
+ connQuality.current = qualityToSignal(q);
615
+ };
616
+ try {
617
+ room.on(RoomEvent.TrackStreamStateChanged, onStreamState);
618
+ room.on(RoomEvent.ConnectionQualityChanged, onQuality);
619
+ } catch {
620
+ return;
621
+ }
622
+ let gov = initGovernor(Date.now());
623
+ const readGetStatsSignals = async () => {
624
+ try {
625
+ const track = targetPublication?.track;
626
+ const stats = await track?.getRTCStatsReport?.();
627
+ if (!stats) return { freezeMs: 0, jitterRising: false };
628
+ let frozenTotalMs = 0;
629
+ let jitterDelaySeconds = 0;
630
+ let jitterEmittedCount = 0;
631
+ stats.forEach((r) => {
632
+ if (r.type === "inbound-rtp" && typeof r.totalFreezesDuration === "number") {
633
+ frozenTotalMs = r.totalFreezesDuration * 1e3;
634
+ }
635
+ if (r.type === "inbound-rtp" && typeof r.jitterBufferDelay === "number" && typeof r.jitterBufferEmittedCount === "number") {
636
+ jitterDelaySeconds += r.jitterBufferDelay;
637
+ jitterEmittedCount += r.jitterBufferEmittedCount;
638
+ }
639
+ });
640
+ const now = Date.now();
641
+ const prev = lastFreezeStat.current;
642
+ lastFreezeStat.current = { frozen: frozenTotalMs, ts: now };
643
+ const trend = stepJitterBufferTrend(jitterTrend.current, {
644
+ delaySeconds: jitterDelaySeconds,
645
+ emittedCount: jitterEmittedCount
646
+ });
647
+ jitterTrend.current = trend.state;
648
+ return {
649
+ freezeMs: prev ? Math.max(0, frozenTotalMs - prev.frozen) : 0,
650
+ jitterRising: trend.rising
651
+ };
652
+ } catch {
653
+ return { freezeMs: 0, jitterRising: false };
654
+ }
655
+ };
656
+ const applyCap = (cap) => {
657
+ try {
658
+ const lowCap = resolveLowCapQuality(
659
+ (targetPublication?.trackInfo?.layers ?? []).map((l) => l.quality)
660
+ );
661
+ targetPublication?.setVideoQuality?.(cap === "low" ? lowCap : VideoQuality.HIGH);
662
+ } catch {
663
+ }
664
+ };
665
+ let disposed = false;
666
+ let tickRunning = false;
667
+ const tick = async () => {
668
+ if (disposed || tickRunning) return;
669
+ tickRunning = true;
670
+ try {
671
+ const rvfc = freezeReading?.() ?? { freezeMsInWindow: 0, inhibited: false };
672
+ const statsSignals = await readGetStatsSignals();
673
+ if (disposed) return;
674
+ const signal = {
675
+ paused: pausedSinceTick.current,
676
+ freezeMsInWindow: Math.max(rvfc.freezeMsInWindow, statsSignals.freezeMs),
677
+ jitterRising: statsSignals.jitterRising,
678
+ connectionQuality: connQuality.current,
679
+ inhibited: rvfc.inhibited
680
+ };
681
+ pausedSinceTick.current = false;
682
+ const { governor, action } = step(gov, signal, Date.now(), config);
683
+ gov = governor;
684
+ if (action) applyCap(action.setCap);
685
+ } catch {
686
+ } finally {
687
+ tickRunning = false;
688
+ }
689
+ };
690
+ applyCap(gov.cap);
691
+ const handle = setInterval(() => void tick(), tickMs);
692
+ return () => {
693
+ disposed = true;
694
+ clearInterval(handle);
695
+ try {
696
+ room.off(RoomEvent.TrackStreamStateChanged, onStreamState);
697
+ room.off(RoomEvent.ConnectionQualityChanged, onQuality);
698
+ } catch {
699
+ }
700
+ };
701
+ }, [enabled, room, videoTrack, freezeReading, config, tickMs]);
702
+ }
703
+
704
+ // ../client/src/react/avatar-video-surface.ts
705
+ function resolveSurfaceLayers(input) {
706
+ return {
707
+ showPoster: Boolean(input.poster),
708
+ showIdleVideo: Boolean(input.idleVideoUrl)
709
+ };
710
+ }
711
+ function isLiveTrackProducing(videoTrack) {
712
+ const mst = videoTrack?.publication?.track?.mediaStreamTrack;
713
+ return mst != null && mst.readyState === "live" && mst.enabled && !mst.muted;
714
+ }
715
+ function isNativeLiveTrackSubscribed(videoTrack) {
716
+ const mst = videoTrack?.publication?.track?.mediaStreamTrack;
717
+ return mst != null && mst.readyState !== "ended";
718
+ }
719
+ function useLiveTrackProducing(videoTrack, isProducing = isLiveTrackProducing) {
720
+ const publication = videoTrack?.publication;
721
+ const [producing, setProducing] = useState(() => isProducing(videoTrack));
722
+ useEffect(() => {
723
+ if (!publication) {
724
+ setProducing(false);
725
+ return;
726
+ }
727
+ const sync = () => setProducing(isProducing(videoTrack));
728
+ sync();
729
+ publication.on(TrackEvent.Muted, sync);
730
+ publication.on(TrackEvent.Unmuted, sync);
731
+ publication.on(TrackEvent.Ended, sync);
732
+ const mst = publication.track?.mediaStreamTrack;
733
+ mst?.addEventListener("ended", sync);
734
+ mst?.addEventListener("mute", sync);
735
+ mst?.addEventListener("unmute", sync);
736
+ return () => {
737
+ publication.off(TrackEvent.Muted, sync);
738
+ publication.off(TrackEvent.Unmuted, sync);
739
+ publication.off(TrackEvent.Ended, sync);
740
+ mst?.removeEventListener("ended", sync);
741
+ mst?.removeEventListener("mute", sync);
742
+ mst?.removeEventListener("unmute", sync);
743
+ };
744
+ }, [publication, videoTrack?.publication?.track, isProducing]);
745
+ return producing;
746
+ }
747
+ function useDebouncedHide(wanted, delayMs) {
748
+ const [shown, setShown] = useState(wanted);
749
+ useEffect(() => {
750
+ if (wanted) {
751
+ setShown(true);
752
+ return;
753
+ }
754
+ if (delayMs <= 0) {
755
+ setShown(false);
756
+ return;
757
+ }
758
+ const timer = window.setTimeout(() => setShown(false), delayMs);
759
+ return () => window.clearTimeout(timer);
760
+ }, [wanted, delayMs]);
761
+ return shown;
762
+ }
763
+
764
+ // ../client/src/react-native/avatar-video-surface.ts
765
+ var IS_ANDROID = Platform.OS === "android";
766
+ var ANDROID_REMOTE_ZORDER = 0;
767
+ function AvatarVideoSurface(props) {
768
+ const {
769
+ idleVideoUrl,
770
+ renderIdleVideo,
771
+ poster = null,
772
+ live = true,
773
+ adaptiveQuality = true,
774
+ fit = "contain",
775
+ crossfadeMs = 500,
776
+ idleReturnDelayMs = 700,
777
+ style,
778
+ children,
779
+ showLiveBadge = true,
780
+ testID
781
+ } = props;
782
+ const { videoTrack, audioTrack } = useVoiceAssistant();
783
+ useAvatarPlayoutDelay(videoTrack, audioTrack);
784
+ const connectionState = useConnectionState();
785
+ const connected = connectionState === ConnectionState.Connected;
786
+ const trackProducing = useLiveTrackProducing(
787
+ videoTrack,
788
+ IS_ANDROID ? isNativeLiveTrackSubscribed : void 0
789
+ );
790
+ useAvatarQualityGovernor({ enabled: adaptiveQuality });
791
+ const liveWanted = live && connected && trackProducing;
792
+ const showLive = useDebouncedHide(liveWanted, connected ? idleReturnDelayMs : 0);
793
+ const liveOpacity = useRef(new Animated.Value(showLive ? 1 : 0)).current;
794
+ useEffect(() => {
795
+ const animation = Animated.timing(liveOpacity, {
796
+ toValue: showLive ? 1 : 0,
797
+ duration: crossfadeMs,
798
+ useNativeDriver: true
799
+ });
800
+ animation.start();
801
+ return () => animation.stop();
802
+ }, [showLive, crossfadeMs, liveOpacity]);
803
+ const floorHidden = IS_ANDROID && showLive;
804
+ const layers = resolveSurfaceLayers({
805
+ idleVideoUrl: floorHidden ? null : idleVideoUrl,
806
+ poster: floorHidden ? null : poster
807
+ });
808
+ const resizeMode = fit === "cover" ? "cover" : "contain";
809
+ const liveDims = videoTrack?.publication?.dimensions ?? null;
810
+ const liveLabel = showLive && liveDims && liveDims.width > 0 && liveDims.height > 0 ? `live \xB7 ${liveDims.width}\xD7${liveDims.height}` : "live";
811
+ const posterLayer = layers.showPoster ? createElement(Image, {
812
+ key: "poster",
813
+ source: { uri: poster },
814
+ resizeMode,
815
+ style: styles.layer,
816
+ accessibilityElementsHidden: true,
817
+ importantForAccessibility: "no-hide-descendants",
818
+ testID: "avatar-poster"
819
+ }) : null;
820
+ const idleLayer = layers.showIdleVideo && renderIdleVideo ? createElement(
821
+ View,
822
+ { key: "idle", style: styles.layer, pointerEvents: "none", testID: "avatar-idle-video" },
823
+ renderIdleVideo({ url: idleVideoUrl, style: styles.layer, resizeMode: fit })
824
+ ) : null;
825
+ const liveVideo = isTrackReference(videoTrack) ? createElement(VideoTrack, {
826
+ trackRef: videoTrack,
827
+ objectFit: fit,
828
+ style: StyleSheet.absoluteFillObject,
829
+ // Remote avatar video is the background layer on Android (below the window);
830
+ // the floor is hidden while it shows so it isn't occluded. Ignored on iOS.
831
+ ...IS_ANDROID ? { zOrder: ANDROID_REMOTE_ZORDER } : {}
832
+ }) : null;
833
+ const frontLayer = IS_ANDROID ? createElement(
834
+ View,
835
+ {
836
+ key: "live",
837
+ style: styles.layer,
838
+ pointerEvents: "none",
839
+ testID: "avatar-live-layer"
840
+ },
841
+ showLive ? liveVideo : null
842
+ ) : createElement(
843
+ Animated.View,
844
+ {
845
+ key: "live",
846
+ style: [styles.layer, { opacity: liveOpacity }],
847
+ pointerEvents: "none",
848
+ testID: "avatar-live-layer"
849
+ },
850
+ liveVideo
851
+ );
852
+ const badge = showLiveBadge && showLive ? createElement(
853
+ View,
854
+ { key: "badge", style: styles.badge, pointerEvents: "none" },
855
+ createElement(View, { key: "dot", style: styles.badgeDot }),
856
+ createElement(Text, { key: "label", style: styles.badgeText }, liveLabel)
857
+ ) : null;
858
+ return createElement(
859
+ View,
860
+ { style: [styles.box, style], testID },
861
+ posterLayer,
862
+ idleLayer,
863
+ frontLayer,
864
+ badge,
865
+ children
866
+ );
867
+ }
868
+ var styles = StyleSheet.create({
869
+ box: {
870
+ position: "relative",
871
+ width: "100%",
872
+ height: "100%",
873
+ overflow: "hidden"
874
+ },
875
+ layer: StyleSheet.absoluteFillObject,
876
+ badge: {
877
+ position: "absolute",
878
+ top: 8,
879
+ left: 8,
880
+ flexDirection: "row",
881
+ alignItems: "center",
882
+ gap: 6,
883
+ paddingHorizontal: 8,
884
+ paddingVertical: 4,
885
+ borderRadius: 999,
886
+ backgroundColor: "rgba(0,0,0,0.55)"
887
+ },
888
+ badgeDot: {
889
+ width: 6,
890
+ height: 6,
891
+ borderRadius: 3,
892
+ backgroundColor: "#34d399"
893
+ },
894
+ badgeText: {
895
+ color: "rgba(255,255,255,0.92)",
896
+ fontSize: 11,
897
+ fontVariant: ["tabular-nums"]
898
+ }
899
+ });
900
+ var DEFAULT_AVATAR_ID = "maria";
901
+ var DEFAULT_BACKGROUND_ID = "plain_white";
902
+ var sessionModeSchema = z.enum(["avatar", "voice"]);
903
+ var DEFAULT_SESSION_MODE = "avatar";
904
+ var DEFAULT_MAX_SESSION_SECONDS = 1800;
905
+ var avatarSourceKindSchema = z.enum(["portrait", "source_video"]);
906
+ var liveKitSttModeSchema = z.enum(["server", "off"]);
907
+ var renderBackendSchema = z.string();
908
+ var sessionLiveEditSchema = z.object({
909
+ rules: z.string().min(1).max(2e3),
910
+ cooldown_seconds: z.number().int().min(5).max(600).optional(),
911
+ // Which machinery runs the re-edit. Absent ⇒ the server default ("editor"). A deploy
912
+ // that cannot provide the requested renderer serves the editor lane and logs it,
913
+ // rather than failing a call that would otherwise have connected fine.
914
+ renderer: z.enum(["editor", "generative"]).optional()
915
+ }).strict();
916
+ var sessionSupportEditsSchema = z.object({
917
+ instruction: z.string().min(1).max(1e3),
918
+ reference_url: z.string().url().optional(),
919
+ live_edit: sessionLiveEditSchema.optional()
920
+ }).strict();
921
+ var LLM_PROVIDERS = ["local", "gemini", "openai"];
922
+ var llmProviderSchema = z.enum(LLM_PROVIDERS);
923
+ var llmConfigSchema = z.object({
924
+ backend: llmProviderSchema.optional(),
925
+ model: z.string().max(200).nullable().optional()
926
+ }).strict();
927
+ var llmSelectionSchema = z.object({
928
+ provider: llmProviderSchema,
929
+ model: z.string().max(200).nullable().optional()
930
+ }).strict();
931
+ var liveKitInitialContextMessageSchema = z.object({
932
+ role: z.enum(["system", "user", "assistant"]),
933
+ content: z.string().min(1).max(4e3)
934
+ }).strict();
935
+ var CARTESIA_TTS_MODELS = [
936
+ "cartesia/sonic-2",
937
+ "cartesia/sonic-2-latest",
938
+ "cartesia/sonic-3",
939
+ "cartesia/sonic-3-latest",
940
+ "cartesia/sonic-turbo",
941
+ "cartesia/sonic-turbo-latest"
942
+ ];
943
+ var cartesiaTtsModelSchema = z.enum(CARTESIA_TTS_MODELS);
944
+ var cartesiaVoiceSpecSchema = z.object({
945
+ provider: z.literal("cartesia"),
946
+ model: cartesiaTtsModelSchema.default("cartesia/sonic-3"),
947
+ voice_id: z.string().min(1).max(120),
948
+ speed: z.number().min(0.5).max(2).nullable().optional(),
949
+ emotion: z.string().min(1).max(80).nullable().optional(),
950
+ language: z.string().min(2).max(16).nullable().optional()
951
+ }).strict();
952
+ var FISH_TTS_MODELS = ["speech-1.6", "s1", "s2-pro", "speech-1.5", "s1-mini"];
953
+ var fishTtsModelSchema = z.enum(FISH_TTS_MODELS);
954
+ var breezeVoiceSpecSchema = z.object({
955
+ provider: z.literal("breezeblue"),
956
+ model: z.string().min(1).max(80).default("bluebell-v1-en"),
957
+ voice_id: z.string().min(1).max(120),
958
+ guidance_scale: z.number().min(1).max(10).nullable().optional(),
959
+ instructions: z.string().min(1).max(1e3).nullable().optional(),
960
+ language: z.string().min(2).max(16).nullable().optional()
961
+ }).strict();
962
+ var fishVoiceSpecSchema = z.object({
963
+ provider: z.literal("fish"),
964
+ model: fishTtsModelSchema.default("speech-1.6"),
965
+ voice_id: z.string().min(1).max(120),
966
+ speed: z.number().min(0.5).max(2).nullable().optional(),
967
+ emotion: z.string().min(1).max(80).nullable().optional(),
968
+ language: z.string().min(2).max(16).nullable().optional()
969
+ }).strict();
970
+ var voiceSpecSchema = z.discriminatedUnion("provider", [
971
+ cartesiaVoiceSpecSchema,
972
+ breezeVoiceSpecSchema,
973
+ fishVoiceSpecSchema
974
+ ]);
975
+ var nullableUrlSchema = z.string().url().nullable();
976
+ var clipTriggerSchema = z.enum([
977
+ "idle",
978
+ "listen",
979
+ "think",
980
+ "directive"
981
+ ]);
982
+ var sessionClipSchema = z.object({
983
+ clip_id: z.string().regex(/^[A-Za-z0-9_-]{1,64}$/, "clip_id must be a slug").refine((id) => id !== "primary", "'primary' is reserved for the avatar's source video"),
984
+ source_video_url: z.string().url().optional(),
985
+ video_cache_id: z.string().min(8).max(160).optional(),
986
+ max_seconds: z.number().min(1).max(10).optional(),
987
+ trigger: clipTriggerSchema.optional(),
988
+ loop: z.boolean().optional(),
989
+ weight: z.number().min(0).max(100).optional(),
990
+ crossfade_ms: z.number().int().min(0).max(1e3).optional(),
991
+ trim_start_ms: z.number().int().min(0).max(2e3).optional(),
992
+ trim_end_ms: z.number().int().min(0).max(2e3).optional(),
993
+ // The cue the character reads to decide this clip. `when` is the public name and the
994
+ // one the docs use; `hint` is the name the wire first shipped under and still accepts.
995
+ // Both are listed because this object is `.strict()` — omitting `when` would make the
996
+ // public name a validation error. Send one, never both.
997
+ when: z.string().min(1).max(120).optional(),
998
+ hint: z.string().min(1).max(120).optional()
999
+ }).strict().refine((clip) => clip.source_video_url || clip.video_cache_id, {
1000
+ message: "a clip needs source_video_url or video_cache_id"
1001
+ }).refine((clip) => !(clip.when && clip.hint), {
1002
+ message: "set `when` or `hint`, not both \u2014 they are the same field",
1003
+ path: ["when"]
1004
+ });
1005
+ var sessionChoreographySchema = z.object({
1006
+ idle_dwell_min_seconds: z.number().min(1).max(60).optional(),
1007
+ idle_dwell_max_seconds: z.number().min(1).max(120).optional(),
1008
+ special_weight: z.number().min(0).max(100).optional(),
1009
+ start_grace_seconds: z.number().min(0).max(60).optional(),
1010
+ crossfade_ms: z.number().int().min(0).max(1e3).optional(),
1011
+ crossfade_easing: z.enum(["linear", "smooth", "ease_out"]).optional(),
1012
+ wrap_crossfade_ms: z.number().int().min(0).max(1e3).optional()
1013
+ }).strict().refine(
1014
+ (c) => c.idle_dwell_min_seconds === void 0 || c.idle_dwell_max_seconds === void 0 || c.idle_dwell_min_seconds <= c.idle_dwell_max_seconds,
1015
+ { message: "idle_dwell_min_seconds must be <= idle_dwell_max_seconds" }
1016
+ );
1017
+ var sessionBehaviorSchema = z.object({
1018
+ gestures_enabled: z.boolean().optional(),
1019
+ gesture_freq: z.enum(["sparse", "balanced", "lively"]).optional()
1020
+ }).strict();
1021
+ var _SCENE_CLIP_ID_RE = /^[A-Za-z0-9_-]{1,64}$/;
1022
+ var sceneIdSchema = z.string().regex(/^[a-z0-9_]{1,40}$/, "scene_id must be a lowercase slug");
1023
+ var sceneTransitionSchema = z.object({
1024
+ clip_id: z.string().regex(_SCENE_CLIP_ID_RE, "clip_id must be a slug"),
1025
+ source_video_url: z.string().url(),
1026
+ from_scene: sceneIdSchema,
1027
+ to_scene: sceneIdSchema,
1028
+ max_seconds: z.number().min(1).max(10).optional()
1029
+ }).strict().superRefine((v, ctx) => {
1030
+ if (v.from_scene === v.to_scene) {
1031
+ ctx.addIssue({ code: "custom", message: "a transition's from_scene must differ from to_scene", path: ["to_scene"] });
1032
+ }
1033
+ });
1034
+ var sceneClusterSchema = z.object({
1035
+ scene_id: sceneIdSchema,
1036
+ hub_clip_id: z.string().regex(_SCENE_CLIP_ID_RE, "hub_clip_id must be a slug"),
1037
+ clips: z.array(sessionClipSchema).min(1).max(4)
1038
+ }).strict();
1039
+ var sceneGraphSchema = z.object({
1040
+ scenes: z.array(sceneClusterSchema).min(1).max(4),
1041
+ transitions: z.array(sceneTransitionSchema).min(2).max(12)
1042
+ }).strict();
1043
+ var transcriptWebhookSchema = z.object({
1044
+ url: z.string().url().max(500),
1045
+ secret: z.string().min(16).max(200)
1046
+ }).strict();
1047
+ var clientMetadataSchema = z.record(z.string().min(1).max(64), z.string().max(200)).refine((value) => Object.keys(value).length <= 16, {
1048
+ message: "client_metadata carries at most 16 entries"
1049
+ });
1050
+ var MAX_SESSION_INSTRUCTIONS_CHARS = 4e3;
1051
+ z.object({
1052
+ avatar_id: z.string().min(1).max(160).default(DEFAULT_AVATAR_ID),
1053
+ background_id: z.string().min(1).max(160).default(DEFAULT_BACKGROUND_ID),
1054
+ mode: sessionModeSchema.default(DEFAULT_SESSION_MODE),
1055
+ create_room: z.boolean().default(true),
1056
+ dispatch_agent: z.boolean().default(true),
1057
+ instructions: z.string().min(1).max(MAX_SESSION_INSTRUCTIONS_CHARS).optional(),
1058
+ initial_context: z.array(liveKitInitialContextMessageSchema).max(32).default([]),
1059
+ initial_say: z.string().min(1).max(1e3).optional(),
1060
+ llm: llmConfigSchema.nullable().optional(),
1061
+ max_session_seconds: z.number().int().min(1).max(DEFAULT_MAX_SESSION_SECONDS).optional(),
1062
+ participant_identity: z.string().min(1).max(160).optional(),
1063
+ participant_name: z.string().max(160).optional(),
1064
+ queue_ticket_id: z.string().min(1).max(160).optional(),
1065
+ portrait_url: nullableUrlSchema.optional(),
1066
+ room_name: z.string().min(1).max(160).optional(),
1067
+ source_kind: avatarSourceKindSchema.default("portrait"),
1068
+ source_video_url: nullableUrlSchema.optional(),
1069
+ stt_mode: liveKitSttModeSchema.default("server"),
1070
+ video_cache_id: z.string().min(1).max(240).nullable().optional(),
1071
+ voice: voiceSpecSchema.nullable().optional(),
1072
+ voice_id: z.string().min(1).max(240).nullable().optional(),
1073
+ // Deliberately UNBOUNDED. This object is `.strict()`, so it rejects rather than trims:
1074
+ // a count cap here does not mean "use fewer clips", it means "there is no call". How
1075
+ // many a session actually warms is decided where the clips are loaded, and loading
1076
+ // fewer is always safe — so the wire must not hold a number too.
1077
+ clip_library: z.array(sessionClipSchema).optional(),
1078
+ choreography: sessionChoreographySchema.optional(),
1079
+ scene_graph: sceneGraphSchema.optional(),
1080
+ behavior: sessionBehaviorSchema.optional(),
1081
+ expression_profile: z.string().min(1).max(40).optional(),
1082
+ render_backend: renderBackendSchema.optional(),
1083
+ support_edits: sessionSupportEditsSchema.optional(),
1084
+ transcript_webhook: transcriptWebhookSchema.optional(),
1085
+ client_metadata: clientMetadataSchema.optional()
1086
+ }).strict().superRefine((value, ctx) => {
1087
+ if (value.support_edits && value.render_backend === "generative") {
1088
+ ctx.addIssue({
1089
+ code: "custom",
1090
+ message: "support_edits needs a source video to edit; it cannot be combined with render_backend='generative'",
1091
+ path: ["support_edits"]
1092
+ });
1093
+ }
1094
+ if (value.support_edits && value.mode === "voice") {
1095
+ ctx.addIssue({
1096
+ code: "custom",
1097
+ message: "support_edits needs a video session; it cannot be combined with mode='voice'",
1098
+ path: ["support_edits"]
1099
+ });
1100
+ }
1101
+ if (value.source_kind === "portrait") {
1102
+ if (value.source_video_url || value.video_cache_id) {
1103
+ ctx.addIssue({
1104
+ code: "custom",
1105
+ message: "source_video_url/video_cache_id require source_kind='source_video'",
1106
+ path: ["source_kind"]
1107
+ });
1108
+ }
1109
+ return;
1110
+ }
1111
+ if (value.portrait_url) {
1112
+ ctx.addIssue({
1113
+ code: "custom",
1114
+ message: "portrait_url cannot be combined with source_kind='source_video'",
1115
+ path: ["portrait_url"]
1116
+ });
1117
+ }
1118
+ if (!value.source_video_url && !value.video_cache_id) {
1119
+ ctx.addIssue({
1120
+ code: "custom",
1121
+ message: "source_kind='source_video' requires source_video_url or video_cache_id",
1122
+ path: ["source_video_url"]
1123
+ });
1124
+ }
1125
+ });
1126
+ z.object({
1127
+ avatarId: z.string().min(1).max(160),
1128
+ backgroundId: z.string().min(1).max(160).default(DEFAULT_BACKGROUND_ID),
1129
+ mode: sessionModeSchema.default(DEFAULT_SESSION_MODE),
1130
+ createRoom: z.boolean().default(true),
1131
+ dispatchAgent: z.boolean().default(true),
1132
+ instructions: z.string().min(1).max(MAX_SESSION_INSTRUCTIONS_CHARS).optional(),
1133
+ initialContext: z.array(liveKitInitialContextMessageSchema).max(32).default([]),
1134
+ initialSay: z.string().min(1).max(1e3).optional(),
1135
+ llm: llmSelectionSchema.nullable().optional(),
1136
+ maxSessionSeconds: z.number().int().min(1).max(DEFAULT_MAX_SESSION_SECONDS).optional(),
1137
+ participantIdentity: z.string().min(1).max(160).optional(),
1138
+ participantName: z.string().max(160).optional(),
1139
+ queueTicketId: z.string().min(1).max(160).optional(),
1140
+ roomName: z.string().min(1).max(160).optional(),
1141
+ sttMode: liveKitSttModeSchema.default("server"),
1142
+ voice: voiceSpecSchema.nullable().optional(),
1143
+ voiceId: z.string().min(1).max(240).nullable().optional(),
1144
+ // Unbounded, for the same reason as `clip_library` on the wire schema above.
1145
+ clipLibrary: z.array(sessionClipSchema).optional(),
1146
+ sceneGraph: sceneGraphSchema.optional(),
1147
+ behavior: sessionBehaviorSchema.optional(),
1148
+ renderBackend: renderBackendSchema.optional(),
1149
+ supportEdits: sessionSupportEditsSchema.optional(),
1150
+ transcriptWebhook: transcriptWebhookSchema.optional(),
1151
+ clientMetadata: clientMetadataSchema.optional()
1152
+ }).strict();
1153
+ z.object({
1154
+ status: z.literal("ready").default("ready"),
1155
+ session_id: z.string().min(1),
1156
+ room_name: z.string().min(1),
1157
+ livekit_url: z.string().min(1),
1158
+ participant_token: z.string().min(1),
1159
+ participant_identity: z.string().min(1),
1160
+ reservation_expires_at: z.string().datetime({ offset: true }),
1161
+ stt_mode: liveKitSttModeSchema.default("server"),
1162
+ room_created: z.boolean().default(false),
1163
+ dispatch_created: z.boolean().default(false),
1164
+ join_timeout_seconds: z.number().int().nonnegative().default(0),
1165
+ idle_timeout_seconds: z.number().int().nonnegative().default(0),
1166
+ max_session_seconds: z.number().int().nonnegative().default(0)
1167
+ }).passthrough();
1168
+ var RTA_LIFECYCLE_TOPIC = "rta.lifecycle";
1169
+ var RTA_TURN_INSTRUCTIONS_ATTR = "rta.turn_instructions";
1170
+ var RTA_CLOSING_TURN_ATTR = "rta.closing_turn";
1171
+ var RTA_TURN_ID_ATTR = "rta.turn_id";
1172
+ var sessionEndReasonSchema = z.enum([
1173
+ "user_ended",
1174
+ "session_cap",
1175
+ "idle",
1176
+ "disconnected",
1177
+ "out_of_credits",
1178
+ "agent_ended",
1179
+ "failed"
1180
+ ]);
1181
+ var approachingEndReasonSchema = z.enum(["session_cap", "idle"]);
1182
+ var sessionClockFrameSchema = z.object({
1183
+ kind: z.literal("session_clock"),
1184
+ started_at_unix_ms: z.number().int().nonnegative(),
1185
+ max_session_seconds: z.number().int().nonnegative(),
1186
+ idle_timeout_seconds: z.number().int().nonnegative()
1187
+ }).strict();
1188
+ var endingFrameSchema = z.object({ kind: z.literal("ending"), reason: approachingEndReasonSchema }).strict();
1189
+ var closingTurnDoneFrameSchema = z.object({ kind: z.literal("closing_turn_done"), turn_id: z.string().min(1) }).strict();
1190
+ var endedFrameSchema = z.object({ kind: z.literal("ended"), reason: sessionEndReasonSchema }).strict();
1191
+ var knownBehaviorStates = ["idle", "listening", "thinking", "speaking"];
1192
+ var behaviorStateFrameSchema = z.object({
1193
+ kind: z.literal("behavior_state"),
1194
+ state: z.string().min(1).max(32),
1195
+ clip_id: z.string().min(1).max(64).optional(),
1196
+ trigger: clipTriggerSchema.optional(),
1197
+ loop: z.boolean().optional(),
1198
+ prev_clip_id: z.string().min(1).max(64).optional(),
1199
+ scene: z.string().min(1).max(40).optional(),
1200
+ scene_transition: z.object({
1201
+ clip_id: z.string().min(1).max(64),
1202
+ from_scene: z.string().min(1).max(40),
1203
+ to_scene: z.string().min(1).max(40)
1204
+ }).optional()
1205
+ }).strip();
1206
+ var clipAckFrameSchema = z.object({
1207
+ kind: z.literal("clip_ack"),
1208
+ request_id: z.string().max(64),
1209
+ accepted: z.boolean(),
1210
+ reason: z.string().max(64)
1211
+ }).strip();
1212
+ var lifecycleServerFrameSchema = z.discriminatedUnion("kind", [
1213
+ sessionClockFrameSchema,
1214
+ endingFrameSchema,
1215
+ closingTurnDoneFrameSchema,
1216
+ endedFrameSchema,
1217
+ behaviorStateFrameSchema,
1218
+ clipAckFrameSchema
1219
+ ]);
1220
+ var liveKitCapacitySnapshotSchema = z.object({
1221
+ // Placement identity + per-worker session ceiling. These are always present on the
1222
+ // wire (the platform serializes them and they come back on the grant, so a customer
1223
+ // already sees them) but are typed OPTIONAL here on purpose: this SDK is the defensive
1224
+ // READER of that wire, so a consumer must tolerate a response variant that omits them
1225
+ // rather than hard-fail parse. `max_sessions_per_gpu` is how many sessions one waking
1226
+ // worker serves — a queue-depth estimate input a consumer reads off the busy response.
1227
+ capacity_pool: z.string().min(1).optional(),
1228
+ agent_name: z.string().min(1).optional(),
1229
+ max_sessions: z.number().int().nonnegative(),
1230
+ max_sessions_per_gpu: z.number().int().positive().optional(),
1231
+ worker_count: z.number().int().nonnegative(),
1232
+ active_sessions: z.number().int().nonnegative(),
1233
+ reserved_sessions: z.number().int().nonnegative(),
1234
+ observed_worker_active_sessions: z.number().int().nonnegative(),
1235
+ available_sessions: z.number().int().nonnegative(),
1236
+ queue_size: z.number().int().nonnegative(),
1237
+ admission_open: z.boolean(),
1238
+ recommended_retry_ms: z.number().int().nonnegative(),
1239
+ load: z.number().min(0).max(1)
1240
+ }).passthrough();
1241
+ z.object({
1242
+ message: z.string().min(1),
1243
+ capacity: liveKitCapacitySnapshotSchema,
1244
+ queue_size: z.number().int().nonnegative(),
1245
+ queue_ticket_id: z.string().min(1).optional(),
1246
+ queue_position: z.number().int().positive().optional(),
1247
+ recommended_retry_ms: z.number().int().nonnegative()
1248
+ }).strict();
1249
+ z.enum([
1250
+ "page_hide",
1251
+ "disconnected",
1252
+ "superseded",
1253
+ "unmount",
1254
+ "manual",
1255
+ "idle_timeout"
1256
+ ]);
1257
+
1258
+ // ../client/src/react/behavior-snapshot.ts
1259
+ function nextBehaviorSnapshot(prev, frame) {
1260
+ const next = {
1261
+ state: frame.state,
1262
+ clipId: frame.clip_id ?? null,
1263
+ trigger: frame.trigger ?? null,
1264
+ loop: frame.loop ?? null,
1265
+ prevClipId: frame.prev_clip_id ?? null
1266
+ };
1267
+ if (prev && prev.state === next.state && prev.clipId === next.clipId && prev.trigger === next.trigger && prev.loop === next.loop && prev.prevClipId === next.prevClipId)
1268
+ return null;
1269
+ return next;
1270
+ }
1271
+
1272
+ // ../client/src/react/session-lifecycle.ts
1273
+ function phaseForReason(reason) {
1274
+ switch (reason) {
1275
+ case DisconnectReason.CLIENT_INITIATED:
1276
+ return "ignore";
1277
+ case DisconnectReason.ROOM_DELETED:
1278
+ case DisconnectReason.SERVER_SHUTDOWN:
1279
+ case DisconnectReason.DUPLICATE_IDENTITY:
1280
+ case DisconnectReason.PARTICIPANT_REMOVED:
1281
+ return "ended";
1282
+ default:
1283
+ return "reconnecting";
1284
+ }
1285
+ }
1286
+ function disconnectAction(reason, _everConnected) {
1287
+ const phase = phaseForReason(reason);
1288
+ if (phase === "ignore") return "reset";
1289
+ if (phase === "ended") return "end";
1290
+ return "reconnect";
1291
+ }
1292
+ function isRecoverableConnectionError(error) {
1293
+ if (!(error instanceof ConnectionError)) return false;
1294
+ switch (error.reason) {
1295
+ case ConnectionErrorReason.ServerUnreachable:
1296
+ case ConnectionErrorReason.InternalError:
1297
+ case ConnectionErrorReason.Timeout:
1298
+ case ConnectionErrorReason.WebSocket:
1299
+ return true;
1300
+ default:
1301
+ return false;
1302
+ }
1303
+ }
1304
+ var RECONNECT_BACKOFF_MS = [800, 2e3, 4e3];
1305
+ var MAX_RECONNECT_ATTEMPTS = 4;
1306
+ function resolveReconnectPolicy(override) {
1307
+ const backoff = override?.backoffMs?.filter((ms) => Number.isFinite(ms) && ms > 0) ?? [];
1308
+ const backoffMs = backoff.length > 0 ? backoff : RECONNECT_BACKOFF_MS;
1309
+ const max = override?.maxAttempts;
1310
+ const maxAttempts = typeof max === "number" && Number.isFinite(max) && max > 0 ? Math.floor(max) : MAX_RECONNECT_ATTEMPTS;
1311
+ return { backoffMs, maxAttempts };
1312
+ }
1313
+ function retryStep(attempt, policy = resolveReconnectPolicy()) {
1314
+ if (attempt >= policy.maxAttempts) return { kind: "give-up" };
1315
+ const delayMs = policy.backoffMs[Math.min(attempt, policy.backoffMs.length - 1)];
1316
+ return { kind: "retry", delayMs, attempt: attempt + 1 };
1317
+ }
1318
+ function idlePhaseFor(args) {
1319
+ const { connected, agentPresent, msSinceActivity, idleTimeoutMs, warnAtMs } = args;
1320
+ if (!connected || !agentPresent || idleTimeoutMs <= 0) return { kind: "inactive" };
1321
+ const remainingMs = idleTimeoutMs - msSinceActivity;
1322
+ if (remainingMs > warnAtMs) return { kind: "live" };
1323
+ const secondsRemaining = Math.max(0, Math.ceil(remainingMs / 1e3));
1324
+ return { kind: "idle-warning", secondsRemaining };
1325
+ }
1326
+ function idleExpired(args) {
1327
+ const { msSinceActivity, idleTimeoutMs } = args;
1328
+ if (idleTimeoutMs <= 0) return false;
1329
+ return msSinceActivity >= idleTimeoutMs;
1330
+ }
1331
+ function needsFreshGrant(recovery) {
1332
+ return recovery.kind === "reconnecting";
1333
+ }
1334
+ function lifecyclePhaseFrom(args) {
1335
+ const { capacity, recovery, idle, connected, agentPresent } = args;
1336
+ if (recovery.kind === "in-place-reconnecting") {
1337
+ return { kind: "reconnectable", reconnecting: true, attempt: 0, strategy: "in-place" };
1338
+ }
1339
+ if (recovery.kind === "refreshing" || recovery.kind === "reconnecting") {
1340
+ return {
1341
+ kind: "reconnectable",
1342
+ reconnecting: true,
1343
+ attempt: recovery.attempt,
1344
+ strategy: "fresh-grant"
1345
+ };
1346
+ }
1347
+ if (recovery.kind === "failed") {
1348
+ return {
1349
+ kind: "reconnectable",
1350
+ reconnecting: false,
1351
+ attempt: MAX_RECONNECT_ATTEMPTS,
1352
+ strategy: "fresh-grant"
1353
+ };
1354
+ }
1355
+ if (recovery.kind === "ended") {
1356
+ return recovery.reason ? { kind: "ended", reason: recovery.reason } : { kind: "ended" };
1357
+ }
1358
+ switch (capacity.kind) {
1359
+ case "idle":
1360
+ return { kind: "idle" };
1361
+ case "connecting":
1362
+ return { kind: "requesting" };
1363
+ case "queued":
1364
+ return { kind: "queued", busy: capacity.busy };
1365
+ case "error":
1366
+ return { kind: "ended", reason: "error" };
1367
+ case "active": {
1368
+ if (!connected || !agentPresent) {
1369
+ return { kind: "connecting", grant: capacity.grant };
1370
+ }
1371
+ if (idle.kind === "idle-warning") {
1372
+ return { kind: "idle-warning", secondsRemaining: idle.secondsRemaining, deadlineAt: idle.deadlineAt };
1373
+ }
1374
+ return { kind: "live" };
1375
+ }
1376
+ default:
1377
+ return { kind: "idle" };
1378
+ }
1379
+ }
1380
+ function isCallActivity(args) {
1381
+ const { assistantState, transcriptionCount, prevTranscriptionCount } = args;
1382
+ if (assistantState === "speaking" || assistantState === "thinking") return true;
1383
+ return transcriptionCount > prevTranscriptionCount;
1384
+ }
1385
+ var DEFAULT_IDLE_SECONDS = 120;
1386
+ var DEFAULT_IDLE_WARN_LEAD_SECONDS = 20;
1387
+ var DEFAULT_TURN_TIMEOUT_SECONDS = 20;
1388
+ function resolveIdleTimeoutMs(args) {
1389
+ const { idleSecondsOption, grantIdleSeconds } = args;
1390
+ if (typeof idleSecondsOption === "number" && idleSecondsOption > 0) {
1391
+ return Math.floor(idleSecondsOption) * 1e3;
1392
+ }
1393
+ if (grantIdleSeconds > 0) return grantIdleSeconds * 1e3;
1394
+ return DEFAULT_IDLE_SECONDS * 1e3;
1395
+ }
1396
+ function resolveWarnBeforeMs(idleTimeoutMs, override, idleWarnLeadSeconds) {
1397
+ if (idleTimeoutMs <= 0) return 0;
1398
+ const maxLeadMs = Math.max(0, idleTimeoutMs - 1e3);
1399
+ let leadMs;
1400
+ if (typeof idleWarnLeadSeconds === "number" && idleWarnLeadSeconds > 0) {
1401
+ leadMs = Math.floor(idleWarnLeadSeconds) * 1e3;
1402
+ } else if (typeof override === "number" && override > 0) {
1403
+ leadMs = override;
1404
+ } else {
1405
+ leadMs = DEFAULT_IDLE_WARN_LEAD_SECONDS * 1e3;
1406
+ }
1407
+ return Math.min(leadMs, maxLeadMs);
1408
+ }
1409
+ function useSessionLifecycle(input) {
1410
+ const {
1411
+ client,
1412
+ session,
1413
+ active = true,
1414
+ idleSeconds,
1415
+ idleWarnLeadSeconds,
1416
+ warnBeforeMs,
1417
+ reconnectBackoffMs,
1418
+ maxReconnectAttempts,
1419
+ autoRetryBusy = true,
1420
+ requestOptions,
1421
+ onBehaviorChange
1422
+ } = input;
1423
+ const reconnectPolicy = useMemo(
1424
+ () => resolveReconnectPolicy({ backoffMs: reconnectBackoffMs, maxAttempts: maxReconnectAttempts }),
1425
+ [reconnectBackoffMs, maxReconnectAttempts]
1426
+ );
1427
+ const reconnectPolicyRef = useRef(reconnectPolicy);
1428
+ reconnectPolicyRef.current = reconnectPolicy;
1429
+ const grantState = useLiveKitAvatarGrant({
1430
+ client,
1431
+ session,
1432
+ active,
1433
+ autoRetryBusy,
1434
+ requestOptions
1435
+ });
1436
+ const capacity = grantState.capacity;
1437
+ const [recovery, setRecovery] = useState({ kind: "connected" });
1438
+ const recoveryRef = useRef(recovery);
1439
+ recoveryRef.current = recovery;
1440
+ const [attempt, setAttempt] = useState(0);
1441
+ const connectedRef = useRef(false);
1442
+ const attemptRef = useRef(0);
1443
+ const timerRef = useRef(null);
1444
+ const manualReconnectPendingRef = useRef(false);
1445
+ const refreshRef = useRef(grantState.refresh);
1446
+ refreshRef.current = grantState.refresh;
1447
+ const releaseRef = useRef(grantState.release);
1448
+ releaseRef.current = grantState.release;
1449
+ const capacityRef = useRef(capacity);
1450
+ capacityRef.current = capacity;
1451
+ const refreshBaselineCapacityRef = useRef(null);
1452
+ const idleTimeoutMs = resolveIdleTimeoutMs({
1453
+ idleSecondsOption: idleSeconds,
1454
+ grantIdleSeconds: grantState.grant?.idle_timeout_seconds ?? 0
1455
+ });
1456
+ const warnAtMs = resolveWarnBeforeMs(idleTimeoutMs, warnBeforeMs, idleWarnLeadSeconds);
1457
+ const [agentPresent, setAgentPresentState] = useState(false);
1458
+ const [connected, setConnected] = useState(false);
1459
+ const lastActivityRef = useRef(Date.now());
1460
+ const leaveRoomRef = useRef(null);
1461
+ const [, setClockTick] = useState(0);
1462
+ const clearTimer = useCallback(() => {
1463
+ if (timerRef.current !== null) {
1464
+ window.clearTimeout(timerRef.current);
1465
+ timerRef.current = null;
1466
+ }
1467
+ }, []);
1468
+ const reset = useCallback(() => {
1469
+ clearTimer();
1470
+ connectedRef.current = false;
1471
+ manualReconnectPendingRef.current = false;
1472
+ refreshBaselineCapacityRef.current = null;
1473
+ attemptRef.current = 0;
1474
+ setAttempt(0);
1475
+ setRecovery({ kind: "connected" });
1476
+ setConnected(false);
1477
+ setAgentPresentState(false);
1478
+ lastActivityRef.current = Date.now();
1479
+ }, [clearTimer]);
1480
+ useEffect(() => {
1481
+ if (!active) reset();
1482
+ }, [active, reset]);
1483
+ const markActivity = useCallback(() => {
1484
+ lastActivityRef.current = Date.now();
1485
+ setClockTick((value) => value + 1);
1486
+ }, []);
1487
+ const stayConnected = useCallback(() => {
1488
+ if (!connectedRef.current) return;
1489
+ lastActivityRef.current = Date.now();
1490
+ setClockTick((value) => value + 1);
1491
+ }, []);
1492
+ const setAgentPresent = useCallback((present) => {
1493
+ setAgentPresentState((prev) => {
1494
+ if (present && !prev) lastActivityRef.current = Date.now();
1495
+ return present;
1496
+ });
1497
+ }, []);
1498
+ const registerLeaveRoom = useCallback((leave) => {
1499
+ leaveRoomRef.current = leave;
1500
+ }, []);
1501
+ const endIdle = useCallback(() => {
1502
+ if (!connectedRef.current) return;
1503
+ clearTimer();
1504
+ connectedRef.current = false;
1505
+ setConnected(false);
1506
+ setAgentPresentState(false);
1507
+ releaseRef.current("idle_timeout");
1508
+ setRecovery({ kind: "ended", reason: "idle" });
1509
+ leaveRoomRef.current?.();
1510
+ }, [clearTimer]);
1511
+ const onConnected = useCallback(() => {
1512
+ clearTimer();
1513
+ connectedRef.current = true;
1514
+ manualReconnectPendingRef.current = false;
1515
+ refreshBaselineCapacityRef.current = null;
1516
+ setConnected(true);
1517
+ attemptRef.current = 0;
1518
+ setAttempt(0);
1519
+ setRecovery({ kind: "connected" });
1520
+ lastActivityRef.current = Date.now();
1521
+ }, [clearTimer]);
1522
+ const onDisconnected = useCallback(
1523
+ (reason) => {
1524
+ if (!active) return;
1525
+ const action = disconnectAction(reason, connectedRef.current);
1526
+ if (action === "noop") return;
1527
+ if (action === "reset") {
1528
+ if (recoveryRef.current.kind === "refreshing") return;
1529
+ reset();
1530
+ return;
1531
+ }
1532
+ connectedRef.current = false;
1533
+ setConnected(false);
1534
+ setAgentPresentState(false);
1535
+ if (action === "end") {
1536
+ manualReconnectPendingRef.current = false;
1537
+ refreshBaselineCapacityRef.current = null;
1538
+ releaseRef.current("disconnected");
1539
+ setRecovery({ kind: "ended" });
1540
+ return;
1541
+ }
1542
+ refreshBaselineCapacityRef.current = null;
1543
+ setRecovery({ kind: "reconnecting", attempt: attemptRef.current });
1544
+ },
1545
+ [active, reset]
1546
+ );
1547
+ const onConnectionError = useCallback(
1548
+ (error) => {
1549
+ if (!active || !isRecoverableConnectionError(error)) return;
1550
+ connectedRef.current = false;
1551
+ setConnected(false);
1552
+ setAgentPresentState(false);
1553
+ setRecovery((prev) => {
1554
+ if (prev.kind === "in-place-reconnecting") return prev;
1555
+ if (prev.kind === "reconnecting" || prev.kind === "failed" || prev.kind === "ended") return prev;
1556
+ refreshBaselineCapacityRef.current = null;
1557
+ return { kind: "reconnecting", attempt: attemptRef.current };
1558
+ });
1559
+ },
1560
+ [active]
1561
+ );
1562
+ useEffect(() => {
1563
+ if (recovery.kind !== "refreshing" || capacity.kind !== "error") return;
1564
+ const baseline = refreshBaselineCapacityRef.current;
1565
+ if (baseline?.kind === "error" && baseline.error === capacity.error) return;
1566
+ refreshBaselineCapacityRef.current = null;
1567
+ setRecovery({ kind: "reconnecting", attempt: recovery.attempt });
1568
+ }, [capacity, recovery]);
1569
+ const onConnectionStateChange = useCallback(
1570
+ (state) => {
1571
+ if (!active) return;
1572
+ if (state === "connected") {
1573
+ onConnected();
1574
+ return;
1575
+ }
1576
+ if (state === "reconnecting" || state === "signalReconnecting") {
1577
+ if (connectedRef.current) {
1578
+ setConnected(false);
1579
+ setRecovery(
1580
+ (prev) => prev.kind === "connected" ? { kind: "in-place-reconnecting" } : prev
1581
+ );
1582
+ }
1583
+ }
1584
+ },
1585
+ [active, onConnected]
1586
+ );
1587
+ useEffect(() => {
1588
+ if (!active || !needsFreshGrant(recovery)) return;
1589
+ const step2 = retryStep(attemptRef.current, reconnectPolicyRef.current);
1590
+ if (step2.kind === "give-up") {
1591
+ manualReconnectPendingRef.current = false;
1592
+ refreshBaselineCapacityRef.current = null;
1593
+ releaseRef.current("disconnected");
1594
+ setRecovery({ kind: "failed" });
1595
+ return;
1596
+ }
1597
+ timerRef.current = window.setTimeout(() => {
1598
+ timerRef.current = null;
1599
+ attemptRef.current = step2.attempt;
1600
+ setAttempt(step2.attempt);
1601
+ refreshBaselineCapacityRef.current = capacityRef.current;
1602
+ setRecovery({ kind: "refreshing", attempt: step2.attempt });
1603
+ refreshRef.current();
1604
+ }, step2.delayMs);
1605
+ return clearTimer;
1606
+ }, [recovery.kind, active, attempt, clearTimer]);
1607
+ const reconnect = useCallback(() => {
1608
+ if (!active || manualReconnectPendingRef.current || recovery.kind === "in-place-reconnecting") return;
1609
+ manualReconnectPendingRef.current = true;
1610
+ clearTimer();
1611
+ attemptRef.current = 0;
1612
+ setAttempt(0);
1613
+ refreshBaselineCapacityRef.current = capacityRef.current;
1614
+ setRecovery({ kind: "refreshing", attempt: 0 });
1615
+ refreshRef.current();
1616
+ }, [active, clearTimer, recovery.kind]);
1617
+ const endIdleRef = useRef(endIdle);
1618
+ endIdleRef.current = endIdle;
1619
+ useEffect(() => {
1620
+ if (!connected || !agentPresent || idleTimeoutMs <= 0 || recovery.kind !== "connected") return;
1621
+ const timer = window.setInterval(() => {
1622
+ if (idleExpired({ msSinceActivity: Date.now() - lastActivityRef.current, idleTimeoutMs })) {
1623
+ endIdleRef.current();
1624
+ return;
1625
+ }
1626
+ setClockTick((value) => value + 1);
1627
+ }, 1e3);
1628
+ return () => window.clearInterval(timer);
1629
+ }, [connected, agentPresent, idleTimeoutMs, recovery.kind]);
1630
+ const timeToDisconnectMs = connected && agentPresent && idleTimeoutMs > 0 && recovery.kind === "connected" ? Math.max(0, idleTimeoutMs - (Date.now() - lastActivityRef.current)) : null;
1631
+ const idle = useMemo(() => {
1632
+ const base = idlePhaseFor({
1633
+ connected,
1634
+ agentPresent,
1635
+ msSinceActivity: Date.now() - lastActivityRef.current,
1636
+ idleTimeoutMs,
1637
+ warnAtMs
1638
+ });
1639
+ if (base.kind === "idle-warning") {
1640
+ return { ...base, deadlineAt: lastActivityRef.current + idleTimeoutMs };
1641
+ }
1642
+ return base;
1643
+ }, [connected, agentPresent, idleTimeoutMs, warnAtMs, timeToDisconnectMs]);
1644
+ const phase = useMemo(
1645
+ () => lifecyclePhaseFrom({ capacity, recovery, idle, connected, agentPresent }),
1646
+ [capacity, recovery, idle, connected, agentPresent]
1647
+ );
1648
+ const onBehaviorChangeRef = useRef(onBehaviorChange);
1649
+ onBehaviorChangeRef.current = onBehaviorChange;
1650
+ const lastBehaviorRef = useRef(null);
1651
+ const behaviorEnabled = Boolean(onBehaviorChange);
1652
+ const onLifecycleData = useMemo(() => {
1653
+ if (!behaviorEnabled) return void 0;
1654
+ return (frame) => {
1655
+ const parsed = lifecycleServerFrameSchema.safeParse(frame);
1656
+ if (!parsed.success || parsed.data.kind !== "behavior_state") return;
1657
+ const next = nextBehaviorSnapshot(lastBehaviorRef.current, parsed.data);
1658
+ if (!next) return;
1659
+ lastBehaviorRef.current = next;
1660
+ onBehaviorChangeRef.current?.(next);
1661
+ };
1662
+ }, [behaviorEnabled]);
1663
+ return useMemo(
1664
+ () => ({
1665
+ phase,
1666
+ grant: grantState.grant,
1667
+ capacity,
1668
+ attempt,
1669
+ timeToDisconnectMs,
1670
+ stayConnected,
1671
+ markActivity,
1672
+ reconnect,
1673
+ onConnected,
1674
+ onDisconnected,
1675
+ onConnectionError,
1676
+ onConnectionStateChange,
1677
+ setAgentPresent,
1678
+ registerLeaveRoom,
1679
+ reset,
1680
+ onLifecycleData
1681
+ }),
1682
+ [
1683
+ phase,
1684
+ grantState.grant,
1685
+ capacity,
1686
+ attempt,
1687
+ timeToDisconnectMs,
1688
+ stayConnected,
1689
+ markActivity,
1690
+ reconnect,
1691
+ onConnected,
1692
+ onDisconnected,
1693
+ onConnectionError,
1694
+ onConnectionStateChange,
1695
+ setAgentPresent,
1696
+ registerLeaveRoom,
1697
+ reset,
1698
+ onLifecycleData
1699
+ ]
1700
+ );
1701
+ }
1702
+ function SessionLifecycleRoomBridge({ lifecycle }) {
1703
+ const {
1704
+ onConnectionStateChange,
1705
+ setAgentPresent,
1706
+ registerLeaveRoom,
1707
+ markActivity,
1708
+ onLifecycleData,
1709
+ registerDataPublisher,
1710
+ registerTurnSender,
1711
+ setTurnState,
1712
+ setMedia
1713
+ } = lifecycle;
1714
+ const connectionState = useConnectionState();
1715
+ const assistant = useVoiceAssistant();
1716
+ const transcriptions = useTranscriptions();
1717
+ const room = useRoomContext();
1718
+ const { send } = useChat();
1719
+ const agentPresent = Boolean(assistant.agent);
1720
+ const assistantState = assistant.state;
1721
+ const videoLive = Boolean(assistant.videoTrack);
1722
+ const audioLive = Boolean(assistant.audioTrack);
1723
+ const transcriptionCount = transcriptions.length;
1724
+ const prevTranscriptionCountRef = useRef(0);
1725
+ useEffect(() => {
1726
+ onConnectionStateChange(connectionState);
1727
+ }, [connectionState, onConnectionStateChange]);
1728
+ useEffect(() => {
1729
+ setAgentPresent(agentPresent);
1730
+ }, [agentPresent, setAgentPresent]);
1731
+ useEffect(() => {
1732
+ const active = isCallActivity({
1733
+ assistantState,
1734
+ transcriptionCount,
1735
+ prevTranscriptionCount: prevTranscriptionCountRef.current
1736
+ });
1737
+ prevTranscriptionCountRef.current = transcriptionCount;
1738
+ if (active) markActivity();
1739
+ }, [assistantState, transcriptionCount, markActivity]);
1740
+ useEffect(() => {
1741
+ const leave = () => {
1742
+ void Promise.resolve(room.disconnect()).catch(() => void 0);
1743
+ };
1744
+ registerLeaveRoom(leave);
1745
+ return () => registerLeaveRoom(null);
1746
+ }, [room, registerLeaveRoom]);
1747
+ useEffect(() => {
1748
+ if (!onLifecycleData) return;
1749
+ const decoder = new TextDecoder();
1750
+ const handler = (payload, _participant, _kind, topic) => {
1751
+ if (topic !== RTA_LIFECYCLE_TOPIC) return;
1752
+ try {
1753
+ onLifecycleData(JSON.parse(decoder.decode(payload)));
1754
+ } catch {
1755
+ }
1756
+ };
1757
+ room.on(RoomEvent.DataReceived, handler);
1758
+ return () => {
1759
+ room.off(RoomEvent.DataReceived, handler);
1760
+ };
1761
+ }, [room, onLifecycleData]);
1762
+ useEffect(() => {
1763
+ if (!registerDataPublisher) return;
1764
+ const encoder = new TextEncoder();
1765
+ const publish = (frame) => {
1766
+ try {
1767
+ void room.localParticipant?.publishData(encoder.encode(JSON.stringify(frame)), { reliable: true, topic: RTA_LIFECYCLE_TOPIC }).catch(() => void 0);
1768
+ } catch {
1769
+ }
1770
+ };
1771
+ registerDataPublisher(publish);
1772
+ return () => registerDataPublisher(null);
1773
+ }, [room, registerDataPublisher]);
1774
+ useEffect(() => {
1775
+ if (!registerTurnSender) return;
1776
+ const sender = (text, opts) => Promise.resolve(send(text, opts)).then(() => void 0);
1777
+ registerTurnSender(sender);
1778
+ return () => registerTurnSender(null);
1779
+ }, [send, registerTurnSender]);
1780
+ useEffect(() => {
1781
+ setTurnState?.(assistantState);
1782
+ }, [assistantState, setTurnState]);
1783
+ useEffect(() => {
1784
+ setMedia?.({ video: videoLive ? "live" : "connecting", audio: audioLive ? "flowing" : "silent" });
1785
+ }, [videoLive, audioLive, setMedia]);
1786
+ return null;
1787
+ }
1788
+
1789
+ // ../client/src/react/grace-window.ts
1790
+ function mapTurnState(assistantState) {
1791
+ switch (assistantState) {
1792
+ case "listening":
1793
+ case "thinking":
1794
+ case "speaking":
1795
+ return assistantState;
1796
+ default:
1797
+ return "quiet";
1798
+ }
1799
+ }
1800
+ function endsAtFrom(args) {
1801
+ const { serverStartedAtUnixMs, maxSessionSeconds } = args;
1802
+ if (serverStartedAtUnixMs == null || !maxSessionSeconds) return null;
1803
+ return serverStartedAtUnixMs + maxSessionSeconds * 1e3;
1804
+ }
1805
+ function sessionRemainingMsFrom(args) {
1806
+ const endsAt = endsAtFrom(args);
1807
+ if (endsAt == null) return null;
1808
+ return Math.max(0, endsAt - args.nowUnixMs);
1809
+ }
1810
+ function approachingEndFrom(args) {
1811
+ const { sessionRemainingMs, approachingEndLeadMs, alreadyFired } = args;
1812
+ if (alreadyFired || sessionRemainingMs == null) return null;
1813
+ if (sessionRemainingMs <= approachingEndLeadMs && sessionRemainingMs > 0) {
1814
+ return { secondsLeft: Math.ceil(sessionRemainingMs / 1e3) };
1815
+ }
1816
+ return null;
1817
+ }
1818
+ function creditsLowFrom(args) {
1819
+ const { creditRemainingMs, creditsLowLeadMs, alreadyFired } = args;
1820
+ if (alreadyFired || creditRemainingMs == null) return null;
1821
+ if (creditRemainingMs <= creditsLowLeadMs && creditRemainingMs > 0) {
1822
+ return { secondsLeft: Math.ceil(creditRemainingMs / 1e3) };
1823
+ }
1824
+ return null;
1825
+ }
1826
+ function nextGraceWindow(args) {
1827
+ const { prev, sessionRemainingMs, graceWindowLeadMs, endsAt, workerEnding, nowMs } = args;
1828
+ switch (prev.kind) {
1829
+ case "spent":
1830
+ return prev;
1831
+ // terminal
1832
+ case "closed": {
1833
+ const clockCrossed = sessionRemainingMs != null && sessionRemainingMs <= graceWindowLeadMs && sessionRemainingMs > 0;
1834
+ if ((workerEnding || clockCrossed) && endsAt != null) {
1835
+ return { kind: "open", reason: "session_cap", deadlineAt: endsAt, msLeft: Math.max(0, endsAt - nowMs) };
1836
+ }
1837
+ return prev;
1838
+ }
1839
+ case "open":
1840
+ if (nowMs >= prev.deadlineAt) return { kind: "spent", delivered: false };
1841
+ return { ...prev, msLeft: Math.max(0, prev.deadlineAt - nowMs) };
1842
+ case "delivering":
1843
+ if (nowMs >= prev.deadlineAt) return { kind: "spent", delivered: false };
1844
+ return prev;
1845
+ }
1846
+ }
1847
+ function resolveEndReason(workerLabel, innerReason) {
1848
+ if (workerLabel) return workerLabel;
1849
+ switch (innerReason) {
1850
+ case "idle":
1851
+ return "idle";
1852
+ case "error":
1853
+ return "failed";
1854
+ default:
1855
+ return "disconnected";
1856
+ }
1857
+ }
1858
+
1859
+ // ../client/src/react/use-realtime-session.ts
1860
+ var DEFAULT_APPROACHING_END_LEAD_SECONDS = 45;
1861
+ var DEFAULT_GRACE_WINDOW_LEAD_SECONDS = 12;
1862
+ var DEFAULT_GRACE_CEILING_SECONDS = 10;
1863
+ var DEFAULT_CREDITS_LOW_LEAD_SECONDS = 300;
1864
+ function positive(value, fallbackSeconds) {
1865
+ return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : fallbackSeconds;
1866
+ }
1867
+ function useRealtimeSession(input) {
1868
+ const {
1869
+ session,
1870
+ maxSessionSeconds,
1871
+ approachingEndLeadSeconds,
1872
+ graceWindowLeadSeconds,
1873
+ graceCeilingSeconds,
1874
+ creditsLowLeadSeconds,
1875
+ turnTimeoutSeconds,
1876
+ creditRemainingMs = null,
1877
+ autoStayConnectedDuringGrace = true,
1878
+ onApproachingEnd,
1879
+ onGraceWindowOpen,
1880
+ onGraceWindowClosed,
1881
+ onIdleWarning,
1882
+ onCreditsLow,
1883
+ onTurnTimeout,
1884
+ onReconnecting,
1885
+ onReconnected,
1886
+ onEnded,
1887
+ onBehaviorChange,
1888
+ onClipResult,
1889
+ // everything else (client/active/idleSeconds/idleWarnLeadSeconds/reconnect/requestOptions…)
1890
+ // flows straight into the inner SSOT hook.
1891
+ ...inner
1892
+ } = input;
1893
+ const effectiveSession = useMemo(
1894
+ () => session && typeof maxSessionSeconds === "number" ? { ...session, maxSessionSeconds } : session,
1895
+ [session, maxSessionSeconds]
1896
+ );
1897
+ const lifecycle = useSessionLifecycle({ ...inner, session: effectiveSession });
1898
+ const [serverClock, setServerClock] = useState(null);
1899
+ const [workerEnding, setWorkerEnding] = useState(false);
1900
+ const [graceWindow, setGraceWindow] = useState({ kind: "closed" });
1901
+ const [turn, setTurn] = useState("quiet");
1902
+ const [media, setMediaState] = useState({ video: "connecting", audio: "silent" });
1903
+ const [behavior, setBehavior] = useState(null);
1904
+ const lastBehaviorRef = useRef(null);
1905
+ const lastLabeledEndReasonRef = useRef(null);
1906
+ const pendingClipsRef = useRef(/* @__PURE__ */ new Map());
1907
+ const cbRef = useRef({ onApproachingEnd, onGraceWindowOpen, onGraceWindowClosed, onIdleWarning, onCreditsLow, onTurnTimeout, onReconnecting, onReconnected, onEnded, onBehaviorChange, onClipResult });
1908
+ cbRef.current = { onApproachingEnd, onGraceWindowOpen, onGraceWindowClosed, onIdleWarning, onCreditsLow, onTurnTimeout, onReconnecting, onReconnected, onEnded, onBehaviorChange, onClipResult };
1909
+ const approachingEndLeadMs = positive(approachingEndLeadSeconds, DEFAULT_APPROACHING_END_LEAD_SECONDS) * 1e3;
1910
+ const graceWindowLeadMs = positive(graceWindowLeadSeconds, DEFAULT_GRACE_WINDOW_LEAD_SECONDS) * 1e3;
1911
+ const graceCeilingMs = positive(graceCeilingSeconds, DEFAULT_GRACE_CEILING_SECONDS) * 1e3;
1912
+ const creditsLowLeadMs = positive(creditsLowLeadSeconds, DEFAULT_CREDITS_LOW_LEAD_SECONDS) * 1e3;
1913
+ const turnTimeoutMs = positive(turnTimeoutSeconds, DEFAULT_TURN_TIMEOUT_SECONDS) * 1e3;
1914
+ const approachingFiredRef = useRef(false);
1915
+ const creditsLowFiredRef = useRef(false);
1916
+ const idleWarnFiredRef = useRef(false);
1917
+ const endedFiredRef = useRef(false);
1918
+ const prevPhaseKindRef = useRef("idle");
1919
+ const graceWindowRef = useRef(graceWindow);
1920
+ graceWindowRef.current = graceWindow;
1921
+ const turnSenderRef = useRef(null);
1922
+ const dataPublisherRef = useRef(null);
1923
+ const lastTurnRef = useRef(null);
1924
+ const [clockTick, setClockTick] = useState(0);
1925
+ const endsAt = endsAtFrom({
1926
+ serverStartedAtUnixMs: serverClock?.startedAtUnixMs ?? null,
1927
+ maxSessionSeconds: serverClock?.maxSessionSeconds ?? null
1928
+ });
1929
+ const sessionRemainingMs = sessionRemainingMsFrom({
1930
+ serverStartedAtUnixMs: serverClock?.startedAtUnixMs ?? null,
1931
+ maxSessionSeconds: serverClock?.maxSessionSeconds ?? null,
1932
+ nowUnixMs: Date.now()
1933
+ });
1934
+ const clocks = {
1935
+ sessionRemainingMs,
1936
+ idleRemainingMs: lifecycle.timeToDisconnectMs,
1937
+ creditRemainingMs
1938
+ };
1939
+ const phaseKind = lifecycle.phase.kind;
1940
+ const stayConnected = lifecycle.stayConnected;
1941
+ const newTurnId = useCallback(() => {
1942
+ try {
1943
+ if (typeof crypto !== "undefined" && typeof crypto.randomUUID === "function") return crypto.randomUUID();
1944
+ } catch {
1945
+ }
1946
+ return `t-${Date.now()}-${clockTick}`;
1947
+ }, [clockTick]);
1948
+ const onLifecycleData = useCallback((frame) => {
1949
+ const parsed = lifecycleServerFrameSchema.safeParse(frame);
1950
+ if (!parsed.success) return;
1951
+ const f = parsed.data;
1952
+ switch (f.kind) {
1953
+ case "session_clock":
1954
+ setServerClock({
1955
+ startedAtUnixMs: f.started_at_unix_ms,
1956
+ maxSessionSeconds: f.max_session_seconds,
1957
+ idleTimeoutSeconds: f.idle_timeout_seconds
1958
+ });
1959
+ break;
1960
+ case "ending":
1961
+ setWorkerEnding(true);
1962
+ break;
1963
+ case "closing_turn_done":
1964
+ setGraceWindow((prev) => {
1965
+ if (prev.kind === "delivering" && prev.turnId === f.turn_id) {
1966
+ cbRef.current.onGraceWindowClosed?.({ reason: "session_cap", delivered: true });
1967
+ return { kind: "spent", delivered: true };
1968
+ }
1969
+ return prev;
1970
+ });
1971
+ break;
1972
+ case "ended":
1973
+ lastLabeledEndReasonRef.current = f.reason;
1974
+ break;
1975
+ case "behavior_state": {
1976
+ const next = nextBehaviorSnapshot(lastBehaviorRef.current, f);
1977
+ if (next) {
1978
+ lastBehaviorRef.current = next;
1979
+ setBehavior(next);
1980
+ cbRef.current.onBehaviorChange?.(next);
1981
+ }
1982
+ break;
1983
+ }
1984
+ case "clip_ack": {
1985
+ const result = { requestId: f.request_id, accepted: f.accepted, reason: f.reason };
1986
+ const pending = pendingClipsRef.current.get(f.request_id);
1987
+ if (pending) {
1988
+ pendingClipsRef.current.delete(f.request_id);
1989
+ clearTimeout(pending.timer);
1990
+ pending.resolve(result);
1991
+ }
1992
+ cbRef.current.onClipResult?.(result);
1993
+ break;
1994
+ }
1995
+ }
1996
+ }, []);
1997
+ const registerDataPublisher = useCallback((publish) => {
1998
+ dataPublisherRef.current = publish;
1999
+ }, []);
2000
+ const registerTurnSender = useCallback((send) => {
2001
+ turnSenderRef.current = send;
2002
+ }, []);
2003
+ const setTurnState = useCallback((state) => {
2004
+ setTurn(mapTurnState(state));
2005
+ }, []);
2006
+ const setMedia = useCallback((next) => {
2007
+ setMediaState((prev) => prev.video === next.video && prev.audio === next.audio ? prev : next);
2008
+ }, []);
2009
+ useEffect(() => {
2010
+ if (phaseKind === "idle" || phaseKind === "ended") return;
2011
+ const timer = window.setInterval(() => {
2012
+ const now = Date.now();
2013
+ const remaining = sessionRemainingMsFrom({
2014
+ serverStartedAtUnixMs: serverClock?.startedAtUnixMs ?? null,
2015
+ maxSessionSeconds: serverClock?.maxSessionSeconds ?? null,
2016
+ nowUnixMs: now
2017
+ });
2018
+ const ends = endsAtFrom({
2019
+ serverStartedAtUnixMs: serverClock?.startedAtUnixMs ?? null,
2020
+ maxSessionSeconds: serverClock?.maxSessionSeconds ?? null
2021
+ });
2022
+ const approaching = approachingEndFrom({ sessionRemainingMs: remaining, approachingEndLeadMs, alreadyFired: approachingFiredRef.current });
2023
+ if (approaching) {
2024
+ approachingFiredRef.current = true;
2025
+ cbRef.current.onApproachingEnd?.({ secondsLeft: approaching.secondsLeft, reason: "session_cap", threshold: approachingEndLeadMs / 1e3 });
2026
+ }
2027
+ const low = creditsLowFrom({ creditRemainingMs, creditsLowLeadMs, alreadyFired: creditsLowFiredRef.current });
2028
+ if (low) {
2029
+ creditsLowFiredRef.current = true;
2030
+ cbRef.current.onCreditsLow?.({ secondsLeft: low.secondsLeft });
2031
+ }
2032
+ setGraceWindow((prev) => {
2033
+ const next = nextGraceWindow({ prev, sessionRemainingMs: remaining, graceWindowLeadMs, endsAt: ends, workerEnding, nowMs: now });
2034
+ if (prev.kind !== "open" && next.kind === "open") {
2035
+ cbRef.current.onGraceWindowOpen?.({ reason: next.reason, deadlineAt: next.deadlineAt, msLeft: next.msLeft });
2036
+ }
2037
+ if (prev.kind !== "spent" && next.kind === "spent" && !next.delivered) {
2038
+ cbRef.current.onGraceWindowClosed?.({ reason: "session_cap", delivered: false });
2039
+ }
2040
+ return next;
2041
+ });
2042
+ if (autoStayConnectedDuringGrace) {
2043
+ const gw = graceWindowRef.current.kind;
2044
+ if (gw === "open" || gw === "delivering") stayConnected();
2045
+ }
2046
+ const turnInfo = lastTurnRef.current;
2047
+ if (turnInfo && now - turnInfo.sentAt >= turnTimeoutMs) {
2048
+ lastTurnRef.current = null;
2049
+ cbRef.current.onTurnTimeout?.({ turnId: turnInfo.id });
2050
+ }
2051
+ setClockTick((value) => value + 1);
2052
+ }, 1e3);
2053
+ return () => window.clearInterval(timer);
2054
+ }, [phaseKind, serverClock, workerEnding, approachingEndLeadMs, graceWindowLeadMs, creditRemainingMs, creditsLowLeadMs, turnTimeoutMs, autoStayConnectedDuringGrace, stayConnected]);
2055
+ useEffect(() => {
2056
+ if (lifecycle.phase.kind === "idle-warning") {
2057
+ if (!idleWarnFiredRef.current) {
2058
+ idleWarnFiredRef.current = true;
2059
+ cbRef.current.onIdleWarning?.({ secondsLeft: lifecycle.phase.secondsRemaining });
2060
+ }
2061
+ } else {
2062
+ idleWarnFiredRef.current = false;
2063
+ }
2064
+ }, [lifecycle.phase]);
2065
+ useEffect(() => {
2066
+ const prev = prevPhaseKindRef.current;
2067
+ if (phaseKind === "reconnectable" && prev !== "reconnectable") {
2068
+ const attempt = lifecycle.phase.kind === "reconnectable" ? lifecycle.phase.attempt : lifecycle.attempt;
2069
+ cbRef.current.onReconnecting?.({ attempt });
2070
+ } else if (prev === "reconnectable" && (phaseKind === "live" || phaseKind === "connecting")) {
2071
+ cbRef.current.onReconnected?.();
2072
+ }
2073
+ prevPhaseKindRef.current = phaseKind;
2074
+ }, [phaseKind, lifecycle.phase, lifecycle.attempt]);
2075
+ useEffect(() => {
2076
+ if (phaseKind === "ended") {
2077
+ if (!endedFiredRef.current) {
2078
+ endedFiredRef.current = true;
2079
+ const inner2 = lifecycle.phase.kind === "ended" ? lifecycle.phase.reason : void 0;
2080
+ cbRef.current.onEnded?.({ reason: resolveEndReason(lastLabeledEndReasonRef.current, inner2) });
2081
+ }
2082
+ } else {
2083
+ endedFiredRef.current = false;
2084
+ }
2085
+ }, [phaseKind, lifecycle.phase]);
2086
+ const sendClosingTurn = useCallback((text, opts) => {
2087
+ const trimmed = text.trim();
2088
+ const gw = graceWindowRef.current;
2089
+ if (gw.kind === "spent") return { ok: false, reason: "already_spent" };
2090
+ if (gw.kind !== "open") return { ok: false, reason: "window_closed" };
2091
+ const sender = turnSenderRef.current;
2092
+ if (!sender || !trimmed) return { ok: false, reason: "not_connected" };
2093
+ const turnId = newTurnId();
2094
+ const attributes = { [RTA_CLOSING_TURN_ATTR]: "1", [RTA_TURN_ID_ATTR]: turnId };
2095
+ if (opts?.instructions) attributes[RTA_TURN_INSTRUCTIONS_ATTR] = opts.instructions;
2096
+ void sender(trimmed, { attributes }).catch(() => void 0);
2097
+ const deadlineAt = (endsAt ?? Date.now()) + graceCeilingMs;
2098
+ setGraceWindow({ kind: "delivering", turnId, deadlineAt });
2099
+ return { ok: true, turnId };
2100
+ }, [endsAt, graceCeilingMs, newTurnId]);
2101
+ const requestGracefulClose = useCallback(() => {
2102
+ dataPublisherRef.current?.({ kind: "request_graceful_close" });
2103
+ }, []);
2104
+ const extend = useCallback((req) => {
2105
+ const publish = dataPublisherRef.current;
2106
+ if (!publish || !(req.addSeconds > 0)) return { ok: false };
2107
+ publish({ kind: "extend", add_seconds: Math.floor(req.addSeconds), ...req.proof ? { proof: req.proof } : {} });
2108
+ approachingFiredRef.current = false;
2109
+ setWorkerEnding(false);
2110
+ setGraceWindow({ kind: "closed" });
2111
+ return { ok: true };
2112
+ }, []);
2113
+ const sendTurn = useCallback(async (text, opts) => {
2114
+ const trimmed = text.trim();
2115
+ const sender = turnSenderRef.current;
2116
+ if (!sender || !trimmed) return;
2117
+ const turnId = newTurnId();
2118
+ lastTurnRef.current = { text: trimmed, opts, id: turnId, sentAt: Date.now() };
2119
+ lifecycle.markActivity();
2120
+ const attributes = opts?.instructions ? { [RTA_TURN_INSTRUCTIONS_ATTR]: opts.instructions } : void 0;
2121
+ await sender(trimmed, attributes ? { attributes } : void 0);
2122
+ }, [lifecycle, newTurnId]);
2123
+ const retryTurn = useCallback(() => {
2124
+ const last = lastTurnRef.current;
2125
+ if (!last) return;
2126
+ void sendTurn(last.text, last.opts);
2127
+ }, [sendTurn]);
2128
+ const end = useCallback((reason) => {
2129
+ if (reason) lastLabeledEndReasonRef.current = reason;
2130
+ requestGracefulClose();
2131
+ lifecycle.reset();
2132
+ }, [lifecycle, requestGracefulClose]);
2133
+ const performClip = useCallback(
2134
+ (clipId, opts) => {
2135
+ const requestId = newTurnId();
2136
+ const publish = dataPublisherRef.current;
2137
+ const trimmed = clipId.trim();
2138
+ if (!publish || !trimmed) {
2139
+ const result = { requestId, accepted: false, reason: "not_connected" };
2140
+ cbRef.current.onClipResult?.(result);
2141
+ return Promise.resolve(result);
2142
+ }
2143
+ return new Promise((resolve) => {
2144
+ const timer = setTimeout(() => {
2145
+ pendingClipsRef.current.delete(requestId);
2146
+ const result = { requestId, accepted: false, reason: "timeout" };
2147
+ cbRef.current.onClipResult?.(result);
2148
+ resolve(result);
2149
+ }, opts?.timeoutMs ?? 5e3);
2150
+ pendingClipsRef.current.set(requestId, { resolve, timer });
2151
+ publish({
2152
+ kind: "clip_request",
2153
+ request_id: requestId,
2154
+ clip_id: trimmed,
2155
+ ...typeof opts?.holdSeconds === "number" ? { hold_seconds: opts.holdSeconds } : {}
2156
+ });
2157
+ });
2158
+ },
2159
+ [newTurnId]
2160
+ );
2161
+ useEffect(
2162
+ () => () => {
2163
+ for (const [requestId, pending] of pendingClipsRef.current) {
2164
+ clearTimeout(pending.timer);
2165
+ pending.resolve({ requestId, accepted: false, reason: "unmounted" });
2166
+ }
2167
+ pendingClipsRef.current.clear();
2168
+ },
2169
+ []
2170
+ );
2171
+ const reset = useCallback(() => {
2172
+ setServerClock(null);
2173
+ setWorkerEnding(false);
2174
+ setGraceWindow({ kind: "closed" });
2175
+ lastLabeledEndReasonRef.current = null;
2176
+ approachingFiredRef.current = false;
2177
+ creditsLowFiredRef.current = false;
2178
+ lastTurnRef.current = null;
2179
+ lifecycle.reset();
2180
+ }, [lifecycle]);
2181
+ return useMemo(
2182
+ () => ({
2183
+ // Spread the inner SSOT verbatim (phase/grant/capacity/attempt/timeToDisconnectMs/
2184
+ // stayConnected/reconnect/markActivity/the sinks) → a true drop-in superset, then
2185
+ // add the realtime-session surface. `reset` + `lifecycle` come AFTER so ours win.
2186
+ ...lifecycle,
2187
+ turn,
2188
+ clocks,
2189
+ endsAt,
2190
+ graceWindow,
2191
+ media,
2192
+ lifecycle,
2193
+ sendClosingTurn,
2194
+ requestGracefulClose,
2195
+ extend,
2196
+ sendTurn,
2197
+ retryTurn,
2198
+ end,
2199
+ behavior,
2200
+ performClip,
2201
+ onLifecycleData,
2202
+ registerDataPublisher,
2203
+ registerTurnSender,
2204
+ setTurnState,
2205
+ setMedia,
2206
+ reset
2207
+ }),
2208
+ [
2209
+ lifecycle,
2210
+ turn,
2211
+ clocks,
2212
+ endsAt,
2213
+ graceWindow,
2214
+ media,
2215
+ sendClosingTurn,
2216
+ requestGracefulClose,
2217
+ extend,
2218
+ sendTurn,
2219
+ retryTurn,
2220
+ end,
2221
+ behavior,
2222
+ performClip,
2223
+ onLifecycleData,
2224
+ registerDataPublisher,
2225
+ registerTurnSender,
2226
+ setTurnState,
2227
+ setMedia,
2228
+ reset
2229
+ ]
2230
+ );
2231
+ }
2232
+
2233
+ // ../client/src/proxy-client.ts
2234
+ var DEFAULT_TIMEOUT_MS = 6e4;
2235
+ var normalize = (url) => url.replace(/\/+$/, "");
2236
+ function createProxyClient(options) {
2237
+ const base = normalize(options.proxyUrl);
2238
+ const doFetch = options.fetch ?? globalThis.fetch?.bind(globalThis);
2239
+ const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
2240
+ const deadline = (caller) => {
2241
+ if (!timeoutMs) return caller;
2242
+ const timer = AbortSignal.timeout(timeoutMs);
2243
+ return caller ? AbortSignal.any([caller, timer]) : timer;
2244
+ };
2245
+ const post = async (path, body, request) => {
2246
+ if (!doFetch) throw new Error("realtime-avatar: no fetch available \u2014 pass one via `fetch`.");
2247
+ return doFetch(`${base}${path}`, {
2248
+ method: "POST",
2249
+ headers: { "content-type": "application/json", ...request?.headers ?? {} },
2250
+ body: JSON.stringify(body),
2251
+ signal: deadline(request?.signal)
2252
+ });
2253
+ };
2254
+ const beacon = (path, body) => {
2255
+ const send = globalThis.navigator?.sendBeacon?.bind(globalThis.navigator);
2256
+ if (!send) return false;
2257
+ return send(`${base}${path}`, new Blob([JSON.stringify(body)], { type: "application/json" }));
2258
+ };
2259
+ return {
2260
+ async createLiveKitSessionOrBusy(input, requestOptions) {
2261
+ const response = await post(
2262
+ "/connect",
2263
+ { avatarId: input.avatarId, mode: input.mode },
2264
+ requestOptions
2265
+ );
2266
+ if (response.status === 429) {
2267
+ const busy = await response.json().catch(() => ({}));
2268
+ return { status: "busy", busy };
2269
+ }
2270
+ if (!response.ok) {
2271
+ const detail = await response.text().catch(() => "");
2272
+ throw new Error(`realtime-avatar: proxy refused the call (${response.status}) ${detail}`.trim());
2273
+ }
2274
+ return { status: "ready", grant: await response.json() };
2275
+ },
2276
+ async releaseLiveKitSession(sessionId, reason, requestOptions) {
2277
+ if (!sessionId) return false;
2278
+ try {
2279
+ const response = await post("/end", { session_id: sessionId, reason }, requestOptions);
2280
+ return response.ok;
2281
+ } catch {
2282
+ return false;
2283
+ }
2284
+ },
2285
+ releaseLiveKitSessionBeacon(sessionId, reason) {
2286
+ if (!sessionId) return false;
2287
+ return beacon("/end", { session_id: sessionId, reason: reason ?? "page_hide" });
2288
+ },
2289
+ async releaseLiveKitQueueTicket(queueTicketId, reason, requestOptions) {
2290
+ if (!queueTicketId) return false;
2291
+ try {
2292
+ const response = await post("/end", { queue_ticket_id: queueTicketId, reason }, requestOptions);
2293
+ return response.ok;
2294
+ } catch {
2295
+ return false;
2296
+ }
2297
+ },
2298
+ releaseLiveKitQueueTicketBeacon(queueTicketId, reason) {
2299
+ if (!queueTicketId) return false;
2300
+ return beacon("/end", { queue_ticket_id: queueTicketId, reason: reason ?? "page_hide" });
2301
+ }
2302
+ };
2303
+ }
2304
+
2305
+ export { AvatarVideoSurface, DEFAULT_APPROACHING_END_LEAD_SECONDS, DEFAULT_AVATAR_PLAYOUT_DELAY_SECONDS, DEFAULT_CREDITS_LOW_LEAD_SECONDS, DEFAULT_GOVERNOR_CONFIG, DEFAULT_GRACE_CEILING_SECONDS, DEFAULT_GRACE_WINDOW_LEAD_SECONDS, DEFAULT_IDLE_SECONDS, DEFAULT_IDLE_WARN_LEAD_SECONDS, DEFAULT_TURN_TIMEOUT_SECONDS, MAX_RECONNECT_ATTEMPTS, MAX_SESSION_INSTRUCTIONS_CHARS, RECONNECT_BACKOFF_MS, RealtimeAvatarLiveKitRoom, SessionLifecycleRoomBridge, capacityErrorFromBusy, capacityStateFromGrant, createProxyClient, isNativeLiveTrackSubscribed, knownBehaviorStates, mapTurnState, sessionBehaviorSchema, sessionClipSchema, splitCallTranscript, useAvatarPlayoutDelay, useAvatarQualityGovernor, useCallTranscript, useLiveKitAvatarGrant, useLiveTrackProducing, useMicLease, useRealtimeAvatarAudioSession, useRealtimeSession, useReleaseMicLeaseOnTrackEnded, useSessionLifecycle };