realtime-avatar 0.25.0 → 0.26.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +78 -0
- package/dist/browser.d.ts +2 -87
- package/dist/browser.js +6 -1
- package/dist/{chunk-DNHHQ5GZ.js → chunk-MSPXJU5P.js} +28 -10
- package/dist/{chunk-GMYGHDY2.js → chunk-V3PQ7RNX.js} +22 -4
- package/dist/express.js +2 -2
- package/dist/hono.js +2 -2
- package/dist/index.d.ts +12 -0
- package/dist/index.js +1 -1
- package/dist/nextjs.js +2 -2
- package/dist/playout-delay-CtRZdtsj.d.ts +168 -0
- package/dist/{proxy-client-O-Mu9JKt.d.ts → proxy-client-C8M_G_Kk.d.ts} +163 -17
- package/dist/react-native.d.ts +3 -3
- package/dist/react-native.js +341 -26
- package/dist/react.d.ts +75 -14
- package/dist/react.js +532 -83
- package/dist/server.js +1 -1
- package/dist/tanstack-start.js +2 -2
- package/dist/tools.d.ts +17 -1
- package/dist/tools.js +28 -8
- package/package.json +1 -1
- package/dist/playout-delay-DWOc8iCl.d.ts +0 -65
package/dist/react-native.js
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { AudioSession, LiveKitRoom, VideoTrack } from '@livekit/react-native';
|
|
2
2
|
export { AudioSession, VideoTrack, registerGlobals } from '@livekit/react-native';
|
|
3
3
|
import { useState, useRef, useEffect, useCallback, createElement, useMemo } from 'react';
|
|
4
|
-
import { useLocalParticipant, useConnectionState, useTranscriptions, useMaybeRoomContext, useVoiceAssistant, isTrackReference, useRoomContext, useChat } from '@livekit/components-react';
|
|
4
|
+
import { useLocalParticipant, useConnectionState, useTranscriptions, useMaybeRoomContext, useVoiceAssistant, isTrackReference, useRoomContext, useChat, useAudioPlayback } from '@livekit/components-react';
|
|
5
5
|
export { useChat, useConnectionState, useLocalParticipant, useRoomContext, useTrackToggle, useTranscriptions, useVoiceAssistant } from '@livekit/components-react';
|
|
6
|
-
import { ConnectionState, isRemoteTrack, VideoQuality, ConnectionQuality, RoomEvent, TrackEvent, Track, ConnectionError, ConnectionErrorReason, DisconnectReason, createLocalVideoTrack, Room } from 'livekit-client';
|
|
6
|
+
import { ConnectionState, isRemoteTrack, VideoQuality, ConnectionQuality, RoomEvent, TrackEvent, Track, ConnectionError, ConnectionErrorReason, ParticipantEvent, DisconnectReason, createLocalVideoTrack, Room } from 'livekit-client';
|
|
7
7
|
export { DisconnectReason, Room, RoomEvent, Track } from 'livekit-client';
|
|
8
8
|
import { Platform, StyleSheet, Animated, Image, View, Text } from 'react-native';
|
|
9
9
|
import { z } from 'zod';
|
|
@@ -698,6 +698,11 @@ function useLiveKitAvatarGrant(input) {
|
|
|
698
698
|
});
|
|
699
699
|
const stateRef = useRef(state);
|
|
700
700
|
stateRef.current = state;
|
|
701
|
+
const wantedRef = useRef(active);
|
|
702
|
+
wantedRef.current = active;
|
|
703
|
+
useEffect(() => () => {
|
|
704
|
+
wantedRef.current = false;
|
|
705
|
+
}, []);
|
|
701
706
|
const [tabVisible, setTabVisible] = useState(
|
|
702
707
|
() => typeof document === "undefined" || document.visibilityState === "visible"
|
|
703
708
|
);
|
|
@@ -738,6 +743,8 @@ function useLiveKitAvatarGrant(input) {
|
|
|
738
743
|
if (cancelled) {
|
|
739
744
|
if (result.status !== "busy") {
|
|
740
745
|
void client.releaseLiveKitSession(result.grant.session_id, "superseded");
|
|
746
|
+
} else if (!wantedRef.current && result.busy.queue_ticket_id) {
|
|
747
|
+
void client.releaseLiveKitQueueTicket(result.busy.queue_ticket_id, "unmount");
|
|
741
748
|
}
|
|
742
749
|
return;
|
|
743
750
|
}
|
|
@@ -821,6 +828,9 @@ function sameCapacityState(a, b) {
|
|
|
821
828
|
return true;
|
|
822
829
|
}
|
|
823
830
|
}
|
|
831
|
+
function publishesMicrophoneByDefault(grant) {
|
|
832
|
+
return grant?.stt_mode === "server";
|
|
833
|
+
}
|
|
824
834
|
var capacityErrorFromBusy = (busy) => {
|
|
825
835
|
return new RealtimeAvatarCapacityError(busy.message, busy);
|
|
826
836
|
};
|
|
@@ -2146,6 +2156,169 @@ function createConnectionHistoryCollector(input) {
|
|
|
2146
2156
|
return { enqueue, flush, dispose };
|
|
2147
2157
|
}
|
|
2148
2158
|
|
|
2159
|
+
// ../browser/src/microphone.ts
|
|
2160
|
+
var HINTS = {
|
|
2161
|
+
"insecure-origin": "The browser only exposes microphones on a secure origin. Serve the page over https, or open it on http://localhost \u2014 a LAN address like http://192.168.1.5 will not do.",
|
|
2162
|
+
"no-answer": "The permission prompt is probably still open, or was dismissed without an answer. Look for it in the address bar, answer it, and try again.",
|
|
2163
|
+
"denied-by-browser": "This site is blocked from using the microphone. Click the icon at the left of the address bar, set Microphone to Allow, and reload.",
|
|
2164
|
+
"denied-by-os": "The operating system is blocking the browser itself, so no site can record. On macOS: System Settings > Privacy & Security > Microphone, enable your browser, then RESTART it \u2014 the change does not apply to a running browser.",
|
|
2165
|
+
"no-device": "No microphone is available. Connect one, and if you are on Bluetooth headphones check they are in a mode that has a microphone.",
|
|
2166
|
+
"device-in-use": "Another application or browser tab is holding the microphone. Close it \u2014 including any call from this page you did not hang up \u2014 and try again.",
|
|
2167
|
+
"device-lost": "The microphone stopped during the call: it was unplugged, or access to it was turned off. Reconnect it or allow access again, then turn the microphone back on.",
|
|
2168
|
+
unknown: "Try again, and if it persists include the message above in your report."
|
|
2169
|
+
};
|
|
2170
|
+
function isOperatingSystemDenial(message) {
|
|
2171
|
+
return /\bby system\b|system permission/i.test(message);
|
|
2172
|
+
}
|
|
2173
|
+
function classify(error) {
|
|
2174
|
+
if (error instanceof TypeError && /mediaDevices|undefined/i.test(error.message)) {
|
|
2175
|
+
return { reason: "insecure-origin", message: error.message };
|
|
2176
|
+
}
|
|
2177
|
+
const name = typeof error === "object" && error !== null && "name" in error ? String(error.name) : "";
|
|
2178
|
+
const message = typeof error === "object" && error !== null && "message" in error && error.message ? String(error.message) : String(error);
|
|
2179
|
+
switch (name) {
|
|
2180
|
+
case "NotAllowedError":
|
|
2181
|
+
case "PermissionDeniedError":
|
|
2182
|
+
return {
|
|
2183
|
+
reason: isOperatingSystemDenial(message) ? "denied-by-os" : "denied-by-browser",
|
|
2184
|
+
message
|
|
2185
|
+
};
|
|
2186
|
+
case "NotFoundError":
|
|
2187
|
+
case "DevicesNotFoundError":
|
|
2188
|
+
case "OverconstrainedError":
|
|
2189
|
+
case "ConstraintNotSatisfiedError":
|
|
2190
|
+
return { reason: "no-device", message };
|
|
2191
|
+
case "NotReadableError":
|
|
2192
|
+
case "TrackStartError":
|
|
2193
|
+
case "AbortError":
|
|
2194
|
+
return { reason: "device-in-use", message };
|
|
2195
|
+
case "SecurityError":
|
|
2196
|
+
return { reason: "insecure-origin", message };
|
|
2197
|
+
default:
|
|
2198
|
+
return { reason: "unknown", message };
|
|
2199
|
+
}
|
|
2200
|
+
}
|
|
2201
|
+
function describeMicrophoneFailure(error) {
|
|
2202
|
+
const { reason, message } = classify(error);
|
|
2203
|
+
return { reason, message, hint: HINTS[reason] };
|
|
2204
|
+
}
|
|
2205
|
+
function microphoneLost(message) {
|
|
2206
|
+
return { reason: "device-lost", message, hint: HINTS["device-lost"] };
|
|
2207
|
+
}
|
|
2208
|
+
|
|
2209
|
+
// ../client/src/react/call-media.ts
|
|
2210
|
+
var DEVICE_LOST_MESSAGE = "The microphone track ended during the call.";
|
|
2211
|
+
function callMicrophoneFrom(facts, intent) {
|
|
2212
|
+
if (!facts.wanted) return { status: "off" };
|
|
2213
|
+
const { publication } = facts;
|
|
2214
|
+
if (publication) {
|
|
2215
|
+
if (publication.ended) {
|
|
2216
|
+
return publication.muted ? { status: "unavailable", ...microphoneLost(DEVICE_LOST_MESSAGE) } : { status: "pending" };
|
|
2217
|
+
}
|
|
2218
|
+
return publication.muted ? { status: "muted" } : { status: "on" };
|
|
2219
|
+
}
|
|
2220
|
+
if (intent.muted) return { status: "muted" };
|
|
2221
|
+
if (facts.enabling) return { status: "pending" };
|
|
2222
|
+
const error = facts.deviceError;
|
|
2223
|
+
if (error) {
|
|
2224
|
+
const problem = describeMicrophoneFailure(error);
|
|
2225
|
+
const blocked = problem.reason === "denied-by-browser" || problem.reason === "denied-by-os" || problem.reason === "insecure-origin";
|
|
2226
|
+
return blocked ? { status: "blocked", ...problem } : { status: "unavailable", ...problem };
|
|
2227
|
+
}
|
|
2228
|
+
return facts.connected ? { status: "pending" } : { status: "off" };
|
|
2229
|
+
}
|
|
2230
|
+
function microphoneFacts(room, wanted, connected, enabling, staleError) {
|
|
2231
|
+
const publication = room.localParticipant.getTrackPublication(Track.Source.Microphone);
|
|
2232
|
+
const track = publication?.track;
|
|
2233
|
+
return {
|
|
2234
|
+
wanted,
|
|
2235
|
+
connected,
|
|
2236
|
+
enabling,
|
|
2237
|
+
publication: track ? { muted: publication.isMuted, ended: track.mediaStreamTrack?.readyState === "ended" } : null,
|
|
2238
|
+
deviceError: room.localParticipant.lastMicrophoneError !== staleError && room.localParticipant.lastMicrophoneError || null
|
|
2239
|
+
};
|
|
2240
|
+
}
|
|
2241
|
+
function useCallMedia(room, options) {
|
|
2242
|
+
const { setMicrophoneFacts, setAudioPlayback, registerMediaControls, microphoneWanted, connectionState, callKey } = options;
|
|
2243
|
+
const connected = connectionState === ConnectionState.Connected;
|
|
2244
|
+
const scope = useMemo(
|
|
2245
|
+
() => ({ room, callKey, stale: room.localParticipant.lastMicrophoneError }),
|
|
2246
|
+
[room, callKey]
|
|
2247
|
+
);
|
|
2248
|
+
const [enabling, setEnabling] = useState(false);
|
|
2249
|
+
const [version, setVersion] = useState(0);
|
|
2250
|
+
const sinksRef = useRef({ setMicrophoneFacts, setAudioPlayback });
|
|
2251
|
+
sinksRef.current = { setMicrophoneFacts, setAudioPlayback };
|
|
2252
|
+
useEffect(() => {
|
|
2253
|
+
if (!setMicrophoneFacts) return;
|
|
2254
|
+
const bump = () => setVersion((value) => value + 1);
|
|
2255
|
+
const participant = room.localParticipant;
|
|
2256
|
+
let watched = null;
|
|
2257
|
+
const watchTrack = () => {
|
|
2258
|
+
watched?.off();
|
|
2259
|
+
const track = participant.getTrackPublication(Track.Source.Microphone)?.track;
|
|
2260
|
+
if (!track) {
|
|
2261
|
+
watched = null;
|
|
2262
|
+
return;
|
|
2263
|
+
}
|
|
2264
|
+
track.on(TrackEvent.Ended, bump).on(TrackEvent.Restarted, bump).on(TrackEvent.Muted, bump).on(TrackEvent.Unmuted, bump);
|
|
2265
|
+
watched = { off: () => {
|
|
2266
|
+
track.off(TrackEvent.Ended, bump).off(TrackEvent.Restarted, bump).off(TrackEvent.Muted, bump).off(TrackEvent.Unmuted, bump);
|
|
2267
|
+
} };
|
|
2268
|
+
};
|
|
2269
|
+
const onPublications = () => {
|
|
2270
|
+
watchTrack();
|
|
2271
|
+
bump();
|
|
2272
|
+
};
|
|
2273
|
+
room.on(RoomEvent.MediaDevicesError, bump).on(RoomEvent.LocalTrackPublished, onPublications).on(RoomEvent.LocalTrackUnpublished, onPublications);
|
|
2274
|
+
participant.on(ParticipantEvent.TrackMuted, bump).on(ParticipantEvent.TrackUnmuted, bump);
|
|
2275
|
+
watchTrack();
|
|
2276
|
+
return () => {
|
|
2277
|
+
watched?.off();
|
|
2278
|
+
room.off(RoomEvent.MediaDevicesError, bump).off(RoomEvent.LocalTrackPublished, onPublications).off(RoomEvent.LocalTrackUnpublished, onPublications);
|
|
2279
|
+
participant.off(ParticipantEvent.TrackMuted, bump).off(ParticipantEvent.TrackUnmuted, bump);
|
|
2280
|
+
};
|
|
2281
|
+
}, [room, setMicrophoneFacts]);
|
|
2282
|
+
useEffect(() => {
|
|
2283
|
+
sinksRef.current.setMicrophoneFacts?.(microphoneFacts(room, microphoneWanted, connected, enabling, scope.stale));
|
|
2284
|
+
}, [room, microphoneWanted, connected, enabling, version, scope]);
|
|
2285
|
+
const { canPlayAudio } = useAudioPlayback(room);
|
|
2286
|
+
useEffect(() => {
|
|
2287
|
+
sinksRef.current.setAudioPlayback?.(!canPlayAudio ? "blocked" : connected ? "allowed" : "unknown");
|
|
2288
|
+
}, [canPlayAudio, connected]);
|
|
2289
|
+
const unlockTriedRef = useRef(null);
|
|
2290
|
+
const connecting = connectionState === ConnectionState.Connecting || connected;
|
|
2291
|
+
useEffect(() => {
|
|
2292
|
+
if (!setAudioPlayback || !connecting || unlockTriedRef.current === room) return;
|
|
2293
|
+
unlockTriedRef.current = room;
|
|
2294
|
+
void room.startAudio().catch(() => void 0);
|
|
2295
|
+
}, [room, setAudioPlayback, connecting]);
|
|
2296
|
+
const controls = useMemo(() => ({
|
|
2297
|
+
startAudio: async () => {
|
|
2298
|
+
try {
|
|
2299
|
+
await room.startAudio();
|
|
2300
|
+
} catch {
|
|
2301
|
+
}
|
|
2302
|
+
return room.canPlaybackAudio;
|
|
2303
|
+
},
|
|
2304
|
+
setMicrophoneEnabled: async (enabled) => {
|
|
2305
|
+
if (enabled) setEnabling(true);
|
|
2306
|
+
try {
|
|
2307
|
+
await room.localParticipant.setMicrophoneEnabled(enabled);
|
|
2308
|
+
} catch {
|
|
2309
|
+
} finally {
|
|
2310
|
+
if (enabled) setEnabling(false);
|
|
2311
|
+
setVersion((value) => value + 1);
|
|
2312
|
+
}
|
|
2313
|
+
}
|
|
2314
|
+
}), [room]);
|
|
2315
|
+
useEffect(() => {
|
|
2316
|
+
if (!registerMediaControls) return;
|
|
2317
|
+
registerMediaControls(controls);
|
|
2318
|
+
return () => registerMediaControls(null);
|
|
2319
|
+
}, [controls, registerMediaControls]);
|
|
2320
|
+
}
|
|
2321
|
+
|
|
2149
2322
|
// ../client/src/react/session-lifecycle.ts
|
|
2150
2323
|
function phaseForReason(reason) {
|
|
2151
2324
|
switch (reason) {
|
|
@@ -2306,10 +2479,13 @@ function useSessionLifecycle(input) {
|
|
|
2306
2479
|
const reconnectPolicyRef = useRef(reconnectPolicy);
|
|
2307
2480
|
reconnectPolicyRef.current = reconnectPolicy;
|
|
2308
2481
|
const connectWatchdogMs = typeof connectWatchdogSeconds === "number" && Number.isFinite(connectWatchdogSeconds) ? Math.max(0, Math.floor(connectWatchdogSeconds)) * 1e3 : DEFAULT_CONNECT_WATCHDOG_SECONDS * 1e3;
|
|
2482
|
+
const [userEnded, setUserEnded] = useState(false);
|
|
2483
|
+
const userEndedRef = useRef(false);
|
|
2484
|
+
const phaseRef = useRef({ kind: "idle" });
|
|
2309
2485
|
const grantState = useLiveKitAvatarGrant({
|
|
2310
2486
|
client,
|
|
2311
2487
|
session,
|
|
2312
|
-
active,
|
|
2488
|
+
active: active && !userEnded,
|
|
2313
2489
|
autoRetryBusy,
|
|
2314
2490
|
requestOptions
|
|
2315
2491
|
});
|
|
@@ -2326,6 +2502,8 @@ function useSessionLifecycle(input) {
|
|
|
2326
2502
|
refreshRef.current = grantState.refresh;
|
|
2327
2503
|
const releaseRef = useRef(grantState.release);
|
|
2328
2504
|
releaseRef.current = grantState.release;
|
|
2505
|
+
const clearRef = useRef(grantState.clear);
|
|
2506
|
+
clearRef.current = grantState.clear;
|
|
2329
2507
|
const capacityRef = useRef(capacity);
|
|
2330
2508
|
capacityRef.current = capacity;
|
|
2331
2509
|
const refreshBaselineCapacityRef = useRef(null);
|
|
@@ -2352,13 +2530,16 @@ function useSessionLifecycle(input) {
|
|
|
2352
2530
|
refreshBaselineCapacityRef.current = null;
|
|
2353
2531
|
attemptRef.current = 0;
|
|
2354
2532
|
setAttempt(0);
|
|
2355
|
-
setRecovery({ kind: "connected" });
|
|
2533
|
+
setRecovery(userEndedRef.current ? { kind: "ended", reason: "user" } : { kind: "connected" });
|
|
2356
2534
|
setConnected(false);
|
|
2357
2535
|
setAgentPresentState(false);
|
|
2358
2536
|
lastActivityRef.current = Date.now();
|
|
2359
2537
|
}, [clearTimer]);
|
|
2360
2538
|
useEffect(() => {
|
|
2361
|
-
if (
|
|
2539
|
+
if (active) return;
|
|
2540
|
+
userEndedRef.current = false;
|
|
2541
|
+
setUserEnded(false);
|
|
2542
|
+
reset();
|
|
2362
2543
|
}, [active, reset]);
|
|
2363
2544
|
const markActivity = useCallback(() => {
|
|
2364
2545
|
lastActivityRef.current = Date.now();
|
|
@@ -2389,6 +2570,7 @@ function useSessionLifecycle(input) {
|
|
|
2389
2570
|
leaveRoomRef.current?.();
|
|
2390
2571
|
}, [clearTimer]);
|
|
2391
2572
|
const onConnected = useCallback(() => {
|
|
2573
|
+
if (userEndedRef.current) return;
|
|
2392
2574
|
clearTimer();
|
|
2393
2575
|
connectedRef.current = true;
|
|
2394
2576
|
manualReconnectPendingRef.current = false;
|
|
@@ -2401,7 +2583,7 @@ function useSessionLifecycle(input) {
|
|
|
2401
2583
|
}, [clearTimer]);
|
|
2402
2584
|
const onDisconnected = useCallback(
|
|
2403
2585
|
(reason) => {
|
|
2404
|
-
if (!active) return;
|
|
2586
|
+
if (!active || userEndedRef.current) return;
|
|
2405
2587
|
const action = disconnectAction(reason, connectedRef.current);
|
|
2406
2588
|
if (action === "noop") return;
|
|
2407
2589
|
if (action === "reset") {
|
|
@@ -2426,7 +2608,7 @@ function useSessionLifecycle(input) {
|
|
|
2426
2608
|
);
|
|
2427
2609
|
const onConnectionError = useCallback(
|
|
2428
2610
|
(error) => {
|
|
2429
|
-
if (!active || !isRecoverableConnectionError(error)) return;
|
|
2611
|
+
if (!active || userEndedRef.current || !isRecoverableConnectionError(error)) return;
|
|
2430
2612
|
connectedRef.current = false;
|
|
2431
2613
|
setConnected(false);
|
|
2432
2614
|
setAgentPresentState(false);
|
|
@@ -2439,6 +2621,9 @@ function useSessionLifecycle(input) {
|
|
|
2439
2621
|
},
|
|
2440
2622
|
[active]
|
|
2441
2623
|
);
|
|
2624
|
+
useEffect(() => {
|
|
2625
|
+
if (capacity.kind === "error" && recovery.kind === "connected") manualReconnectPendingRef.current = false;
|
|
2626
|
+
}, [capacity, recovery]);
|
|
2442
2627
|
useEffect(() => {
|
|
2443
2628
|
if (recovery.kind !== "refreshing" || capacity.kind !== "error") return;
|
|
2444
2629
|
const baseline = refreshBaselineCapacityRef.current;
|
|
@@ -2448,7 +2633,7 @@ function useSessionLifecycle(input) {
|
|
|
2448
2633
|
}, [capacity, recovery]);
|
|
2449
2634
|
const onConnectionStateChange = useCallback(
|
|
2450
2635
|
(state) => {
|
|
2451
|
-
if (!active) return;
|
|
2636
|
+
if (!active || userEndedRef.current) return;
|
|
2452
2637
|
if (state === "connected") {
|
|
2453
2638
|
onConnected();
|
|
2454
2639
|
return;
|
|
@@ -2496,8 +2681,30 @@ function useSessionLifecycle(input) {
|
|
|
2496
2681
|
}, step2.delayMs);
|
|
2497
2682
|
return clearTimer;
|
|
2498
2683
|
}, [recovery.kind, active, attempt, clearTimer]);
|
|
2684
|
+
const end = useCallback(() => {
|
|
2685
|
+
if (!active || userEndedRef.current || phaseRef.current.kind === "ended") return;
|
|
2686
|
+
userEndedRef.current = true;
|
|
2687
|
+
clearTimer();
|
|
2688
|
+
connectedRef.current = false;
|
|
2689
|
+
manualReconnectPendingRef.current = false;
|
|
2690
|
+
refreshBaselineCapacityRef.current = null;
|
|
2691
|
+
setUserEnded(true);
|
|
2692
|
+
setConnected(false);
|
|
2693
|
+
setAgentPresentState(false);
|
|
2694
|
+
setRecovery({ kind: "ended", reason: "user" });
|
|
2695
|
+
clearRef.current();
|
|
2696
|
+
leaveRoomRef.current?.();
|
|
2697
|
+
}, [active, clearTimer]);
|
|
2499
2698
|
const reconnect = useCallback(() => {
|
|
2500
|
-
if (!active
|
|
2699
|
+
if (!active) return;
|
|
2700
|
+
if (userEndedRef.current) {
|
|
2701
|
+
userEndedRef.current = false;
|
|
2702
|
+
setUserEnded(false);
|
|
2703
|
+
reset();
|
|
2704
|
+
manualReconnectPendingRef.current = true;
|
|
2705
|
+
return;
|
|
2706
|
+
}
|
|
2707
|
+
if (manualReconnectPendingRef.current || recovery.kind === "in-place-reconnecting") return;
|
|
2501
2708
|
manualReconnectPendingRef.current = true;
|
|
2502
2709
|
clearTimer();
|
|
2503
2710
|
attemptRef.current = 0;
|
|
@@ -2506,7 +2713,7 @@ function useSessionLifecycle(input) {
|
|
|
2506
2713
|
releaseRef.current("superseded");
|
|
2507
2714
|
setRecovery({ kind: "refreshing", attempt: 0 });
|
|
2508
2715
|
refreshRef.current();
|
|
2509
|
-
}, [active, clearTimer, recovery.kind]);
|
|
2716
|
+
}, [active, clearTimer, recovery.kind, reset]);
|
|
2510
2717
|
const endIdleRef = useRef(endIdle);
|
|
2511
2718
|
endIdleRef.current = endIdle;
|
|
2512
2719
|
useEffect(() => {
|
|
@@ -2538,6 +2745,7 @@ function useSessionLifecycle(input) {
|
|
|
2538
2745
|
() => lifecyclePhaseFrom({ capacity, recovery, idle, connected, agentPresent }),
|
|
2539
2746
|
[capacity, recovery, idle, connected, agentPresent]
|
|
2540
2747
|
);
|
|
2748
|
+
phaseRef.current = phase;
|
|
2541
2749
|
const onBehaviorChangeRef = useRef(onBehaviorChange);
|
|
2542
2750
|
onBehaviorChangeRef.current = onBehaviorChange;
|
|
2543
2751
|
const lastBehaviorRef = useRef(null);
|
|
@@ -2563,6 +2771,7 @@ function useSessionLifecycle(input) {
|
|
|
2563
2771
|
stayConnected,
|
|
2564
2772
|
markActivity,
|
|
2565
2773
|
reconnect,
|
|
2774
|
+
end,
|
|
2566
2775
|
onConnected,
|
|
2567
2776
|
onDisconnected,
|
|
2568
2777
|
onConnectionError,
|
|
@@ -2581,6 +2790,7 @@ function useSessionLifecycle(input) {
|
|
|
2581
2790
|
stayConnected,
|
|
2582
2791
|
markActivity,
|
|
2583
2792
|
reconnect,
|
|
2793
|
+
end,
|
|
2584
2794
|
onConnected,
|
|
2585
2795
|
onDisconnected,
|
|
2586
2796
|
onConnectionError,
|
|
@@ -2595,7 +2805,8 @@ function useSessionLifecycle(input) {
|
|
|
2595
2805
|
function SessionLifecycleRoomBridge({
|
|
2596
2806
|
lifecycle,
|
|
2597
2807
|
onConnectionDetailsChange,
|
|
2598
|
-
onMediaModeChange
|
|
2808
|
+
onMediaModeChange,
|
|
2809
|
+
microphone
|
|
2599
2810
|
}) {
|
|
2600
2811
|
const {
|
|
2601
2812
|
onConnectionStateChange,
|
|
@@ -2606,12 +2817,23 @@ function SessionLifecycleRoomBridge({
|
|
|
2606
2817
|
registerDataPublisher,
|
|
2607
2818
|
registerTurnSender,
|
|
2608
2819
|
setTurnState,
|
|
2609
|
-
setMedia
|
|
2820
|
+
setMedia,
|
|
2821
|
+
setMicrophoneFacts,
|
|
2822
|
+
setAudioPlayback,
|
|
2823
|
+
registerMediaControls
|
|
2610
2824
|
} = lifecycle;
|
|
2611
2825
|
const connectionState = useConnectionState();
|
|
2612
2826
|
const assistant = useVoiceAssistant();
|
|
2613
2827
|
const transcriptions = useTranscriptions();
|
|
2614
2828
|
const room = useRoomContext();
|
|
2829
|
+
useCallMedia(room, {
|
|
2830
|
+
setMicrophoneFacts,
|
|
2831
|
+
setAudioPlayback,
|
|
2832
|
+
registerMediaControls,
|
|
2833
|
+
microphoneWanted: microphone ?? publishesMicrophoneByDefault(lifecycle.grant),
|
|
2834
|
+
connectionState,
|
|
2835
|
+
callKey: lifecycle.grant?.session_id
|
|
2836
|
+
});
|
|
2615
2837
|
const mediaModeCallbackRef = useRef(onMediaModeChange);
|
|
2616
2838
|
mediaModeCallbackRef.current = onMediaModeChange;
|
|
2617
2839
|
const mediaModeEnabled = onMediaModeChange !== void 0 && lifecycle.grant !== null;
|
|
@@ -2885,6 +3107,8 @@ function resolveEndReason(workerLabel, innerReason) {
|
|
|
2885
3107
|
return "idle";
|
|
2886
3108
|
case "error":
|
|
2887
3109
|
return "failed";
|
|
3110
|
+
case "user":
|
|
3111
|
+
return "user_ended";
|
|
2888
3112
|
default:
|
|
2889
3113
|
return "disconnected";
|
|
2890
3114
|
}
|
|
@@ -2940,6 +3164,16 @@ function useRealtimeSession(input) {
|
|
|
2940
3164
|
const [turn, setTurn] = useState("quiet");
|
|
2941
3165
|
const [media, setMediaState] = useState({ video: "connecting", audio: "silent" });
|
|
2942
3166
|
const [behavior, setBehavior] = useState(null);
|
|
3167
|
+
const [microphoneFacts2, setMicrophoneFactsState] = useState({
|
|
3168
|
+
wanted: false,
|
|
3169
|
+
connected: false,
|
|
3170
|
+
enabling: false,
|
|
3171
|
+
publication: null,
|
|
3172
|
+
deviceError: null
|
|
3173
|
+
});
|
|
3174
|
+
const [microphoneMuted, setMicrophoneMuted] = useState(false);
|
|
3175
|
+
const [audioPlayback, setAudioPlaybackState] = useState("unknown");
|
|
3176
|
+
const mediaControlsRef = useRef(null);
|
|
2943
3177
|
const lastBehaviorRef = useRef(null);
|
|
2944
3178
|
const lastLabeledEndReasonRef = useRef(null);
|
|
2945
3179
|
const pendingClipsRef = useRef(/* @__PURE__ */ new Map());
|
|
@@ -3046,6 +3280,31 @@ function useRealtimeSession(input) {
|
|
|
3046
3280
|
const setMedia = useCallback((next) => {
|
|
3047
3281
|
setMediaState((prev) => prev.video === next.video && prev.audio === next.audio ? prev : next);
|
|
3048
3282
|
}, []);
|
|
3283
|
+
const microphoneFactsRef = useRef(microphoneFacts2);
|
|
3284
|
+
microphoneFactsRef.current = microphoneFacts2;
|
|
3285
|
+
const setMicrophoneFacts = useCallback((facts) => setMicrophoneFactsState(facts), []);
|
|
3286
|
+
const setAudioPlayback = useCallback((playback) => setAudioPlaybackState(playback), []);
|
|
3287
|
+
const registerMediaControls = useCallback((controls) => {
|
|
3288
|
+
mediaControlsRef.current = controls;
|
|
3289
|
+
}, []);
|
|
3290
|
+
const startAudio = useCallback(async () => {
|
|
3291
|
+
const controls = mediaControlsRef.current;
|
|
3292
|
+
return controls ? controls.startAudio() : false;
|
|
3293
|
+
}, []);
|
|
3294
|
+
const callEnded = lifecycle.phase.kind === "ended";
|
|
3295
|
+
const callEndedRef = useRef(callEnded);
|
|
3296
|
+
callEndedRef.current = callEnded;
|
|
3297
|
+
const setMicrophoneEnabled = useCallback(async (enabled) => {
|
|
3298
|
+
if (callEndedRef.current) return;
|
|
3299
|
+
setMicrophoneMuted(!enabled);
|
|
3300
|
+
if (!microphoneFactsRef.current.connected) return;
|
|
3301
|
+
await mediaControlsRef.current?.setMicrophoneEnabled(enabled);
|
|
3302
|
+
}, []);
|
|
3303
|
+
const microphone = useMemo(
|
|
3304
|
+
() => callEnded ? { status: "off" } : callMicrophoneFrom(microphoneFacts2, { muted: microphoneMuted }),
|
|
3305
|
+
[callEnded, microphoneFacts2, microphoneMuted]
|
|
3306
|
+
);
|
|
3307
|
+
const callAudioPlayback = callEnded ? "unknown" : audioPlayback;
|
|
3049
3308
|
useEffect(() => {
|
|
3050
3309
|
if (phaseKind === "idle" || phaseKind === "ended") return;
|
|
3051
3310
|
const timer = window.setInterval(() => {
|
|
@@ -3190,11 +3449,23 @@ function useRealtimeSession(input) {
|
|
|
3190
3449
|
void sendResolvedTurn(last.text, last.attributes, last.id).catch(() => void 0);
|
|
3191
3450
|
}, [sendResolvedTurn]);
|
|
3192
3451
|
const end = useCallback((reason) => {
|
|
3452
|
+
if (phaseKind === "ended") return;
|
|
3193
3453
|
if (reason) lastLabeledEndReasonRef.current = reason;
|
|
3194
3454
|
lastTurnRef.current = null;
|
|
3195
3455
|
requestGracefulClose();
|
|
3196
|
-
lifecycle.
|
|
3197
|
-
}, [lifecycle, requestGracefulClose]);
|
|
3456
|
+
lifecycle.end();
|
|
3457
|
+
}, [lifecycle, phaseKind, requestGracefulClose]);
|
|
3458
|
+
const active = inner.active ?? true;
|
|
3459
|
+
useEffect(() => {
|
|
3460
|
+
if (active) return;
|
|
3461
|
+
lastLabeledEndReasonRef.current = null;
|
|
3462
|
+
setMicrophoneMuted(false);
|
|
3463
|
+
}, [active]);
|
|
3464
|
+
const innerReconnect = lifecycle.reconnect;
|
|
3465
|
+
const reconnect = useCallback(() => {
|
|
3466
|
+
if (phaseKind === "ended") lastLabeledEndReasonRef.current = null;
|
|
3467
|
+
innerReconnect();
|
|
3468
|
+
}, [innerReconnect, phaseKind]);
|
|
3198
3469
|
const performAction = useCallback(
|
|
3199
3470
|
(actionId, opts) => {
|
|
3200
3471
|
const requestId = newTurnId();
|
|
@@ -3240,6 +3511,7 @@ function useRealtimeSession(input) {
|
|
|
3240
3511
|
approachingFiredRef.current = false;
|
|
3241
3512
|
creditsLowFiredRef.current = false;
|
|
3242
3513
|
lastTurnRef.current = null;
|
|
3514
|
+
setMicrophoneMuted(false);
|
|
3243
3515
|
lifecycle.reset();
|
|
3244
3516
|
}, [lifecycle]);
|
|
3245
3517
|
return useMemo(
|
|
@@ -3253,6 +3525,12 @@ function useRealtimeSession(input) {
|
|
|
3253
3525
|
endsAt,
|
|
3254
3526
|
graceWindow,
|
|
3255
3527
|
media,
|
|
3528
|
+
microphone,
|
|
3529
|
+
audioPlayback: callAudioPlayback,
|
|
3530
|
+
startAudio,
|
|
3531
|
+
setMicrophoneEnabled,
|
|
3532
|
+
microphoneMuted,
|
|
3533
|
+
reconnect,
|
|
3256
3534
|
lifecycle,
|
|
3257
3535
|
sendClosingTurn,
|
|
3258
3536
|
requestGracefulClose,
|
|
@@ -3268,6 +3546,9 @@ function useRealtimeSession(input) {
|
|
|
3268
3546
|
registerTurnSender,
|
|
3269
3547
|
setTurnState,
|
|
3270
3548
|
setMedia,
|
|
3549
|
+
setMicrophoneFacts,
|
|
3550
|
+
setAudioPlayback,
|
|
3551
|
+
registerMediaControls,
|
|
3271
3552
|
reset
|
|
3272
3553
|
}),
|
|
3273
3554
|
[
|
|
@@ -3277,6 +3558,12 @@ function useRealtimeSession(input) {
|
|
|
3277
3558
|
endsAt,
|
|
3278
3559
|
graceWindow,
|
|
3279
3560
|
media,
|
|
3561
|
+
microphone,
|
|
3562
|
+
callAudioPlayback,
|
|
3563
|
+
startAudio,
|
|
3564
|
+
setMicrophoneEnabled,
|
|
3565
|
+
microphoneMuted,
|
|
3566
|
+
reconnect,
|
|
3280
3567
|
sendClosingTurn,
|
|
3281
3568
|
requestGracefulClose,
|
|
3282
3569
|
extend,
|
|
@@ -3291,6 +3578,9 @@ function useRealtimeSession(input) {
|
|
|
3291
3578
|
registerTurnSender,
|
|
3292
3579
|
setTurnState,
|
|
3293
3580
|
setMedia,
|
|
3581
|
+
setMicrophoneFacts,
|
|
3582
|
+
setAudioPlayback,
|
|
3583
|
+
registerMediaControls,
|
|
3294
3584
|
reset
|
|
3295
3585
|
]
|
|
3296
3586
|
);
|
|
@@ -3305,6 +3595,8 @@ function createProxyClient(options) {
|
|
|
3305
3595
|
const base = normalize(options.proxyUrl);
|
|
3306
3596
|
const doFetch = options.fetch ?? globalThis.fetch?.bind(globalThis);
|
|
3307
3597
|
const timeoutMs = options.timeoutMs ?? PROXY_CLIENT_TIMEOUT_MS;
|
|
3598
|
+
const credentials = options.credentials;
|
|
3599
|
+
const withCredentials = credentials === void 0 ? {} : { credentials };
|
|
3308
3600
|
const post = async (path, body, request) => {
|
|
3309
3601
|
if (!doFetch) throw new Error("realtime-avatar: no fetch available \u2014 pass one via `fetch`.");
|
|
3310
3602
|
const caller = request?.signal;
|
|
@@ -3312,6 +3604,7 @@ function createProxyClient(options) {
|
|
|
3312
3604
|
try {
|
|
3313
3605
|
return await doFetch(`${base}${path}`, {
|
|
3314
3606
|
method: "POST",
|
|
3607
|
+
...withCredentials,
|
|
3315
3608
|
headers: { "content-type": "application/json", ...request?.headers ?? {} },
|
|
3316
3609
|
body: JSON.stringify(body),
|
|
3317
3610
|
signal: timer && caller ? AbortSignal.any([caller, timer]) : timer ?? caller
|
|
@@ -3321,18 +3614,38 @@ function createProxyClient(options) {
|
|
|
3321
3614
|
throw cause;
|
|
3322
3615
|
}
|
|
3323
3616
|
};
|
|
3324
|
-
const
|
|
3617
|
+
const pageHide = (body) => {
|
|
3618
|
+
const page = globalThis.location?.href;
|
|
3619
|
+
let target;
|
|
3620
|
+
try {
|
|
3621
|
+
target = new URL(`${base}/end`, page);
|
|
3622
|
+
} catch {
|
|
3623
|
+
return false;
|
|
3624
|
+
}
|
|
3625
|
+
const payload = JSON.stringify(body);
|
|
3325
3626
|
const send = globalThis.navigator?.sendBeacon?.bind(globalThis.navigator);
|
|
3326
|
-
|
|
3327
|
-
|
|
3627
|
+
const sameOrigin = page !== void 0 && target.origin === new URL(page).origin;
|
|
3628
|
+
if (send && (sameOrigin || credentials === "include")) {
|
|
3629
|
+
if (send(target.href, new Blob([payload], { type: "application/json" }))) return true;
|
|
3630
|
+
}
|
|
3631
|
+
if (!doFetch) return false;
|
|
3632
|
+
try {
|
|
3633
|
+
void doFetch(target.href, {
|
|
3634
|
+
method: "POST",
|
|
3635
|
+
keepalive: true,
|
|
3636
|
+
...withCredentials,
|
|
3637
|
+
headers: { "content-type": "application/json" },
|
|
3638
|
+
body: payload
|
|
3639
|
+
}).catch(() => void 0);
|
|
3640
|
+
return true;
|
|
3641
|
+
} catch {
|
|
3642
|
+
return false;
|
|
3643
|
+
}
|
|
3328
3644
|
};
|
|
3329
3645
|
return {
|
|
3330
3646
|
async createLiveKitSessionOrBusy(input, requestOptions) {
|
|
3331
|
-
const
|
|
3332
|
-
|
|
3333
|
-
{ avatarId: input.avatarId, mode: input.mode },
|
|
3334
|
-
requestOptions
|
|
3335
|
-
);
|
|
3647
|
+
const connect = { avatarId: input.avatarId, mode: input.mode };
|
|
3648
|
+
const response = await post("/connect", connect, requestOptions);
|
|
3336
3649
|
if (response.status === 429) {
|
|
3337
3650
|
const value = await response.clone().json().catch(() => null);
|
|
3338
3651
|
if (value && typeof value === "object" && !Array.isArray(value)) {
|
|
@@ -3350,7 +3663,8 @@ function createProxyClient(options) {
|
|
|
3350
3663
|
async releaseLiveKitSession(sessionId, reason, requestOptions) {
|
|
3351
3664
|
if (!sessionId) return false;
|
|
3352
3665
|
try {
|
|
3353
|
-
const
|
|
3666
|
+
const body = { session_id: sessionId, reason };
|
|
3667
|
+
const response = await post("/end", body, requestOptions);
|
|
3354
3668
|
return response.ok;
|
|
3355
3669
|
} catch {
|
|
3356
3670
|
return false;
|
|
@@ -3358,12 +3672,13 @@ function createProxyClient(options) {
|
|
|
3358
3672
|
},
|
|
3359
3673
|
releaseLiveKitSessionBeacon(sessionId, reason) {
|
|
3360
3674
|
if (!sessionId) return false;
|
|
3361
|
-
return
|
|
3675
|
+
return pageHide({ session_id: sessionId, reason: reason ?? "page_hide" });
|
|
3362
3676
|
},
|
|
3363
3677
|
async releaseLiveKitQueueTicket(queueTicketId, reason, requestOptions) {
|
|
3364
3678
|
if (!queueTicketId) return false;
|
|
3365
3679
|
try {
|
|
3366
|
-
const
|
|
3680
|
+
const body = { queue_ticket_id: queueTicketId, reason };
|
|
3681
|
+
const response = await post("/end", body, requestOptions);
|
|
3367
3682
|
return response.ok;
|
|
3368
3683
|
} catch {
|
|
3369
3684
|
return false;
|
|
@@ -3371,7 +3686,7 @@ function createProxyClient(options) {
|
|
|
3371
3686
|
},
|
|
3372
3687
|
releaseLiveKitQueueTicketBeacon(queueTicketId, reason) {
|
|
3373
3688
|
if (!queueTicketId) return false;
|
|
3374
|
-
return
|
|
3689
|
+
return pageHide({ queue_ticket_id: queueTicketId, reason: reason ?? "page_hide" });
|
|
3375
3690
|
}
|
|
3376
3691
|
};
|
|
3377
3692
|
}
|