realtime-avatar 0.25.0 → 0.26.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,9 +1,9 @@
1
1
  import { AudioSession, LiveKitRoom, VideoTrack } from '@livekit/react-native';
2
2
  export { AudioSession, VideoTrack, registerGlobals } from '@livekit/react-native';
3
3
  import { useState, useRef, useEffect, useCallback, createElement, useMemo } from 'react';
4
- import { useLocalParticipant, useConnectionState, useTranscriptions, useMaybeRoomContext, useVoiceAssistant, isTrackReference, useRoomContext, useChat } from '@livekit/components-react';
4
+ import { useLocalParticipant, useConnectionState, useTranscriptions, useMaybeRoomContext, useVoiceAssistant, isTrackReference, useRoomContext, useChat, useAudioPlayback } from '@livekit/components-react';
5
5
  export { useChat, useConnectionState, useLocalParticipant, useRoomContext, useTrackToggle, useTranscriptions, useVoiceAssistant } from '@livekit/components-react';
6
- import { ConnectionState, isRemoteTrack, VideoQuality, ConnectionQuality, RoomEvent, TrackEvent, Track, ConnectionError, ConnectionErrorReason, DisconnectReason, createLocalVideoTrack, Room } from 'livekit-client';
6
+ import { ConnectionState, isRemoteTrack, VideoQuality, ConnectionQuality, RoomEvent, TrackEvent, Track, ConnectionError, ConnectionErrorReason, ParticipantEvent, DisconnectReason, createLocalVideoTrack, Room } from 'livekit-client';
7
7
  export { DisconnectReason, Room, RoomEvent, Track } from 'livekit-client';
8
8
  import { Platform, StyleSheet, Animated, Image, View, Text } from 'react-native';
9
9
  import { z } from 'zod';
@@ -698,6 +698,11 @@ function useLiveKitAvatarGrant(input) {
698
698
  });
699
699
  const stateRef = useRef(state);
700
700
  stateRef.current = state;
701
+ const wantedRef = useRef(active);
702
+ wantedRef.current = active;
703
+ useEffect(() => () => {
704
+ wantedRef.current = false;
705
+ }, []);
701
706
  const [tabVisible, setTabVisible] = useState(
702
707
  () => typeof document === "undefined" || document.visibilityState === "visible"
703
708
  );
@@ -738,6 +743,8 @@ function useLiveKitAvatarGrant(input) {
738
743
  if (cancelled) {
739
744
  if (result.status !== "busy") {
740
745
  void client.releaseLiveKitSession(result.grant.session_id, "superseded");
746
+ } else if (!wantedRef.current && result.busy.queue_ticket_id) {
747
+ void client.releaseLiveKitQueueTicket(result.busy.queue_ticket_id, "unmount");
741
748
  }
742
749
  return;
743
750
  }
@@ -821,6 +828,9 @@ function sameCapacityState(a, b) {
821
828
  return true;
822
829
  }
823
830
  }
831
+ function publishesMicrophoneByDefault(grant) {
832
+ return grant?.stt_mode === "server";
833
+ }
824
834
  var capacityErrorFromBusy = (busy) => {
825
835
  return new RealtimeAvatarCapacityError(busy.message, busy);
826
836
  };
@@ -2146,6 +2156,169 @@ function createConnectionHistoryCollector(input) {
2146
2156
  return { enqueue, flush, dispose };
2147
2157
  }
2148
2158
 
2159
+ // ../browser/src/microphone.ts
2160
+ var HINTS = {
2161
+ "insecure-origin": "The browser only exposes microphones on a secure origin. Serve the page over https, or open it on http://localhost \u2014 a LAN address like http://192.168.1.5 will not do.",
2162
+ "no-answer": "The permission prompt is probably still open, or was dismissed without an answer. Look for it in the address bar, answer it, and try again.",
2163
+ "denied-by-browser": "This site is blocked from using the microphone. Click the icon at the left of the address bar, set Microphone to Allow, and reload.",
2164
+ "denied-by-os": "The operating system is blocking the browser itself, so no site can record. On macOS: System Settings > Privacy & Security > Microphone, enable your browser, then RESTART it \u2014 the change does not apply to a running browser.",
2165
+ "no-device": "No microphone is available. Connect one, and if you are on Bluetooth headphones check they are in a mode that has a microphone.",
2166
+ "device-in-use": "Another application or browser tab is holding the microphone. Close it \u2014 including any call from this page you did not hang up \u2014 and try again.",
2167
+ "device-lost": "The microphone stopped during the call: it was unplugged, or access to it was turned off. Reconnect it or allow access again, then turn the microphone back on.",
2168
+ unknown: "Try again, and if it persists include the message above in your report."
2169
+ };
2170
+ function isOperatingSystemDenial(message) {
2171
+ return /\bby system\b|system permission/i.test(message);
2172
+ }
2173
+ function classify(error) {
2174
+ if (error instanceof TypeError && /mediaDevices|undefined/i.test(error.message)) {
2175
+ return { reason: "insecure-origin", message: error.message };
2176
+ }
2177
+ const name = typeof error === "object" && error !== null && "name" in error ? String(error.name) : "";
2178
+ const message = typeof error === "object" && error !== null && "message" in error && error.message ? String(error.message) : String(error);
2179
+ switch (name) {
2180
+ case "NotAllowedError":
2181
+ case "PermissionDeniedError":
2182
+ return {
2183
+ reason: isOperatingSystemDenial(message) ? "denied-by-os" : "denied-by-browser",
2184
+ message
2185
+ };
2186
+ case "NotFoundError":
2187
+ case "DevicesNotFoundError":
2188
+ case "OverconstrainedError":
2189
+ case "ConstraintNotSatisfiedError":
2190
+ return { reason: "no-device", message };
2191
+ case "NotReadableError":
2192
+ case "TrackStartError":
2193
+ case "AbortError":
2194
+ return { reason: "device-in-use", message };
2195
+ case "SecurityError":
2196
+ return { reason: "insecure-origin", message };
2197
+ default:
2198
+ return { reason: "unknown", message };
2199
+ }
2200
+ }
2201
+ function describeMicrophoneFailure(error) {
2202
+ const { reason, message } = classify(error);
2203
+ return { reason, message, hint: HINTS[reason] };
2204
+ }
2205
+ function microphoneLost(message) {
2206
+ return { reason: "device-lost", message, hint: HINTS["device-lost"] };
2207
+ }
2208
+
2209
+ // ../client/src/react/call-media.ts
2210
+ var DEVICE_LOST_MESSAGE = "The microphone track ended during the call.";
2211
+ function callMicrophoneFrom(facts, intent) {
2212
+ if (!facts.wanted) return { status: "off" };
2213
+ const { publication } = facts;
2214
+ if (publication) {
2215
+ if (publication.ended) {
2216
+ return publication.muted ? { status: "unavailable", ...microphoneLost(DEVICE_LOST_MESSAGE) } : { status: "pending" };
2217
+ }
2218
+ return publication.muted ? { status: "muted" } : { status: "on" };
2219
+ }
2220
+ if (intent.muted) return { status: "muted" };
2221
+ if (facts.enabling) return { status: "pending" };
2222
+ const error = facts.deviceError;
2223
+ if (error) {
2224
+ const problem = describeMicrophoneFailure(error);
2225
+ const blocked = problem.reason === "denied-by-browser" || problem.reason === "denied-by-os" || problem.reason === "insecure-origin";
2226
+ return blocked ? { status: "blocked", ...problem } : { status: "unavailable", ...problem };
2227
+ }
2228
+ return facts.connected ? { status: "pending" } : { status: "off" };
2229
+ }
2230
+ function microphoneFacts(room, wanted, connected, enabling, staleError) {
2231
+ const publication = room.localParticipant.getTrackPublication(Track.Source.Microphone);
2232
+ const track = publication?.track;
2233
+ return {
2234
+ wanted,
2235
+ connected,
2236
+ enabling,
2237
+ publication: track ? { muted: publication.isMuted, ended: track.mediaStreamTrack?.readyState === "ended" } : null,
2238
+ deviceError: room.localParticipant.lastMicrophoneError !== staleError && room.localParticipant.lastMicrophoneError || null
2239
+ };
2240
+ }
2241
+ function useCallMedia(room, options) {
2242
+ const { setMicrophoneFacts, setAudioPlayback, registerMediaControls, microphoneWanted, connectionState, callKey } = options;
2243
+ const connected = connectionState === ConnectionState.Connected;
2244
+ const scope = useMemo(
2245
+ () => ({ room, callKey, stale: room.localParticipant.lastMicrophoneError }),
2246
+ [room, callKey]
2247
+ );
2248
+ const [enabling, setEnabling] = useState(false);
2249
+ const [version, setVersion] = useState(0);
2250
+ const sinksRef = useRef({ setMicrophoneFacts, setAudioPlayback });
2251
+ sinksRef.current = { setMicrophoneFacts, setAudioPlayback };
2252
+ useEffect(() => {
2253
+ if (!setMicrophoneFacts) return;
2254
+ const bump = () => setVersion((value) => value + 1);
2255
+ const participant = room.localParticipant;
2256
+ let watched = null;
2257
+ const watchTrack = () => {
2258
+ watched?.off();
2259
+ const track = participant.getTrackPublication(Track.Source.Microphone)?.track;
2260
+ if (!track) {
2261
+ watched = null;
2262
+ return;
2263
+ }
2264
+ track.on(TrackEvent.Ended, bump).on(TrackEvent.Restarted, bump).on(TrackEvent.Muted, bump).on(TrackEvent.Unmuted, bump);
2265
+ watched = { off: () => {
2266
+ track.off(TrackEvent.Ended, bump).off(TrackEvent.Restarted, bump).off(TrackEvent.Muted, bump).off(TrackEvent.Unmuted, bump);
2267
+ } };
2268
+ };
2269
+ const onPublications = () => {
2270
+ watchTrack();
2271
+ bump();
2272
+ };
2273
+ room.on(RoomEvent.MediaDevicesError, bump).on(RoomEvent.LocalTrackPublished, onPublications).on(RoomEvent.LocalTrackUnpublished, onPublications);
2274
+ participant.on(ParticipantEvent.TrackMuted, bump).on(ParticipantEvent.TrackUnmuted, bump);
2275
+ watchTrack();
2276
+ return () => {
2277
+ watched?.off();
2278
+ room.off(RoomEvent.MediaDevicesError, bump).off(RoomEvent.LocalTrackPublished, onPublications).off(RoomEvent.LocalTrackUnpublished, onPublications);
2279
+ participant.off(ParticipantEvent.TrackMuted, bump).off(ParticipantEvent.TrackUnmuted, bump);
2280
+ };
2281
+ }, [room, setMicrophoneFacts]);
2282
+ useEffect(() => {
2283
+ sinksRef.current.setMicrophoneFacts?.(microphoneFacts(room, microphoneWanted, connected, enabling, scope.stale));
2284
+ }, [room, microphoneWanted, connected, enabling, version, scope]);
2285
+ const { canPlayAudio } = useAudioPlayback(room);
2286
+ useEffect(() => {
2287
+ sinksRef.current.setAudioPlayback?.(!canPlayAudio ? "blocked" : connected ? "allowed" : "unknown");
2288
+ }, [canPlayAudio, connected]);
2289
+ const unlockTriedRef = useRef(null);
2290
+ const connecting = connectionState === ConnectionState.Connecting || connected;
2291
+ useEffect(() => {
2292
+ if (!setAudioPlayback || !connecting || unlockTriedRef.current === room) return;
2293
+ unlockTriedRef.current = room;
2294
+ void room.startAudio().catch(() => void 0);
2295
+ }, [room, setAudioPlayback, connecting]);
2296
+ const controls = useMemo(() => ({
2297
+ startAudio: async () => {
2298
+ try {
2299
+ await room.startAudio();
2300
+ } catch {
2301
+ }
2302
+ return room.canPlaybackAudio;
2303
+ },
2304
+ setMicrophoneEnabled: async (enabled) => {
2305
+ if (enabled) setEnabling(true);
2306
+ try {
2307
+ await room.localParticipant.setMicrophoneEnabled(enabled);
2308
+ } catch {
2309
+ } finally {
2310
+ if (enabled) setEnabling(false);
2311
+ setVersion((value) => value + 1);
2312
+ }
2313
+ }
2314
+ }), [room]);
2315
+ useEffect(() => {
2316
+ if (!registerMediaControls) return;
2317
+ registerMediaControls(controls);
2318
+ return () => registerMediaControls(null);
2319
+ }, [controls, registerMediaControls]);
2320
+ }
2321
+
2149
2322
  // ../client/src/react/session-lifecycle.ts
2150
2323
  function phaseForReason(reason) {
2151
2324
  switch (reason) {
@@ -2306,10 +2479,13 @@ function useSessionLifecycle(input) {
2306
2479
  const reconnectPolicyRef = useRef(reconnectPolicy);
2307
2480
  reconnectPolicyRef.current = reconnectPolicy;
2308
2481
  const connectWatchdogMs = typeof connectWatchdogSeconds === "number" && Number.isFinite(connectWatchdogSeconds) ? Math.max(0, Math.floor(connectWatchdogSeconds)) * 1e3 : DEFAULT_CONNECT_WATCHDOG_SECONDS * 1e3;
2482
+ const [userEnded, setUserEnded] = useState(false);
2483
+ const userEndedRef = useRef(false);
2484
+ const phaseRef = useRef({ kind: "idle" });
2309
2485
  const grantState = useLiveKitAvatarGrant({
2310
2486
  client,
2311
2487
  session,
2312
- active,
2488
+ active: active && !userEnded,
2313
2489
  autoRetryBusy,
2314
2490
  requestOptions
2315
2491
  });
@@ -2326,6 +2502,8 @@ function useSessionLifecycle(input) {
2326
2502
  refreshRef.current = grantState.refresh;
2327
2503
  const releaseRef = useRef(grantState.release);
2328
2504
  releaseRef.current = grantState.release;
2505
+ const clearRef = useRef(grantState.clear);
2506
+ clearRef.current = grantState.clear;
2329
2507
  const capacityRef = useRef(capacity);
2330
2508
  capacityRef.current = capacity;
2331
2509
  const refreshBaselineCapacityRef = useRef(null);
@@ -2352,13 +2530,16 @@ function useSessionLifecycle(input) {
2352
2530
  refreshBaselineCapacityRef.current = null;
2353
2531
  attemptRef.current = 0;
2354
2532
  setAttempt(0);
2355
- setRecovery({ kind: "connected" });
2533
+ setRecovery(userEndedRef.current ? { kind: "ended", reason: "user" } : { kind: "connected" });
2356
2534
  setConnected(false);
2357
2535
  setAgentPresentState(false);
2358
2536
  lastActivityRef.current = Date.now();
2359
2537
  }, [clearTimer]);
2360
2538
  useEffect(() => {
2361
- if (!active) reset();
2539
+ if (active) return;
2540
+ userEndedRef.current = false;
2541
+ setUserEnded(false);
2542
+ reset();
2362
2543
  }, [active, reset]);
2363
2544
  const markActivity = useCallback(() => {
2364
2545
  lastActivityRef.current = Date.now();
@@ -2389,6 +2570,7 @@ function useSessionLifecycle(input) {
2389
2570
  leaveRoomRef.current?.();
2390
2571
  }, [clearTimer]);
2391
2572
  const onConnected = useCallback(() => {
2573
+ if (userEndedRef.current) return;
2392
2574
  clearTimer();
2393
2575
  connectedRef.current = true;
2394
2576
  manualReconnectPendingRef.current = false;
@@ -2401,7 +2583,7 @@ function useSessionLifecycle(input) {
2401
2583
  }, [clearTimer]);
2402
2584
  const onDisconnected = useCallback(
2403
2585
  (reason) => {
2404
- if (!active) return;
2586
+ if (!active || userEndedRef.current) return;
2405
2587
  const action = disconnectAction(reason, connectedRef.current);
2406
2588
  if (action === "noop") return;
2407
2589
  if (action === "reset") {
@@ -2426,7 +2608,7 @@ function useSessionLifecycle(input) {
2426
2608
  );
2427
2609
  const onConnectionError = useCallback(
2428
2610
  (error) => {
2429
- if (!active || !isRecoverableConnectionError(error)) return;
2611
+ if (!active || userEndedRef.current || !isRecoverableConnectionError(error)) return;
2430
2612
  connectedRef.current = false;
2431
2613
  setConnected(false);
2432
2614
  setAgentPresentState(false);
@@ -2439,6 +2621,9 @@ function useSessionLifecycle(input) {
2439
2621
  },
2440
2622
  [active]
2441
2623
  );
2624
+ useEffect(() => {
2625
+ if (capacity.kind === "error" && recovery.kind === "connected") manualReconnectPendingRef.current = false;
2626
+ }, [capacity, recovery]);
2442
2627
  useEffect(() => {
2443
2628
  if (recovery.kind !== "refreshing" || capacity.kind !== "error") return;
2444
2629
  const baseline = refreshBaselineCapacityRef.current;
@@ -2448,7 +2633,7 @@ function useSessionLifecycle(input) {
2448
2633
  }, [capacity, recovery]);
2449
2634
  const onConnectionStateChange = useCallback(
2450
2635
  (state) => {
2451
- if (!active) return;
2636
+ if (!active || userEndedRef.current) return;
2452
2637
  if (state === "connected") {
2453
2638
  onConnected();
2454
2639
  return;
@@ -2496,8 +2681,30 @@ function useSessionLifecycle(input) {
2496
2681
  }, step2.delayMs);
2497
2682
  return clearTimer;
2498
2683
  }, [recovery.kind, active, attempt, clearTimer]);
2684
+ const end = useCallback(() => {
2685
+ if (!active || userEndedRef.current || phaseRef.current.kind === "ended") return;
2686
+ userEndedRef.current = true;
2687
+ clearTimer();
2688
+ connectedRef.current = false;
2689
+ manualReconnectPendingRef.current = false;
2690
+ refreshBaselineCapacityRef.current = null;
2691
+ setUserEnded(true);
2692
+ setConnected(false);
2693
+ setAgentPresentState(false);
2694
+ setRecovery({ kind: "ended", reason: "user" });
2695
+ clearRef.current();
2696
+ leaveRoomRef.current?.();
2697
+ }, [active, clearTimer]);
2499
2698
  const reconnect = useCallback(() => {
2500
- if (!active || manualReconnectPendingRef.current || recovery.kind === "in-place-reconnecting") return;
2699
+ if (!active) return;
2700
+ if (userEndedRef.current) {
2701
+ userEndedRef.current = false;
2702
+ setUserEnded(false);
2703
+ reset();
2704
+ manualReconnectPendingRef.current = true;
2705
+ return;
2706
+ }
2707
+ if (manualReconnectPendingRef.current || recovery.kind === "in-place-reconnecting") return;
2501
2708
  manualReconnectPendingRef.current = true;
2502
2709
  clearTimer();
2503
2710
  attemptRef.current = 0;
@@ -2506,7 +2713,7 @@ function useSessionLifecycle(input) {
2506
2713
  releaseRef.current("superseded");
2507
2714
  setRecovery({ kind: "refreshing", attempt: 0 });
2508
2715
  refreshRef.current();
2509
- }, [active, clearTimer, recovery.kind]);
2716
+ }, [active, clearTimer, recovery.kind, reset]);
2510
2717
  const endIdleRef = useRef(endIdle);
2511
2718
  endIdleRef.current = endIdle;
2512
2719
  useEffect(() => {
@@ -2538,6 +2745,7 @@ function useSessionLifecycle(input) {
2538
2745
  () => lifecyclePhaseFrom({ capacity, recovery, idle, connected, agentPresent }),
2539
2746
  [capacity, recovery, idle, connected, agentPresent]
2540
2747
  );
2748
+ phaseRef.current = phase;
2541
2749
  const onBehaviorChangeRef = useRef(onBehaviorChange);
2542
2750
  onBehaviorChangeRef.current = onBehaviorChange;
2543
2751
  const lastBehaviorRef = useRef(null);
@@ -2563,6 +2771,7 @@ function useSessionLifecycle(input) {
2563
2771
  stayConnected,
2564
2772
  markActivity,
2565
2773
  reconnect,
2774
+ end,
2566
2775
  onConnected,
2567
2776
  onDisconnected,
2568
2777
  onConnectionError,
@@ -2581,6 +2790,7 @@ function useSessionLifecycle(input) {
2581
2790
  stayConnected,
2582
2791
  markActivity,
2583
2792
  reconnect,
2793
+ end,
2584
2794
  onConnected,
2585
2795
  onDisconnected,
2586
2796
  onConnectionError,
@@ -2595,7 +2805,8 @@ function useSessionLifecycle(input) {
2595
2805
  function SessionLifecycleRoomBridge({
2596
2806
  lifecycle,
2597
2807
  onConnectionDetailsChange,
2598
- onMediaModeChange
2808
+ onMediaModeChange,
2809
+ microphone
2599
2810
  }) {
2600
2811
  const {
2601
2812
  onConnectionStateChange,
@@ -2606,12 +2817,23 @@ function SessionLifecycleRoomBridge({
2606
2817
  registerDataPublisher,
2607
2818
  registerTurnSender,
2608
2819
  setTurnState,
2609
- setMedia
2820
+ setMedia,
2821
+ setMicrophoneFacts,
2822
+ setAudioPlayback,
2823
+ registerMediaControls
2610
2824
  } = lifecycle;
2611
2825
  const connectionState = useConnectionState();
2612
2826
  const assistant = useVoiceAssistant();
2613
2827
  const transcriptions = useTranscriptions();
2614
2828
  const room = useRoomContext();
2829
+ useCallMedia(room, {
2830
+ setMicrophoneFacts,
2831
+ setAudioPlayback,
2832
+ registerMediaControls,
2833
+ microphoneWanted: microphone ?? publishesMicrophoneByDefault(lifecycle.grant),
2834
+ connectionState,
2835
+ callKey: lifecycle.grant?.session_id
2836
+ });
2615
2837
  const mediaModeCallbackRef = useRef(onMediaModeChange);
2616
2838
  mediaModeCallbackRef.current = onMediaModeChange;
2617
2839
  const mediaModeEnabled = onMediaModeChange !== void 0 && lifecycle.grant !== null;
@@ -2885,6 +3107,8 @@ function resolveEndReason(workerLabel, innerReason) {
2885
3107
  return "idle";
2886
3108
  case "error":
2887
3109
  return "failed";
3110
+ case "user":
3111
+ return "user_ended";
2888
3112
  default:
2889
3113
  return "disconnected";
2890
3114
  }
@@ -2940,6 +3164,16 @@ function useRealtimeSession(input) {
2940
3164
  const [turn, setTurn] = useState("quiet");
2941
3165
  const [media, setMediaState] = useState({ video: "connecting", audio: "silent" });
2942
3166
  const [behavior, setBehavior] = useState(null);
3167
+ const [microphoneFacts2, setMicrophoneFactsState] = useState({
3168
+ wanted: false,
3169
+ connected: false,
3170
+ enabling: false,
3171
+ publication: null,
3172
+ deviceError: null
3173
+ });
3174
+ const [microphoneMuted, setMicrophoneMuted] = useState(false);
3175
+ const [audioPlayback, setAudioPlaybackState] = useState("unknown");
3176
+ const mediaControlsRef = useRef(null);
2943
3177
  const lastBehaviorRef = useRef(null);
2944
3178
  const lastLabeledEndReasonRef = useRef(null);
2945
3179
  const pendingClipsRef = useRef(/* @__PURE__ */ new Map());
@@ -3046,6 +3280,31 @@ function useRealtimeSession(input) {
3046
3280
  const setMedia = useCallback((next) => {
3047
3281
  setMediaState((prev) => prev.video === next.video && prev.audio === next.audio ? prev : next);
3048
3282
  }, []);
3283
+ const microphoneFactsRef = useRef(microphoneFacts2);
3284
+ microphoneFactsRef.current = microphoneFacts2;
3285
+ const setMicrophoneFacts = useCallback((facts) => setMicrophoneFactsState(facts), []);
3286
+ const setAudioPlayback = useCallback((playback) => setAudioPlaybackState(playback), []);
3287
+ const registerMediaControls = useCallback((controls) => {
3288
+ mediaControlsRef.current = controls;
3289
+ }, []);
3290
+ const startAudio = useCallback(async () => {
3291
+ const controls = mediaControlsRef.current;
3292
+ return controls ? controls.startAudio() : false;
3293
+ }, []);
3294
+ const callEnded = lifecycle.phase.kind === "ended";
3295
+ const callEndedRef = useRef(callEnded);
3296
+ callEndedRef.current = callEnded;
3297
+ const setMicrophoneEnabled = useCallback(async (enabled) => {
3298
+ if (callEndedRef.current) return;
3299
+ setMicrophoneMuted(!enabled);
3300
+ if (!microphoneFactsRef.current.connected) return;
3301
+ await mediaControlsRef.current?.setMicrophoneEnabled(enabled);
3302
+ }, []);
3303
+ const microphone = useMemo(
3304
+ () => callEnded ? { status: "off" } : callMicrophoneFrom(microphoneFacts2, { muted: microphoneMuted }),
3305
+ [callEnded, microphoneFacts2, microphoneMuted]
3306
+ );
3307
+ const callAudioPlayback = callEnded ? "unknown" : audioPlayback;
3049
3308
  useEffect(() => {
3050
3309
  if (phaseKind === "idle" || phaseKind === "ended") return;
3051
3310
  const timer = window.setInterval(() => {
@@ -3190,11 +3449,23 @@ function useRealtimeSession(input) {
3190
3449
  void sendResolvedTurn(last.text, last.attributes, last.id).catch(() => void 0);
3191
3450
  }, [sendResolvedTurn]);
3192
3451
  const end = useCallback((reason) => {
3452
+ if (phaseKind === "ended") return;
3193
3453
  if (reason) lastLabeledEndReasonRef.current = reason;
3194
3454
  lastTurnRef.current = null;
3195
3455
  requestGracefulClose();
3196
- lifecycle.reset();
3197
- }, [lifecycle, requestGracefulClose]);
3456
+ lifecycle.end();
3457
+ }, [lifecycle, phaseKind, requestGracefulClose]);
3458
+ const active = inner.active ?? true;
3459
+ useEffect(() => {
3460
+ if (active) return;
3461
+ lastLabeledEndReasonRef.current = null;
3462
+ setMicrophoneMuted(false);
3463
+ }, [active]);
3464
+ const innerReconnect = lifecycle.reconnect;
3465
+ const reconnect = useCallback(() => {
3466
+ if (phaseKind === "ended") lastLabeledEndReasonRef.current = null;
3467
+ innerReconnect();
3468
+ }, [innerReconnect, phaseKind]);
3198
3469
  const performAction = useCallback(
3199
3470
  (actionId, opts) => {
3200
3471
  const requestId = newTurnId();
@@ -3240,6 +3511,7 @@ function useRealtimeSession(input) {
3240
3511
  approachingFiredRef.current = false;
3241
3512
  creditsLowFiredRef.current = false;
3242
3513
  lastTurnRef.current = null;
3514
+ setMicrophoneMuted(false);
3243
3515
  lifecycle.reset();
3244
3516
  }, [lifecycle]);
3245
3517
  return useMemo(
@@ -3253,6 +3525,12 @@ function useRealtimeSession(input) {
3253
3525
  endsAt,
3254
3526
  graceWindow,
3255
3527
  media,
3528
+ microphone,
3529
+ audioPlayback: callAudioPlayback,
3530
+ startAudio,
3531
+ setMicrophoneEnabled,
3532
+ microphoneMuted,
3533
+ reconnect,
3256
3534
  lifecycle,
3257
3535
  sendClosingTurn,
3258
3536
  requestGracefulClose,
@@ -3268,6 +3546,9 @@ function useRealtimeSession(input) {
3268
3546
  registerTurnSender,
3269
3547
  setTurnState,
3270
3548
  setMedia,
3549
+ setMicrophoneFacts,
3550
+ setAudioPlayback,
3551
+ registerMediaControls,
3271
3552
  reset
3272
3553
  }),
3273
3554
  [
@@ -3277,6 +3558,12 @@ function useRealtimeSession(input) {
3277
3558
  endsAt,
3278
3559
  graceWindow,
3279
3560
  media,
3561
+ microphone,
3562
+ callAudioPlayback,
3563
+ startAudio,
3564
+ setMicrophoneEnabled,
3565
+ microphoneMuted,
3566
+ reconnect,
3280
3567
  sendClosingTurn,
3281
3568
  requestGracefulClose,
3282
3569
  extend,
@@ -3291,6 +3578,9 @@ function useRealtimeSession(input) {
3291
3578
  registerTurnSender,
3292
3579
  setTurnState,
3293
3580
  setMedia,
3581
+ setMicrophoneFacts,
3582
+ setAudioPlayback,
3583
+ registerMediaControls,
3294
3584
  reset
3295
3585
  ]
3296
3586
  );
@@ -3305,6 +3595,8 @@ function createProxyClient(options) {
3305
3595
  const base = normalize(options.proxyUrl);
3306
3596
  const doFetch = options.fetch ?? globalThis.fetch?.bind(globalThis);
3307
3597
  const timeoutMs = options.timeoutMs ?? PROXY_CLIENT_TIMEOUT_MS;
3598
+ const credentials = options.credentials;
3599
+ const withCredentials = credentials === void 0 ? {} : { credentials };
3308
3600
  const post = async (path, body, request) => {
3309
3601
  if (!doFetch) throw new Error("realtime-avatar: no fetch available \u2014 pass one via `fetch`.");
3310
3602
  const caller = request?.signal;
@@ -3312,6 +3604,7 @@ function createProxyClient(options) {
3312
3604
  try {
3313
3605
  return await doFetch(`${base}${path}`, {
3314
3606
  method: "POST",
3607
+ ...withCredentials,
3315
3608
  headers: { "content-type": "application/json", ...request?.headers ?? {} },
3316
3609
  body: JSON.stringify(body),
3317
3610
  signal: timer && caller ? AbortSignal.any([caller, timer]) : timer ?? caller
@@ -3321,18 +3614,38 @@ function createProxyClient(options) {
3321
3614
  throw cause;
3322
3615
  }
3323
3616
  };
3324
- const beacon = (path, body) => {
3617
+ const pageHide = (body) => {
3618
+ const page = globalThis.location?.href;
3619
+ let target;
3620
+ try {
3621
+ target = new URL(`${base}/end`, page);
3622
+ } catch {
3623
+ return false;
3624
+ }
3625
+ const payload = JSON.stringify(body);
3325
3626
  const send = globalThis.navigator?.sendBeacon?.bind(globalThis.navigator);
3326
- if (!send) return false;
3327
- return send(`${base}${path}`, new Blob([JSON.stringify(body)], { type: "application/json" }));
3627
+ const sameOrigin = page !== void 0 && target.origin === new URL(page).origin;
3628
+ if (send && (sameOrigin || credentials === "include")) {
3629
+ if (send(target.href, new Blob([payload], { type: "application/json" }))) return true;
3630
+ }
3631
+ if (!doFetch) return false;
3632
+ try {
3633
+ void doFetch(target.href, {
3634
+ method: "POST",
3635
+ keepalive: true,
3636
+ ...withCredentials,
3637
+ headers: { "content-type": "application/json" },
3638
+ body: payload
3639
+ }).catch(() => void 0);
3640
+ return true;
3641
+ } catch {
3642
+ return false;
3643
+ }
3328
3644
  };
3329
3645
  return {
3330
3646
  async createLiveKitSessionOrBusy(input, requestOptions) {
3331
- const response = await post(
3332
- "/connect",
3333
- { avatarId: input.avatarId, mode: input.mode },
3334
- requestOptions
3335
- );
3647
+ const connect = { avatarId: input.avatarId, mode: input.mode };
3648
+ const response = await post("/connect", connect, requestOptions);
3336
3649
  if (response.status === 429) {
3337
3650
  const value = await response.clone().json().catch(() => null);
3338
3651
  if (value && typeof value === "object" && !Array.isArray(value)) {
@@ -3350,7 +3663,8 @@ function createProxyClient(options) {
3350
3663
  async releaseLiveKitSession(sessionId, reason, requestOptions) {
3351
3664
  if (!sessionId) return false;
3352
3665
  try {
3353
- const response = await post("/end", { session_id: sessionId, reason }, requestOptions);
3666
+ const body = { session_id: sessionId, reason };
3667
+ const response = await post("/end", body, requestOptions);
3354
3668
  return response.ok;
3355
3669
  } catch {
3356
3670
  return false;
@@ -3358,12 +3672,13 @@ function createProxyClient(options) {
3358
3672
  },
3359
3673
  releaseLiveKitSessionBeacon(sessionId, reason) {
3360
3674
  if (!sessionId) return false;
3361
- return beacon("/end", { session_id: sessionId, reason: reason ?? "page_hide" });
3675
+ return pageHide({ session_id: sessionId, reason: reason ?? "page_hide" });
3362
3676
  },
3363
3677
  async releaseLiveKitQueueTicket(queueTicketId, reason, requestOptions) {
3364
3678
  if (!queueTicketId) return false;
3365
3679
  try {
3366
- const response = await post("/end", { queue_ticket_id: queueTicketId, reason }, requestOptions);
3680
+ const body = { queue_ticket_id: queueTicketId, reason };
3681
+ const response = await post("/end", body, requestOptions);
3367
3682
  return response.ok;
3368
3683
  } catch {
3369
3684
  return false;
@@ -3371,7 +3686,7 @@ function createProxyClient(options) {
3371
3686
  },
3372
3687
  releaseLiveKitQueueTicketBeacon(queueTicketId, reason) {
3373
3688
  if (!queueTicketId) return false;
3374
- return beacon("/end", { queue_ticket_id: queueTicketId, reason: reason ?? "page_hide" });
3689
+ return pageHide({ queue_ticket_id: queueTicketId, reason: reason ?? "page_hide" });
3375
3690
  }
3376
3691
  };
3377
3692
  }