realtime-avatar 0.24.1 → 0.26.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/react.js CHANGED
@@ -1,7 +1,7 @@
1
1
  "use client";
2
- import { useRef, useState, useEffect, useCallback, useMemo, createElement, Fragment } from 'react';
3
- import { useRoomContext, useConnectionState, useLocalParticipant, useTranscriptions, RoomAudioRenderer, LiveKitRoom, useMaybeRoomContext, useVoiceAssistant, VideoTrack, useChat } from '@livekit/components-react';
4
- import { ConnectionState, isRemoteTrack, VideoQuality, ConnectionQuality, RoomEvent, TrackEvent, Track, ConnectionError, ConnectionErrorReason, DisconnectReason, createLocalVideoTrack, Room } from 'livekit-client';
2
+ import { useRef, useMemo, useState, useEffect, useCallback, createElement, Fragment } from 'react';
3
+ import { useRoomContext, useConnectionState, useLocalParticipant, useTranscriptions, RoomAudioRenderer, LiveKitRoom, useMaybeRoomContext, useVoiceAssistant, VideoTrack, useChat, useAudioPlayback } from '@livekit/components-react';
4
+ import { ConnectionState, isRemoteTrack, VideoQuality, ConnectionQuality, RoomEvent, TrackEvent, Track, ConnectionError, ConnectionErrorReason, ParticipantEvent, DisconnectReason, createLocalVideoTrack, Room } from 'livekit-client';
5
5
  import { z } from 'zod';
6
6
  import { jsxs, jsx } from 'react/jsx-runtime';
7
7
 
@@ -27,14 +27,26 @@ function candidateIdentities(room) {
27
27
  const agents = all.filter((id) => /^agent[-_]/i.test(id));
28
28
  return [...agents, ...all.filter((id) => !agents.includes(id))];
29
29
  }
30
+ function rpcCode(error) {
31
+ return typeof error === "object" && error !== null && "code" in error ? error.code : void 0;
32
+ }
30
33
  function methodMissing(error) {
31
- const code = error?.code;
32
- if (code === "UNSUPPORTED_METHOD" || code === "RECIPIENT_NOT_FOUND" || code === 1401 || code === 1402) {
34
+ const code = rpcCode(error);
35
+ if (code === "UNSUPPORTED_METHOD" || code === "RECIPIENT_NOT_FOUND" || code === 1400 || code === 1401) {
33
36
  return true;
34
37
  }
35
38
  const message = error instanceof Error ? error.message : String(error ?? "");
36
39
  return /not supported|unsupported_method|recipient_not_found|recipient not found/i.test(message);
37
40
  }
41
+ var TRANSIENT_RPC_CODES = /* @__PURE__ */ new Set([1501, 1502, 1503, 1505]);
42
+ var ToolRegistrationError = class extends Error {
43
+ name = "ToolRegistrationError";
44
+ retryable;
45
+ constructor(message, options) {
46
+ super(message, options.cause === void 0 ? void 0 : { cause: options.cause });
47
+ this.retryable = options.retryable;
48
+ }
49
+ };
38
50
  var sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
39
51
  function buildManifest(tools) {
40
52
  const descriptors = [];
@@ -64,8 +76,9 @@ async function attachAvatarTools(room, tools, options = {}) {
64
76
  const { descriptors, rejected } = buildManifest(tools);
65
77
  const payload = JSON.stringify({ tools: descriptors });
66
78
  if (payload.length > MAX_MANIFEST_BYTES) {
67
- throw new Error(
68
- `tool manifest is ${payload.length} bytes, over the ${MAX_MANIFEST_BYTES} limit \u2014 shorten descriptions or register fewer tools`
79
+ throw new ToolRegistrationError(
80
+ `tool manifest is ${payload.length} bytes, over the ${MAX_MANIFEST_BYTES} limit \u2014 shorten descriptions or register fewer tools`,
81
+ { retryable: false }
69
82
  );
70
83
  }
71
84
  options.signal?.throwIfAborted();
@@ -148,14 +161,21 @@ async function attachAvatarTools(room, tools, options = {}) {
148
161
  } catch (error) {
149
162
  activeCaller = void 0;
150
163
  lastError = error;
151
- if (!methodMissing(error)) throw error;
164
+ if (!methodMissing(error)) {
165
+ if (options.signal?.aborted) throw error;
166
+ throw new ToolRegistrationError(
167
+ `tool registration failed: ${error instanceof Error ? error.message : String(error)}`,
168
+ { retryable: TRANSIENT_RPC_CODES.has(rpcCode(error)), cause: error }
169
+ );
170
+ }
152
171
  }
153
172
  }
154
173
  if (Date.now() >= deadline) break;
155
174
  await sleep(200);
156
175
  }
157
- throw new Error(
158
- sawCandidate ? `no participant accepted the tool manifest \u2014 either this session was not minted with the \`client_tools\` capability (a server-side grant), or the agent never armed registration within ${options.timeoutMs ?? 8e3}ms` + (lastError instanceof Error ? ` (last error: ${lastError.message})` : "") : "no remote participant joined the room before the timeout \u2014 nothing to register with"
176
+ throw new ToolRegistrationError(
177
+ sawCandidate ? `no participant accepted the tool manifest \u2014 either this session was not minted with the \`client_tools\` capability (a server-side grant), or the agent never armed registration within ${options.timeoutMs ?? 8e3}ms` + (lastError instanceof Error ? ` (last error: ${lastError.message})` : "") : "no remote participant joined the room before the timeout \u2014 nothing to register with",
178
+ { retryable: true, cause: lastError ?? void 0 }
159
179
  );
160
180
  } catch (error) {
161
181
  dispose();
@@ -164,11 +184,31 @@ async function attachAvatarTools(room, tools, options = {}) {
164
184
  }
165
185
 
166
186
  // ../client/src/react/use-character-tools.ts
187
+ var CHARACTER_TOOLS_RETRY_DELAYS_MS = [1e3, 3e3];
188
+ function manifestKey(tools) {
189
+ return JSON.stringify(Object.entries(tools).map(([name, tool]) => [name, tool.description, tool.parameters ?? null]));
190
+ }
167
191
  function useCharacterTools(tools, options = {}) {
168
192
  const room = useRoomContext();
169
193
  const connection = useConnectionState();
170
194
  const callback = useRef(options.onResult);
171
195
  callback.current = options.onResult;
196
+ const latest = useRef(tools);
197
+ latest.current = tools;
198
+ const key = manifestKey(tools);
199
+ const stable = useMemo(() => Object.fromEntries(
200
+ Object.entries(latest.current).map(([name, tool]) => [
201
+ name,
202
+ // A tool without a callable `execute` is passed as it is, so the manifest still rejects it.
203
+ typeof tool.execute !== "function" ? tool : {
204
+ description: tool.description,
205
+ parameters: tool.parameters,
206
+ execute: (args, context) => latest.current[name]?.execute(args, context)
207
+ }
208
+ ])
209
+ // `key` IS the dependency: it changes exactly when the manifest does.
210
+ // eslint-disable-next-line react-hooks/exhaustive-deps
211
+ ), [key]);
172
212
  const [state, setState] = useState({ status: "idle", registered: [], rejected: [] });
173
213
  useEffect(() => {
174
214
  if (connection !== ConnectionState.Connected) {
@@ -176,28 +216,42 @@ function useCharacterTools(tools, options = {}) {
176
216
  return;
177
217
  }
178
218
  const abort = new AbortController();
179
- setState({ status: "registering", registered: [], rejected: [] });
180
- void attachAvatarTools(room, tools, {
181
- signal: abort.signal,
182
- onResult: (event) => {
183
- if (!abort.signal.aborted) callback.current?.(event);
184
- }
185
- }).then((result) => {
186
- if (abort.signal.aborted) {
187
- result.dispose();
188
- return;
189
- }
190
- setState({ status: "ready", registered: result.accepted, rejected: result.rejected });
191
- }).catch((error) => {
192
- if (!abort.signal.aborted) setState({
193
- status: "error",
194
- registered: [],
195
- rejected: [],
196
- error: error instanceof Error ? error.message : "Tool registration failed"
219
+ let timer;
220
+ const register = (attempt) => {
221
+ setState({ status: "registering", registered: [], rejected: [], attempt });
222
+ void attachAvatarTools(room, stable, {
223
+ signal: abort.signal,
224
+ onResult: (event) => {
225
+ if (!abort.signal.aborted) callback.current?.(event);
226
+ }
227
+ }).then((result) => {
228
+ if (abort.signal.aborted) {
229
+ result.dispose();
230
+ return;
231
+ }
232
+ setState({ status: "ready", registered: result.accepted, rejected: result.rejected, attempt });
233
+ }).catch((error) => {
234
+ if (abort.signal.aborted) return;
235
+ const delay = CHARACTER_TOOLS_RETRY_DELAYS_MS[attempt - 1];
236
+ if (error instanceof ToolRegistrationError && error.retryable && delay !== void 0) {
237
+ timer = setTimeout(() => register(attempt + 1), delay);
238
+ return;
239
+ }
240
+ setState({
241
+ status: "error",
242
+ registered: [],
243
+ rejected: [],
244
+ attempt,
245
+ error: error instanceof Error ? error.message : "Tool registration failed"
246
+ });
197
247
  });
198
- });
199
- return () => abort.abort();
200
- }, [room, tools, connection]);
248
+ };
249
+ register(1);
250
+ return () => {
251
+ abort.abort();
252
+ if (timer !== void 0) clearTimeout(timer);
253
+ };
254
+ }, [room, stable, connection]);
201
255
  return state;
202
256
  }
203
257
 
@@ -568,7 +622,13 @@ function normalizeError(input, overrides) {
568
622
  };
569
623
  }
570
624
 
625
+ // ../http-client/src/errors.ts
626
+ function isRequestId(value) {
627
+ return typeof value === "string" && /^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/i.test(value);
628
+ }
629
+
571
630
  // ../client/src/errors.ts
631
+ var TIMEOUT_CODE = "upstream_timeout";
572
632
  var RealtimeAvatarApiError = class _RealtimeAvatarApiError extends Error {
573
633
  constructor(message, status, body, response, metadata = {}) {
574
634
  super(message);
@@ -580,6 +640,7 @@ var RealtimeAvatarApiError = class _RealtimeAvatarApiError extends Error {
580
640
  this.retryable = metadata.retryable;
581
641
  this.billingUrl = metadata.billingUrl;
582
642
  this.rawMessage = metadata.rawMessage ?? null;
643
+ this.requestId = metadata.requestId;
583
644
  }
584
645
  status;
585
646
  body;
@@ -589,6 +650,8 @@ var RealtimeAvatarApiError = class _RealtimeAvatarApiError extends Error {
589
650
  billingUrl;
590
651
  /** Debug detail returned by the API. Do not render directly in user-facing UI. */
591
652
  rawMessage;
653
+ /** The platform's correlation ID for this attempt. Quote it to support. */
654
+ requestId;
592
655
  get isBillingRequired() {
593
656
  return this.status === 402 || this.code === "insufficient_credits" || this.code === "spend_limit_exceeded";
594
657
  }
@@ -597,14 +660,31 @@ var RealtimeAvatarApiError = class _RealtimeAvatarApiError extends Error {
597
660
  const body = contentType.includes("application/json") ? await response.clone().json().catch(() => null) : await response.clone().text().catch(() => "");
598
661
  const rawMessage = (extractErrorMessage(body) ?? response.statusText) || null;
599
662
  const metadata = extractErrorMetadata(body, response.status, rawMessage);
663
+ const bodyRequestId = readRecord(body)?.requestId;
664
+ const headerRequestId = response.headers.get("x-request-id");
600
665
  return new _RealtimeAvatarApiError(
601
666
  metadata.message,
602
667
  response.status,
603
668
  body,
604
669
  response,
605
- metadata
670
+ {
671
+ ...metadata,
672
+ requestId: isRequestId(bodyRequestId) ? bodyRequestId : isRequestId(headerRequestId) ? headerRequestId : void 0
673
+ }
606
674
  );
607
675
  }
676
+ /**
677
+ * A request that got no answer before its deadline. The status is the one the vocabulary pairs
678
+ * with the code, so `normalizeRealtimeAvatarError` reads it back as the same failure.
679
+ */
680
+ static timeout(timeoutMs) {
681
+ const { message, retryable, statuses: [status] } = ERROR_SEMANTICS[TIMEOUT_CODE];
682
+ return new _RealtimeAvatarApiError(message, status, null, null, {
683
+ code: TIMEOUT_CODE,
684
+ retryable,
685
+ rawMessage: `No response within ${timeoutMs}ms.`
686
+ });
687
+ }
608
688
  };
609
689
  var RealtimeAvatarCapacityError = class extends Error {
610
690
  constructor(message, busy) {
@@ -826,6 +906,11 @@ function useLiveKitAvatarGrant(input) {
826
906
  });
827
907
  const stateRef = useRef(state);
828
908
  stateRef.current = state;
909
+ const wantedRef = useRef(active);
910
+ wantedRef.current = active;
911
+ useEffect(() => () => {
912
+ wantedRef.current = false;
913
+ }, []);
829
914
  const [tabVisible, setTabVisible] = useState(
830
915
  () => typeof document === "undefined" || document.visibilityState === "visible"
831
916
  );
@@ -866,6 +951,8 @@ function useLiveKitAvatarGrant(input) {
866
951
  if (cancelled) {
867
952
  if (result.status !== "busy") {
868
953
  void client.releaseLiveKitSession(result.grant.session_id, "superseded");
954
+ } else if (!wantedRef.current && result.busy.queue_ticket_id) {
955
+ void client.releaseLiveKitQueueTicket(result.busy.queue_ticket_id, "unmount");
869
956
  }
870
957
  return;
871
958
  }
@@ -990,13 +1077,16 @@ function RealtimeAvatarLiveKitRoom(props) {
990
1077
  serverUrl: grant?.livekit_url,
991
1078
  token: grant?.participant_token,
992
1079
  connect: shouldConnect,
993
- audio: audio ?? grant?.stt_mode === "server",
1080
+ audio: audio ?? publishesMicrophoneByDefault(grant),
994
1081
  video: cameraPublishOptions(grant, video),
995
1082
  options: roomOptions
996
1083
  },
997
1084
  createElement(Fragment, null, cushion, children, roomAudio)
998
1085
  );
999
1086
  }
1087
+ function publishesMicrophoneByDefault(grant) {
1088
+ return grant?.stt_mode === "server";
1089
+ }
1000
1090
  var capacityErrorFromBusy = (busy) => {
1001
1091
  return new RealtimeAvatarCapacityError(busy.message, busy);
1002
1092
  };
@@ -1728,16 +1818,16 @@ function AvatarVideoSurface(props) {
1728
1818
  !live || connectionState === ConnectionState.Disconnected
1729
1819
  );
1730
1820
  useLiveResumeOnProducing(liveWrapRef, trackProducing);
1731
- const fitClass = fit === "cover" ? AVATAR_VIDEO_FIT_COVER : AVATAR_VIDEO_FIT_CONTAIN;
1821
+ const fitStyle = fit === "cover" ? AVATAR_VIDEO_FIT_COVER : AVATAR_VIDEO_FIT_CONTAIN;
1732
1822
  const liveLabel = showLive && liveDims && liveDims.width > 0 && liveDims.height > 0 ? `${liveDims.width}\xD7${liveDims.height}` : null;
1733
1823
  const layers = resolveSurfaceLayers({ idleVideoUrl, poster });
1734
- const backFitClass = fitClass;
1824
+ const backFitStyle = fitStyle;
1735
1825
  const posterLayer = layers.showPoster ? createElement("img", {
1736
1826
  key: "poster",
1737
1827
  src: poster,
1738
1828
  alt: "",
1739
1829
  "aria-hidden": true,
1740
- className: joinClass(AVATAR_VIDEO_LAYER, backFitClass),
1830
+ style: backFitStyle,
1741
1831
  "data-testid": "avatar-poster"
1742
1832
  }) : null;
1743
1833
  const idleLayer = layers.showIdleVideo ? createElement("video", {
@@ -1755,7 +1845,7 @@ function AvatarVideoSurface(props) {
1755
1845
  playsInline: true,
1756
1846
  autoPlay: true,
1757
1847
  preload: "auto",
1758
- className: joinClass(AVATAR_VIDEO_LAYER, backFitClass),
1848
+ style: backFitStyle,
1759
1849
  "data-testid": "avatar-idle-video"
1760
1850
  }) : null;
1761
1851
  const frontLayer = createElement(
@@ -1763,17 +1853,17 @@ function AvatarVideoSurface(props) {
1763
1853
  {
1764
1854
  key: "live",
1765
1855
  ref: liveWrapRef,
1766
- className: joinClass(AVATAR_VIDEO_LIVE_WRAP, showLive ? void 0 : "pointer-events-none"),
1767
1856
  style: {
1857
+ ...AVATAR_VIDEO_LIVE_WRAP,
1768
1858
  opacity: showLive ? 1 : 0,
1769
- transitionDuration: "0ms"
1859
+ pointerEvents: showLive ? void 0 : "none"
1770
1860
  },
1771
1861
  "data-testid": "avatar-live-layer",
1772
1862
  "aria-hidden": !showLive
1773
1863
  },
1774
1864
  videoTrack ? createElement(VideoTrack, {
1775
1865
  trackRef: videoTrack,
1776
- className: joinClass(AVATAR_VIDEO_LAYER, fitClass),
1866
+ style: fitStyle,
1777
1867
  playsInline: true,
1778
1868
  autoPlay: true,
1779
1869
  muted: true,
@@ -1782,16 +1872,16 @@ function AvatarVideoSurface(props) {
1782
1872
  );
1783
1873
  const badge = showLiveBadge && showLive ? createElement(
1784
1874
  "span",
1785
- { key: "badge", className: AVATAR_VIDEO_BADGE },
1786
- createElement("span", { key: "dot", className: AVATAR_VIDEO_BADGE_DOT, "aria-hidden": true }),
1875
+ { key: "badge", style: AVATAR_VIDEO_BADGE },
1876
+ createElement("span", { key: "dot", style: AVATAR_VIDEO_BADGE_DOT, "aria-hidden": true }),
1787
1877
  liveLabel ? `live \xB7 ${liveLabel}` : "live"
1788
1878
  ) : null;
1789
1879
  return createElement(
1790
1880
  "div",
1791
1881
  {
1792
- className: joinClass(AVATAR_VIDEO_BOX, className),
1793
- style: boxAspect ? { aspectRatio: boxAspect, ...style } : style,
1794
- "data-testid": testId
1882
+ className,
1883
+ style: boxAspect ? { ...AVATAR_VIDEO_BOX, aspectRatio: boxAspect, ...style } : { ...AVATAR_VIDEO_BOX, ...style },
1884
+ "data-testid": testId ?? "avatar-video-surface"
1795
1885
  },
1796
1886
  // Deepest → shallowest: poster floor (the always-present face), the idle clip
1797
1887
  // over it, then the live video. The poster guarantees the surface is never a
@@ -1800,7 +1890,9 @@ function AvatarVideoSurface(props) {
1800
1890
  idleLayer,
1801
1891
  frontLayer,
1802
1892
  badge,
1803
- children
1893
+ // Above BOTH media layers and the badge. Appended bare, a consumer's positioned child with
1894
+ // no z-index painted under the z-20 live layer, which then took its clicks once live.
1895
+ children == null || children === false ? null : createElement("div", { key: "overlay", style: AVATAR_VIDEO_OVERLAY, "data-testid": "avatar-overlay" }, children)
1804
1896
  );
1805
1897
  }
1806
1898
  function resolveSurfaceLayers(input) {
@@ -2112,16 +2204,45 @@ function useLiveResumeOnProducing(wrapRef, producing) {
2112
2204
  };
2113
2205
  }, [wrapRef, producing]);
2114
2206
  }
2115
- function joinClass(...parts) {
2116
- return parts.filter(Boolean).join(" ");
2117
- }
2118
- var AVATAR_VIDEO_BOX = "relative size-full overflow-hidden";
2119
- var AVATAR_VIDEO_LAYER = "absolute inset-0 size-full transform-gpu [image-rendering:auto] [object-position:center_22%]";
2120
- var AVATAR_VIDEO_FIT_CONTAIN = "object-contain";
2121
- var AVATAR_VIDEO_FIT_COVER = "object-cover";
2122
- var AVATAR_VIDEO_LIVE_WRAP = "absolute inset-0 z-20 size-full transition-opacity ease-out";
2123
- var AVATAR_VIDEO_BADGE = "absolute top-2 right-2 z-30 inline-flex items-center gap-1.5 rounded-md bg-black/55 px-1.5 py-0.5 font-mono text-[10px] text-white/85 backdrop-blur-sm";
2124
- var AVATAR_VIDEO_BADGE_DOT = "size-1.5 rounded-full bg-emerald-400";
2207
+ var AVATAR_VIDEO_BOX = { position: "relative", width: "100%", height: "100%", overflow: "hidden", isolation: "isolate" };
2208
+ var AVATAR_VIDEO_LAYER = {
2209
+ position: "absolute",
2210
+ inset: 0,
2211
+ width: "100%",
2212
+ height: "100%",
2213
+ transform: "translateZ(0)",
2214
+ imageRendering: "auto",
2215
+ objectPosition: "center 22%"
2216
+ };
2217
+ var AVATAR_VIDEO_FIT_CONTAIN = { ...AVATAR_VIDEO_LAYER, objectFit: "contain" };
2218
+ var AVATAR_VIDEO_FIT_COVER = { ...AVATAR_VIDEO_LAYER, objectFit: "cover" };
2219
+ var AVATAR_VIDEO_LIVE_WRAP = {
2220
+ position: "absolute",
2221
+ inset: 0,
2222
+ zIndex: 20,
2223
+ width: "100%",
2224
+ height: "100%",
2225
+ transitionProperty: "opacity",
2226
+ transitionDuration: "0ms"
2227
+ };
2228
+ var AVATAR_VIDEO_BADGE = {
2229
+ position: "absolute",
2230
+ top: 8,
2231
+ right: 8,
2232
+ zIndex: 30,
2233
+ display: "inline-flex",
2234
+ alignItems: "center",
2235
+ gap: 6,
2236
+ borderRadius: 6,
2237
+ background: "rgb(0 0 0 / 0.55)",
2238
+ padding: "2px 6px",
2239
+ fontFamily: "ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, monospace",
2240
+ fontSize: 10,
2241
+ color: "rgb(255 255 255 / 0.85)",
2242
+ backdropFilter: "blur(8px)"
2243
+ };
2244
+ var AVATAR_VIDEO_BADGE_DOT = { width: 6, height: 6, borderRadius: 9999, background: "#34d399" };
2245
+ var AVATAR_VIDEO_OVERLAY = { position: "absolute", inset: 0, zIndex: 40 };
2125
2246
  var RECORDED_MEDIA_MODES = ["audio", "video", "audio_video"];
2126
2247
  var recordingModeSchema = z.enum(["off", ...RECORDED_MEDIA_MODES, "participants"]).describe("Server-owned recording policy; omitted means off. Legacy audio includes user and avatar audio. participants saves one continuous audio/video file per participant, without mixing their voices.");
2127
2248
  var RECORDING_STATUSES = ["pending", "recording", "processing", "ready", "failed", "expired"];
@@ -2655,6 +2776,169 @@ function createConnectionHistoryCollector(input) {
2655
2776
  return { enqueue, flush, dispose };
2656
2777
  }
2657
2778
 
2779
+ // ../browser/src/microphone.ts
2780
+ var HINTS = {
2781
+ "insecure-origin": "The browser only exposes microphones on a secure origin. Serve the page over https, or open it on http://localhost \u2014 a LAN address like http://192.168.1.5 will not do.",
2782
+ "no-answer": "The permission prompt is probably still open, or was dismissed without an answer. Look for it in the address bar, answer it, and try again.",
2783
+ "denied-by-browser": "This site is blocked from using the microphone. Click the icon at the left of the address bar, set Microphone to Allow, and reload.",
2784
+ "denied-by-os": "The operating system is blocking the browser itself, so no site can record. On macOS: System Settings > Privacy & Security > Microphone, enable your browser, then RESTART it \u2014 the change does not apply to a running browser.",
2785
+ "no-device": "No microphone is available. Connect one, and if you are on Bluetooth headphones check they are in a mode that has a microphone.",
2786
+ "device-in-use": "Another application or browser tab is holding the microphone. Close it \u2014 including any call from this page you did not hang up \u2014 and try again.",
2787
+ "device-lost": "The microphone stopped during the call: it was unplugged, or access to it was turned off. Reconnect it or allow access again, then turn the microphone back on.",
2788
+ unknown: "Try again, and if it persists include the message above in your report."
2789
+ };
2790
+ function isOperatingSystemDenial(message) {
2791
+ return /\bby system\b|system permission/i.test(message);
2792
+ }
2793
+ function classify(error) {
2794
+ if (error instanceof TypeError && /mediaDevices|undefined/i.test(error.message)) {
2795
+ return { reason: "insecure-origin", message: error.message };
2796
+ }
2797
+ const name = typeof error === "object" && error !== null && "name" in error ? String(error.name) : "";
2798
+ const message = typeof error === "object" && error !== null && "message" in error && error.message ? String(error.message) : String(error);
2799
+ switch (name) {
2800
+ case "NotAllowedError":
2801
+ case "PermissionDeniedError":
2802
+ return {
2803
+ reason: isOperatingSystemDenial(message) ? "denied-by-os" : "denied-by-browser",
2804
+ message
2805
+ };
2806
+ case "NotFoundError":
2807
+ case "DevicesNotFoundError":
2808
+ case "OverconstrainedError":
2809
+ case "ConstraintNotSatisfiedError":
2810
+ return { reason: "no-device", message };
2811
+ case "NotReadableError":
2812
+ case "TrackStartError":
2813
+ case "AbortError":
2814
+ return { reason: "device-in-use", message };
2815
+ case "SecurityError":
2816
+ return { reason: "insecure-origin", message };
2817
+ default:
2818
+ return { reason: "unknown", message };
2819
+ }
2820
+ }
2821
+ function describeMicrophoneFailure(error) {
2822
+ const { reason, message } = classify(error);
2823
+ return { reason, message, hint: HINTS[reason] };
2824
+ }
2825
+ function microphoneLost(message) {
2826
+ return { reason: "device-lost", message, hint: HINTS["device-lost"] };
2827
+ }
2828
+
2829
+ // ../client/src/react/call-media.ts
2830
+ var DEVICE_LOST_MESSAGE = "The microphone track ended during the call.";
2831
+ function callMicrophoneFrom(facts, intent) {
2832
+ if (!facts.wanted) return { status: "off" };
2833
+ const { publication } = facts;
2834
+ if (publication) {
2835
+ if (publication.ended) {
2836
+ return publication.muted ? { status: "unavailable", ...microphoneLost(DEVICE_LOST_MESSAGE) } : { status: "pending" };
2837
+ }
2838
+ return publication.muted ? { status: "muted" } : { status: "on" };
2839
+ }
2840
+ if (intent.muted) return { status: "muted" };
2841
+ if (facts.enabling) return { status: "pending" };
2842
+ const error = facts.deviceError;
2843
+ if (error) {
2844
+ const problem = describeMicrophoneFailure(error);
2845
+ const blocked = problem.reason === "denied-by-browser" || problem.reason === "denied-by-os" || problem.reason === "insecure-origin";
2846
+ return blocked ? { status: "blocked", ...problem } : { status: "unavailable", ...problem };
2847
+ }
2848
+ return facts.connected ? { status: "pending" } : { status: "off" };
2849
+ }
2850
+ function microphoneFacts(room, wanted, connected, enabling, staleError) {
2851
+ const publication = room.localParticipant.getTrackPublication(Track.Source.Microphone);
2852
+ const track = publication?.track;
2853
+ return {
2854
+ wanted,
2855
+ connected,
2856
+ enabling,
2857
+ publication: track ? { muted: publication.isMuted, ended: track.mediaStreamTrack?.readyState === "ended" } : null,
2858
+ deviceError: room.localParticipant.lastMicrophoneError !== staleError && room.localParticipant.lastMicrophoneError || null
2859
+ };
2860
+ }
2861
+ function useCallMedia(room, options) {
2862
+ const { setMicrophoneFacts, setAudioPlayback, registerMediaControls, microphoneWanted, connectionState, callKey } = options;
2863
+ const connected = connectionState === ConnectionState.Connected;
2864
+ const scope = useMemo(
2865
+ () => ({ room, callKey, stale: room.localParticipant.lastMicrophoneError }),
2866
+ [room, callKey]
2867
+ );
2868
+ const [enabling, setEnabling] = useState(false);
2869
+ const [version, setVersion] = useState(0);
2870
+ const sinksRef = useRef({ setMicrophoneFacts, setAudioPlayback });
2871
+ sinksRef.current = { setMicrophoneFacts, setAudioPlayback };
2872
+ useEffect(() => {
2873
+ if (!setMicrophoneFacts) return;
2874
+ const bump = () => setVersion((value) => value + 1);
2875
+ const participant = room.localParticipant;
2876
+ let watched = null;
2877
+ const watchTrack = () => {
2878
+ watched?.off();
2879
+ const track = participant.getTrackPublication(Track.Source.Microphone)?.track;
2880
+ if (!track) {
2881
+ watched = null;
2882
+ return;
2883
+ }
2884
+ track.on(TrackEvent.Ended, bump).on(TrackEvent.Restarted, bump).on(TrackEvent.Muted, bump).on(TrackEvent.Unmuted, bump);
2885
+ watched = { off: () => {
2886
+ track.off(TrackEvent.Ended, bump).off(TrackEvent.Restarted, bump).off(TrackEvent.Muted, bump).off(TrackEvent.Unmuted, bump);
2887
+ } };
2888
+ };
2889
+ const onPublications = () => {
2890
+ watchTrack();
2891
+ bump();
2892
+ };
2893
+ room.on(RoomEvent.MediaDevicesError, bump).on(RoomEvent.LocalTrackPublished, onPublications).on(RoomEvent.LocalTrackUnpublished, onPublications);
2894
+ participant.on(ParticipantEvent.TrackMuted, bump).on(ParticipantEvent.TrackUnmuted, bump);
2895
+ watchTrack();
2896
+ return () => {
2897
+ watched?.off();
2898
+ room.off(RoomEvent.MediaDevicesError, bump).off(RoomEvent.LocalTrackPublished, onPublications).off(RoomEvent.LocalTrackUnpublished, onPublications);
2899
+ participant.off(ParticipantEvent.TrackMuted, bump).off(ParticipantEvent.TrackUnmuted, bump);
2900
+ };
2901
+ }, [room, setMicrophoneFacts]);
2902
+ useEffect(() => {
2903
+ sinksRef.current.setMicrophoneFacts?.(microphoneFacts(room, microphoneWanted, connected, enabling, scope.stale));
2904
+ }, [room, microphoneWanted, connected, enabling, version, scope]);
2905
+ const { canPlayAudio } = useAudioPlayback(room);
2906
+ useEffect(() => {
2907
+ sinksRef.current.setAudioPlayback?.(!canPlayAudio ? "blocked" : connected ? "allowed" : "unknown");
2908
+ }, [canPlayAudio, connected]);
2909
+ const unlockTriedRef = useRef(null);
2910
+ const connecting = connectionState === ConnectionState.Connecting || connected;
2911
+ useEffect(() => {
2912
+ if (!setAudioPlayback || !connecting || unlockTriedRef.current === room) return;
2913
+ unlockTriedRef.current = room;
2914
+ void room.startAudio().catch(() => void 0);
2915
+ }, [room, setAudioPlayback, connecting]);
2916
+ const controls = useMemo(() => ({
2917
+ startAudio: async () => {
2918
+ try {
2919
+ await room.startAudio();
2920
+ } catch {
2921
+ }
2922
+ return room.canPlaybackAudio;
2923
+ },
2924
+ setMicrophoneEnabled: async (enabled) => {
2925
+ if (enabled) setEnabling(true);
2926
+ try {
2927
+ await room.localParticipant.setMicrophoneEnabled(enabled);
2928
+ } catch {
2929
+ } finally {
2930
+ if (enabled) setEnabling(false);
2931
+ setVersion((value) => value + 1);
2932
+ }
2933
+ }
2934
+ }), [room]);
2935
+ useEffect(() => {
2936
+ if (!registerMediaControls) return;
2937
+ registerMediaControls(controls);
2938
+ return () => registerMediaControls(null);
2939
+ }, [controls, registerMediaControls]);
2940
+ }
2941
+
2658
2942
  // ../client/src/react/session-lifecycle.ts
2659
2943
  function phaseForReason(reason) {
2660
2944
  switch (reason) {
@@ -2820,10 +3104,13 @@ function useSessionLifecycle(input) {
2820
3104
  const reconnectPolicyRef = useRef(reconnectPolicy);
2821
3105
  reconnectPolicyRef.current = reconnectPolicy;
2822
3106
  const connectWatchdogMs = typeof connectWatchdogSeconds === "number" && Number.isFinite(connectWatchdogSeconds) ? Math.max(0, Math.floor(connectWatchdogSeconds)) * 1e3 : DEFAULT_CONNECT_WATCHDOG_SECONDS * 1e3;
3107
+ const [userEnded, setUserEnded] = useState(false);
3108
+ const userEndedRef = useRef(false);
3109
+ const phaseRef = useRef({ kind: "idle" });
2823
3110
  const grantState = useLiveKitAvatarGrant({
2824
3111
  client,
2825
3112
  session,
2826
- active,
3113
+ active: active && !userEnded,
2827
3114
  autoRetryBusy,
2828
3115
  requestOptions
2829
3116
  });
@@ -2840,6 +3127,8 @@ function useSessionLifecycle(input) {
2840
3127
  refreshRef.current = grantState.refresh;
2841
3128
  const releaseRef = useRef(grantState.release);
2842
3129
  releaseRef.current = grantState.release;
3130
+ const clearRef = useRef(grantState.clear);
3131
+ clearRef.current = grantState.clear;
2843
3132
  const capacityRef = useRef(capacity);
2844
3133
  capacityRef.current = capacity;
2845
3134
  const refreshBaselineCapacityRef = useRef(null);
@@ -2866,13 +3155,16 @@ function useSessionLifecycle(input) {
2866
3155
  refreshBaselineCapacityRef.current = null;
2867
3156
  attemptRef.current = 0;
2868
3157
  setAttempt(0);
2869
- setRecovery({ kind: "connected" });
3158
+ setRecovery(userEndedRef.current ? { kind: "ended", reason: "user" } : { kind: "connected" });
2870
3159
  setConnected(false);
2871
3160
  setAgentPresentState(false);
2872
3161
  lastActivityRef.current = Date.now();
2873
3162
  }, [clearTimer]);
2874
3163
  useEffect(() => {
2875
- if (!active) reset();
3164
+ if (active) return;
3165
+ userEndedRef.current = false;
3166
+ setUserEnded(false);
3167
+ reset();
2876
3168
  }, [active, reset]);
2877
3169
  const markActivity = useCallback(() => {
2878
3170
  lastActivityRef.current = Date.now();
@@ -2903,6 +3195,7 @@ function useSessionLifecycle(input) {
2903
3195
  leaveRoomRef.current?.();
2904
3196
  }, [clearTimer]);
2905
3197
  const onConnected = useCallback(() => {
3198
+ if (userEndedRef.current) return;
2906
3199
  clearTimer();
2907
3200
  connectedRef.current = true;
2908
3201
  manualReconnectPendingRef.current = false;
@@ -2915,7 +3208,7 @@ function useSessionLifecycle(input) {
2915
3208
  }, [clearTimer]);
2916
3209
  const onDisconnected = useCallback(
2917
3210
  (reason) => {
2918
- if (!active) return;
3211
+ if (!active || userEndedRef.current) return;
2919
3212
  const action = disconnectAction(reason, connectedRef.current);
2920
3213
  if (action === "noop") return;
2921
3214
  if (action === "reset") {
@@ -2940,7 +3233,7 @@ function useSessionLifecycle(input) {
2940
3233
  );
2941
3234
  const onConnectionError = useCallback(
2942
3235
  (error) => {
2943
- if (!active || !isRecoverableConnectionError(error)) return;
3236
+ if (!active || userEndedRef.current || !isRecoverableConnectionError(error)) return;
2944
3237
  connectedRef.current = false;
2945
3238
  setConnected(false);
2946
3239
  setAgentPresentState(false);
@@ -2953,6 +3246,9 @@ function useSessionLifecycle(input) {
2953
3246
  },
2954
3247
  [active]
2955
3248
  );
3249
+ useEffect(() => {
3250
+ if (capacity.kind === "error" && recovery.kind === "connected") manualReconnectPendingRef.current = false;
3251
+ }, [capacity, recovery]);
2956
3252
  useEffect(() => {
2957
3253
  if (recovery.kind !== "refreshing" || capacity.kind !== "error") return;
2958
3254
  const baseline = refreshBaselineCapacityRef.current;
@@ -2962,7 +3258,7 @@ function useSessionLifecycle(input) {
2962
3258
  }, [capacity, recovery]);
2963
3259
  const onConnectionStateChange = useCallback(
2964
3260
  (state) => {
2965
- if (!active) return;
3261
+ if (!active || userEndedRef.current) return;
2966
3262
  if (state === "connected") {
2967
3263
  onConnected();
2968
3264
  return;
@@ -3010,8 +3306,30 @@ function useSessionLifecycle(input) {
3010
3306
  }, step2.delayMs);
3011
3307
  return clearTimer;
3012
3308
  }, [recovery.kind, active, attempt, clearTimer]);
3309
+ const end = useCallback(() => {
3310
+ if (!active || userEndedRef.current || phaseRef.current.kind === "ended") return;
3311
+ userEndedRef.current = true;
3312
+ clearTimer();
3313
+ connectedRef.current = false;
3314
+ manualReconnectPendingRef.current = false;
3315
+ refreshBaselineCapacityRef.current = null;
3316
+ setUserEnded(true);
3317
+ setConnected(false);
3318
+ setAgentPresentState(false);
3319
+ setRecovery({ kind: "ended", reason: "user" });
3320
+ clearRef.current();
3321
+ leaveRoomRef.current?.();
3322
+ }, [active, clearTimer]);
3013
3323
  const reconnect = useCallback(() => {
3014
- if (!active || manualReconnectPendingRef.current || recovery.kind === "in-place-reconnecting") return;
3324
+ if (!active) return;
3325
+ if (userEndedRef.current) {
3326
+ userEndedRef.current = false;
3327
+ setUserEnded(false);
3328
+ reset();
3329
+ manualReconnectPendingRef.current = true;
3330
+ return;
3331
+ }
3332
+ if (manualReconnectPendingRef.current || recovery.kind === "in-place-reconnecting") return;
3015
3333
  manualReconnectPendingRef.current = true;
3016
3334
  clearTimer();
3017
3335
  attemptRef.current = 0;
@@ -3020,7 +3338,7 @@ function useSessionLifecycle(input) {
3020
3338
  releaseRef.current("superseded");
3021
3339
  setRecovery({ kind: "refreshing", attempt: 0 });
3022
3340
  refreshRef.current();
3023
- }, [active, clearTimer, recovery.kind]);
3341
+ }, [active, clearTimer, recovery.kind, reset]);
3024
3342
  const endIdleRef = useRef(endIdle);
3025
3343
  endIdleRef.current = endIdle;
3026
3344
  useEffect(() => {
@@ -3052,6 +3370,7 @@ function useSessionLifecycle(input) {
3052
3370
  () => lifecyclePhaseFrom({ capacity, recovery, idle, connected, agentPresent }),
3053
3371
  [capacity, recovery, idle, connected, agentPresent]
3054
3372
  );
3373
+ phaseRef.current = phase;
3055
3374
  const onBehaviorChangeRef = useRef(onBehaviorChange);
3056
3375
  onBehaviorChangeRef.current = onBehaviorChange;
3057
3376
  const lastBehaviorRef = useRef(null);
@@ -3077,6 +3396,7 @@ function useSessionLifecycle(input) {
3077
3396
  stayConnected,
3078
3397
  markActivity,
3079
3398
  reconnect,
3399
+ end,
3080
3400
  onConnected,
3081
3401
  onDisconnected,
3082
3402
  onConnectionError,
@@ -3095,6 +3415,7 @@ function useSessionLifecycle(input) {
3095
3415
  stayConnected,
3096
3416
  markActivity,
3097
3417
  reconnect,
3418
+ end,
3098
3419
  onConnected,
3099
3420
  onDisconnected,
3100
3421
  onConnectionError,
@@ -3109,7 +3430,8 @@ function useSessionLifecycle(input) {
3109
3430
  function SessionLifecycleRoomBridge({
3110
3431
  lifecycle,
3111
3432
  onConnectionDetailsChange,
3112
- onMediaModeChange
3433
+ onMediaModeChange,
3434
+ microphone
3113
3435
  }) {
3114
3436
  const {
3115
3437
  onConnectionStateChange,
@@ -3120,12 +3442,23 @@ function SessionLifecycleRoomBridge({
3120
3442
  registerDataPublisher,
3121
3443
  registerTurnSender,
3122
3444
  setTurnState,
3123
- setMedia
3445
+ setMedia,
3446
+ setMicrophoneFacts,
3447
+ setAudioPlayback,
3448
+ registerMediaControls
3124
3449
  } = lifecycle;
3125
3450
  const connectionState = useConnectionState();
3126
3451
  const assistant = useVoiceAssistant();
3127
3452
  const transcriptions = useTranscriptions();
3128
3453
  const room = useRoomContext();
3454
+ useCallMedia(room, {
3455
+ setMicrophoneFacts,
3456
+ setAudioPlayback,
3457
+ registerMediaControls,
3458
+ microphoneWanted: microphone ?? publishesMicrophoneByDefault(lifecycle.grant),
3459
+ connectionState,
3460
+ callKey: lifecycle.grant?.session_id
3461
+ });
3129
3462
  const mediaModeCallbackRef = useRef(onMediaModeChange);
3130
3463
  mediaModeCallbackRef.current = onMediaModeChange;
3131
3464
  const mediaModeEnabled = onMediaModeChange !== void 0 && lifecycle.grant !== null;
@@ -3399,6 +3732,8 @@ function resolveEndReason(workerLabel, innerReason) {
3399
3732
  return "idle";
3400
3733
  case "error":
3401
3734
  return "failed";
3735
+ case "user":
3736
+ return "user_ended";
3402
3737
  default:
3403
3738
  return "disconnected";
3404
3739
  }
@@ -3454,6 +3789,16 @@ function useRealtimeSession(input) {
3454
3789
  const [turn, setTurn] = useState("quiet");
3455
3790
  const [media, setMediaState] = useState({ video: "connecting", audio: "silent" });
3456
3791
  const [behavior, setBehavior] = useState(null);
3792
+ const [microphoneFacts2, setMicrophoneFactsState] = useState({
3793
+ wanted: false,
3794
+ connected: false,
3795
+ enabling: false,
3796
+ publication: null,
3797
+ deviceError: null
3798
+ });
3799
+ const [microphoneMuted, setMicrophoneMuted] = useState(false);
3800
+ const [audioPlayback, setAudioPlaybackState] = useState("unknown");
3801
+ const mediaControlsRef = useRef(null);
3457
3802
  const lastBehaviorRef = useRef(null);
3458
3803
  const lastLabeledEndReasonRef = useRef(null);
3459
3804
  const pendingClipsRef = useRef(/* @__PURE__ */ new Map());
@@ -3560,6 +3905,31 @@ function useRealtimeSession(input) {
3560
3905
  const setMedia = useCallback((next) => {
3561
3906
  setMediaState((prev) => prev.video === next.video && prev.audio === next.audio ? prev : next);
3562
3907
  }, []);
3908
+ const microphoneFactsRef = useRef(microphoneFacts2);
3909
+ microphoneFactsRef.current = microphoneFacts2;
3910
+ const setMicrophoneFacts = useCallback((facts) => setMicrophoneFactsState(facts), []);
3911
+ const setAudioPlayback = useCallback((playback) => setAudioPlaybackState(playback), []);
3912
+ const registerMediaControls = useCallback((controls) => {
3913
+ mediaControlsRef.current = controls;
3914
+ }, []);
3915
+ const startAudio = useCallback(async () => {
3916
+ const controls = mediaControlsRef.current;
3917
+ return controls ? controls.startAudio() : false;
3918
+ }, []);
3919
+ const callEnded = lifecycle.phase.kind === "ended";
3920
+ const callEndedRef = useRef(callEnded);
3921
+ callEndedRef.current = callEnded;
3922
+ const setMicrophoneEnabled = useCallback(async (enabled) => {
3923
+ if (callEndedRef.current) return;
3924
+ setMicrophoneMuted(!enabled);
3925
+ if (!microphoneFactsRef.current.connected) return;
3926
+ await mediaControlsRef.current?.setMicrophoneEnabled(enabled);
3927
+ }, []);
3928
+ const microphone = useMemo(
3929
+ () => callEnded ? { status: "off" } : callMicrophoneFrom(microphoneFacts2, { muted: microphoneMuted }),
3930
+ [callEnded, microphoneFacts2, microphoneMuted]
3931
+ );
3932
+ const callAudioPlayback = callEnded ? "unknown" : audioPlayback;
3563
3933
  useEffect(() => {
3564
3934
  if (phaseKind === "idle" || phaseKind === "ended") return;
3565
3935
  const timer = window.setInterval(() => {
@@ -3704,11 +4074,23 @@ function useRealtimeSession(input) {
3704
4074
  void sendResolvedTurn(last.text, last.attributes, last.id).catch(() => void 0);
3705
4075
  }, [sendResolvedTurn]);
3706
4076
  const end = useCallback((reason) => {
4077
+ if (phaseKind === "ended") return;
3707
4078
  if (reason) lastLabeledEndReasonRef.current = reason;
3708
4079
  lastTurnRef.current = null;
3709
4080
  requestGracefulClose();
3710
- lifecycle.reset();
3711
- }, [lifecycle, requestGracefulClose]);
4081
+ lifecycle.end();
4082
+ }, [lifecycle, phaseKind, requestGracefulClose]);
4083
+ const active = inner.active ?? true;
4084
+ useEffect(() => {
4085
+ if (active) return;
4086
+ lastLabeledEndReasonRef.current = null;
4087
+ setMicrophoneMuted(false);
4088
+ }, [active]);
4089
+ const innerReconnect = lifecycle.reconnect;
4090
+ const reconnect = useCallback(() => {
4091
+ if (phaseKind === "ended") lastLabeledEndReasonRef.current = null;
4092
+ innerReconnect();
4093
+ }, [innerReconnect, phaseKind]);
3712
4094
  const performAction = useCallback(
3713
4095
  (actionId, opts) => {
3714
4096
  const requestId = newTurnId();
@@ -3754,6 +4136,7 @@ function useRealtimeSession(input) {
3754
4136
  approachingFiredRef.current = false;
3755
4137
  creditsLowFiredRef.current = false;
3756
4138
  lastTurnRef.current = null;
4139
+ setMicrophoneMuted(false);
3757
4140
  lifecycle.reset();
3758
4141
  }, [lifecycle]);
3759
4142
  return useMemo(
@@ -3767,6 +4150,12 @@ function useRealtimeSession(input) {
3767
4150
  endsAt,
3768
4151
  graceWindow,
3769
4152
  media,
4153
+ microphone,
4154
+ audioPlayback: callAudioPlayback,
4155
+ startAudio,
4156
+ setMicrophoneEnabled,
4157
+ microphoneMuted,
4158
+ reconnect,
3770
4159
  lifecycle,
3771
4160
  sendClosingTurn,
3772
4161
  requestGracefulClose,
@@ -3782,6 +4171,9 @@ function useRealtimeSession(input) {
3782
4171
  registerTurnSender,
3783
4172
  setTurnState,
3784
4173
  setMedia,
4174
+ setMicrophoneFacts,
4175
+ setAudioPlayback,
4176
+ registerMediaControls,
3785
4177
  reset
3786
4178
  }),
3787
4179
  [
@@ -3791,6 +4183,12 @@ function useRealtimeSession(input) {
3791
4183
  endsAt,
3792
4184
  graceWindow,
3793
4185
  media,
4186
+ microphone,
4187
+ callAudioPlayback,
4188
+ startAudio,
4189
+ setMicrophoneEnabled,
4190
+ microphoneMuted,
4191
+ reconnect,
3794
4192
  sendClosingTurn,
3795
4193
  requestGracefulClose,
3796
4194
  extend,
@@ -3805,6 +4203,9 @@ function useRealtimeSession(input) {
3805
4203
  registerTurnSender,
3806
4204
  setTurnState,
3807
4205
  setMedia,
4206
+ setMicrophoneFacts,
4207
+ setAudioPlayback,
4208
+ registerMediaControls,
3808
4209
  reset
3809
4210
  ]
3810
4211
  );
@@ -3837,9 +4238,31 @@ function handleFor(session) {
3837
4238
  session.sendClosingTurn(text);
3838
4239
  },
3839
4240
  keepAlive: session.stayConnected,
3840
- end: () => session.end("user_ended")
4241
+ end: () => session.end("user_ended"),
4242
+ microphone: session.microphone,
4243
+ audio: session.audioPlayback,
4244
+ startAudio: session.startAudio,
4245
+ setMicrophoneEnabled: session.setMicrophoneEnabled,
4246
+ retryMicrophone: () => session.setMicrophoneEnabled(true)
3841
4247
  };
3842
4248
  }
4249
+ var UNLOCK_REGION = {
4250
+ position: "absolute",
4251
+ top: 16,
4252
+ left: "50%",
4253
+ transform: "translateX(-50%)",
4254
+ zIndex: 1
4255
+ };
4256
+ var UNLOCK_BUTTON = {
4257
+ padding: "10px 16px",
4258
+ border: "none",
4259
+ borderRadius: 9999,
4260
+ background: "rgb(0 0 0 / 0.7)",
4261
+ color: "white",
4262
+ font: "inherit",
4263
+ fontSize: 14,
4264
+ cursor: "pointer"
4265
+ };
3843
4266
  function useAvatarCall(props) {
3844
4267
  const sessionRef = useRef(null);
3845
4268
  const session = useRealtimeSession({
@@ -3863,11 +4286,36 @@ function useAvatarCall(props) {
3863
4286
  lastStatusRef.current = call.status;
3864
4287
  onStatusChange?.(call.status);
3865
4288
  }, [call.status, onStatusChange]);
4289
+ const microphone = call.microphone;
4290
+ const problemKey = microphone.status === "blocked" || microphone.status === "unavailable" ? `${microphone.status}:${microphone.reason}:${microphone.message}` : null;
4291
+ const lastProblemRef = useRef(null);
4292
+ const onMicrophoneProblemRef = useRef(props.onMicrophoneProblem);
4293
+ onMicrophoneProblemRef.current = props.onMicrophoneProblem;
4294
+ useEffect(() => {
4295
+ if (problemKey === lastProblemRef.current) return;
4296
+ lastProblemRef.current = problemKey;
4297
+ if (microphone.status === "blocked" || microphone.status === "unavailable") onMicrophoneProblemRef.current?.(microphone);
4298
+ }, [problemKey]);
4299
+ const prompt = props.audioUnlockPrompt ?? true;
4300
+ const unlock = prompt === false ? null : createElement(
4301
+ "div",
4302
+ { key: "audio-unlock-region", role: "status", "aria-live": "polite", style: UNLOCK_REGION, "data-testid": "avatar-audio-unlock-region" },
4303
+ call.audio !== "blocked" ? null : typeof prompt === "function" ? prompt(call) : createElement(
4304
+ "button",
4305
+ { type: "button", style: UNLOCK_BUTTON, onClick: () => {
4306
+ void call.startAudio();
4307
+ }, "data-testid": "avatar-audio-unlock" },
4308
+ "Tap to turn on sound"
4309
+ )
4310
+ );
4311
+ const overlay = props.children ? props.children(call) : null;
3866
4312
  const view = createElement(
3867
4313
  RealtimeAvatarLiveKitRoom,
3868
4314
  {
3869
4315
  grant: session.grant,
3870
- audio: props.listen !== false,
4316
+ // LiveKit captures the microphone on connect from this value, so a mute chosen while
4317
+ // waiting is honoured rather than overridden when the call goes live.
4318
+ audio: props.listen !== false && !session.microphoneMuted,
3871
4319
  onConnected: session.onConnected,
3872
4320
  onDisconnected: session.onDisconnected,
3873
4321
  onError: session.onConnectionError
@@ -3875,6 +4323,7 @@ function useAvatarCall(props) {
3875
4323
  createElement(SessionLifecycleRoomBridge, {
3876
4324
  key: "bridge",
3877
4325
  lifecycle: session,
4326
+ microphone: props.listen !== false,
3878
4327
  onConnectionDetailsChange: props.onConnectionDetailsChange
3879
4328
  }),
3880
4329
  createElement(
@@ -3884,9 +4333,10 @@ function useAvatarCall(props) {
3884
4333
  idleVideoUrl: props.idleVideoUrl ?? null,
3885
4334
  poster: props.poster ?? null,
3886
4335
  fit: props.fit ?? "cover",
3887
- className: props.className
4336
+ className: props.className,
4337
+ style: props.style
3888
4338
  },
3889
- props.children ? props.children(call) : null
4339
+ overlay === null && unlock === null ? null : createElement(Fragment, null, overlay, unlock)
3890
4340
  )
3891
4341
  );
3892
4342
  return { call, view };
@@ -3896,39 +4346,66 @@ function AvatarCall(props) {
3896
4346
  return view;
3897
4347
  }
3898
4348
 
4349
+ // ../http-client/src/retry.ts
4350
+ var PROXY_CLIENT_TIMEOUT_MS = 6e4;
4351
+
3899
4352
  // ../client/src/proxy-client.ts
3900
- var DEFAULT_TIMEOUT_MS = 6e4;
3901
4353
  var normalize = (url) => url.replace(/\/+$/, "");
3902
4354
  function createProxyClient(options) {
3903
4355
  const base = normalize(options.proxyUrl);
3904
4356
  const doFetch = options.fetch ?? globalThis.fetch?.bind(globalThis);
3905
- const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
3906
- const deadline = (caller) => {
3907
- if (!timeoutMs) return caller;
3908
- const timer = AbortSignal.timeout(timeoutMs);
3909
- return caller ? AbortSignal.any([caller, timer]) : timer;
3910
- };
4357
+ const timeoutMs = options.timeoutMs ?? PROXY_CLIENT_TIMEOUT_MS;
4358
+ const credentials = options.credentials;
4359
+ const withCredentials = credentials === void 0 ? {} : { credentials };
3911
4360
  const post = async (path, body, request) => {
3912
4361
  if (!doFetch) throw new Error("realtime-avatar: no fetch available \u2014 pass one via `fetch`.");
3913
- return doFetch(`${base}${path}`, {
3914
- method: "POST",
3915
- headers: { "content-type": "application/json", ...request?.headers ?? {} },
3916
- body: JSON.stringify(body),
3917
- signal: deadline(request?.signal)
3918
- });
4362
+ const caller = request?.signal;
4363
+ const timer = timeoutMs ? AbortSignal.timeout(timeoutMs) : void 0;
4364
+ try {
4365
+ return await doFetch(`${base}${path}`, {
4366
+ method: "POST",
4367
+ ...withCredentials,
4368
+ headers: { "content-type": "application/json", ...request?.headers ?? {} },
4369
+ body: JSON.stringify(body),
4370
+ signal: timer && caller ? AbortSignal.any([caller, timer]) : timer ?? caller
4371
+ });
4372
+ } catch (cause) {
4373
+ if (timer?.aborted && !caller?.aborted) throw RealtimeAvatarApiError.timeout(timeoutMs);
4374
+ throw cause;
4375
+ }
3919
4376
  };
3920
- const beacon = (path, body) => {
4377
+ const pageHide = (body) => {
4378
+ const page = globalThis.location?.href;
4379
+ let target;
4380
+ try {
4381
+ target = new URL(`${base}/end`, page);
4382
+ } catch {
4383
+ return false;
4384
+ }
4385
+ const payload = JSON.stringify(body);
3921
4386
  const send = globalThis.navigator?.sendBeacon?.bind(globalThis.navigator);
3922
- if (!send) return false;
3923
- return send(`${base}${path}`, new Blob([JSON.stringify(body)], { type: "application/json" }));
4387
+ const sameOrigin = page !== void 0 && target.origin === new URL(page).origin;
4388
+ if (send && (sameOrigin || credentials === "include")) {
4389
+ if (send(target.href, new Blob([payload], { type: "application/json" }))) return true;
4390
+ }
4391
+ if (!doFetch) return false;
4392
+ try {
4393
+ void doFetch(target.href, {
4394
+ method: "POST",
4395
+ keepalive: true,
4396
+ ...withCredentials,
4397
+ headers: { "content-type": "application/json" },
4398
+ body: payload
4399
+ }).catch(() => void 0);
4400
+ return true;
4401
+ } catch {
4402
+ return false;
4403
+ }
3924
4404
  };
3925
4405
  return {
3926
4406
  async createLiveKitSessionOrBusy(input, requestOptions) {
3927
- const response = await post(
3928
- "/connect",
3929
- { avatarId: input.avatarId, mode: input.mode },
3930
- requestOptions
3931
- );
4407
+ const connect = { avatarId: input.avatarId, mode: input.mode };
4408
+ const response = await post("/connect", connect, requestOptions);
3932
4409
  if (response.status === 429) {
3933
4410
  const value = await response.clone().json().catch(() => null);
3934
4411
  if (value && typeof value === "object" && !Array.isArray(value)) {
@@ -3946,7 +4423,8 @@ function createProxyClient(options) {
3946
4423
  async releaseLiveKitSession(sessionId, reason, requestOptions) {
3947
4424
  if (!sessionId) return false;
3948
4425
  try {
3949
- const response = await post("/end", { session_id: sessionId, reason }, requestOptions);
4426
+ const body = { session_id: sessionId, reason };
4427
+ const response = await post("/end", body, requestOptions);
3950
4428
  return response.ok;
3951
4429
  } catch {
3952
4430
  return false;
@@ -3954,12 +4432,13 @@ function createProxyClient(options) {
3954
4432
  },
3955
4433
  releaseLiveKitSessionBeacon(sessionId, reason) {
3956
4434
  if (!sessionId) return false;
3957
- return beacon("/end", { session_id: sessionId, reason: reason ?? "page_hide" });
4435
+ return pageHide({ session_id: sessionId, reason: reason ?? "page_hide" });
3958
4436
  },
3959
4437
  async releaseLiveKitQueueTicket(queueTicketId, reason, requestOptions) {
3960
4438
  if (!queueTicketId) return false;
3961
4439
  try {
3962
- const response = await post("/end", { queue_ticket_id: queueTicketId, reason }, requestOptions);
4440
+ const body = { queue_ticket_id: queueTicketId, reason };
4441
+ const response = await post("/end", body, requestOptions);
3963
4442
  return response.ok;
3964
4443
  } catch {
3965
4444
  return false;
@@ -3967,7 +4446,7 @@ function createProxyClient(options) {
3967
4446
  },
3968
4447
  releaseLiveKitQueueTicketBeacon(queueTicketId, reason) {
3969
4448
  if (!queueTicketId) return false;
3970
- return beacon("/end", { queue_ticket_id: queueTicketId, reason: reason ?? "page_hide" });
4449
+ return pageHide({ queue_ticket_id: queueTicketId, reason: reason ?? "page_hide" });
3971
4450
  }
3972
4451
  };
3973
4452
  }