realtime-avatar 0.24.1 → 0.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +78 -0
- package/dist/browser.d.ts +2 -87
- package/dist/browser.js +6 -1
- package/dist/{chunk-EODL5S3A.js → chunk-HMESE54B.js} +46 -10
- package/dist/chunk-XJPGQMEW.js +161 -0
- package/dist/express.d.ts +1 -1
- package/dist/express.js +5 -3
- package/dist/hono.d.ts +1 -1
- package/dist/hono.js +2 -2
- package/dist/index.d.ts +24 -1
- package/dist/index.js +1 -1
- package/dist/nextjs.d.ts +1 -1
- package/dist/nextjs.js +2 -2
- package/dist/playout-delay-CtRZdtsj.d.ts +168 -0
- package/dist/{proxy-client-DlFXaXtn.d.ts → proxy-client-C8M_G_Kk.d.ts} +168 -18
- package/dist/react-native.d.ts +3 -3
- package/dist/react-native.js +385 -40
- package/dist/react.d.ts +75 -14
- package/dist/react.js +576 -97
- package/dist/server.js +1 -1
- package/dist/tanstack-start.d.ts +1 -1
- package/dist/tanstack-start.js +2 -2
- package/dist/tools.d.ts +17 -1
- package/dist/tools.js +28 -8
- package/dist/{types-CpqMHna3.d.ts → types-C7B7jOx6.d.ts} +7 -0
- package/package.json +1 -1
- package/dist/chunk-4PWWEERK.js +0 -87
- package/dist/playout-delay-DWOc8iCl.d.ts +0 -65
package/dist/react.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
"use client";
|
|
2
|
-
import { useRef, useState, useEffect, useCallback,
|
|
3
|
-
import { useRoomContext, useConnectionState, useLocalParticipant, useTranscriptions, RoomAudioRenderer, LiveKitRoom, useMaybeRoomContext, useVoiceAssistant, VideoTrack, useChat } from '@livekit/components-react';
|
|
4
|
-
import { ConnectionState, isRemoteTrack, VideoQuality, ConnectionQuality, RoomEvent, TrackEvent, Track, ConnectionError, ConnectionErrorReason, DisconnectReason, createLocalVideoTrack, Room } from 'livekit-client';
|
|
2
|
+
import { useRef, useMemo, useState, useEffect, useCallback, createElement, Fragment } from 'react';
|
|
3
|
+
import { useRoomContext, useConnectionState, useLocalParticipant, useTranscriptions, RoomAudioRenderer, LiveKitRoom, useMaybeRoomContext, useVoiceAssistant, VideoTrack, useChat, useAudioPlayback } from '@livekit/components-react';
|
|
4
|
+
import { ConnectionState, isRemoteTrack, VideoQuality, ConnectionQuality, RoomEvent, TrackEvent, Track, ConnectionError, ConnectionErrorReason, ParticipantEvent, DisconnectReason, createLocalVideoTrack, Room } from 'livekit-client';
|
|
5
5
|
import { z } from 'zod';
|
|
6
6
|
import { jsxs, jsx } from 'react/jsx-runtime';
|
|
7
7
|
|
|
@@ -27,14 +27,26 @@ function candidateIdentities(room) {
|
|
|
27
27
|
const agents = all.filter((id) => /^agent[-_]/i.test(id));
|
|
28
28
|
return [...agents, ...all.filter((id) => !agents.includes(id))];
|
|
29
29
|
}
|
|
30
|
+
function rpcCode(error) {
|
|
31
|
+
return typeof error === "object" && error !== null && "code" in error ? error.code : void 0;
|
|
32
|
+
}
|
|
30
33
|
function methodMissing(error) {
|
|
31
|
-
const code = error
|
|
32
|
-
if (code === "UNSUPPORTED_METHOD" || code === "RECIPIENT_NOT_FOUND" || code ===
|
|
34
|
+
const code = rpcCode(error);
|
|
35
|
+
if (code === "UNSUPPORTED_METHOD" || code === "RECIPIENT_NOT_FOUND" || code === 1400 || code === 1401) {
|
|
33
36
|
return true;
|
|
34
37
|
}
|
|
35
38
|
const message = error instanceof Error ? error.message : String(error ?? "");
|
|
36
39
|
return /not supported|unsupported_method|recipient_not_found|recipient not found/i.test(message);
|
|
37
40
|
}
|
|
41
|
+
var TRANSIENT_RPC_CODES = /* @__PURE__ */ new Set([1501, 1502, 1503, 1505]);
|
|
42
|
+
var ToolRegistrationError = class extends Error {
|
|
43
|
+
name = "ToolRegistrationError";
|
|
44
|
+
retryable;
|
|
45
|
+
constructor(message, options) {
|
|
46
|
+
super(message, options.cause === void 0 ? void 0 : { cause: options.cause });
|
|
47
|
+
this.retryable = options.retryable;
|
|
48
|
+
}
|
|
49
|
+
};
|
|
38
50
|
var sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
39
51
|
function buildManifest(tools) {
|
|
40
52
|
const descriptors = [];
|
|
@@ -64,8 +76,9 @@ async function attachAvatarTools(room, tools, options = {}) {
|
|
|
64
76
|
const { descriptors, rejected } = buildManifest(tools);
|
|
65
77
|
const payload = JSON.stringify({ tools: descriptors });
|
|
66
78
|
if (payload.length > MAX_MANIFEST_BYTES) {
|
|
67
|
-
throw new
|
|
68
|
-
`tool manifest is ${payload.length} bytes, over the ${MAX_MANIFEST_BYTES} limit \u2014 shorten descriptions or register fewer tools
|
|
79
|
+
throw new ToolRegistrationError(
|
|
80
|
+
`tool manifest is ${payload.length} bytes, over the ${MAX_MANIFEST_BYTES} limit \u2014 shorten descriptions or register fewer tools`,
|
|
81
|
+
{ retryable: false }
|
|
69
82
|
);
|
|
70
83
|
}
|
|
71
84
|
options.signal?.throwIfAborted();
|
|
@@ -148,14 +161,21 @@ async function attachAvatarTools(room, tools, options = {}) {
|
|
|
148
161
|
} catch (error) {
|
|
149
162
|
activeCaller = void 0;
|
|
150
163
|
lastError = error;
|
|
151
|
-
if (!methodMissing(error))
|
|
164
|
+
if (!methodMissing(error)) {
|
|
165
|
+
if (options.signal?.aborted) throw error;
|
|
166
|
+
throw new ToolRegistrationError(
|
|
167
|
+
`tool registration failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
168
|
+
{ retryable: TRANSIENT_RPC_CODES.has(rpcCode(error)), cause: error }
|
|
169
|
+
);
|
|
170
|
+
}
|
|
152
171
|
}
|
|
153
172
|
}
|
|
154
173
|
if (Date.now() >= deadline) break;
|
|
155
174
|
await sleep(200);
|
|
156
175
|
}
|
|
157
|
-
throw new
|
|
158
|
-
sawCandidate ? `no participant accepted the tool manifest \u2014 either this session was not minted with the \`client_tools\` capability (a server-side grant), or the agent never armed registration within ${options.timeoutMs ?? 8e3}ms` + (lastError instanceof Error ? ` (last error: ${lastError.message})` : "") : "no remote participant joined the room before the timeout \u2014 nothing to register with"
|
|
176
|
+
throw new ToolRegistrationError(
|
|
177
|
+
sawCandidate ? `no participant accepted the tool manifest \u2014 either this session was not minted with the \`client_tools\` capability (a server-side grant), or the agent never armed registration within ${options.timeoutMs ?? 8e3}ms` + (lastError instanceof Error ? ` (last error: ${lastError.message})` : "") : "no remote participant joined the room before the timeout \u2014 nothing to register with",
|
|
178
|
+
{ retryable: true, cause: lastError ?? void 0 }
|
|
159
179
|
);
|
|
160
180
|
} catch (error) {
|
|
161
181
|
dispose();
|
|
@@ -164,11 +184,31 @@ async function attachAvatarTools(room, tools, options = {}) {
|
|
|
164
184
|
}
|
|
165
185
|
|
|
166
186
|
// ../client/src/react/use-character-tools.ts
|
|
187
|
+
var CHARACTER_TOOLS_RETRY_DELAYS_MS = [1e3, 3e3];
|
|
188
|
+
function manifestKey(tools) {
|
|
189
|
+
return JSON.stringify(Object.entries(tools).map(([name, tool]) => [name, tool.description, tool.parameters ?? null]));
|
|
190
|
+
}
|
|
167
191
|
function useCharacterTools(tools, options = {}) {
|
|
168
192
|
const room = useRoomContext();
|
|
169
193
|
const connection = useConnectionState();
|
|
170
194
|
const callback = useRef(options.onResult);
|
|
171
195
|
callback.current = options.onResult;
|
|
196
|
+
const latest = useRef(tools);
|
|
197
|
+
latest.current = tools;
|
|
198
|
+
const key = manifestKey(tools);
|
|
199
|
+
const stable = useMemo(() => Object.fromEntries(
|
|
200
|
+
Object.entries(latest.current).map(([name, tool]) => [
|
|
201
|
+
name,
|
|
202
|
+
// A tool without a callable `execute` is passed as it is, so the manifest still rejects it.
|
|
203
|
+
typeof tool.execute !== "function" ? tool : {
|
|
204
|
+
description: tool.description,
|
|
205
|
+
parameters: tool.parameters,
|
|
206
|
+
execute: (args, context) => latest.current[name]?.execute(args, context)
|
|
207
|
+
}
|
|
208
|
+
])
|
|
209
|
+
// `key` IS the dependency: it changes exactly when the manifest does.
|
|
210
|
+
// eslint-disable-next-line react-hooks/exhaustive-deps
|
|
211
|
+
), [key]);
|
|
172
212
|
const [state, setState] = useState({ status: "idle", registered: [], rejected: [] });
|
|
173
213
|
useEffect(() => {
|
|
174
214
|
if (connection !== ConnectionState.Connected) {
|
|
@@ -176,28 +216,42 @@ function useCharacterTools(tools, options = {}) {
|
|
|
176
216
|
return;
|
|
177
217
|
}
|
|
178
218
|
const abort = new AbortController();
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
219
|
+
let timer;
|
|
220
|
+
const register = (attempt) => {
|
|
221
|
+
setState({ status: "registering", registered: [], rejected: [], attempt });
|
|
222
|
+
void attachAvatarTools(room, stable, {
|
|
223
|
+
signal: abort.signal,
|
|
224
|
+
onResult: (event) => {
|
|
225
|
+
if (!abort.signal.aborted) callback.current?.(event);
|
|
226
|
+
}
|
|
227
|
+
}).then((result) => {
|
|
228
|
+
if (abort.signal.aborted) {
|
|
229
|
+
result.dispose();
|
|
230
|
+
return;
|
|
231
|
+
}
|
|
232
|
+
setState({ status: "ready", registered: result.accepted, rejected: result.rejected, attempt });
|
|
233
|
+
}).catch((error) => {
|
|
234
|
+
if (abort.signal.aborted) return;
|
|
235
|
+
const delay = CHARACTER_TOOLS_RETRY_DELAYS_MS[attempt - 1];
|
|
236
|
+
if (error instanceof ToolRegistrationError && error.retryable && delay !== void 0) {
|
|
237
|
+
timer = setTimeout(() => register(attempt + 1), delay);
|
|
238
|
+
return;
|
|
239
|
+
}
|
|
240
|
+
setState({
|
|
241
|
+
status: "error",
|
|
242
|
+
registered: [],
|
|
243
|
+
rejected: [],
|
|
244
|
+
attempt,
|
|
245
|
+
error: error instanceof Error ? error.message : "Tool registration failed"
|
|
246
|
+
});
|
|
197
247
|
});
|
|
198
|
-
}
|
|
199
|
-
|
|
200
|
-
|
|
248
|
+
};
|
|
249
|
+
register(1);
|
|
250
|
+
return () => {
|
|
251
|
+
abort.abort();
|
|
252
|
+
if (timer !== void 0) clearTimeout(timer);
|
|
253
|
+
};
|
|
254
|
+
}, [room, stable, connection]);
|
|
201
255
|
return state;
|
|
202
256
|
}
|
|
203
257
|
|
|
@@ -568,7 +622,13 @@ function normalizeError(input, overrides) {
|
|
|
568
622
|
};
|
|
569
623
|
}
|
|
570
624
|
|
|
625
|
+
// ../http-client/src/errors.ts
|
|
626
|
+
function isRequestId(value) {
|
|
627
|
+
return typeof value === "string" && /^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/i.test(value);
|
|
628
|
+
}
|
|
629
|
+
|
|
571
630
|
// ../client/src/errors.ts
|
|
631
|
+
var TIMEOUT_CODE = "upstream_timeout";
|
|
572
632
|
var RealtimeAvatarApiError = class _RealtimeAvatarApiError extends Error {
|
|
573
633
|
constructor(message, status, body, response, metadata = {}) {
|
|
574
634
|
super(message);
|
|
@@ -580,6 +640,7 @@ var RealtimeAvatarApiError = class _RealtimeAvatarApiError extends Error {
|
|
|
580
640
|
this.retryable = metadata.retryable;
|
|
581
641
|
this.billingUrl = metadata.billingUrl;
|
|
582
642
|
this.rawMessage = metadata.rawMessage ?? null;
|
|
643
|
+
this.requestId = metadata.requestId;
|
|
583
644
|
}
|
|
584
645
|
status;
|
|
585
646
|
body;
|
|
@@ -589,6 +650,8 @@ var RealtimeAvatarApiError = class _RealtimeAvatarApiError extends Error {
|
|
|
589
650
|
billingUrl;
|
|
590
651
|
/** Debug detail returned by the API. Do not render directly in user-facing UI. */
|
|
591
652
|
rawMessage;
|
|
653
|
+
/** The platform's correlation ID for this attempt. Quote it to support. */
|
|
654
|
+
requestId;
|
|
592
655
|
get isBillingRequired() {
|
|
593
656
|
return this.status === 402 || this.code === "insufficient_credits" || this.code === "spend_limit_exceeded";
|
|
594
657
|
}
|
|
@@ -597,14 +660,31 @@ var RealtimeAvatarApiError = class _RealtimeAvatarApiError extends Error {
|
|
|
597
660
|
const body = contentType.includes("application/json") ? await response.clone().json().catch(() => null) : await response.clone().text().catch(() => "");
|
|
598
661
|
const rawMessage = (extractErrorMessage(body) ?? response.statusText) || null;
|
|
599
662
|
const metadata = extractErrorMetadata(body, response.status, rawMessage);
|
|
663
|
+
const bodyRequestId = readRecord(body)?.requestId;
|
|
664
|
+
const headerRequestId = response.headers.get("x-request-id");
|
|
600
665
|
return new _RealtimeAvatarApiError(
|
|
601
666
|
metadata.message,
|
|
602
667
|
response.status,
|
|
603
668
|
body,
|
|
604
669
|
response,
|
|
605
|
-
|
|
670
|
+
{
|
|
671
|
+
...metadata,
|
|
672
|
+
requestId: isRequestId(bodyRequestId) ? bodyRequestId : isRequestId(headerRequestId) ? headerRequestId : void 0
|
|
673
|
+
}
|
|
606
674
|
);
|
|
607
675
|
}
|
|
676
|
+
/**
|
|
677
|
+
* A request that got no answer before its deadline. The status is the one the vocabulary pairs
|
|
678
|
+
* with the code, so `normalizeRealtimeAvatarError` reads it back as the same failure.
|
|
679
|
+
*/
|
|
680
|
+
static timeout(timeoutMs) {
|
|
681
|
+
const { message, retryable, statuses: [status] } = ERROR_SEMANTICS[TIMEOUT_CODE];
|
|
682
|
+
return new _RealtimeAvatarApiError(message, status, null, null, {
|
|
683
|
+
code: TIMEOUT_CODE,
|
|
684
|
+
retryable,
|
|
685
|
+
rawMessage: `No response within ${timeoutMs}ms.`
|
|
686
|
+
});
|
|
687
|
+
}
|
|
608
688
|
};
|
|
609
689
|
var RealtimeAvatarCapacityError = class extends Error {
|
|
610
690
|
constructor(message, busy) {
|
|
@@ -826,6 +906,11 @@ function useLiveKitAvatarGrant(input) {
|
|
|
826
906
|
});
|
|
827
907
|
const stateRef = useRef(state);
|
|
828
908
|
stateRef.current = state;
|
|
909
|
+
const wantedRef = useRef(active);
|
|
910
|
+
wantedRef.current = active;
|
|
911
|
+
useEffect(() => () => {
|
|
912
|
+
wantedRef.current = false;
|
|
913
|
+
}, []);
|
|
829
914
|
const [tabVisible, setTabVisible] = useState(
|
|
830
915
|
() => typeof document === "undefined" || document.visibilityState === "visible"
|
|
831
916
|
);
|
|
@@ -866,6 +951,8 @@ function useLiveKitAvatarGrant(input) {
|
|
|
866
951
|
if (cancelled) {
|
|
867
952
|
if (result.status !== "busy") {
|
|
868
953
|
void client.releaseLiveKitSession(result.grant.session_id, "superseded");
|
|
954
|
+
} else if (!wantedRef.current && result.busy.queue_ticket_id) {
|
|
955
|
+
void client.releaseLiveKitQueueTicket(result.busy.queue_ticket_id, "unmount");
|
|
869
956
|
}
|
|
870
957
|
return;
|
|
871
958
|
}
|
|
@@ -990,13 +1077,16 @@ function RealtimeAvatarLiveKitRoom(props) {
|
|
|
990
1077
|
serverUrl: grant?.livekit_url,
|
|
991
1078
|
token: grant?.participant_token,
|
|
992
1079
|
connect: shouldConnect,
|
|
993
|
-
audio: audio ?? grant
|
|
1080
|
+
audio: audio ?? publishesMicrophoneByDefault(grant),
|
|
994
1081
|
video: cameraPublishOptions(grant, video),
|
|
995
1082
|
options: roomOptions
|
|
996
1083
|
},
|
|
997
1084
|
createElement(Fragment, null, cushion, children, roomAudio)
|
|
998
1085
|
);
|
|
999
1086
|
}
|
|
1087
|
+
function publishesMicrophoneByDefault(grant) {
|
|
1088
|
+
return grant?.stt_mode === "server";
|
|
1089
|
+
}
|
|
1000
1090
|
var capacityErrorFromBusy = (busy) => {
|
|
1001
1091
|
return new RealtimeAvatarCapacityError(busy.message, busy);
|
|
1002
1092
|
};
|
|
@@ -1728,16 +1818,16 @@ function AvatarVideoSurface(props) {
|
|
|
1728
1818
|
!live || connectionState === ConnectionState.Disconnected
|
|
1729
1819
|
);
|
|
1730
1820
|
useLiveResumeOnProducing(liveWrapRef, trackProducing);
|
|
1731
|
-
const
|
|
1821
|
+
const fitStyle = fit === "cover" ? AVATAR_VIDEO_FIT_COVER : AVATAR_VIDEO_FIT_CONTAIN;
|
|
1732
1822
|
const liveLabel = showLive && liveDims && liveDims.width > 0 && liveDims.height > 0 ? `${liveDims.width}\xD7${liveDims.height}` : null;
|
|
1733
1823
|
const layers = resolveSurfaceLayers({ idleVideoUrl, poster });
|
|
1734
|
-
const
|
|
1824
|
+
const backFitStyle = fitStyle;
|
|
1735
1825
|
const posterLayer = layers.showPoster ? createElement("img", {
|
|
1736
1826
|
key: "poster",
|
|
1737
1827
|
src: poster,
|
|
1738
1828
|
alt: "",
|
|
1739
1829
|
"aria-hidden": true,
|
|
1740
|
-
|
|
1830
|
+
style: backFitStyle,
|
|
1741
1831
|
"data-testid": "avatar-poster"
|
|
1742
1832
|
}) : null;
|
|
1743
1833
|
const idleLayer = layers.showIdleVideo ? createElement("video", {
|
|
@@ -1755,7 +1845,7 @@ function AvatarVideoSurface(props) {
|
|
|
1755
1845
|
playsInline: true,
|
|
1756
1846
|
autoPlay: true,
|
|
1757
1847
|
preload: "auto",
|
|
1758
|
-
|
|
1848
|
+
style: backFitStyle,
|
|
1759
1849
|
"data-testid": "avatar-idle-video"
|
|
1760
1850
|
}) : null;
|
|
1761
1851
|
const frontLayer = createElement(
|
|
@@ -1763,17 +1853,17 @@ function AvatarVideoSurface(props) {
|
|
|
1763
1853
|
{
|
|
1764
1854
|
key: "live",
|
|
1765
1855
|
ref: liveWrapRef,
|
|
1766
|
-
className: joinClass(AVATAR_VIDEO_LIVE_WRAP, showLive ? void 0 : "pointer-events-none"),
|
|
1767
1856
|
style: {
|
|
1857
|
+
...AVATAR_VIDEO_LIVE_WRAP,
|
|
1768
1858
|
opacity: showLive ? 1 : 0,
|
|
1769
|
-
|
|
1859
|
+
pointerEvents: showLive ? void 0 : "none"
|
|
1770
1860
|
},
|
|
1771
1861
|
"data-testid": "avatar-live-layer",
|
|
1772
1862
|
"aria-hidden": !showLive
|
|
1773
1863
|
},
|
|
1774
1864
|
videoTrack ? createElement(VideoTrack, {
|
|
1775
1865
|
trackRef: videoTrack,
|
|
1776
|
-
|
|
1866
|
+
style: fitStyle,
|
|
1777
1867
|
playsInline: true,
|
|
1778
1868
|
autoPlay: true,
|
|
1779
1869
|
muted: true,
|
|
@@ -1782,16 +1872,16 @@ function AvatarVideoSurface(props) {
|
|
|
1782
1872
|
);
|
|
1783
1873
|
const badge = showLiveBadge && showLive ? createElement(
|
|
1784
1874
|
"span",
|
|
1785
|
-
{ key: "badge",
|
|
1786
|
-
createElement("span", { key: "dot",
|
|
1875
|
+
{ key: "badge", style: AVATAR_VIDEO_BADGE },
|
|
1876
|
+
createElement("span", { key: "dot", style: AVATAR_VIDEO_BADGE_DOT, "aria-hidden": true }),
|
|
1787
1877
|
liveLabel ? `live \xB7 ${liveLabel}` : "live"
|
|
1788
1878
|
) : null;
|
|
1789
1879
|
return createElement(
|
|
1790
1880
|
"div",
|
|
1791
1881
|
{
|
|
1792
|
-
className
|
|
1793
|
-
style: boxAspect ? { aspectRatio: boxAspect, ...style } : style,
|
|
1794
|
-
"data-testid": testId
|
|
1882
|
+
className,
|
|
1883
|
+
style: boxAspect ? { ...AVATAR_VIDEO_BOX, aspectRatio: boxAspect, ...style } : { ...AVATAR_VIDEO_BOX, ...style },
|
|
1884
|
+
"data-testid": testId ?? "avatar-video-surface"
|
|
1795
1885
|
},
|
|
1796
1886
|
// Deepest → shallowest: poster floor (the always-present face), the idle clip
|
|
1797
1887
|
// over it, then the live video. The poster guarantees the surface is never a
|
|
@@ -1800,7 +1890,9 @@ function AvatarVideoSurface(props) {
|
|
|
1800
1890
|
idleLayer,
|
|
1801
1891
|
frontLayer,
|
|
1802
1892
|
badge,
|
|
1803
|
-
|
|
1893
|
+
// Above BOTH media layers and the badge. Appended bare, a consumer's positioned child with
|
|
1894
|
+
// no z-index painted under the z-20 live layer, which then took its clicks once live.
|
|
1895
|
+
children == null || children === false ? null : createElement("div", { key: "overlay", style: AVATAR_VIDEO_OVERLAY, "data-testid": "avatar-overlay" }, children)
|
|
1804
1896
|
);
|
|
1805
1897
|
}
|
|
1806
1898
|
function resolveSurfaceLayers(input) {
|
|
@@ -2112,16 +2204,45 @@ function useLiveResumeOnProducing(wrapRef, producing) {
|
|
|
2112
2204
|
};
|
|
2113
2205
|
}, [wrapRef, producing]);
|
|
2114
2206
|
}
|
|
2115
|
-
|
|
2116
|
-
|
|
2117
|
-
|
|
2118
|
-
|
|
2119
|
-
|
|
2120
|
-
|
|
2121
|
-
|
|
2122
|
-
|
|
2123
|
-
|
|
2124
|
-
|
|
2207
|
+
var AVATAR_VIDEO_BOX = { position: "relative", width: "100%", height: "100%", overflow: "hidden", isolation: "isolate" };
|
|
2208
|
+
var AVATAR_VIDEO_LAYER = {
|
|
2209
|
+
position: "absolute",
|
|
2210
|
+
inset: 0,
|
|
2211
|
+
width: "100%",
|
|
2212
|
+
height: "100%",
|
|
2213
|
+
transform: "translateZ(0)",
|
|
2214
|
+
imageRendering: "auto",
|
|
2215
|
+
objectPosition: "center 22%"
|
|
2216
|
+
};
|
|
2217
|
+
var AVATAR_VIDEO_FIT_CONTAIN = { ...AVATAR_VIDEO_LAYER, objectFit: "contain" };
|
|
2218
|
+
var AVATAR_VIDEO_FIT_COVER = { ...AVATAR_VIDEO_LAYER, objectFit: "cover" };
|
|
2219
|
+
var AVATAR_VIDEO_LIVE_WRAP = {
|
|
2220
|
+
position: "absolute",
|
|
2221
|
+
inset: 0,
|
|
2222
|
+
zIndex: 20,
|
|
2223
|
+
width: "100%",
|
|
2224
|
+
height: "100%",
|
|
2225
|
+
transitionProperty: "opacity",
|
|
2226
|
+
transitionDuration: "0ms"
|
|
2227
|
+
};
|
|
2228
|
+
var AVATAR_VIDEO_BADGE = {
|
|
2229
|
+
position: "absolute",
|
|
2230
|
+
top: 8,
|
|
2231
|
+
right: 8,
|
|
2232
|
+
zIndex: 30,
|
|
2233
|
+
display: "inline-flex",
|
|
2234
|
+
alignItems: "center",
|
|
2235
|
+
gap: 6,
|
|
2236
|
+
borderRadius: 6,
|
|
2237
|
+
background: "rgb(0 0 0 / 0.55)",
|
|
2238
|
+
padding: "2px 6px",
|
|
2239
|
+
fontFamily: "ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, monospace",
|
|
2240
|
+
fontSize: 10,
|
|
2241
|
+
color: "rgb(255 255 255 / 0.85)",
|
|
2242
|
+
backdropFilter: "blur(8px)"
|
|
2243
|
+
};
|
|
2244
|
+
var AVATAR_VIDEO_BADGE_DOT = { width: 6, height: 6, borderRadius: 9999, background: "#34d399" };
|
|
2245
|
+
var AVATAR_VIDEO_OVERLAY = { position: "absolute", inset: 0, zIndex: 40 };
|
|
2125
2246
|
var RECORDED_MEDIA_MODES = ["audio", "video", "audio_video"];
|
|
2126
2247
|
var recordingModeSchema = z.enum(["off", ...RECORDED_MEDIA_MODES, "participants"]).describe("Server-owned recording policy; omitted means off. Legacy audio includes user and avatar audio. participants saves one continuous audio/video file per participant, without mixing their voices.");
|
|
2127
2248
|
var RECORDING_STATUSES = ["pending", "recording", "processing", "ready", "failed", "expired"];
|
|
@@ -2655,6 +2776,169 @@ function createConnectionHistoryCollector(input) {
|
|
|
2655
2776
|
return { enqueue, flush, dispose };
|
|
2656
2777
|
}
|
|
2657
2778
|
|
|
2779
|
+
// ../browser/src/microphone.ts
|
|
2780
|
+
var HINTS = {
|
|
2781
|
+
"insecure-origin": "The browser only exposes microphones on a secure origin. Serve the page over https, or open it on http://localhost \u2014 a LAN address like http://192.168.1.5 will not do.",
|
|
2782
|
+
"no-answer": "The permission prompt is probably still open, or was dismissed without an answer. Look for it in the address bar, answer it, and try again.",
|
|
2783
|
+
"denied-by-browser": "This site is blocked from using the microphone. Click the icon at the left of the address bar, set Microphone to Allow, and reload.",
|
|
2784
|
+
"denied-by-os": "The operating system is blocking the browser itself, so no site can record. On macOS: System Settings > Privacy & Security > Microphone, enable your browser, then RESTART it \u2014 the change does not apply to a running browser.",
|
|
2785
|
+
"no-device": "No microphone is available. Connect one, and if you are on Bluetooth headphones check they are in a mode that has a microphone.",
|
|
2786
|
+
"device-in-use": "Another application or browser tab is holding the microphone. Close it \u2014 including any call from this page you did not hang up \u2014 and try again.",
|
|
2787
|
+
"device-lost": "The microphone stopped during the call: it was unplugged, or access to it was turned off. Reconnect it or allow access again, then turn the microphone back on.",
|
|
2788
|
+
unknown: "Try again, and if it persists include the message above in your report."
|
|
2789
|
+
};
|
|
2790
|
+
function isOperatingSystemDenial(message) {
|
|
2791
|
+
return /\bby system\b|system permission/i.test(message);
|
|
2792
|
+
}
|
|
2793
|
+
function classify(error) {
|
|
2794
|
+
if (error instanceof TypeError && /mediaDevices|undefined/i.test(error.message)) {
|
|
2795
|
+
return { reason: "insecure-origin", message: error.message };
|
|
2796
|
+
}
|
|
2797
|
+
const name = typeof error === "object" && error !== null && "name" in error ? String(error.name) : "";
|
|
2798
|
+
const message = typeof error === "object" && error !== null && "message" in error && error.message ? String(error.message) : String(error);
|
|
2799
|
+
switch (name) {
|
|
2800
|
+
case "NotAllowedError":
|
|
2801
|
+
case "PermissionDeniedError":
|
|
2802
|
+
return {
|
|
2803
|
+
reason: isOperatingSystemDenial(message) ? "denied-by-os" : "denied-by-browser",
|
|
2804
|
+
message
|
|
2805
|
+
};
|
|
2806
|
+
case "NotFoundError":
|
|
2807
|
+
case "DevicesNotFoundError":
|
|
2808
|
+
case "OverconstrainedError":
|
|
2809
|
+
case "ConstraintNotSatisfiedError":
|
|
2810
|
+
return { reason: "no-device", message };
|
|
2811
|
+
case "NotReadableError":
|
|
2812
|
+
case "TrackStartError":
|
|
2813
|
+
case "AbortError":
|
|
2814
|
+
return { reason: "device-in-use", message };
|
|
2815
|
+
case "SecurityError":
|
|
2816
|
+
return { reason: "insecure-origin", message };
|
|
2817
|
+
default:
|
|
2818
|
+
return { reason: "unknown", message };
|
|
2819
|
+
}
|
|
2820
|
+
}
|
|
2821
|
+
function describeMicrophoneFailure(error) {
|
|
2822
|
+
const { reason, message } = classify(error);
|
|
2823
|
+
return { reason, message, hint: HINTS[reason] };
|
|
2824
|
+
}
|
|
2825
|
+
function microphoneLost(message) {
|
|
2826
|
+
return { reason: "device-lost", message, hint: HINTS["device-lost"] };
|
|
2827
|
+
}
|
|
2828
|
+
|
|
2829
|
+
// ../client/src/react/call-media.ts
|
|
2830
|
+
var DEVICE_LOST_MESSAGE = "The microphone track ended during the call.";
|
|
2831
|
+
function callMicrophoneFrom(facts, intent) {
|
|
2832
|
+
if (!facts.wanted) return { status: "off" };
|
|
2833
|
+
const { publication } = facts;
|
|
2834
|
+
if (publication) {
|
|
2835
|
+
if (publication.ended) {
|
|
2836
|
+
return publication.muted ? { status: "unavailable", ...microphoneLost(DEVICE_LOST_MESSAGE) } : { status: "pending" };
|
|
2837
|
+
}
|
|
2838
|
+
return publication.muted ? { status: "muted" } : { status: "on" };
|
|
2839
|
+
}
|
|
2840
|
+
if (intent.muted) return { status: "muted" };
|
|
2841
|
+
if (facts.enabling) return { status: "pending" };
|
|
2842
|
+
const error = facts.deviceError;
|
|
2843
|
+
if (error) {
|
|
2844
|
+
const problem = describeMicrophoneFailure(error);
|
|
2845
|
+
const blocked = problem.reason === "denied-by-browser" || problem.reason === "denied-by-os" || problem.reason === "insecure-origin";
|
|
2846
|
+
return blocked ? { status: "blocked", ...problem } : { status: "unavailable", ...problem };
|
|
2847
|
+
}
|
|
2848
|
+
return facts.connected ? { status: "pending" } : { status: "off" };
|
|
2849
|
+
}
|
|
2850
|
+
function microphoneFacts(room, wanted, connected, enabling, staleError) {
|
|
2851
|
+
const publication = room.localParticipant.getTrackPublication(Track.Source.Microphone);
|
|
2852
|
+
const track = publication?.track;
|
|
2853
|
+
return {
|
|
2854
|
+
wanted,
|
|
2855
|
+
connected,
|
|
2856
|
+
enabling,
|
|
2857
|
+
publication: track ? { muted: publication.isMuted, ended: track.mediaStreamTrack?.readyState === "ended" } : null,
|
|
2858
|
+
deviceError: room.localParticipant.lastMicrophoneError !== staleError && room.localParticipant.lastMicrophoneError || null
|
|
2859
|
+
};
|
|
2860
|
+
}
|
|
2861
|
+
function useCallMedia(room, options) {
|
|
2862
|
+
const { setMicrophoneFacts, setAudioPlayback, registerMediaControls, microphoneWanted, connectionState, callKey } = options;
|
|
2863
|
+
const connected = connectionState === ConnectionState.Connected;
|
|
2864
|
+
const scope = useMemo(
|
|
2865
|
+
() => ({ room, callKey, stale: room.localParticipant.lastMicrophoneError }),
|
|
2866
|
+
[room, callKey]
|
|
2867
|
+
);
|
|
2868
|
+
const [enabling, setEnabling] = useState(false);
|
|
2869
|
+
const [version, setVersion] = useState(0);
|
|
2870
|
+
const sinksRef = useRef({ setMicrophoneFacts, setAudioPlayback });
|
|
2871
|
+
sinksRef.current = { setMicrophoneFacts, setAudioPlayback };
|
|
2872
|
+
useEffect(() => {
|
|
2873
|
+
if (!setMicrophoneFacts) return;
|
|
2874
|
+
const bump = () => setVersion((value) => value + 1);
|
|
2875
|
+
const participant = room.localParticipant;
|
|
2876
|
+
let watched = null;
|
|
2877
|
+
const watchTrack = () => {
|
|
2878
|
+
watched?.off();
|
|
2879
|
+
const track = participant.getTrackPublication(Track.Source.Microphone)?.track;
|
|
2880
|
+
if (!track) {
|
|
2881
|
+
watched = null;
|
|
2882
|
+
return;
|
|
2883
|
+
}
|
|
2884
|
+
track.on(TrackEvent.Ended, bump).on(TrackEvent.Restarted, bump).on(TrackEvent.Muted, bump).on(TrackEvent.Unmuted, bump);
|
|
2885
|
+
watched = { off: () => {
|
|
2886
|
+
track.off(TrackEvent.Ended, bump).off(TrackEvent.Restarted, bump).off(TrackEvent.Muted, bump).off(TrackEvent.Unmuted, bump);
|
|
2887
|
+
} };
|
|
2888
|
+
};
|
|
2889
|
+
const onPublications = () => {
|
|
2890
|
+
watchTrack();
|
|
2891
|
+
bump();
|
|
2892
|
+
};
|
|
2893
|
+
room.on(RoomEvent.MediaDevicesError, bump).on(RoomEvent.LocalTrackPublished, onPublications).on(RoomEvent.LocalTrackUnpublished, onPublications);
|
|
2894
|
+
participant.on(ParticipantEvent.TrackMuted, bump).on(ParticipantEvent.TrackUnmuted, bump);
|
|
2895
|
+
watchTrack();
|
|
2896
|
+
return () => {
|
|
2897
|
+
watched?.off();
|
|
2898
|
+
room.off(RoomEvent.MediaDevicesError, bump).off(RoomEvent.LocalTrackPublished, onPublications).off(RoomEvent.LocalTrackUnpublished, onPublications);
|
|
2899
|
+
participant.off(ParticipantEvent.TrackMuted, bump).off(ParticipantEvent.TrackUnmuted, bump);
|
|
2900
|
+
};
|
|
2901
|
+
}, [room, setMicrophoneFacts]);
|
|
2902
|
+
useEffect(() => {
|
|
2903
|
+
sinksRef.current.setMicrophoneFacts?.(microphoneFacts(room, microphoneWanted, connected, enabling, scope.stale));
|
|
2904
|
+
}, [room, microphoneWanted, connected, enabling, version, scope]);
|
|
2905
|
+
const { canPlayAudio } = useAudioPlayback(room);
|
|
2906
|
+
useEffect(() => {
|
|
2907
|
+
sinksRef.current.setAudioPlayback?.(!canPlayAudio ? "blocked" : connected ? "allowed" : "unknown");
|
|
2908
|
+
}, [canPlayAudio, connected]);
|
|
2909
|
+
const unlockTriedRef = useRef(null);
|
|
2910
|
+
const connecting = connectionState === ConnectionState.Connecting || connected;
|
|
2911
|
+
useEffect(() => {
|
|
2912
|
+
if (!setAudioPlayback || !connecting || unlockTriedRef.current === room) return;
|
|
2913
|
+
unlockTriedRef.current = room;
|
|
2914
|
+
void room.startAudio().catch(() => void 0);
|
|
2915
|
+
}, [room, setAudioPlayback, connecting]);
|
|
2916
|
+
const controls = useMemo(() => ({
|
|
2917
|
+
startAudio: async () => {
|
|
2918
|
+
try {
|
|
2919
|
+
await room.startAudio();
|
|
2920
|
+
} catch {
|
|
2921
|
+
}
|
|
2922
|
+
return room.canPlaybackAudio;
|
|
2923
|
+
},
|
|
2924
|
+
setMicrophoneEnabled: async (enabled) => {
|
|
2925
|
+
if (enabled) setEnabling(true);
|
|
2926
|
+
try {
|
|
2927
|
+
await room.localParticipant.setMicrophoneEnabled(enabled);
|
|
2928
|
+
} catch {
|
|
2929
|
+
} finally {
|
|
2930
|
+
if (enabled) setEnabling(false);
|
|
2931
|
+
setVersion((value) => value + 1);
|
|
2932
|
+
}
|
|
2933
|
+
}
|
|
2934
|
+
}), [room]);
|
|
2935
|
+
useEffect(() => {
|
|
2936
|
+
if (!registerMediaControls) return;
|
|
2937
|
+
registerMediaControls(controls);
|
|
2938
|
+
return () => registerMediaControls(null);
|
|
2939
|
+
}, [controls, registerMediaControls]);
|
|
2940
|
+
}
|
|
2941
|
+
|
|
2658
2942
|
// ../client/src/react/session-lifecycle.ts
|
|
2659
2943
|
function phaseForReason(reason) {
|
|
2660
2944
|
switch (reason) {
|
|
@@ -2820,10 +3104,13 @@ function useSessionLifecycle(input) {
|
|
|
2820
3104
|
const reconnectPolicyRef = useRef(reconnectPolicy);
|
|
2821
3105
|
reconnectPolicyRef.current = reconnectPolicy;
|
|
2822
3106
|
const connectWatchdogMs = typeof connectWatchdogSeconds === "number" && Number.isFinite(connectWatchdogSeconds) ? Math.max(0, Math.floor(connectWatchdogSeconds)) * 1e3 : DEFAULT_CONNECT_WATCHDOG_SECONDS * 1e3;
|
|
3107
|
+
const [userEnded, setUserEnded] = useState(false);
|
|
3108
|
+
const userEndedRef = useRef(false);
|
|
3109
|
+
const phaseRef = useRef({ kind: "idle" });
|
|
2823
3110
|
const grantState = useLiveKitAvatarGrant({
|
|
2824
3111
|
client,
|
|
2825
3112
|
session,
|
|
2826
|
-
active,
|
|
3113
|
+
active: active && !userEnded,
|
|
2827
3114
|
autoRetryBusy,
|
|
2828
3115
|
requestOptions
|
|
2829
3116
|
});
|
|
@@ -2840,6 +3127,8 @@ function useSessionLifecycle(input) {
|
|
|
2840
3127
|
refreshRef.current = grantState.refresh;
|
|
2841
3128
|
const releaseRef = useRef(grantState.release);
|
|
2842
3129
|
releaseRef.current = grantState.release;
|
|
3130
|
+
const clearRef = useRef(grantState.clear);
|
|
3131
|
+
clearRef.current = grantState.clear;
|
|
2843
3132
|
const capacityRef = useRef(capacity);
|
|
2844
3133
|
capacityRef.current = capacity;
|
|
2845
3134
|
const refreshBaselineCapacityRef = useRef(null);
|
|
@@ -2866,13 +3155,16 @@ function useSessionLifecycle(input) {
|
|
|
2866
3155
|
refreshBaselineCapacityRef.current = null;
|
|
2867
3156
|
attemptRef.current = 0;
|
|
2868
3157
|
setAttempt(0);
|
|
2869
|
-
setRecovery({ kind: "connected" });
|
|
3158
|
+
setRecovery(userEndedRef.current ? { kind: "ended", reason: "user" } : { kind: "connected" });
|
|
2870
3159
|
setConnected(false);
|
|
2871
3160
|
setAgentPresentState(false);
|
|
2872
3161
|
lastActivityRef.current = Date.now();
|
|
2873
3162
|
}, [clearTimer]);
|
|
2874
3163
|
useEffect(() => {
|
|
2875
|
-
if (
|
|
3164
|
+
if (active) return;
|
|
3165
|
+
userEndedRef.current = false;
|
|
3166
|
+
setUserEnded(false);
|
|
3167
|
+
reset();
|
|
2876
3168
|
}, [active, reset]);
|
|
2877
3169
|
const markActivity = useCallback(() => {
|
|
2878
3170
|
lastActivityRef.current = Date.now();
|
|
@@ -2903,6 +3195,7 @@ function useSessionLifecycle(input) {
|
|
|
2903
3195
|
leaveRoomRef.current?.();
|
|
2904
3196
|
}, [clearTimer]);
|
|
2905
3197
|
const onConnected = useCallback(() => {
|
|
3198
|
+
if (userEndedRef.current) return;
|
|
2906
3199
|
clearTimer();
|
|
2907
3200
|
connectedRef.current = true;
|
|
2908
3201
|
manualReconnectPendingRef.current = false;
|
|
@@ -2915,7 +3208,7 @@ function useSessionLifecycle(input) {
|
|
|
2915
3208
|
}, [clearTimer]);
|
|
2916
3209
|
const onDisconnected = useCallback(
|
|
2917
3210
|
(reason) => {
|
|
2918
|
-
if (!active) return;
|
|
3211
|
+
if (!active || userEndedRef.current) return;
|
|
2919
3212
|
const action = disconnectAction(reason, connectedRef.current);
|
|
2920
3213
|
if (action === "noop") return;
|
|
2921
3214
|
if (action === "reset") {
|
|
@@ -2940,7 +3233,7 @@ function useSessionLifecycle(input) {
|
|
|
2940
3233
|
);
|
|
2941
3234
|
const onConnectionError = useCallback(
|
|
2942
3235
|
(error) => {
|
|
2943
|
-
if (!active || !isRecoverableConnectionError(error)) return;
|
|
3236
|
+
if (!active || userEndedRef.current || !isRecoverableConnectionError(error)) return;
|
|
2944
3237
|
connectedRef.current = false;
|
|
2945
3238
|
setConnected(false);
|
|
2946
3239
|
setAgentPresentState(false);
|
|
@@ -2953,6 +3246,9 @@ function useSessionLifecycle(input) {
|
|
|
2953
3246
|
},
|
|
2954
3247
|
[active]
|
|
2955
3248
|
);
|
|
3249
|
+
useEffect(() => {
|
|
3250
|
+
if (capacity.kind === "error" && recovery.kind === "connected") manualReconnectPendingRef.current = false;
|
|
3251
|
+
}, [capacity, recovery]);
|
|
2956
3252
|
useEffect(() => {
|
|
2957
3253
|
if (recovery.kind !== "refreshing" || capacity.kind !== "error") return;
|
|
2958
3254
|
const baseline = refreshBaselineCapacityRef.current;
|
|
@@ -2962,7 +3258,7 @@ function useSessionLifecycle(input) {
|
|
|
2962
3258
|
}, [capacity, recovery]);
|
|
2963
3259
|
const onConnectionStateChange = useCallback(
|
|
2964
3260
|
(state) => {
|
|
2965
|
-
if (!active) return;
|
|
3261
|
+
if (!active || userEndedRef.current) return;
|
|
2966
3262
|
if (state === "connected") {
|
|
2967
3263
|
onConnected();
|
|
2968
3264
|
return;
|
|
@@ -3010,8 +3306,30 @@ function useSessionLifecycle(input) {
|
|
|
3010
3306
|
}, step2.delayMs);
|
|
3011
3307
|
return clearTimer;
|
|
3012
3308
|
}, [recovery.kind, active, attempt, clearTimer]);
|
|
3309
|
+
const end = useCallback(() => {
|
|
3310
|
+
if (!active || userEndedRef.current || phaseRef.current.kind === "ended") return;
|
|
3311
|
+
userEndedRef.current = true;
|
|
3312
|
+
clearTimer();
|
|
3313
|
+
connectedRef.current = false;
|
|
3314
|
+
manualReconnectPendingRef.current = false;
|
|
3315
|
+
refreshBaselineCapacityRef.current = null;
|
|
3316
|
+
setUserEnded(true);
|
|
3317
|
+
setConnected(false);
|
|
3318
|
+
setAgentPresentState(false);
|
|
3319
|
+
setRecovery({ kind: "ended", reason: "user" });
|
|
3320
|
+
clearRef.current();
|
|
3321
|
+
leaveRoomRef.current?.();
|
|
3322
|
+
}, [active, clearTimer]);
|
|
3013
3323
|
const reconnect = useCallback(() => {
|
|
3014
|
-
if (!active
|
|
3324
|
+
if (!active) return;
|
|
3325
|
+
if (userEndedRef.current) {
|
|
3326
|
+
userEndedRef.current = false;
|
|
3327
|
+
setUserEnded(false);
|
|
3328
|
+
reset();
|
|
3329
|
+
manualReconnectPendingRef.current = true;
|
|
3330
|
+
return;
|
|
3331
|
+
}
|
|
3332
|
+
if (manualReconnectPendingRef.current || recovery.kind === "in-place-reconnecting") return;
|
|
3015
3333
|
manualReconnectPendingRef.current = true;
|
|
3016
3334
|
clearTimer();
|
|
3017
3335
|
attemptRef.current = 0;
|
|
@@ -3020,7 +3338,7 @@ function useSessionLifecycle(input) {
|
|
|
3020
3338
|
releaseRef.current("superseded");
|
|
3021
3339
|
setRecovery({ kind: "refreshing", attempt: 0 });
|
|
3022
3340
|
refreshRef.current();
|
|
3023
|
-
}, [active, clearTimer, recovery.kind]);
|
|
3341
|
+
}, [active, clearTimer, recovery.kind, reset]);
|
|
3024
3342
|
const endIdleRef = useRef(endIdle);
|
|
3025
3343
|
endIdleRef.current = endIdle;
|
|
3026
3344
|
useEffect(() => {
|
|
@@ -3052,6 +3370,7 @@ function useSessionLifecycle(input) {
|
|
|
3052
3370
|
() => lifecyclePhaseFrom({ capacity, recovery, idle, connected, agentPresent }),
|
|
3053
3371
|
[capacity, recovery, idle, connected, agentPresent]
|
|
3054
3372
|
);
|
|
3373
|
+
phaseRef.current = phase;
|
|
3055
3374
|
const onBehaviorChangeRef = useRef(onBehaviorChange);
|
|
3056
3375
|
onBehaviorChangeRef.current = onBehaviorChange;
|
|
3057
3376
|
const lastBehaviorRef = useRef(null);
|
|
@@ -3077,6 +3396,7 @@ function useSessionLifecycle(input) {
|
|
|
3077
3396
|
stayConnected,
|
|
3078
3397
|
markActivity,
|
|
3079
3398
|
reconnect,
|
|
3399
|
+
end,
|
|
3080
3400
|
onConnected,
|
|
3081
3401
|
onDisconnected,
|
|
3082
3402
|
onConnectionError,
|
|
@@ -3095,6 +3415,7 @@ function useSessionLifecycle(input) {
|
|
|
3095
3415
|
stayConnected,
|
|
3096
3416
|
markActivity,
|
|
3097
3417
|
reconnect,
|
|
3418
|
+
end,
|
|
3098
3419
|
onConnected,
|
|
3099
3420
|
onDisconnected,
|
|
3100
3421
|
onConnectionError,
|
|
@@ -3109,7 +3430,8 @@ function useSessionLifecycle(input) {
|
|
|
3109
3430
|
function SessionLifecycleRoomBridge({
|
|
3110
3431
|
lifecycle,
|
|
3111
3432
|
onConnectionDetailsChange,
|
|
3112
|
-
onMediaModeChange
|
|
3433
|
+
onMediaModeChange,
|
|
3434
|
+
microphone
|
|
3113
3435
|
}) {
|
|
3114
3436
|
const {
|
|
3115
3437
|
onConnectionStateChange,
|
|
@@ -3120,12 +3442,23 @@ function SessionLifecycleRoomBridge({
|
|
|
3120
3442
|
registerDataPublisher,
|
|
3121
3443
|
registerTurnSender,
|
|
3122
3444
|
setTurnState,
|
|
3123
|
-
setMedia
|
|
3445
|
+
setMedia,
|
|
3446
|
+
setMicrophoneFacts,
|
|
3447
|
+
setAudioPlayback,
|
|
3448
|
+
registerMediaControls
|
|
3124
3449
|
} = lifecycle;
|
|
3125
3450
|
const connectionState = useConnectionState();
|
|
3126
3451
|
const assistant = useVoiceAssistant();
|
|
3127
3452
|
const transcriptions = useTranscriptions();
|
|
3128
3453
|
const room = useRoomContext();
|
|
3454
|
+
useCallMedia(room, {
|
|
3455
|
+
setMicrophoneFacts,
|
|
3456
|
+
setAudioPlayback,
|
|
3457
|
+
registerMediaControls,
|
|
3458
|
+
microphoneWanted: microphone ?? publishesMicrophoneByDefault(lifecycle.grant),
|
|
3459
|
+
connectionState,
|
|
3460
|
+
callKey: lifecycle.grant?.session_id
|
|
3461
|
+
});
|
|
3129
3462
|
const mediaModeCallbackRef = useRef(onMediaModeChange);
|
|
3130
3463
|
mediaModeCallbackRef.current = onMediaModeChange;
|
|
3131
3464
|
const mediaModeEnabled = onMediaModeChange !== void 0 && lifecycle.grant !== null;
|
|
@@ -3399,6 +3732,8 @@ function resolveEndReason(workerLabel, innerReason) {
|
|
|
3399
3732
|
return "idle";
|
|
3400
3733
|
case "error":
|
|
3401
3734
|
return "failed";
|
|
3735
|
+
case "user":
|
|
3736
|
+
return "user_ended";
|
|
3402
3737
|
default:
|
|
3403
3738
|
return "disconnected";
|
|
3404
3739
|
}
|
|
@@ -3454,6 +3789,16 @@ function useRealtimeSession(input) {
|
|
|
3454
3789
|
const [turn, setTurn] = useState("quiet");
|
|
3455
3790
|
const [media, setMediaState] = useState({ video: "connecting", audio: "silent" });
|
|
3456
3791
|
const [behavior, setBehavior] = useState(null);
|
|
3792
|
+
const [microphoneFacts2, setMicrophoneFactsState] = useState({
|
|
3793
|
+
wanted: false,
|
|
3794
|
+
connected: false,
|
|
3795
|
+
enabling: false,
|
|
3796
|
+
publication: null,
|
|
3797
|
+
deviceError: null
|
|
3798
|
+
});
|
|
3799
|
+
const [microphoneMuted, setMicrophoneMuted] = useState(false);
|
|
3800
|
+
const [audioPlayback, setAudioPlaybackState] = useState("unknown");
|
|
3801
|
+
const mediaControlsRef = useRef(null);
|
|
3457
3802
|
const lastBehaviorRef = useRef(null);
|
|
3458
3803
|
const lastLabeledEndReasonRef = useRef(null);
|
|
3459
3804
|
const pendingClipsRef = useRef(/* @__PURE__ */ new Map());
|
|
@@ -3560,6 +3905,31 @@ function useRealtimeSession(input) {
|
|
|
3560
3905
|
const setMedia = useCallback((next) => {
|
|
3561
3906
|
setMediaState((prev) => prev.video === next.video && prev.audio === next.audio ? prev : next);
|
|
3562
3907
|
}, []);
|
|
3908
|
+
const microphoneFactsRef = useRef(microphoneFacts2);
|
|
3909
|
+
microphoneFactsRef.current = microphoneFacts2;
|
|
3910
|
+
const setMicrophoneFacts = useCallback((facts) => setMicrophoneFactsState(facts), []);
|
|
3911
|
+
const setAudioPlayback = useCallback((playback) => setAudioPlaybackState(playback), []);
|
|
3912
|
+
const registerMediaControls = useCallback((controls) => {
|
|
3913
|
+
mediaControlsRef.current = controls;
|
|
3914
|
+
}, []);
|
|
3915
|
+
const startAudio = useCallback(async () => {
|
|
3916
|
+
const controls = mediaControlsRef.current;
|
|
3917
|
+
return controls ? controls.startAudio() : false;
|
|
3918
|
+
}, []);
|
|
3919
|
+
const callEnded = lifecycle.phase.kind === "ended";
|
|
3920
|
+
const callEndedRef = useRef(callEnded);
|
|
3921
|
+
callEndedRef.current = callEnded;
|
|
3922
|
+
const setMicrophoneEnabled = useCallback(async (enabled) => {
|
|
3923
|
+
if (callEndedRef.current) return;
|
|
3924
|
+
setMicrophoneMuted(!enabled);
|
|
3925
|
+
if (!microphoneFactsRef.current.connected) return;
|
|
3926
|
+
await mediaControlsRef.current?.setMicrophoneEnabled(enabled);
|
|
3927
|
+
}, []);
|
|
3928
|
+
const microphone = useMemo(
|
|
3929
|
+
() => callEnded ? { status: "off" } : callMicrophoneFrom(microphoneFacts2, { muted: microphoneMuted }),
|
|
3930
|
+
[callEnded, microphoneFacts2, microphoneMuted]
|
|
3931
|
+
);
|
|
3932
|
+
const callAudioPlayback = callEnded ? "unknown" : audioPlayback;
|
|
3563
3933
|
useEffect(() => {
|
|
3564
3934
|
if (phaseKind === "idle" || phaseKind === "ended") return;
|
|
3565
3935
|
const timer = window.setInterval(() => {
|
|
@@ -3704,11 +4074,23 @@ function useRealtimeSession(input) {
|
|
|
3704
4074
|
void sendResolvedTurn(last.text, last.attributes, last.id).catch(() => void 0);
|
|
3705
4075
|
}, [sendResolvedTurn]);
|
|
3706
4076
|
const end = useCallback((reason) => {
|
|
4077
|
+
if (phaseKind === "ended") return;
|
|
3707
4078
|
if (reason) lastLabeledEndReasonRef.current = reason;
|
|
3708
4079
|
lastTurnRef.current = null;
|
|
3709
4080
|
requestGracefulClose();
|
|
3710
|
-
lifecycle.
|
|
3711
|
-
}, [lifecycle, requestGracefulClose]);
|
|
4081
|
+
lifecycle.end();
|
|
4082
|
+
}, [lifecycle, phaseKind, requestGracefulClose]);
|
|
4083
|
+
const active = inner.active ?? true;
|
|
4084
|
+
useEffect(() => {
|
|
4085
|
+
if (active) return;
|
|
4086
|
+
lastLabeledEndReasonRef.current = null;
|
|
4087
|
+
setMicrophoneMuted(false);
|
|
4088
|
+
}, [active]);
|
|
4089
|
+
const innerReconnect = lifecycle.reconnect;
|
|
4090
|
+
const reconnect = useCallback(() => {
|
|
4091
|
+
if (phaseKind === "ended") lastLabeledEndReasonRef.current = null;
|
|
4092
|
+
innerReconnect();
|
|
4093
|
+
}, [innerReconnect, phaseKind]);
|
|
3712
4094
|
const performAction = useCallback(
|
|
3713
4095
|
(actionId, opts) => {
|
|
3714
4096
|
const requestId = newTurnId();
|
|
@@ -3754,6 +4136,7 @@ function useRealtimeSession(input) {
|
|
|
3754
4136
|
approachingFiredRef.current = false;
|
|
3755
4137
|
creditsLowFiredRef.current = false;
|
|
3756
4138
|
lastTurnRef.current = null;
|
|
4139
|
+
setMicrophoneMuted(false);
|
|
3757
4140
|
lifecycle.reset();
|
|
3758
4141
|
}, [lifecycle]);
|
|
3759
4142
|
return useMemo(
|
|
@@ -3767,6 +4150,12 @@ function useRealtimeSession(input) {
|
|
|
3767
4150
|
endsAt,
|
|
3768
4151
|
graceWindow,
|
|
3769
4152
|
media,
|
|
4153
|
+
microphone,
|
|
4154
|
+
audioPlayback: callAudioPlayback,
|
|
4155
|
+
startAudio,
|
|
4156
|
+
setMicrophoneEnabled,
|
|
4157
|
+
microphoneMuted,
|
|
4158
|
+
reconnect,
|
|
3770
4159
|
lifecycle,
|
|
3771
4160
|
sendClosingTurn,
|
|
3772
4161
|
requestGracefulClose,
|
|
@@ -3782,6 +4171,9 @@ function useRealtimeSession(input) {
|
|
|
3782
4171
|
registerTurnSender,
|
|
3783
4172
|
setTurnState,
|
|
3784
4173
|
setMedia,
|
|
4174
|
+
setMicrophoneFacts,
|
|
4175
|
+
setAudioPlayback,
|
|
4176
|
+
registerMediaControls,
|
|
3785
4177
|
reset
|
|
3786
4178
|
}),
|
|
3787
4179
|
[
|
|
@@ -3791,6 +4183,12 @@ function useRealtimeSession(input) {
|
|
|
3791
4183
|
endsAt,
|
|
3792
4184
|
graceWindow,
|
|
3793
4185
|
media,
|
|
4186
|
+
microphone,
|
|
4187
|
+
callAudioPlayback,
|
|
4188
|
+
startAudio,
|
|
4189
|
+
setMicrophoneEnabled,
|
|
4190
|
+
microphoneMuted,
|
|
4191
|
+
reconnect,
|
|
3794
4192
|
sendClosingTurn,
|
|
3795
4193
|
requestGracefulClose,
|
|
3796
4194
|
extend,
|
|
@@ -3805,6 +4203,9 @@ function useRealtimeSession(input) {
|
|
|
3805
4203
|
registerTurnSender,
|
|
3806
4204
|
setTurnState,
|
|
3807
4205
|
setMedia,
|
|
4206
|
+
setMicrophoneFacts,
|
|
4207
|
+
setAudioPlayback,
|
|
4208
|
+
registerMediaControls,
|
|
3808
4209
|
reset
|
|
3809
4210
|
]
|
|
3810
4211
|
);
|
|
@@ -3837,9 +4238,31 @@ function handleFor(session) {
|
|
|
3837
4238
|
session.sendClosingTurn(text);
|
|
3838
4239
|
},
|
|
3839
4240
|
keepAlive: session.stayConnected,
|
|
3840
|
-
end: () => session.end("user_ended")
|
|
4241
|
+
end: () => session.end("user_ended"),
|
|
4242
|
+
microphone: session.microphone,
|
|
4243
|
+
audio: session.audioPlayback,
|
|
4244
|
+
startAudio: session.startAudio,
|
|
4245
|
+
setMicrophoneEnabled: session.setMicrophoneEnabled,
|
|
4246
|
+
retryMicrophone: () => session.setMicrophoneEnabled(true)
|
|
3841
4247
|
};
|
|
3842
4248
|
}
|
|
4249
|
+
var UNLOCK_REGION = {
|
|
4250
|
+
position: "absolute",
|
|
4251
|
+
top: 16,
|
|
4252
|
+
left: "50%",
|
|
4253
|
+
transform: "translateX(-50%)",
|
|
4254
|
+
zIndex: 1
|
|
4255
|
+
};
|
|
4256
|
+
var UNLOCK_BUTTON = {
|
|
4257
|
+
padding: "10px 16px",
|
|
4258
|
+
border: "none",
|
|
4259
|
+
borderRadius: 9999,
|
|
4260
|
+
background: "rgb(0 0 0 / 0.7)",
|
|
4261
|
+
color: "white",
|
|
4262
|
+
font: "inherit",
|
|
4263
|
+
fontSize: 14,
|
|
4264
|
+
cursor: "pointer"
|
|
4265
|
+
};
|
|
3843
4266
|
function useAvatarCall(props) {
|
|
3844
4267
|
const sessionRef = useRef(null);
|
|
3845
4268
|
const session = useRealtimeSession({
|
|
@@ -3863,11 +4286,36 @@ function useAvatarCall(props) {
|
|
|
3863
4286
|
lastStatusRef.current = call.status;
|
|
3864
4287
|
onStatusChange?.(call.status);
|
|
3865
4288
|
}, [call.status, onStatusChange]);
|
|
4289
|
+
const microphone = call.microphone;
|
|
4290
|
+
const problemKey = microphone.status === "blocked" || microphone.status === "unavailable" ? `${microphone.status}:${microphone.reason}:${microphone.message}` : null;
|
|
4291
|
+
const lastProblemRef = useRef(null);
|
|
4292
|
+
const onMicrophoneProblemRef = useRef(props.onMicrophoneProblem);
|
|
4293
|
+
onMicrophoneProblemRef.current = props.onMicrophoneProblem;
|
|
4294
|
+
useEffect(() => {
|
|
4295
|
+
if (problemKey === lastProblemRef.current) return;
|
|
4296
|
+
lastProblemRef.current = problemKey;
|
|
4297
|
+
if (microphone.status === "blocked" || microphone.status === "unavailable") onMicrophoneProblemRef.current?.(microphone);
|
|
4298
|
+
}, [problemKey]);
|
|
4299
|
+
const prompt = props.audioUnlockPrompt ?? true;
|
|
4300
|
+
const unlock = prompt === false ? null : createElement(
|
|
4301
|
+
"div",
|
|
4302
|
+
{ key: "audio-unlock-region", role: "status", "aria-live": "polite", style: UNLOCK_REGION, "data-testid": "avatar-audio-unlock-region" },
|
|
4303
|
+
call.audio !== "blocked" ? null : typeof prompt === "function" ? prompt(call) : createElement(
|
|
4304
|
+
"button",
|
|
4305
|
+
{ type: "button", style: UNLOCK_BUTTON, onClick: () => {
|
|
4306
|
+
void call.startAudio();
|
|
4307
|
+
}, "data-testid": "avatar-audio-unlock" },
|
|
4308
|
+
"Tap to turn on sound"
|
|
4309
|
+
)
|
|
4310
|
+
);
|
|
4311
|
+
const overlay = props.children ? props.children(call) : null;
|
|
3866
4312
|
const view = createElement(
|
|
3867
4313
|
RealtimeAvatarLiveKitRoom,
|
|
3868
4314
|
{
|
|
3869
4315
|
grant: session.grant,
|
|
3870
|
-
|
|
4316
|
+
// LiveKit captures the microphone on connect from this value, so a mute chosen while
|
|
4317
|
+
// waiting is honoured rather than overridden when the call goes live.
|
|
4318
|
+
audio: props.listen !== false && !session.microphoneMuted,
|
|
3871
4319
|
onConnected: session.onConnected,
|
|
3872
4320
|
onDisconnected: session.onDisconnected,
|
|
3873
4321
|
onError: session.onConnectionError
|
|
@@ -3875,6 +4323,7 @@ function useAvatarCall(props) {
|
|
|
3875
4323
|
createElement(SessionLifecycleRoomBridge, {
|
|
3876
4324
|
key: "bridge",
|
|
3877
4325
|
lifecycle: session,
|
|
4326
|
+
microphone: props.listen !== false,
|
|
3878
4327
|
onConnectionDetailsChange: props.onConnectionDetailsChange
|
|
3879
4328
|
}),
|
|
3880
4329
|
createElement(
|
|
@@ -3884,9 +4333,10 @@ function useAvatarCall(props) {
|
|
|
3884
4333
|
idleVideoUrl: props.idleVideoUrl ?? null,
|
|
3885
4334
|
poster: props.poster ?? null,
|
|
3886
4335
|
fit: props.fit ?? "cover",
|
|
3887
|
-
className: props.className
|
|
4336
|
+
className: props.className,
|
|
4337
|
+
style: props.style
|
|
3888
4338
|
},
|
|
3889
|
-
|
|
4339
|
+
overlay === null && unlock === null ? null : createElement(Fragment, null, overlay, unlock)
|
|
3890
4340
|
)
|
|
3891
4341
|
);
|
|
3892
4342
|
return { call, view };
|
|
@@ -3896,39 +4346,66 @@ function AvatarCall(props) {
|
|
|
3896
4346
|
return view;
|
|
3897
4347
|
}
|
|
3898
4348
|
|
|
4349
|
+
// ../http-client/src/retry.ts
|
|
4350
|
+
var PROXY_CLIENT_TIMEOUT_MS = 6e4;
|
|
4351
|
+
|
|
3899
4352
|
// ../client/src/proxy-client.ts
|
|
3900
|
-
var DEFAULT_TIMEOUT_MS = 6e4;
|
|
3901
4353
|
var normalize = (url) => url.replace(/\/+$/, "");
|
|
3902
4354
|
function createProxyClient(options) {
|
|
3903
4355
|
const base = normalize(options.proxyUrl);
|
|
3904
4356
|
const doFetch = options.fetch ?? globalThis.fetch?.bind(globalThis);
|
|
3905
|
-
const timeoutMs = options.timeoutMs ??
|
|
3906
|
-
const
|
|
3907
|
-
|
|
3908
|
-
const timer = AbortSignal.timeout(timeoutMs);
|
|
3909
|
-
return caller ? AbortSignal.any([caller, timer]) : timer;
|
|
3910
|
-
};
|
|
4357
|
+
const timeoutMs = options.timeoutMs ?? PROXY_CLIENT_TIMEOUT_MS;
|
|
4358
|
+
const credentials = options.credentials;
|
|
4359
|
+
const withCredentials = credentials === void 0 ? {} : { credentials };
|
|
3911
4360
|
const post = async (path, body, request) => {
|
|
3912
4361
|
if (!doFetch) throw new Error("realtime-avatar: no fetch available \u2014 pass one via `fetch`.");
|
|
3913
|
-
|
|
3914
|
-
|
|
3915
|
-
|
|
3916
|
-
|
|
3917
|
-
|
|
3918
|
-
|
|
4362
|
+
const caller = request?.signal;
|
|
4363
|
+
const timer = timeoutMs ? AbortSignal.timeout(timeoutMs) : void 0;
|
|
4364
|
+
try {
|
|
4365
|
+
return await doFetch(`${base}${path}`, {
|
|
4366
|
+
method: "POST",
|
|
4367
|
+
...withCredentials,
|
|
4368
|
+
headers: { "content-type": "application/json", ...request?.headers ?? {} },
|
|
4369
|
+
body: JSON.stringify(body),
|
|
4370
|
+
signal: timer && caller ? AbortSignal.any([caller, timer]) : timer ?? caller
|
|
4371
|
+
});
|
|
4372
|
+
} catch (cause) {
|
|
4373
|
+
if (timer?.aborted && !caller?.aborted) throw RealtimeAvatarApiError.timeout(timeoutMs);
|
|
4374
|
+
throw cause;
|
|
4375
|
+
}
|
|
3919
4376
|
};
|
|
3920
|
-
const
|
|
4377
|
+
const pageHide = (body) => {
|
|
4378
|
+
const page = globalThis.location?.href;
|
|
4379
|
+
let target;
|
|
4380
|
+
try {
|
|
4381
|
+
target = new URL(`${base}/end`, page);
|
|
4382
|
+
} catch {
|
|
4383
|
+
return false;
|
|
4384
|
+
}
|
|
4385
|
+
const payload = JSON.stringify(body);
|
|
3921
4386
|
const send = globalThis.navigator?.sendBeacon?.bind(globalThis.navigator);
|
|
3922
|
-
|
|
3923
|
-
|
|
4387
|
+
const sameOrigin = page !== void 0 && target.origin === new URL(page).origin;
|
|
4388
|
+
if (send && (sameOrigin || credentials === "include")) {
|
|
4389
|
+
if (send(target.href, new Blob([payload], { type: "application/json" }))) return true;
|
|
4390
|
+
}
|
|
4391
|
+
if (!doFetch) return false;
|
|
4392
|
+
try {
|
|
4393
|
+
void doFetch(target.href, {
|
|
4394
|
+
method: "POST",
|
|
4395
|
+
keepalive: true,
|
|
4396
|
+
...withCredentials,
|
|
4397
|
+
headers: { "content-type": "application/json" },
|
|
4398
|
+
body: payload
|
|
4399
|
+
}).catch(() => void 0);
|
|
4400
|
+
return true;
|
|
4401
|
+
} catch {
|
|
4402
|
+
return false;
|
|
4403
|
+
}
|
|
3924
4404
|
};
|
|
3925
4405
|
return {
|
|
3926
4406
|
async createLiveKitSessionOrBusy(input, requestOptions) {
|
|
3927
|
-
const
|
|
3928
|
-
|
|
3929
|
-
{ avatarId: input.avatarId, mode: input.mode },
|
|
3930
|
-
requestOptions
|
|
3931
|
-
);
|
|
4407
|
+
const connect = { avatarId: input.avatarId, mode: input.mode };
|
|
4408
|
+
const response = await post("/connect", connect, requestOptions);
|
|
3932
4409
|
if (response.status === 429) {
|
|
3933
4410
|
const value = await response.clone().json().catch(() => null);
|
|
3934
4411
|
if (value && typeof value === "object" && !Array.isArray(value)) {
|
|
@@ -3946,7 +4423,8 @@ function createProxyClient(options) {
|
|
|
3946
4423
|
async releaseLiveKitSession(sessionId, reason, requestOptions) {
|
|
3947
4424
|
if (!sessionId) return false;
|
|
3948
4425
|
try {
|
|
3949
|
-
const
|
|
4426
|
+
const body = { session_id: sessionId, reason };
|
|
4427
|
+
const response = await post("/end", body, requestOptions);
|
|
3950
4428
|
return response.ok;
|
|
3951
4429
|
} catch {
|
|
3952
4430
|
return false;
|
|
@@ -3954,12 +4432,13 @@ function createProxyClient(options) {
|
|
|
3954
4432
|
},
|
|
3955
4433
|
releaseLiveKitSessionBeacon(sessionId, reason) {
|
|
3956
4434
|
if (!sessionId) return false;
|
|
3957
|
-
return
|
|
4435
|
+
return pageHide({ session_id: sessionId, reason: reason ?? "page_hide" });
|
|
3958
4436
|
},
|
|
3959
4437
|
async releaseLiveKitQueueTicket(queueTicketId, reason, requestOptions) {
|
|
3960
4438
|
if (!queueTicketId) return false;
|
|
3961
4439
|
try {
|
|
3962
|
-
const
|
|
4440
|
+
const body = { queue_ticket_id: queueTicketId, reason };
|
|
4441
|
+
const response = await post("/end", body, requestOptions);
|
|
3963
4442
|
return response.ok;
|
|
3964
4443
|
} catch {
|
|
3965
4444
|
return false;
|
|
@@ -3967,7 +4446,7 @@ function createProxyClient(options) {
|
|
|
3967
4446
|
},
|
|
3968
4447
|
releaseLiveKitQueueTicketBeacon(queueTicketId, reason) {
|
|
3969
4448
|
if (!queueTicketId) return false;
|
|
3970
|
-
return
|
|
4449
|
+
return pageHide({ queue_ticket_id: queueTicketId, reason: reason ?? "page_hide" });
|
|
3971
4450
|
}
|
|
3972
4451
|
};
|
|
3973
4452
|
}
|