realtime-avatar 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +176 -0
- package/dist/browser.d.ts +203 -0
- package/dist/browser.js +172 -0
- package/dist/express.d.ts +28 -0
- package/dist/express.js +510 -0
- package/dist/hono.d.ts +19 -0
- package/dist/hono.js +498 -0
- package/dist/index.d.ts +192 -0
- package/dist/index.js +453 -0
- package/dist/nextjs.d.ts +18 -0
- package/dist/nextjs.js +498 -0
- package/dist/proxy-client-cZyX50-O.d.ts +1508 -0
- package/dist/react-native.d.ts +103 -0
- package/dist/react-native.js +2305 -0
- package/dist/react.d.ts +107 -0
- package/dist/react.js +2610 -0
- package/dist/server-only-guard.d.ts +2 -0
- package/dist/server-only-guard.js +4 -0
- package/dist/server.d.ts +2 -0
- package/dist/server.js +453 -0
- package/dist/tanstack-start.d.ts +24 -0
- package/dist/tanstack-start.js +498 -0
- package/dist/tools.d.ts +111 -0
- package/dist/tools.js +121 -0
- package/dist/types-C_EMPwN7.d.ts +831 -0
- package/dist/types-E8SrD6sv.d.ts +48 -0
- package/package.json +150 -0
|
@@ -0,0 +1,1508 @@
|
|
|
1
|
+
import { TrackReferenceOrPlaceholder, useTranscriptions, useConnectionState, LiveKitRoomProps } from '@livekit/components-react';
|
|
2
|
+
import { CSSProperties, ReactNode, ReactElement } from 'react';
|
|
3
|
+
import { DisconnectReason, SendTextOptions } from 'livekit-client';
|
|
4
|
+
import { z } from 'zod';
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Wire schemas for the public Realtime Avatar API.
|
|
8
|
+
*
|
|
9
|
+
* Code only, by policy: this file carries schemas and no narrative. Prose is where
|
|
10
|
+
* boundaries are hardest to hold, because a sentence explaining WHY a field exists tends
|
|
11
|
+
* to describe the thing on the other side of it.
|
|
12
|
+
*
|
|
13
|
+
* What may appear here is bounded by `public-surface.txt` and enforced by
|
|
14
|
+
* `npm run surface`: every export must be named on that allowlist, so a new one is
|
|
15
|
+
* refused until someone decides it is public. Documentation lives at
|
|
16
|
+
* https://realtimeavatar.ai/docs.
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
declare const llmProviderSchema: z.ZodEnum<{
|
|
20
|
+
local: "local";
|
|
21
|
+
gemini: "gemini";
|
|
22
|
+
openai: "openai";
|
|
23
|
+
}>;
|
|
24
|
+
declare const llmSelectionSchema: z.ZodObject<{
|
|
25
|
+
provider: z.ZodEnum<{
|
|
26
|
+
local: "local";
|
|
27
|
+
gemini: "gemini";
|
|
28
|
+
openai: "openai";
|
|
29
|
+
}>;
|
|
30
|
+
model: z.ZodOptional<z.ZodNullable<z.ZodString>>;
|
|
31
|
+
}, z.core.$strict>;
|
|
32
|
+
declare const cartesiaTtsModelSchema: z.ZodEnum<{
|
|
33
|
+
"cartesia/sonic-2": "cartesia/sonic-2";
|
|
34
|
+
"cartesia/sonic-2-latest": "cartesia/sonic-2-latest";
|
|
35
|
+
"cartesia/sonic-3": "cartesia/sonic-3";
|
|
36
|
+
"cartesia/sonic-3-latest": "cartesia/sonic-3-latest";
|
|
37
|
+
"cartesia/sonic-turbo": "cartesia/sonic-turbo";
|
|
38
|
+
"cartesia/sonic-turbo-latest": "cartesia/sonic-turbo-latest";
|
|
39
|
+
}>;
|
|
40
|
+
declare const fishTtsModelSchema: z.ZodEnum<{
|
|
41
|
+
"speech-1.6": "speech-1.6";
|
|
42
|
+
s1: "s1";
|
|
43
|
+
"s2-pro": "s2-pro";
|
|
44
|
+
"speech-1.5": "speech-1.5";
|
|
45
|
+
"s1-mini": "s1-mini";
|
|
46
|
+
}>;
|
|
47
|
+
declare const voiceSpecSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
48
|
+
provider: z.ZodLiteral<"cartesia">;
|
|
49
|
+
model: z.ZodDefault<z.ZodEnum<{
|
|
50
|
+
"cartesia/sonic-2": "cartesia/sonic-2";
|
|
51
|
+
"cartesia/sonic-2-latest": "cartesia/sonic-2-latest";
|
|
52
|
+
"cartesia/sonic-3": "cartesia/sonic-3";
|
|
53
|
+
"cartesia/sonic-3-latest": "cartesia/sonic-3-latest";
|
|
54
|
+
"cartesia/sonic-turbo": "cartesia/sonic-turbo";
|
|
55
|
+
"cartesia/sonic-turbo-latest": "cartesia/sonic-turbo-latest";
|
|
56
|
+
}>>;
|
|
57
|
+
voice_id: z.ZodString;
|
|
58
|
+
speed: z.ZodOptional<z.ZodNullable<z.ZodNumber>>;
|
|
59
|
+
emotion: z.ZodOptional<z.ZodNullable<z.ZodString>>;
|
|
60
|
+
language: z.ZodOptional<z.ZodNullable<z.ZodString>>;
|
|
61
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
62
|
+
provider: z.ZodLiteral<"breezeblue">;
|
|
63
|
+
model: z.ZodDefault<z.ZodString>;
|
|
64
|
+
voice_id: z.ZodString;
|
|
65
|
+
guidance_scale: z.ZodOptional<z.ZodNullable<z.ZodNumber>>;
|
|
66
|
+
instructions: z.ZodOptional<z.ZodNullable<z.ZodString>>;
|
|
67
|
+
language: z.ZodOptional<z.ZodNullable<z.ZodString>>;
|
|
68
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
69
|
+
provider: z.ZodLiteral<"fish">;
|
|
70
|
+
model: z.ZodDefault<z.ZodEnum<{
|
|
71
|
+
"speech-1.6": "speech-1.6";
|
|
72
|
+
s1: "s1";
|
|
73
|
+
"s2-pro": "s2-pro";
|
|
74
|
+
"speech-1.5": "speech-1.5";
|
|
75
|
+
"s1-mini": "s1-mini";
|
|
76
|
+
}>>;
|
|
77
|
+
voice_id: z.ZodString;
|
|
78
|
+
speed: z.ZodOptional<z.ZodNullable<z.ZodNumber>>;
|
|
79
|
+
emotion: z.ZodOptional<z.ZodNullable<z.ZodString>>;
|
|
80
|
+
language: z.ZodOptional<z.ZodNullable<z.ZodString>>;
|
|
81
|
+
}, z.core.$strict>], "provider">;
|
|
82
|
+
declare const sessionClipSchema: z.ZodObject<{
|
|
83
|
+
clip_id: z.ZodString;
|
|
84
|
+
source_video_url: z.ZodOptional<z.ZodString>;
|
|
85
|
+
video_cache_id: z.ZodOptional<z.ZodString>;
|
|
86
|
+
max_seconds: z.ZodOptional<z.ZodNumber>;
|
|
87
|
+
trigger: z.ZodOptional<z.ZodEnum<{
|
|
88
|
+
idle: "idle";
|
|
89
|
+
listen: "listen";
|
|
90
|
+
think: "think";
|
|
91
|
+
directive: "directive";
|
|
92
|
+
}>>;
|
|
93
|
+
loop: z.ZodOptional<z.ZodBoolean>;
|
|
94
|
+
weight: z.ZodOptional<z.ZodNumber>;
|
|
95
|
+
crossfade_ms: z.ZodOptional<z.ZodNumber>;
|
|
96
|
+
trim_start_ms: z.ZodOptional<z.ZodNumber>;
|
|
97
|
+
trim_end_ms: z.ZodOptional<z.ZodNumber>;
|
|
98
|
+
when: z.ZodOptional<z.ZodString>;
|
|
99
|
+
hint: z.ZodOptional<z.ZodString>;
|
|
100
|
+
}, z.core.$strict>;
|
|
101
|
+
type SessionClip = z.infer<typeof sessionClipSchema>;
|
|
102
|
+
declare const sessionBehaviorSchema: z.ZodObject<{
|
|
103
|
+
gestures_enabled: z.ZodOptional<z.ZodBoolean>;
|
|
104
|
+
gesture_freq: z.ZodOptional<z.ZodEnum<{
|
|
105
|
+
sparse: "sparse";
|
|
106
|
+
balanced: "balanced";
|
|
107
|
+
lively: "lively";
|
|
108
|
+
}>>;
|
|
109
|
+
}, z.core.$strict>;
|
|
110
|
+
type SessionBehavior = z.infer<typeof sessionBehaviorSchema>;
|
|
111
|
+
/**
|
|
112
|
+
* The cap the mint enforces on `instructions`. Exported so an app can budget its prompt
|
|
113
|
+
* assembly against the real number instead of probing for it with a binary search.
|
|
114
|
+
*/
|
|
115
|
+
declare const MAX_SESSION_INSTRUCTIONS_CHARS = 4000;
|
|
116
|
+
declare const liveKitSessionRequestSchema: z.ZodObject<{
|
|
117
|
+
avatarId: z.ZodString;
|
|
118
|
+
backgroundId: z.ZodDefault<z.ZodString>;
|
|
119
|
+
mode: z.ZodDefault<z.ZodEnum<{
|
|
120
|
+
avatar: "avatar";
|
|
121
|
+
voice: "voice";
|
|
122
|
+
}>>;
|
|
123
|
+
createRoom: z.ZodDefault<z.ZodBoolean>;
|
|
124
|
+
dispatchAgent: z.ZodDefault<z.ZodBoolean>;
|
|
125
|
+
instructions: z.ZodOptional<z.ZodString>;
|
|
126
|
+
initialContext: z.ZodDefault<z.ZodArray<z.ZodObject<{
|
|
127
|
+
role: z.ZodEnum<{
|
|
128
|
+
system: "system";
|
|
129
|
+
user: "user";
|
|
130
|
+
assistant: "assistant";
|
|
131
|
+
}>;
|
|
132
|
+
content: z.ZodString;
|
|
133
|
+
}, z.core.$strict>>>;
|
|
134
|
+
initialSay: z.ZodOptional<z.ZodString>;
|
|
135
|
+
llm: z.ZodOptional<z.ZodNullable<z.ZodObject<{
|
|
136
|
+
provider: z.ZodEnum<{
|
|
137
|
+
local: "local";
|
|
138
|
+
gemini: "gemini";
|
|
139
|
+
openai: "openai";
|
|
140
|
+
}>;
|
|
141
|
+
model: z.ZodOptional<z.ZodNullable<z.ZodString>>;
|
|
142
|
+
}, z.core.$strict>>>;
|
|
143
|
+
maxSessionSeconds: z.ZodOptional<z.ZodNumber>;
|
|
144
|
+
participantIdentity: z.ZodOptional<z.ZodString>;
|
|
145
|
+
participantName: z.ZodOptional<z.ZodString>;
|
|
146
|
+
queueTicketId: z.ZodOptional<z.ZodString>;
|
|
147
|
+
roomName: z.ZodOptional<z.ZodString>;
|
|
148
|
+
sttMode: z.ZodDefault<z.ZodEnum<{
|
|
149
|
+
server: "server";
|
|
150
|
+
off: "off";
|
|
151
|
+
}>>;
|
|
152
|
+
voice: z.ZodOptional<z.ZodNullable<z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
153
|
+
provider: z.ZodLiteral<"cartesia">;
|
|
154
|
+
model: z.ZodDefault<z.ZodEnum<{
|
|
155
|
+
"cartesia/sonic-2": "cartesia/sonic-2";
|
|
156
|
+
"cartesia/sonic-2-latest": "cartesia/sonic-2-latest";
|
|
157
|
+
"cartesia/sonic-3": "cartesia/sonic-3";
|
|
158
|
+
"cartesia/sonic-3-latest": "cartesia/sonic-3-latest";
|
|
159
|
+
"cartesia/sonic-turbo": "cartesia/sonic-turbo";
|
|
160
|
+
"cartesia/sonic-turbo-latest": "cartesia/sonic-turbo-latest";
|
|
161
|
+
}>>;
|
|
162
|
+
voice_id: z.ZodString;
|
|
163
|
+
speed: z.ZodOptional<z.ZodNullable<z.ZodNumber>>;
|
|
164
|
+
emotion: z.ZodOptional<z.ZodNullable<z.ZodString>>;
|
|
165
|
+
language: z.ZodOptional<z.ZodNullable<z.ZodString>>;
|
|
166
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
167
|
+
provider: z.ZodLiteral<"breezeblue">;
|
|
168
|
+
model: z.ZodDefault<z.ZodString>;
|
|
169
|
+
voice_id: z.ZodString;
|
|
170
|
+
guidance_scale: z.ZodOptional<z.ZodNullable<z.ZodNumber>>;
|
|
171
|
+
instructions: z.ZodOptional<z.ZodNullable<z.ZodString>>;
|
|
172
|
+
language: z.ZodOptional<z.ZodNullable<z.ZodString>>;
|
|
173
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
174
|
+
provider: z.ZodLiteral<"fish">;
|
|
175
|
+
model: z.ZodDefault<z.ZodEnum<{
|
|
176
|
+
"speech-1.6": "speech-1.6";
|
|
177
|
+
s1: "s1";
|
|
178
|
+
"s2-pro": "s2-pro";
|
|
179
|
+
"speech-1.5": "speech-1.5";
|
|
180
|
+
"s1-mini": "s1-mini";
|
|
181
|
+
}>>;
|
|
182
|
+
voice_id: z.ZodString;
|
|
183
|
+
speed: z.ZodOptional<z.ZodNullable<z.ZodNumber>>;
|
|
184
|
+
emotion: z.ZodOptional<z.ZodNullable<z.ZodString>>;
|
|
185
|
+
language: z.ZodOptional<z.ZodNullable<z.ZodString>>;
|
|
186
|
+
}, z.core.$strict>], "provider">>>;
|
|
187
|
+
voiceId: z.ZodOptional<z.ZodNullable<z.ZodString>>;
|
|
188
|
+
clipLibrary: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
189
|
+
clip_id: z.ZodString;
|
|
190
|
+
source_video_url: z.ZodOptional<z.ZodString>;
|
|
191
|
+
video_cache_id: z.ZodOptional<z.ZodString>;
|
|
192
|
+
max_seconds: z.ZodOptional<z.ZodNumber>;
|
|
193
|
+
trigger: z.ZodOptional<z.ZodEnum<{
|
|
194
|
+
idle: "idle";
|
|
195
|
+
listen: "listen";
|
|
196
|
+
think: "think";
|
|
197
|
+
directive: "directive";
|
|
198
|
+
}>>;
|
|
199
|
+
loop: z.ZodOptional<z.ZodBoolean>;
|
|
200
|
+
weight: z.ZodOptional<z.ZodNumber>;
|
|
201
|
+
crossfade_ms: z.ZodOptional<z.ZodNumber>;
|
|
202
|
+
trim_start_ms: z.ZodOptional<z.ZodNumber>;
|
|
203
|
+
trim_end_ms: z.ZodOptional<z.ZodNumber>;
|
|
204
|
+
when: z.ZodOptional<z.ZodString>;
|
|
205
|
+
hint: z.ZodOptional<z.ZodString>;
|
|
206
|
+
}, z.core.$strict>>>;
|
|
207
|
+
sceneGraph: z.ZodOptional<z.ZodObject<{
|
|
208
|
+
scenes: z.ZodArray<z.ZodObject<{
|
|
209
|
+
scene_id: z.ZodString;
|
|
210
|
+
hub_clip_id: z.ZodString;
|
|
211
|
+
clips: z.ZodArray<z.ZodObject<{
|
|
212
|
+
clip_id: z.ZodString;
|
|
213
|
+
source_video_url: z.ZodOptional<z.ZodString>;
|
|
214
|
+
video_cache_id: z.ZodOptional<z.ZodString>;
|
|
215
|
+
max_seconds: z.ZodOptional<z.ZodNumber>;
|
|
216
|
+
trigger: z.ZodOptional<z.ZodEnum<{
|
|
217
|
+
idle: "idle";
|
|
218
|
+
listen: "listen";
|
|
219
|
+
think: "think";
|
|
220
|
+
directive: "directive";
|
|
221
|
+
}>>;
|
|
222
|
+
loop: z.ZodOptional<z.ZodBoolean>;
|
|
223
|
+
weight: z.ZodOptional<z.ZodNumber>;
|
|
224
|
+
crossfade_ms: z.ZodOptional<z.ZodNumber>;
|
|
225
|
+
trim_start_ms: z.ZodOptional<z.ZodNumber>;
|
|
226
|
+
trim_end_ms: z.ZodOptional<z.ZodNumber>;
|
|
227
|
+
when: z.ZodOptional<z.ZodString>;
|
|
228
|
+
hint: z.ZodOptional<z.ZodString>;
|
|
229
|
+
}, z.core.$strict>>;
|
|
230
|
+
}, z.core.$strict>>;
|
|
231
|
+
transitions: z.ZodArray<z.ZodObject<{
|
|
232
|
+
clip_id: z.ZodString;
|
|
233
|
+
source_video_url: z.ZodString;
|
|
234
|
+
from_scene: z.ZodString;
|
|
235
|
+
to_scene: z.ZodString;
|
|
236
|
+
max_seconds: z.ZodOptional<z.ZodNumber>;
|
|
237
|
+
}, z.core.$strict>>;
|
|
238
|
+
}, z.core.$strict>>;
|
|
239
|
+
behavior: z.ZodOptional<z.ZodObject<{
|
|
240
|
+
gestures_enabled: z.ZodOptional<z.ZodBoolean>;
|
|
241
|
+
gesture_freq: z.ZodOptional<z.ZodEnum<{
|
|
242
|
+
sparse: "sparse";
|
|
243
|
+
balanced: "balanced";
|
|
244
|
+
lively: "lively";
|
|
245
|
+
}>>;
|
|
246
|
+
}, z.core.$strict>>;
|
|
247
|
+
renderBackend: z.ZodOptional<z.ZodString>;
|
|
248
|
+
supportEdits: z.ZodOptional<z.ZodObject<{
|
|
249
|
+
instruction: z.ZodString;
|
|
250
|
+
reference_url: z.ZodOptional<z.ZodString>;
|
|
251
|
+
live_edit: z.ZodOptional<z.ZodObject<{
|
|
252
|
+
rules: z.ZodString;
|
|
253
|
+
cooldown_seconds: z.ZodOptional<z.ZodNumber>;
|
|
254
|
+
renderer: z.ZodOptional<z.ZodEnum<{
|
|
255
|
+
generative: "generative";
|
|
256
|
+
editor: "editor";
|
|
257
|
+
}>>;
|
|
258
|
+
}, z.core.$strict>>;
|
|
259
|
+
}, z.core.$strict>>;
|
|
260
|
+
transcriptWebhook: z.ZodOptional<z.ZodObject<{
|
|
261
|
+
url: z.ZodString;
|
|
262
|
+
secret: z.ZodString;
|
|
263
|
+
}, z.core.$strict>>;
|
|
264
|
+
clientMetadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
|
|
265
|
+
}, z.core.$strict>;
|
|
266
|
+
declare const liveKitSessionGrantSchema: z.ZodObject<{
|
|
267
|
+
status: z.ZodDefault<z.ZodLiteral<"ready">>;
|
|
268
|
+
session_id: z.ZodString;
|
|
269
|
+
room_name: z.ZodString;
|
|
270
|
+
livekit_url: z.ZodString;
|
|
271
|
+
participant_token: z.ZodString;
|
|
272
|
+
participant_identity: z.ZodString;
|
|
273
|
+
reservation_expires_at: z.ZodString;
|
|
274
|
+
stt_mode: z.ZodDefault<z.ZodEnum<{
|
|
275
|
+
server: "server";
|
|
276
|
+
off: "off";
|
|
277
|
+
}>>;
|
|
278
|
+
room_created: z.ZodDefault<z.ZodBoolean>;
|
|
279
|
+
dispatch_created: z.ZodDefault<z.ZodBoolean>;
|
|
280
|
+
join_timeout_seconds: z.ZodDefault<z.ZodNumber>;
|
|
281
|
+
idle_timeout_seconds: z.ZodDefault<z.ZodNumber>;
|
|
282
|
+
max_session_seconds: z.ZodDefault<z.ZodNumber>;
|
|
283
|
+
}, z.core.$loose>;
|
|
284
|
+
declare const sessionEndReasonSchema: z.ZodEnum<{
|
|
285
|
+
idle: "idle";
|
|
286
|
+
disconnected: "disconnected";
|
|
287
|
+
failed: "failed";
|
|
288
|
+
user_ended: "user_ended";
|
|
289
|
+
session_cap: "session_cap";
|
|
290
|
+
out_of_credits: "out_of_credits";
|
|
291
|
+
agent_ended: "agent_ended";
|
|
292
|
+
}>;
|
|
293
|
+
type SessionEndReasonLabel = z.infer<typeof sessionEndReasonSchema>;
|
|
294
|
+
declare const approachingEndReasonSchema: z.ZodEnum<{
|
|
295
|
+
idle: "idle";
|
|
296
|
+
session_cap: "session_cap";
|
|
297
|
+
}>;
|
|
298
|
+
type ApproachingEndReason = z.infer<typeof approachingEndReasonSchema>;
|
|
299
|
+
declare const knownBehaviorStates: readonly ["idle", "listening", "thinking", "speaking"];
|
|
300
|
+
type KnownBehaviorState = (typeof knownBehaviorStates)[number];
|
|
301
|
+
declare const capacityBusyResponseSchema: z.ZodObject<{
|
|
302
|
+
message: z.ZodString;
|
|
303
|
+
capacity: z.ZodObject<{
|
|
304
|
+
capacity_pool: z.ZodOptional<z.ZodString>;
|
|
305
|
+
agent_name: z.ZodOptional<z.ZodString>;
|
|
306
|
+
max_sessions: z.ZodNumber;
|
|
307
|
+
max_sessions_per_gpu: z.ZodOptional<z.ZodNumber>;
|
|
308
|
+
worker_count: z.ZodNumber;
|
|
309
|
+
active_sessions: z.ZodNumber;
|
|
310
|
+
reserved_sessions: z.ZodNumber;
|
|
311
|
+
observed_worker_active_sessions: z.ZodNumber;
|
|
312
|
+
available_sessions: z.ZodNumber;
|
|
313
|
+
queue_size: z.ZodNumber;
|
|
314
|
+
admission_open: z.ZodBoolean;
|
|
315
|
+
recommended_retry_ms: z.ZodNumber;
|
|
316
|
+
load: z.ZodNumber;
|
|
317
|
+
}, z.core.$loose>;
|
|
318
|
+
queue_size: z.ZodNumber;
|
|
319
|
+
queue_ticket_id: z.ZodOptional<z.ZodString>;
|
|
320
|
+
queue_position: z.ZodOptional<z.ZodNumber>;
|
|
321
|
+
recommended_retry_ms: z.ZodNumber;
|
|
322
|
+
}, z.core.$strict>;
|
|
323
|
+
declare const liveKitSessionReleaseReasonSchema: z.ZodEnum<{
|
|
324
|
+
page_hide: "page_hide";
|
|
325
|
+
disconnected: "disconnected";
|
|
326
|
+
superseded: "superseded";
|
|
327
|
+
unmount: "unmount";
|
|
328
|
+
manual: "manual";
|
|
329
|
+
idle_timeout: "idle_timeout";
|
|
330
|
+
}>;
|
|
331
|
+
type LLMProvider = z.infer<typeof llmProviderSchema>;
|
|
332
|
+
type LLMSelection = z.infer<typeof llmSelectionSchema>;
|
|
333
|
+
type LLMSelectionForProvider<TProvider extends LLMProvider = LLMProvider> = TProvider extends LLMProvider ? Omit<LLMSelection, "provider"> & {
|
|
334
|
+
provider: TProvider;
|
|
335
|
+
} : never;
|
|
336
|
+
type CartesiaTtsModel = z.infer<typeof cartesiaTtsModelSchema>;
|
|
337
|
+
type FishTtsModel = z.infer<typeof fishTtsModelSchema>;
|
|
338
|
+
type VoiceSpec = z.infer<typeof voiceSpecSchema>;
|
|
339
|
+
type VoiceSpecInput = z.input<typeof voiceSpecSchema>;
|
|
340
|
+
type LiveKitSessionRequestInput = z.input<typeof liveKitSessionRequestSchema>;
|
|
341
|
+
type LiveKitSessionGrant = z.infer<typeof liveKitSessionGrantSchema>;
|
|
342
|
+
type CapacityBusyResponse = z.infer<typeof capacityBusyResponseSchema>;
|
|
343
|
+
type LiveKitSessionReleaseReason = z.infer<typeof liveKitSessionReleaseReasonSchema>;
|
|
344
|
+
|
|
345
|
+
type LiveKitSessionRequest<TLlmProvider extends LLMProvider = LLMProvider> = Omit<LiveKitSessionRequestInput, "llm" | "voice"> & {
|
|
346
|
+
llm?: LLMSelectionForProvider<TLlmProvider> | null;
|
|
347
|
+
voice?: VoiceSpecInput | null;
|
|
348
|
+
};
|
|
349
|
+
|
|
350
|
+
/** The subscriber-cap levels we drive (map to LiveKit VideoQuality LOW/HIGH). */
|
|
351
|
+
type QualityCap = "low" | "high";
|
|
352
|
+
/** The governor's finite states (the video-quality design doc). */
|
|
353
|
+
type GovernorState = "opening" | "opening_high" | "cap_low_sticky" | "cap_low_eligible" | "probing_up" | "cap_high_stable";
|
|
354
|
+
/**
|
|
355
|
+
* One normalized input the hook feeds each tick. The hook translates ALL of
|
|
356
|
+
* {TrackStreamStateChanged, ConnectionQualityChanged, getStats, rVFC freeze} into
|
|
357
|
+
* this shape, so the core never sees a vendor type.
|
|
358
|
+
*/
|
|
359
|
+
interface GovernorSignal {
|
|
360
|
+
/** SFU congestion controller paused the track since the last tick — Tier-0, the
|
|
361
|
+
* single most trustworthy downgrade trigger (server already confirmed congestion). */
|
|
362
|
+
paused: boolean;
|
|
363
|
+
/** Frozen milliseconds observed in the trailing freeze window W (max of inbound-rtp
|
|
364
|
+
* delta and the rVFC-derived gap — Safari coverage). */
|
|
365
|
+
freezeMsInWindow: number;
|
|
366
|
+
/** jitterBufferDelay is trending up — the earliest LEADING pre-freeze sign. */
|
|
367
|
+
jitterRising: boolean;
|
|
368
|
+
/** LiveKit ConnectionQuality — a LAGGING corroborator only (jitter/RTT are disabled
|
|
369
|
+
* in its score + known false-Poor bugs). Blocks up-probes; never a sole trigger. */
|
|
370
|
+
connectionQuality: "excellent" | "good" | "poor" | "lost" | "unknown";
|
|
371
|
+
/** The tab is hidden / track muted / freezes correlate with local CPU not network —
|
|
372
|
+
* the false-positive fence (our own Dia-freeze lesson). When true the machine is
|
|
373
|
+
* frozen: no signal is trusted, no transition fires. */
|
|
374
|
+
inhibited: boolean;
|
|
375
|
+
}
|
|
376
|
+
/** The side effect the hook must apply after a step (absent = leave the cap alone). */
|
|
377
|
+
interface GovernorAction {
|
|
378
|
+
setCap: QualityCap;
|
|
379
|
+
}
|
|
380
|
+
interface GovernorConfig {
|
|
381
|
+
/** Where the cap OPENS. "low" (default): the shipped posture — first impression is
|
|
382
|
+
* never a freeze, at the cost of a visible soft start (openingDwellMs + cleanMs
|
|
383
|
+
* before the first up-probe, plus the LOW→HIGH rung switch's decoder-reconfigure
|
|
384
|
+
* keyframe pop). "high": first impression is never a RAMP — the session opens at
|
|
385
|
+
* the top rung already under the STRICT probation bar (probationFreezeMs, paused
|
|
386
|
+
* = instant), so a link that cannot afford it is demoted within one tick of
|
|
387
|
+
* evidence and then follows the normal ladder with a failure on record. Pick
|
|
388
|
+
* "high" for surfaces where the opening softness reads as a defect (the canary
|
|
389
|
+
* demo comparison, 2026-08-13) and the link population skews capable. */
|
|
390
|
+
openingCap: QualityCap;
|
|
391
|
+
/** Frozen-ms in window W that forces an immediate downgrade from a stable cap. */
|
|
392
|
+
downgradeFreezeMs: number;
|
|
393
|
+
/** Stricter frozen-ms bar DURING probation — kill a bad upgrade fast. */
|
|
394
|
+
probationFreezeMs: number;
|
|
395
|
+
/** Hold at low at SESSION START before an up-probe is considered. Separate from
|
|
396
|
+
* dwellBaseMs because the opening carries no failure evidence — the 8s recovery
|
|
397
|
+
* dwell was being served to every fresh call, and prod measurement
|
|
398
|
+
* (call_connect_profile, 2026-08-05) showed the cost: median 14s at the small
|
|
399
|
+
* rung before first upgrade, 56% of 30s windows never upgrading at all. Opening
|
|
400
|
+
* low is kept (first impression is never a freeze); paying the POST-FAILURE
|
|
401
|
+
* penalty before any failure is not. */
|
|
402
|
+
openingDwellMs: number;
|
|
403
|
+
/** Base minimum hold at low before an up-probe is considered (grows on failure). */
|
|
404
|
+
dwellBaseMs: number;
|
|
405
|
+
/** Cap on the exponential dwell backoff — never pin low permanently. */
|
|
406
|
+
dwellMaxMs: number;
|
|
407
|
+
/** Continuously-healthy window required before raising the cap. 3s: still well above
|
|
408
|
+
* the sub-second downgrade reaction (the asymmetry that prevents flap), but short
|
|
409
|
+
* enough that a clean link reaches the probe at ~5s from session start
|
|
410
|
+
* (openingDwellMs + cleanMs) instead of 13s under the old 8s+5s posture. */
|
|
411
|
+
cleanMs: number;
|
|
412
|
+
/** Probation length after raising the cap before committing to high. */
|
|
413
|
+
probeMs: number;
|
|
414
|
+
/** Sustained-healthy-at-low duration that resets the failure count (link improved). */
|
|
415
|
+
healthyResetMs: number;
|
|
416
|
+
}
|
|
417
|
+
/** The grounded defaults (Meet <10s recovery + GCC +5%/−15% step asymmetry). */
|
|
418
|
+
declare const DEFAULT_GOVERNOR_CONFIG: GovernorConfig;
|
|
419
|
+
/** The full reducer state (immutable; `step` returns a new object on change). */
|
|
420
|
+
interface Governor {
|
|
421
|
+
state: GovernorState;
|
|
422
|
+
cap: QualityCap;
|
|
423
|
+
/** Consecutive failed up-probes — drives the exponential dwell backoff. */
|
|
424
|
+
failures: number;
|
|
425
|
+
/** Wall-clock ms the current state was entered (for dwell/clean/probe timing). */
|
|
426
|
+
enteredAtMs: number;
|
|
427
|
+
/** Wall-clock ms of the last healthy tick in the current low period (clean-window
|
|
428
|
+
* accumulation); null until the first healthy tick after entering low. */
|
|
429
|
+
healthySinceMs: number | null;
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
/** How the media is fit into the surface box. Mirrors CSS `object-fit`. */
|
|
433
|
+
type AvatarVideoFit = "contain" | "cover";
|
|
434
|
+
type AvatarVideoSurfaceProps = {
|
|
435
|
+
/**
|
|
436
|
+
* The avatar's idle/ambient clip, natively looped, layered OVER the
|
|
437
|
+
* {@link poster} floor. When a realtime turn is not producing media — including
|
|
438
|
+
* after a session disconnect — the surface rests on this clip. Pass `null` for
|
|
439
|
+
* avatars with no idle clip; the poster floor then shows at rest.
|
|
440
|
+
*/
|
|
441
|
+
idleVideoUrl: string | null;
|
|
442
|
+
/**
|
|
443
|
+
* The avatar's PORTRAIT (its face) — the DEEPEST floor, rendered behind the idle
|
|
444
|
+
* clip and the live video whenever provided. It paints IMMEDIATELY on connect
|
|
445
|
+
* (an `<img>` decodes before a `<video>` reaches its first frame) and is the
|
|
446
|
+
* guaranteed no-frames backdrop, so the surface is NEVER a black box while
|
|
447
|
+
* connecting / listening / idle / between turns. With an idle clip it sits behind
|
|
448
|
+
* it; with none it is the resting floor. Pass `null` to fall through to the
|
|
449
|
+
* caller's own branded floor. Rendered with the same fit/box as the video layers
|
|
450
|
+
* so swaps are in place.
|
|
451
|
+
*/
|
|
452
|
+
poster?: string | null;
|
|
453
|
+
/**
|
|
454
|
+
* The caller's INTENT to show the live stream — typically "the avatar is
|
|
455
|
+
* actively producing media this turn". The surface still gates this on the
|
|
456
|
+
* LiveKit connection state and a subscribed, PRODUCING video track, so a stale
|
|
457
|
+
* `true` after a drop can never reveal a frozen/black live layer: it falls back
|
|
458
|
+
* to the idle clip in the SAME box. Defaults to `true` (show live whenever a
|
|
459
|
+
* connected, producing track exists).
|
|
460
|
+
*/
|
|
461
|
+
live?: boolean;
|
|
462
|
+
/**
|
|
463
|
+
* Enable the subscriber-side fast-down/sticky-up quality governor. It starts on
|
|
464
|
+
* the lower simulcast rung, reacts immediately to an SFU pause or decoded-frame
|
|
465
|
+
* freeze, and only probes the full layer after a clean dwell. Default true.
|
|
466
|
+
*/
|
|
467
|
+
adaptiveQuality?: boolean;
|
|
468
|
+
/**
|
|
469
|
+
* Opening bet for the governor when `adaptiveQuality` is on. `"high"` opens on the
|
|
470
|
+
* FULL simulcast layer under probation — sharp from frame 1, no LOW→HIGH layer-walk —
|
|
471
|
+
* and instant-demotes a cold link within one tick; `"low"` is the reactive soft-open.
|
|
472
|
+
* Unset ⇒ the fleet default (`DEFAULT_GOVERNOR_CONFIG.openingCap`, currently `"low"`).
|
|
473
|
+
* `<AvatarCall>` sets this to `"high"` only for rtx6000-pool sessions (see
|
|
474
|
+
* opening-cap-policy.ts); everything else keeps the soft-open.
|
|
475
|
+
*/
|
|
476
|
+
openingCap?: QualityCap;
|
|
477
|
+
/** `object-fit` for BOTH layers. Both layers always use the SAME fit + box so
|
|
478
|
+
* the front (live) fully covers the back (idle) — no peek-through. */
|
|
479
|
+
fit?: AvatarVideoFit;
|
|
480
|
+
/**
|
|
481
|
+
* @deprecated No-op. The surface FILLS its container (`size-full`); the CONSUMER
|
|
482
|
+
* owns the aspect box. Self-pinning a native ratio INSIDE the surface could not
|
|
483
|
+
* survive an indefinite-height ancestor (it collapsed to zero height — the
|
|
484
|
+
* black-screen bug), so aspect ownership moved out to the caller: wrap the
|
|
485
|
+
* surface in a box with a definite size (e.g. `aspect-[9/16]` + a height seed).
|
|
486
|
+
* Accepted for source compatibility; ignored.
|
|
487
|
+
*/
|
|
488
|
+
pinNativeAspect?: boolean;
|
|
489
|
+
/** CSS `aspect-ratio` placeholder for the box (a size-jump guard on first paint).
|
|
490
|
+
* Applied to the fill box when set; the caller's own aspect box still wins. */
|
|
491
|
+
aspectRatio?: string;
|
|
492
|
+
/**
|
|
493
|
+
* @deprecated Retained for source compatibility; the idle↔live handover is a SWAP,
|
|
494
|
+
* not a blend. Both layers show the same body from independent clocks, so any
|
|
495
|
+
* overlap is a double exposure rather than a softening (measured: 31.8% of the
|
|
496
|
+
* picture differs at the median cursor pair). The layers are made to agree at the
|
|
497
|
+
* ANCHOR instead, where a swap is invisible — which is also the falsifiable test:
|
|
498
|
+
* 0 and 500 must look identical. Accepted; ignored.
|
|
499
|
+
*/
|
|
500
|
+
crossfadeMs?: number;
|
|
501
|
+
/**
|
|
502
|
+
* Debounce (ms) before dropping BACK to the idle clip once the live layer stops
|
|
503
|
+
* being shown (turn end), so back-to-back turns don't flash the idle clip for a
|
|
504
|
+
* frame between them. Default 700. A disconnect bypasses this and reverts to idle
|
|
505
|
+
* immediately (a dead room is never held). Showing the live layer is always
|
|
506
|
+
* immediate — only the hide is debounced.
|
|
507
|
+
*/
|
|
508
|
+
idleReturnDelayMs?: number;
|
|
509
|
+
/**
|
|
510
|
+
* Maximum time without a newly presented decoded frame before the live layer is
|
|
511
|
+
* treated as stalled and immediately replaced by the idle/poster floor. Default 800ms.
|
|
512
|
+
*/
|
|
513
|
+
frameStallMs?: number;
|
|
514
|
+
/** Extra className for the box (the layers fill it). */
|
|
515
|
+
className?: string;
|
|
516
|
+
/** Extra inline style for the box. */
|
|
517
|
+
style?: CSSProperties;
|
|
518
|
+
/** Overlay content rendered above both media layers (badges, chrome, scrims). */
|
|
519
|
+
children?: ReactNode;
|
|
520
|
+
/** Surface a small "live · WxH" badge when the live layer is shown. Default true. */
|
|
521
|
+
showLiveBadge?: boolean;
|
|
522
|
+
/** Test id for the box. */
|
|
523
|
+
"data-testid"?: string;
|
|
524
|
+
};
|
|
525
|
+
/**
|
|
526
|
+
* The avatar video surface: a single box that FILLS its container and renders the
|
|
527
|
+
* looping idle clip and the realtime LiveKit video as two PIXEL-ALIGNED layers,
|
|
528
|
+
* with an anchor-aligned SWAP between them and an automatic fall-back to the idle
|
|
529
|
+
* clip whenever the realtime session is not connected or producing (max-duration
|
|
530
|
+
* end, network drop, agent gone, turn-end mute).
|
|
531
|
+
*
|
|
532
|
+
* ONE BODY, ONE POSE. Only one media layer ever runs: the idle clip is paused while
|
|
533
|
+
* the live layer covers it, and re-enters at its anchor (frame 0) when it takes over.
|
|
534
|
+
* That is what lets the handover be a swap with no blending — see the front-layer and
|
|
535
|
+
* `useIdleWhileResting` notes for the measurements behind it.
|
|
536
|
+
*
|
|
537
|
+
* This is the SSOT for "is the avatar live right now": it reads the LiveKit
|
|
538
|
+
* connection state ({@link useConnectionState}) and the bound avatar track
|
|
539
|
+
* ({@link useVoiceAssistant}) directly, so a caller's stale `live` intent can
|
|
540
|
+
* never leave a frozen/black frame on screen — the surface always reverts to the
|
|
541
|
+
* idle loop in the SAME box. Must be rendered inside a LiveKit `RoomContext`
|
|
542
|
+
* (e.g. under `RealtimeAvatarLiveKitRoom`).
|
|
543
|
+
*
|
|
544
|
+
* LAYOUT CONTRACT: the surface FILLS its container (`size-full`) — it does NOT pin
|
|
545
|
+
* its own aspect. The CONSUMER owns the aspect box: wrap the surface in an element
|
|
546
|
+
* with a DEFINITE size (e.g. `aspect-[9/16]` plus a height seed). Self-pinning a
|
|
547
|
+
* native ratio inside the surface (`h-full w-auto`) collapsed to zero height
|
|
548
|
+
* against an indefinite-height (flex) ancestor — the black-screen bug — so that
|
|
549
|
+
* ownership moved out. Both media layers are `absolute inset-0` over the filled box
|
|
550
|
+
* and share the SAME `object-fit`, so the front layer fully covers the back with no
|
|
551
|
+
* sliver.
|
|
552
|
+
*/
|
|
553
|
+
declare function AvatarVideoSurface(props: AvatarVideoSurfaceProps): ReactElement;
|
|
554
|
+
/** Which back (non-live) layers the surface renders, deepest first. */
|
|
555
|
+
type SurfaceLayers = {
|
|
556
|
+
/** Render the poster `<img>` — the deepest floor, the always-present face. */
|
|
557
|
+
showPoster: boolean;
|
|
558
|
+
/** Render the idle clip `<video>`, layered OVER the poster. */
|
|
559
|
+
showIdleVideo: boolean;
|
|
560
|
+
};
|
|
561
|
+
/**
|
|
562
|
+
* The COARSE producing gate the React Native (Android) surface twin passes to
|
|
563
|
+
* {@link useLiveTrackProducing} in place of {@link isLiveTrackProducing}.
|
|
564
|
+
*
|
|
565
|
+
* On Android #446 mounts the live `<VideoTrack>` (a below-window SurfaceView) ONLY
|
|
566
|
+
* while it's producing, so this predicate is the upstream switch that decides
|
|
567
|
+
* whether the video shows at all — and the strict {@link isLiveTrackProducing}
|
|
568
|
+
* reads it false there. react-native-webrtc's REMOTE `MediaStreamTrack` does not
|
|
569
|
+
* drive the browser-shaped micro-signals that predicate depends on: `enabled` is a
|
|
570
|
+
* LOCAL playback toggle (not frame flow), and a remote track's `muted` can read
|
|
571
|
+
* `true` for an entire producing turn (or never emit `unmute`). So on a real
|
|
572
|
+
* Android device the strict test returned false while frames flowed and the avatar
|
|
573
|
+
* froze on its poster ("voice only / static image").
|
|
574
|
+
*
|
|
575
|
+
* We therefore gate on the one fact react-native-webrtc reports reliably: a
|
|
576
|
+
* subscribed remote track whose underlying `MediaStreamTrack` has not ENDED. The
|
|
577
|
+
* trade-off is that a turn-end no longer crossfades back to the idle clip on
|
|
578
|
+
* Android (the live layer stays up between turns) — an acceptable price for the
|
|
579
|
+
* video actually appearing. The room's connection state still unmounts the layer on
|
|
580
|
+
* disconnect, and the track's `ended` event (wired in {@link useLiveTrackProducing})
|
|
581
|
+
* still tears it down. iOS keeps {@link isLiveTrackProducing}: its in-tree UIView
|
|
582
|
+
* and DOM-shaped track behave like the web, where the strict signal works today.
|
|
583
|
+
*
|
|
584
|
+
* Pure + DOM-free (reads only `mediaStreamTrack.readyState`), so it lives here with
|
|
585
|
+
* its web twin and is unit-tested in the shared suite rather than the RN module.
|
|
586
|
+
*/
|
|
587
|
+
declare function isNativeLiveTrackSubscribed(videoTrack: TrackReferenceOrPlaceholder | undefined): boolean;
|
|
588
|
+
/**
|
|
589
|
+
* Reactive {@link isLiveTrackProducing}: returns whether the avatar's live video
|
|
590
|
+
* is producing frames, and re-renders the surface when that changes.
|
|
591
|
+
*
|
|
592
|
+
* `useVoiceAssistant` re-renders when the track APPEARS/DISAPPEARS, but NOT when a
|
|
593
|
+
* present track merely mutes — so a plain read in render would be stale and leave
|
|
594
|
+
* the frozen last frame up after a turn ends. We subscribe to the publication's
|
|
595
|
+
* own lifecycle events — the SDK's `TrackEvent.Muted`/`Unmuted` (turn end/start)
|
|
596
|
+
* and `Ended` (track torn down) — PLUS the underlying MediaStreamTrack's
|
|
597
|
+
* mute/unmute/ended (which fire even when the publication's mute flag is stale),
|
|
598
|
+
* and recompute on each, so a turn-end crossfades back to the idle loop
|
|
599
|
+
* immediately. Unconditional (safe with no track) and re-subscribes whenever the
|
|
600
|
+
* underlying publication/track changes.
|
|
601
|
+
*
|
|
602
|
+
* Exported (not just the pure {@link isLiveTrackProducing}) because it is
|
|
603
|
+
* DOM-free — publication events + MediaStreamTrack events exist on React Native's
|
|
604
|
+
* WebRTC shim too — so the react-native surface twin reuses THIS hook (the event
|
|
605
|
+
* wiring never drifts between platforms) but may pass its OWN `isProducing`
|
|
606
|
+
* predicate: react-native-webrtc's remote track does not drive the browser-shaped
|
|
607
|
+
* `enabled`/`muted` flags {@link isLiveTrackProducing} reads, so Android supplies a
|
|
608
|
+
* coarser subscribed-and-not-ended test (see the RN surface twin). Defaults to
|
|
609
|
+
* {@link isLiveTrackProducing} so every web caller is unchanged.
|
|
610
|
+
*/
|
|
611
|
+
declare function useLiveTrackProducing(videoTrack: TrackReferenceOrPlaceholder | undefined, isProducing?: (videoTrack: TrackReferenceOrPlaceholder | undefined) => boolean): boolean;
|
|
612
|
+
/** A `<video>` narrowed to the members the live-playback keeper touches. */
|
|
613
|
+
type PlayableVideoElement = {
|
|
614
|
+
paused: boolean;
|
|
615
|
+
play: () => Promise<void> | void;
|
|
616
|
+
addEventListener: (type: string, listener: () => void) => void;
|
|
617
|
+
removeEventListener: (type: string, listener: () => void) => void;
|
|
618
|
+
};
|
|
619
|
+
/** Controls a live `<video>`'s continuous playback across turns (the freeze fix). */
|
|
620
|
+
type LivePlaybackKeeper = {
|
|
621
|
+
/** Bind (or rebind) the element to keep playing; pass `null` on detach. */
|
|
622
|
+
bind: (video: PlayableVideoElement | null) => void;
|
|
623
|
+
/** Re-check playback after a tab/page resume. */
|
|
624
|
+
resume: () => void;
|
|
625
|
+
/** Stop keeping the element alive and drop its listeners. */
|
|
626
|
+
dispose: () => void;
|
|
627
|
+
};
|
|
628
|
+
|
|
629
|
+
declare class RealtimeAvatarCapacityError extends Error {
|
|
630
|
+
readonly busy: CapacityBusyResponse;
|
|
631
|
+
constructor(message: string, busy: CapacityBusyResponse);
|
|
632
|
+
get queueSize(): number;
|
|
633
|
+
get queuePosition(): number | undefined;
|
|
634
|
+
get queueTicketId(): string | undefined;
|
|
635
|
+
get recommendedRetryMs(): number;
|
|
636
|
+
}
|
|
637
|
+
|
|
638
|
+
/**
|
|
639
|
+
* What the React bindings actually need from a client: mint a call, and give the slot back.
|
|
640
|
+
*
|
|
641
|
+
* They used to demand `RealtimeAvatarClient` itself, and that is why `AvatarCall` could not be
|
|
642
|
+
* used at all. The component is exported at runtime from `realtime-avatar/react` and required
|
|
643
|
+
* `client: RealtimeAvatarClient` — but that package exports no client constructor, because the
|
|
644
|
+
* class carries an API-key path and keeping it out of the browser is what took `apiKey`/`Bearer`
|
|
645
|
+
* to zero occurrences in the shipped bundles. So the prop named a type no consumer could produce.
|
|
646
|
+
*
|
|
647
|
+
* The fix is nominal, not structural: the bindings depend on this INTERFACE instead of on that
|
|
648
|
+
* class. Anything with these five methods satisfies it — a fake in a test, or a thin fetch wrapper
|
|
649
|
+
* over an app's own proxy routes, which is what an integrator writes anyway.
|
|
650
|
+
*
|
|
651
|
+
* The carried class itself is GONE as of 2026-08-26. Once nothing named it, a module-graph walk
|
|
652
|
+
* (`npm run reachable`) showed it and six files with it were reachable from nothing at all: 1,230
|
|
653
|
+
* lines of a second HTTP client, a second error path and a second key parser, shipping nowhere.
|
|
654
|
+
* So this interface is now the only client contract in the repo, which is the point.
|
|
655
|
+
*
|
|
656
|
+
* The LLM-provider generic is KEPT. It pairs the client with `session: LiveKitSessionRequest<T>`,
|
|
657
|
+
* so a client built for one provider set still cannot be handed a session requesting another —
|
|
658
|
+
* that check was never what made the prop unusable, and giving it up would have bought nothing.
|
|
659
|
+
*/
|
|
660
|
+
|
|
661
|
+
/** Per-request escape hatches. Lives here, not on the client, so the React half never imports it from there. */
|
|
662
|
+
type RealtimeAvatarRequestOptions = {
|
|
663
|
+
signal?: AbortSignal;
|
|
664
|
+
headers?: HeadersInit;
|
|
665
|
+
};
|
|
666
|
+
/**
|
|
667
|
+
* Capacity exhaustion is a VALUE here, not a throw — `queued` is not an error, and modelling it
|
|
668
|
+
* as one is the most common bad first impression an integration makes.
|
|
669
|
+
*/
|
|
670
|
+
type LiveKitSessionStartResult = {
|
|
671
|
+
status: "ready";
|
|
672
|
+
grant: LiveKitSessionGrant;
|
|
673
|
+
} | {
|
|
674
|
+
status: "busy";
|
|
675
|
+
busy: CapacityBusyResponse;
|
|
676
|
+
};
|
|
677
|
+
interface AvatarSessionClient<TLlmProvider extends LLMProvider = LLMProvider> {
|
|
678
|
+
/** Mint a call, or report that every slot is busy. Never throws for capacity. */
|
|
679
|
+
createLiveKitSessionOrBusy(input: LiveKitSessionRequest<TLlmProvider>, options?: RealtimeAvatarRequestOptions): Promise<LiveKitSessionStartResult>;
|
|
680
|
+
/**
|
|
681
|
+
* Free a started call's slot. Returns `false` rather than throwing — a release that is lost
|
|
682
|
+
* is a slower release, never a broken page, because the join timeout is the backstop.
|
|
683
|
+
*/
|
|
684
|
+
releaseLiveKitSession(sessionId: string, reason?: LiveKitSessionReleaseReason, options?: RealtimeAvatarRequestOptions): Promise<boolean>;
|
|
685
|
+
/**
|
|
686
|
+
* The `pagehide` path. `sendBeacon` is the one send that outlives a closing page, and it is
|
|
687
|
+
* synchronous-or-nothing — hence a `boolean` return and no promise, so the caller can fall
|
|
688
|
+
* back to {@link releaseLiveKitSession} when the browser has no beacon.
|
|
689
|
+
*/
|
|
690
|
+
releaseLiveKitSessionBeacon(sessionId: string, reason?: LiveKitSessionReleaseReason): boolean;
|
|
691
|
+
/**
|
|
692
|
+
* A queued call holds no session id yet, so {@link releaseLiveKitSession} cannot free it.
|
|
693
|
+
* This is the queue's own release.
|
|
694
|
+
*/
|
|
695
|
+
releaseLiveKitQueueTicket(queueTicketId: string, reason?: LiveKitSessionReleaseReason, options?: RealtimeAvatarRequestOptions): Promise<boolean>;
|
|
696
|
+
releaseLiveKitQueueTicketBeacon(queueTicketId: string, reason?: LiveKitSessionReleaseReason): boolean;
|
|
697
|
+
}
|
|
698
|
+
|
|
699
|
+
/**
|
|
700
|
+
* One room transcription entry as surfaced by {@link useTranscriptions} — a LiveKit `TextStreamData`
|
|
701
|
+
* carrying the spoken `text` plus `participantInfo.identity` (WHO spoke) and `streamInfo`.
|
|
702
|
+
*/
|
|
703
|
+
type CallTranscriptSegment = ReturnType<typeof useTranscriptions>[number];
|
|
704
|
+
/**
|
|
705
|
+
* A live call's transcript, split by who spoke — so BOTH sides are first-class, not just the agent's.
|
|
706
|
+
* See {@link useCallTranscript}.
|
|
707
|
+
*/
|
|
708
|
+
type CallTranscript = {
|
|
709
|
+
/**
|
|
710
|
+
* The USER's spoken lines (their server-STT). Already on the wire — the worker's room input path
|
|
711
|
+
* publishes user input transcriptions to the room, attributed to the user participant — but
|
|
712
|
+
* previously only reachable by hand-filtering {@link useTranscriptions}. Empty until the user speaks.
|
|
713
|
+
*/
|
|
714
|
+
user: CallTranscriptSegment[];
|
|
715
|
+
/** The AGENT/character's spoken lines (every room transcription that isn't the local user's). */
|
|
716
|
+
agent: CallTranscriptSegment[];
|
|
717
|
+
};
|
|
718
|
+
/**
|
|
719
|
+
* Split room transcription entries into the user's vs the agent's, by participant identity.
|
|
720
|
+
*
|
|
721
|
+
* In an avatar call the browser user is the LOCAL participant and the character is a remote (agent)
|
|
722
|
+
* participant, so a segment authored by the local identity is the user's and everything else is the
|
|
723
|
+
* agent's. Splitting on the identity carried by each `TextStreamData` is exact (no id-scheme or
|
|
724
|
+
* ordering heuristic). When the local identity isn't known yet (pre-connect) nothing is attributed to
|
|
725
|
+
* the user — safer than a false positive. Pure + exported so it's unit-testable without a live room.
|
|
726
|
+
*/
|
|
727
|
+
declare function splitCallTranscript(all: CallTranscriptSegment[], localIdentity: string | null | undefined): CallTranscript;
|
|
728
|
+
/**
|
|
729
|
+
* The live call transcript, split into the user's and the agent's spoken lines.
|
|
730
|
+
*
|
|
731
|
+
* The user's side was always published to the room by the worker, but only the agent's was
|
|
732
|
+
* conveniently exposed (via {@link useVoiceAssistant}); this surfaces BOTH so adopters can build a
|
|
733
|
+
* two-sided call recap / caption history. Both arrays are empty before anyone speaks. Must be called
|
|
734
|
+
* inside a LiveKit room context (e.g. under {@link RealtimeAvatarLiveKitRoom}).
|
|
735
|
+
*/
|
|
736
|
+
declare function useCallTranscript(): CallTranscript;
|
|
737
|
+
/**
|
|
738
|
+
* In-room companion to {@link useMicLease}: fires the lease's PRECISE release the
|
|
739
|
+
* moment this call's local microphone MediaStreamTrack actually reaches `ended`
|
|
740
|
+
* (the true hardware-release signal) — not merely on React unmount. Pass the
|
|
741
|
+
* `token` returned by `useMicLease`.
|
|
742
|
+
*
|
|
743
|
+
* Why it exists: `Room.disconnect()` stops the local track on a NON-awaited path
|
|
744
|
+
* (livekit-client 2.19.2), so after a call's React teardown the browser keeps the
|
|
745
|
+
* capture device for a beat. Handing the tab-global mic lease to the next call at
|
|
746
|
+
* unmount can therefore let a rapid redial's getUserMedia collide with the still-
|
|
747
|
+
* closing device. This hook holds the lease across that gap and releases it on the
|
|
748
|
+
* track's own `ended` event; `useMicLease` carries a timeout backstop so a track
|
|
749
|
+
* that never emits `ended` can't wedge the lease. MUST be rendered inside the room
|
|
750
|
+
* (needs `useLocalParticipant`), e.g. under {@link RealtimeAvatarLiveKitRoom}.
|
|
751
|
+
*/
|
|
752
|
+
declare function useReleaseMicLeaseOnTrackEnded(token: symbol): void;
|
|
753
|
+
/**
|
|
754
|
+
* Receiver-side playout delay (SECONDS) for the avatar media tracks.
|
|
755
|
+
*
|
|
756
|
+
* The avatar is published from a GPU datacenter and reaches viewers over the
|
|
757
|
+
* public internet, where a few percent of packet loss is normal. A shallow
|
|
758
|
+
* receiver buffer stalls (freezes) on every loss while it waits for a
|
|
759
|
+
* retransmit; a deeper buffer recovers the lost packets BEFORE playout, so the
|
|
760
|
+
* viewer sees smooth 25fps instead of freezing. Measured at ~5% loss this took
|
|
761
|
+
* the stream from ~11fps with multi-second freezes to a steady 25fps with zero
|
|
762
|
+
* freezes.
|
|
763
|
+
*
|
|
764
|
+
* 0.5s is applied via the native {@link RemoteTrack.setPlayoutDelay}, which sets
|
|
765
|
+
* the receiver's `playoutDelayHint` — the same Chromium jitter-buffer knob the
|
|
766
|
+
* old hand-rolled `jitterBufferTarget` reached into (the spec renamed
|
|
767
|
+
* playoutDelayHint→jitterBufferTarget; both influence the same buffer depth),
|
|
768
|
+
* but typed and SDK-owned so there is no cast or feature-probe. The hint is NOT
|
|
769
|
+
* free at the open: the receiver holds the FIRST video frame toward the target
|
|
770
|
+
* (~250-450ms of measured TTFF), a deliberate cost paid once for the freeze-free
|
|
771
|
+
* steady state above. The open-small-then-ramp variant that tried to dodge that
|
|
772
|
+
* cost was shipped, failed twice (the rVFC ramp trigger is throttled on the
|
|
773
|
+
* opacity:0-mounted element, stranding the shallow buffer), and was deliberately
|
|
774
|
+
* removed in 2262e4c7b — do not re-propose it. No-op on browsers that don't
|
|
775
|
+
* support the hint (the SDK warns and moves on).
|
|
776
|
+
*/
|
|
777
|
+
declare const DEFAULT_AVATAR_PLAYOUT_DELAY_SECONDS = 0.5;
|
|
778
|
+
/**
|
|
779
|
+
* Sets the avatar AUDIO and VIDEO tracks' native playout delay so the stream has
|
|
780
|
+
* a real de-jitter / loss-recovery cushion for smooth, freeze-free playback.
|
|
781
|
+
*
|
|
782
|
+
* Both tracks are kept at the SAME delay so the two streams stay lip-synced
|
|
783
|
+
* (WebRTC syncs the pair to the LARGER of the two receivers' hints; an unequal
|
|
784
|
+
* video-only buffer would make the lips lag the audio). Pass the `videoTrack`
|
|
785
|
+
* and `audioTrack` from {@link useVoiceAssistant}. A `RemoteTrack` is required —
|
|
786
|
+
* placeholders/local tracks lack `setPlayoutDelay` and are skipped.
|
|
787
|
+
*/
|
|
788
|
+
declare function useAvatarPlayoutDelay(videoTrack: TrackReferenceOrPlaceholder | undefined, audioTrack: TrackReferenceOrPlaceholder | undefined, delaySeconds?: number): void;
|
|
789
|
+
type LiveKitAvatarGrantStatus = "idle" | "requesting" | "ready" | "busy" | "failed";
|
|
790
|
+
type LiveKitConnectionStatus = ReturnType<typeof useConnectionState>;
|
|
791
|
+
/**
|
|
792
|
+
* A clean, adopter-facing discriminated capacity signal derived from the raw
|
|
793
|
+
* grant {@link LiveKitAvatarGrantStatus}. The point is that a transient capacity
|
|
794
|
+
* wait ("all GPU slots busy, queued behind N others, auto-retrying") is NOT an
|
|
795
|
+
* error — it is a normal, self-healing waiting state. Only a genuinely failed
|
|
796
|
+
* grant maps to `error`. Adopters should render `queued`/`connecting` calmly
|
|
797
|
+
* (spinner + queue position) and reserve error UI for the `error` variant.
|
|
798
|
+
*
|
|
799
|
+
* - `idle` — no active session requested.
|
|
800
|
+
* - `connecting`— a grant fetch is in flight (no prior queue placement).
|
|
801
|
+
* - `queued` — all slots are busy; the client holds a queue ticket and is
|
|
802
|
+
* auto-retrying. Carries the raw {@link CapacityBusyResponse}
|
|
803
|
+
* (queue position/size, recommended retry delay). NOT an error.
|
|
804
|
+
* - `active` — a grant is held; the room can connect.
|
|
805
|
+
* - `error` — the grant request genuinely failed (not capacity-related).
|
|
806
|
+
*/
|
|
807
|
+
type LiveKitCapacityState = {
|
|
808
|
+
kind: "idle";
|
|
809
|
+
} | {
|
|
810
|
+
kind: "connecting";
|
|
811
|
+
} | {
|
|
812
|
+
kind: "queued";
|
|
813
|
+
busy: CapacityBusyResponse;
|
|
814
|
+
} | {
|
|
815
|
+
kind: "active";
|
|
816
|
+
grant: LiveKitSessionGrant;
|
|
817
|
+
} | {
|
|
818
|
+
kind: "error";
|
|
819
|
+
error: Error;
|
|
820
|
+
};
|
|
821
|
+
type LiveKitAvatarGrantState = {
|
|
822
|
+
status: LiveKitAvatarGrantStatus;
|
|
823
|
+
grant: LiveKitSessionGrant | null;
|
|
824
|
+
busy: CapacityBusyResponse | null;
|
|
825
|
+
error: Error | null;
|
|
826
|
+
/**
|
|
827
|
+
* Discriminated capacity signal — prefer this over `status` in UI: it
|
|
828
|
+
* separates a normal capacity wait (`queued`) from a real failure (`error`).
|
|
829
|
+
*/
|
|
830
|
+
capacity: LiveKitCapacityState;
|
|
831
|
+
refresh: () => void;
|
|
832
|
+
clear: () => void;
|
|
833
|
+
/**
|
|
834
|
+
* Eagerly release the CURRENTLY-HELD session's capacity lease (best-effort,
|
|
835
|
+
* fire-and-forget). Wire this to the LiveKit room's terminal `Disconnected`
|
|
836
|
+
* event so a session that ends for good frees its GPU slot immediately instead
|
|
837
|
+
* of lingering until the worker reconcile. A no-op when no grant is held.
|
|
838
|
+
* Releasing on a supersede (reconnect) and on unmount/tab-close is automatic.
|
|
839
|
+
*/
|
|
840
|
+
release: (reason?: LiveKitSessionReleaseReason) => void;
|
|
841
|
+
};
|
|
842
|
+
/** Maps the raw grant status into the adopter-facing discriminated capacity signal. */
|
|
843
|
+
declare function capacityStateFromGrant(state: Pick<LiveKitAvatarGrantState, "status" | "grant" | "busy" | "error">): LiveKitCapacityState;
|
|
844
|
+
type UseLiveKitAvatarGrantInput<TLlmProvider extends LLMProvider = LLMProvider> = {
|
|
845
|
+
client: AvatarSessionClient<TLlmProvider>;
|
|
846
|
+
session: LiveKitSessionRequest<TLlmProvider> | null | undefined;
|
|
847
|
+
active?: boolean;
|
|
848
|
+
autoRetryBusy?: boolean;
|
|
849
|
+
requestOptions?: RealtimeAvatarRequestOptions;
|
|
850
|
+
/**
|
|
851
|
+
* Optional LiveKit host URL (`wss://…`) to pre-warm (DNS + TCP + TLS) while
|
|
852
|
+
* the grant POST is in flight — the only way to cover a browser's FIRST-ever
|
|
853
|
+
* call, before any landed grant has persisted a host hint. ADVISORY ONLY:
|
|
854
|
+
* the room always connects to the freshly-granted `livekit_url`, never this
|
|
855
|
+
* value, so a wrong hint costs one wasted HEAD request and nothing else.
|
|
856
|
+
*/
|
|
857
|
+
serverUrlHint?: string;
|
|
858
|
+
};
|
|
859
|
+
/**
|
|
860
|
+
* Requests a Realtime Avatar LiveKit grant, then lets LiveKit's own React
|
|
861
|
+
* components own the room connection, tracks, agent state, and media controls.
|
|
862
|
+
*/
|
|
863
|
+
declare function useLiveKitAvatarGrant<TLlmProvider extends LLMProvider = LLMProvider>(input: UseLiveKitAvatarGrantInput<TLlmProvider>): LiveKitAvatarGrantState;
|
|
864
|
+
type RealtimeAvatarLiveKitRoomProps = Omit<LiveKitRoomProps, "serverUrl" | "token" | "connect" | "audio" | "video" | "options"> & {
|
|
865
|
+
grant: LiveKitSessionGrant | null | undefined;
|
|
866
|
+
connect?: boolean;
|
|
867
|
+
/** Native LiveKit audio publish option. Defaults to server-STT sessions only. */
|
|
868
|
+
audio?: LiveKitRoomProps["audio"];
|
|
869
|
+
/** Native LiveKit video publish option. Defaults to false for avatar calls. */
|
|
870
|
+
video?: LiveKitRoomProps["video"];
|
|
871
|
+
/** Passed directly to LiveKitRoom. */
|
|
872
|
+
options?: LiveKitRoomProps["options"];
|
|
873
|
+
renderRoomAudio?: boolean;
|
|
874
|
+
children?: ReactNode;
|
|
875
|
+
};
|
|
876
|
+
/** Thin typed bridge from a Realtime Avatar grant into LiveKitRoom. */
|
|
877
|
+
declare function RealtimeAvatarLiveKitRoom(props: RealtimeAvatarLiveKitRoomProps): ReactElement;
|
|
878
|
+
declare const capacityErrorFromBusy: (busy: CapacityBusyResponse) => RealtimeAvatarCapacityError;
|
|
879
|
+
|
|
880
|
+
/**
|
|
881
|
+
* Avatar nonverbal-behavior snapshot + its SSOT derivation.
|
|
882
|
+
*
|
|
883
|
+
* This is the ONE place that turns a decoded `behavior_state` frame into the app-facing
|
|
884
|
+
* snapshot. Both {@link useSessionLifecycle} (opt-in `onBehaviorChange`) and
|
|
885
|
+
* {@link useRealtimeSession} (`behavior` state + `onBehaviorChange`) route through
|
|
886
|
+
* {@link nextBehaviorSnapshot}, so the extract + change-diff live exactly once.
|
|
887
|
+
*
|
|
888
|
+
* A LEAF module (no hook imports) so both hooks depend on it without an import cycle.
|
|
889
|
+
*/
|
|
890
|
+
/**
|
|
891
|
+
* The avatar's live nonverbal behavior (multi-clip choreography). `state` is an OPEN
|
|
892
|
+
* string so newer workers can ship new states without breaking old apps — narrow with
|
|
893
|
+
* `knownBehaviorStates` and treat unknown values like `"idle"`. `clipId` is the internal
|
|
894
|
+
* render clip currently playing (a debug/admin detail), or `null` before the first frame.
|
|
895
|
+
*/
|
|
896
|
+
type BehaviorSnapshot = {
|
|
897
|
+
state: string;
|
|
898
|
+
clipId: string | null;
|
|
899
|
+
/** WIRE clip model of the current clip — the same trigger vocabulary the session
|
|
900
|
+
* minted with ("idle" | "listen" | "directive"; "think" is retired and never emitted). `null` on pre-Tier-1
|
|
901
|
+
* workers (the fields simply aren't on the frame yet). */
|
|
902
|
+
trigger: string | null;
|
|
903
|
+
/** Whether the current clip is a looping resting state (vs a one-shot). */
|
|
904
|
+
loop: boolean | null;
|
|
905
|
+
/** The clip the last seam faded FROM (`null` before the first seam). */
|
|
906
|
+
prevClipId: string | null;
|
|
907
|
+
};
|
|
908
|
+
|
|
909
|
+
/**
|
|
910
|
+
* Wired up by CONSUMERS, not by this package. Nothing inside the carry calls it — checked
|
|
911
|
+
* 2026-08-26 by searching for `pendingTurn` and `replay`, which appear nowhere else here. The
|
|
912
|
+
* upstream studio imports it to gate its own turn replay on reconnect, which is why it is on the
|
|
913
|
+
* public `/react` surface (exported 2026-08-27): the hooks own the reconnect, but the pending
|
|
914
|
+
* turn lives in the app, so the app is the one that has to ask.
|
|
915
|
+
*
|
|
916
|
+
* Whether a re-queued turn should replay onto a freshly-(re)connected room. Pure
|
|
917
|
+
* so the "replay exactly once, keyed on the NEW session_id" guard is testable.
|
|
918
|
+
*
|
|
919
|
+
* Replays only when ALL hold:
|
|
920
|
+
* - the connect was a RECOVERY (we were reconnecting), not a first connect;
|
|
921
|
+
* - there is a pending turn to replay;
|
|
922
|
+
* - the fresh grant has a session_id;
|
|
923
|
+
* - that session_id hasn't already replayed (a flapping reconnect that re-fires
|
|
924
|
+
* onConnected for the SAME session must not double-send).
|
|
925
|
+
*/
|
|
926
|
+
declare function shouldReplayPendingTurn(args: {
|
|
927
|
+
wasReconnecting: boolean;
|
|
928
|
+
hasPendingTurn: boolean;
|
|
929
|
+
sessionId: string | null | undefined;
|
|
930
|
+
lastReplayedSessionId: string | null | undefined;
|
|
931
|
+
}): boolean;
|
|
932
|
+
declare const RECONNECT_BACKOFF_MS: number[];
|
|
933
|
+
declare const MAX_RECONNECT_ATTEMPTS = 4;
|
|
934
|
+
type RetryStep = {
|
|
935
|
+
kind: "retry";
|
|
936
|
+
delayMs: number;
|
|
937
|
+
attempt: number;
|
|
938
|
+
} | {
|
|
939
|
+
kind: "give-up";
|
|
940
|
+
};
|
|
941
|
+
/**
|
|
942
|
+
* The reconnect policy: the backoff schedule + the give-up bound. Both are CLIENT-
|
|
943
|
+
* UX-tier knobs (not infra), so they are configurable on the hook with the audited
|
|
944
|
+
* defaults above. A non-positive / empty override falls back to the default.
|
|
945
|
+
*/
|
|
946
|
+
type ReconnectPolicy = {
|
|
947
|
+
backoffMs: number[];
|
|
948
|
+
maxAttempts: number;
|
|
949
|
+
};
|
|
950
|
+
/**
|
|
951
|
+
* Why a session reached the terminal {@link SessionLifecyclePhase} `ended` arm.
|
|
952
|
+
* OPTIONAL on the phase (back-compat: a bare `{ kind: "ended" }` still type-checks
|
|
953
|
+
* and renders), so adopters can disambiguate the end without forking the union:
|
|
954
|
+
* - `idle`: the CLIENT-AUTHORITATIVE idle clock expired and the SDK ended the
|
|
955
|
+
* session itself (released the lease + left the room → the worker's disconnect release
|
|
956
|
+
* frees the GPU). This is the reason the idle-warning countdown is now TRUE.
|
|
957
|
+
* - `disconnected`: a deliberate server/peer end or an exhausted auto-reconnect.
|
|
958
|
+
* - `error`: a grant failure surfaced as a terminal phase (no lease, no retry).
|
|
959
|
+
*/
|
|
960
|
+
type SessionEndReason = "idle" | "disconnected" | "error";
|
|
961
|
+
type SessionLifecyclePhase = {
|
|
962
|
+
kind: "idle";
|
|
963
|
+
} | {
|
|
964
|
+
kind: "requesting";
|
|
965
|
+
} | {
|
|
966
|
+
kind: "queued";
|
|
967
|
+
busy: CapacityBusyResponse;
|
|
968
|
+
} | {
|
|
969
|
+
kind: "connecting";
|
|
970
|
+
grant: LiveKitSessionGrant;
|
|
971
|
+
} | {
|
|
972
|
+
kind: "live";
|
|
973
|
+
} | {
|
|
974
|
+
kind: "idle-warning";
|
|
975
|
+
secondsRemaining: number;
|
|
976
|
+
deadlineAt: number;
|
|
977
|
+
} | {
|
|
978
|
+
kind: "reconnectable";
|
|
979
|
+
reconnecting: boolean;
|
|
980
|
+
attempt: number;
|
|
981
|
+
/** Native LiveKit recovery keeps the same grant; fresh-grant recovery re-mints. */
|
|
982
|
+
strategy: "in-place" | "fresh-grant";
|
|
983
|
+
} | {
|
|
984
|
+
kind: "ended";
|
|
985
|
+
reason?: SessionEndReason;
|
|
986
|
+
};
|
|
987
|
+
type SessionLifecyclePhaseKind = SessionLifecyclePhase["kind"];
|
|
988
|
+
/**
|
|
989
|
+
* The recovery sub-state, the bridge between the pure recovery classifiers above
|
|
990
|
+
* and the unified phase. Kept as a small explicit union so the reducer below can
|
|
991
|
+
* compose it without re-deriving the recover-vs-end table.
|
|
992
|
+
*/
|
|
993
|
+
type RecoveryState = {
|
|
994
|
+
kind: "connected";
|
|
995
|
+
} | {
|
|
996
|
+
kind: "in-place-reconnecting";
|
|
997
|
+
} | {
|
|
998
|
+
kind: "refreshing";
|
|
999
|
+
attempt: number;
|
|
1000
|
+
} | {
|
|
1001
|
+
kind: "reconnecting";
|
|
1002
|
+
attempt: number;
|
|
1003
|
+
} | {
|
|
1004
|
+
kind: "failed";
|
|
1005
|
+
} | {
|
|
1006
|
+
kind: "ended";
|
|
1007
|
+
reason?: SessionEndReason;
|
|
1008
|
+
};
|
|
1009
|
+
/**
|
|
1010
|
+
* The CLIENT-OWNED idle budget (seconds): how long "quiet" before the client-
|
|
1011
|
+
* enforced idle end. This is the AUTHORITY for the reap decision (the worker reap
|
|
1012
|
+
* is off). Falls back to the grant's `idle_timeout_seconds` if the app omits it,
|
|
1013
|
+
* then to this default. Matches docs/SDK_SESSION_API.md (`idleSeconds = 120`).
|
|
1014
|
+
*/
|
|
1015
|
+
declare const DEFAULT_IDLE_SECONDS = 120;
|
|
1016
|
+
/**
|
|
1017
|
+
* How early the `idle-warning` phase fires before the idle end (seconds). Replaces
|
|
1018
|
+
* the legacy `min(30s, idle/3)` magic; clamped so the lead is always < the idle
|
|
1019
|
+
* budget. Matches docs/SDK_SESSION_API.md (`idleWarnLeadSeconds = 20`).
|
|
1020
|
+
*/
|
|
1021
|
+
declare const DEFAULT_IDLE_WARN_LEAD_SECONDS = 20;
|
|
1022
|
+
/**
|
|
1023
|
+
* The per-turn "no response" watchdog (seconds). The SDK owns this as the SINGLE
|
|
1024
|
+
* source for the turn timeout so adopters (the studio controller) thread ONE knob
|
|
1025
|
+
* instead of hardcoding a second literal. Matches docs/SDK_SESSION_API.md
|
|
1026
|
+
* (`turnTimeoutSeconds = 20`).
|
|
1027
|
+
*/
|
|
1028
|
+
declare const DEFAULT_TURN_TIMEOUT_SECONDS = 20;
|
|
1029
|
+
type UseSessionLifecycleInput<T extends LLMProvider = LLMProvider> = {
|
|
1030
|
+
client: AvatarSessionClient<T>;
|
|
1031
|
+
session: LiveKitSessionRequest<T> | null | undefined;
|
|
1032
|
+
active?: boolean;
|
|
1033
|
+
/**
|
|
1034
|
+
* The CLIENT-OWNED idle budget (seconds) — the authority for the client-enforced
|
|
1035
|
+
* idle end. Falls back to the grant's `idle_timeout_seconds`, then
|
|
1036
|
+
* {@link DEFAULT_IDLE_SECONDS}. Validated positive (a non-positive value falls
|
|
1037
|
+
* back).
|
|
1038
|
+
*/
|
|
1039
|
+
idleSeconds?: number;
|
|
1040
|
+
/**
|
|
1041
|
+
* How early the `idle-warning` phase fires before the idle end (seconds). Default
|
|
1042
|
+
* {@link DEFAULT_IDLE_WARN_LEAD_SECONDS}; clamped so the lead is always strictly
|
|
1043
|
+
* less than the idle budget.
|
|
1044
|
+
*/
|
|
1045
|
+
idleWarnLeadSeconds?: number;
|
|
1046
|
+
/**
|
|
1047
|
+
* @deprecated Use {@link idleWarnLeadSeconds}. The legacy ms warn-window override;
|
|
1048
|
+
* still honored (clamped to the idle budget) for back-compat. `idleWarnLeadSeconds`
|
|
1049
|
+
* wins when both are set.
|
|
1050
|
+
*/
|
|
1051
|
+
warnBeforeMs?: number;
|
|
1052
|
+
/** Auto-reconnect backoff schedule (ms). Default {@link RECONNECT_BACKOFF_MS}. */
|
|
1053
|
+
reconnectBackoffMs?: number[];
|
|
1054
|
+
/** Auto-reconnect give-up bound. Default {@link MAX_RECONNECT_ATTEMPTS}. */
|
|
1055
|
+
maxReconnectAttempts?: number;
|
|
1056
|
+
/** Forwarded to {@link useLiveKitAvatarGrant}. Defaults true (queue auto-retry). */
|
|
1057
|
+
autoRetryBusy?: boolean;
|
|
1058
|
+
requestOptions?: UseLiveKitAvatarGrantInput<T>["requestOptions"];
|
|
1059
|
+
/**
|
|
1060
|
+
* Opt-in tap for the avatar's live nonverbal behavior (multi-clip choreography). Fires
|
|
1061
|
+
* ONLY on a real change of {@link BehaviorSnapshot}, via the SAME `nextBehaviorSnapshot`
|
|
1062
|
+
* derivation {@link useRealtimeSession} uses for its `behavior` state — one source of truth.
|
|
1063
|
+
*
|
|
1064
|
+
* Omit it and the SDK never subscribes to behavior frames (zero overhead). Treat the
|
|
1065
|
+
* snapshot as DEBUG/admin signal: `clipId` is an internal render-clip id, not a
|
|
1066
|
+
* product-facing value — gate any UI that shows it (e.g. behind an admin flag).
|
|
1067
|
+
*/
|
|
1068
|
+
onBehaviorChange?: (snapshot: BehaviorSnapshot) => void;
|
|
1069
|
+
};
|
|
1070
|
+
type SessionLifecycleApi = {
|
|
1071
|
+
phase: SessionLifecyclePhase;
|
|
1072
|
+
/** The held grant for RealtimeAvatarLiveKitRoom (null until connecting+). */
|
|
1073
|
+
grant: LiveKitSessionGrant | null;
|
|
1074
|
+
/** The underlying capacity signal (queue position/size, error) — unchanged. */
|
|
1075
|
+
capacity: LiveKitCapacityState;
|
|
1076
|
+
/** The DOM recovery contract: auto-reconnect attempts since the last connect. */
|
|
1077
|
+
attempt: number;
|
|
1078
|
+
/**
|
|
1079
|
+
* ms until the CLIENT-ENFORCED idle end, or null when not connected/live. Drives
|
|
1080
|
+
* the countdown — and it is TRUE: at zero the SDK ends the session (`ended{idle}`).
|
|
1081
|
+
* This is the consumer's leverageable window (read it to act in-character before
|
|
1082
|
+
* the line closes; `stayConnected()` to extend; or let it run out).
|
|
1083
|
+
*/
|
|
1084
|
+
timeToDisconnectMs: number | null;
|
|
1085
|
+
/**
|
|
1086
|
+
* EXTEND the session: reset the CLIENT idle clock (the load-bearing effect that
|
|
1087
|
+
* postpones the client-enforced end). idle-warning → live. Idempotent,
|
|
1088
|
+
* fire-and-forget, no-op if not connected.
|
|
1089
|
+
*/
|
|
1090
|
+
stayConnected: () => void;
|
|
1091
|
+
/**
|
|
1092
|
+
* Mark organic activity (text-turn start, mic unmute, inbound transcription) —
|
|
1093
|
+
* resets the CLIENT idle clock (postpones the client-enforced end).
|
|
1094
|
+
*/
|
|
1095
|
+
markActivity: () => void;
|
|
1096
|
+
/** Manual reconnect (the Reconnect button): resets the attempt budget and re-mints. */
|
|
1097
|
+
reconnect: () => void;
|
|
1098
|
+
onConnected: () => void;
|
|
1099
|
+
onDisconnected: (reason?: DisconnectReason) => void;
|
|
1100
|
+
/**
|
|
1101
|
+
* Wire to the room's onError callback. Only retryable LiveKit transport errors
|
|
1102
|
+
* enter recovery; media-device/autoplay/client-cancel errors remain UI errors.
|
|
1103
|
+
*/
|
|
1104
|
+
onConnectionError: (error: Error) => void;
|
|
1105
|
+
/**
|
|
1106
|
+
* Wire to onConnectionStateChange so LiveKit's IN-PLACE reconnect
|
|
1107
|
+
* (signalReconnecting → reconnecting → connected) flips the phase too — fixes
|
|
1108
|
+
* the "invisible mid-flight reconnect" gap on the text path.
|
|
1109
|
+
*/
|
|
1110
|
+
onConnectionStateChange: (state: LiveKitConnectionStatus) => void;
|
|
1111
|
+
/** Feed the bound-agent signal (useVoiceAssistant().agent presence) from in-room. */
|
|
1112
|
+
setAgentPresent: (present: boolean) => void;
|
|
1113
|
+
/**
|
|
1114
|
+
* Register the in-room room-leave handle (room.disconnect). The in-room bridge
|
|
1115
|
+
* supplies it because the Room lives below this hook. This is what makes idle
|
|
1116
|
+
* CLIENT-AUTHORITATIVE: when the idle clock expires the hook calls this to LEAVE
|
|
1117
|
+
* the room, which (with the worker's disconnect release) ends the worker
|
|
1118
|
+
* session and frees the GPU — no leaked session, no waiting on a server reap that
|
|
1119
|
+
* never fires. Pass null to unregister.
|
|
1120
|
+
*/
|
|
1121
|
+
registerLeaveRoom: (leave: (() => void) | null) => void;
|
|
1122
|
+
/** Reset to idle (mode/avatar change). */
|
|
1123
|
+
reset: () => void;
|
|
1124
|
+
};
|
|
1125
|
+
declare function useSessionLifecycle<T extends LLMProvider = LLMProvider>(input: UseSessionLifecycleInput<T>): SessionLifecycleApi;
|
|
1126
|
+
/**
|
|
1127
|
+
* The optional realtime-session sinks the bridge ALSO fills when given the richer
|
|
1128
|
+
* {@link RealtimeSessionApi} (vs a bare {@link SessionLifecycleApi}). All optional, so
|
|
1129
|
+
* mounting the bridge with either surface type-checks — the extra wiring is inert when
|
|
1130
|
+
* the sink is absent. This keeps ONE bridge for both `useSessionLifecycle` and
|
|
1131
|
+
* `useRealtimeSession`.
|
|
1132
|
+
*/
|
|
1133
|
+
type RealtimeSessionRoomSinks = Partial<{
|
|
1134
|
+
/** Inbound `rta.lifecycle` data frames (DataReceived on the lifecycle topic). */
|
|
1135
|
+
onLifecycleData: (frame: unknown) => void;
|
|
1136
|
+
/** Publisher for client→worker frames (request_graceful_close / extend). */
|
|
1137
|
+
registerDataPublisher: (publish: ((frame: unknown) => void) | null) => void;
|
|
1138
|
+
/** The turn sender (`useChat().send`) used for closing + normal turns. */
|
|
1139
|
+
registerTurnSender: (send: ((text: string, opts?: SendTextOptions) => Promise<void>) | null) => void;
|
|
1140
|
+
/** Live assistant state → the `turn` micro-state. */
|
|
1141
|
+
setTurnState: (state: string | null | undefined) => void;
|
|
1142
|
+
/** Honest media liveness from the bound agent's tracks. */
|
|
1143
|
+
setMedia: (media: {
|
|
1144
|
+
video: "live" | "stalled" | "connecting";
|
|
1145
|
+
audio: "flowing" | "silent";
|
|
1146
|
+
}) => void;
|
|
1147
|
+
}>;
|
|
1148
|
+
type SessionLifecycleRoomBridgeProps = {
|
|
1149
|
+
lifecycle: Pick<SessionLifecycleApi, "onConnectionStateChange" | "setAgentPresent" | "registerLeaveRoom" | "markActivity"> & RealtimeSessionRoomSinks;
|
|
1150
|
+
};
|
|
1151
|
+
/**
|
|
1152
|
+
* Wire the in-room LiveKit facts into a {@link useSessionLifecycle} instance:
|
|
1153
|
+
* - `useConnectionState()` → `onConnectionStateChange` (in-place reconnect).
|
|
1154
|
+
* - `useVoiceAssistant().agent` presence → `setAgentPresent` (idle clock gate).
|
|
1155
|
+
* - `useVoiceAssistant().state` + `useTranscriptions()` → `markActivity` (the
|
|
1156
|
+
* IDLE×CALL fix: a voice call has no per-turn text send to reset the client idle
|
|
1157
|
+
* clock, so without this an ACTIVE call is falsely reaped at the 120s client idle
|
|
1158
|
+
* end. We reset on REAL activity only — the agent actively producing
|
|
1159
|
+
* (`speaking`/`thinking`) or a fresh user transcript (the user spoke) — NOT on the
|
|
1160
|
+
* resting `listening` state, so a genuinely SILENT call still reaps as intended.
|
|
1161
|
+
* On the text path the per-turn send already marks activity; this is harmless +
|
|
1162
|
+
* redundant there, and load-bearing on the call path — one DRY signal for both).
|
|
1163
|
+
* - `room.disconnect()` → `registerLeaveRoom` (so the CLIENT-AUTHORITATIVE idle end
|
|
1164
|
+
* actually LEAVES the room → `the worker's disconnect release` frees the GPU).
|
|
1165
|
+
*
|
|
1166
|
+
* Renders nothing. Mount it once inside RealtimeAvatarLiveKitRoom.
|
|
1167
|
+
*/
|
|
1168
|
+
declare function SessionLifecycleRoomBridge({ lifecycle }: SessionLifecycleRoomBridgeProps): null;
|
|
1169
|
+
|
|
1170
|
+
/**
|
|
1171
|
+
* The terminal, LABELED end reason surfaced to the app via `onEnded`. This is the
|
|
1172
|
+
* contracts {@link SessionEndReasonLabel} (a superset of the client-internal
|
|
1173
|
+
* {@link SessionEndReason}) — the worker stamps the precise label on its `ended`
|
|
1174
|
+
* frame; {@link resolveEndReason} falls back to mapping the internal reason when no
|
|
1175
|
+
* frame arrived (an old worker), so the app always gets a usable reason.
|
|
1176
|
+
*/
|
|
1177
|
+
type EndReason = SessionEndReasonLabel;
|
|
1178
|
+
/**
|
|
1179
|
+
* The three DISTINCT session clocks — never conflated, each with its own authority:
|
|
1180
|
+
* - `sessionRemainingMs`: to the hard cap. SERVER-MIRRORED off the worker
|
|
1181
|
+
* `session_clock` frame (a client-only timer drifts across the cold-start gap and
|
|
1182
|
+
* would cut the goodbye mid-syllable). `null` until the frame lands ("unknown").
|
|
1183
|
+
* - `idleRemainingMs`: to the CLIENT-AUTHORITATIVE idle end (alias of the lifecycle
|
|
1184
|
+
* `timeToDisconnectMs`; the SDK enforces this one).
|
|
1185
|
+
* - `creditRemainingMs`: an app-supplied passthrough — the SDK never fetches or
|
|
1186
|
+
* decides billing, it only surfaces the balance + fires `onCreditsLow`.
|
|
1187
|
+
*/
|
|
1188
|
+
type SessionClocks = {
|
|
1189
|
+
sessionRemainingMs: number | null;
|
|
1190
|
+
idleRemainingMs: number | null;
|
|
1191
|
+
creditRemainingMs: number | null;
|
|
1192
|
+
};
|
|
1193
|
+
/**
|
|
1194
|
+
* The grace-window state machine. Drives the guaranteed closing line:
|
|
1195
|
+
* - `closed`: no end imminent.
|
|
1196
|
+
* - `open`: the SOFT end is reached — send the ONE closing turn NOW. `deadlineAt`
|
|
1197
|
+
* is the hard cap (`endsAt`); past it an unused window is `spent{false}`.
|
|
1198
|
+
* - `delivering`: the closing turn is in flight / being spoken. `deadlineAt` is the
|
|
1199
|
+
* worker's hard ceiling (`endsAt + graceCeilingMs`); past it → `spent{false}`.
|
|
1200
|
+
* - `spent`: terminal — `delivered:true` on the worker `closing_turn_done` frame,
|
|
1201
|
+
* `delivered:false` if a deadline passed first (the line was cut / never sent).
|
|
1202
|
+
*/
|
|
1203
|
+
type GraceWindowState = {
|
|
1204
|
+
kind: "closed";
|
|
1205
|
+
} | {
|
|
1206
|
+
kind: "open";
|
|
1207
|
+
reason: ApproachingEndReason;
|
|
1208
|
+
deadlineAt: number;
|
|
1209
|
+
msLeft: number;
|
|
1210
|
+
} | {
|
|
1211
|
+
kind: "delivering";
|
|
1212
|
+
turnId: string;
|
|
1213
|
+
deadlineAt: number;
|
|
1214
|
+
} | {
|
|
1215
|
+
kind: "spent";
|
|
1216
|
+
delivered: boolean;
|
|
1217
|
+
};
|
|
1218
|
+
/** The live turn micro-state, mapped from `useVoiceAssistant().state`. */
|
|
1219
|
+
type TurnState = "listening" | "thinking" | "speaking" | "quiet";
|
|
1220
|
+
/**
|
|
1221
|
+
* Map the LiveKit voice-assistant state onto the 4-value turn micro-state. Only the
|
|
1222
|
+
* three "doing something" states pass through; everything else
|
|
1223
|
+
* (`initializing`/`connecting`/`disconnected`/`idle`/unknown) reads as `quiet`.
|
|
1224
|
+
*/
|
|
1225
|
+
declare function mapTurnState(assistantState: string | null | undefined): TurnState;
|
|
1226
|
+
|
|
1227
|
+
/** How early `onApproachingEnd` fires before the hard cap (room to compose the goodbye). */
|
|
1228
|
+
declare const DEFAULT_APPROACHING_END_LEAD_SECONDS = 45;
|
|
1229
|
+
/** How early the grace window opens before the cap (fits LLM excuse + send RTT + TTS + playout). */
|
|
1230
|
+
declare const DEFAULT_GRACE_WINDOW_LEAD_SECONDS = 12;
|
|
1231
|
+
/** Client mirror of the worker's hard grace ceiling — how long a closing line may run past the cap. */
|
|
1232
|
+
declare const DEFAULT_GRACE_CEILING_SECONDS = 10;
|
|
1233
|
+
/** Surface "low on minutes" at ~5 min of the app-supplied credit balance. */
|
|
1234
|
+
declare const DEFAULT_CREDITS_LOW_LEAD_SECONDS = 300;
|
|
1235
|
+
type ClosingTurnResult = {
|
|
1236
|
+
ok: true;
|
|
1237
|
+
turnId: string;
|
|
1238
|
+
} | {
|
|
1239
|
+
ok: false;
|
|
1240
|
+
reason: "window_closed" | "not_connected" | "already_spent";
|
|
1241
|
+
};
|
|
1242
|
+
type ExtendResult = {
|
|
1243
|
+
ok: boolean;
|
|
1244
|
+
};
|
|
1245
|
+
type ApproachingEndEvent = {
|
|
1246
|
+
secondsLeft: number;
|
|
1247
|
+
reason: ApproachingEndReason;
|
|
1248
|
+
threshold: number;
|
|
1249
|
+
};
|
|
1250
|
+
type GraceWindowOpenEvent = {
|
|
1251
|
+
reason: ApproachingEndReason;
|
|
1252
|
+
deadlineAt: number;
|
|
1253
|
+
msLeft: number;
|
|
1254
|
+
};
|
|
1255
|
+
type GraceWindowClosedEvent = {
|
|
1256
|
+
reason: ApproachingEndReason;
|
|
1257
|
+
delivered: boolean;
|
|
1258
|
+
};
|
|
1259
|
+
type IdleWarningEvent = {
|
|
1260
|
+
secondsLeft: number;
|
|
1261
|
+
};
|
|
1262
|
+
type CreditsLowEvent = {
|
|
1263
|
+
secondsLeft: number;
|
|
1264
|
+
};
|
|
1265
|
+
type TurnTimeoutEvent = {
|
|
1266
|
+
turnId: string | null;
|
|
1267
|
+
};
|
|
1268
|
+
type ReconnectingEvent = {
|
|
1269
|
+
attempt: number;
|
|
1270
|
+
};
|
|
1271
|
+
type EndedEvent = {
|
|
1272
|
+
reason: EndReason;
|
|
1273
|
+
};
|
|
1274
|
+
type RealtimeSessionMedia = {
|
|
1275
|
+
video: "live" | "stalled" | "connecting";
|
|
1276
|
+
audio: "flowing" | "silent";
|
|
1277
|
+
};
|
|
1278
|
+
|
|
1279
|
+
/** The worker's verdict on a gesture request (every request is answered or times out). */
|
|
1280
|
+
type ClipResult = {
|
|
1281
|
+
requestId: string;
|
|
1282
|
+
accepted: boolean;
|
|
1283
|
+
reason: string;
|
|
1284
|
+
};
|
|
1285
|
+
/**
|
|
1286
|
+
* The `useRealtimeSession` input. ADDITIVE over {@link UseSessionLifecycleInput}
|
|
1287
|
+
* (whose `client`/`session`/`active`/`idleSeconds`/`idleWarnLeadSeconds`/reconnect
|
|
1288
|
+
* knobs are reused VERBATIM — see the "Timeouts & budgets" group). The new fields
|
|
1289
|
+
* are the credit passthrough, the 5 moment callbacks, and the lead-second knobs.
|
|
1290
|
+
*
|
|
1291
|
+
* DESIGN PRINCIPLE — maximum developer control: every timeout/budget is a knob with a
|
|
1292
|
+
* safe default. Client-owned clocks (idle + every lead) are set freely; the hard cap
|
|
1293
|
+
* is a REQUEST (`maxSessionSeconds`, bounded by the platform max + server-enforced for
|
|
1294
|
+
* billing/GPU safety). The app wires only the NARRATIVE callbacks + copy.
|
|
1295
|
+
*/
|
|
1296
|
+
type UseRealtimeSessionInput<T extends LLMProvider = LLMProvider> = UseSessionLifecycleInput<T> & {
|
|
1297
|
+
/** Echoed by the app on its own events; the SDK never inspects it. */
|
|
1298
|
+
clientMetadata?: Record<string, unknown>;
|
|
1299
|
+
/**
|
|
1300
|
+
* Requested hard cap (seconds). Injected into the session request → the platform
|
|
1301
|
+
* grant → the worker enforces it. Bounded by the platform max (billing/GPU safety);
|
|
1302
|
+
* falls back to the grant's `max_session_seconds`. `idleSeconds`/`idleWarnLeadSeconds`
|
|
1303
|
+
* are inherited from {@link UseSessionLifecycleInput} (client-authoritative).
|
|
1304
|
+
*/
|
|
1305
|
+
maxSessionSeconds?: number;
|
|
1306
|
+
/** Lead before the cap for `onApproachingEnd`. Default {@link DEFAULT_APPROACHING_END_LEAD_SECONDS}. */
|
|
1307
|
+
approachingEndLeadSeconds?: number;
|
|
1308
|
+
/** Lead before the cap for the grace window to open. Default {@link DEFAULT_GRACE_WINDOW_LEAD_SECONDS}. */
|
|
1309
|
+
graceWindowLeadSeconds?: number;
|
|
1310
|
+
/** Client mirror of the worker grace ceiling. Default {@link DEFAULT_GRACE_CEILING_SECONDS}. */
|
|
1311
|
+
graceCeilingSeconds?: number;
|
|
1312
|
+
/** Lead before the credit balance runs out for `onCreditsLow`. Default {@link DEFAULT_CREDITS_LOW_LEAD_SECONDS}. */
|
|
1313
|
+
creditsLowLeadSeconds?: number;
|
|
1314
|
+
/** Per-turn "no response" watchdog (seconds). Default {@link DEFAULT_TURN_TIMEOUT_SECONDS}. */
|
|
1315
|
+
turnTimeoutSeconds?: number;
|
|
1316
|
+
/** App-supplied credit balance (ms) — surfaced on `clocks.creditRemainingMs` + drives `onCreditsLow`. */
|
|
1317
|
+
creditRemainingMs?: number | null;
|
|
1318
|
+
/**
|
|
1319
|
+
* Auto-hold the idle clock (`stayConnected`) while the grace window is open/delivering
|
|
1320
|
+
* so the idle reaper can't preempt the guaranteed closing line. Default true.
|
|
1321
|
+
*/
|
|
1322
|
+
autoStayConnectedDuringGrace?: boolean;
|
|
1323
|
+
onApproachingEnd?: (e: ApproachingEndEvent) => void;
|
|
1324
|
+
onGraceWindowOpen?: (e: GraceWindowOpenEvent) => void;
|
|
1325
|
+
onGraceWindowClosed?: (e: GraceWindowClosedEvent) => void;
|
|
1326
|
+
onIdleWarning?: (e: IdleWarningEvent) => void;
|
|
1327
|
+
onCreditsLow?: (e: CreditsLowEvent) => void;
|
|
1328
|
+
onTurnTimeout?: (e: TurnTimeoutEvent) => void;
|
|
1329
|
+
onReconnecting?: (e: ReconnectingEvent) => void;
|
|
1330
|
+
onReconnected?: () => void;
|
|
1331
|
+
onEnded?: (e: EndedEvent) => void;
|
|
1332
|
+
/** The avatar's behavior changed (listening/thinking/idle/special clips). */
|
|
1333
|
+
onBehaviorChange?: (b: BehaviorSnapshot) => void;
|
|
1334
|
+
/** A clip request was answered (also resolved on the performClip promise). */
|
|
1335
|
+
onClipResult?: (r: ClipResult) => void;
|
|
1336
|
+
};
|
|
1337
|
+
/**
|
|
1338
|
+
* The composed realtime-session surface. `phase` is the {@link SessionLifecyclePhase}
|
|
1339
|
+
* SSOT VERBATIM (no union fork). Adds the three clocks, the grace window, the live turn
|
|
1340
|
+
* micro-state, honest media, and the leverage actions. The in-room sinks
|
|
1341
|
+
* (`onLifecycleData`/`registerDataPublisher`/`registerTurnSender`/`setTurnState`/`setMedia`)
|
|
1342
|
+
* are filled by {@link SessionLifecycleRoomBridge} — the app just mounts the bridge.
|
|
1343
|
+
*/
|
|
1344
|
+
/**
|
|
1345
|
+
* The composed realtime-session surface. A strict SUPERSET of {@link SessionLifecycleApi}
|
|
1346
|
+
* (so it is a DROP-IN replacement everywhere the inner hook was used — `phase`/`grant`/
|
|
1347
|
+
* `capacity`/`stayConnected`/`reconnect`/`timeToDisconnectMs`/the sinks all pass through
|
|
1348
|
+
* verbatim), plus the three clocks, the grace window, the live turn micro-state, honest
|
|
1349
|
+
* media, and the new leverage actions. The realtime-session sinks
|
|
1350
|
+
* (`onLifecycleData`/`registerDataPublisher`/`registerTurnSender`/`setTurnState`/`setMedia`)
|
|
1351
|
+
* are filled by {@link SessionLifecycleRoomBridge} — the app just mounts the bridge.
|
|
1352
|
+
*/
|
|
1353
|
+
type RealtimeSessionApi = SessionLifecycleApi & {
|
|
1354
|
+
/** Live turn micro-state, mapped from `useVoiceAssistant().state`. */
|
|
1355
|
+
turn: TurnState;
|
|
1356
|
+
/** The three DISTINCT clocks (session/idle/credit), never conflated. */
|
|
1357
|
+
clocks: SessionClocks;
|
|
1358
|
+
/** Wall-clock hard-cap deadline (server-mirrored), or null until the clock frame lands. */
|
|
1359
|
+
endsAt: number | null;
|
|
1360
|
+
/** The grace-window state machine driving the guaranteed closing line. */
|
|
1361
|
+
graceWindow: GraceWindowState;
|
|
1362
|
+
/** Honest media liveness from the bound agent's tracks. */
|
|
1363
|
+
media: RealtimeSessionMedia;
|
|
1364
|
+
/** The inner SSOT surface, for explicit access (it is also spread at top level). */
|
|
1365
|
+
lifecycle: SessionLifecycleApi;
|
|
1366
|
+
/** Speak ONE final in-character line VERBATIM, then end. Guaranteed delivered before the cut. */
|
|
1367
|
+
sendClosingTurn: (text: string, opts?: {
|
|
1368
|
+
instructions?: string;
|
|
1369
|
+
}) => ClosingTurnResult;
|
|
1370
|
+
/** Ask the worker to wind down + close gracefully now (publishes request_graceful_close). */
|
|
1371
|
+
requestGracefulClose: () => void;
|
|
1372
|
+
/** Request a billable, guarded cap extension (the app owns who-pays; the worker validates). */
|
|
1373
|
+
extend: (req: {
|
|
1374
|
+
addSeconds: number;
|
|
1375
|
+
proof?: string;
|
|
1376
|
+
}) => ExtendResult;
|
|
1377
|
+
/** Send a normal turn THROUGH the SDK (arms the turn-timeout watchdog + enables retryTurn). */
|
|
1378
|
+
sendTurn: (text: string, opts?: {
|
|
1379
|
+
instructions?: string;
|
|
1380
|
+
}) => Promise<void>;
|
|
1381
|
+
/** Re-send the last turn sent through the SDK (the "no response" recovery). */
|
|
1382
|
+
retryTurn: () => void;
|
|
1383
|
+
/** End gracefully now (the user tapped End). */
|
|
1384
|
+
end: (reason?: EndReason) => void;
|
|
1385
|
+
/** The avatar's live nonverbal behavior, or null pre-choreo (see {@link BehaviorSnapshot}). */
|
|
1386
|
+
behavior: BehaviorSnapshot | null;
|
|
1387
|
+
/**
|
|
1388
|
+
* Ask the character to PERFORM a clip by id — a gesture arc plays once (e.g. the
|
|
1389
|
+
* gift moment); a `special`-role clip pins for `holdSeconds` (3-20, default 8).
|
|
1390
|
+
* Scheduled at the next seamless swap point, never a hard cut. Resolves with the
|
|
1391
|
+
* worker's verdict (`accepted:false` + reason on refusal / not connected /
|
|
1392
|
+
* timeout) — never rejects, so the app can always fall back to text-only behavior.
|
|
1393
|
+
*/
|
|
1394
|
+
performClip: (clipId: string, opts?: {
|
|
1395
|
+
holdSeconds?: number;
|
|
1396
|
+
timeoutMs?: number;
|
|
1397
|
+
}) => Promise<ClipResult>;
|
|
1398
|
+
/** Inbound `rta.lifecycle` frames (the bridge decodes RoomEvent.DataReceived). */
|
|
1399
|
+
onLifecycleData: (frame: unknown) => void;
|
|
1400
|
+
/** Publisher for client→worker frames (request_graceful_close / extend). */
|
|
1401
|
+
registerDataPublisher: (publish: ((frame: unknown) => void) | null) => void;
|
|
1402
|
+
/** The turn sender (the bridge's `useChat().send`) for closing + normal turns. */
|
|
1403
|
+
registerTurnSender: (send: ((text: string, opts?: SendTextOptions) => Promise<void>) | null) => void;
|
|
1404
|
+
/** Live assistant state → `turn`. */
|
|
1405
|
+
setTurnState: (state: string | null | undefined) => void;
|
|
1406
|
+
/** Honest media liveness from the bound agent's tracks. */
|
|
1407
|
+
setMedia: (media: RealtimeSessionMedia) => void;
|
|
1408
|
+
};
|
|
1409
|
+
declare function useRealtimeSession<T extends LLMProvider = LLMProvider>(input: UseRealtimeSessionInput<T>): RealtimeSessionApi;
|
|
1410
|
+
|
|
1411
|
+
/** The player's freeze verdict for the trailing window, in milliseconds. The app
|
|
1412
|
+
* computes this from its rVFC telemetry (call-telemetry `analyzeCallVideo`) and hands
|
|
1413
|
+
* a GETTER so the hook samples on its own tick — the app owns no timing. */
|
|
1414
|
+
type FreezeReadingFn = () => {
|
|
1415
|
+
/** Frozen ms observed in the trailing window (rVFC-derived; 0 = smooth). */
|
|
1416
|
+
freezeMsInWindow: number;
|
|
1417
|
+
/** The tab is hidden OR the freeze correlates with local CPU, not the network — the
|
|
1418
|
+
* false-positive fence. When true the governor is frozen (a downgrade can't fix a
|
|
1419
|
+
* decode/paint stall — our Dia-freeze lesson). */
|
|
1420
|
+
inhibited: boolean;
|
|
1421
|
+
};
|
|
1422
|
+
interface UseAvatarQualityGovernorInput {
|
|
1423
|
+
/** Master switch (product policy — the player's feature flag). Off ⇒ inert, no tick,
|
|
1424
|
+
* no subscriptions, the cap is never touched (byte-identical to today). */
|
|
1425
|
+
enabled: boolean;
|
|
1426
|
+
/** The player's rVFC freeze reading getter (see FreezeReadingFn). Optional: without
|
|
1427
|
+
* it the governor still reacts to Paused + getStats freezes, just without the
|
|
1428
|
+
* cross-browser rVFC signal. */
|
|
1429
|
+
freezeReading?: FreezeReadingFn;
|
|
1430
|
+
/** Governor timing overrides (tests / tuning). Defaults are the grounded constants. */
|
|
1431
|
+
config?: GovernorConfig;
|
|
1432
|
+
/** Poll cadence (ms). Default 1000 — the governor tick. */
|
|
1433
|
+
tickMs?: number;
|
|
1434
|
+
}
|
|
1435
|
+
/**
|
|
1436
|
+
* Drive the adaptive quality governor for the avatar's subscribed video track.
|
|
1437
|
+
*
|
|
1438
|
+
* Returns nothing the app must act on — it is a pure side-effect hook (like
|
|
1439
|
+
* useCallTelemetry). Mount it once inside the call body; it self-tears-down.
|
|
1440
|
+
*/
|
|
1441
|
+
declare function useAvatarQualityGovernor(input: UseAvatarQualityGovernorInput): void;
|
|
1442
|
+
|
|
1443
|
+
/** How long the deferred (`ended`-gated) release waits for the track's `ended`
|
|
1444
|
+
* event before handing the lease on anyway. A backstop against a track that
|
|
1445
|
+
* never fires `ended`; comfortably longer than a normal stop (~tens of ms). */
|
|
1446
|
+
declare const MIC_LEASE_ENDED_TIMEOUT_MS = 1500;
|
|
1447
|
+
type MicLease = {
|
|
1448
|
+
/** True once THIS room owns the lease — fold into the mic-intent signal. */
|
|
1449
|
+
held: boolean;
|
|
1450
|
+
/** Stable per-mount lease token. Pass to {@link useReleaseMicLeaseOnTrackEnded}
|
|
1451
|
+
* (rendered INSIDE the room) to release on the real hardware `ended` signal. */
|
|
1452
|
+
token: symbol;
|
|
1453
|
+
};
|
|
1454
|
+
/**
|
|
1455
|
+
* React binding for the tab-global mic lease. Pass `want=true` when this room
|
|
1456
|
+
* intends to hold the live microphone (a voice-first / speak-mode call); pass
|
|
1457
|
+
* `false` for a text-mode room that must not touch the mic. Returns `{ held,
|
|
1458
|
+
* token }` — the caller folds `held` into its single mic-intent signal
|
|
1459
|
+
* (`micLive = want && held`), so a rapid redial keeps its mic OFF until the
|
|
1460
|
+
* previous call's room has released, and no two rooms contend for getUserMedia.
|
|
1461
|
+
*
|
|
1462
|
+
* RELEASE is DEFERRED to the local mic track's real `ended` event when an in-room
|
|
1463
|
+
* {@link useReleaseMicLeaseOnTrackEnded} is mounted with this `token`: on unmount
|
|
1464
|
+
* / `want=false` the lease is not handed on until the browser has actually let go
|
|
1465
|
+
* of the capture device (or a safety timeout). Without that companion hook it
|
|
1466
|
+
* falls back to releasing on unmount (still correct for the dominant overlap
|
|
1467
|
+
* case, just coarser). Uncontended, acquisition resolves in a microtask.
|
|
1468
|
+
*/
|
|
1469
|
+
declare function useMicLease(want: boolean): MicLease;
|
|
1470
|
+
|
|
1471
|
+
/**
|
|
1472
|
+
* The client `AvatarCall` and the hooks ask for, talking to YOUR proxy route.
|
|
1473
|
+
*
|
|
1474
|
+
* Everything in this package is keyless by construction, and this is no exception: it holds a
|
|
1475
|
+
* URL, not a credential. Your route holds the key and decides the call; this only relays who to
|
|
1476
|
+
* call and, later, that the call is over.
|
|
1477
|
+
*
|
|
1478
|
+
* It exists because the prop was unsatisfiable without it. `AvatarCall` requires
|
|
1479
|
+
* `client: AvatarSessionClient`, and until now nothing in the published package could produce
|
|
1480
|
+
* one — the only implementation lived in a key-bearing class that is deliberately not exported
|
|
1481
|
+
* to browsers. So the flagship component typechecked, shipped, and could not be used.
|
|
1482
|
+
*
|
|
1483
|
+
* Pair it with `realtime-avatar/nextjs` (or `/hono`, `/express`, `/tanstack-start`) mounted at
|
|
1484
|
+
* the same prefix. Those adapters serve `POST …/connect` and `POST …/end`, which is exactly
|
|
1485
|
+
* what the five methods below call.
|
|
1486
|
+
*/
|
|
1487
|
+
interface ProxyClientOptions {
|
|
1488
|
+
/**
|
|
1489
|
+
* Where your route is mounted, e.g. `/api/realtime-avatar`.
|
|
1490
|
+
*
|
|
1491
|
+
* Same-origin and relative is the normal case. React Native has no page origin, so pass an
|
|
1492
|
+
* ABSOLUTE url there or every request resolves against nothing.
|
|
1493
|
+
*/
|
|
1494
|
+
proxyUrl: string;
|
|
1495
|
+
/** Swap the transport — a test double, or a fetch that carries your session cookie. */
|
|
1496
|
+
fetch?: typeof globalThis.fetch;
|
|
1497
|
+
/**
|
|
1498
|
+
* Per-request deadline, default 60s, `0` to disable.
|
|
1499
|
+
*
|
|
1500
|
+
* Not optional in spirit: a proxy that accepts the connection and then never answers leaves
|
|
1501
|
+
* a promise that never settles, which presents as a page stuck on "connecting" with no error
|
|
1502
|
+
* and a call slot held until the join timeout reclaims it.
|
|
1503
|
+
*/
|
|
1504
|
+
timeoutMs?: number;
|
|
1505
|
+
}
|
|
1506
|
+
declare function createProxyClient(options: ProxyClientOptions): AvatarSessionClient;
|
|
1507
|
+
|
|
1508
|
+
export { RECONNECT_BACKOFF_MS as $, type AvatarSessionClient as A, type BehaviorSnapshot as B, type CallTranscript as C, DEFAULT_APPROACHING_END_LEAD_SECONDS as D, type EndReason as E, type FishTtsModel as F, type Governor as G, type GraceWindowOpenEvent as H, type GraceWindowState as I, type IdleWarningEvent as J, type KnownBehaviorState as K, type LLMProvider as L, type LLMSelection as M, type LiveKitAvatarGrantState as N, type LiveKitAvatarGrantStatus as O, type LiveKitCapacityState as P, type LiveKitConnectionStatus as Q, type LiveKitSessionRequest as R, type LiveKitSessionStartResult as S, type LivePlaybackKeeper as T, MAX_RECONNECT_ATTEMPTS as U, MAX_SESSION_INSTRUCTIONS_CHARS as V, MIC_LEASE_ENDED_TIMEOUT_MS as W, type MicLease as X, type PlayableVideoElement as Y, type ProxyClientOptions as Z, type QualityCap as _, type AvatarVideoFit as a, RealtimeAvatarCapacityError as a0, RealtimeAvatarLiveKitRoom as a1, type RealtimeAvatarLiveKitRoomProps as a2, type RealtimeAvatarRequestOptions as a3, type RealtimeSessionApi as a4, type RealtimeSessionMedia as a5, type RealtimeSessionRoomSinks as a6, type ReconnectPolicy as a7, type ReconnectingEvent as a8, type RecoveryState as a9, shouldReplayPendingTurn as aA, splitCallTranscript as aB, useAvatarPlayoutDelay as aC, useAvatarQualityGovernor as aD, useCallTranscript as aE, useLiveKitAvatarGrant as aF, useMicLease as aG, useRealtimeSession as aH, useReleaseMicLeaseOnTrackEnded as aI, useSessionLifecycle as aJ, type LiveKitSessionGrant as aK, isNativeLiveTrackSubscribed as aL, useLiveTrackProducing as aM, type RetryStep as aa, type SessionBehavior as ab, type SessionClip as ac, type SessionClocks as ad, type SessionEndReason as ae, type SessionLifecycleApi as af, type SessionLifecyclePhase as ag, type SessionLifecyclePhaseKind as ah, SessionLifecycleRoomBridge as ai, type SessionLifecycleRoomBridgeProps as aj, type SurfaceLayers as ak, type TurnState as al, type TurnTimeoutEvent as am, type UseAvatarQualityGovernorInput as an, type UseLiveKitAvatarGrantInput as ao, type UseRealtimeSessionInput as ap, type UseSessionLifecycleInput as aq, type VoiceSpec as ar, type VoiceSpecInput as as, capacityErrorFromBusy as at, capacityStateFromGrant as au, createProxyClient as av, knownBehaviorStates as aw, mapTurnState as ax, sessionBehaviorSchema as ay, sessionClipSchema as az, type ApproachingEndEvent as b, type ApproachingEndReason as c, AvatarVideoSurface as d, type AvatarVideoSurfaceProps as e, type CallTranscriptSegment as f, type CartesiaTtsModel as g, type ClipResult as h, type ClosingTurnResult as i, type CreditsLowEvent as j, DEFAULT_AVATAR_PLAYOUT_DELAY_SECONDS as k, DEFAULT_CREDITS_LOW_LEAD_SECONDS as l, DEFAULT_GOVERNOR_CONFIG as m, DEFAULT_GRACE_CEILING_SECONDS as n, DEFAULT_GRACE_WINDOW_LEAD_SECONDS as o, DEFAULT_IDLE_SECONDS as p, DEFAULT_IDLE_WARN_LEAD_SECONDS as q, DEFAULT_TURN_TIMEOUT_SECONDS as r, type EndedEvent as s, type ExtendResult as t, type FreezeReadingFn as u, type GovernorAction as v, type GovernorConfig as w, type GovernorSignal as x, type GovernorState as y, type GraceWindowClosedEvent as z };
|