realtime-avatar 0.33.0 → 0.34.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +56 -1
- package/dist/browser.d.ts +4 -89
- package/dist/browser.js +362 -1
- package/dist/chunk-6KQS7CTR.js +1 -0
- package/dist/{chunk-VLYUMP3I.js → chunk-GPGNAQZV.js} +401 -170
- package/dist/{chunk-DM5TWRC4.js → chunk-OVW4N5NH.js} +1 -13
- package/dist/express.d.ts +2 -2
- package/dist/express.js +2 -2
- package/dist/hono.d.ts +2 -2
- package/dist/hono.js +2 -2
- package/dist/index.d.ts +330 -3
- package/dist/index.js +2 -1
- package/dist/nextjs.d.ts +2 -2
- package/dist/nextjs.js +2 -2
- package/dist/{playout-delay-CtRZdtsj.d.ts → playout-delay-BzoM1BMN.d.ts} +135 -1
- package/dist/{proxy-client-DJhYiiKX.d.ts → proxy-client-CiTOjkg8.d.ts} +67 -3
- package/dist/react-native.d.ts +3 -3
- package/dist/react-native.js +614 -36
- package/dist/react.d.ts +12 -22
- package/dist/react.js +618 -37
- package/dist/server.d.ts +2 -2
- package/dist/server.js +2 -1
- package/dist/tanstack-start.d.ts +2 -2
- package/dist/tanstack-start.js +2 -2
- package/dist/{types-DygYBvSv.d.ts → types-Cdi7KWWj.d.ts} +1 -1
- package/dist/{types-VoejsHij.d.ts → types-Cg_i1jhk.d.ts} +207 -18
- package/package.json +3 -3
package/dist/react.js
CHANGED
|
@@ -1461,7 +1461,7 @@ function stepNetworkQuality(previous, sample, nowMs, options) {
|
|
|
1461
1461
|
}
|
|
1462
1462
|
|
|
1463
1463
|
// ../client/src/react/network-quality-binding.ts
|
|
1464
|
-
function bindNetworkQuality(publication, inhibited,
|
|
1464
|
+
function bindNetworkQuality(publication, inhibited, notify2, tickMs) {
|
|
1465
1465
|
let disposed = false;
|
|
1466
1466
|
let running = false;
|
|
1467
1467
|
let state = initNetworkQuality(performance.now());
|
|
@@ -1475,7 +1475,7 @@ function bindNetworkQuality(publication, inhibited, notify, tickMs) {
|
|
|
1475
1475
|
};
|
|
1476
1476
|
const emit = (status) => {
|
|
1477
1477
|
try {
|
|
1478
|
-
|
|
1478
|
+
notify2(status);
|
|
1479
1479
|
} catch {
|
|
1480
1480
|
}
|
|
1481
1481
|
};
|
|
@@ -2359,15 +2359,107 @@ var schema5 = z.object({
|
|
|
2359
2359
|
"request_id": schema22.optional()
|
|
2360
2360
|
});
|
|
2361
2361
|
var voiceErrorFrameSchema = schema5;
|
|
2362
|
-
var schema04 = z.
|
|
2363
|
-
var schema13 = z.
|
|
2364
|
-
var schema23 = z.
|
|
2365
|
-
var schema33 = z.
|
|
2366
|
-
var schema43 = z.
|
|
2367
|
-
var schema52 = z.
|
|
2368
|
-
var
|
|
2362
|
+
var schema04 = z.string().min(1).max(200);
|
|
2363
|
+
var schema13 = z.literal(1);
|
|
2364
|
+
var schema23 = z.string().min(1).max(200);
|
|
2365
|
+
var schema33 = z.int().min(1).max(9007199254740991);
|
|
2366
|
+
var schema43 = z.literal(24e3);
|
|
2367
|
+
var schema52 = z.literal("pcm_s16le");
|
|
2368
|
+
var schema6 = z.object({
|
|
2369
|
+
"session_id": schema04,
|
|
2370
|
+
"generation_id": schema04,
|
|
2371
|
+
"version": schema13,
|
|
2372
|
+
"utterance_id": schema23,
|
|
2373
|
+
"stream_id": schema04,
|
|
2374
|
+
"epoch": schema33,
|
|
2375
|
+
"sample_rate": schema43,
|
|
2376
|
+
"channels": schema13,
|
|
2377
|
+
"encoding": schema52
|
|
2378
|
+
});
|
|
2379
|
+
var schema7 = z.enum(["started", "progress", "finished", "cancelled"]);
|
|
2380
|
+
var schema8 = z.int().min(0).max(144e4);
|
|
2381
|
+
z.strictObject({
|
|
2382
|
+
"session_id": schema04,
|
|
2383
|
+
"generation_id": schema04,
|
|
2384
|
+
"utterance_id": schema23,
|
|
2385
|
+
"stream_id": schema04,
|
|
2386
|
+
"epoch": schema33,
|
|
2387
|
+
"status": schema7,
|
|
2388
|
+
"samples": schema8
|
|
2389
|
+
});
|
|
2390
|
+
var schema10 = z.object({
|
|
2391
|
+
"version": schema13,
|
|
2392
|
+
"session_id": schema04,
|
|
2393
|
+
"epoch": schema33
|
|
2394
|
+
});
|
|
2395
|
+
var schema11 = z.literal("external_llm_error");
|
|
2396
|
+
var schema122 = z.enum(["external_llm_refused", "external_llm_failed"]);
|
|
2397
|
+
var schema132 = z.string();
|
|
2398
|
+
var schema14 = z.literal(true);
|
|
2399
|
+
var schema15 = z.object({
|
|
2400
|
+
"kind": schema11,
|
|
2401
|
+
"code": schema122,
|
|
2402
|
+
"message": schema132,
|
|
2403
|
+
"fatal": schema14
|
|
2404
|
+
});
|
|
2405
|
+
var correlatedAudioHeaderSchema = schema6;
|
|
2406
|
+
var correlatedAudioCancelSchema = schema10;
|
|
2407
|
+
var externalLlmErrorFrameSchema = schema15;
|
|
2408
|
+
var CORRELATED_AUDIO_CONTRACT = {
|
|
2409
|
+
"topic": "rta.audio.v1",
|
|
2410
|
+
"receiptMethod": "rta.audio.receipt.v1",
|
|
2411
|
+
"cancelMethod": "rta.audio.cancel.v1",
|
|
2412
|
+
"maxSamples": 144e4,
|
|
2413
|
+
"maxChunkBytes": 48e3,
|
|
2414
|
+
"maxBufferedSamples": 48e3
|
|
2415
|
+
};
|
|
2416
|
+
var schema05 = z.string().min(1).max(200);
|
|
2417
|
+
var schema16 = z.literal(1);
|
|
2418
|
+
var schema24 = z.enum(["respond", "recovery", "initiative"]);
|
|
2419
|
+
var schema34 = z.string().min(1).max(32);
|
|
2420
|
+
var schema44 = z.union([schema24, schema34]);
|
|
2421
|
+
var schema53 = z.enum(["draft", "accepted"]);
|
|
2422
|
+
var schema62 = z.literal("external");
|
|
2423
|
+
var schema72 = z.object({
|
|
2424
|
+
"renderer": schema62
|
|
2425
|
+
});
|
|
2426
|
+
z.object({
|
|
2427
|
+
"session_id": schema05,
|
|
2428
|
+
"generation_id": schema05,
|
|
2429
|
+
"version": schema16,
|
|
2430
|
+
"purpose": schema44,
|
|
2431
|
+
"admission": schema53,
|
|
2432
|
+
"visible_character": schema72.optional()
|
|
2433
|
+
});
|
|
2434
|
+
var schema92 = z.string().min(1).max(200);
|
|
2435
|
+
z.object({
|
|
2436
|
+
"generation_id": schema05,
|
|
2437
|
+
"utterance_id": schema92
|
|
2438
|
+
});
|
|
2439
|
+
var schema112 = z.literal("external_performance");
|
|
2440
|
+
var schema123 = z.enum(["accepted", "dropped", "cancelled", "audio_released"]);
|
|
2441
|
+
var schema133 = z.union([schema123, schema34]);
|
|
2442
|
+
var schema142 = z.literal("unavailable");
|
|
2443
|
+
var schema152 = z.literal("unavailable");
|
|
2444
|
+
var schema162 = z.object({
|
|
2445
|
+
"session_id": schema05,
|
|
2446
|
+
"generation_id": schema05,
|
|
2447
|
+
"kind": schema112,
|
|
2448
|
+
"version": schema16,
|
|
2449
|
+
"status": schema133,
|
|
2450
|
+
"audible_checkpoint": schema142,
|
|
2451
|
+
"audio_mapping": schema152
|
|
2452
|
+
});
|
|
2453
|
+
var externalPerformanceFrameSchema = schema162;
|
|
2454
|
+
var schema06 = z.enum(["s1", "s2-pro", "s2.1-pro", "drama-3"]);
|
|
2455
|
+
var schema17 = z.enum(["speech-1.6", "speech-1.5", "s1-mini"]);
|
|
2456
|
+
var schema25 = z.union([schema06, schema17]);
|
|
2457
|
+
var schema35 = z.enum(["whisper"]);
|
|
2458
|
+
var schema45 = z.null();
|
|
2459
|
+
var schema54 = z.union([schema35, schema45]);
|
|
2460
|
+
var fishTtsModelSchema = schema25;
|
|
2369
2461
|
var DEFAULT_FISH_TTS_MODEL = "s2.1-pro";
|
|
2370
|
-
var fishDeliverySchema =
|
|
2462
|
+
var fishDeliverySchema = schema54;
|
|
2371
2463
|
var FISH_DELIVERY_MODELS = { "whisper": ["drama-3"] };
|
|
2372
2464
|
|
|
2373
2465
|
// ../client/src/wire.ts
|
|
@@ -2683,7 +2775,9 @@ var lifecycleServerFrameSchema = z.discriminatedUnion("kind", [
|
|
|
2683
2775
|
endedFrameSchema,
|
|
2684
2776
|
behaviorStateFrameSchema,
|
|
2685
2777
|
clipAckFrameSchema,
|
|
2686
|
-
voiceErrorFrameSchema2
|
|
2778
|
+
voiceErrorFrameSchema2,
|
|
2779
|
+
externalPerformanceFrameSchema,
|
|
2780
|
+
externalLlmErrorFrameSchema
|
|
2687
2781
|
]);
|
|
2688
2782
|
var liveKitCapacitySnapshotSchema = z.object({
|
|
2689
2783
|
// Placement identity + per-worker session ceiling. These are always present on the
|
|
@@ -2965,6 +3059,7 @@ function useCallMedia(room, options) {
|
|
|
2965
3059
|
}, [room, setAudioPlayback, connecting]);
|
|
2966
3060
|
const controls = useMemo(() => ({
|
|
2967
3061
|
startAudio: async () => {
|
|
3062
|
+
if (options.startAudioOverride) return options.startAudioOverride();
|
|
2968
3063
|
try {
|
|
2969
3064
|
await room.startAudio();
|
|
2970
3065
|
} catch {
|
|
@@ -2981,13 +3076,372 @@ function useCallMedia(room, options) {
|
|
|
2981
3076
|
setVersion((value) => value + 1);
|
|
2982
3077
|
}
|
|
2983
3078
|
}
|
|
2984
|
-
}), [room]);
|
|
3079
|
+
}), [room, options.startAudioOverride]);
|
|
2985
3080
|
useEffect(() => {
|
|
2986
3081
|
if (!registerMediaControls) return;
|
|
2987
3082
|
registerMediaControls(controls);
|
|
2988
3083
|
return () => registerMediaControls(null);
|
|
2989
3084
|
}, [controls, registerMediaControls]);
|
|
2990
3085
|
}
|
|
3086
|
+
var attachments = /* @__PURE__ */ new WeakSet();
|
|
3087
|
+
var bindings = /* @__PURE__ */ new WeakMap();
|
|
3088
|
+
function notify(callback) {
|
|
3089
|
+
try {
|
|
3090
|
+
void Promise.resolve(callback()).catch(() => {
|
|
3091
|
+
});
|
|
3092
|
+
} catch {
|
|
3093
|
+
}
|
|
3094
|
+
}
|
|
3095
|
+
function attachCorrelatedAudio(room, options) {
|
|
3096
|
+
if (attachments.has(room)) throw new Error("Correlated audio is already attached to this room");
|
|
3097
|
+
if (!options.sessionId || !options.agentIdentity) throw new Error("Correlated audio requires the held session and bound agent");
|
|
3098
|
+
attachments.add(room);
|
|
3099
|
+
let context;
|
|
3100
|
+
let disposed = false;
|
|
3101
|
+
const previous = bindings.get(room);
|
|
3102
|
+
const binding = previous?.sessionId === options.sessionId ? previous : { sessionId: options.sessionId, epoch: 1, unavailable: false, seen: /* @__PURE__ */ new Set() };
|
|
3103
|
+
bindings.set(room, binding);
|
|
3104
|
+
const sources = [];
|
|
3105
|
+
const active = (source) => sources.includes(source);
|
|
3106
|
+
const buffered = () => sources.reduce((sum, source) => sum + source.received - source.consumed, 0);
|
|
3107
|
+
let nextStart = 0;
|
|
3108
|
+
let blocked;
|
|
3109
|
+
let closing;
|
|
3110
|
+
const seen = binding.seen;
|
|
3111
|
+
const waiters = /* @__PURE__ */ new Set();
|
|
3112
|
+
const wake = () => {
|
|
3113
|
+
for (const resolve of waiters) resolve();
|
|
3114
|
+
waiters.clear();
|
|
3115
|
+
};
|
|
3116
|
+
const identity = (header) => ({
|
|
3117
|
+
session_id: header.session_id,
|
|
3118
|
+
generation_id: header.generation_id,
|
|
3119
|
+
utterance_id: header.utterance_id,
|
|
3120
|
+
stream_id: header.stream_id,
|
|
3121
|
+
epoch: header.epoch
|
|
3122
|
+
});
|
|
3123
|
+
const position = (current, state) => {
|
|
3124
|
+
notify(() => options.onPosition({ ...identity(current.header), samples: current.consumed, sampleRate: current.header.sample_rate, state }));
|
|
3125
|
+
};
|
|
3126
|
+
const receipt = (current, status) => {
|
|
3127
|
+
const payload = { ...identity(current.header), status, samples: current.consumed };
|
|
3128
|
+
if (current.rpcBusy) {
|
|
3129
|
+
current.pendingReceipt = payload;
|
|
3130
|
+
return;
|
|
3131
|
+
}
|
|
3132
|
+
const send = async (next) => {
|
|
3133
|
+
current.rpcBusy = true;
|
|
3134
|
+
try {
|
|
3135
|
+
await room.localParticipant.performRpc({
|
|
3136
|
+
destinationIdentity: options.agentIdentity,
|
|
3137
|
+
method: CORRELATED_AUDIO_CONTRACT.receiptMethod,
|
|
3138
|
+
payload: JSON.stringify(next),
|
|
3139
|
+
responseTimeout: 2e3
|
|
3140
|
+
});
|
|
3141
|
+
} catch {
|
|
3142
|
+
if (!disposed && next.epoch === binding.epoch) invalidate("connection_lost");
|
|
3143
|
+
} finally {
|
|
3144
|
+
current.rpcBusy = false;
|
|
3145
|
+
const pending = current.pendingReceipt;
|
|
3146
|
+
current.pendingReceipt = void 0;
|
|
3147
|
+
if (pending && !disposed && pending.epoch === binding.epoch && !binding.unavailable) void send(pending);
|
|
3148
|
+
}
|
|
3149
|
+
};
|
|
3150
|
+
void send(payload);
|
|
3151
|
+
};
|
|
3152
|
+
const clear = () => {
|
|
3153
|
+
for (const current of sources.splice(0)) {
|
|
3154
|
+
current.abort.abort();
|
|
3155
|
+
for (const segment of current.segments) {
|
|
3156
|
+
segment.node.onended = null;
|
|
3157
|
+
segment.node.stop();
|
|
3158
|
+
segment.node.disconnect();
|
|
3159
|
+
}
|
|
3160
|
+
current.segments.length = 0;
|
|
3161
|
+
}
|
|
3162
|
+
nextStart = context?.currentTime ?? 0;
|
|
3163
|
+
wake();
|
|
3164
|
+
};
|
|
3165
|
+
const invalidate = (reason) => {
|
|
3166
|
+
if (disposed) return;
|
|
3167
|
+
if (reason === "connection_lost") {
|
|
3168
|
+
if (binding.unavailable) return;
|
|
3169
|
+
binding.unavailable = true;
|
|
3170
|
+
}
|
|
3171
|
+
if (reason === "invalid_stream" || reason === "disposed") for (const current of sources) receipt(current, "cancelled");
|
|
3172
|
+
clear();
|
|
3173
|
+
notify(() => options.onInvalidated(reason));
|
|
3174
|
+
};
|
|
3175
|
+
const tick = (reportCredit = true) => {
|
|
3176
|
+
if (!context || context.state !== "running" || disposed || binding.unavailable) return;
|
|
3177
|
+
for (const current of [...sources]) {
|
|
3178
|
+
if (!active(current)) continue;
|
|
3179
|
+
let consumed = current.consumed;
|
|
3180
|
+
for (const segment of current.segments) {
|
|
3181
|
+
if (context.currentTime < segment.start) break;
|
|
3182
|
+
const samples = Math.min(segment.length, Math.max(0, Math.floor((context.currentTime - segment.start) * current.header.sample_rate + 1e-6)));
|
|
3183
|
+
consumed = Math.max(consumed, segment.offset + samples);
|
|
3184
|
+
if (!current.started && samples > 0) {
|
|
3185
|
+
current.started = true;
|
|
3186
|
+
receipt(current, "started");
|
|
3187
|
+
position(current, "playing");
|
|
3188
|
+
current.lastPosition = current.consumed;
|
|
3189
|
+
}
|
|
3190
|
+
}
|
|
3191
|
+
current.consumed = consumed;
|
|
3192
|
+
while (current.segments.length && current.segments[0].offset + current.segments[0].length <= consumed) {
|
|
3193
|
+
const segment = current.segments.shift();
|
|
3194
|
+
if (segment) {
|
|
3195
|
+
segment.node.onended = null;
|
|
3196
|
+
segment.node.disconnect();
|
|
3197
|
+
}
|
|
3198
|
+
}
|
|
3199
|
+
if (current.started && current.eof && consumed === current.received && consumed > 0) {
|
|
3200
|
+
position(current, "finished");
|
|
3201
|
+
if (!active(current)) continue;
|
|
3202
|
+
receipt(current, "finished");
|
|
3203
|
+
sources.splice(sources.indexOf(current), 1);
|
|
3204
|
+
wake();
|
|
3205
|
+
} else {
|
|
3206
|
+
if (current.started && consumed > current.lastPosition) {
|
|
3207
|
+
current.lastPosition = consumed;
|
|
3208
|
+
position(current, "playing");
|
|
3209
|
+
}
|
|
3210
|
+
if (!active(current)) continue;
|
|
3211
|
+
if (reportCredit && consumed > current.lastReported) {
|
|
3212
|
+
current.lastReported = consumed;
|
|
3213
|
+
receipt(current, "progress");
|
|
3214
|
+
}
|
|
3215
|
+
wake();
|
|
3216
|
+
}
|
|
3217
|
+
}
|
|
3218
|
+
};
|
|
3219
|
+
const playbackState = () => {
|
|
3220
|
+
const next = context?.state !== "running";
|
|
3221
|
+
if (next !== blocked) {
|
|
3222
|
+
blocked = next;
|
|
3223
|
+
notify(() => options.onPlaybackBlocked?.(next ? startAudio : null));
|
|
3224
|
+
}
|
|
3225
|
+
tick();
|
|
3226
|
+
};
|
|
3227
|
+
const getContext = () => {
|
|
3228
|
+
if (!context) {
|
|
3229
|
+
context = new AudioContext({ sampleRate: correlatedAudioHeaderSchema.shape.sample_rate.value });
|
|
3230
|
+
context.onstatechange = playbackState;
|
|
3231
|
+
playbackState();
|
|
3232
|
+
}
|
|
3233
|
+
return context;
|
|
3234
|
+
};
|
|
3235
|
+
async function startAudio() {
|
|
3236
|
+
if (disposed || binding.unavailable) throw new Error("Correlated audio binding is no longer available");
|
|
3237
|
+
const ctx = getContext();
|
|
3238
|
+
await ctx.resume();
|
|
3239
|
+
playbackState();
|
|
3240
|
+
if (ctx.state !== "running") throw new Error("Correlated audio playback is blocked");
|
|
3241
|
+
}
|
|
3242
|
+
async function receive(reader, participant) {
|
|
3243
|
+
if (disposed || binding.unavailable || room.state !== "connected" || participant.identity !== options.agentIdentity) return;
|
|
3244
|
+
let value;
|
|
3245
|
+
try {
|
|
3246
|
+
value = JSON.parse(reader.info.attributes?.["rta.audio"] ?? "");
|
|
3247
|
+
} catch {
|
|
3248
|
+
return;
|
|
3249
|
+
}
|
|
3250
|
+
const parsed = correlatedAudioHeaderSchema.safeParse(value);
|
|
3251
|
+
if (!parsed.success || parsed.data.session_id !== options.sessionId || parsed.data.epoch < binding.epoch) return;
|
|
3252
|
+
const header = parsed.data;
|
|
3253
|
+
if (header.stream_id !== reader.info.id) return;
|
|
3254
|
+
if (header.epoch > binding.epoch) {
|
|
3255
|
+
binding.epoch = header.epoch;
|
|
3256
|
+
invalidate("cancelled");
|
|
3257
|
+
}
|
|
3258
|
+
const key = `${header.epoch}/${header.stream_id}`;
|
|
3259
|
+
if (seen.has(key)) return;
|
|
3260
|
+
if (sources.length >= 128 || seen.size >= 1e4) {
|
|
3261
|
+
invalidate("invalid_stream");
|
|
3262
|
+
return;
|
|
3263
|
+
}
|
|
3264
|
+
seen.add(key);
|
|
3265
|
+
const previousSource = sources.at(-1);
|
|
3266
|
+
const current = { header, abort: new AbortController(), segments: [], received: 0, consumed: 0, started: false, eof: false, lastReported: 0, lastPosition: -1, rpcBusy: false };
|
|
3267
|
+
sources.push(current);
|
|
3268
|
+
let carry;
|
|
3269
|
+
try {
|
|
3270
|
+
if (options.prepare) await options.prepare(header, current.abort.signal);
|
|
3271
|
+
if (!active(current) || current.abort.signal.aborted) return;
|
|
3272
|
+
const ctx = getContext();
|
|
3273
|
+
while (active(current) && (previousSource && active(previousSource) && !previousSource.eof || buffered() > CORRELATED_AUDIO_CONTRACT.maxBufferedSamples - CORRELATED_AUDIO_CONTRACT.maxChunkBytes / 2)) {
|
|
3274
|
+
await new Promise((resolve) => waiters.add(resolve));
|
|
3275
|
+
}
|
|
3276
|
+
if (!active(current)) return;
|
|
3277
|
+
for await (const chunk of reader.withAbortSignal(current.abort.signal)) {
|
|
3278
|
+
if (!active(current)) break;
|
|
3279
|
+
if (chunk.byteLength > CORRELATED_AUDIO_CONTRACT.maxChunkBytes) throw new Error("PCM chunk exceeds contract");
|
|
3280
|
+
const count = Math.floor((chunk.byteLength + (carry === void 0 ? 0 : 1)) / 2);
|
|
3281
|
+
if (current.received + count > CORRELATED_AUDIO_CONTRACT.maxSamples) throw new Error("PCM stream exceeds contract");
|
|
3282
|
+
while (active(current) && buffered() + count > CORRELATED_AUDIO_CONTRACT.maxBufferedSamples) {
|
|
3283
|
+
await new Promise((resolve) => waiters.add(resolve));
|
|
3284
|
+
}
|
|
3285
|
+
if (!active(current)) break;
|
|
3286
|
+
if (!count) {
|
|
3287
|
+
if (chunk.length) carry = chunk[0];
|
|
3288
|
+
continue;
|
|
3289
|
+
}
|
|
3290
|
+
const buffer = ctx.createBuffer(header.channels, count, header.sample_rate);
|
|
3291
|
+
const data = buffer.getChannelData(0);
|
|
3292
|
+
let cursor = 0;
|
|
3293
|
+
for (let i = 0; i < count; i++) {
|
|
3294
|
+
const low = carry ?? chunk[cursor++];
|
|
3295
|
+
carry = void 0;
|
|
3296
|
+
const raw = low | chunk[cursor++] << 8;
|
|
3297
|
+
data[i] = (raw >= 32768 ? raw - 65536 : raw) / 32768;
|
|
3298
|
+
}
|
|
3299
|
+
if (cursor < chunk.length) carry = chunk[cursor];
|
|
3300
|
+
const node = ctx.createBufferSource();
|
|
3301
|
+
node.buffer = buffer;
|
|
3302
|
+
node.connect(ctx.destination);
|
|
3303
|
+
const start = Math.max(ctx.currentTime, nextStart);
|
|
3304
|
+
current.segments.push({ node, start, offset: current.received, length: count });
|
|
3305
|
+
current.received += count;
|
|
3306
|
+
nextStart = start + count / header.sample_rate;
|
|
3307
|
+
node.onended = () => tick();
|
|
3308
|
+
node.start(start);
|
|
3309
|
+
while (active(current) && buffered() > CORRELATED_AUDIO_CONTRACT.maxBufferedSamples - CORRELATED_AUDIO_CONTRACT.maxChunkBytes / 2) {
|
|
3310
|
+
await new Promise((resolve) => waiters.add(resolve));
|
|
3311
|
+
}
|
|
3312
|
+
}
|
|
3313
|
+
if (!active(current)) return;
|
|
3314
|
+
if (carry !== void 0 || current.received === 0) throw new Error("Incomplete PCM stream");
|
|
3315
|
+
current.eof = true;
|
|
3316
|
+
wake();
|
|
3317
|
+
tick(false);
|
|
3318
|
+
} catch {
|
|
3319
|
+
if (active(current)) invalidate("invalid_stream");
|
|
3320
|
+
}
|
|
3321
|
+
}
|
|
3322
|
+
const cancel = async (data) => {
|
|
3323
|
+
if (disposed || binding.unavailable || room.state !== "connected" || data.callerIdentity !== options.agentIdentity) throw new Error("Unbound audio cancellation");
|
|
3324
|
+
const value = JSON.parse(data.payload);
|
|
3325
|
+
const frame2 = correlatedAudioCancelSchema.parse(value);
|
|
3326
|
+
if (frame2.session_id !== options.sessionId) throw new Error("Wrong audio session");
|
|
3327
|
+
if (frame2.epoch <= binding.epoch) return "";
|
|
3328
|
+
binding.epoch = frame2.epoch;
|
|
3329
|
+
invalidate("cancelled");
|
|
3330
|
+
return "";
|
|
3331
|
+
};
|
|
3332
|
+
const lost = () => invalidate("connection_lost");
|
|
3333
|
+
const changed = () => {
|
|
3334
|
+
if (["disconnected", "reconnecting", "signalReconnecting"].includes(room.state)) lost();
|
|
3335
|
+
};
|
|
3336
|
+
const participantLeft = (participant) => {
|
|
3337
|
+
if (participant.identity === options.agentIdentity) lost();
|
|
3338
|
+
};
|
|
3339
|
+
let registered = false;
|
|
3340
|
+
try {
|
|
3341
|
+
room.registerByteStreamHandler(CORRELATED_AUDIO_CONTRACT.topic, (reader, participant) => {
|
|
3342
|
+
void receive(reader, participant);
|
|
3343
|
+
});
|
|
3344
|
+
registered = true;
|
|
3345
|
+
room.localParticipant.registerRpcMethod(CORRELATED_AUDIO_CONTRACT.cancelMethod, cancel);
|
|
3346
|
+
} catch (error) {
|
|
3347
|
+
if (registered) room.unregisterByteStreamHandler(CORRELATED_AUDIO_CONTRACT.topic);
|
|
3348
|
+
attachments.delete(room);
|
|
3349
|
+
throw error;
|
|
3350
|
+
}
|
|
3351
|
+
room.on(RoomEvent.Reconnecting, lost);
|
|
3352
|
+
room.on(RoomEvent.SignalReconnecting, lost);
|
|
3353
|
+
room.on(RoomEvent.Disconnected, lost);
|
|
3354
|
+
room.on(RoomEvent.ConnectionStateChanged, changed);
|
|
3355
|
+
room.on(RoomEvent.ParticipantDisconnected, participantLeft);
|
|
3356
|
+
const timer = setInterval(() => tick(), 100);
|
|
3357
|
+
let frame = 0;
|
|
3358
|
+
const render = () => {
|
|
3359
|
+
if (disposed) return;
|
|
3360
|
+
tick(false);
|
|
3361
|
+
frame = requestAnimationFrame(render);
|
|
3362
|
+
};
|
|
3363
|
+
frame = requestAnimationFrame(render);
|
|
3364
|
+
return {
|
|
3365
|
+
startAudio,
|
|
3366
|
+
dispose() {
|
|
3367
|
+
if (closing) return closing;
|
|
3368
|
+
invalidate("disposed");
|
|
3369
|
+
disposed = true;
|
|
3370
|
+
clearInterval(timer);
|
|
3371
|
+
cancelAnimationFrame(frame);
|
|
3372
|
+
room.unregisterByteStreamHandler(CORRELATED_AUDIO_CONTRACT.topic);
|
|
3373
|
+
room.localParticipant.unregisterRpcMethod(CORRELATED_AUDIO_CONTRACT.cancelMethod);
|
|
3374
|
+
room.off(RoomEvent.Reconnecting, lost);
|
|
3375
|
+
room.off(RoomEvent.SignalReconnecting, lost);
|
|
3376
|
+
room.off(RoomEvent.Disconnected, lost);
|
|
3377
|
+
room.off(RoomEvent.ConnectionStateChanged, changed);
|
|
3378
|
+
room.off(RoomEvent.ParticipantDisconnected, participantLeft);
|
|
3379
|
+
attachments.delete(room);
|
|
3380
|
+
if (context) context.onstatechange = null;
|
|
3381
|
+
closing = context?.close().catch(() => {
|
|
3382
|
+
}) ?? Promise.resolve();
|
|
3383
|
+
return closing;
|
|
3384
|
+
}
|
|
3385
|
+
};
|
|
3386
|
+
}
|
|
3387
|
+
|
|
3388
|
+
// ../client/src/external-performance.ts
|
|
3389
|
+
var EXTERNAL_PERFORMANCE_KIND = externalPerformanceFrameSchema.shape.kind.value;
|
|
3390
|
+
var EXTERNAL_PERFORMANCE_TIMING = Object.freeze({
|
|
3391
|
+
receiverPlayout: "unsupported",
|
|
3392
|
+
audibleCheckpoint: externalPerformanceFrameSchema.shape.audible_checkpoint.value,
|
|
3393
|
+
audioMapping: externalPerformanceFrameSchema.shape.audio_mapping.value
|
|
3394
|
+
});
|
|
3395
|
+
var statusSchema = externalPerformanceFrameSchema.shape.status;
|
|
3396
|
+
var KNOWN_EXTERNAL_PERFORMANCE_STATUSES = Object.freeze([...statusSchema.options[0].options]);
|
|
3397
|
+
var knownStatuses = new Set(KNOWN_EXTERNAL_PERFORMANCE_STATUSES);
|
|
3398
|
+
function isObservable(frame) {
|
|
3399
|
+
return knownStatuses.has(frame.status);
|
|
3400
|
+
}
|
|
3401
|
+
var NEXT = {
|
|
3402
|
+
unseen: /* @__PURE__ */ new Set(["accepted", "dropped"]),
|
|
3403
|
+
accepted: /* @__PURE__ */ new Set(["audio_released", "cancelled"]),
|
|
3404
|
+
audio_released: /* @__PURE__ */ new Set(["cancelled"]),
|
|
3405
|
+
dropped: /* @__PURE__ */ new Set(),
|
|
3406
|
+
cancelled: /* @__PURE__ */ new Set(),
|
|
3407
|
+
retired: /* @__PURE__ */ new Set()
|
|
3408
|
+
};
|
|
3409
|
+
var LEDGER_LIMIT = 4096;
|
|
3410
|
+
function deliver(callback) {
|
|
3411
|
+
try {
|
|
3412
|
+
void Promise.resolve(callback()).catch(() => {
|
|
3413
|
+
});
|
|
3414
|
+
} catch {
|
|
3415
|
+
}
|
|
3416
|
+
}
|
|
3417
|
+
function createExternalPerformanceObserver(sessionId, callbacks) {
|
|
3418
|
+
const generations = /* @__PURE__ */ new Map();
|
|
3419
|
+
let state = "live";
|
|
3420
|
+
return {
|
|
3421
|
+
receive(frame) {
|
|
3422
|
+
if (state !== "live" || frame.session_id !== sessionId || !isObservable(frame)) return;
|
|
3423
|
+
const previous = generations.get(frame.generation_id) ?? "unseen";
|
|
3424
|
+
if (!NEXT[previous].has(frame.status)) return;
|
|
3425
|
+
if (previous === "unseen" && generations.size >= LEDGER_LIMIT) {
|
|
3426
|
+
const oldest = generations.keys().next();
|
|
3427
|
+
if (!oldest.done) generations.delete(oldest.value);
|
|
3428
|
+
}
|
|
3429
|
+
generations.set(frame.generation_id, frame.status);
|
|
3430
|
+
deliver(() => callbacks.onFrame(frame));
|
|
3431
|
+
},
|
|
3432
|
+
invalidate(reason) {
|
|
3433
|
+
if (state === "closed" || state === "paused" && reason === "connection_lost") return;
|
|
3434
|
+
state = reason === "connection_lost" ? "paused" : "closed";
|
|
3435
|
+
if (state === "closed") generations.clear();
|
|
3436
|
+
else for (const id of generations.keys()) generations.set(id, "retired");
|
|
3437
|
+
deliver(() => callbacks.onInvalidated(reason));
|
|
3438
|
+
},
|
|
3439
|
+
/** The call is connected to its agent again. Only a connection loss can be resumed. */
|
|
3440
|
+
resume() {
|
|
3441
|
+
if (state === "paused") state = "live";
|
|
3442
|
+
}
|
|
3443
|
+
};
|
|
3444
|
+
}
|
|
2991
3445
|
|
|
2992
3446
|
// ../client/src/react/session-lifecycle.ts
|
|
2993
3447
|
function phaseForReason(reason) {
|
|
@@ -3183,6 +3637,32 @@ function useSessionLifecycle(input) {
|
|
|
3183
3637
|
const grantIdRef = useRef(null);
|
|
3184
3638
|
grantIdRef.current = grantId;
|
|
3185
3639
|
const retiredGrantRef = useRef(null);
|
|
3640
|
+
const performanceEnabled = input.externalPerformance !== void 0;
|
|
3641
|
+
const performanceCallbacksRef = useRef(input.externalPerformance);
|
|
3642
|
+
if (input.externalPerformance) performanceCallbacksRef.current = input.externalPerformance;
|
|
3643
|
+
const performanceRef = useRef(null);
|
|
3644
|
+
const endedPerformanceGrantRef = useRef(null);
|
|
3645
|
+
useEffect(() => {
|
|
3646
|
+
if (!grantId || !performanceEnabled || endedPerformanceGrantRef.current === grantId) return;
|
|
3647
|
+
const observer = createExternalPerformanceObserver(grantId, {
|
|
3648
|
+
onFrame: (frame) => performanceCallbacksRef.current?.onFrame(frame),
|
|
3649
|
+
onInvalidated: (reason) => {
|
|
3650
|
+
if (reason === "ended") endedPerformanceGrantRef.current = grantId;
|
|
3651
|
+
performanceCallbacksRef.current?.onInvalidated(reason);
|
|
3652
|
+
}
|
|
3653
|
+
});
|
|
3654
|
+
performanceRef.current = observer;
|
|
3655
|
+
return () => {
|
|
3656
|
+
observer.invalidate("binding_changed");
|
|
3657
|
+
if (performanceRef.current === observer) performanceRef.current = null;
|
|
3658
|
+
};
|
|
3659
|
+
}, [grantId, performanceEnabled]);
|
|
3660
|
+
const invalidateExternalPerformance = useCallback((reason) => {
|
|
3661
|
+
if (grantId === grantIdRef.current) performanceRef.current?.invalidate(reason);
|
|
3662
|
+
}, [grantId]);
|
|
3663
|
+
const resumeExternalPerformance = useCallback(() => {
|
|
3664
|
+
if (grantId === grantIdRef.current) performanceRef.current?.resume();
|
|
3665
|
+
}, [grantId]);
|
|
3186
3666
|
const [recovery, setRecovery] = useState({ kind: "connected" });
|
|
3187
3667
|
const recoveryRef = useRef(recovery);
|
|
3188
3668
|
recoveryRef.current = recovery;
|
|
@@ -3228,6 +3708,7 @@ function useSessionLifecycle(input) {
|
|
|
3228
3708
|
}
|
|
3229
3709
|
}, []);
|
|
3230
3710
|
const reset = useCallback(() => {
|
|
3711
|
+
invalidateExternalPerformance("binding_changed");
|
|
3231
3712
|
clearTimer();
|
|
3232
3713
|
connectedRef.current = false;
|
|
3233
3714
|
manualReconnectPendingRef.current = false;
|
|
@@ -3242,7 +3723,7 @@ function useSessionLifecycle(input) {
|
|
|
3242
3723
|
setConnected(false);
|
|
3243
3724
|
setAgentPresentState(false);
|
|
3244
3725
|
lastActivityRef.current = Date.now();
|
|
3245
|
-
}, [clearTimer]);
|
|
3726
|
+
}, [clearTimer, invalidateExternalPerformance]);
|
|
3246
3727
|
useEffect(() => {
|
|
3247
3728
|
if (active) return;
|
|
3248
3729
|
userEndedRef.current = false;
|
|
@@ -3276,6 +3757,7 @@ function useSessionLifecycle(input) {
|
|
|
3276
3757
|
}, [grantId]);
|
|
3277
3758
|
const endIdle = useCallback(() => {
|
|
3278
3759
|
if (!connectedRef.current) return;
|
|
3760
|
+
invalidateExternalPerformance("ended");
|
|
3279
3761
|
clearTimer();
|
|
3280
3762
|
connectedRef.current = false;
|
|
3281
3763
|
setConnected(false);
|
|
@@ -3283,7 +3765,7 @@ function useSessionLifecycle(input) {
|
|
|
3283
3765
|
releaseRef.current("idle_timeout");
|
|
3284
3766
|
setRecovery({ kind: "ended", reason: "idle" });
|
|
3285
3767
|
leaveRoomRef.current?.();
|
|
3286
|
-
}, [clearTimer]);
|
|
3768
|
+
}, [clearTimer, invalidateExternalPerformance]);
|
|
3287
3769
|
const onConnected = useCallback(() => {
|
|
3288
3770
|
if (grantId === null || grantIdRef.current !== grantId) return;
|
|
3289
3771
|
if (userEndedRef.current || readinessFailureRef.current || retiredGrantRef.current !== null && retiredGrantRef.current === grantIdRef.current) return;
|
|
@@ -3301,6 +3783,7 @@ function useSessionLifecycle(input) {
|
|
|
3301
3783
|
if (grantId === null || grantIdRef.current !== grantId) return;
|
|
3302
3784
|
if (!active || userEndedRef.current || readinessFailureRef.current) return;
|
|
3303
3785
|
const action = disconnectAction(reason, connectedRef.current);
|
|
3786
|
+
invalidateExternalPerformance(action === "end" ? "ended" : "connection_lost");
|
|
3304
3787
|
if (action === "noop") return;
|
|
3305
3788
|
if (action === "reset") {
|
|
3306
3789
|
if (recoveryRef.current.kind === "refreshing" || recoveryRef.current.kind === "reconnecting") return;
|
|
@@ -3320,12 +3803,13 @@ function useSessionLifecycle(input) {
|
|
|
3320
3803
|
refreshBaselineCapacityRef.current = null;
|
|
3321
3804
|
setRecovery({ kind: "reconnecting", attempt: attemptRef.current });
|
|
3322
3805
|
},
|
|
3323
|
-
[active, reset, grantId]
|
|
3806
|
+
[active, reset, grantId, invalidateExternalPerformance]
|
|
3324
3807
|
);
|
|
3325
3808
|
const onConnectionError = useCallback(
|
|
3326
3809
|
(error) => {
|
|
3327
3810
|
if (grantId === null || grantIdRef.current !== grantId) return;
|
|
3328
3811
|
if (!active || userEndedRef.current || readinessFailureRef.current || !isRecoverableConnectionError(error)) return;
|
|
3812
|
+
invalidateExternalPerformance("connection_lost");
|
|
3329
3813
|
connectedRef.current = false;
|
|
3330
3814
|
setConnected(false);
|
|
3331
3815
|
setAgentPresentState(false);
|
|
@@ -3336,7 +3820,7 @@ function useSessionLifecycle(input) {
|
|
|
3336
3820
|
return { kind: "reconnecting", attempt: attemptRef.current };
|
|
3337
3821
|
});
|
|
3338
3822
|
},
|
|
3339
|
-
[active, grantId]
|
|
3823
|
+
[active, grantId, invalidateExternalPerformance]
|
|
3340
3824
|
);
|
|
3341
3825
|
useEffect(() => {
|
|
3342
3826
|
if (capacity.kind === "error" && recovery.kind === "connected") manualReconnectPendingRef.current = false;
|
|
@@ -3356,6 +3840,9 @@ function useSessionLifecycle(input) {
|
|
|
3356
3840
|
onConnected();
|
|
3357
3841
|
return;
|
|
3358
3842
|
}
|
|
3843
|
+
if (state === "disconnected" && connectedRef.current || state === "reconnecting" || state === "signalReconnecting") {
|
|
3844
|
+
invalidateExternalPerformance("connection_lost");
|
|
3845
|
+
}
|
|
3359
3846
|
if (state === "reconnecting" || state === "signalReconnecting") {
|
|
3360
3847
|
if (connectedRef.current) {
|
|
3361
3848
|
setConnected(false);
|
|
@@ -3365,7 +3852,7 @@ function useSessionLifecycle(input) {
|
|
|
3365
3852
|
}
|
|
3366
3853
|
}
|
|
3367
3854
|
},
|
|
3368
|
-
[active, onConnected, grantId]
|
|
3855
|
+
[active, onConnected, grantId, invalidateExternalPerformance]
|
|
3369
3856
|
);
|
|
3370
3857
|
useEffect(() => {
|
|
3371
3858
|
if (!active || userEnded || readinessFailed || !connected || !agentPresent || !grantState.grant) return;
|
|
@@ -3378,6 +3865,7 @@ function useSessionLifecycle(input) {
|
|
|
3378
3865
|
const failReadiness = useCallback((code) => {
|
|
3379
3866
|
if (userEndedRef.current || readinessFailureRef.current) return;
|
|
3380
3867
|
readinessFailureRef.current = code;
|
|
3868
|
+
invalidateExternalPerformance("ended");
|
|
3381
3869
|
clearTimer();
|
|
3382
3870
|
connectedRef.current = false;
|
|
3383
3871
|
manualReconnectPendingRef.current = false;
|
|
@@ -3390,7 +3878,7 @@ function useSessionLifecycle(input) {
|
|
|
3390
3878
|
releaseRef.current("disconnected");
|
|
3391
3879
|
setReadinessFailed(true);
|
|
3392
3880
|
leaveRoomRef.current?.();
|
|
3393
|
-
}, [clearTimer]);
|
|
3881
|
+
}, [clearTimer, invalidateExternalPerformance]);
|
|
3394
3882
|
useEffect(() => {
|
|
3395
3883
|
if (!active || userEnded || readinessFailed) return;
|
|
3396
3884
|
if (recovery.kind === "ended" || recovery.kind === "failed") return;
|
|
@@ -3471,6 +3959,7 @@ function useSessionLifecycle(input) {
|
|
|
3471
3959
|
const end = useCallback(() => {
|
|
3472
3960
|
if (!active || userEndedRef.current || phaseRef.current.kind === "ended") return;
|
|
3473
3961
|
userEndedRef.current = true;
|
|
3962
|
+
invalidateExternalPerformance("ended");
|
|
3474
3963
|
clearTimer();
|
|
3475
3964
|
connectedRef.current = false;
|
|
3476
3965
|
manualReconnectPendingRef.current = false;
|
|
@@ -3481,7 +3970,7 @@ function useSessionLifecycle(input) {
|
|
|
3481
3970
|
setRecovery({ kind: "ended", reason: "user" });
|
|
3482
3971
|
clearRef.current();
|
|
3483
3972
|
leaveRoomRef.current?.();
|
|
3484
|
-
}, [active, clearTimer]);
|
|
3973
|
+
}, [active, clearTimer, invalidateExternalPerformance]);
|
|
3485
3974
|
const reconnect = useCallback(() => {
|
|
3486
3975
|
if (!active) return;
|
|
3487
3976
|
if (readinessFailureRef.current) {
|
|
@@ -3551,16 +4040,22 @@ function useSessionLifecycle(input) {
|
|
|
3551
4040
|
const lastBehaviorRef = useRef(null);
|
|
3552
4041
|
const behaviorEnabled = Boolean(onBehaviorChange);
|
|
3553
4042
|
const onLifecycleData = useMemo(() => {
|
|
3554
|
-
if (!behaviorEnabled) return void 0;
|
|
4043
|
+
if (!behaviorEnabled && !performanceEnabled) return void 0;
|
|
3555
4044
|
return (frame) => {
|
|
3556
4045
|
const parsed = lifecycleServerFrameSchema.safeParse(frame);
|
|
3557
|
-
if (!parsed.success
|
|
4046
|
+
if (!parsed.success) return;
|
|
4047
|
+
if (parsed.data.kind === "external_performance") {
|
|
4048
|
+
if (grantId === grantIdRef.current && parsed.data.session_id === grantId) performanceRef.current?.receive(parsed.data);
|
|
4049
|
+
return;
|
|
4050
|
+
}
|
|
4051
|
+
if (parsed.data.kind === "ended") invalidateExternalPerformance("ended");
|
|
4052
|
+
if (!behaviorEnabled || parsed.data.kind !== "behavior_state") return;
|
|
3558
4053
|
const next = nextBehaviorSnapshot(lastBehaviorRef.current, parsed.data);
|
|
3559
4054
|
if (!next) return;
|
|
3560
4055
|
lastBehaviorRef.current = next;
|
|
3561
4056
|
onBehaviorChangeRef.current?.(next);
|
|
3562
4057
|
};
|
|
3563
|
-
}, [behaviorEnabled]);
|
|
4058
|
+
}, [behaviorEnabled, performanceEnabled, grantId, invalidateExternalPerformance]);
|
|
3564
4059
|
return useMemo(
|
|
3565
4060
|
() => ({
|
|
3566
4061
|
phase,
|
|
@@ -3579,7 +4074,9 @@ function useSessionLifecycle(input) {
|
|
|
3579
4074
|
setAgentPresent,
|
|
3580
4075
|
registerLeaveRoom,
|
|
3581
4076
|
reset,
|
|
3582
|
-
onLifecycleData
|
|
4077
|
+
onLifecycleData,
|
|
4078
|
+
invalidateExternalPerformance: performanceEnabled ? invalidateExternalPerformance : void 0,
|
|
4079
|
+
resumeExternalPerformance: performanceEnabled ? resumeExternalPerformance : void 0
|
|
3583
4080
|
}),
|
|
3584
4081
|
[
|
|
3585
4082
|
phase,
|
|
@@ -3598,7 +4095,10 @@ function useSessionLifecycle(input) {
|
|
|
3598
4095
|
setAgentPresent,
|
|
3599
4096
|
registerLeaveRoom,
|
|
3600
4097
|
reset,
|
|
3601
|
-
onLifecycleData
|
|
4098
|
+
onLifecycleData,
|
|
4099
|
+
invalidateExternalPerformance,
|
|
4100
|
+
resumeExternalPerformance,
|
|
4101
|
+
performanceEnabled
|
|
3602
4102
|
]
|
|
3603
4103
|
);
|
|
3604
4104
|
}
|
|
@@ -3627,9 +4127,45 @@ function SessionLifecycleRoomBridge({
|
|
|
3627
4127
|
const transcriptions = useTranscriptions();
|
|
3628
4128
|
const room = useRoomContext();
|
|
3629
4129
|
const roomMatchesGrant = lifecycle.grant === void 0 || lifecycle.grant !== null && room.name === lifecycle.grant.room_name;
|
|
4130
|
+
const correlatedRef = useRef(null);
|
|
4131
|
+
const correlatedObserver = useRef(lifecycle.correlatedAudio);
|
|
4132
|
+
correlatedObserver.current = lifecycle.correlatedAudio;
|
|
4133
|
+
const correlatedEnabled = lifecycle.correlatedAudio !== void 0;
|
|
4134
|
+
const correlatedSessionId = lifecycle.grant?.session_id;
|
|
4135
|
+
const startCorrelatedAudio = useCallback(async () => {
|
|
4136
|
+
const player = correlatedRef.current;
|
|
4137
|
+
if (!player) return false;
|
|
4138
|
+
try {
|
|
4139
|
+
await player.startAudio();
|
|
4140
|
+
return true;
|
|
4141
|
+
} catch {
|
|
4142
|
+
return false;
|
|
4143
|
+
}
|
|
4144
|
+
}, []);
|
|
4145
|
+
useEffect(() => {
|
|
4146
|
+
if (!correlatedEnabled || !correlatedSessionId || !roomMatchesGrant || !assistant.agent) return;
|
|
4147
|
+
const player = attachCorrelatedAudio(room, {
|
|
4148
|
+
sessionId: correlatedSessionId,
|
|
4149
|
+
agentIdentity: assistant.agent.identity,
|
|
4150
|
+
prepare: (header, signal) => correlatedObserver.current?.prepare?.(header, signal) ?? Promise.resolve(),
|
|
4151
|
+
onPosition: (position) => correlatedObserver.current?.onPosition(position),
|
|
4152
|
+
onInvalidated: (reason) => correlatedObserver.current?.onInvalidated(reason),
|
|
4153
|
+
onPlaybackBlocked: (unblock) => {
|
|
4154
|
+
setAudioPlayback?.(unblock ? "blocked" : "allowed");
|
|
4155
|
+
correlatedObserver.current?.onPlaybackBlocked?.(unblock);
|
|
4156
|
+
}
|
|
4157
|
+
});
|
|
4158
|
+
correlatedRef.current = player;
|
|
4159
|
+
return () => {
|
|
4160
|
+
correlatedRef.current = null;
|
|
4161
|
+
void player.dispose();
|
|
4162
|
+
setAudioPlayback?.("unknown");
|
|
4163
|
+
};
|
|
4164
|
+
}, [room, assistant.agent, correlatedEnabled, correlatedSessionId, roomMatchesGrant, setAudioPlayback]);
|
|
3630
4165
|
useCallMedia(room, {
|
|
3631
4166
|
setMicrophoneFacts,
|
|
3632
|
-
setAudioPlayback,
|
|
4167
|
+
setAudioPlayback: correlatedEnabled ? void 0 : setAudioPlayback,
|
|
4168
|
+
startAudioOverride: correlatedEnabled ? startCorrelatedAudio : void 0,
|
|
3633
4169
|
registerMediaControls,
|
|
3634
4170
|
microphoneWanted: microphone ?? publishesMicrophoneByDefault(lifecycle.grant),
|
|
3635
4171
|
connectionState,
|
|
@@ -3643,7 +4179,7 @@ function SessionLifecycleRoomBridge({
|
|
|
3643
4179
|
if (!mediaModeEnabled) return;
|
|
3644
4180
|
const agent = assistant.agent;
|
|
3645
4181
|
let previous;
|
|
3646
|
-
const
|
|
4182
|
+
const deliver2 = (mode) => {
|
|
3647
4183
|
if (mode === previous) return;
|
|
3648
4184
|
previous = mode;
|
|
3649
4185
|
try {
|
|
@@ -3654,7 +4190,7 @@ function SessionLifecycleRoomBridge({
|
|
|
3654
4190
|
};
|
|
3655
4191
|
const read = () => {
|
|
3656
4192
|
const value = agent?.attributes?.["rta.media_mode"];
|
|
3657
|
-
|
|
4193
|
+
deliver2(value === "video" ? "avatar" : value === "voice" ? "voice" : null);
|
|
3658
4194
|
};
|
|
3659
4195
|
const changed = (_attributes, participant) => {
|
|
3660
4196
|
if (participant === agent) read();
|
|
@@ -3663,7 +4199,7 @@ function SessionLifecycleRoomBridge({
|
|
|
3663
4199
|
read();
|
|
3664
4200
|
return () => {
|
|
3665
4201
|
room.off(RoomEvent.ParticipantAttributesChanged, changed);
|
|
3666
|
-
|
|
4202
|
+
deliver2(null);
|
|
3667
4203
|
};
|
|
3668
4204
|
}, [room, assistant.agent, mediaModeEnabled, mediaModeSessionId]);
|
|
3669
4205
|
const { send } = useChat();
|
|
@@ -3710,7 +4246,7 @@ function SessionLifecycleRoomBridge({
|
|
|
3710
4246
|
let closed = false;
|
|
3711
4247
|
let pending = false;
|
|
3712
4248
|
let previous;
|
|
3713
|
-
const
|
|
4249
|
+
const deliver2 = (details) => {
|
|
3714
4250
|
connectionHistoryRef.current?.enqueue(details);
|
|
3715
4251
|
try {
|
|
3716
4252
|
if (onConnectionDetailsChange) void Promise.resolve(callback?.(details)).catch(() => {
|
|
@@ -3737,7 +4273,7 @@ function SessionLifecycleRoomBridge({
|
|
|
3737
4273
|
const last = previous;
|
|
3738
4274
|
if (last?.connectionState === next.connectionState && last.localQuality === next.localQuality && ["audio", "video"].every((kind) => last[kind]?.publisherQuality === next[kind]?.publisherQuality && last[kind]?.streamState === next[kind]?.streamState)) return;
|
|
3739
4275
|
previous = next;
|
|
3740
|
-
|
|
4276
|
+
deliver2(next);
|
|
3741
4277
|
});
|
|
3742
4278
|
};
|
|
3743
4279
|
const onQuality = (_quality, participant) => {
|
|
@@ -3767,7 +4303,7 @@ function SessionLifecycleRoomBridge({
|
|
|
3767
4303
|
room.off(RoomEvent.TrackStreamStateChanged, onPublication);
|
|
3768
4304
|
room.off(RoomEvent.TrackSubscribed, onTrack);
|
|
3769
4305
|
room.off(RoomEvent.TrackUnsubscribed, onTrack);
|
|
3770
|
-
|
|
4306
|
+
deliver2(null);
|
|
3771
4307
|
};
|
|
3772
4308
|
}, [room, detailsSessionId, detailsEnabled]);
|
|
3773
4309
|
useEffect(() => {
|
|
@@ -3804,18 +4340,56 @@ function SessionLifecycleRoomBridge({
|
|
|
3804
4340
|
useEffect(() => {
|
|
3805
4341
|
if (!onLifecycleData) return;
|
|
3806
4342
|
const decoder = new TextDecoder();
|
|
3807
|
-
const handler = (payload,
|
|
4343
|
+
const handler = (payload, participant, _kind, topic) => {
|
|
3808
4344
|
if (topic !== RTA_LIFECYCLE_TOPIC) return;
|
|
3809
4345
|
try {
|
|
3810
|
-
|
|
4346
|
+
const frame = JSON.parse(decoder.decode(payload));
|
|
4347
|
+
if (typeof frame === "object" && frame !== null && "kind" in frame && (frame.kind === EXTERNAL_PERFORMANCE_KIND || frame.kind === "ended") && (!assistant.agent || participant !== assistant.agent || !roomMatchesGrant || room.state !== "connected")) return;
|
|
4348
|
+
if (typeof frame === "object" && frame !== null && "kind" in frame && frame.kind === "ended" && lifecycleServerFrameSchema.safeParse(frame).success) {
|
|
4349
|
+
void correlatedRef.current?.dispose();
|
|
4350
|
+
correlatedRef.current = null;
|
|
4351
|
+
}
|
|
4352
|
+
onLifecycleData(frame);
|
|
3811
4353
|
} catch {
|
|
3812
4354
|
}
|
|
3813
4355
|
};
|
|
4356
|
+
const invalidate = () => lifecycle.invalidateExternalPerformance?.("connection_lost");
|
|
4357
|
+
const changed = () => {
|
|
4358
|
+
if (room.state === "disconnected" || room.state === "reconnecting" || room.state === "signalReconnecting") invalidate();
|
|
4359
|
+
};
|
|
3814
4360
|
room.on(RoomEvent.DataReceived, handler);
|
|
4361
|
+
if (lifecycle.invalidateExternalPerformance) {
|
|
4362
|
+
room.on(RoomEvent.Reconnecting, invalidate);
|
|
4363
|
+
room.on(RoomEvent.SignalReconnecting, invalidate);
|
|
4364
|
+
room.on(RoomEvent.Disconnected, invalidate);
|
|
4365
|
+
room.on(RoomEvent.ConnectionStateChanged, changed);
|
|
4366
|
+
}
|
|
3815
4367
|
return () => {
|
|
3816
4368
|
room.off(RoomEvent.DataReceived, handler);
|
|
4369
|
+
room.off(RoomEvent.Reconnecting, invalidate);
|
|
4370
|
+
room.off(RoomEvent.SignalReconnecting, invalidate);
|
|
4371
|
+
room.off(RoomEvent.Disconnected, invalidate);
|
|
4372
|
+
room.off(RoomEvent.ConnectionStateChanged, changed);
|
|
4373
|
+
};
|
|
4374
|
+
}, [room, onLifecycleData, assistant.agent, roomMatchesGrant, lifecycle.invalidateExternalPerformance]);
|
|
4375
|
+
useEffect(() => {
|
|
4376
|
+
if (!assistant.agent) return;
|
|
4377
|
+
return () => lifecycle.invalidateExternalPerformance?.("connection_lost");
|
|
4378
|
+
}, [assistant.agent, lifecycle.invalidateExternalPerformance]);
|
|
4379
|
+
const resumeExternalPerformance = lifecycle.resumeExternalPerformance;
|
|
4380
|
+
useEffect(() => {
|
|
4381
|
+
if (!resumeExternalPerformance) return;
|
|
4382
|
+
const resumeIfBound = () => {
|
|
4383
|
+
if (assistant.agent && roomMatchesGrant && room.state === "connected") resumeExternalPerformance();
|
|
4384
|
+
};
|
|
4385
|
+
resumeIfBound();
|
|
4386
|
+
room.on(RoomEvent.Reconnected, resumeIfBound);
|
|
4387
|
+
room.on(RoomEvent.ConnectionStateChanged, resumeIfBound);
|
|
4388
|
+
return () => {
|
|
4389
|
+
room.off(RoomEvent.Reconnected, resumeIfBound);
|
|
4390
|
+
room.off(RoomEvent.ConnectionStateChanged, resumeIfBound);
|
|
3817
4391
|
};
|
|
3818
|
-
}, [room,
|
|
4392
|
+
}, [room, assistant.agent, roomMatchesGrant, resumeExternalPerformance]);
|
|
3819
4393
|
useEffect(() => {
|
|
3820
4394
|
if (!registerDataPublisher) return;
|
|
3821
4395
|
const encoder = new TextEncoder();
|
|
@@ -3951,6 +4525,7 @@ function useRealtimeSession(input) {
|
|
|
3951
4525
|
onBehaviorChange,
|
|
3952
4526
|
onClipResult,
|
|
3953
4527
|
onVoiceError,
|
|
4528
|
+
correlatedAudio,
|
|
3954
4529
|
// everything else (client/active/idleSeconds/idleWarnLeadSeconds/reconnect/requestOptions…)
|
|
3955
4530
|
// flows straight into the inner SSOT hook.
|
|
3956
4531
|
...inner
|
|
@@ -4023,6 +4598,7 @@ function useRealtimeSession(input) {
|
|
|
4023
4598
|
return `t-${Date.now()}-${++turnSequenceRef.current}`;
|
|
4024
4599
|
}, []);
|
|
4025
4600
|
const onLifecycleData = useCallback((frame) => {
|
|
4601
|
+
lifecycle.onLifecycleData?.(frame);
|
|
4026
4602
|
const parsed = lifecycleServerFrameSchema.safeParse(frame);
|
|
4027
4603
|
if (!parsed.success) return;
|
|
4028
4604
|
const f = parsed.data;
|
|
@@ -4082,7 +4658,7 @@ function useRealtimeSession(input) {
|
|
|
4082
4658
|
break;
|
|
4083
4659
|
}
|
|
4084
4660
|
}
|
|
4085
|
-
}, []);
|
|
4661
|
+
}, [lifecycle.onLifecycleData]);
|
|
4086
4662
|
const registerDataPublisher = useCallback((publish) => {
|
|
4087
4663
|
dataPublisherRef.current = publish;
|
|
4088
4664
|
}, []);
|
|
@@ -4348,6 +4924,7 @@ function useRealtimeSession(input) {
|
|
|
4348
4924
|
// stayConnected/reconnect/markActivity/the sinks) → a true drop-in superset, then
|
|
4349
4925
|
// add the realtime-session surface. `reset` + `lifecycle` come AFTER so ours win.
|
|
4350
4926
|
...lifecycle,
|
|
4927
|
+
correlatedAudio,
|
|
4351
4928
|
turn,
|
|
4352
4929
|
clocks,
|
|
4353
4930
|
endsAt,
|
|
@@ -4382,6 +4959,7 @@ function useRealtimeSession(input) {
|
|
|
4382
4959
|
}),
|
|
4383
4960
|
[
|
|
4384
4961
|
lifecycle,
|
|
4962
|
+
correlatedAudio,
|
|
4385
4963
|
turn,
|
|
4386
4964
|
clocks,
|
|
4387
4965
|
endsAt,
|
|
@@ -4482,7 +5060,9 @@ function useAvatarCall(props) {
|
|
|
4482
5060
|
onIdleWarning: props.onQuiet ? ({ secondsLeft }) => props.onQuiet?.({ secondsLeft }) : void 0,
|
|
4483
5061
|
onCreditsLow: props.onLowBalance ? ({ secondsLeft }) => props.onLowBalance?.({ secondsLeft }) : void 0,
|
|
4484
5062
|
onEnded: props.onEnded ? ({ reason }) => props.onEnded?.({ reason }) : void 0,
|
|
4485
|
-
onVoiceError: props.onVoiceError ? (error) => props.onVoiceError?.(error) : void 0
|
|
5063
|
+
onVoiceError: props.onVoiceError ? (error) => props.onVoiceError?.(error) : void 0,
|
|
5064
|
+
externalPerformance: props.externalPerformance,
|
|
5065
|
+
correlatedAudio: props.correlatedAudio
|
|
4486
5066
|
});
|
|
4487
5067
|
sessionRef.current = session;
|
|
4488
5068
|
const call = handleFor(session);
|
|
@@ -4520,6 +5100,7 @@ function useAvatarCall(props) {
|
|
|
4520
5100
|
RealtimeAvatarLiveKitRoom,
|
|
4521
5101
|
{
|
|
4522
5102
|
grant: session.grant,
|
|
5103
|
+
renderRoomAudio: props.correlatedAudio === void 0,
|
|
4523
5104
|
// LiveKit captures the microphone on connect from this value, so a mute chosen while
|
|
4524
5105
|
// waiting is honoured rather than overridden when the call goes live.
|
|
4525
5106
|
audio: props.listen !== false && !session.microphoneMuted,
|
|
@@ -4919,4 +5500,4 @@ function RecordingPlayer({ assets, className }) {
|
|
|
4919
5500
|
] });
|
|
4920
5501
|
}
|
|
4921
5502
|
|
|
4922
|
-
export { AdaptivePlayoutController, AvatarCall, AvatarVideoSurface, DEFAULT_APPROACHING_END_LEAD_SECONDS, DEFAULT_AVATAR_PLAYOUT_DELAY_SECONDS, DEFAULT_AVATAR_UNSTABLE_STALL_MS, DEFAULT_CREDITS_LOW_LEAD_SECONDS, DEFAULT_FISH_TTS_MODEL, DEFAULT_GOVERNOR_CONFIG, DEFAULT_GRACE_CEILING_SECONDS, DEFAULT_GRACE_WINDOW_LEAD_SECONDS, DEFAULT_IDLE_SECONDS, DEFAULT_IDLE_WARN_LEAD_SECONDS, DEFAULT_READY_TIMEOUT_SECONDS, DEFAULT_TURN_TIMEOUT_SECONDS, FISH_DELIVERY_MODELS, MAX_RECONNECT_ATTEMPTS, MAX_SESSION_INSTRUCTIONS_CHARS, MIC_LEASE_ENDED_TIMEOUT_MS, NETWORK_QUALITY_POLICY, RECONNECT_BACKOFF_MS, RealtimeAvatarCapacityError, RealtimeAvatarLiveKitRoom, RecordingPlayer, SessionLifecycleRoomBridge, capacityErrorFromBusy, capacityStateFromGrant, createProxyClient, fishTtsModelSchema, knownBehaviorStates, mapTurnState, readInboundRtp, sessionBehaviorSchema, sessionClipSchema, shouldReplayPendingTurn, splitCallTranscript, useAvatarAdaptivePlayoutDelay, useAvatarCall, useAvatarCamera, useAvatarPlayoutDelay, useAvatarQualityGovernor, useCallTranscript, useCharacterTools, useLiveKitAvatarGrant, useMicLease, useRealtimeSession, useReleaseMicLeaseOnTrackEnded, useSessionLifecycle };
|
|
5503
|
+
export { AdaptivePlayoutController, AvatarCall, AvatarVideoSurface, DEFAULT_APPROACHING_END_LEAD_SECONDS, DEFAULT_AVATAR_PLAYOUT_DELAY_SECONDS, DEFAULT_AVATAR_UNSTABLE_STALL_MS, DEFAULT_CREDITS_LOW_LEAD_SECONDS, DEFAULT_FISH_TTS_MODEL, DEFAULT_GOVERNOR_CONFIG, DEFAULT_GRACE_CEILING_SECONDS, DEFAULT_GRACE_WINDOW_LEAD_SECONDS, DEFAULT_IDLE_SECONDS, DEFAULT_IDLE_WARN_LEAD_SECONDS, DEFAULT_READY_TIMEOUT_SECONDS, DEFAULT_TURN_TIMEOUT_SECONDS, EXTERNAL_PERFORMANCE_TIMING, FISH_DELIVERY_MODELS, KNOWN_EXTERNAL_PERFORMANCE_STATUSES, MAX_RECONNECT_ATTEMPTS, MAX_SESSION_INSTRUCTIONS_CHARS, MIC_LEASE_ENDED_TIMEOUT_MS, NETWORK_QUALITY_POLICY, RECONNECT_BACKOFF_MS, RealtimeAvatarCapacityError, RealtimeAvatarLiveKitRoom, RecordingPlayer, SessionLifecycleRoomBridge, capacityErrorFromBusy, capacityStateFromGrant, createProxyClient, fishTtsModelSchema, knownBehaviorStates, mapTurnState, readInboundRtp, sessionBehaviorSchema, sessionClipSchema, shouldReplayPendingTurn, splitCallTranscript, useAvatarAdaptivePlayoutDelay, useAvatarCall, useAvatarCamera, useAvatarPlayoutDelay, useAvatarQualityGovernor, useCallTranscript, useCharacterTools, useLiveKitAvatarGrant, useMicLease, useRealtimeSession, useReleaseMicLeaseOnTrackEnded, useSessionLifecycle };
|