realtime-avatar 0.33.0 → 0.35.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/react.js CHANGED
@@ -1461,7 +1461,7 @@ function stepNetworkQuality(previous, sample, nowMs, options) {
1461
1461
  }
1462
1462
 
1463
1463
  // ../client/src/react/network-quality-binding.ts
1464
- function bindNetworkQuality(publication, inhibited, notify, tickMs) {
1464
+ function bindNetworkQuality(publication, inhibited, notify2, tickMs) {
1465
1465
  let disposed = false;
1466
1466
  let running = false;
1467
1467
  let state = initNetworkQuality(performance.now());
@@ -1475,7 +1475,7 @@ function bindNetworkQuality(publication, inhibited, notify, tickMs) {
1475
1475
  };
1476
1476
  const emit = (status) => {
1477
1477
  try {
1478
- notify(status);
1478
+ notify2(status);
1479
1479
  } catch {
1480
1480
  }
1481
1481
  };
@@ -2359,15 +2359,116 @@ var schema5 = z.object({
2359
2359
  "request_id": schema22.optional()
2360
2360
  });
2361
2361
  var voiceErrorFrameSchema = schema5;
2362
- var schema04 = z.enum(["s1", "s2-pro", "s2.1-pro", "drama-3"]);
2363
- var schema13 = z.enum(["speech-1.6", "speech-1.5", "s1-mini"]);
2364
- var schema23 = z.union([schema04, schema13]);
2365
- var schema33 = z.enum(["whisper"]);
2366
- var schema43 = z.null();
2367
- var schema52 = z.union([schema33, schema43]);
2368
- var fishTtsModelSchema = schema23;
2362
+ var schema04 = z.string().min(1).max(200);
2363
+ var schema13 = z.literal(1);
2364
+ var schema23 = z.string().min(1).max(200);
2365
+ var schema33 = z.int().min(1).max(9007199254740991);
2366
+ var schema43 = z.literal(24e3);
2367
+ var schema52 = z.literal("pcm_s16le");
2368
+ var schema6 = z.object({
2369
+ "session_id": schema04,
2370
+ "generation_id": schema04,
2371
+ "version": schema13,
2372
+ "utterance_id": schema23,
2373
+ "stream_id": schema04,
2374
+ "epoch": schema33,
2375
+ "sample_rate": schema43,
2376
+ "channels": schema13,
2377
+ "encoding": schema52
2378
+ });
2379
+ var schema7 = z.enum(["started", "progress", "finished", "cancelled"]);
2380
+ var schema8 = z.int().min(0).max(144e4);
2381
+ z.strictObject({
2382
+ "session_id": schema04,
2383
+ "generation_id": schema04,
2384
+ "utterance_id": schema23,
2385
+ "stream_id": schema04,
2386
+ "epoch": schema33,
2387
+ "status": schema7,
2388
+ "samples": schema8
2389
+ });
2390
+ var schema10 = z.object({
2391
+ "version": schema13,
2392
+ "session_id": schema04,
2393
+ "epoch": schema33
2394
+ });
2395
+ var schema11 = z.literal("external_llm_error");
2396
+ var schema122 = z.enum(["external_llm_refused", "external_llm_failed"]);
2397
+ var schema132 = z.string();
2398
+ var schema14 = z.literal(true);
2399
+ var schema15 = z.object({
2400
+ "kind": schema11,
2401
+ "code": schema122,
2402
+ "message": schema132,
2403
+ "fatal": schema14
2404
+ });
2405
+ var correlatedAudioHeaderSchema = schema6;
2406
+ var correlatedAudioCancelSchema = schema10;
2407
+ var externalLlmErrorFrameSchema = schema15;
2408
+ var CORRELATED_AUDIO_CONTRACT = {
2409
+ "topic": "rta.audio.v1",
2410
+ "receiptMethod": "rta.audio.receipt.v1",
2411
+ "cancelMethod": "rta.audio.cancel.v1",
2412
+ "maxSamples": 144e4,
2413
+ "maxChunkBytes": 48e3,
2414
+ "maxBufferedSamples": 48e3
2415
+ };
2416
+ var schema05 = z.string().min(1).max(200);
2417
+ var schema16 = z.literal(1);
2418
+ var schema24 = z.enum(["respond", "recovery", "initiative"]);
2419
+ var schema34 = z.string().min(1).max(32);
2420
+ var schema44 = z.union([schema24, schema34]);
2421
+ var schema53 = z.enum(["draft", "accepted"]);
2422
+ var schema62 = z.literal("speech_segments_v1");
2423
+ var schema72 = z.literal("external");
2424
+ var schema82 = z.object({
2425
+ "renderer": schema72
2426
+ });
2427
+ z.object({
2428
+ "session_id": schema05,
2429
+ "generation_id": schema05,
2430
+ "version": schema16,
2431
+ "purpose": schema44,
2432
+ "admission": schema53,
2433
+ "text_output": schema62.optional(),
2434
+ "visible_character": schema82.optional()
2435
+ });
2436
+ var schema102 = z.string().min(1).max(200);
2437
+ z.object({
2438
+ "generation_id": schema05,
2439
+ "utterance_id": schema102
2440
+ });
2441
+ var schema123 = z.literal("external_performance");
2442
+ var schema133 = z.enum(["accepted", "dropped", "cancelled", "audio_released"]);
2443
+ var schema142 = z.union([schema133, schema34]);
2444
+ var schema152 = z.literal("unavailable");
2445
+ var schema162 = z.literal("unavailable");
2446
+ var schema17 = z.object({
2447
+ "session_id": schema05,
2448
+ "generation_id": schema05,
2449
+ "kind": schema123,
2450
+ "version": schema16,
2451
+ "status": schema142,
2452
+ "audible_checkpoint": schema152,
2453
+ "audio_mapping": schema162
2454
+ });
2455
+ var schema18 = z.string().regex(new RegExp("^(?:[^\\uD800-\\uDFFF]|[\\uD800-\\uDBFF][\\uDC00-\\uDFFF])*$")).min(1).max(5e3);
2456
+ z.strictObject({
2457
+ "version": schema16,
2458
+ "generation_id": schema05,
2459
+ "utterance_id": schema102,
2460
+ "text": schema18
2461
+ });
2462
+ var externalPerformanceFrameSchema = schema17;
2463
+ var schema06 = z.enum(["s1", "s2-pro", "s2.1-pro", "drama-3"]);
2464
+ var schema110 = z.enum(["speech-1.6", "speech-1.5", "s1-mini"]);
2465
+ var schema25 = z.union([schema06, schema110]);
2466
+ var schema35 = z.enum(["whisper"]);
2467
+ var schema45 = z.null();
2468
+ var schema54 = z.union([schema35, schema45]);
2469
+ var fishTtsModelSchema = schema25;
2369
2470
  var DEFAULT_FISH_TTS_MODEL = "s2.1-pro";
2370
- var fishDeliverySchema = schema52;
2471
+ var fishDeliverySchema = schema54;
2371
2472
  var FISH_DELIVERY_MODELS = { "whisper": ["drama-3"] };
2372
2473
 
2373
2474
  // ../client/src/wire.ts
@@ -2683,7 +2784,9 @@ var lifecycleServerFrameSchema = z.discriminatedUnion("kind", [
2683
2784
  endedFrameSchema,
2684
2785
  behaviorStateFrameSchema,
2685
2786
  clipAckFrameSchema,
2686
- voiceErrorFrameSchema2
2787
+ voiceErrorFrameSchema2,
2788
+ externalPerformanceFrameSchema,
2789
+ externalLlmErrorFrameSchema
2687
2790
  ]);
2688
2791
  var liveKitCapacitySnapshotSchema = z.object({
2689
2792
  // Placement identity + per-worker session ceiling. These are always present on the
@@ -2965,6 +3068,7 @@ function useCallMedia(room, options) {
2965
3068
  }, [room, setAudioPlayback, connecting]);
2966
3069
  const controls = useMemo(() => ({
2967
3070
  startAudio: async () => {
3071
+ if (options.startAudioOverride) return options.startAudioOverride();
2968
3072
  try {
2969
3073
  await room.startAudio();
2970
3074
  } catch {
@@ -2981,13 +3085,372 @@ function useCallMedia(room, options) {
2981
3085
  setVersion((value) => value + 1);
2982
3086
  }
2983
3087
  }
2984
- }), [room]);
3088
+ }), [room, options.startAudioOverride]);
2985
3089
  useEffect(() => {
2986
3090
  if (!registerMediaControls) return;
2987
3091
  registerMediaControls(controls);
2988
3092
  return () => registerMediaControls(null);
2989
3093
  }, [controls, registerMediaControls]);
2990
3094
  }
3095
+ var attachments = /* @__PURE__ */ new WeakSet();
3096
+ var bindings = /* @__PURE__ */ new WeakMap();
3097
+ function notify(callback) {
3098
+ try {
3099
+ void Promise.resolve(callback()).catch(() => {
3100
+ });
3101
+ } catch {
3102
+ }
3103
+ }
3104
+ function attachCorrelatedAudio(room, options) {
3105
+ if (attachments.has(room)) throw new Error("Correlated audio is already attached to this room");
3106
+ if (!options.sessionId || !options.agentIdentity) throw new Error("Correlated audio requires the held session and bound agent");
3107
+ attachments.add(room);
3108
+ let context;
3109
+ let disposed = false;
3110
+ const previous = bindings.get(room);
3111
+ const binding = previous?.sessionId === options.sessionId ? previous : { sessionId: options.sessionId, epoch: 1, unavailable: false, seen: /* @__PURE__ */ new Set() };
3112
+ bindings.set(room, binding);
3113
+ const sources = [];
3114
+ const active = (source) => sources.includes(source);
3115
+ const buffered = () => sources.reduce((sum, source) => sum + source.received - source.consumed, 0);
3116
+ let nextStart = 0;
3117
+ let blocked;
3118
+ let closing;
3119
+ const seen = binding.seen;
3120
+ const waiters = /* @__PURE__ */ new Set();
3121
+ const wake = () => {
3122
+ for (const resolve of waiters) resolve();
3123
+ waiters.clear();
3124
+ };
3125
+ const identity = (header) => ({
3126
+ session_id: header.session_id,
3127
+ generation_id: header.generation_id,
3128
+ utterance_id: header.utterance_id,
3129
+ stream_id: header.stream_id,
3130
+ epoch: header.epoch
3131
+ });
3132
+ const position = (current, state) => {
3133
+ notify(() => options.onPosition({ ...identity(current.header), samples: current.consumed, sampleRate: current.header.sample_rate, state }));
3134
+ };
3135
+ const receipt = (current, status) => {
3136
+ const payload = { ...identity(current.header), status, samples: current.consumed };
3137
+ if (current.rpcBusy) {
3138
+ current.pendingReceipt = payload;
3139
+ return;
3140
+ }
3141
+ const send = async (next) => {
3142
+ current.rpcBusy = true;
3143
+ try {
3144
+ await room.localParticipant.performRpc({
3145
+ destinationIdentity: options.agentIdentity,
3146
+ method: CORRELATED_AUDIO_CONTRACT.receiptMethod,
3147
+ payload: JSON.stringify(next),
3148
+ responseTimeout: 2e3
3149
+ });
3150
+ } catch {
3151
+ if (!disposed && next.epoch === binding.epoch) invalidate("connection_lost");
3152
+ } finally {
3153
+ current.rpcBusy = false;
3154
+ const pending = current.pendingReceipt;
3155
+ current.pendingReceipt = void 0;
3156
+ if (pending && !disposed && pending.epoch === binding.epoch && !binding.unavailable) void send(pending);
3157
+ }
3158
+ };
3159
+ void send(payload);
3160
+ };
3161
+ const clear = () => {
3162
+ for (const current of sources.splice(0)) {
3163
+ current.abort.abort();
3164
+ for (const segment of current.segments) {
3165
+ segment.node.onended = null;
3166
+ segment.node.stop();
3167
+ segment.node.disconnect();
3168
+ }
3169
+ current.segments.length = 0;
3170
+ }
3171
+ nextStart = context?.currentTime ?? 0;
3172
+ wake();
3173
+ };
3174
+ const invalidate = (reason) => {
3175
+ if (disposed) return;
3176
+ if (reason === "connection_lost") {
3177
+ if (binding.unavailable) return;
3178
+ binding.unavailable = true;
3179
+ }
3180
+ if (reason === "invalid_stream" || reason === "disposed") for (const current of sources) receipt(current, "cancelled");
3181
+ clear();
3182
+ notify(() => options.onInvalidated(reason));
3183
+ };
3184
+ const tick = (reportCredit = true) => {
3185
+ if (!context || context.state !== "running" || disposed || binding.unavailable) return;
3186
+ for (const current of [...sources]) {
3187
+ if (!active(current)) continue;
3188
+ let consumed = current.consumed;
3189
+ for (const segment of current.segments) {
3190
+ if (context.currentTime < segment.start) break;
3191
+ const samples = Math.min(segment.length, Math.max(0, Math.floor((context.currentTime - segment.start) * current.header.sample_rate + 1e-6)));
3192
+ consumed = Math.max(consumed, segment.offset + samples);
3193
+ if (!current.started && samples > 0) {
3194
+ current.started = true;
3195
+ receipt(current, "started");
3196
+ position(current, "playing");
3197
+ current.lastPosition = current.consumed;
3198
+ }
3199
+ }
3200
+ current.consumed = consumed;
3201
+ while (current.segments.length && current.segments[0].offset + current.segments[0].length <= consumed) {
3202
+ const segment = current.segments.shift();
3203
+ if (segment) {
3204
+ segment.node.onended = null;
3205
+ segment.node.disconnect();
3206
+ }
3207
+ }
3208
+ if (current.started && current.eof && consumed === current.received && consumed > 0) {
3209
+ position(current, "finished");
3210
+ if (!active(current)) continue;
3211
+ receipt(current, "finished");
3212
+ sources.splice(sources.indexOf(current), 1);
3213
+ wake();
3214
+ } else {
3215
+ if (current.started && consumed > current.lastPosition) {
3216
+ current.lastPosition = consumed;
3217
+ position(current, "playing");
3218
+ }
3219
+ if (!active(current)) continue;
3220
+ if (reportCredit && consumed > current.lastReported) {
3221
+ current.lastReported = consumed;
3222
+ receipt(current, "progress");
3223
+ }
3224
+ wake();
3225
+ }
3226
+ }
3227
+ };
3228
+ const playbackState = () => {
3229
+ const next = context?.state !== "running";
3230
+ if (next !== blocked) {
3231
+ blocked = next;
3232
+ notify(() => options.onPlaybackBlocked?.(next ? startAudio : null));
3233
+ }
3234
+ tick();
3235
+ };
3236
+ const getContext = () => {
3237
+ if (!context) {
3238
+ context = new AudioContext({ sampleRate: correlatedAudioHeaderSchema.shape.sample_rate.value });
3239
+ context.onstatechange = playbackState;
3240
+ playbackState();
3241
+ }
3242
+ return context;
3243
+ };
3244
+ async function startAudio() {
3245
+ if (disposed || binding.unavailable) throw new Error("Correlated audio binding is no longer available");
3246
+ const ctx = getContext();
3247
+ await ctx.resume();
3248
+ playbackState();
3249
+ if (ctx.state !== "running") throw new Error("Correlated audio playback is blocked");
3250
+ }
3251
+ async function receive(reader, participant) {
3252
+ if (disposed || binding.unavailable || room.state !== "connected" || participant.identity !== options.agentIdentity) return;
3253
+ let value;
3254
+ try {
3255
+ value = JSON.parse(reader.info.attributes?.["rta.audio"] ?? "");
3256
+ } catch {
3257
+ return;
3258
+ }
3259
+ const parsed = correlatedAudioHeaderSchema.safeParse(value);
3260
+ if (!parsed.success || parsed.data.session_id !== options.sessionId || parsed.data.epoch < binding.epoch) return;
3261
+ const header = parsed.data;
3262
+ if (header.stream_id !== reader.info.id) return;
3263
+ if (header.epoch > binding.epoch) {
3264
+ binding.epoch = header.epoch;
3265
+ invalidate("cancelled");
3266
+ }
3267
+ const key = `${header.epoch}/${header.stream_id}`;
3268
+ if (seen.has(key)) return;
3269
+ if (sources.length >= 128 || seen.size >= 1e4) {
3270
+ invalidate("invalid_stream");
3271
+ return;
3272
+ }
3273
+ seen.add(key);
3274
+ const previousSource = sources.at(-1);
3275
+ const current = { header, abort: new AbortController(), segments: [], received: 0, consumed: 0, started: false, eof: false, lastReported: 0, lastPosition: -1, rpcBusy: false };
3276
+ sources.push(current);
3277
+ let carry;
3278
+ try {
3279
+ if (options.prepare) await options.prepare(header, current.abort.signal);
3280
+ if (!active(current) || current.abort.signal.aborted) return;
3281
+ const ctx = getContext();
3282
+ while (active(current) && (previousSource && active(previousSource) && !previousSource.eof || buffered() > CORRELATED_AUDIO_CONTRACT.maxBufferedSamples - CORRELATED_AUDIO_CONTRACT.maxChunkBytes / 2)) {
3283
+ await new Promise((resolve) => waiters.add(resolve));
3284
+ }
3285
+ if (!active(current)) return;
3286
+ for await (const chunk of reader.withAbortSignal(current.abort.signal)) {
3287
+ if (!active(current)) break;
3288
+ if (chunk.byteLength > CORRELATED_AUDIO_CONTRACT.maxChunkBytes) throw new Error("PCM chunk exceeds contract");
3289
+ const count = Math.floor((chunk.byteLength + (carry === void 0 ? 0 : 1)) / 2);
3290
+ if (current.received + count > CORRELATED_AUDIO_CONTRACT.maxSamples) throw new Error("PCM stream exceeds contract");
3291
+ while (active(current) && buffered() + count > CORRELATED_AUDIO_CONTRACT.maxBufferedSamples) {
3292
+ await new Promise((resolve) => waiters.add(resolve));
3293
+ }
3294
+ if (!active(current)) break;
3295
+ if (!count) {
3296
+ if (chunk.length) carry = chunk[0];
3297
+ continue;
3298
+ }
3299
+ const buffer = ctx.createBuffer(header.channels, count, header.sample_rate);
3300
+ const data = buffer.getChannelData(0);
3301
+ let cursor = 0;
3302
+ for (let i = 0; i < count; i++) {
3303
+ const low = carry ?? chunk[cursor++];
3304
+ carry = void 0;
3305
+ const raw = low | chunk[cursor++] << 8;
3306
+ data[i] = (raw >= 32768 ? raw - 65536 : raw) / 32768;
3307
+ }
3308
+ if (cursor < chunk.length) carry = chunk[cursor];
3309
+ const node = ctx.createBufferSource();
3310
+ node.buffer = buffer;
3311
+ node.connect(ctx.destination);
3312
+ const start = Math.max(ctx.currentTime, nextStart);
3313
+ current.segments.push({ node, start, offset: current.received, length: count });
3314
+ current.received += count;
3315
+ nextStart = start + count / header.sample_rate;
3316
+ node.onended = () => tick();
3317
+ node.start(start);
3318
+ while (active(current) && buffered() > CORRELATED_AUDIO_CONTRACT.maxBufferedSamples - CORRELATED_AUDIO_CONTRACT.maxChunkBytes / 2) {
3319
+ await new Promise((resolve) => waiters.add(resolve));
3320
+ }
3321
+ }
3322
+ if (!active(current)) return;
3323
+ if (carry !== void 0 || current.received === 0) throw new Error("Incomplete PCM stream");
3324
+ current.eof = true;
3325
+ wake();
3326
+ tick(false);
3327
+ } catch {
3328
+ if (active(current)) invalidate("invalid_stream");
3329
+ }
3330
+ }
3331
+ const cancel = async (data) => {
3332
+ if (disposed || binding.unavailable || room.state !== "connected" || data.callerIdentity !== options.agentIdentity) throw new Error("Unbound audio cancellation");
3333
+ const value = JSON.parse(data.payload);
3334
+ const frame2 = correlatedAudioCancelSchema.parse(value);
3335
+ if (frame2.session_id !== options.sessionId) throw new Error("Wrong audio session");
3336
+ if (frame2.epoch <= binding.epoch) return "";
3337
+ binding.epoch = frame2.epoch;
3338
+ invalidate("cancelled");
3339
+ return "";
3340
+ };
3341
+ const lost = () => invalidate("connection_lost");
3342
+ const changed = () => {
3343
+ if (["disconnected", "reconnecting", "signalReconnecting"].includes(room.state)) lost();
3344
+ };
3345
+ const participantLeft = (participant) => {
3346
+ if (participant.identity === options.agentIdentity) lost();
3347
+ };
3348
+ let registered = false;
3349
+ try {
3350
+ room.registerByteStreamHandler(CORRELATED_AUDIO_CONTRACT.topic, (reader, participant) => {
3351
+ void receive(reader, participant);
3352
+ });
3353
+ registered = true;
3354
+ room.localParticipant.registerRpcMethod(CORRELATED_AUDIO_CONTRACT.cancelMethod, cancel);
3355
+ } catch (error) {
3356
+ if (registered) room.unregisterByteStreamHandler(CORRELATED_AUDIO_CONTRACT.topic);
3357
+ attachments.delete(room);
3358
+ throw error;
3359
+ }
3360
+ room.on(RoomEvent.Reconnecting, lost);
3361
+ room.on(RoomEvent.SignalReconnecting, lost);
3362
+ room.on(RoomEvent.Disconnected, lost);
3363
+ room.on(RoomEvent.ConnectionStateChanged, changed);
3364
+ room.on(RoomEvent.ParticipantDisconnected, participantLeft);
3365
+ const timer = setInterval(() => tick(), 100);
3366
+ let frame = 0;
3367
+ const render = () => {
3368
+ if (disposed) return;
3369
+ tick(false);
3370
+ frame = requestAnimationFrame(render);
3371
+ };
3372
+ frame = requestAnimationFrame(render);
3373
+ return {
3374
+ startAudio,
3375
+ dispose() {
3376
+ if (closing) return closing;
3377
+ invalidate("disposed");
3378
+ disposed = true;
3379
+ clearInterval(timer);
3380
+ cancelAnimationFrame(frame);
3381
+ room.unregisterByteStreamHandler(CORRELATED_AUDIO_CONTRACT.topic);
3382
+ room.localParticipant.unregisterRpcMethod(CORRELATED_AUDIO_CONTRACT.cancelMethod);
3383
+ room.off(RoomEvent.Reconnecting, lost);
3384
+ room.off(RoomEvent.SignalReconnecting, lost);
3385
+ room.off(RoomEvent.Disconnected, lost);
3386
+ room.off(RoomEvent.ConnectionStateChanged, changed);
3387
+ room.off(RoomEvent.ParticipantDisconnected, participantLeft);
3388
+ attachments.delete(room);
3389
+ if (context) context.onstatechange = null;
3390
+ closing = context?.close().catch(() => {
3391
+ }) ?? Promise.resolve();
3392
+ return closing;
3393
+ }
3394
+ };
3395
+ }
3396
+
3397
+ // ../client/src/external-performance.ts
3398
+ var EXTERNAL_PERFORMANCE_KIND = externalPerformanceFrameSchema.shape.kind.value;
3399
+ var EXTERNAL_PERFORMANCE_TIMING = Object.freeze({
3400
+ receiverPlayout: "unsupported",
3401
+ audibleCheckpoint: externalPerformanceFrameSchema.shape.audible_checkpoint.value,
3402
+ audioMapping: externalPerformanceFrameSchema.shape.audio_mapping.value
3403
+ });
3404
+ var statusSchema = externalPerformanceFrameSchema.shape.status;
3405
+ var KNOWN_EXTERNAL_PERFORMANCE_STATUSES = Object.freeze([...statusSchema.options[0].options]);
3406
+ var knownStatuses = new Set(KNOWN_EXTERNAL_PERFORMANCE_STATUSES);
3407
+ function isObservable(frame) {
3408
+ return knownStatuses.has(frame.status);
3409
+ }
3410
+ var NEXT = {
3411
+ unseen: /* @__PURE__ */ new Set(["accepted", "dropped"]),
3412
+ accepted: /* @__PURE__ */ new Set(["audio_released", "cancelled"]),
3413
+ audio_released: /* @__PURE__ */ new Set(["cancelled"]),
3414
+ dropped: /* @__PURE__ */ new Set(),
3415
+ cancelled: /* @__PURE__ */ new Set(),
3416
+ retired: /* @__PURE__ */ new Set()
3417
+ };
3418
+ var LEDGER_LIMIT = 4096;
3419
+ function deliver(callback) {
3420
+ try {
3421
+ void Promise.resolve(callback()).catch(() => {
3422
+ });
3423
+ } catch {
3424
+ }
3425
+ }
3426
+ function createExternalPerformanceObserver(sessionId, callbacks) {
3427
+ const generations = /* @__PURE__ */ new Map();
3428
+ let state = "live";
3429
+ return {
3430
+ receive(frame) {
3431
+ if (state !== "live" || frame.session_id !== sessionId || !isObservable(frame)) return;
3432
+ const previous = generations.get(frame.generation_id) ?? "unseen";
3433
+ if (!NEXT[previous].has(frame.status)) return;
3434
+ if (previous === "unseen" && generations.size >= LEDGER_LIMIT) {
3435
+ const oldest = generations.keys().next();
3436
+ if (!oldest.done) generations.delete(oldest.value);
3437
+ }
3438
+ generations.set(frame.generation_id, frame.status);
3439
+ deliver(() => callbacks.onFrame(frame));
3440
+ },
3441
+ invalidate(reason) {
3442
+ if (state === "closed" || state === "paused" && reason === "connection_lost") return;
3443
+ state = reason === "connection_lost" ? "paused" : "closed";
3444
+ if (state === "closed") generations.clear();
3445
+ else for (const id of generations.keys()) generations.set(id, "retired");
3446
+ deliver(() => callbacks.onInvalidated(reason));
3447
+ },
3448
+ /** The call is connected to its agent again. Only a connection loss can be resumed. */
3449
+ resume() {
3450
+ if (state === "paused") state = "live";
3451
+ }
3452
+ };
3453
+ }
2991
3454
 
2992
3455
  // ../client/src/react/session-lifecycle.ts
2993
3456
  function phaseForReason(reason) {
@@ -3183,6 +3646,32 @@ function useSessionLifecycle(input) {
3183
3646
  const grantIdRef = useRef(null);
3184
3647
  grantIdRef.current = grantId;
3185
3648
  const retiredGrantRef = useRef(null);
3649
+ const performanceEnabled = input.externalPerformance !== void 0;
3650
+ const performanceCallbacksRef = useRef(input.externalPerformance);
3651
+ if (input.externalPerformance) performanceCallbacksRef.current = input.externalPerformance;
3652
+ const performanceRef = useRef(null);
3653
+ const endedPerformanceGrantRef = useRef(null);
3654
+ useEffect(() => {
3655
+ if (!grantId || !performanceEnabled || endedPerformanceGrantRef.current === grantId) return;
3656
+ const observer = createExternalPerformanceObserver(grantId, {
3657
+ onFrame: (frame) => performanceCallbacksRef.current?.onFrame(frame),
3658
+ onInvalidated: (reason) => {
3659
+ if (reason === "ended") endedPerformanceGrantRef.current = grantId;
3660
+ performanceCallbacksRef.current?.onInvalidated(reason);
3661
+ }
3662
+ });
3663
+ performanceRef.current = observer;
3664
+ return () => {
3665
+ observer.invalidate("binding_changed");
3666
+ if (performanceRef.current === observer) performanceRef.current = null;
3667
+ };
3668
+ }, [grantId, performanceEnabled]);
3669
+ const invalidateExternalPerformance = useCallback((reason) => {
3670
+ if (grantId === grantIdRef.current) performanceRef.current?.invalidate(reason);
3671
+ }, [grantId]);
3672
+ const resumeExternalPerformance = useCallback(() => {
3673
+ if (grantId === grantIdRef.current) performanceRef.current?.resume();
3674
+ }, [grantId]);
3186
3675
  const [recovery, setRecovery] = useState({ kind: "connected" });
3187
3676
  const recoveryRef = useRef(recovery);
3188
3677
  recoveryRef.current = recovery;
@@ -3228,6 +3717,7 @@ function useSessionLifecycle(input) {
3228
3717
  }
3229
3718
  }, []);
3230
3719
  const reset = useCallback(() => {
3720
+ invalidateExternalPerformance("binding_changed");
3231
3721
  clearTimer();
3232
3722
  connectedRef.current = false;
3233
3723
  manualReconnectPendingRef.current = false;
@@ -3242,7 +3732,7 @@ function useSessionLifecycle(input) {
3242
3732
  setConnected(false);
3243
3733
  setAgentPresentState(false);
3244
3734
  lastActivityRef.current = Date.now();
3245
- }, [clearTimer]);
3735
+ }, [clearTimer, invalidateExternalPerformance]);
3246
3736
  useEffect(() => {
3247
3737
  if (active) return;
3248
3738
  userEndedRef.current = false;
@@ -3276,6 +3766,7 @@ function useSessionLifecycle(input) {
3276
3766
  }, [grantId]);
3277
3767
  const endIdle = useCallback(() => {
3278
3768
  if (!connectedRef.current) return;
3769
+ invalidateExternalPerformance("ended");
3279
3770
  clearTimer();
3280
3771
  connectedRef.current = false;
3281
3772
  setConnected(false);
@@ -3283,7 +3774,7 @@ function useSessionLifecycle(input) {
3283
3774
  releaseRef.current("idle_timeout");
3284
3775
  setRecovery({ kind: "ended", reason: "idle" });
3285
3776
  leaveRoomRef.current?.();
3286
- }, [clearTimer]);
3777
+ }, [clearTimer, invalidateExternalPerformance]);
3287
3778
  const onConnected = useCallback(() => {
3288
3779
  if (grantId === null || grantIdRef.current !== grantId) return;
3289
3780
  if (userEndedRef.current || readinessFailureRef.current || retiredGrantRef.current !== null && retiredGrantRef.current === grantIdRef.current) return;
@@ -3301,6 +3792,7 @@ function useSessionLifecycle(input) {
3301
3792
  if (grantId === null || grantIdRef.current !== grantId) return;
3302
3793
  if (!active || userEndedRef.current || readinessFailureRef.current) return;
3303
3794
  const action = disconnectAction(reason, connectedRef.current);
3795
+ invalidateExternalPerformance(action === "end" ? "ended" : "connection_lost");
3304
3796
  if (action === "noop") return;
3305
3797
  if (action === "reset") {
3306
3798
  if (recoveryRef.current.kind === "refreshing" || recoveryRef.current.kind === "reconnecting") return;
@@ -3320,12 +3812,13 @@ function useSessionLifecycle(input) {
3320
3812
  refreshBaselineCapacityRef.current = null;
3321
3813
  setRecovery({ kind: "reconnecting", attempt: attemptRef.current });
3322
3814
  },
3323
- [active, reset, grantId]
3815
+ [active, reset, grantId, invalidateExternalPerformance]
3324
3816
  );
3325
3817
  const onConnectionError = useCallback(
3326
3818
  (error) => {
3327
3819
  if (grantId === null || grantIdRef.current !== grantId) return;
3328
3820
  if (!active || userEndedRef.current || readinessFailureRef.current || !isRecoverableConnectionError(error)) return;
3821
+ invalidateExternalPerformance("connection_lost");
3329
3822
  connectedRef.current = false;
3330
3823
  setConnected(false);
3331
3824
  setAgentPresentState(false);
@@ -3336,7 +3829,7 @@ function useSessionLifecycle(input) {
3336
3829
  return { kind: "reconnecting", attempt: attemptRef.current };
3337
3830
  });
3338
3831
  },
3339
- [active, grantId]
3832
+ [active, grantId, invalidateExternalPerformance]
3340
3833
  );
3341
3834
  useEffect(() => {
3342
3835
  if (capacity.kind === "error" && recovery.kind === "connected") manualReconnectPendingRef.current = false;
@@ -3356,6 +3849,9 @@ function useSessionLifecycle(input) {
3356
3849
  onConnected();
3357
3850
  return;
3358
3851
  }
3852
+ if (state === "disconnected" && connectedRef.current || state === "reconnecting" || state === "signalReconnecting") {
3853
+ invalidateExternalPerformance("connection_lost");
3854
+ }
3359
3855
  if (state === "reconnecting" || state === "signalReconnecting") {
3360
3856
  if (connectedRef.current) {
3361
3857
  setConnected(false);
@@ -3365,7 +3861,7 @@ function useSessionLifecycle(input) {
3365
3861
  }
3366
3862
  }
3367
3863
  },
3368
- [active, onConnected, grantId]
3864
+ [active, onConnected, grantId, invalidateExternalPerformance]
3369
3865
  );
3370
3866
  useEffect(() => {
3371
3867
  if (!active || userEnded || readinessFailed || !connected || !agentPresent || !grantState.grant) return;
@@ -3378,6 +3874,7 @@ function useSessionLifecycle(input) {
3378
3874
  const failReadiness = useCallback((code) => {
3379
3875
  if (userEndedRef.current || readinessFailureRef.current) return;
3380
3876
  readinessFailureRef.current = code;
3877
+ invalidateExternalPerformance("ended");
3381
3878
  clearTimer();
3382
3879
  connectedRef.current = false;
3383
3880
  manualReconnectPendingRef.current = false;
@@ -3390,7 +3887,7 @@ function useSessionLifecycle(input) {
3390
3887
  releaseRef.current("disconnected");
3391
3888
  setReadinessFailed(true);
3392
3889
  leaveRoomRef.current?.();
3393
- }, [clearTimer]);
3890
+ }, [clearTimer, invalidateExternalPerformance]);
3394
3891
  useEffect(() => {
3395
3892
  if (!active || userEnded || readinessFailed) return;
3396
3893
  if (recovery.kind === "ended" || recovery.kind === "failed") return;
@@ -3471,6 +3968,7 @@ function useSessionLifecycle(input) {
3471
3968
  const end = useCallback(() => {
3472
3969
  if (!active || userEndedRef.current || phaseRef.current.kind === "ended") return;
3473
3970
  userEndedRef.current = true;
3971
+ invalidateExternalPerformance("ended");
3474
3972
  clearTimer();
3475
3973
  connectedRef.current = false;
3476
3974
  manualReconnectPendingRef.current = false;
@@ -3481,7 +3979,7 @@ function useSessionLifecycle(input) {
3481
3979
  setRecovery({ kind: "ended", reason: "user" });
3482
3980
  clearRef.current();
3483
3981
  leaveRoomRef.current?.();
3484
- }, [active, clearTimer]);
3982
+ }, [active, clearTimer, invalidateExternalPerformance]);
3485
3983
  const reconnect = useCallback(() => {
3486
3984
  if (!active) return;
3487
3985
  if (readinessFailureRef.current) {
@@ -3551,16 +4049,22 @@ function useSessionLifecycle(input) {
3551
4049
  const lastBehaviorRef = useRef(null);
3552
4050
  const behaviorEnabled = Boolean(onBehaviorChange);
3553
4051
  const onLifecycleData = useMemo(() => {
3554
- if (!behaviorEnabled) return void 0;
4052
+ if (!behaviorEnabled && !performanceEnabled) return void 0;
3555
4053
  return (frame) => {
3556
4054
  const parsed = lifecycleServerFrameSchema.safeParse(frame);
3557
- if (!parsed.success || parsed.data.kind !== "behavior_state") return;
4055
+ if (!parsed.success) return;
4056
+ if (parsed.data.kind === "external_performance") {
4057
+ if (grantId === grantIdRef.current && parsed.data.session_id === grantId) performanceRef.current?.receive(parsed.data);
4058
+ return;
4059
+ }
4060
+ if (parsed.data.kind === "ended") invalidateExternalPerformance("ended");
4061
+ if (!behaviorEnabled || parsed.data.kind !== "behavior_state") return;
3558
4062
  const next = nextBehaviorSnapshot(lastBehaviorRef.current, parsed.data);
3559
4063
  if (!next) return;
3560
4064
  lastBehaviorRef.current = next;
3561
4065
  onBehaviorChangeRef.current?.(next);
3562
4066
  };
3563
- }, [behaviorEnabled]);
4067
+ }, [behaviorEnabled, performanceEnabled, grantId, invalidateExternalPerformance]);
3564
4068
  return useMemo(
3565
4069
  () => ({
3566
4070
  phase,
@@ -3579,7 +4083,9 @@ function useSessionLifecycle(input) {
3579
4083
  setAgentPresent,
3580
4084
  registerLeaveRoom,
3581
4085
  reset,
3582
- onLifecycleData
4086
+ onLifecycleData,
4087
+ invalidateExternalPerformance: performanceEnabled ? invalidateExternalPerformance : void 0,
4088
+ resumeExternalPerformance: performanceEnabled ? resumeExternalPerformance : void 0
3583
4089
  }),
3584
4090
  [
3585
4091
  phase,
@@ -3598,7 +4104,10 @@ function useSessionLifecycle(input) {
3598
4104
  setAgentPresent,
3599
4105
  registerLeaveRoom,
3600
4106
  reset,
3601
- onLifecycleData
4107
+ onLifecycleData,
4108
+ invalidateExternalPerformance,
4109
+ resumeExternalPerformance,
4110
+ performanceEnabled
3602
4111
  ]
3603
4112
  );
3604
4113
  }
@@ -3627,9 +4136,45 @@ function SessionLifecycleRoomBridge({
3627
4136
  const transcriptions = useTranscriptions();
3628
4137
  const room = useRoomContext();
3629
4138
  const roomMatchesGrant = lifecycle.grant === void 0 || lifecycle.grant !== null && room.name === lifecycle.grant.room_name;
4139
+ const correlatedRef = useRef(null);
4140
+ const correlatedObserver = useRef(lifecycle.correlatedAudio);
4141
+ correlatedObserver.current = lifecycle.correlatedAudio;
4142
+ const correlatedEnabled = lifecycle.correlatedAudio !== void 0;
4143
+ const correlatedSessionId = lifecycle.grant?.session_id;
4144
+ const startCorrelatedAudio = useCallback(async () => {
4145
+ const player = correlatedRef.current;
4146
+ if (!player) return false;
4147
+ try {
4148
+ await player.startAudio();
4149
+ return true;
4150
+ } catch {
4151
+ return false;
4152
+ }
4153
+ }, []);
4154
+ useEffect(() => {
4155
+ if (!correlatedEnabled || !correlatedSessionId || !roomMatchesGrant || !assistant.agent) return;
4156
+ const player = attachCorrelatedAudio(room, {
4157
+ sessionId: correlatedSessionId,
4158
+ agentIdentity: assistant.agent.identity,
4159
+ prepare: (header, signal) => correlatedObserver.current?.prepare?.(header, signal) ?? Promise.resolve(),
4160
+ onPosition: (position) => correlatedObserver.current?.onPosition(position),
4161
+ onInvalidated: (reason) => correlatedObserver.current?.onInvalidated(reason),
4162
+ onPlaybackBlocked: (unblock) => {
4163
+ setAudioPlayback?.(unblock ? "blocked" : "allowed");
4164
+ correlatedObserver.current?.onPlaybackBlocked?.(unblock);
4165
+ }
4166
+ });
4167
+ correlatedRef.current = player;
4168
+ return () => {
4169
+ correlatedRef.current = null;
4170
+ void player.dispose();
4171
+ setAudioPlayback?.("unknown");
4172
+ };
4173
+ }, [room, assistant.agent, correlatedEnabled, correlatedSessionId, roomMatchesGrant, setAudioPlayback]);
3630
4174
  useCallMedia(room, {
3631
4175
  setMicrophoneFacts,
3632
- setAudioPlayback,
4176
+ setAudioPlayback: correlatedEnabled ? void 0 : setAudioPlayback,
4177
+ startAudioOverride: correlatedEnabled ? startCorrelatedAudio : void 0,
3633
4178
  registerMediaControls,
3634
4179
  microphoneWanted: microphone ?? publishesMicrophoneByDefault(lifecycle.grant),
3635
4180
  connectionState,
@@ -3643,7 +4188,7 @@ function SessionLifecycleRoomBridge({
3643
4188
  if (!mediaModeEnabled) return;
3644
4189
  const agent = assistant.agent;
3645
4190
  let previous;
3646
- const deliver = (mode) => {
4191
+ const deliver2 = (mode) => {
3647
4192
  if (mode === previous) return;
3648
4193
  previous = mode;
3649
4194
  try {
@@ -3654,7 +4199,7 @@ function SessionLifecycleRoomBridge({
3654
4199
  };
3655
4200
  const read = () => {
3656
4201
  const value = agent?.attributes?.["rta.media_mode"];
3657
- deliver(value === "video" ? "avatar" : value === "voice" ? "voice" : null);
4202
+ deliver2(value === "video" ? "avatar" : value === "voice" ? "voice" : null);
3658
4203
  };
3659
4204
  const changed = (_attributes, participant) => {
3660
4205
  if (participant === agent) read();
@@ -3663,7 +4208,7 @@ function SessionLifecycleRoomBridge({
3663
4208
  read();
3664
4209
  return () => {
3665
4210
  room.off(RoomEvent.ParticipantAttributesChanged, changed);
3666
- deliver(null);
4211
+ deliver2(null);
3667
4212
  };
3668
4213
  }, [room, assistant.agent, mediaModeEnabled, mediaModeSessionId]);
3669
4214
  const { send } = useChat();
@@ -3710,7 +4255,7 @@ function SessionLifecycleRoomBridge({
3710
4255
  let closed = false;
3711
4256
  let pending = false;
3712
4257
  let previous;
3713
- const deliver = (details) => {
4258
+ const deliver2 = (details) => {
3714
4259
  connectionHistoryRef.current?.enqueue(details);
3715
4260
  try {
3716
4261
  if (onConnectionDetailsChange) void Promise.resolve(callback?.(details)).catch(() => {
@@ -3737,7 +4282,7 @@ function SessionLifecycleRoomBridge({
3737
4282
  const last = previous;
3738
4283
  if (last?.connectionState === next.connectionState && last.localQuality === next.localQuality && ["audio", "video"].every((kind) => last[kind]?.publisherQuality === next[kind]?.publisherQuality && last[kind]?.streamState === next[kind]?.streamState)) return;
3739
4284
  previous = next;
3740
- deliver(next);
4285
+ deliver2(next);
3741
4286
  });
3742
4287
  };
3743
4288
  const onQuality = (_quality, participant) => {
@@ -3767,7 +4312,7 @@ function SessionLifecycleRoomBridge({
3767
4312
  room.off(RoomEvent.TrackStreamStateChanged, onPublication);
3768
4313
  room.off(RoomEvent.TrackSubscribed, onTrack);
3769
4314
  room.off(RoomEvent.TrackUnsubscribed, onTrack);
3770
- deliver(null);
4315
+ deliver2(null);
3771
4316
  };
3772
4317
  }, [room, detailsSessionId, detailsEnabled]);
3773
4318
  useEffect(() => {
@@ -3804,18 +4349,56 @@ function SessionLifecycleRoomBridge({
3804
4349
  useEffect(() => {
3805
4350
  if (!onLifecycleData) return;
3806
4351
  const decoder = new TextDecoder();
3807
- const handler = (payload, _participant, _kind, topic) => {
4352
+ const handler = (payload, participant, _kind, topic) => {
3808
4353
  if (topic !== RTA_LIFECYCLE_TOPIC) return;
3809
4354
  try {
3810
- onLifecycleData(JSON.parse(decoder.decode(payload)));
4355
+ const frame = JSON.parse(decoder.decode(payload));
4356
+ if (typeof frame === "object" && frame !== null && "kind" in frame && (frame.kind === EXTERNAL_PERFORMANCE_KIND || frame.kind === "ended") && (!assistant.agent || participant !== assistant.agent || !roomMatchesGrant || room.state !== "connected")) return;
4357
+ if (typeof frame === "object" && frame !== null && "kind" in frame && frame.kind === "ended" && lifecycleServerFrameSchema.safeParse(frame).success) {
4358
+ void correlatedRef.current?.dispose();
4359
+ correlatedRef.current = null;
4360
+ }
4361
+ onLifecycleData(frame);
3811
4362
  } catch {
3812
4363
  }
3813
4364
  };
4365
+ const invalidate = () => lifecycle.invalidateExternalPerformance?.("connection_lost");
4366
+ const changed = () => {
4367
+ if (room.state === "disconnected" || room.state === "reconnecting" || room.state === "signalReconnecting") invalidate();
4368
+ };
3814
4369
  room.on(RoomEvent.DataReceived, handler);
4370
+ if (lifecycle.invalidateExternalPerformance) {
4371
+ room.on(RoomEvent.Reconnecting, invalidate);
4372
+ room.on(RoomEvent.SignalReconnecting, invalidate);
4373
+ room.on(RoomEvent.Disconnected, invalidate);
4374
+ room.on(RoomEvent.ConnectionStateChanged, changed);
4375
+ }
3815
4376
  return () => {
3816
4377
  room.off(RoomEvent.DataReceived, handler);
4378
+ room.off(RoomEvent.Reconnecting, invalidate);
4379
+ room.off(RoomEvent.SignalReconnecting, invalidate);
4380
+ room.off(RoomEvent.Disconnected, invalidate);
4381
+ room.off(RoomEvent.ConnectionStateChanged, changed);
4382
+ };
4383
+ }, [room, onLifecycleData, assistant.agent, roomMatchesGrant, lifecycle.invalidateExternalPerformance]);
4384
+ useEffect(() => {
4385
+ if (!assistant.agent) return;
4386
+ return () => lifecycle.invalidateExternalPerformance?.("connection_lost");
4387
+ }, [assistant.agent, lifecycle.invalidateExternalPerformance]);
4388
+ const resumeExternalPerformance = lifecycle.resumeExternalPerformance;
4389
+ useEffect(() => {
4390
+ if (!resumeExternalPerformance) return;
4391
+ const resumeIfBound = () => {
4392
+ if (assistant.agent && roomMatchesGrant && room.state === "connected") resumeExternalPerformance();
4393
+ };
4394
+ resumeIfBound();
4395
+ room.on(RoomEvent.Reconnected, resumeIfBound);
4396
+ room.on(RoomEvent.ConnectionStateChanged, resumeIfBound);
4397
+ return () => {
4398
+ room.off(RoomEvent.Reconnected, resumeIfBound);
4399
+ room.off(RoomEvent.ConnectionStateChanged, resumeIfBound);
3817
4400
  };
3818
- }, [room, onLifecycleData]);
4401
+ }, [room, assistant.agent, roomMatchesGrant, resumeExternalPerformance]);
3819
4402
  useEffect(() => {
3820
4403
  if (!registerDataPublisher) return;
3821
4404
  const encoder = new TextEncoder();
@@ -3951,6 +4534,7 @@ function useRealtimeSession(input) {
3951
4534
  onBehaviorChange,
3952
4535
  onClipResult,
3953
4536
  onVoiceError,
4537
+ correlatedAudio,
3954
4538
  // everything else (client/active/idleSeconds/idleWarnLeadSeconds/reconnect/requestOptions…)
3955
4539
  // flows straight into the inner SSOT hook.
3956
4540
  ...inner
@@ -4023,6 +4607,7 @@ function useRealtimeSession(input) {
4023
4607
  return `t-${Date.now()}-${++turnSequenceRef.current}`;
4024
4608
  }, []);
4025
4609
  const onLifecycleData = useCallback((frame) => {
4610
+ lifecycle.onLifecycleData?.(frame);
4026
4611
  const parsed = lifecycleServerFrameSchema.safeParse(frame);
4027
4612
  if (!parsed.success) return;
4028
4613
  const f = parsed.data;
@@ -4082,7 +4667,7 @@ function useRealtimeSession(input) {
4082
4667
  break;
4083
4668
  }
4084
4669
  }
4085
- }, []);
4670
+ }, [lifecycle.onLifecycleData]);
4086
4671
  const registerDataPublisher = useCallback((publish) => {
4087
4672
  dataPublisherRef.current = publish;
4088
4673
  }, []);
@@ -4348,6 +4933,7 @@ function useRealtimeSession(input) {
4348
4933
  // stayConnected/reconnect/markActivity/the sinks) → a true drop-in superset, then
4349
4934
  // add the realtime-session surface. `reset` + `lifecycle` come AFTER so ours win.
4350
4935
  ...lifecycle,
4936
+ correlatedAudio,
4351
4937
  turn,
4352
4938
  clocks,
4353
4939
  endsAt,
@@ -4382,6 +4968,7 @@ function useRealtimeSession(input) {
4382
4968
  }),
4383
4969
  [
4384
4970
  lifecycle,
4971
+ correlatedAudio,
4385
4972
  turn,
4386
4973
  clocks,
4387
4974
  endsAt,
@@ -4482,7 +5069,9 @@ function useAvatarCall(props) {
4482
5069
  onIdleWarning: props.onQuiet ? ({ secondsLeft }) => props.onQuiet?.({ secondsLeft }) : void 0,
4483
5070
  onCreditsLow: props.onLowBalance ? ({ secondsLeft }) => props.onLowBalance?.({ secondsLeft }) : void 0,
4484
5071
  onEnded: props.onEnded ? ({ reason }) => props.onEnded?.({ reason }) : void 0,
4485
- onVoiceError: props.onVoiceError ? (error) => props.onVoiceError?.(error) : void 0
5072
+ onVoiceError: props.onVoiceError ? (error) => props.onVoiceError?.(error) : void 0,
5073
+ externalPerformance: props.externalPerformance,
5074
+ correlatedAudio: props.correlatedAudio
4486
5075
  });
4487
5076
  sessionRef.current = session;
4488
5077
  const call = handleFor(session);
@@ -4520,6 +5109,7 @@ function useAvatarCall(props) {
4520
5109
  RealtimeAvatarLiveKitRoom,
4521
5110
  {
4522
5111
  grant: session.grant,
5112
+ renderRoomAudio: props.correlatedAudio === void 0,
4523
5113
  // LiveKit captures the microphone on connect from this value, so a mute chosen while
4524
5114
  // waiting is honoured rather than overridden when the call goes live.
4525
5115
  audio: props.listen !== false && !session.microphoneMuted,
@@ -4919,4 +5509,4 @@ function RecordingPlayer({ assets, className }) {
4919
5509
  ] });
4920
5510
  }
4921
5511
 
4922
- export { AdaptivePlayoutController, AvatarCall, AvatarVideoSurface, DEFAULT_APPROACHING_END_LEAD_SECONDS, DEFAULT_AVATAR_PLAYOUT_DELAY_SECONDS, DEFAULT_AVATAR_UNSTABLE_STALL_MS, DEFAULT_CREDITS_LOW_LEAD_SECONDS, DEFAULT_FISH_TTS_MODEL, DEFAULT_GOVERNOR_CONFIG, DEFAULT_GRACE_CEILING_SECONDS, DEFAULT_GRACE_WINDOW_LEAD_SECONDS, DEFAULT_IDLE_SECONDS, DEFAULT_IDLE_WARN_LEAD_SECONDS, DEFAULT_READY_TIMEOUT_SECONDS, DEFAULT_TURN_TIMEOUT_SECONDS, FISH_DELIVERY_MODELS, MAX_RECONNECT_ATTEMPTS, MAX_SESSION_INSTRUCTIONS_CHARS, MIC_LEASE_ENDED_TIMEOUT_MS, NETWORK_QUALITY_POLICY, RECONNECT_BACKOFF_MS, RealtimeAvatarCapacityError, RealtimeAvatarLiveKitRoom, RecordingPlayer, SessionLifecycleRoomBridge, capacityErrorFromBusy, capacityStateFromGrant, createProxyClient, fishTtsModelSchema, knownBehaviorStates, mapTurnState, readInboundRtp, sessionBehaviorSchema, sessionClipSchema, shouldReplayPendingTurn, splitCallTranscript, useAvatarAdaptivePlayoutDelay, useAvatarCall, useAvatarCamera, useAvatarPlayoutDelay, useAvatarQualityGovernor, useCallTranscript, useCharacterTools, useLiveKitAvatarGrant, useMicLease, useRealtimeSession, useReleaseMicLeaseOnTrackEnded, useSessionLifecycle };
5512
+ export { AdaptivePlayoutController, AvatarCall, AvatarVideoSurface, DEFAULT_APPROACHING_END_LEAD_SECONDS, DEFAULT_AVATAR_PLAYOUT_DELAY_SECONDS, DEFAULT_AVATAR_UNSTABLE_STALL_MS, DEFAULT_CREDITS_LOW_LEAD_SECONDS, DEFAULT_FISH_TTS_MODEL, DEFAULT_GOVERNOR_CONFIG, DEFAULT_GRACE_CEILING_SECONDS, DEFAULT_GRACE_WINDOW_LEAD_SECONDS, DEFAULT_IDLE_SECONDS, DEFAULT_IDLE_WARN_LEAD_SECONDS, DEFAULT_READY_TIMEOUT_SECONDS, DEFAULT_TURN_TIMEOUT_SECONDS, EXTERNAL_PERFORMANCE_TIMING, FISH_DELIVERY_MODELS, KNOWN_EXTERNAL_PERFORMANCE_STATUSES, MAX_RECONNECT_ATTEMPTS, MAX_SESSION_INSTRUCTIONS_CHARS, MIC_LEASE_ENDED_TIMEOUT_MS, NETWORK_QUALITY_POLICY, RECONNECT_BACKOFF_MS, RealtimeAvatarCapacityError, RealtimeAvatarLiveKitRoom, RecordingPlayer, SessionLifecycleRoomBridge, capacityErrorFromBusy, capacityStateFromGrant, createProxyClient, fishTtsModelSchema, knownBehaviorStates, mapTurnState, readInboundRtp, sessionBehaviorSchema, sessionClipSchema, shouldReplayPendingTurn, splitCallTranscript, useAvatarAdaptivePlayoutDelay, useAvatarCall, useAvatarCamera, useAvatarPlayoutDelay, useAvatarQualityGovernor, useCallTranscript, useCharacterTools, useLiveKitAvatarGrant, useMicLease, useRealtimeSession, useReleaseMicLeaseOnTrackEnded, useSessionLifecycle };