realtime-avatar 0.33.0 → 0.35.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +56 -1
- package/dist/browser.d.ts +4 -89
- package/dist/browser.js +362 -1
- package/dist/chunk-6KQS7CTR.js +1 -0
- package/dist/{chunk-DM5TWRC4.js → chunk-ARJEAQQT.js} +1 -13
- package/dist/{chunk-VLYUMP3I.js → chunk-WHYDSORD.js} +426 -170
- package/dist/express.d.ts +2 -2
- package/dist/express.js +2 -2
- package/dist/hono.d.ts +2 -2
- package/dist/hono.js +2 -2
- package/dist/index.d.ts +338 -3
- package/dist/index.js +2 -1
- package/dist/nextjs.d.ts +2 -2
- package/dist/nextjs.js +2 -2
- package/dist/{playout-delay-CtRZdtsj.d.ts → playout-delay-BzoM1BMN.d.ts} +135 -1
- package/dist/{proxy-client-DJhYiiKX.d.ts → proxy-client-CiTOjkg8.d.ts} +67 -3
- package/dist/react-native.d.ts +3 -3
- package/dist/react-native.js +623 -36
- package/dist/react.d.ts +12 -22
- package/dist/react.js +627 -37
- package/dist/server.d.ts +2 -2
- package/dist/server.js +2 -1
- package/dist/tanstack-start.d.ts +2 -2
- package/dist/tanstack-start.js +2 -2
- package/dist/{types-DygYBvSv.d.ts → types-CUvcV0Wq.d.ts} +1 -1
- package/dist/{types-VoejsHij.d.ts → types-CpnLuiJU.d.ts} +220 -18
- package/package.json +3 -3
package/dist/react.js
CHANGED
|
@@ -1461,7 +1461,7 @@ function stepNetworkQuality(previous, sample, nowMs, options) {
|
|
|
1461
1461
|
}
|
|
1462
1462
|
|
|
1463
1463
|
// ../client/src/react/network-quality-binding.ts
|
|
1464
|
-
function bindNetworkQuality(publication, inhibited,
|
|
1464
|
+
function bindNetworkQuality(publication, inhibited, notify2, tickMs) {
|
|
1465
1465
|
let disposed = false;
|
|
1466
1466
|
let running = false;
|
|
1467
1467
|
let state = initNetworkQuality(performance.now());
|
|
@@ -1475,7 +1475,7 @@ function bindNetworkQuality(publication, inhibited, notify, tickMs) {
|
|
|
1475
1475
|
};
|
|
1476
1476
|
const emit = (status) => {
|
|
1477
1477
|
try {
|
|
1478
|
-
|
|
1478
|
+
notify2(status);
|
|
1479
1479
|
} catch {
|
|
1480
1480
|
}
|
|
1481
1481
|
};
|
|
@@ -2359,15 +2359,116 @@ var schema5 = z.object({
|
|
|
2359
2359
|
"request_id": schema22.optional()
|
|
2360
2360
|
});
|
|
2361
2361
|
var voiceErrorFrameSchema = schema5;
|
|
2362
|
-
var schema04 = z.
|
|
2363
|
-
var schema13 = z.
|
|
2364
|
-
var schema23 = z.
|
|
2365
|
-
var schema33 = z.
|
|
2366
|
-
var schema43 = z.
|
|
2367
|
-
var schema52 = z.
|
|
2368
|
-
var
|
|
2362
|
+
var schema04 = z.string().min(1).max(200);
|
|
2363
|
+
var schema13 = z.literal(1);
|
|
2364
|
+
var schema23 = z.string().min(1).max(200);
|
|
2365
|
+
var schema33 = z.int().min(1).max(9007199254740991);
|
|
2366
|
+
var schema43 = z.literal(24e3);
|
|
2367
|
+
var schema52 = z.literal("pcm_s16le");
|
|
2368
|
+
var schema6 = z.object({
|
|
2369
|
+
"session_id": schema04,
|
|
2370
|
+
"generation_id": schema04,
|
|
2371
|
+
"version": schema13,
|
|
2372
|
+
"utterance_id": schema23,
|
|
2373
|
+
"stream_id": schema04,
|
|
2374
|
+
"epoch": schema33,
|
|
2375
|
+
"sample_rate": schema43,
|
|
2376
|
+
"channels": schema13,
|
|
2377
|
+
"encoding": schema52
|
|
2378
|
+
});
|
|
2379
|
+
var schema7 = z.enum(["started", "progress", "finished", "cancelled"]);
|
|
2380
|
+
var schema8 = z.int().min(0).max(144e4);
|
|
2381
|
+
z.strictObject({
|
|
2382
|
+
"session_id": schema04,
|
|
2383
|
+
"generation_id": schema04,
|
|
2384
|
+
"utterance_id": schema23,
|
|
2385
|
+
"stream_id": schema04,
|
|
2386
|
+
"epoch": schema33,
|
|
2387
|
+
"status": schema7,
|
|
2388
|
+
"samples": schema8
|
|
2389
|
+
});
|
|
2390
|
+
var schema10 = z.object({
|
|
2391
|
+
"version": schema13,
|
|
2392
|
+
"session_id": schema04,
|
|
2393
|
+
"epoch": schema33
|
|
2394
|
+
});
|
|
2395
|
+
var schema11 = z.literal("external_llm_error");
|
|
2396
|
+
var schema122 = z.enum(["external_llm_refused", "external_llm_failed"]);
|
|
2397
|
+
var schema132 = z.string();
|
|
2398
|
+
var schema14 = z.literal(true);
|
|
2399
|
+
var schema15 = z.object({
|
|
2400
|
+
"kind": schema11,
|
|
2401
|
+
"code": schema122,
|
|
2402
|
+
"message": schema132,
|
|
2403
|
+
"fatal": schema14
|
|
2404
|
+
});
|
|
2405
|
+
var correlatedAudioHeaderSchema = schema6;
|
|
2406
|
+
var correlatedAudioCancelSchema = schema10;
|
|
2407
|
+
var externalLlmErrorFrameSchema = schema15;
|
|
2408
|
+
var CORRELATED_AUDIO_CONTRACT = {
|
|
2409
|
+
"topic": "rta.audio.v1",
|
|
2410
|
+
"receiptMethod": "rta.audio.receipt.v1",
|
|
2411
|
+
"cancelMethod": "rta.audio.cancel.v1",
|
|
2412
|
+
"maxSamples": 144e4,
|
|
2413
|
+
"maxChunkBytes": 48e3,
|
|
2414
|
+
"maxBufferedSamples": 48e3
|
|
2415
|
+
};
|
|
2416
|
+
var schema05 = z.string().min(1).max(200);
|
|
2417
|
+
var schema16 = z.literal(1);
|
|
2418
|
+
var schema24 = z.enum(["respond", "recovery", "initiative"]);
|
|
2419
|
+
var schema34 = z.string().min(1).max(32);
|
|
2420
|
+
var schema44 = z.union([schema24, schema34]);
|
|
2421
|
+
var schema53 = z.enum(["draft", "accepted"]);
|
|
2422
|
+
var schema62 = z.literal("speech_segments_v1");
|
|
2423
|
+
var schema72 = z.literal("external");
|
|
2424
|
+
var schema82 = z.object({
|
|
2425
|
+
"renderer": schema72
|
|
2426
|
+
});
|
|
2427
|
+
z.object({
|
|
2428
|
+
"session_id": schema05,
|
|
2429
|
+
"generation_id": schema05,
|
|
2430
|
+
"version": schema16,
|
|
2431
|
+
"purpose": schema44,
|
|
2432
|
+
"admission": schema53,
|
|
2433
|
+
"text_output": schema62.optional(),
|
|
2434
|
+
"visible_character": schema82.optional()
|
|
2435
|
+
});
|
|
2436
|
+
var schema102 = z.string().min(1).max(200);
|
|
2437
|
+
z.object({
|
|
2438
|
+
"generation_id": schema05,
|
|
2439
|
+
"utterance_id": schema102
|
|
2440
|
+
});
|
|
2441
|
+
var schema123 = z.literal("external_performance");
|
|
2442
|
+
var schema133 = z.enum(["accepted", "dropped", "cancelled", "audio_released"]);
|
|
2443
|
+
var schema142 = z.union([schema133, schema34]);
|
|
2444
|
+
var schema152 = z.literal("unavailable");
|
|
2445
|
+
var schema162 = z.literal("unavailable");
|
|
2446
|
+
var schema17 = z.object({
|
|
2447
|
+
"session_id": schema05,
|
|
2448
|
+
"generation_id": schema05,
|
|
2449
|
+
"kind": schema123,
|
|
2450
|
+
"version": schema16,
|
|
2451
|
+
"status": schema142,
|
|
2452
|
+
"audible_checkpoint": schema152,
|
|
2453
|
+
"audio_mapping": schema162
|
|
2454
|
+
});
|
|
2455
|
+
var schema18 = z.string().regex(new RegExp("^(?:[^\\uD800-\\uDFFF]|[\\uD800-\\uDBFF][\\uDC00-\\uDFFF])*$")).min(1).max(5e3);
|
|
2456
|
+
z.strictObject({
|
|
2457
|
+
"version": schema16,
|
|
2458
|
+
"generation_id": schema05,
|
|
2459
|
+
"utterance_id": schema102,
|
|
2460
|
+
"text": schema18
|
|
2461
|
+
});
|
|
2462
|
+
var externalPerformanceFrameSchema = schema17;
|
|
2463
|
+
var schema06 = z.enum(["s1", "s2-pro", "s2.1-pro", "drama-3"]);
|
|
2464
|
+
var schema110 = z.enum(["speech-1.6", "speech-1.5", "s1-mini"]);
|
|
2465
|
+
var schema25 = z.union([schema06, schema110]);
|
|
2466
|
+
var schema35 = z.enum(["whisper"]);
|
|
2467
|
+
var schema45 = z.null();
|
|
2468
|
+
var schema54 = z.union([schema35, schema45]);
|
|
2469
|
+
var fishTtsModelSchema = schema25;
|
|
2369
2470
|
var DEFAULT_FISH_TTS_MODEL = "s2.1-pro";
|
|
2370
|
-
var fishDeliverySchema =
|
|
2471
|
+
var fishDeliverySchema = schema54;
|
|
2371
2472
|
var FISH_DELIVERY_MODELS = { "whisper": ["drama-3"] };
|
|
2372
2473
|
|
|
2373
2474
|
// ../client/src/wire.ts
|
|
@@ -2683,7 +2784,9 @@ var lifecycleServerFrameSchema = z.discriminatedUnion("kind", [
|
|
|
2683
2784
|
endedFrameSchema,
|
|
2684
2785
|
behaviorStateFrameSchema,
|
|
2685
2786
|
clipAckFrameSchema,
|
|
2686
|
-
voiceErrorFrameSchema2
|
|
2787
|
+
voiceErrorFrameSchema2,
|
|
2788
|
+
externalPerformanceFrameSchema,
|
|
2789
|
+
externalLlmErrorFrameSchema
|
|
2687
2790
|
]);
|
|
2688
2791
|
var liveKitCapacitySnapshotSchema = z.object({
|
|
2689
2792
|
// Placement identity + per-worker session ceiling. These are always present on the
|
|
@@ -2965,6 +3068,7 @@ function useCallMedia(room, options) {
|
|
|
2965
3068
|
}, [room, setAudioPlayback, connecting]);
|
|
2966
3069
|
const controls = useMemo(() => ({
|
|
2967
3070
|
startAudio: async () => {
|
|
3071
|
+
if (options.startAudioOverride) return options.startAudioOverride();
|
|
2968
3072
|
try {
|
|
2969
3073
|
await room.startAudio();
|
|
2970
3074
|
} catch {
|
|
@@ -2981,13 +3085,372 @@ function useCallMedia(room, options) {
|
|
|
2981
3085
|
setVersion((value) => value + 1);
|
|
2982
3086
|
}
|
|
2983
3087
|
}
|
|
2984
|
-
}), [room]);
|
|
3088
|
+
}), [room, options.startAudioOverride]);
|
|
2985
3089
|
useEffect(() => {
|
|
2986
3090
|
if (!registerMediaControls) return;
|
|
2987
3091
|
registerMediaControls(controls);
|
|
2988
3092
|
return () => registerMediaControls(null);
|
|
2989
3093
|
}, [controls, registerMediaControls]);
|
|
2990
3094
|
}
|
|
3095
|
+
var attachments = /* @__PURE__ */ new WeakSet();
|
|
3096
|
+
var bindings = /* @__PURE__ */ new WeakMap();
|
|
3097
|
+
function notify(callback) {
|
|
3098
|
+
try {
|
|
3099
|
+
void Promise.resolve(callback()).catch(() => {
|
|
3100
|
+
});
|
|
3101
|
+
} catch {
|
|
3102
|
+
}
|
|
3103
|
+
}
|
|
3104
|
+
function attachCorrelatedAudio(room, options) {
|
|
3105
|
+
if (attachments.has(room)) throw new Error("Correlated audio is already attached to this room");
|
|
3106
|
+
if (!options.sessionId || !options.agentIdentity) throw new Error("Correlated audio requires the held session and bound agent");
|
|
3107
|
+
attachments.add(room);
|
|
3108
|
+
let context;
|
|
3109
|
+
let disposed = false;
|
|
3110
|
+
const previous = bindings.get(room);
|
|
3111
|
+
const binding = previous?.sessionId === options.sessionId ? previous : { sessionId: options.sessionId, epoch: 1, unavailable: false, seen: /* @__PURE__ */ new Set() };
|
|
3112
|
+
bindings.set(room, binding);
|
|
3113
|
+
const sources = [];
|
|
3114
|
+
const active = (source) => sources.includes(source);
|
|
3115
|
+
const buffered = () => sources.reduce((sum, source) => sum + source.received - source.consumed, 0);
|
|
3116
|
+
let nextStart = 0;
|
|
3117
|
+
let blocked;
|
|
3118
|
+
let closing;
|
|
3119
|
+
const seen = binding.seen;
|
|
3120
|
+
const waiters = /* @__PURE__ */ new Set();
|
|
3121
|
+
const wake = () => {
|
|
3122
|
+
for (const resolve of waiters) resolve();
|
|
3123
|
+
waiters.clear();
|
|
3124
|
+
};
|
|
3125
|
+
const identity = (header) => ({
|
|
3126
|
+
session_id: header.session_id,
|
|
3127
|
+
generation_id: header.generation_id,
|
|
3128
|
+
utterance_id: header.utterance_id,
|
|
3129
|
+
stream_id: header.stream_id,
|
|
3130
|
+
epoch: header.epoch
|
|
3131
|
+
});
|
|
3132
|
+
const position = (current, state) => {
|
|
3133
|
+
notify(() => options.onPosition({ ...identity(current.header), samples: current.consumed, sampleRate: current.header.sample_rate, state }));
|
|
3134
|
+
};
|
|
3135
|
+
const receipt = (current, status) => {
|
|
3136
|
+
const payload = { ...identity(current.header), status, samples: current.consumed };
|
|
3137
|
+
if (current.rpcBusy) {
|
|
3138
|
+
current.pendingReceipt = payload;
|
|
3139
|
+
return;
|
|
3140
|
+
}
|
|
3141
|
+
const send = async (next) => {
|
|
3142
|
+
current.rpcBusy = true;
|
|
3143
|
+
try {
|
|
3144
|
+
await room.localParticipant.performRpc({
|
|
3145
|
+
destinationIdentity: options.agentIdentity,
|
|
3146
|
+
method: CORRELATED_AUDIO_CONTRACT.receiptMethod,
|
|
3147
|
+
payload: JSON.stringify(next),
|
|
3148
|
+
responseTimeout: 2e3
|
|
3149
|
+
});
|
|
3150
|
+
} catch {
|
|
3151
|
+
if (!disposed && next.epoch === binding.epoch) invalidate("connection_lost");
|
|
3152
|
+
} finally {
|
|
3153
|
+
current.rpcBusy = false;
|
|
3154
|
+
const pending = current.pendingReceipt;
|
|
3155
|
+
current.pendingReceipt = void 0;
|
|
3156
|
+
if (pending && !disposed && pending.epoch === binding.epoch && !binding.unavailable) void send(pending);
|
|
3157
|
+
}
|
|
3158
|
+
};
|
|
3159
|
+
void send(payload);
|
|
3160
|
+
};
|
|
3161
|
+
const clear = () => {
|
|
3162
|
+
for (const current of sources.splice(0)) {
|
|
3163
|
+
current.abort.abort();
|
|
3164
|
+
for (const segment of current.segments) {
|
|
3165
|
+
segment.node.onended = null;
|
|
3166
|
+
segment.node.stop();
|
|
3167
|
+
segment.node.disconnect();
|
|
3168
|
+
}
|
|
3169
|
+
current.segments.length = 0;
|
|
3170
|
+
}
|
|
3171
|
+
nextStart = context?.currentTime ?? 0;
|
|
3172
|
+
wake();
|
|
3173
|
+
};
|
|
3174
|
+
const invalidate = (reason) => {
|
|
3175
|
+
if (disposed) return;
|
|
3176
|
+
if (reason === "connection_lost") {
|
|
3177
|
+
if (binding.unavailable) return;
|
|
3178
|
+
binding.unavailable = true;
|
|
3179
|
+
}
|
|
3180
|
+
if (reason === "invalid_stream" || reason === "disposed") for (const current of sources) receipt(current, "cancelled");
|
|
3181
|
+
clear();
|
|
3182
|
+
notify(() => options.onInvalidated(reason));
|
|
3183
|
+
};
|
|
3184
|
+
const tick = (reportCredit = true) => {
|
|
3185
|
+
if (!context || context.state !== "running" || disposed || binding.unavailable) return;
|
|
3186
|
+
for (const current of [...sources]) {
|
|
3187
|
+
if (!active(current)) continue;
|
|
3188
|
+
let consumed = current.consumed;
|
|
3189
|
+
for (const segment of current.segments) {
|
|
3190
|
+
if (context.currentTime < segment.start) break;
|
|
3191
|
+
const samples = Math.min(segment.length, Math.max(0, Math.floor((context.currentTime - segment.start) * current.header.sample_rate + 1e-6)));
|
|
3192
|
+
consumed = Math.max(consumed, segment.offset + samples);
|
|
3193
|
+
if (!current.started && samples > 0) {
|
|
3194
|
+
current.started = true;
|
|
3195
|
+
receipt(current, "started");
|
|
3196
|
+
position(current, "playing");
|
|
3197
|
+
current.lastPosition = current.consumed;
|
|
3198
|
+
}
|
|
3199
|
+
}
|
|
3200
|
+
current.consumed = consumed;
|
|
3201
|
+
while (current.segments.length && current.segments[0].offset + current.segments[0].length <= consumed) {
|
|
3202
|
+
const segment = current.segments.shift();
|
|
3203
|
+
if (segment) {
|
|
3204
|
+
segment.node.onended = null;
|
|
3205
|
+
segment.node.disconnect();
|
|
3206
|
+
}
|
|
3207
|
+
}
|
|
3208
|
+
if (current.started && current.eof && consumed === current.received && consumed > 0) {
|
|
3209
|
+
position(current, "finished");
|
|
3210
|
+
if (!active(current)) continue;
|
|
3211
|
+
receipt(current, "finished");
|
|
3212
|
+
sources.splice(sources.indexOf(current), 1);
|
|
3213
|
+
wake();
|
|
3214
|
+
} else {
|
|
3215
|
+
if (current.started && consumed > current.lastPosition) {
|
|
3216
|
+
current.lastPosition = consumed;
|
|
3217
|
+
position(current, "playing");
|
|
3218
|
+
}
|
|
3219
|
+
if (!active(current)) continue;
|
|
3220
|
+
if (reportCredit && consumed > current.lastReported) {
|
|
3221
|
+
current.lastReported = consumed;
|
|
3222
|
+
receipt(current, "progress");
|
|
3223
|
+
}
|
|
3224
|
+
wake();
|
|
3225
|
+
}
|
|
3226
|
+
}
|
|
3227
|
+
};
|
|
3228
|
+
const playbackState = () => {
|
|
3229
|
+
const next = context?.state !== "running";
|
|
3230
|
+
if (next !== blocked) {
|
|
3231
|
+
blocked = next;
|
|
3232
|
+
notify(() => options.onPlaybackBlocked?.(next ? startAudio : null));
|
|
3233
|
+
}
|
|
3234
|
+
tick();
|
|
3235
|
+
};
|
|
3236
|
+
const getContext = () => {
|
|
3237
|
+
if (!context) {
|
|
3238
|
+
context = new AudioContext({ sampleRate: correlatedAudioHeaderSchema.shape.sample_rate.value });
|
|
3239
|
+
context.onstatechange = playbackState;
|
|
3240
|
+
playbackState();
|
|
3241
|
+
}
|
|
3242
|
+
return context;
|
|
3243
|
+
};
|
|
3244
|
+
async function startAudio() {
|
|
3245
|
+
if (disposed || binding.unavailable) throw new Error("Correlated audio binding is no longer available");
|
|
3246
|
+
const ctx = getContext();
|
|
3247
|
+
await ctx.resume();
|
|
3248
|
+
playbackState();
|
|
3249
|
+
if (ctx.state !== "running") throw new Error("Correlated audio playback is blocked");
|
|
3250
|
+
}
|
|
3251
|
+
async function receive(reader, participant) {
|
|
3252
|
+
if (disposed || binding.unavailable || room.state !== "connected" || participant.identity !== options.agentIdentity) return;
|
|
3253
|
+
let value;
|
|
3254
|
+
try {
|
|
3255
|
+
value = JSON.parse(reader.info.attributes?.["rta.audio"] ?? "");
|
|
3256
|
+
} catch {
|
|
3257
|
+
return;
|
|
3258
|
+
}
|
|
3259
|
+
const parsed = correlatedAudioHeaderSchema.safeParse(value);
|
|
3260
|
+
if (!parsed.success || parsed.data.session_id !== options.sessionId || parsed.data.epoch < binding.epoch) return;
|
|
3261
|
+
const header = parsed.data;
|
|
3262
|
+
if (header.stream_id !== reader.info.id) return;
|
|
3263
|
+
if (header.epoch > binding.epoch) {
|
|
3264
|
+
binding.epoch = header.epoch;
|
|
3265
|
+
invalidate("cancelled");
|
|
3266
|
+
}
|
|
3267
|
+
const key = `${header.epoch}/${header.stream_id}`;
|
|
3268
|
+
if (seen.has(key)) return;
|
|
3269
|
+
if (sources.length >= 128 || seen.size >= 1e4) {
|
|
3270
|
+
invalidate("invalid_stream");
|
|
3271
|
+
return;
|
|
3272
|
+
}
|
|
3273
|
+
seen.add(key);
|
|
3274
|
+
const previousSource = sources.at(-1);
|
|
3275
|
+
const current = { header, abort: new AbortController(), segments: [], received: 0, consumed: 0, started: false, eof: false, lastReported: 0, lastPosition: -1, rpcBusy: false };
|
|
3276
|
+
sources.push(current);
|
|
3277
|
+
let carry;
|
|
3278
|
+
try {
|
|
3279
|
+
if (options.prepare) await options.prepare(header, current.abort.signal);
|
|
3280
|
+
if (!active(current) || current.abort.signal.aborted) return;
|
|
3281
|
+
const ctx = getContext();
|
|
3282
|
+
while (active(current) && (previousSource && active(previousSource) && !previousSource.eof || buffered() > CORRELATED_AUDIO_CONTRACT.maxBufferedSamples - CORRELATED_AUDIO_CONTRACT.maxChunkBytes / 2)) {
|
|
3283
|
+
await new Promise((resolve) => waiters.add(resolve));
|
|
3284
|
+
}
|
|
3285
|
+
if (!active(current)) return;
|
|
3286
|
+
for await (const chunk of reader.withAbortSignal(current.abort.signal)) {
|
|
3287
|
+
if (!active(current)) break;
|
|
3288
|
+
if (chunk.byteLength > CORRELATED_AUDIO_CONTRACT.maxChunkBytes) throw new Error("PCM chunk exceeds contract");
|
|
3289
|
+
const count = Math.floor((chunk.byteLength + (carry === void 0 ? 0 : 1)) / 2);
|
|
3290
|
+
if (current.received + count > CORRELATED_AUDIO_CONTRACT.maxSamples) throw new Error("PCM stream exceeds contract");
|
|
3291
|
+
while (active(current) && buffered() + count > CORRELATED_AUDIO_CONTRACT.maxBufferedSamples) {
|
|
3292
|
+
await new Promise((resolve) => waiters.add(resolve));
|
|
3293
|
+
}
|
|
3294
|
+
if (!active(current)) break;
|
|
3295
|
+
if (!count) {
|
|
3296
|
+
if (chunk.length) carry = chunk[0];
|
|
3297
|
+
continue;
|
|
3298
|
+
}
|
|
3299
|
+
const buffer = ctx.createBuffer(header.channels, count, header.sample_rate);
|
|
3300
|
+
const data = buffer.getChannelData(0);
|
|
3301
|
+
let cursor = 0;
|
|
3302
|
+
for (let i = 0; i < count; i++) {
|
|
3303
|
+
const low = carry ?? chunk[cursor++];
|
|
3304
|
+
carry = void 0;
|
|
3305
|
+
const raw = low | chunk[cursor++] << 8;
|
|
3306
|
+
data[i] = (raw >= 32768 ? raw - 65536 : raw) / 32768;
|
|
3307
|
+
}
|
|
3308
|
+
if (cursor < chunk.length) carry = chunk[cursor];
|
|
3309
|
+
const node = ctx.createBufferSource();
|
|
3310
|
+
node.buffer = buffer;
|
|
3311
|
+
node.connect(ctx.destination);
|
|
3312
|
+
const start = Math.max(ctx.currentTime, nextStart);
|
|
3313
|
+
current.segments.push({ node, start, offset: current.received, length: count });
|
|
3314
|
+
current.received += count;
|
|
3315
|
+
nextStart = start + count / header.sample_rate;
|
|
3316
|
+
node.onended = () => tick();
|
|
3317
|
+
node.start(start);
|
|
3318
|
+
while (active(current) && buffered() > CORRELATED_AUDIO_CONTRACT.maxBufferedSamples - CORRELATED_AUDIO_CONTRACT.maxChunkBytes / 2) {
|
|
3319
|
+
await new Promise((resolve) => waiters.add(resolve));
|
|
3320
|
+
}
|
|
3321
|
+
}
|
|
3322
|
+
if (!active(current)) return;
|
|
3323
|
+
if (carry !== void 0 || current.received === 0) throw new Error("Incomplete PCM stream");
|
|
3324
|
+
current.eof = true;
|
|
3325
|
+
wake();
|
|
3326
|
+
tick(false);
|
|
3327
|
+
} catch {
|
|
3328
|
+
if (active(current)) invalidate("invalid_stream");
|
|
3329
|
+
}
|
|
3330
|
+
}
|
|
3331
|
+
const cancel = async (data) => {
|
|
3332
|
+
if (disposed || binding.unavailable || room.state !== "connected" || data.callerIdentity !== options.agentIdentity) throw new Error("Unbound audio cancellation");
|
|
3333
|
+
const value = JSON.parse(data.payload);
|
|
3334
|
+
const frame2 = correlatedAudioCancelSchema.parse(value);
|
|
3335
|
+
if (frame2.session_id !== options.sessionId) throw new Error("Wrong audio session");
|
|
3336
|
+
if (frame2.epoch <= binding.epoch) return "";
|
|
3337
|
+
binding.epoch = frame2.epoch;
|
|
3338
|
+
invalidate("cancelled");
|
|
3339
|
+
return "";
|
|
3340
|
+
};
|
|
3341
|
+
const lost = () => invalidate("connection_lost");
|
|
3342
|
+
const changed = () => {
|
|
3343
|
+
if (["disconnected", "reconnecting", "signalReconnecting"].includes(room.state)) lost();
|
|
3344
|
+
};
|
|
3345
|
+
const participantLeft = (participant) => {
|
|
3346
|
+
if (participant.identity === options.agentIdentity) lost();
|
|
3347
|
+
};
|
|
3348
|
+
let registered = false;
|
|
3349
|
+
try {
|
|
3350
|
+
room.registerByteStreamHandler(CORRELATED_AUDIO_CONTRACT.topic, (reader, participant) => {
|
|
3351
|
+
void receive(reader, participant);
|
|
3352
|
+
});
|
|
3353
|
+
registered = true;
|
|
3354
|
+
room.localParticipant.registerRpcMethod(CORRELATED_AUDIO_CONTRACT.cancelMethod, cancel);
|
|
3355
|
+
} catch (error) {
|
|
3356
|
+
if (registered) room.unregisterByteStreamHandler(CORRELATED_AUDIO_CONTRACT.topic);
|
|
3357
|
+
attachments.delete(room);
|
|
3358
|
+
throw error;
|
|
3359
|
+
}
|
|
3360
|
+
room.on(RoomEvent.Reconnecting, lost);
|
|
3361
|
+
room.on(RoomEvent.SignalReconnecting, lost);
|
|
3362
|
+
room.on(RoomEvent.Disconnected, lost);
|
|
3363
|
+
room.on(RoomEvent.ConnectionStateChanged, changed);
|
|
3364
|
+
room.on(RoomEvent.ParticipantDisconnected, participantLeft);
|
|
3365
|
+
const timer = setInterval(() => tick(), 100);
|
|
3366
|
+
let frame = 0;
|
|
3367
|
+
const render = () => {
|
|
3368
|
+
if (disposed) return;
|
|
3369
|
+
tick(false);
|
|
3370
|
+
frame = requestAnimationFrame(render);
|
|
3371
|
+
};
|
|
3372
|
+
frame = requestAnimationFrame(render);
|
|
3373
|
+
return {
|
|
3374
|
+
startAudio,
|
|
3375
|
+
dispose() {
|
|
3376
|
+
if (closing) return closing;
|
|
3377
|
+
invalidate("disposed");
|
|
3378
|
+
disposed = true;
|
|
3379
|
+
clearInterval(timer);
|
|
3380
|
+
cancelAnimationFrame(frame);
|
|
3381
|
+
room.unregisterByteStreamHandler(CORRELATED_AUDIO_CONTRACT.topic);
|
|
3382
|
+
room.localParticipant.unregisterRpcMethod(CORRELATED_AUDIO_CONTRACT.cancelMethod);
|
|
3383
|
+
room.off(RoomEvent.Reconnecting, lost);
|
|
3384
|
+
room.off(RoomEvent.SignalReconnecting, lost);
|
|
3385
|
+
room.off(RoomEvent.Disconnected, lost);
|
|
3386
|
+
room.off(RoomEvent.ConnectionStateChanged, changed);
|
|
3387
|
+
room.off(RoomEvent.ParticipantDisconnected, participantLeft);
|
|
3388
|
+
attachments.delete(room);
|
|
3389
|
+
if (context) context.onstatechange = null;
|
|
3390
|
+
closing = context?.close().catch(() => {
|
|
3391
|
+
}) ?? Promise.resolve();
|
|
3392
|
+
return closing;
|
|
3393
|
+
}
|
|
3394
|
+
};
|
|
3395
|
+
}
|
|
3396
|
+
|
|
3397
|
+
// ../client/src/external-performance.ts
|
|
3398
|
+
var EXTERNAL_PERFORMANCE_KIND = externalPerformanceFrameSchema.shape.kind.value;
|
|
3399
|
+
var EXTERNAL_PERFORMANCE_TIMING = Object.freeze({
|
|
3400
|
+
receiverPlayout: "unsupported",
|
|
3401
|
+
audibleCheckpoint: externalPerformanceFrameSchema.shape.audible_checkpoint.value,
|
|
3402
|
+
audioMapping: externalPerformanceFrameSchema.shape.audio_mapping.value
|
|
3403
|
+
});
|
|
3404
|
+
var statusSchema = externalPerformanceFrameSchema.shape.status;
|
|
3405
|
+
var KNOWN_EXTERNAL_PERFORMANCE_STATUSES = Object.freeze([...statusSchema.options[0].options]);
|
|
3406
|
+
var knownStatuses = new Set(KNOWN_EXTERNAL_PERFORMANCE_STATUSES);
|
|
3407
|
+
function isObservable(frame) {
|
|
3408
|
+
return knownStatuses.has(frame.status);
|
|
3409
|
+
}
|
|
3410
|
+
var NEXT = {
|
|
3411
|
+
unseen: /* @__PURE__ */ new Set(["accepted", "dropped"]),
|
|
3412
|
+
accepted: /* @__PURE__ */ new Set(["audio_released", "cancelled"]),
|
|
3413
|
+
audio_released: /* @__PURE__ */ new Set(["cancelled"]),
|
|
3414
|
+
dropped: /* @__PURE__ */ new Set(),
|
|
3415
|
+
cancelled: /* @__PURE__ */ new Set(),
|
|
3416
|
+
retired: /* @__PURE__ */ new Set()
|
|
3417
|
+
};
|
|
3418
|
+
var LEDGER_LIMIT = 4096;
|
|
3419
|
+
function deliver(callback) {
|
|
3420
|
+
try {
|
|
3421
|
+
void Promise.resolve(callback()).catch(() => {
|
|
3422
|
+
});
|
|
3423
|
+
} catch {
|
|
3424
|
+
}
|
|
3425
|
+
}
|
|
3426
|
+
function createExternalPerformanceObserver(sessionId, callbacks) {
|
|
3427
|
+
const generations = /* @__PURE__ */ new Map();
|
|
3428
|
+
let state = "live";
|
|
3429
|
+
return {
|
|
3430
|
+
receive(frame) {
|
|
3431
|
+
if (state !== "live" || frame.session_id !== sessionId || !isObservable(frame)) return;
|
|
3432
|
+
const previous = generations.get(frame.generation_id) ?? "unseen";
|
|
3433
|
+
if (!NEXT[previous].has(frame.status)) return;
|
|
3434
|
+
if (previous === "unseen" && generations.size >= LEDGER_LIMIT) {
|
|
3435
|
+
const oldest = generations.keys().next();
|
|
3436
|
+
if (!oldest.done) generations.delete(oldest.value);
|
|
3437
|
+
}
|
|
3438
|
+
generations.set(frame.generation_id, frame.status);
|
|
3439
|
+
deliver(() => callbacks.onFrame(frame));
|
|
3440
|
+
},
|
|
3441
|
+
invalidate(reason) {
|
|
3442
|
+
if (state === "closed" || state === "paused" && reason === "connection_lost") return;
|
|
3443
|
+
state = reason === "connection_lost" ? "paused" : "closed";
|
|
3444
|
+
if (state === "closed") generations.clear();
|
|
3445
|
+
else for (const id of generations.keys()) generations.set(id, "retired");
|
|
3446
|
+
deliver(() => callbacks.onInvalidated(reason));
|
|
3447
|
+
},
|
|
3448
|
+
/** The call is connected to its agent again. Only a connection loss can be resumed. */
|
|
3449
|
+
resume() {
|
|
3450
|
+
if (state === "paused") state = "live";
|
|
3451
|
+
}
|
|
3452
|
+
};
|
|
3453
|
+
}
|
|
2991
3454
|
|
|
2992
3455
|
// ../client/src/react/session-lifecycle.ts
|
|
2993
3456
|
function phaseForReason(reason) {
|
|
@@ -3183,6 +3646,32 @@ function useSessionLifecycle(input) {
|
|
|
3183
3646
|
const grantIdRef = useRef(null);
|
|
3184
3647
|
grantIdRef.current = grantId;
|
|
3185
3648
|
const retiredGrantRef = useRef(null);
|
|
3649
|
+
const performanceEnabled = input.externalPerformance !== void 0;
|
|
3650
|
+
const performanceCallbacksRef = useRef(input.externalPerformance);
|
|
3651
|
+
if (input.externalPerformance) performanceCallbacksRef.current = input.externalPerformance;
|
|
3652
|
+
const performanceRef = useRef(null);
|
|
3653
|
+
const endedPerformanceGrantRef = useRef(null);
|
|
3654
|
+
useEffect(() => {
|
|
3655
|
+
if (!grantId || !performanceEnabled || endedPerformanceGrantRef.current === grantId) return;
|
|
3656
|
+
const observer = createExternalPerformanceObserver(grantId, {
|
|
3657
|
+
onFrame: (frame) => performanceCallbacksRef.current?.onFrame(frame),
|
|
3658
|
+
onInvalidated: (reason) => {
|
|
3659
|
+
if (reason === "ended") endedPerformanceGrantRef.current = grantId;
|
|
3660
|
+
performanceCallbacksRef.current?.onInvalidated(reason);
|
|
3661
|
+
}
|
|
3662
|
+
});
|
|
3663
|
+
performanceRef.current = observer;
|
|
3664
|
+
return () => {
|
|
3665
|
+
observer.invalidate("binding_changed");
|
|
3666
|
+
if (performanceRef.current === observer) performanceRef.current = null;
|
|
3667
|
+
};
|
|
3668
|
+
}, [grantId, performanceEnabled]);
|
|
3669
|
+
const invalidateExternalPerformance = useCallback((reason) => {
|
|
3670
|
+
if (grantId === grantIdRef.current) performanceRef.current?.invalidate(reason);
|
|
3671
|
+
}, [grantId]);
|
|
3672
|
+
const resumeExternalPerformance = useCallback(() => {
|
|
3673
|
+
if (grantId === grantIdRef.current) performanceRef.current?.resume();
|
|
3674
|
+
}, [grantId]);
|
|
3186
3675
|
const [recovery, setRecovery] = useState({ kind: "connected" });
|
|
3187
3676
|
const recoveryRef = useRef(recovery);
|
|
3188
3677
|
recoveryRef.current = recovery;
|
|
@@ -3228,6 +3717,7 @@ function useSessionLifecycle(input) {
|
|
|
3228
3717
|
}
|
|
3229
3718
|
}, []);
|
|
3230
3719
|
const reset = useCallback(() => {
|
|
3720
|
+
invalidateExternalPerformance("binding_changed");
|
|
3231
3721
|
clearTimer();
|
|
3232
3722
|
connectedRef.current = false;
|
|
3233
3723
|
manualReconnectPendingRef.current = false;
|
|
@@ -3242,7 +3732,7 @@ function useSessionLifecycle(input) {
|
|
|
3242
3732
|
setConnected(false);
|
|
3243
3733
|
setAgentPresentState(false);
|
|
3244
3734
|
lastActivityRef.current = Date.now();
|
|
3245
|
-
}, [clearTimer]);
|
|
3735
|
+
}, [clearTimer, invalidateExternalPerformance]);
|
|
3246
3736
|
useEffect(() => {
|
|
3247
3737
|
if (active) return;
|
|
3248
3738
|
userEndedRef.current = false;
|
|
@@ -3276,6 +3766,7 @@ function useSessionLifecycle(input) {
|
|
|
3276
3766
|
}, [grantId]);
|
|
3277
3767
|
const endIdle = useCallback(() => {
|
|
3278
3768
|
if (!connectedRef.current) return;
|
|
3769
|
+
invalidateExternalPerformance("ended");
|
|
3279
3770
|
clearTimer();
|
|
3280
3771
|
connectedRef.current = false;
|
|
3281
3772
|
setConnected(false);
|
|
@@ -3283,7 +3774,7 @@ function useSessionLifecycle(input) {
|
|
|
3283
3774
|
releaseRef.current("idle_timeout");
|
|
3284
3775
|
setRecovery({ kind: "ended", reason: "idle" });
|
|
3285
3776
|
leaveRoomRef.current?.();
|
|
3286
|
-
}, [clearTimer]);
|
|
3777
|
+
}, [clearTimer, invalidateExternalPerformance]);
|
|
3287
3778
|
const onConnected = useCallback(() => {
|
|
3288
3779
|
if (grantId === null || grantIdRef.current !== grantId) return;
|
|
3289
3780
|
if (userEndedRef.current || readinessFailureRef.current || retiredGrantRef.current !== null && retiredGrantRef.current === grantIdRef.current) return;
|
|
@@ -3301,6 +3792,7 @@ function useSessionLifecycle(input) {
|
|
|
3301
3792
|
if (grantId === null || grantIdRef.current !== grantId) return;
|
|
3302
3793
|
if (!active || userEndedRef.current || readinessFailureRef.current) return;
|
|
3303
3794
|
const action = disconnectAction(reason, connectedRef.current);
|
|
3795
|
+
invalidateExternalPerformance(action === "end" ? "ended" : "connection_lost");
|
|
3304
3796
|
if (action === "noop") return;
|
|
3305
3797
|
if (action === "reset") {
|
|
3306
3798
|
if (recoveryRef.current.kind === "refreshing" || recoveryRef.current.kind === "reconnecting") return;
|
|
@@ -3320,12 +3812,13 @@ function useSessionLifecycle(input) {
|
|
|
3320
3812
|
refreshBaselineCapacityRef.current = null;
|
|
3321
3813
|
setRecovery({ kind: "reconnecting", attempt: attemptRef.current });
|
|
3322
3814
|
},
|
|
3323
|
-
[active, reset, grantId]
|
|
3815
|
+
[active, reset, grantId, invalidateExternalPerformance]
|
|
3324
3816
|
);
|
|
3325
3817
|
const onConnectionError = useCallback(
|
|
3326
3818
|
(error) => {
|
|
3327
3819
|
if (grantId === null || grantIdRef.current !== grantId) return;
|
|
3328
3820
|
if (!active || userEndedRef.current || readinessFailureRef.current || !isRecoverableConnectionError(error)) return;
|
|
3821
|
+
invalidateExternalPerformance("connection_lost");
|
|
3329
3822
|
connectedRef.current = false;
|
|
3330
3823
|
setConnected(false);
|
|
3331
3824
|
setAgentPresentState(false);
|
|
@@ -3336,7 +3829,7 @@ function useSessionLifecycle(input) {
|
|
|
3336
3829
|
return { kind: "reconnecting", attempt: attemptRef.current };
|
|
3337
3830
|
});
|
|
3338
3831
|
},
|
|
3339
|
-
[active, grantId]
|
|
3832
|
+
[active, grantId, invalidateExternalPerformance]
|
|
3340
3833
|
);
|
|
3341
3834
|
useEffect(() => {
|
|
3342
3835
|
if (capacity.kind === "error" && recovery.kind === "connected") manualReconnectPendingRef.current = false;
|
|
@@ -3356,6 +3849,9 @@ function useSessionLifecycle(input) {
|
|
|
3356
3849
|
onConnected();
|
|
3357
3850
|
return;
|
|
3358
3851
|
}
|
|
3852
|
+
if (state === "disconnected" && connectedRef.current || state === "reconnecting" || state === "signalReconnecting") {
|
|
3853
|
+
invalidateExternalPerformance("connection_lost");
|
|
3854
|
+
}
|
|
3359
3855
|
if (state === "reconnecting" || state === "signalReconnecting") {
|
|
3360
3856
|
if (connectedRef.current) {
|
|
3361
3857
|
setConnected(false);
|
|
@@ -3365,7 +3861,7 @@ function useSessionLifecycle(input) {
|
|
|
3365
3861
|
}
|
|
3366
3862
|
}
|
|
3367
3863
|
},
|
|
3368
|
-
[active, onConnected, grantId]
|
|
3864
|
+
[active, onConnected, grantId, invalidateExternalPerformance]
|
|
3369
3865
|
);
|
|
3370
3866
|
useEffect(() => {
|
|
3371
3867
|
if (!active || userEnded || readinessFailed || !connected || !agentPresent || !grantState.grant) return;
|
|
@@ -3378,6 +3874,7 @@ function useSessionLifecycle(input) {
|
|
|
3378
3874
|
const failReadiness = useCallback((code) => {
|
|
3379
3875
|
if (userEndedRef.current || readinessFailureRef.current) return;
|
|
3380
3876
|
readinessFailureRef.current = code;
|
|
3877
|
+
invalidateExternalPerformance("ended");
|
|
3381
3878
|
clearTimer();
|
|
3382
3879
|
connectedRef.current = false;
|
|
3383
3880
|
manualReconnectPendingRef.current = false;
|
|
@@ -3390,7 +3887,7 @@ function useSessionLifecycle(input) {
|
|
|
3390
3887
|
releaseRef.current("disconnected");
|
|
3391
3888
|
setReadinessFailed(true);
|
|
3392
3889
|
leaveRoomRef.current?.();
|
|
3393
|
-
}, [clearTimer]);
|
|
3890
|
+
}, [clearTimer, invalidateExternalPerformance]);
|
|
3394
3891
|
useEffect(() => {
|
|
3395
3892
|
if (!active || userEnded || readinessFailed) return;
|
|
3396
3893
|
if (recovery.kind === "ended" || recovery.kind === "failed") return;
|
|
@@ -3471,6 +3968,7 @@ function useSessionLifecycle(input) {
|
|
|
3471
3968
|
const end = useCallback(() => {
|
|
3472
3969
|
if (!active || userEndedRef.current || phaseRef.current.kind === "ended") return;
|
|
3473
3970
|
userEndedRef.current = true;
|
|
3971
|
+
invalidateExternalPerformance("ended");
|
|
3474
3972
|
clearTimer();
|
|
3475
3973
|
connectedRef.current = false;
|
|
3476
3974
|
manualReconnectPendingRef.current = false;
|
|
@@ -3481,7 +3979,7 @@ function useSessionLifecycle(input) {
|
|
|
3481
3979
|
setRecovery({ kind: "ended", reason: "user" });
|
|
3482
3980
|
clearRef.current();
|
|
3483
3981
|
leaveRoomRef.current?.();
|
|
3484
|
-
}, [active, clearTimer]);
|
|
3982
|
+
}, [active, clearTimer, invalidateExternalPerformance]);
|
|
3485
3983
|
const reconnect = useCallback(() => {
|
|
3486
3984
|
if (!active) return;
|
|
3487
3985
|
if (readinessFailureRef.current) {
|
|
@@ -3551,16 +4049,22 @@ function useSessionLifecycle(input) {
|
|
|
3551
4049
|
const lastBehaviorRef = useRef(null);
|
|
3552
4050
|
const behaviorEnabled = Boolean(onBehaviorChange);
|
|
3553
4051
|
const onLifecycleData = useMemo(() => {
|
|
3554
|
-
if (!behaviorEnabled) return void 0;
|
|
4052
|
+
if (!behaviorEnabled && !performanceEnabled) return void 0;
|
|
3555
4053
|
return (frame) => {
|
|
3556
4054
|
const parsed = lifecycleServerFrameSchema.safeParse(frame);
|
|
3557
|
-
if (!parsed.success
|
|
4055
|
+
if (!parsed.success) return;
|
|
4056
|
+
if (parsed.data.kind === "external_performance") {
|
|
4057
|
+
if (grantId === grantIdRef.current && parsed.data.session_id === grantId) performanceRef.current?.receive(parsed.data);
|
|
4058
|
+
return;
|
|
4059
|
+
}
|
|
4060
|
+
if (parsed.data.kind === "ended") invalidateExternalPerformance("ended");
|
|
4061
|
+
if (!behaviorEnabled || parsed.data.kind !== "behavior_state") return;
|
|
3558
4062
|
const next = nextBehaviorSnapshot(lastBehaviorRef.current, parsed.data);
|
|
3559
4063
|
if (!next) return;
|
|
3560
4064
|
lastBehaviorRef.current = next;
|
|
3561
4065
|
onBehaviorChangeRef.current?.(next);
|
|
3562
4066
|
};
|
|
3563
|
-
}, [behaviorEnabled]);
|
|
4067
|
+
}, [behaviorEnabled, performanceEnabled, grantId, invalidateExternalPerformance]);
|
|
3564
4068
|
return useMemo(
|
|
3565
4069
|
() => ({
|
|
3566
4070
|
phase,
|
|
@@ -3579,7 +4083,9 @@ function useSessionLifecycle(input) {
|
|
|
3579
4083
|
setAgentPresent,
|
|
3580
4084
|
registerLeaveRoom,
|
|
3581
4085
|
reset,
|
|
3582
|
-
onLifecycleData
|
|
4086
|
+
onLifecycleData,
|
|
4087
|
+
invalidateExternalPerformance: performanceEnabled ? invalidateExternalPerformance : void 0,
|
|
4088
|
+
resumeExternalPerformance: performanceEnabled ? resumeExternalPerformance : void 0
|
|
3583
4089
|
}),
|
|
3584
4090
|
[
|
|
3585
4091
|
phase,
|
|
@@ -3598,7 +4104,10 @@ function useSessionLifecycle(input) {
|
|
|
3598
4104
|
setAgentPresent,
|
|
3599
4105
|
registerLeaveRoom,
|
|
3600
4106
|
reset,
|
|
3601
|
-
onLifecycleData
|
|
4107
|
+
onLifecycleData,
|
|
4108
|
+
invalidateExternalPerformance,
|
|
4109
|
+
resumeExternalPerformance,
|
|
4110
|
+
performanceEnabled
|
|
3602
4111
|
]
|
|
3603
4112
|
);
|
|
3604
4113
|
}
|
|
@@ -3627,9 +4136,45 @@ function SessionLifecycleRoomBridge({
|
|
|
3627
4136
|
const transcriptions = useTranscriptions();
|
|
3628
4137
|
const room = useRoomContext();
|
|
3629
4138
|
const roomMatchesGrant = lifecycle.grant === void 0 || lifecycle.grant !== null && room.name === lifecycle.grant.room_name;
|
|
4139
|
+
const correlatedRef = useRef(null);
|
|
4140
|
+
const correlatedObserver = useRef(lifecycle.correlatedAudio);
|
|
4141
|
+
correlatedObserver.current = lifecycle.correlatedAudio;
|
|
4142
|
+
const correlatedEnabled = lifecycle.correlatedAudio !== void 0;
|
|
4143
|
+
const correlatedSessionId = lifecycle.grant?.session_id;
|
|
4144
|
+
const startCorrelatedAudio = useCallback(async () => {
|
|
4145
|
+
const player = correlatedRef.current;
|
|
4146
|
+
if (!player) return false;
|
|
4147
|
+
try {
|
|
4148
|
+
await player.startAudio();
|
|
4149
|
+
return true;
|
|
4150
|
+
} catch {
|
|
4151
|
+
return false;
|
|
4152
|
+
}
|
|
4153
|
+
}, []);
|
|
4154
|
+
useEffect(() => {
|
|
4155
|
+
if (!correlatedEnabled || !correlatedSessionId || !roomMatchesGrant || !assistant.agent) return;
|
|
4156
|
+
const player = attachCorrelatedAudio(room, {
|
|
4157
|
+
sessionId: correlatedSessionId,
|
|
4158
|
+
agentIdentity: assistant.agent.identity,
|
|
4159
|
+
prepare: (header, signal) => correlatedObserver.current?.prepare?.(header, signal) ?? Promise.resolve(),
|
|
4160
|
+
onPosition: (position) => correlatedObserver.current?.onPosition(position),
|
|
4161
|
+
onInvalidated: (reason) => correlatedObserver.current?.onInvalidated(reason),
|
|
4162
|
+
onPlaybackBlocked: (unblock) => {
|
|
4163
|
+
setAudioPlayback?.(unblock ? "blocked" : "allowed");
|
|
4164
|
+
correlatedObserver.current?.onPlaybackBlocked?.(unblock);
|
|
4165
|
+
}
|
|
4166
|
+
});
|
|
4167
|
+
correlatedRef.current = player;
|
|
4168
|
+
return () => {
|
|
4169
|
+
correlatedRef.current = null;
|
|
4170
|
+
void player.dispose();
|
|
4171
|
+
setAudioPlayback?.("unknown");
|
|
4172
|
+
};
|
|
4173
|
+
}, [room, assistant.agent, correlatedEnabled, correlatedSessionId, roomMatchesGrant, setAudioPlayback]);
|
|
3630
4174
|
useCallMedia(room, {
|
|
3631
4175
|
setMicrophoneFacts,
|
|
3632
|
-
setAudioPlayback,
|
|
4176
|
+
setAudioPlayback: correlatedEnabled ? void 0 : setAudioPlayback,
|
|
4177
|
+
startAudioOverride: correlatedEnabled ? startCorrelatedAudio : void 0,
|
|
3633
4178
|
registerMediaControls,
|
|
3634
4179
|
microphoneWanted: microphone ?? publishesMicrophoneByDefault(lifecycle.grant),
|
|
3635
4180
|
connectionState,
|
|
@@ -3643,7 +4188,7 @@ function SessionLifecycleRoomBridge({
|
|
|
3643
4188
|
if (!mediaModeEnabled) return;
|
|
3644
4189
|
const agent = assistant.agent;
|
|
3645
4190
|
let previous;
|
|
3646
|
-
const
|
|
4191
|
+
const deliver2 = (mode) => {
|
|
3647
4192
|
if (mode === previous) return;
|
|
3648
4193
|
previous = mode;
|
|
3649
4194
|
try {
|
|
@@ -3654,7 +4199,7 @@ function SessionLifecycleRoomBridge({
|
|
|
3654
4199
|
};
|
|
3655
4200
|
const read = () => {
|
|
3656
4201
|
const value = agent?.attributes?.["rta.media_mode"];
|
|
3657
|
-
|
|
4202
|
+
deliver2(value === "video" ? "avatar" : value === "voice" ? "voice" : null);
|
|
3658
4203
|
};
|
|
3659
4204
|
const changed = (_attributes, participant) => {
|
|
3660
4205
|
if (participant === agent) read();
|
|
@@ -3663,7 +4208,7 @@ function SessionLifecycleRoomBridge({
|
|
|
3663
4208
|
read();
|
|
3664
4209
|
return () => {
|
|
3665
4210
|
room.off(RoomEvent.ParticipantAttributesChanged, changed);
|
|
3666
|
-
|
|
4211
|
+
deliver2(null);
|
|
3667
4212
|
};
|
|
3668
4213
|
}, [room, assistant.agent, mediaModeEnabled, mediaModeSessionId]);
|
|
3669
4214
|
const { send } = useChat();
|
|
@@ -3710,7 +4255,7 @@ function SessionLifecycleRoomBridge({
|
|
|
3710
4255
|
let closed = false;
|
|
3711
4256
|
let pending = false;
|
|
3712
4257
|
let previous;
|
|
3713
|
-
const
|
|
4258
|
+
const deliver2 = (details) => {
|
|
3714
4259
|
connectionHistoryRef.current?.enqueue(details);
|
|
3715
4260
|
try {
|
|
3716
4261
|
if (onConnectionDetailsChange) void Promise.resolve(callback?.(details)).catch(() => {
|
|
@@ -3737,7 +4282,7 @@ function SessionLifecycleRoomBridge({
|
|
|
3737
4282
|
const last = previous;
|
|
3738
4283
|
if (last?.connectionState === next.connectionState && last.localQuality === next.localQuality && ["audio", "video"].every((kind) => last[kind]?.publisherQuality === next[kind]?.publisherQuality && last[kind]?.streamState === next[kind]?.streamState)) return;
|
|
3739
4284
|
previous = next;
|
|
3740
|
-
|
|
4285
|
+
deliver2(next);
|
|
3741
4286
|
});
|
|
3742
4287
|
};
|
|
3743
4288
|
const onQuality = (_quality, participant) => {
|
|
@@ -3767,7 +4312,7 @@ function SessionLifecycleRoomBridge({
|
|
|
3767
4312
|
room.off(RoomEvent.TrackStreamStateChanged, onPublication);
|
|
3768
4313
|
room.off(RoomEvent.TrackSubscribed, onTrack);
|
|
3769
4314
|
room.off(RoomEvent.TrackUnsubscribed, onTrack);
|
|
3770
|
-
|
|
4315
|
+
deliver2(null);
|
|
3771
4316
|
};
|
|
3772
4317
|
}, [room, detailsSessionId, detailsEnabled]);
|
|
3773
4318
|
useEffect(() => {
|
|
@@ -3804,18 +4349,56 @@ function SessionLifecycleRoomBridge({
|
|
|
3804
4349
|
useEffect(() => {
|
|
3805
4350
|
if (!onLifecycleData) return;
|
|
3806
4351
|
const decoder = new TextDecoder();
|
|
3807
|
-
const handler = (payload,
|
|
4352
|
+
const handler = (payload, participant, _kind, topic) => {
|
|
3808
4353
|
if (topic !== RTA_LIFECYCLE_TOPIC) return;
|
|
3809
4354
|
try {
|
|
3810
|
-
|
|
4355
|
+
const frame = JSON.parse(decoder.decode(payload));
|
|
4356
|
+
if (typeof frame === "object" && frame !== null && "kind" in frame && (frame.kind === EXTERNAL_PERFORMANCE_KIND || frame.kind === "ended") && (!assistant.agent || participant !== assistant.agent || !roomMatchesGrant || room.state !== "connected")) return;
|
|
4357
|
+
if (typeof frame === "object" && frame !== null && "kind" in frame && frame.kind === "ended" && lifecycleServerFrameSchema.safeParse(frame).success) {
|
|
4358
|
+
void correlatedRef.current?.dispose();
|
|
4359
|
+
correlatedRef.current = null;
|
|
4360
|
+
}
|
|
4361
|
+
onLifecycleData(frame);
|
|
3811
4362
|
} catch {
|
|
3812
4363
|
}
|
|
3813
4364
|
};
|
|
4365
|
+
const invalidate = () => lifecycle.invalidateExternalPerformance?.("connection_lost");
|
|
4366
|
+
const changed = () => {
|
|
4367
|
+
if (room.state === "disconnected" || room.state === "reconnecting" || room.state === "signalReconnecting") invalidate();
|
|
4368
|
+
};
|
|
3814
4369
|
room.on(RoomEvent.DataReceived, handler);
|
|
4370
|
+
if (lifecycle.invalidateExternalPerformance) {
|
|
4371
|
+
room.on(RoomEvent.Reconnecting, invalidate);
|
|
4372
|
+
room.on(RoomEvent.SignalReconnecting, invalidate);
|
|
4373
|
+
room.on(RoomEvent.Disconnected, invalidate);
|
|
4374
|
+
room.on(RoomEvent.ConnectionStateChanged, changed);
|
|
4375
|
+
}
|
|
3815
4376
|
return () => {
|
|
3816
4377
|
room.off(RoomEvent.DataReceived, handler);
|
|
4378
|
+
room.off(RoomEvent.Reconnecting, invalidate);
|
|
4379
|
+
room.off(RoomEvent.SignalReconnecting, invalidate);
|
|
4380
|
+
room.off(RoomEvent.Disconnected, invalidate);
|
|
4381
|
+
room.off(RoomEvent.ConnectionStateChanged, changed);
|
|
4382
|
+
};
|
|
4383
|
+
}, [room, onLifecycleData, assistant.agent, roomMatchesGrant, lifecycle.invalidateExternalPerformance]);
|
|
4384
|
+
useEffect(() => {
|
|
4385
|
+
if (!assistant.agent) return;
|
|
4386
|
+
return () => lifecycle.invalidateExternalPerformance?.("connection_lost");
|
|
4387
|
+
}, [assistant.agent, lifecycle.invalidateExternalPerformance]);
|
|
4388
|
+
const resumeExternalPerformance = lifecycle.resumeExternalPerformance;
|
|
4389
|
+
useEffect(() => {
|
|
4390
|
+
if (!resumeExternalPerformance) return;
|
|
4391
|
+
const resumeIfBound = () => {
|
|
4392
|
+
if (assistant.agent && roomMatchesGrant && room.state === "connected") resumeExternalPerformance();
|
|
4393
|
+
};
|
|
4394
|
+
resumeIfBound();
|
|
4395
|
+
room.on(RoomEvent.Reconnected, resumeIfBound);
|
|
4396
|
+
room.on(RoomEvent.ConnectionStateChanged, resumeIfBound);
|
|
4397
|
+
return () => {
|
|
4398
|
+
room.off(RoomEvent.Reconnected, resumeIfBound);
|
|
4399
|
+
room.off(RoomEvent.ConnectionStateChanged, resumeIfBound);
|
|
3817
4400
|
};
|
|
3818
|
-
}, [room,
|
|
4401
|
+
}, [room, assistant.agent, roomMatchesGrant, resumeExternalPerformance]);
|
|
3819
4402
|
useEffect(() => {
|
|
3820
4403
|
if (!registerDataPublisher) return;
|
|
3821
4404
|
const encoder = new TextEncoder();
|
|
@@ -3951,6 +4534,7 @@ function useRealtimeSession(input) {
|
|
|
3951
4534
|
onBehaviorChange,
|
|
3952
4535
|
onClipResult,
|
|
3953
4536
|
onVoiceError,
|
|
4537
|
+
correlatedAudio,
|
|
3954
4538
|
// everything else (client/active/idleSeconds/idleWarnLeadSeconds/reconnect/requestOptions…)
|
|
3955
4539
|
// flows straight into the inner SSOT hook.
|
|
3956
4540
|
...inner
|
|
@@ -4023,6 +4607,7 @@ function useRealtimeSession(input) {
|
|
|
4023
4607
|
return `t-${Date.now()}-${++turnSequenceRef.current}`;
|
|
4024
4608
|
}, []);
|
|
4025
4609
|
const onLifecycleData = useCallback((frame) => {
|
|
4610
|
+
lifecycle.onLifecycleData?.(frame);
|
|
4026
4611
|
const parsed = lifecycleServerFrameSchema.safeParse(frame);
|
|
4027
4612
|
if (!parsed.success) return;
|
|
4028
4613
|
const f = parsed.data;
|
|
@@ -4082,7 +4667,7 @@ function useRealtimeSession(input) {
|
|
|
4082
4667
|
break;
|
|
4083
4668
|
}
|
|
4084
4669
|
}
|
|
4085
|
-
}, []);
|
|
4670
|
+
}, [lifecycle.onLifecycleData]);
|
|
4086
4671
|
const registerDataPublisher = useCallback((publish) => {
|
|
4087
4672
|
dataPublisherRef.current = publish;
|
|
4088
4673
|
}, []);
|
|
@@ -4348,6 +4933,7 @@ function useRealtimeSession(input) {
|
|
|
4348
4933
|
// stayConnected/reconnect/markActivity/the sinks) → a true drop-in superset, then
|
|
4349
4934
|
// add the realtime-session surface. `reset` + `lifecycle` come AFTER so ours win.
|
|
4350
4935
|
...lifecycle,
|
|
4936
|
+
correlatedAudio,
|
|
4351
4937
|
turn,
|
|
4352
4938
|
clocks,
|
|
4353
4939
|
endsAt,
|
|
@@ -4382,6 +4968,7 @@ function useRealtimeSession(input) {
|
|
|
4382
4968
|
}),
|
|
4383
4969
|
[
|
|
4384
4970
|
lifecycle,
|
|
4971
|
+
correlatedAudio,
|
|
4385
4972
|
turn,
|
|
4386
4973
|
clocks,
|
|
4387
4974
|
endsAt,
|
|
@@ -4482,7 +5069,9 @@ function useAvatarCall(props) {
|
|
|
4482
5069
|
onIdleWarning: props.onQuiet ? ({ secondsLeft }) => props.onQuiet?.({ secondsLeft }) : void 0,
|
|
4483
5070
|
onCreditsLow: props.onLowBalance ? ({ secondsLeft }) => props.onLowBalance?.({ secondsLeft }) : void 0,
|
|
4484
5071
|
onEnded: props.onEnded ? ({ reason }) => props.onEnded?.({ reason }) : void 0,
|
|
4485
|
-
onVoiceError: props.onVoiceError ? (error) => props.onVoiceError?.(error) : void 0
|
|
5072
|
+
onVoiceError: props.onVoiceError ? (error) => props.onVoiceError?.(error) : void 0,
|
|
5073
|
+
externalPerformance: props.externalPerformance,
|
|
5074
|
+
correlatedAudio: props.correlatedAudio
|
|
4486
5075
|
});
|
|
4487
5076
|
sessionRef.current = session;
|
|
4488
5077
|
const call = handleFor(session);
|
|
@@ -4520,6 +5109,7 @@ function useAvatarCall(props) {
|
|
|
4520
5109
|
RealtimeAvatarLiveKitRoom,
|
|
4521
5110
|
{
|
|
4522
5111
|
grant: session.grant,
|
|
5112
|
+
renderRoomAudio: props.correlatedAudio === void 0,
|
|
4523
5113
|
// LiveKit captures the microphone on connect from this value, so a mute chosen while
|
|
4524
5114
|
// waiting is honoured rather than overridden when the call goes live.
|
|
4525
5115
|
audio: props.listen !== false && !session.microphoneMuted,
|
|
@@ -4919,4 +5509,4 @@ function RecordingPlayer({ assets, className }) {
|
|
|
4919
5509
|
] });
|
|
4920
5510
|
}
|
|
4921
5511
|
|
|
4922
|
-
export { AdaptivePlayoutController, AvatarCall, AvatarVideoSurface, DEFAULT_APPROACHING_END_LEAD_SECONDS, DEFAULT_AVATAR_PLAYOUT_DELAY_SECONDS, DEFAULT_AVATAR_UNSTABLE_STALL_MS, DEFAULT_CREDITS_LOW_LEAD_SECONDS, DEFAULT_FISH_TTS_MODEL, DEFAULT_GOVERNOR_CONFIG, DEFAULT_GRACE_CEILING_SECONDS, DEFAULT_GRACE_WINDOW_LEAD_SECONDS, DEFAULT_IDLE_SECONDS, DEFAULT_IDLE_WARN_LEAD_SECONDS, DEFAULT_READY_TIMEOUT_SECONDS, DEFAULT_TURN_TIMEOUT_SECONDS, FISH_DELIVERY_MODELS, MAX_RECONNECT_ATTEMPTS, MAX_SESSION_INSTRUCTIONS_CHARS, MIC_LEASE_ENDED_TIMEOUT_MS, NETWORK_QUALITY_POLICY, RECONNECT_BACKOFF_MS, RealtimeAvatarCapacityError, RealtimeAvatarLiveKitRoom, RecordingPlayer, SessionLifecycleRoomBridge, capacityErrorFromBusy, capacityStateFromGrant, createProxyClient, fishTtsModelSchema, knownBehaviorStates, mapTurnState, readInboundRtp, sessionBehaviorSchema, sessionClipSchema, shouldReplayPendingTurn, splitCallTranscript, useAvatarAdaptivePlayoutDelay, useAvatarCall, useAvatarCamera, useAvatarPlayoutDelay, useAvatarQualityGovernor, useCallTranscript, useCharacterTools, useLiveKitAvatarGrant, useMicLease, useRealtimeSession, useReleaseMicLeaseOnTrackEnded, useSessionLifecycle };
|
|
5512
|
+
export { AdaptivePlayoutController, AvatarCall, AvatarVideoSurface, DEFAULT_APPROACHING_END_LEAD_SECONDS, DEFAULT_AVATAR_PLAYOUT_DELAY_SECONDS, DEFAULT_AVATAR_UNSTABLE_STALL_MS, DEFAULT_CREDITS_LOW_LEAD_SECONDS, DEFAULT_FISH_TTS_MODEL, DEFAULT_GOVERNOR_CONFIG, DEFAULT_GRACE_CEILING_SECONDS, DEFAULT_GRACE_WINDOW_LEAD_SECONDS, DEFAULT_IDLE_SECONDS, DEFAULT_IDLE_WARN_LEAD_SECONDS, DEFAULT_READY_TIMEOUT_SECONDS, DEFAULT_TURN_TIMEOUT_SECONDS, EXTERNAL_PERFORMANCE_TIMING, FISH_DELIVERY_MODELS, KNOWN_EXTERNAL_PERFORMANCE_STATUSES, MAX_RECONNECT_ATTEMPTS, MAX_SESSION_INSTRUCTIONS_CHARS, MIC_LEASE_ENDED_TIMEOUT_MS, NETWORK_QUALITY_POLICY, RECONNECT_BACKOFF_MS, RealtimeAvatarCapacityError, RealtimeAvatarLiveKitRoom, RecordingPlayer, SessionLifecycleRoomBridge, capacityErrorFromBusy, capacityStateFromGrant, createProxyClient, fishTtsModelSchema, knownBehaviorStates, mapTurnState, readInboundRtp, sessionBehaviorSchema, sessionClipSchema, shouldReplayPendingTurn, splitCallTranscript, useAvatarAdaptivePlayoutDelay, useAvatarCall, useAvatarCamera, useAvatarPlayoutDelay, useAvatarQualityGovernor, useCallTranscript, useCharacterTools, useLiveKitAvatarGrant, useMicLease, useRealtimeSession, useReleaseMicLeaseOnTrackEnded, useSessionLifecycle };
|