realtime-avatar 0.32.0 → 0.34.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +56 -1
- package/dist/browser.d.ts +4 -89
- package/dist/browser.js +362 -1
- package/dist/chunk-6KQS7CTR.js +1 -0
- package/dist/{chunk-NKS5ZIWN.js → chunk-GPGNAQZV.js} +413 -170
- package/dist/{chunk-NDCZABOK.js → chunk-OVW4N5NH.js} +1 -13
- package/dist/express.d.ts +2 -2
- package/dist/express.js +2 -2
- package/dist/hono.d.ts +2 -2
- package/dist/hono.js +2 -2
- package/dist/index.d.ts +338 -3
- package/dist/index.js +2 -1
- package/dist/nextjs.d.ts +2 -2
- package/dist/nextjs.js +2 -2
- package/dist/{playout-delay-CtRZdtsj.d.ts → playout-delay-BzoM1BMN.d.ts} +135 -1
- package/dist/{proxy-client-BlaFxR6S.d.ts → proxy-client-CiTOjkg8.d.ts} +81 -3
- package/dist/react-native.d.ts +3 -3
- package/dist/react-native.js +624 -34
- package/dist/react.d.ts +12 -22
- package/dist/react.js +628 -35
- package/dist/server.d.ts +2 -2
- package/dist/server.js +2 -1
- package/dist/tanstack-start.d.ts +2 -2
- package/dist/tanstack-start.js +2 -2
- package/dist/{types-su27kEw0.d.ts → types-Cdi7KWWj.d.ts} +1 -1
- package/dist/{types-BSSqsu_Z.d.ts → types-Cg_i1jhk.d.ts} +209 -18
- package/package.json +3 -3
package/dist/react.js
CHANGED
|
@@ -1461,7 +1461,7 @@ function stepNetworkQuality(previous, sample, nowMs, options) {
|
|
|
1461
1461
|
}
|
|
1462
1462
|
|
|
1463
1463
|
// ../client/src/react/network-quality-binding.ts
|
|
1464
|
-
function bindNetworkQuality(publication, inhibited,
|
|
1464
|
+
function bindNetworkQuality(publication, inhibited, notify2, tickMs) {
|
|
1465
1465
|
let disposed = false;
|
|
1466
1466
|
let running = false;
|
|
1467
1467
|
let state = initNetworkQuality(performance.now());
|
|
@@ -1475,7 +1475,7 @@ function bindNetworkQuality(publication, inhibited, notify, tickMs) {
|
|
|
1475
1475
|
};
|
|
1476
1476
|
const emit = (status) => {
|
|
1477
1477
|
try {
|
|
1478
|
-
|
|
1478
|
+
notify2(status);
|
|
1479
1479
|
} catch {
|
|
1480
1480
|
}
|
|
1481
1481
|
};
|
|
@@ -2359,11 +2359,108 @@ var schema5 = z.object({
|
|
|
2359
2359
|
"request_id": schema22.optional()
|
|
2360
2360
|
});
|
|
2361
2361
|
var voiceErrorFrameSchema = schema5;
|
|
2362
|
-
var schema04 = z.
|
|
2363
|
-
var schema13 = z.
|
|
2364
|
-
var schema23 = z.
|
|
2365
|
-
var
|
|
2362
|
+
var schema04 = z.string().min(1).max(200);
|
|
2363
|
+
var schema13 = z.literal(1);
|
|
2364
|
+
var schema23 = z.string().min(1).max(200);
|
|
2365
|
+
var schema33 = z.int().min(1).max(9007199254740991);
|
|
2366
|
+
var schema43 = z.literal(24e3);
|
|
2367
|
+
var schema52 = z.literal("pcm_s16le");
|
|
2368
|
+
var schema6 = z.object({
|
|
2369
|
+
"session_id": schema04,
|
|
2370
|
+
"generation_id": schema04,
|
|
2371
|
+
"version": schema13,
|
|
2372
|
+
"utterance_id": schema23,
|
|
2373
|
+
"stream_id": schema04,
|
|
2374
|
+
"epoch": schema33,
|
|
2375
|
+
"sample_rate": schema43,
|
|
2376
|
+
"channels": schema13,
|
|
2377
|
+
"encoding": schema52
|
|
2378
|
+
});
|
|
2379
|
+
var schema7 = z.enum(["started", "progress", "finished", "cancelled"]);
|
|
2380
|
+
var schema8 = z.int().min(0).max(144e4);
|
|
2381
|
+
z.strictObject({
|
|
2382
|
+
"session_id": schema04,
|
|
2383
|
+
"generation_id": schema04,
|
|
2384
|
+
"utterance_id": schema23,
|
|
2385
|
+
"stream_id": schema04,
|
|
2386
|
+
"epoch": schema33,
|
|
2387
|
+
"status": schema7,
|
|
2388
|
+
"samples": schema8
|
|
2389
|
+
});
|
|
2390
|
+
var schema10 = z.object({
|
|
2391
|
+
"version": schema13,
|
|
2392
|
+
"session_id": schema04,
|
|
2393
|
+
"epoch": schema33
|
|
2394
|
+
});
|
|
2395
|
+
var schema11 = z.literal("external_llm_error");
|
|
2396
|
+
var schema122 = z.enum(["external_llm_refused", "external_llm_failed"]);
|
|
2397
|
+
var schema132 = z.string();
|
|
2398
|
+
var schema14 = z.literal(true);
|
|
2399
|
+
var schema15 = z.object({
|
|
2400
|
+
"kind": schema11,
|
|
2401
|
+
"code": schema122,
|
|
2402
|
+
"message": schema132,
|
|
2403
|
+
"fatal": schema14
|
|
2404
|
+
});
|
|
2405
|
+
var correlatedAudioHeaderSchema = schema6;
|
|
2406
|
+
var correlatedAudioCancelSchema = schema10;
|
|
2407
|
+
var externalLlmErrorFrameSchema = schema15;
|
|
2408
|
+
var CORRELATED_AUDIO_CONTRACT = {
|
|
2409
|
+
"topic": "rta.audio.v1",
|
|
2410
|
+
"receiptMethod": "rta.audio.receipt.v1",
|
|
2411
|
+
"cancelMethod": "rta.audio.cancel.v1",
|
|
2412
|
+
"maxSamples": 144e4,
|
|
2413
|
+
"maxChunkBytes": 48e3,
|
|
2414
|
+
"maxBufferedSamples": 48e3
|
|
2415
|
+
};
|
|
2416
|
+
var schema05 = z.string().min(1).max(200);
|
|
2417
|
+
var schema16 = z.literal(1);
|
|
2418
|
+
var schema24 = z.enum(["respond", "recovery", "initiative"]);
|
|
2419
|
+
var schema34 = z.string().min(1).max(32);
|
|
2420
|
+
var schema44 = z.union([schema24, schema34]);
|
|
2421
|
+
var schema53 = z.enum(["draft", "accepted"]);
|
|
2422
|
+
var schema62 = z.literal("external");
|
|
2423
|
+
var schema72 = z.object({
|
|
2424
|
+
"renderer": schema62
|
|
2425
|
+
});
|
|
2426
|
+
z.object({
|
|
2427
|
+
"session_id": schema05,
|
|
2428
|
+
"generation_id": schema05,
|
|
2429
|
+
"version": schema16,
|
|
2430
|
+
"purpose": schema44,
|
|
2431
|
+
"admission": schema53,
|
|
2432
|
+
"visible_character": schema72.optional()
|
|
2433
|
+
});
|
|
2434
|
+
var schema92 = z.string().min(1).max(200);
|
|
2435
|
+
z.object({
|
|
2436
|
+
"generation_id": schema05,
|
|
2437
|
+
"utterance_id": schema92
|
|
2438
|
+
});
|
|
2439
|
+
var schema112 = z.literal("external_performance");
|
|
2440
|
+
var schema123 = z.enum(["accepted", "dropped", "cancelled", "audio_released"]);
|
|
2441
|
+
var schema133 = z.union([schema123, schema34]);
|
|
2442
|
+
var schema142 = z.literal("unavailable");
|
|
2443
|
+
var schema152 = z.literal("unavailable");
|
|
2444
|
+
var schema162 = z.object({
|
|
2445
|
+
"session_id": schema05,
|
|
2446
|
+
"generation_id": schema05,
|
|
2447
|
+
"kind": schema112,
|
|
2448
|
+
"version": schema16,
|
|
2449
|
+
"status": schema133,
|
|
2450
|
+
"audible_checkpoint": schema142,
|
|
2451
|
+
"audio_mapping": schema152
|
|
2452
|
+
});
|
|
2453
|
+
var externalPerformanceFrameSchema = schema162;
|
|
2454
|
+
var schema06 = z.enum(["s1", "s2-pro", "s2.1-pro", "drama-3"]);
|
|
2455
|
+
var schema17 = z.enum(["speech-1.6", "speech-1.5", "s1-mini"]);
|
|
2456
|
+
var schema25 = z.union([schema06, schema17]);
|
|
2457
|
+
var schema35 = z.enum(["whisper"]);
|
|
2458
|
+
var schema45 = z.null();
|
|
2459
|
+
var schema54 = z.union([schema35, schema45]);
|
|
2460
|
+
var fishTtsModelSchema = schema25;
|
|
2366
2461
|
var DEFAULT_FISH_TTS_MODEL = "s2.1-pro";
|
|
2462
|
+
var fishDeliverySchema = schema54;
|
|
2463
|
+
var FISH_DELIVERY_MODELS = { "whisper": ["drama-3"] };
|
|
2367
2464
|
|
|
2368
2465
|
// ../client/src/wire.ts
|
|
2369
2466
|
var DEFAULT_AVATAR_ID = "maria";
|
|
@@ -2432,8 +2529,15 @@ var fishVoiceSpecSchema = z.object({
|
|
|
2432
2529
|
voice_id: z.string().min(1).max(120),
|
|
2433
2530
|
speed: z.number().min(0.5).max(2).nullable().optional(),
|
|
2434
2531
|
emotion: z.string().min(1).max(80).nullable().optional(),
|
|
2435
|
-
language: z.string().min(2).max(16).nullable().optional()
|
|
2436
|
-
|
|
2532
|
+
language: z.string().min(2).max(16).nullable().optional(),
|
|
2533
|
+
delivery: fishDeliverySchema.optional()
|
|
2534
|
+
}).strict().superRefine((voice, ctx) => {
|
|
2535
|
+
if (!voice.delivery) return;
|
|
2536
|
+
const models = FISH_DELIVERY_MODELS[voice.delivery];
|
|
2537
|
+
if (!models.includes(voice.model)) {
|
|
2538
|
+
ctx.addIssue({ code: "custom", path: ["delivery"], message: `delivery "${voice.delivery}" needs model ${models.join(" | ")}` });
|
|
2539
|
+
}
|
|
2540
|
+
});
|
|
2437
2541
|
var voiceSpecSchema = z.discriminatedUnion("provider", [
|
|
2438
2542
|
cartesiaVoiceSpecSchema,
|
|
2439
2543
|
breezeVoiceSpecSchema,
|
|
@@ -2671,7 +2775,9 @@ var lifecycleServerFrameSchema = z.discriminatedUnion("kind", [
|
|
|
2671
2775
|
endedFrameSchema,
|
|
2672
2776
|
behaviorStateFrameSchema,
|
|
2673
2777
|
clipAckFrameSchema,
|
|
2674
|
-
voiceErrorFrameSchema2
|
|
2778
|
+
voiceErrorFrameSchema2,
|
|
2779
|
+
externalPerformanceFrameSchema,
|
|
2780
|
+
externalLlmErrorFrameSchema
|
|
2675
2781
|
]);
|
|
2676
2782
|
var liveKitCapacitySnapshotSchema = z.object({
|
|
2677
2783
|
// Placement identity + per-worker session ceiling. These are always present on the
|
|
@@ -2953,6 +3059,7 @@ function useCallMedia(room, options) {
|
|
|
2953
3059
|
}, [room, setAudioPlayback, connecting]);
|
|
2954
3060
|
const controls = useMemo(() => ({
|
|
2955
3061
|
startAudio: async () => {
|
|
3062
|
+
if (options.startAudioOverride) return options.startAudioOverride();
|
|
2956
3063
|
try {
|
|
2957
3064
|
await room.startAudio();
|
|
2958
3065
|
} catch {
|
|
@@ -2969,13 +3076,372 @@ function useCallMedia(room, options) {
|
|
|
2969
3076
|
setVersion((value) => value + 1);
|
|
2970
3077
|
}
|
|
2971
3078
|
}
|
|
2972
|
-
}), [room]);
|
|
3079
|
+
}), [room, options.startAudioOverride]);
|
|
2973
3080
|
useEffect(() => {
|
|
2974
3081
|
if (!registerMediaControls) return;
|
|
2975
3082
|
registerMediaControls(controls);
|
|
2976
3083
|
return () => registerMediaControls(null);
|
|
2977
3084
|
}, [controls, registerMediaControls]);
|
|
2978
3085
|
}
|
|
3086
|
+
var attachments = /* @__PURE__ */ new WeakSet();
|
|
3087
|
+
var bindings = /* @__PURE__ */ new WeakMap();
|
|
3088
|
+
function notify(callback) {
|
|
3089
|
+
try {
|
|
3090
|
+
void Promise.resolve(callback()).catch(() => {
|
|
3091
|
+
});
|
|
3092
|
+
} catch {
|
|
3093
|
+
}
|
|
3094
|
+
}
|
|
3095
|
+
function attachCorrelatedAudio(room, options) {
|
|
3096
|
+
if (attachments.has(room)) throw new Error("Correlated audio is already attached to this room");
|
|
3097
|
+
if (!options.sessionId || !options.agentIdentity) throw new Error("Correlated audio requires the held session and bound agent");
|
|
3098
|
+
attachments.add(room);
|
|
3099
|
+
let context;
|
|
3100
|
+
let disposed = false;
|
|
3101
|
+
const previous = bindings.get(room);
|
|
3102
|
+
const binding = previous?.sessionId === options.sessionId ? previous : { sessionId: options.sessionId, epoch: 1, unavailable: false, seen: /* @__PURE__ */ new Set() };
|
|
3103
|
+
bindings.set(room, binding);
|
|
3104
|
+
const sources = [];
|
|
3105
|
+
const active = (source) => sources.includes(source);
|
|
3106
|
+
const buffered = () => sources.reduce((sum, source) => sum + source.received - source.consumed, 0);
|
|
3107
|
+
let nextStart = 0;
|
|
3108
|
+
let blocked;
|
|
3109
|
+
let closing;
|
|
3110
|
+
const seen = binding.seen;
|
|
3111
|
+
const waiters = /* @__PURE__ */ new Set();
|
|
3112
|
+
const wake = () => {
|
|
3113
|
+
for (const resolve of waiters) resolve();
|
|
3114
|
+
waiters.clear();
|
|
3115
|
+
};
|
|
3116
|
+
const identity = (header) => ({
|
|
3117
|
+
session_id: header.session_id,
|
|
3118
|
+
generation_id: header.generation_id,
|
|
3119
|
+
utterance_id: header.utterance_id,
|
|
3120
|
+
stream_id: header.stream_id,
|
|
3121
|
+
epoch: header.epoch
|
|
3122
|
+
});
|
|
3123
|
+
const position = (current, state) => {
|
|
3124
|
+
notify(() => options.onPosition({ ...identity(current.header), samples: current.consumed, sampleRate: current.header.sample_rate, state }));
|
|
3125
|
+
};
|
|
3126
|
+
const receipt = (current, status) => {
|
|
3127
|
+
const payload = { ...identity(current.header), status, samples: current.consumed };
|
|
3128
|
+
if (current.rpcBusy) {
|
|
3129
|
+
current.pendingReceipt = payload;
|
|
3130
|
+
return;
|
|
3131
|
+
}
|
|
3132
|
+
const send = async (next) => {
|
|
3133
|
+
current.rpcBusy = true;
|
|
3134
|
+
try {
|
|
3135
|
+
await room.localParticipant.performRpc({
|
|
3136
|
+
destinationIdentity: options.agentIdentity,
|
|
3137
|
+
method: CORRELATED_AUDIO_CONTRACT.receiptMethod,
|
|
3138
|
+
payload: JSON.stringify(next),
|
|
3139
|
+
responseTimeout: 2e3
|
|
3140
|
+
});
|
|
3141
|
+
} catch {
|
|
3142
|
+
if (!disposed && next.epoch === binding.epoch) invalidate("connection_lost");
|
|
3143
|
+
} finally {
|
|
3144
|
+
current.rpcBusy = false;
|
|
3145
|
+
const pending = current.pendingReceipt;
|
|
3146
|
+
current.pendingReceipt = void 0;
|
|
3147
|
+
if (pending && !disposed && pending.epoch === binding.epoch && !binding.unavailable) void send(pending);
|
|
3148
|
+
}
|
|
3149
|
+
};
|
|
3150
|
+
void send(payload);
|
|
3151
|
+
};
|
|
3152
|
+
const clear = () => {
|
|
3153
|
+
for (const current of sources.splice(0)) {
|
|
3154
|
+
current.abort.abort();
|
|
3155
|
+
for (const segment of current.segments) {
|
|
3156
|
+
segment.node.onended = null;
|
|
3157
|
+
segment.node.stop();
|
|
3158
|
+
segment.node.disconnect();
|
|
3159
|
+
}
|
|
3160
|
+
current.segments.length = 0;
|
|
3161
|
+
}
|
|
3162
|
+
nextStart = context?.currentTime ?? 0;
|
|
3163
|
+
wake();
|
|
3164
|
+
};
|
|
3165
|
+
const invalidate = (reason) => {
|
|
3166
|
+
if (disposed) return;
|
|
3167
|
+
if (reason === "connection_lost") {
|
|
3168
|
+
if (binding.unavailable) return;
|
|
3169
|
+
binding.unavailable = true;
|
|
3170
|
+
}
|
|
3171
|
+
if (reason === "invalid_stream" || reason === "disposed") for (const current of sources) receipt(current, "cancelled");
|
|
3172
|
+
clear();
|
|
3173
|
+
notify(() => options.onInvalidated(reason));
|
|
3174
|
+
};
|
|
3175
|
+
const tick = (reportCredit = true) => {
|
|
3176
|
+
if (!context || context.state !== "running" || disposed || binding.unavailable) return;
|
|
3177
|
+
for (const current of [...sources]) {
|
|
3178
|
+
if (!active(current)) continue;
|
|
3179
|
+
let consumed = current.consumed;
|
|
3180
|
+
for (const segment of current.segments) {
|
|
3181
|
+
if (context.currentTime < segment.start) break;
|
|
3182
|
+
const samples = Math.min(segment.length, Math.max(0, Math.floor((context.currentTime - segment.start) * current.header.sample_rate + 1e-6)));
|
|
3183
|
+
consumed = Math.max(consumed, segment.offset + samples);
|
|
3184
|
+
if (!current.started && samples > 0) {
|
|
3185
|
+
current.started = true;
|
|
3186
|
+
receipt(current, "started");
|
|
3187
|
+
position(current, "playing");
|
|
3188
|
+
current.lastPosition = current.consumed;
|
|
3189
|
+
}
|
|
3190
|
+
}
|
|
3191
|
+
current.consumed = consumed;
|
|
3192
|
+
while (current.segments.length && current.segments[0].offset + current.segments[0].length <= consumed) {
|
|
3193
|
+
const segment = current.segments.shift();
|
|
3194
|
+
if (segment) {
|
|
3195
|
+
segment.node.onended = null;
|
|
3196
|
+
segment.node.disconnect();
|
|
3197
|
+
}
|
|
3198
|
+
}
|
|
3199
|
+
if (current.started && current.eof && consumed === current.received && consumed > 0) {
|
|
3200
|
+
position(current, "finished");
|
|
3201
|
+
if (!active(current)) continue;
|
|
3202
|
+
receipt(current, "finished");
|
|
3203
|
+
sources.splice(sources.indexOf(current), 1);
|
|
3204
|
+
wake();
|
|
3205
|
+
} else {
|
|
3206
|
+
if (current.started && consumed > current.lastPosition) {
|
|
3207
|
+
current.lastPosition = consumed;
|
|
3208
|
+
position(current, "playing");
|
|
3209
|
+
}
|
|
3210
|
+
if (!active(current)) continue;
|
|
3211
|
+
if (reportCredit && consumed > current.lastReported) {
|
|
3212
|
+
current.lastReported = consumed;
|
|
3213
|
+
receipt(current, "progress");
|
|
3214
|
+
}
|
|
3215
|
+
wake();
|
|
3216
|
+
}
|
|
3217
|
+
}
|
|
3218
|
+
};
|
|
3219
|
+
const playbackState = () => {
|
|
3220
|
+
const next = context?.state !== "running";
|
|
3221
|
+
if (next !== blocked) {
|
|
3222
|
+
blocked = next;
|
|
3223
|
+
notify(() => options.onPlaybackBlocked?.(next ? startAudio : null));
|
|
3224
|
+
}
|
|
3225
|
+
tick();
|
|
3226
|
+
};
|
|
3227
|
+
const getContext = () => {
|
|
3228
|
+
if (!context) {
|
|
3229
|
+
context = new AudioContext({ sampleRate: correlatedAudioHeaderSchema.shape.sample_rate.value });
|
|
3230
|
+
context.onstatechange = playbackState;
|
|
3231
|
+
playbackState();
|
|
3232
|
+
}
|
|
3233
|
+
return context;
|
|
3234
|
+
};
|
|
3235
|
+
async function startAudio() {
|
|
3236
|
+
if (disposed || binding.unavailable) throw new Error("Correlated audio binding is no longer available");
|
|
3237
|
+
const ctx = getContext();
|
|
3238
|
+
await ctx.resume();
|
|
3239
|
+
playbackState();
|
|
3240
|
+
if (ctx.state !== "running") throw new Error("Correlated audio playback is blocked");
|
|
3241
|
+
}
|
|
3242
|
+
async function receive(reader, participant) {
|
|
3243
|
+
if (disposed || binding.unavailable || room.state !== "connected" || participant.identity !== options.agentIdentity) return;
|
|
3244
|
+
let value;
|
|
3245
|
+
try {
|
|
3246
|
+
value = JSON.parse(reader.info.attributes?.["rta.audio"] ?? "");
|
|
3247
|
+
} catch {
|
|
3248
|
+
return;
|
|
3249
|
+
}
|
|
3250
|
+
const parsed = correlatedAudioHeaderSchema.safeParse(value);
|
|
3251
|
+
if (!parsed.success || parsed.data.session_id !== options.sessionId || parsed.data.epoch < binding.epoch) return;
|
|
3252
|
+
const header = parsed.data;
|
|
3253
|
+
if (header.stream_id !== reader.info.id) return;
|
|
3254
|
+
if (header.epoch > binding.epoch) {
|
|
3255
|
+
binding.epoch = header.epoch;
|
|
3256
|
+
invalidate("cancelled");
|
|
3257
|
+
}
|
|
3258
|
+
const key = `${header.epoch}/${header.stream_id}`;
|
|
3259
|
+
if (seen.has(key)) return;
|
|
3260
|
+
if (sources.length >= 128 || seen.size >= 1e4) {
|
|
3261
|
+
invalidate("invalid_stream");
|
|
3262
|
+
return;
|
|
3263
|
+
}
|
|
3264
|
+
seen.add(key);
|
|
3265
|
+
const previousSource = sources.at(-1);
|
|
3266
|
+
const current = { header, abort: new AbortController(), segments: [], received: 0, consumed: 0, started: false, eof: false, lastReported: 0, lastPosition: -1, rpcBusy: false };
|
|
3267
|
+
sources.push(current);
|
|
3268
|
+
let carry;
|
|
3269
|
+
try {
|
|
3270
|
+
if (options.prepare) await options.prepare(header, current.abort.signal);
|
|
3271
|
+
if (!active(current) || current.abort.signal.aborted) return;
|
|
3272
|
+
const ctx = getContext();
|
|
3273
|
+
while (active(current) && (previousSource && active(previousSource) && !previousSource.eof || buffered() > CORRELATED_AUDIO_CONTRACT.maxBufferedSamples - CORRELATED_AUDIO_CONTRACT.maxChunkBytes / 2)) {
|
|
3274
|
+
await new Promise((resolve) => waiters.add(resolve));
|
|
3275
|
+
}
|
|
3276
|
+
if (!active(current)) return;
|
|
3277
|
+
for await (const chunk of reader.withAbortSignal(current.abort.signal)) {
|
|
3278
|
+
if (!active(current)) break;
|
|
3279
|
+
if (chunk.byteLength > CORRELATED_AUDIO_CONTRACT.maxChunkBytes) throw new Error("PCM chunk exceeds contract");
|
|
3280
|
+
const count = Math.floor((chunk.byteLength + (carry === void 0 ? 0 : 1)) / 2);
|
|
3281
|
+
if (current.received + count > CORRELATED_AUDIO_CONTRACT.maxSamples) throw new Error("PCM stream exceeds contract");
|
|
3282
|
+
while (active(current) && buffered() + count > CORRELATED_AUDIO_CONTRACT.maxBufferedSamples) {
|
|
3283
|
+
await new Promise((resolve) => waiters.add(resolve));
|
|
3284
|
+
}
|
|
3285
|
+
if (!active(current)) break;
|
|
3286
|
+
if (!count) {
|
|
3287
|
+
if (chunk.length) carry = chunk[0];
|
|
3288
|
+
continue;
|
|
3289
|
+
}
|
|
3290
|
+
const buffer = ctx.createBuffer(header.channels, count, header.sample_rate);
|
|
3291
|
+
const data = buffer.getChannelData(0);
|
|
3292
|
+
let cursor = 0;
|
|
3293
|
+
for (let i = 0; i < count; i++) {
|
|
3294
|
+
const low = carry ?? chunk[cursor++];
|
|
3295
|
+
carry = void 0;
|
|
3296
|
+
const raw = low | chunk[cursor++] << 8;
|
|
3297
|
+
data[i] = (raw >= 32768 ? raw - 65536 : raw) / 32768;
|
|
3298
|
+
}
|
|
3299
|
+
if (cursor < chunk.length) carry = chunk[cursor];
|
|
3300
|
+
const node = ctx.createBufferSource();
|
|
3301
|
+
node.buffer = buffer;
|
|
3302
|
+
node.connect(ctx.destination);
|
|
3303
|
+
const start = Math.max(ctx.currentTime, nextStart);
|
|
3304
|
+
current.segments.push({ node, start, offset: current.received, length: count });
|
|
3305
|
+
current.received += count;
|
|
3306
|
+
nextStart = start + count / header.sample_rate;
|
|
3307
|
+
node.onended = () => tick();
|
|
3308
|
+
node.start(start);
|
|
3309
|
+
while (active(current) && buffered() > CORRELATED_AUDIO_CONTRACT.maxBufferedSamples - CORRELATED_AUDIO_CONTRACT.maxChunkBytes / 2) {
|
|
3310
|
+
await new Promise((resolve) => waiters.add(resolve));
|
|
3311
|
+
}
|
|
3312
|
+
}
|
|
3313
|
+
if (!active(current)) return;
|
|
3314
|
+
if (carry !== void 0 || current.received === 0) throw new Error("Incomplete PCM stream");
|
|
3315
|
+
current.eof = true;
|
|
3316
|
+
wake();
|
|
3317
|
+
tick(false);
|
|
3318
|
+
} catch {
|
|
3319
|
+
if (active(current)) invalidate("invalid_stream");
|
|
3320
|
+
}
|
|
3321
|
+
}
|
|
3322
|
+
const cancel = async (data) => {
|
|
3323
|
+
if (disposed || binding.unavailable || room.state !== "connected" || data.callerIdentity !== options.agentIdentity) throw new Error("Unbound audio cancellation");
|
|
3324
|
+
const value = JSON.parse(data.payload);
|
|
3325
|
+
const frame2 = correlatedAudioCancelSchema.parse(value);
|
|
3326
|
+
if (frame2.session_id !== options.sessionId) throw new Error("Wrong audio session");
|
|
3327
|
+
if (frame2.epoch <= binding.epoch) return "";
|
|
3328
|
+
binding.epoch = frame2.epoch;
|
|
3329
|
+
invalidate("cancelled");
|
|
3330
|
+
return "";
|
|
3331
|
+
};
|
|
3332
|
+
const lost = () => invalidate("connection_lost");
|
|
3333
|
+
const changed = () => {
|
|
3334
|
+
if (["disconnected", "reconnecting", "signalReconnecting"].includes(room.state)) lost();
|
|
3335
|
+
};
|
|
3336
|
+
const participantLeft = (participant) => {
|
|
3337
|
+
if (participant.identity === options.agentIdentity) lost();
|
|
3338
|
+
};
|
|
3339
|
+
let registered = false;
|
|
3340
|
+
try {
|
|
3341
|
+
room.registerByteStreamHandler(CORRELATED_AUDIO_CONTRACT.topic, (reader, participant) => {
|
|
3342
|
+
void receive(reader, participant);
|
|
3343
|
+
});
|
|
3344
|
+
registered = true;
|
|
3345
|
+
room.localParticipant.registerRpcMethod(CORRELATED_AUDIO_CONTRACT.cancelMethod, cancel);
|
|
3346
|
+
} catch (error) {
|
|
3347
|
+
if (registered) room.unregisterByteStreamHandler(CORRELATED_AUDIO_CONTRACT.topic);
|
|
3348
|
+
attachments.delete(room);
|
|
3349
|
+
throw error;
|
|
3350
|
+
}
|
|
3351
|
+
room.on(RoomEvent.Reconnecting, lost);
|
|
3352
|
+
room.on(RoomEvent.SignalReconnecting, lost);
|
|
3353
|
+
room.on(RoomEvent.Disconnected, lost);
|
|
3354
|
+
room.on(RoomEvent.ConnectionStateChanged, changed);
|
|
3355
|
+
room.on(RoomEvent.ParticipantDisconnected, participantLeft);
|
|
3356
|
+
const timer = setInterval(() => tick(), 100);
|
|
3357
|
+
let frame = 0;
|
|
3358
|
+
const render = () => {
|
|
3359
|
+
if (disposed) return;
|
|
3360
|
+
tick(false);
|
|
3361
|
+
frame = requestAnimationFrame(render);
|
|
3362
|
+
};
|
|
3363
|
+
frame = requestAnimationFrame(render);
|
|
3364
|
+
return {
|
|
3365
|
+
startAudio,
|
|
3366
|
+
dispose() {
|
|
3367
|
+
if (closing) return closing;
|
|
3368
|
+
invalidate("disposed");
|
|
3369
|
+
disposed = true;
|
|
3370
|
+
clearInterval(timer);
|
|
3371
|
+
cancelAnimationFrame(frame);
|
|
3372
|
+
room.unregisterByteStreamHandler(CORRELATED_AUDIO_CONTRACT.topic);
|
|
3373
|
+
room.localParticipant.unregisterRpcMethod(CORRELATED_AUDIO_CONTRACT.cancelMethod);
|
|
3374
|
+
room.off(RoomEvent.Reconnecting, lost);
|
|
3375
|
+
room.off(RoomEvent.SignalReconnecting, lost);
|
|
3376
|
+
room.off(RoomEvent.Disconnected, lost);
|
|
3377
|
+
room.off(RoomEvent.ConnectionStateChanged, changed);
|
|
3378
|
+
room.off(RoomEvent.ParticipantDisconnected, participantLeft);
|
|
3379
|
+
attachments.delete(room);
|
|
3380
|
+
if (context) context.onstatechange = null;
|
|
3381
|
+
closing = context?.close().catch(() => {
|
|
3382
|
+
}) ?? Promise.resolve();
|
|
3383
|
+
return closing;
|
|
3384
|
+
}
|
|
3385
|
+
};
|
|
3386
|
+
}
|
|
3387
|
+
|
|
3388
|
+
// ../client/src/external-performance.ts
|
|
3389
|
+
var EXTERNAL_PERFORMANCE_KIND = externalPerformanceFrameSchema.shape.kind.value;
|
|
3390
|
+
var EXTERNAL_PERFORMANCE_TIMING = Object.freeze({
|
|
3391
|
+
receiverPlayout: "unsupported",
|
|
3392
|
+
audibleCheckpoint: externalPerformanceFrameSchema.shape.audible_checkpoint.value,
|
|
3393
|
+
audioMapping: externalPerformanceFrameSchema.shape.audio_mapping.value
|
|
3394
|
+
});
|
|
3395
|
+
var statusSchema = externalPerformanceFrameSchema.shape.status;
|
|
3396
|
+
var KNOWN_EXTERNAL_PERFORMANCE_STATUSES = Object.freeze([...statusSchema.options[0].options]);
|
|
3397
|
+
var knownStatuses = new Set(KNOWN_EXTERNAL_PERFORMANCE_STATUSES);
|
|
3398
|
+
function isObservable(frame) {
|
|
3399
|
+
return knownStatuses.has(frame.status);
|
|
3400
|
+
}
|
|
3401
|
+
var NEXT = {
|
|
3402
|
+
unseen: /* @__PURE__ */ new Set(["accepted", "dropped"]),
|
|
3403
|
+
accepted: /* @__PURE__ */ new Set(["audio_released", "cancelled"]),
|
|
3404
|
+
audio_released: /* @__PURE__ */ new Set(["cancelled"]),
|
|
3405
|
+
dropped: /* @__PURE__ */ new Set(),
|
|
3406
|
+
cancelled: /* @__PURE__ */ new Set(),
|
|
3407
|
+
retired: /* @__PURE__ */ new Set()
|
|
3408
|
+
};
|
|
3409
|
+
var LEDGER_LIMIT = 4096;
|
|
3410
|
+
function deliver(callback) {
|
|
3411
|
+
try {
|
|
3412
|
+
void Promise.resolve(callback()).catch(() => {
|
|
3413
|
+
});
|
|
3414
|
+
} catch {
|
|
3415
|
+
}
|
|
3416
|
+
}
|
|
3417
|
+
function createExternalPerformanceObserver(sessionId, callbacks) {
|
|
3418
|
+
const generations = /* @__PURE__ */ new Map();
|
|
3419
|
+
let state = "live";
|
|
3420
|
+
return {
|
|
3421
|
+
receive(frame) {
|
|
3422
|
+
if (state !== "live" || frame.session_id !== sessionId || !isObservable(frame)) return;
|
|
3423
|
+
const previous = generations.get(frame.generation_id) ?? "unseen";
|
|
3424
|
+
if (!NEXT[previous].has(frame.status)) return;
|
|
3425
|
+
if (previous === "unseen" && generations.size >= LEDGER_LIMIT) {
|
|
3426
|
+
const oldest = generations.keys().next();
|
|
3427
|
+
if (!oldest.done) generations.delete(oldest.value);
|
|
3428
|
+
}
|
|
3429
|
+
generations.set(frame.generation_id, frame.status);
|
|
3430
|
+
deliver(() => callbacks.onFrame(frame));
|
|
3431
|
+
},
|
|
3432
|
+
invalidate(reason) {
|
|
3433
|
+
if (state === "closed" || state === "paused" && reason === "connection_lost") return;
|
|
3434
|
+
state = reason === "connection_lost" ? "paused" : "closed";
|
|
3435
|
+
if (state === "closed") generations.clear();
|
|
3436
|
+
else for (const id of generations.keys()) generations.set(id, "retired");
|
|
3437
|
+
deliver(() => callbacks.onInvalidated(reason));
|
|
3438
|
+
},
|
|
3439
|
+
/** The call is connected to its agent again. Only a connection loss can be resumed. */
|
|
3440
|
+
resume() {
|
|
3441
|
+
if (state === "paused") state = "live";
|
|
3442
|
+
}
|
|
3443
|
+
};
|
|
3444
|
+
}
|
|
2979
3445
|
|
|
2980
3446
|
// ../client/src/react/session-lifecycle.ts
|
|
2981
3447
|
function phaseForReason(reason) {
|
|
@@ -3171,6 +3637,32 @@ function useSessionLifecycle(input) {
|
|
|
3171
3637
|
const grantIdRef = useRef(null);
|
|
3172
3638
|
grantIdRef.current = grantId;
|
|
3173
3639
|
const retiredGrantRef = useRef(null);
|
|
3640
|
+
const performanceEnabled = input.externalPerformance !== void 0;
|
|
3641
|
+
const performanceCallbacksRef = useRef(input.externalPerformance);
|
|
3642
|
+
if (input.externalPerformance) performanceCallbacksRef.current = input.externalPerformance;
|
|
3643
|
+
const performanceRef = useRef(null);
|
|
3644
|
+
const endedPerformanceGrantRef = useRef(null);
|
|
3645
|
+
useEffect(() => {
|
|
3646
|
+
if (!grantId || !performanceEnabled || endedPerformanceGrantRef.current === grantId) return;
|
|
3647
|
+
const observer = createExternalPerformanceObserver(grantId, {
|
|
3648
|
+
onFrame: (frame) => performanceCallbacksRef.current?.onFrame(frame),
|
|
3649
|
+
onInvalidated: (reason) => {
|
|
3650
|
+
if (reason === "ended") endedPerformanceGrantRef.current = grantId;
|
|
3651
|
+
performanceCallbacksRef.current?.onInvalidated(reason);
|
|
3652
|
+
}
|
|
3653
|
+
});
|
|
3654
|
+
performanceRef.current = observer;
|
|
3655
|
+
return () => {
|
|
3656
|
+
observer.invalidate("binding_changed");
|
|
3657
|
+
if (performanceRef.current === observer) performanceRef.current = null;
|
|
3658
|
+
};
|
|
3659
|
+
}, [grantId, performanceEnabled]);
|
|
3660
|
+
const invalidateExternalPerformance = useCallback((reason) => {
|
|
3661
|
+
if (grantId === grantIdRef.current) performanceRef.current?.invalidate(reason);
|
|
3662
|
+
}, [grantId]);
|
|
3663
|
+
const resumeExternalPerformance = useCallback(() => {
|
|
3664
|
+
if (grantId === grantIdRef.current) performanceRef.current?.resume();
|
|
3665
|
+
}, [grantId]);
|
|
3174
3666
|
const [recovery, setRecovery] = useState({ kind: "connected" });
|
|
3175
3667
|
const recoveryRef = useRef(recovery);
|
|
3176
3668
|
recoveryRef.current = recovery;
|
|
@@ -3216,6 +3708,7 @@ function useSessionLifecycle(input) {
|
|
|
3216
3708
|
}
|
|
3217
3709
|
}, []);
|
|
3218
3710
|
const reset = useCallback(() => {
|
|
3711
|
+
invalidateExternalPerformance("binding_changed");
|
|
3219
3712
|
clearTimer();
|
|
3220
3713
|
connectedRef.current = false;
|
|
3221
3714
|
manualReconnectPendingRef.current = false;
|
|
@@ -3230,7 +3723,7 @@ function useSessionLifecycle(input) {
|
|
|
3230
3723
|
setConnected(false);
|
|
3231
3724
|
setAgentPresentState(false);
|
|
3232
3725
|
lastActivityRef.current = Date.now();
|
|
3233
|
-
}, [clearTimer]);
|
|
3726
|
+
}, [clearTimer, invalidateExternalPerformance]);
|
|
3234
3727
|
useEffect(() => {
|
|
3235
3728
|
if (active) return;
|
|
3236
3729
|
userEndedRef.current = false;
|
|
@@ -3264,6 +3757,7 @@ function useSessionLifecycle(input) {
|
|
|
3264
3757
|
}, [grantId]);
|
|
3265
3758
|
const endIdle = useCallback(() => {
|
|
3266
3759
|
if (!connectedRef.current) return;
|
|
3760
|
+
invalidateExternalPerformance("ended");
|
|
3267
3761
|
clearTimer();
|
|
3268
3762
|
connectedRef.current = false;
|
|
3269
3763
|
setConnected(false);
|
|
@@ -3271,7 +3765,7 @@ function useSessionLifecycle(input) {
|
|
|
3271
3765
|
releaseRef.current("idle_timeout");
|
|
3272
3766
|
setRecovery({ kind: "ended", reason: "idle" });
|
|
3273
3767
|
leaveRoomRef.current?.();
|
|
3274
|
-
}, [clearTimer]);
|
|
3768
|
+
}, [clearTimer, invalidateExternalPerformance]);
|
|
3275
3769
|
const onConnected = useCallback(() => {
|
|
3276
3770
|
if (grantId === null || grantIdRef.current !== grantId) return;
|
|
3277
3771
|
if (userEndedRef.current || readinessFailureRef.current || retiredGrantRef.current !== null && retiredGrantRef.current === grantIdRef.current) return;
|
|
@@ -3289,6 +3783,7 @@ function useSessionLifecycle(input) {
|
|
|
3289
3783
|
if (grantId === null || grantIdRef.current !== grantId) return;
|
|
3290
3784
|
if (!active || userEndedRef.current || readinessFailureRef.current) return;
|
|
3291
3785
|
const action = disconnectAction(reason, connectedRef.current);
|
|
3786
|
+
invalidateExternalPerformance(action === "end" ? "ended" : "connection_lost");
|
|
3292
3787
|
if (action === "noop") return;
|
|
3293
3788
|
if (action === "reset") {
|
|
3294
3789
|
if (recoveryRef.current.kind === "refreshing" || recoveryRef.current.kind === "reconnecting") return;
|
|
@@ -3308,12 +3803,13 @@ function useSessionLifecycle(input) {
|
|
|
3308
3803
|
refreshBaselineCapacityRef.current = null;
|
|
3309
3804
|
setRecovery({ kind: "reconnecting", attempt: attemptRef.current });
|
|
3310
3805
|
},
|
|
3311
|
-
[active, reset, grantId]
|
|
3806
|
+
[active, reset, grantId, invalidateExternalPerformance]
|
|
3312
3807
|
);
|
|
3313
3808
|
const onConnectionError = useCallback(
|
|
3314
3809
|
(error) => {
|
|
3315
3810
|
if (grantId === null || grantIdRef.current !== grantId) return;
|
|
3316
3811
|
if (!active || userEndedRef.current || readinessFailureRef.current || !isRecoverableConnectionError(error)) return;
|
|
3812
|
+
invalidateExternalPerformance("connection_lost");
|
|
3317
3813
|
connectedRef.current = false;
|
|
3318
3814
|
setConnected(false);
|
|
3319
3815
|
setAgentPresentState(false);
|
|
@@ -3324,7 +3820,7 @@ function useSessionLifecycle(input) {
|
|
|
3324
3820
|
return { kind: "reconnecting", attempt: attemptRef.current };
|
|
3325
3821
|
});
|
|
3326
3822
|
},
|
|
3327
|
-
[active, grantId]
|
|
3823
|
+
[active, grantId, invalidateExternalPerformance]
|
|
3328
3824
|
);
|
|
3329
3825
|
useEffect(() => {
|
|
3330
3826
|
if (capacity.kind === "error" && recovery.kind === "connected") manualReconnectPendingRef.current = false;
|
|
@@ -3344,6 +3840,9 @@ function useSessionLifecycle(input) {
|
|
|
3344
3840
|
onConnected();
|
|
3345
3841
|
return;
|
|
3346
3842
|
}
|
|
3843
|
+
if (state === "disconnected" && connectedRef.current || state === "reconnecting" || state === "signalReconnecting") {
|
|
3844
|
+
invalidateExternalPerformance("connection_lost");
|
|
3845
|
+
}
|
|
3347
3846
|
if (state === "reconnecting" || state === "signalReconnecting") {
|
|
3348
3847
|
if (connectedRef.current) {
|
|
3349
3848
|
setConnected(false);
|
|
@@ -3353,7 +3852,7 @@ function useSessionLifecycle(input) {
|
|
|
3353
3852
|
}
|
|
3354
3853
|
}
|
|
3355
3854
|
},
|
|
3356
|
-
[active, onConnected, grantId]
|
|
3855
|
+
[active, onConnected, grantId, invalidateExternalPerformance]
|
|
3357
3856
|
);
|
|
3358
3857
|
useEffect(() => {
|
|
3359
3858
|
if (!active || userEnded || readinessFailed || !connected || !agentPresent || !grantState.grant) return;
|
|
@@ -3366,6 +3865,7 @@ function useSessionLifecycle(input) {
|
|
|
3366
3865
|
const failReadiness = useCallback((code) => {
|
|
3367
3866
|
if (userEndedRef.current || readinessFailureRef.current) return;
|
|
3368
3867
|
readinessFailureRef.current = code;
|
|
3868
|
+
invalidateExternalPerformance("ended");
|
|
3369
3869
|
clearTimer();
|
|
3370
3870
|
connectedRef.current = false;
|
|
3371
3871
|
manualReconnectPendingRef.current = false;
|
|
@@ -3378,7 +3878,7 @@ function useSessionLifecycle(input) {
|
|
|
3378
3878
|
releaseRef.current("disconnected");
|
|
3379
3879
|
setReadinessFailed(true);
|
|
3380
3880
|
leaveRoomRef.current?.();
|
|
3381
|
-
}, [clearTimer]);
|
|
3881
|
+
}, [clearTimer, invalidateExternalPerformance]);
|
|
3382
3882
|
useEffect(() => {
|
|
3383
3883
|
if (!active || userEnded || readinessFailed) return;
|
|
3384
3884
|
if (recovery.kind === "ended" || recovery.kind === "failed") return;
|
|
@@ -3459,6 +3959,7 @@ function useSessionLifecycle(input) {
|
|
|
3459
3959
|
const end = useCallback(() => {
|
|
3460
3960
|
if (!active || userEndedRef.current || phaseRef.current.kind === "ended") return;
|
|
3461
3961
|
userEndedRef.current = true;
|
|
3962
|
+
invalidateExternalPerformance("ended");
|
|
3462
3963
|
clearTimer();
|
|
3463
3964
|
connectedRef.current = false;
|
|
3464
3965
|
manualReconnectPendingRef.current = false;
|
|
@@ -3469,7 +3970,7 @@ function useSessionLifecycle(input) {
|
|
|
3469
3970
|
setRecovery({ kind: "ended", reason: "user" });
|
|
3470
3971
|
clearRef.current();
|
|
3471
3972
|
leaveRoomRef.current?.();
|
|
3472
|
-
}, [active, clearTimer]);
|
|
3973
|
+
}, [active, clearTimer, invalidateExternalPerformance]);
|
|
3473
3974
|
const reconnect = useCallback(() => {
|
|
3474
3975
|
if (!active) return;
|
|
3475
3976
|
if (readinessFailureRef.current) {
|
|
@@ -3539,16 +4040,22 @@ function useSessionLifecycle(input) {
|
|
|
3539
4040
|
const lastBehaviorRef = useRef(null);
|
|
3540
4041
|
const behaviorEnabled = Boolean(onBehaviorChange);
|
|
3541
4042
|
const onLifecycleData = useMemo(() => {
|
|
3542
|
-
if (!behaviorEnabled) return void 0;
|
|
4043
|
+
if (!behaviorEnabled && !performanceEnabled) return void 0;
|
|
3543
4044
|
return (frame) => {
|
|
3544
4045
|
const parsed = lifecycleServerFrameSchema.safeParse(frame);
|
|
3545
|
-
if (!parsed.success
|
|
4046
|
+
if (!parsed.success) return;
|
|
4047
|
+
if (parsed.data.kind === "external_performance") {
|
|
4048
|
+
if (grantId === grantIdRef.current && parsed.data.session_id === grantId) performanceRef.current?.receive(parsed.data);
|
|
4049
|
+
return;
|
|
4050
|
+
}
|
|
4051
|
+
if (parsed.data.kind === "ended") invalidateExternalPerformance("ended");
|
|
4052
|
+
if (!behaviorEnabled || parsed.data.kind !== "behavior_state") return;
|
|
3546
4053
|
const next = nextBehaviorSnapshot(lastBehaviorRef.current, parsed.data);
|
|
3547
4054
|
if (!next) return;
|
|
3548
4055
|
lastBehaviorRef.current = next;
|
|
3549
4056
|
onBehaviorChangeRef.current?.(next);
|
|
3550
4057
|
};
|
|
3551
|
-
}, [behaviorEnabled]);
|
|
4058
|
+
}, [behaviorEnabled, performanceEnabled, grantId, invalidateExternalPerformance]);
|
|
3552
4059
|
return useMemo(
|
|
3553
4060
|
() => ({
|
|
3554
4061
|
phase,
|
|
@@ -3567,7 +4074,9 @@ function useSessionLifecycle(input) {
|
|
|
3567
4074
|
setAgentPresent,
|
|
3568
4075
|
registerLeaveRoom,
|
|
3569
4076
|
reset,
|
|
3570
|
-
onLifecycleData
|
|
4077
|
+
onLifecycleData,
|
|
4078
|
+
invalidateExternalPerformance: performanceEnabled ? invalidateExternalPerformance : void 0,
|
|
4079
|
+
resumeExternalPerformance: performanceEnabled ? resumeExternalPerformance : void 0
|
|
3571
4080
|
}),
|
|
3572
4081
|
[
|
|
3573
4082
|
phase,
|
|
@@ -3586,7 +4095,10 @@ function useSessionLifecycle(input) {
|
|
|
3586
4095
|
setAgentPresent,
|
|
3587
4096
|
registerLeaveRoom,
|
|
3588
4097
|
reset,
|
|
3589
|
-
onLifecycleData
|
|
4098
|
+
onLifecycleData,
|
|
4099
|
+
invalidateExternalPerformance,
|
|
4100
|
+
resumeExternalPerformance,
|
|
4101
|
+
performanceEnabled
|
|
3590
4102
|
]
|
|
3591
4103
|
);
|
|
3592
4104
|
}
|
|
@@ -3615,9 +4127,45 @@ function SessionLifecycleRoomBridge({
|
|
|
3615
4127
|
const transcriptions = useTranscriptions();
|
|
3616
4128
|
const room = useRoomContext();
|
|
3617
4129
|
const roomMatchesGrant = lifecycle.grant === void 0 || lifecycle.grant !== null && room.name === lifecycle.grant.room_name;
|
|
4130
|
+
const correlatedRef = useRef(null);
|
|
4131
|
+
const correlatedObserver = useRef(lifecycle.correlatedAudio);
|
|
4132
|
+
correlatedObserver.current = lifecycle.correlatedAudio;
|
|
4133
|
+
const correlatedEnabled = lifecycle.correlatedAudio !== void 0;
|
|
4134
|
+
const correlatedSessionId = lifecycle.grant?.session_id;
|
|
4135
|
+
const startCorrelatedAudio = useCallback(async () => {
|
|
4136
|
+
const player = correlatedRef.current;
|
|
4137
|
+
if (!player) return false;
|
|
4138
|
+
try {
|
|
4139
|
+
await player.startAudio();
|
|
4140
|
+
return true;
|
|
4141
|
+
} catch {
|
|
4142
|
+
return false;
|
|
4143
|
+
}
|
|
4144
|
+
}, []);
|
|
4145
|
+
useEffect(() => {
|
|
4146
|
+
if (!correlatedEnabled || !correlatedSessionId || !roomMatchesGrant || !assistant.agent) return;
|
|
4147
|
+
const player = attachCorrelatedAudio(room, {
|
|
4148
|
+
sessionId: correlatedSessionId,
|
|
4149
|
+
agentIdentity: assistant.agent.identity,
|
|
4150
|
+
prepare: (header, signal) => correlatedObserver.current?.prepare?.(header, signal) ?? Promise.resolve(),
|
|
4151
|
+
onPosition: (position) => correlatedObserver.current?.onPosition(position),
|
|
4152
|
+
onInvalidated: (reason) => correlatedObserver.current?.onInvalidated(reason),
|
|
4153
|
+
onPlaybackBlocked: (unblock) => {
|
|
4154
|
+
setAudioPlayback?.(unblock ? "blocked" : "allowed");
|
|
4155
|
+
correlatedObserver.current?.onPlaybackBlocked?.(unblock);
|
|
4156
|
+
}
|
|
4157
|
+
});
|
|
4158
|
+
correlatedRef.current = player;
|
|
4159
|
+
return () => {
|
|
4160
|
+
correlatedRef.current = null;
|
|
4161
|
+
void player.dispose();
|
|
4162
|
+
setAudioPlayback?.("unknown");
|
|
4163
|
+
};
|
|
4164
|
+
}, [room, assistant.agent, correlatedEnabled, correlatedSessionId, roomMatchesGrant, setAudioPlayback]);
|
|
3618
4165
|
useCallMedia(room, {
|
|
3619
4166
|
setMicrophoneFacts,
|
|
3620
|
-
setAudioPlayback,
|
|
4167
|
+
setAudioPlayback: correlatedEnabled ? void 0 : setAudioPlayback,
|
|
4168
|
+
startAudioOverride: correlatedEnabled ? startCorrelatedAudio : void 0,
|
|
3621
4169
|
registerMediaControls,
|
|
3622
4170
|
microphoneWanted: microphone ?? publishesMicrophoneByDefault(lifecycle.grant),
|
|
3623
4171
|
connectionState,
|
|
@@ -3631,7 +4179,7 @@ function SessionLifecycleRoomBridge({
|
|
|
3631
4179
|
if (!mediaModeEnabled) return;
|
|
3632
4180
|
const agent = assistant.agent;
|
|
3633
4181
|
let previous;
|
|
3634
|
-
const
|
|
4182
|
+
const deliver2 = (mode) => {
|
|
3635
4183
|
if (mode === previous) return;
|
|
3636
4184
|
previous = mode;
|
|
3637
4185
|
try {
|
|
@@ -3642,7 +4190,7 @@ function SessionLifecycleRoomBridge({
|
|
|
3642
4190
|
};
|
|
3643
4191
|
const read = () => {
|
|
3644
4192
|
const value = agent?.attributes?.["rta.media_mode"];
|
|
3645
|
-
|
|
4193
|
+
deliver2(value === "video" ? "avatar" : value === "voice" ? "voice" : null);
|
|
3646
4194
|
};
|
|
3647
4195
|
const changed = (_attributes, participant) => {
|
|
3648
4196
|
if (participant === agent) read();
|
|
@@ -3651,7 +4199,7 @@ function SessionLifecycleRoomBridge({
|
|
|
3651
4199
|
read();
|
|
3652
4200
|
return () => {
|
|
3653
4201
|
room.off(RoomEvent.ParticipantAttributesChanged, changed);
|
|
3654
|
-
|
|
4202
|
+
deliver2(null);
|
|
3655
4203
|
};
|
|
3656
4204
|
}, [room, assistant.agent, mediaModeEnabled, mediaModeSessionId]);
|
|
3657
4205
|
const { send } = useChat();
|
|
@@ -3698,7 +4246,7 @@ function SessionLifecycleRoomBridge({
|
|
|
3698
4246
|
let closed = false;
|
|
3699
4247
|
let pending = false;
|
|
3700
4248
|
let previous;
|
|
3701
|
-
const
|
|
4249
|
+
const deliver2 = (details) => {
|
|
3702
4250
|
connectionHistoryRef.current?.enqueue(details);
|
|
3703
4251
|
try {
|
|
3704
4252
|
if (onConnectionDetailsChange) void Promise.resolve(callback?.(details)).catch(() => {
|
|
@@ -3725,7 +4273,7 @@ function SessionLifecycleRoomBridge({
|
|
|
3725
4273
|
const last = previous;
|
|
3726
4274
|
if (last?.connectionState === next.connectionState && last.localQuality === next.localQuality && ["audio", "video"].every((kind) => last[kind]?.publisherQuality === next[kind]?.publisherQuality && last[kind]?.streamState === next[kind]?.streamState)) return;
|
|
3727
4275
|
previous = next;
|
|
3728
|
-
|
|
4276
|
+
deliver2(next);
|
|
3729
4277
|
});
|
|
3730
4278
|
};
|
|
3731
4279
|
const onQuality = (_quality, participant) => {
|
|
@@ -3755,7 +4303,7 @@ function SessionLifecycleRoomBridge({
|
|
|
3755
4303
|
room.off(RoomEvent.TrackStreamStateChanged, onPublication);
|
|
3756
4304
|
room.off(RoomEvent.TrackSubscribed, onTrack);
|
|
3757
4305
|
room.off(RoomEvent.TrackUnsubscribed, onTrack);
|
|
3758
|
-
|
|
4306
|
+
deliver2(null);
|
|
3759
4307
|
};
|
|
3760
4308
|
}, [room, detailsSessionId, detailsEnabled]);
|
|
3761
4309
|
useEffect(() => {
|
|
@@ -3792,18 +4340,56 @@ function SessionLifecycleRoomBridge({
|
|
|
3792
4340
|
useEffect(() => {
|
|
3793
4341
|
if (!onLifecycleData) return;
|
|
3794
4342
|
const decoder = new TextDecoder();
|
|
3795
|
-
const handler = (payload,
|
|
4343
|
+
const handler = (payload, participant, _kind, topic) => {
|
|
3796
4344
|
if (topic !== RTA_LIFECYCLE_TOPIC) return;
|
|
3797
4345
|
try {
|
|
3798
|
-
|
|
4346
|
+
const frame = JSON.parse(decoder.decode(payload));
|
|
4347
|
+
if (typeof frame === "object" && frame !== null && "kind" in frame && (frame.kind === EXTERNAL_PERFORMANCE_KIND || frame.kind === "ended") && (!assistant.agent || participant !== assistant.agent || !roomMatchesGrant || room.state !== "connected")) return;
|
|
4348
|
+
if (typeof frame === "object" && frame !== null && "kind" in frame && frame.kind === "ended" && lifecycleServerFrameSchema.safeParse(frame).success) {
|
|
4349
|
+
void correlatedRef.current?.dispose();
|
|
4350
|
+
correlatedRef.current = null;
|
|
4351
|
+
}
|
|
4352
|
+
onLifecycleData(frame);
|
|
3799
4353
|
} catch {
|
|
3800
4354
|
}
|
|
3801
4355
|
};
|
|
4356
|
+
const invalidate = () => lifecycle.invalidateExternalPerformance?.("connection_lost");
|
|
4357
|
+
const changed = () => {
|
|
4358
|
+
if (room.state === "disconnected" || room.state === "reconnecting" || room.state === "signalReconnecting") invalidate();
|
|
4359
|
+
};
|
|
3802
4360
|
room.on(RoomEvent.DataReceived, handler);
|
|
4361
|
+
if (lifecycle.invalidateExternalPerformance) {
|
|
4362
|
+
room.on(RoomEvent.Reconnecting, invalidate);
|
|
4363
|
+
room.on(RoomEvent.SignalReconnecting, invalidate);
|
|
4364
|
+
room.on(RoomEvent.Disconnected, invalidate);
|
|
4365
|
+
room.on(RoomEvent.ConnectionStateChanged, changed);
|
|
4366
|
+
}
|
|
3803
4367
|
return () => {
|
|
3804
4368
|
room.off(RoomEvent.DataReceived, handler);
|
|
4369
|
+
room.off(RoomEvent.Reconnecting, invalidate);
|
|
4370
|
+
room.off(RoomEvent.SignalReconnecting, invalidate);
|
|
4371
|
+
room.off(RoomEvent.Disconnected, invalidate);
|
|
4372
|
+
room.off(RoomEvent.ConnectionStateChanged, changed);
|
|
4373
|
+
};
|
|
4374
|
+
}, [room, onLifecycleData, assistant.agent, roomMatchesGrant, lifecycle.invalidateExternalPerformance]);
|
|
4375
|
+
useEffect(() => {
|
|
4376
|
+
if (!assistant.agent) return;
|
|
4377
|
+
return () => lifecycle.invalidateExternalPerformance?.("connection_lost");
|
|
4378
|
+
}, [assistant.agent, lifecycle.invalidateExternalPerformance]);
|
|
4379
|
+
const resumeExternalPerformance = lifecycle.resumeExternalPerformance;
|
|
4380
|
+
useEffect(() => {
|
|
4381
|
+
if (!resumeExternalPerformance) return;
|
|
4382
|
+
const resumeIfBound = () => {
|
|
4383
|
+
if (assistant.agent && roomMatchesGrant && room.state === "connected") resumeExternalPerformance();
|
|
4384
|
+
};
|
|
4385
|
+
resumeIfBound();
|
|
4386
|
+
room.on(RoomEvent.Reconnected, resumeIfBound);
|
|
4387
|
+
room.on(RoomEvent.ConnectionStateChanged, resumeIfBound);
|
|
4388
|
+
return () => {
|
|
4389
|
+
room.off(RoomEvent.Reconnected, resumeIfBound);
|
|
4390
|
+
room.off(RoomEvent.ConnectionStateChanged, resumeIfBound);
|
|
3805
4391
|
};
|
|
3806
|
-
}, [room,
|
|
4392
|
+
}, [room, assistant.agent, roomMatchesGrant, resumeExternalPerformance]);
|
|
3807
4393
|
useEffect(() => {
|
|
3808
4394
|
if (!registerDataPublisher) return;
|
|
3809
4395
|
const encoder = new TextEncoder();
|
|
@@ -3939,6 +4525,7 @@ function useRealtimeSession(input) {
|
|
|
3939
4525
|
onBehaviorChange,
|
|
3940
4526
|
onClipResult,
|
|
3941
4527
|
onVoiceError,
|
|
4528
|
+
correlatedAudio,
|
|
3942
4529
|
// everything else (client/active/idleSeconds/idleWarnLeadSeconds/reconnect/requestOptions…)
|
|
3943
4530
|
// flows straight into the inner SSOT hook.
|
|
3944
4531
|
...inner
|
|
@@ -4011,6 +4598,7 @@ function useRealtimeSession(input) {
|
|
|
4011
4598
|
return `t-${Date.now()}-${++turnSequenceRef.current}`;
|
|
4012
4599
|
}, []);
|
|
4013
4600
|
const onLifecycleData = useCallback((frame) => {
|
|
4601
|
+
lifecycle.onLifecycleData?.(frame);
|
|
4014
4602
|
const parsed = lifecycleServerFrameSchema.safeParse(frame);
|
|
4015
4603
|
if (!parsed.success) return;
|
|
4016
4604
|
const f = parsed.data;
|
|
@@ -4070,7 +4658,7 @@ function useRealtimeSession(input) {
|
|
|
4070
4658
|
break;
|
|
4071
4659
|
}
|
|
4072
4660
|
}
|
|
4073
|
-
}, []);
|
|
4661
|
+
}, [lifecycle.onLifecycleData]);
|
|
4074
4662
|
const registerDataPublisher = useCallback((publish) => {
|
|
4075
4663
|
dataPublisherRef.current = publish;
|
|
4076
4664
|
}, []);
|
|
@@ -4336,6 +4924,7 @@ function useRealtimeSession(input) {
|
|
|
4336
4924
|
// stayConnected/reconnect/markActivity/the sinks) → a true drop-in superset, then
|
|
4337
4925
|
// add the realtime-session surface. `reset` + `lifecycle` come AFTER so ours win.
|
|
4338
4926
|
...lifecycle,
|
|
4927
|
+
correlatedAudio,
|
|
4339
4928
|
turn,
|
|
4340
4929
|
clocks,
|
|
4341
4930
|
endsAt,
|
|
@@ -4370,6 +4959,7 @@ function useRealtimeSession(input) {
|
|
|
4370
4959
|
}),
|
|
4371
4960
|
[
|
|
4372
4961
|
lifecycle,
|
|
4962
|
+
correlatedAudio,
|
|
4373
4963
|
turn,
|
|
4374
4964
|
clocks,
|
|
4375
4965
|
endsAt,
|
|
@@ -4470,7 +5060,9 @@ function useAvatarCall(props) {
|
|
|
4470
5060
|
onIdleWarning: props.onQuiet ? ({ secondsLeft }) => props.onQuiet?.({ secondsLeft }) : void 0,
|
|
4471
5061
|
onCreditsLow: props.onLowBalance ? ({ secondsLeft }) => props.onLowBalance?.({ secondsLeft }) : void 0,
|
|
4472
5062
|
onEnded: props.onEnded ? ({ reason }) => props.onEnded?.({ reason }) : void 0,
|
|
4473
|
-
onVoiceError: props.onVoiceError ? (error) => props.onVoiceError?.(error) : void 0
|
|
5063
|
+
onVoiceError: props.onVoiceError ? (error) => props.onVoiceError?.(error) : void 0,
|
|
5064
|
+
externalPerformance: props.externalPerformance,
|
|
5065
|
+
correlatedAudio: props.correlatedAudio
|
|
4474
5066
|
});
|
|
4475
5067
|
sessionRef.current = session;
|
|
4476
5068
|
const call = handleFor(session);
|
|
@@ -4508,6 +5100,7 @@ function useAvatarCall(props) {
|
|
|
4508
5100
|
RealtimeAvatarLiveKitRoom,
|
|
4509
5101
|
{
|
|
4510
5102
|
grant: session.grant,
|
|
5103
|
+
renderRoomAudio: props.correlatedAudio === void 0,
|
|
4511
5104
|
// LiveKit captures the microphone on connect from this value, so a mute chosen while
|
|
4512
5105
|
// waiting is honoured rather than overridden when the call goes live.
|
|
4513
5106
|
audio: props.listen !== false && !session.microphoneMuted,
|
|
@@ -4907,4 +5500,4 @@ function RecordingPlayer({ assets, className }) {
|
|
|
4907
5500
|
] });
|
|
4908
5501
|
}
|
|
4909
5502
|
|
|
4910
|
-
export { AdaptivePlayoutController, AvatarCall, AvatarVideoSurface, DEFAULT_APPROACHING_END_LEAD_SECONDS, DEFAULT_AVATAR_PLAYOUT_DELAY_SECONDS, DEFAULT_AVATAR_UNSTABLE_STALL_MS, DEFAULT_CREDITS_LOW_LEAD_SECONDS, DEFAULT_FISH_TTS_MODEL, DEFAULT_GOVERNOR_CONFIG, DEFAULT_GRACE_CEILING_SECONDS, DEFAULT_GRACE_WINDOW_LEAD_SECONDS, DEFAULT_IDLE_SECONDS, DEFAULT_IDLE_WARN_LEAD_SECONDS, DEFAULT_READY_TIMEOUT_SECONDS, DEFAULT_TURN_TIMEOUT_SECONDS, MAX_RECONNECT_ATTEMPTS, MAX_SESSION_INSTRUCTIONS_CHARS, MIC_LEASE_ENDED_TIMEOUT_MS, NETWORK_QUALITY_POLICY, RECONNECT_BACKOFF_MS, RealtimeAvatarCapacityError, RealtimeAvatarLiveKitRoom, RecordingPlayer, SessionLifecycleRoomBridge, capacityErrorFromBusy, capacityStateFromGrant, createProxyClient, fishTtsModelSchema, knownBehaviorStates, mapTurnState, readInboundRtp, sessionBehaviorSchema, sessionClipSchema, shouldReplayPendingTurn, splitCallTranscript, useAvatarAdaptivePlayoutDelay, useAvatarCall, useAvatarCamera, useAvatarPlayoutDelay, useAvatarQualityGovernor, useCallTranscript, useCharacterTools, useLiveKitAvatarGrant, useMicLease, useRealtimeSession, useReleaseMicLeaseOnTrackEnded, useSessionLifecycle };
|
|
5503
|
+
export { AdaptivePlayoutController, AvatarCall, AvatarVideoSurface, DEFAULT_APPROACHING_END_LEAD_SECONDS, DEFAULT_AVATAR_PLAYOUT_DELAY_SECONDS, DEFAULT_AVATAR_UNSTABLE_STALL_MS, DEFAULT_CREDITS_LOW_LEAD_SECONDS, DEFAULT_FISH_TTS_MODEL, DEFAULT_GOVERNOR_CONFIG, DEFAULT_GRACE_CEILING_SECONDS, DEFAULT_GRACE_WINDOW_LEAD_SECONDS, DEFAULT_IDLE_SECONDS, DEFAULT_IDLE_WARN_LEAD_SECONDS, DEFAULT_READY_TIMEOUT_SECONDS, DEFAULT_TURN_TIMEOUT_SECONDS, EXTERNAL_PERFORMANCE_TIMING, FISH_DELIVERY_MODELS, KNOWN_EXTERNAL_PERFORMANCE_STATUSES, MAX_RECONNECT_ATTEMPTS, MAX_SESSION_INSTRUCTIONS_CHARS, MIC_LEASE_ENDED_TIMEOUT_MS, NETWORK_QUALITY_POLICY, RECONNECT_BACKOFF_MS, RealtimeAvatarCapacityError, RealtimeAvatarLiveKitRoom, RecordingPlayer, SessionLifecycleRoomBridge, capacityErrorFromBusy, capacityStateFromGrant, createProxyClient, fishTtsModelSchema, knownBehaviorStates, mapTurnState, readInboundRtp, sessionBehaviorSchema, sessionClipSchema, shouldReplayPendingTurn, splitCallTranscript, useAvatarAdaptivePlayoutDelay, useAvatarCall, useAvatarCamera, useAvatarPlayoutDelay, useAvatarQualityGovernor, useCallTranscript, useCharacterTools, useLiveKitAvatarGrant, useMicLease, useRealtimeSession, useReleaseMicLeaseOnTrackEnded, useSessionLifecycle };
|