realtime-avatar 0.33.0 → 0.34.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +56 -1
- package/dist/browser.d.ts +4 -89
- package/dist/browser.js +362 -1
- package/dist/chunk-6KQS7CTR.js +1 -0
- package/dist/{chunk-VLYUMP3I.js → chunk-GPGNAQZV.js} +401 -170
- package/dist/{chunk-DM5TWRC4.js → chunk-OVW4N5NH.js} +1 -13
- package/dist/express.d.ts +2 -2
- package/dist/express.js +2 -2
- package/dist/hono.d.ts +2 -2
- package/dist/hono.js +2 -2
- package/dist/index.d.ts +330 -3
- package/dist/index.js +2 -1
- package/dist/nextjs.d.ts +2 -2
- package/dist/nextjs.js +2 -2
- package/dist/{playout-delay-CtRZdtsj.d.ts → playout-delay-BzoM1BMN.d.ts} +135 -1
- package/dist/{proxy-client-DJhYiiKX.d.ts → proxy-client-CiTOjkg8.d.ts} +67 -3
- package/dist/react-native.d.ts +3 -3
- package/dist/react-native.js +614 -36
- package/dist/react.d.ts +12 -22
- package/dist/react.js +618 -37
- package/dist/server.d.ts +2 -2
- package/dist/server.js +2 -1
- package/dist/tanstack-start.d.ts +2 -2
- package/dist/tanstack-start.js +2 -2
- package/dist/{types-DygYBvSv.d.ts → types-Cdi7KWWj.d.ts} +1 -1
- package/dist/{types-VoejsHij.d.ts → types-Cg_i1jhk.d.ts} +207 -18
- package/package.json +3 -3
package/dist/react-native.js
CHANGED
|
@@ -1251,7 +1251,7 @@ function stepNetworkQuality(previous, sample, nowMs, options) {
|
|
|
1251
1251
|
}
|
|
1252
1252
|
|
|
1253
1253
|
// ../client/src/react/network-quality-binding.ts
|
|
1254
|
-
function bindNetworkQuality(publication, inhibited,
|
|
1254
|
+
function bindNetworkQuality(publication, inhibited, notify2, tickMs) {
|
|
1255
1255
|
let disposed = false;
|
|
1256
1256
|
let running = false;
|
|
1257
1257
|
let state = initNetworkQuality(performance.now());
|
|
@@ -1265,7 +1265,7 @@ function bindNetworkQuality(publication, inhibited, notify, tickMs) {
|
|
|
1265
1265
|
};
|
|
1266
1266
|
const emit = (status) => {
|
|
1267
1267
|
try {
|
|
1268
|
-
|
|
1268
|
+
notify2(status);
|
|
1269
1269
|
} catch {
|
|
1270
1270
|
}
|
|
1271
1271
|
};
|
|
@@ -1787,15 +1787,107 @@ var schema5 = z.object({
|
|
|
1787
1787
|
"request_id": schema22.optional()
|
|
1788
1788
|
});
|
|
1789
1789
|
var voiceErrorFrameSchema = schema5;
|
|
1790
|
-
var schema04 = z.
|
|
1791
|
-
var schema13 = z.
|
|
1792
|
-
var schema23 = z.
|
|
1793
|
-
var schema33 = z.
|
|
1794
|
-
var schema43 = z.
|
|
1795
|
-
var schema52 = z.
|
|
1796
|
-
var
|
|
1790
|
+
var schema04 = z.string().min(1).max(200);
|
|
1791
|
+
var schema13 = z.literal(1);
|
|
1792
|
+
var schema23 = z.string().min(1).max(200);
|
|
1793
|
+
var schema33 = z.int().min(1).max(9007199254740991);
|
|
1794
|
+
var schema43 = z.literal(24e3);
|
|
1795
|
+
var schema52 = z.literal("pcm_s16le");
|
|
1796
|
+
var schema6 = z.object({
|
|
1797
|
+
"session_id": schema04,
|
|
1798
|
+
"generation_id": schema04,
|
|
1799
|
+
"version": schema13,
|
|
1800
|
+
"utterance_id": schema23,
|
|
1801
|
+
"stream_id": schema04,
|
|
1802
|
+
"epoch": schema33,
|
|
1803
|
+
"sample_rate": schema43,
|
|
1804
|
+
"channels": schema13,
|
|
1805
|
+
"encoding": schema52
|
|
1806
|
+
});
|
|
1807
|
+
var schema7 = z.enum(["started", "progress", "finished", "cancelled"]);
|
|
1808
|
+
var schema8 = z.int().min(0).max(144e4);
|
|
1809
|
+
z.strictObject({
|
|
1810
|
+
"session_id": schema04,
|
|
1811
|
+
"generation_id": schema04,
|
|
1812
|
+
"utterance_id": schema23,
|
|
1813
|
+
"stream_id": schema04,
|
|
1814
|
+
"epoch": schema33,
|
|
1815
|
+
"status": schema7,
|
|
1816
|
+
"samples": schema8
|
|
1817
|
+
});
|
|
1818
|
+
var schema10 = z.object({
|
|
1819
|
+
"version": schema13,
|
|
1820
|
+
"session_id": schema04,
|
|
1821
|
+
"epoch": schema33
|
|
1822
|
+
});
|
|
1823
|
+
var schema11 = z.literal("external_llm_error");
|
|
1824
|
+
var schema122 = z.enum(["external_llm_refused", "external_llm_failed"]);
|
|
1825
|
+
var schema132 = z.string();
|
|
1826
|
+
var schema14 = z.literal(true);
|
|
1827
|
+
var schema15 = z.object({
|
|
1828
|
+
"kind": schema11,
|
|
1829
|
+
"code": schema122,
|
|
1830
|
+
"message": schema132,
|
|
1831
|
+
"fatal": schema14
|
|
1832
|
+
});
|
|
1833
|
+
var correlatedAudioHeaderSchema = schema6;
|
|
1834
|
+
var correlatedAudioCancelSchema = schema10;
|
|
1835
|
+
var externalLlmErrorFrameSchema = schema15;
|
|
1836
|
+
var CORRELATED_AUDIO_CONTRACT = {
|
|
1837
|
+
"topic": "rta.audio.v1",
|
|
1838
|
+
"receiptMethod": "rta.audio.receipt.v1",
|
|
1839
|
+
"cancelMethod": "rta.audio.cancel.v1",
|
|
1840
|
+
"maxSamples": 144e4,
|
|
1841
|
+
"maxChunkBytes": 48e3,
|
|
1842
|
+
"maxBufferedSamples": 48e3
|
|
1843
|
+
};
|
|
1844
|
+
var schema05 = z.string().min(1).max(200);
|
|
1845
|
+
var schema16 = z.literal(1);
|
|
1846
|
+
var schema24 = z.enum(["respond", "recovery", "initiative"]);
|
|
1847
|
+
var schema34 = z.string().min(1).max(32);
|
|
1848
|
+
var schema44 = z.union([schema24, schema34]);
|
|
1849
|
+
var schema53 = z.enum(["draft", "accepted"]);
|
|
1850
|
+
var schema62 = z.literal("external");
|
|
1851
|
+
var schema72 = z.object({
|
|
1852
|
+
"renderer": schema62
|
|
1853
|
+
});
|
|
1854
|
+
z.object({
|
|
1855
|
+
"session_id": schema05,
|
|
1856
|
+
"generation_id": schema05,
|
|
1857
|
+
"version": schema16,
|
|
1858
|
+
"purpose": schema44,
|
|
1859
|
+
"admission": schema53,
|
|
1860
|
+
"visible_character": schema72.optional()
|
|
1861
|
+
});
|
|
1862
|
+
var schema92 = z.string().min(1).max(200);
|
|
1863
|
+
z.object({
|
|
1864
|
+
"generation_id": schema05,
|
|
1865
|
+
"utterance_id": schema92
|
|
1866
|
+
});
|
|
1867
|
+
var schema112 = z.literal("external_performance");
|
|
1868
|
+
var schema123 = z.enum(["accepted", "dropped", "cancelled", "audio_released"]);
|
|
1869
|
+
var schema133 = z.union([schema123, schema34]);
|
|
1870
|
+
var schema142 = z.literal("unavailable");
|
|
1871
|
+
var schema152 = z.literal("unavailable");
|
|
1872
|
+
var schema162 = z.object({
|
|
1873
|
+
"session_id": schema05,
|
|
1874
|
+
"generation_id": schema05,
|
|
1875
|
+
"kind": schema112,
|
|
1876
|
+
"version": schema16,
|
|
1877
|
+
"status": schema133,
|
|
1878
|
+
"audible_checkpoint": schema142,
|
|
1879
|
+
"audio_mapping": schema152
|
|
1880
|
+
});
|
|
1881
|
+
var externalPerformanceFrameSchema = schema162;
|
|
1882
|
+
var schema06 = z.enum(["s1", "s2-pro", "s2.1-pro", "drama-3"]);
|
|
1883
|
+
var schema17 = z.enum(["speech-1.6", "speech-1.5", "s1-mini"]);
|
|
1884
|
+
var schema25 = z.union([schema06, schema17]);
|
|
1885
|
+
var schema35 = z.enum(["whisper"]);
|
|
1886
|
+
var schema45 = z.null();
|
|
1887
|
+
var schema54 = z.union([schema35, schema45]);
|
|
1888
|
+
var fishTtsModelSchema = schema25;
|
|
1797
1889
|
var DEFAULT_FISH_TTS_MODEL = "s2.1-pro";
|
|
1798
|
-
var fishDeliverySchema =
|
|
1890
|
+
var fishDeliverySchema = schema54;
|
|
1799
1891
|
var FISH_DELIVERY_MODELS = { "whisper": ["drama-3"] };
|
|
1800
1892
|
|
|
1801
1893
|
// ../client/src/wire.ts
|
|
@@ -2111,7 +2203,9 @@ var lifecycleServerFrameSchema = z.discriminatedUnion("kind", [
|
|
|
2111
2203
|
endedFrameSchema,
|
|
2112
2204
|
behaviorStateFrameSchema,
|
|
2113
2205
|
clipAckFrameSchema,
|
|
2114
|
-
voiceErrorFrameSchema2
|
|
2206
|
+
voiceErrorFrameSchema2,
|
|
2207
|
+
externalPerformanceFrameSchema,
|
|
2208
|
+
externalLlmErrorFrameSchema
|
|
2115
2209
|
]);
|
|
2116
2210
|
var liveKitCapacitySnapshotSchema = z.object({
|
|
2117
2211
|
// Placement identity + per-worker session ceiling. These are always present on the
|
|
@@ -2393,6 +2487,7 @@ function useCallMedia(room, options) {
|
|
|
2393
2487
|
}, [room, setAudioPlayback, connecting]);
|
|
2394
2488
|
const controls = useMemo(() => ({
|
|
2395
2489
|
startAudio: async () => {
|
|
2490
|
+
if (options.startAudioOverride) return options.startAudioOverride();
|
|
2396
2491
|
try {
|
|
2397
2492
|
await room.startAudio();
|
|
2398
2493
|
} catch {
|
|
@@ -2409,13 +2504,372 @@ function useCallMedia(room, options) {
|
|
|
2409
2504
|
setVersion((value) => value + 1);
|
|
2410
2505
|
}
|
|
2411
2506
|
}
|
|
2412
|
-
}), [room]);
|
|
2507
|
+
}), [room, options.startAudioOverride]);
|
|
2413
2508
|
useEffect(() => {
|
|
2414
2509
|
if (!registerMediaControls) return;
|
|
2415
2510
|
registerMediaControls(controls);
|
|
2416
2511
|
return () => registerMediaControls(null);
|
|
2417
2512
|
}, [controls, registerMediaControls]);
|
|
2418
2513
|
}
|
|
2514
|
+
var attachments = /* @__PURE__ */ new WeakSet();
|
|
2515
|
+
var bindings = /* @__PURE__ */ new WeakMap();
|
|
2516
|
+
function notify(callback) {
|
|
2517
|
+
try {
|
|
2518
|
+
void Promise.resolve(callback()).catch(() => {
|
|
2519
|
+
});
|
|
2520
|
+
} catch {
|
|
2521
|
+
}
|
|
2522
|
+
}
|
|
2523
|
+
function attachCorrelatedAudio(room, options) {
|
|
2524
|
+
if (attachments.has(room)) throw new Error("Correlated audio is already attached to this room");
|
|
2525
|
+
if (!options.sessionId || !options.agentIdentity) throw new Error("Correlated audio requires the held session and bound agent");
|
|
2526
|
+
attachments.add(room);
|
|
2527
|
+
let context;
|
|
2528
|
+
let disposed = false;
|
|
2529
|
+
const previous = bindings.get(room);
|
|
2530
|
+
const binding = previous?.sessionId === options.sessionId ? previous : { sessionId: options.sessionId, epoch: 1, unavailable: false, seen: /* @__PURE__ */ new Set() };
|
|
2531
|
+
bindings.set(room, binding);
|
|
2532
|
+
const sources = [];
|
|
2533
|
+
const active = (source) => sources.includes(source);
|
|
2534
|
+
const buffered = () => sources.reduce((sum, source) => sum + source.received - source.consumed, 0);
|
|
2535
|
+
let nextStart = 0;
|
|
2536
|
+
let blocked;
|
|
2537
|
+
let closing;
|
|
2538
|
+
const seen = binding.seen;
|
|
2539
|
+
const waiters = /* @__PURE__ */ new Set();
|
|
2540
|
+
const wake = () => {
|
|
2541
|
+
for (const resolve of waiters) resolve();
|
|
2542
|
+
waiters.clear();
|
|
2543
|
+
};
|
|
2544
|
+
const identity = (header) => ({
|
|
2545
|
+
session_id: header.session_id,
|
|
2546
|
+
generation_id: header.generation_id,
|
|
2547
|
+
utterance_id: header.utterance_id,
|
|
2548
|
+
stream_id: header.stream_id,
|
|
2549
|
+
epoch: header.epoch
|
|
2550
|
+
});
|
|
2551
|
+
const position = (current, state) => {
|
|
2552
|
+
notify(() => options.onPosition({ ...identity(current.header), samples: current.consumed, sampleRate: current.header.sample_rate, state }));
|
|
2553
|
+
};
|
|
2554
|
+
const receipt = (current, status) => {
|
|
2555
|
+
const payload = { ...identity(current.header), status, samples: current.consumed };
|
|
2556
|
+
if (current.rpcBusy) {
|
|
2557
|
+
current.pendingReceipt = payload;
|
|
2558
|
+
return;
|
|
2559
|
+
}
|
|
2560
|
+
const send = async (next) => {
|
|
2561
|
+
current.rpcBusy = true;
|
|
2562
|
+
try {
|
|
2563
|
+
await room.localParticipant.performRpc({
|
|
2564
|
+
destinationIdentity: options.agentIdentity,
|
|
2565
|
+
method: CORRELATED_AUDIO_CONTRACT.receiptMethod,
|
|
2566
|
+
payload: JSON.stringify(next),
|
|
2567
|
+
responseTimeout: 2e3
|
|
2568
|
+
});
|
|
2569
|
+
} catch {
|
|
2570
|
+
if (!disposed && next.epoch === binding.epoch) invalidate("connection_lost");
|
|
2571
|
+
} finally {
|
|
2572
|
+
current.rpcBusy = false;
|
|
2573
|
+
const pending = current.pendingReceipt;
|
|
2574
|
+
current.pendingReceipt = void 0;
|
|
2575
|
+
if (pending && !disposed && pending.epoch === binding.epoch && !binding.unavailable) void send(pending);
|
|
2576
|
+
}
|
|
2577
|
+
};
|
|
2578
|
+
void send(payload);
|
|
2579
|
+
};
|
|
2580
|
+
const clear = () => {
|
|
2581
|
+
for (const current of sources.splice(0)) {
|
|
2582
|
+
current.abort.abort();
|
|
2583
|
+
for (const segment of current.segments) {
|
|
2584
|
+
segment.node.onended = null;
|
|
2585
|
+
segment.node.stop();
|
|
2586
|
+
segment.node.disconnect();
|
|
2587
|
+
}
|
|
2588
|
+
current.segments.length = 0;
|
|
2589
|
+
}
|
|
2590
|
+
nextStart = context?.currentTime ?? 0;
|
|
2591
|
+
wake();
|
|
2592
|
+
};
|
|
2593
|
+
const invalidate = (reason) => {
|
|
2594
|
+
if (disposed) return;
|
|
2595
|
+
if (reason === "connection_lost") {
|
|
2596
|
+
if (binding.unavailable) return;
|
|
2597
|
+
binding.unavailable = true;
|
|
2598
|
+
}
|
|
2599
|
+
if (reason === "invalid_stream" || reason === "disposed") for (const current of sources) receipt(current, "cancelled");
|
|
2600
|
+
clear();
|
|
2601
|
+
notify(() => options.onInvalidated(reason));
|
|
2602
|
+
};
|
|
2603
|
+
const tick = (reportCredit = true) => {
|
|
2604
|
+
if (!context || context.state !== "running" || disposed || binding.unavailable) return;
|
|
2605
|
+
for (const current of [...sources]) {
|
|
2606
|
+
if (!active(current)) continue;
|
|
2607
|
+
let consumed = current.consumed;
|
|
2608
|
+
for (const segment of current.segments) {
|
|
2609
|
+
if (context.currentTime < segment.start) break;
|
|
2610
|
+
const samples = Math.min(segment.length, Math.max(0, Math.floor((context.currentTime - segment.start) * current.header.sample_rate + 1e-6)));
|
|
2611
|
+
consumed = Math.max(consumed, segment.offset + samples);
|
|
2612
|
+
if (!current.started && samples > 0) {
|
|
2613
|
+
current.started = true;
|
|
2614
|
+
receipt(current, "started");
|
|
2615
|
+
position(current, "playing");
|
|
2616
|
+
current.lastPosition = current.consumed;
|
|
2617
|
+
}
|
|
2618
|
+
}
|
|
2619
|
+
current.consumed = consumed;
|
|
2620
|
+
while (current.segments.length && current.segments[0].offset + current.segments[0].length <= consumed) {
|
|
2621
|
+
const segment = current.segments.shift();
|
|
2622
|
+
if (segment) {
|
|
2623
|
+
segment.node.onended = null;
|
|
2624
|
+
segment.node.disconnect();
|
|
2625
|
+
}
|
|
2626
|
+
}
|
|
2627
|
+
if (current.started && current.eof && consumed === current.received && consumed > 0) {
|
|
2628
|
+
position(current, "finished");
|
|
2629
|
+
if (!active(current)) continue;
|
|
2630
|
+
receipt(current, "finished");
|
|
2631
|
+
sources.splice(sources.indexOf(current), 1);
|
|
2632
|
+
wake();
|
|
2633
|
+
} else {
|
|
2634
|
+
if (current.started && consumed > current.lastPosition) {
|
|
2635
|
+
current.lastPosition = consumed;
|
|
2636
|
+
position(current, "playing");
|
|
2637
|
+
}
|
|
2638
|
+
if (!active(current)) continue;
|
|
2639
|
+
if (reportCredit && consumed > current.lastReported) {
|
|
2640
|
+
current.lastReported = consumed;
|
|
2641
|
+
receipt(current, "progress");
|
|
2642
|
+
}
|
|
2643
|
+
wake();
|
|
2644
|
+
}
|
|
2645
|
+
}
|
|
2646
|
+
};
|
|
2647
|
+
const playbackState = () => {
|
|
2648
|
+
const next = context?.state !== "running";
|
|
2649
|
+
if (next !== blocked) {
|
|
2650
|
+
blocked = next;
|
|
2651
|
+
notify(() => options.onPlaybackBlocked?.(next ? startAudio : null));
|
|
2652
|
+
}
|
|
2653
|
+
tick();
|
|
2654
|
+
};
|
|
2655
|
+
const getContext = () => {
|
|
2656
|
+
if (!context) {
|
|
2657
|
+
context = new AudioContext({ sampleRate: correlatedAudioHeaderSchema.shape.sample_rate.value });
|
|
2658
|
+
context.onstatechange = playbackState;
|
|
2659
|
+
playbackState();
|
|
2660
|
+
}
|
|
2661
|
+
return context;
|
|
2662
|
+
};
|
|
2663
|
+
async function startAudio() {
|
|
2664
|
+
if (disposed || binding.unavailable) throw new Error("Correlated audio binding is no longer available");
|
|
2665
|
+
const ctx = getContext();
|
|
2666
|
+
await ctx.resume();
|
|
2667
|
+
playbackState();
|
|
2668
|
+
if (ctx.state !== "running") throw new Error("Correlated audio playback is blocked");
|
|
2669
|
+
}
|
|
2670
|
+
async function receive(reader, participant) {
|
|
2671
|
+
if (disposed || binding.unavailable || room.state !== "connected" || participant.identity !== options.agentIdentity) return;
|
|
2672
|
+
let value;
|
|
2673
|
+
try {
|
|
2674
|
+
value = JSON.parse(reader.info.attributes?.["rta.audio"] ?? "");
|
|
2675
|
+
} catch {
|
|
2676
|
+
return;
|
|
2677
|
+
}
|
|
2678
|
+
const parsed = correlatedAudioHeaderSchema.safeParse(value);
|
|
2679
|
+
if (!parsed.success || parsed.data.session_id !== options.sessionId || parsed.data.epoch < binding.epoch) return;
|
|
2680
|
+
const header = parsed.data;
|
|
2681
|
+
if (header.stream_id !== reader.info.id) return;
|
|
2682
|
+
if (header.epoch > binding.epoch) {
|
|
2683
|
+
binding.epoch = header.epoch;
|
|
2684
|
+
invalidate("cancelled");
|
|
2685
|
+
}
|
|
2686
|
+
const key = `${header.epoch}/${header.stream_id}`;
|
|
2687
|
+
if (seen.has(key)) return;
|
|
2688
|
+
if (sources.length >= 128 || seen.size >= 1e4) {
|
|
2689
|
+
invalidate("invalid_stream");
|
|
2690
|
+
return;
|
|
2691
|
+
}
|
|
2692
|
+
seen.add(key);
|
|
2693
|
+
const previousSource = sources.at(-1);
|
|
2694
|
+
const current = { header, abort: new AbortController(), segments: [], received: 0, consumed: 0, started: false, eof: false, lastReported: 0, lastPosition: -1, rpcBusy: false };
|
|
2695
|
+
sources.push(current);
|
|
2696
|
+
let carry;
|
|
2697
|
+
try {
|
|
2698
|
+
if (options.prepare) await options.prepare(header, current.abort.signal);
|
|
2699
|
+
if (!active(current) || current.abort.signal.aborted) return;
|
|
2700
|
+
const ctx = getContext();
|
|
2701
|
+
while (active(current) && (previousSource && active(previousSource) && !previousSource.eof || buffered() > CORRELATED_AUDIO_CONTRACT.maxBufferedSamples - CORRELATED_AUDIO_CONTRACT.maxChunkBytes / 2)) {
|
|
2702
|
+
await new Promise((resolve) => waiters.add(resolve));
|
|
2703
|
+
}
|
|
2704
|
+
if (!active(current)) return;
|
|
2705
|
+
for await (const chunk of reader.withAbortSignal(current.abort.signal)) {
|
|
2706
|
+
if (!active(current)) break;
|
|
2707
|
+
if (chunk.byteLength > CORRELATED_AUDIO_CONTRACT.maxChunkBytes) throw new Error("PCM chunk exceeds contract");
|
|
2708
|
+
const count = Math.floor((chunk.byteLength + (carry === void 0 ? 0 : 1)) / 2);
|
|
2709
|
+
if (current.received + count > CORRELATED_AUDIO_CONTRACT.maxSamples) throw new Error("PCM stream exceeds contract");
|
|
2710
|
+
while (active(current) && buffered() + count > CORRELATED_AUDIO_CONTRACT.maxBufferedSamples) {
|
|
2711
|
+
await new Promise((resolve) => waiters.add(resolve));
|
|
2712
|
+
}
|
|
2713
|
+
if (!active(current)) break;
|
|
2714
|
+
if (!count) {
|
|
2715
|
+
if (chunk.length) carry = chunk[0];
|
|
2716
|
+
continue;
|
|
2717
|
+
}
|
|
2718
|
+
const buffer = ctx.createBuffer(header.channels, count, header.sample_rate);
|
|
2719
|
+
const data = buffer.getChannelData(0);
|
|
2720
|
+
let cursor = 0;
|
|
2721
|
+
for (let i = 0; i < count; i++) {
|
|
2722
|
+
const low = carry ?? chunk[cursor++];
|
|
2723
|
+
carry = void 0;
|
|
2724
|
+
const raw = low | chunk[cursor++] << 8;
|
|
2725
|
+
data[i] = (raw >= 32768 ? raw - 65536 : raw) / 32768;
|
|
2726
|
+
}
|
|
2727
|
+
if (cursor < chunk.length) carry = chunk[cursor];
|
|
2728
|
+
const node = ctx.createBufferSource();
|
|
2729
|
+
node.buffer = buffer;
|
|
2730
|
+
node.connect(ctx.destination);
|
|
2731
|
+
const start = Math.max(ctx.currentTime, nextStart);
|
|
2732
|
+
current.segments.push({ node, start, offset: current.received, length: count });
|
|
2733
|
+
current.received += count;
|
|
2734
|
+
nextStart = start + count / header.sample_rate;
|
|
2735
|
+
node.onended = () => tick();
|
|
2736
|
+
node.start(start);
|
|
2737
|
+
while (active(current) && buffered() > CORRELATED_AUDIO_CONTRACT.maxBufferedSamples - CORRELATED_AUDIO_CONTRACT.maxChunkBytes / 2) {
|
|
2738
|
+
await new Promise((resolve) => waiters.add(resolve));
|
|
2739
|
+
}
|
|
2740
|
+
}
|
|
2741
|
+
if (!active(current)) return;
|
|
2742
|
+
if (carry !== void 0 || current.received === 0) throw new Error("Incomplete PCM stream");
|
|
2743
|
+
current.eof = true;
|
|
2744
|
+
wake();
|
|
2745
|
+
tick(false);
|
|
2746
|
+
} catch {
|
|
2747
|
+
if (active(current)) invalidate("invalid_stream");
|
|
2748
|
+
}
|
|
2749
|
+
}
|
|
2750
|
+
const cancel = async (data) => {
|
|
2751
|
+
if (disposed || binding.unavailable || room.state !== "connected" || data.callerIdentity !== options.agentIdentity) throw new Error("Unbound audio cancellation");
|
|
2752
|
+
const value = JSON.parse(data.payload);
|
|
2753
|
+
const frame2 = correlatedAudioCancelSchema.parse(value);
|
|
2754
|
+
if (frame2.session_id !== options.sessionId) throw new Error("Wrong audio session");
|
|
2755
|
+
if (frame2.epoch <= binding.epoch) return "";
|
|
2756
|
+
binding.epoch = frame2.epoch;
|
|
2757
|
+
invalidate("cancelled");
|
|
2758
|
+
return "";
|
|
2759
|
+
};
|
|
2760
|
+
const lost = () => invalidate("connection_lost");
|
|
2761
|
+
const changed = () => {
|
|
2762
|
+
if (["disconnected", "reconnecting", "signalReconnecting"].includes(room.state)) lost();
|
|
2763
|
+
};
|
|
2764
|
+
const participantLeft = (participant) => {
|
|
2765
|
+
if (participant.identity === options.agentIdentity) lost();
|
|
2766
|
+
};
|
|
2767
|
+
let registered = false;
|
|
2768
|
+
try {
|
|
2769
|
+
room.registerByteStreamHandler(CORRELATED_AUDIO_CONTRACT.topic, (reader, participant) => {
|
|
2770
|
+
void receive(reader, participant);
|
|
2771
|
+
});
|
|
2772
|
+
registered = true;
|
|
2773
|
+
room.localParticipant.registerRpcMethod(CORRELATED_AUDIO_CONTRACT.cancelMethod, cancel);
|
|
2774
|
+
} catch (error) {
|
|
2775
|
+
if (registered) room.unregisterByteStreamHandler(CORRELATED_AUDIO_CONTRACT.topic);
|
|
2776
|
+
attachments.delete(room);
|
|
2777
|
+
throw error;
|
|
2778
|
+
}
|
|
2779
|
+
room.on(RoomEvent.Reconnecting, lost);
|
|
2780
|
+
room.on(RoomEvent.SignalReconnecting, lost);
|
|
2781
|
+
room.on(RoomEvent.Disconnected, lost);
|
|
2782
|
+
room.on(RoomEvent.ConnectionStateChanged, changed);
|
|
2783
|
+
room.on(RoomEvent.ParticipantDisconnected, participantLeft);
|
|
2784
|
+
const timer = setInterval(() => tick(), 100);
|
|
2785
|
+
let frame = 0;
|
|
2786
|
+
const render = () => {
|
|
2787
|
+
if (disposed) return;
|
|
2788
|
+
tick(false);
|
|
2789
|
+
frame = requestAnimationFrame(render);
|
|
2790
|
+
};
|
|
2791
|
+
frame = requestAnimationFrame(render);
|
|
2792
|
+
return {
|
|
2793
|
+
startAudio,
|
|
2794
|
+
dispose() {
|
|
2795
|
+
if (closing) return closing;
|
|
2796
|
+
invalidate("disposed");
|
|
2797
|
+
disposed = true;
|
|
2798
|
+
clearInterval(timer);
|
|
2799
|
+
cancelAnimationFrame(frame);
|
|
2800
|
+
room.unregisterByteStreamHandler(CORRELATED_AUDIO_CONTRACT.topic);
|
|
2801
|
+
room.localParticipant.unregisterRpcMethod(CORRELATED_AUDIO_CONTRACT.cancelMethod);
|
|
2802
|
+
room.off(RoomEvent.Reconnecting, lost);
|
|
2803
|
+
room.off(RoomEvent.SignalReconnecting, lost);
|
|
2804
|
+
room.off(RoomEvent.Disconnected, lost);
|
|
2805
|
+
room.off(RoomEvent.ConnectionStateChanged, changed);
|
|
2806
|
+
room.off(RoomEvent.ParticipantDisconnected, participantLeft);
|
|
2807
|
+
attachments.delete(room);
|
|
2808
|
+
if (context) context.onstatechange = null;
|
|
2809
|
+
closing = context?.close().catch(() => {
|
|
2810
|
+
}) ?? Promise.resolve();
|
|
2811
|
+
return closing;
|
|
2812
|
+
}
|
|
2813
|
+
};
|
|
2814
|
+
}
|
|
2815
|
+
|
|
2816
|
+
// ../client/src/external-performance.ts
|
|
2817
|
+
var EXTERNAL_PERFORMANCE_KIND = externalPerformanceFrameSchema.shape.kind.value;
|
|
2818
|
+
var EXTERNAL_PERFORMANCE_TIMING = Object.freeze({
|
|
2819
|
+
receiverPlayout: "unsupported",
|
|
2820
|
+
audibleCheckpoint: externalPerformanceFrameSchema.shape.audible_checkpoint.value,
|
|
2821
|
+
audioMapping: externalPerformanceFrameSchema.shape.audio_mapping.value
|
|
2822
|
+
});
|
|
2823
|
+
var statusSchema = externalPerformanceFrameSchema.shape.status;
|
|
2824
|
+
var KNOWN_EXTERNAL_PERFORMANCE_STATUSES = Object.freeze([...statusSchema.options[0].options]);
|
|
2825
|
+
var knownStatuses = new Set(KNOWN_EXTERNAL_PERFORMANCE_STATUSES);
|
|
2826
|
+
function isObservable(frame) {
|
|
2827
|
+
return knownStatuses.has(frame.status);
|
|
2828
|
+
}
|
|
2829
|
+
var NEXT = {
|
|
2830
|
+
unseen: /* @__PURE__ */ new Set(["accepted", "dropped"]),
|
|
2831
|
+
accepted: /* @__PURE__ */ new Set(["audio_released", "cancelled"]),
|
|
2832
|
+
audio_released: /* @__PURE__ */ new Set(["cancelled"]),
|
|
2833
|
+
dropped: /* @__PURE__ */ new Set(),
|
|
2834
|
+
cancelled: /* @__PURE__ */ new Set(),
|
|
2835
|
+
retired: /* @__PURE__ */ new Set()
|
|
2836
|
+
};
|
|
2837
|
+
var LEDGER_LIMIT = 4096;
|
|
2838
|
+
function deliver(callback) {
|
|
2839
|
+
try {
|
|
2840
|
+
void Promise.resolve(callback()).catch(() => {
|
|
2841
|
+
});
|
|
2842
|
+
} catch {
|
|
2843
|
+
}
|
|
2844
|
+
}
|
|
2845
|
+
function createExternalPerformanceObserver(sessionId, callbacks) {
|
|
2846
|
+
const generations = /* @__PURE__ */ new Map();
|
|
2847
|
+
let state = "live";
|
|
2848
|
+
return {
|
|
2849
|
+
receive(frame) {
|
|
2850
|
+
if (state !== "live" || frame.session_id !== sessionId || !isObservable(frame)) return;
|
|
2851
|
+
const previous = generations.get(frame.generation_id) ?? "unseen";
|
|
2852
|
+
if (!NEXT[previous].has(frame.status)) return;
|
|
2853
|
+
if (previous === "unseen" && generations.size >= LEDGER_LIMIT) {
|
|
2854
|
+
const oldest = generations.keys().next();
|
|
2855
|
+
if (!oldest.done) generations.delete(oldest.value);
|
|
2856
|
+
}
|
|
2857
|
+
generations.set(frame.generation_id, frame.status);
|
|
2858
|
+
deliver(() => callbacks.onFrame(frame));
|
|
2859
|
+
},
|
|
2860
|
+
invalidate(reason) {
|
|
2861
|
+
if (state === "closed" || state === "paused" && reason === "connection_lost") return;
|
|
2862
|
+
state = reason === "connection_lost" ? "paused" : "closed";
|
|
2863
|
+
if (state === "closed") generations.clear();
|
|
2864
|
+
else for (const id of generations.keys()) generations.set(id, "retired");
|
|
2865
|
+
deliver(() => callbacks.onInvalidated(reason));
|
|
2866
|
+
},
|
|
2867
|
+
/** The call is connected to its agent again. Only a connection loss can be resumed. */
|
|
2868
|
+
resume() {
|
|
2869
|
+
if (state === "paused") state = "live";
|
|
2870
|
+
}
|
|
2871
|
+
};
|
|
2872
|
+
}
|
|
2419
2873
|
|
|
2420
2874
|
// ../client/src/react/session-lifecycle.ts
|
|
2421
2875
|
function phaseForReason(reason) {
|
|
@@ -2606,6 +3060,32 @@ function useSessionLifecycle(input) {
|
|
|
2606
3060
|
const grantIdRef = useRef(null);
|
|
2607
3061
|
grantIdRef.current = grantId;
|
|
2608
3062
|
const retiredGrantRef = useRef(null);
|
|
3063
|
+
const performanceEnabled = input.externalPerformance !== void 0;
|
|
3064
|
+
const performanceCallbacksRef = useRef(input.externalPerformance);
|
|
3065
|
+
if (input.externalPerformance) performanceCallbacksRef.current = input.externalPerformance;
|
|
3066
|
+
const performanceRef = useRef(null);
|
|
3067
|
+
const endedPerformanceGrantRef = useRef(null);
|
|
3068
|
+
useEffect(() => {
|
|
3069
|
+
if (!grantId || !performanceEnabled || endedPerformanceGrantRef.current === grantId) return;
|
|
3070
|
+
const observer = createExternalPerformanceObserver(grantId, {
|
|
3071
|
+
onFrame: (frame) => performanceCallbacksRef.current?.onFrame(frame),
|
|
3072
|
+
onInvalidated: (reason) => {
|
|
3073
|
+
if (reason === "ended") endedPerformanceGrantRef.current = grantId;
|
|
3074
|
+
performanceCallbacksRef.current?.onInvalidated(reason);
|
|
3075
|
+
}
|
|
3076
|
+
});
|
|
3077
|
+
performanceRef.current = observer;
|
|
3078
|
+
return () => {
|
|
3079
|
+
observer.invalidate("binding_changed");
|
|
3080
|
+
if (performanceRef.current === observer) performanceRef.current = null;
|
|
3081
|
+
};
|
|
3082
|
+
}, [grantId, performanceEnabled]);
|
|
3083
|
+
const invalidateExternalPerformance = useCallback((reason) => {
|
|
3084
|
+
if (grantId === grantIdRef.current) performanceRef.current?.invalidate(reason);
|
|
3085
|
+
}, [grantId]);
|
|
3086
|
+
const resumeExternalPerformance = useCallback(() => {
|
|
3087
|
+
if (grantId === grantIdRef.current) performanceRef.current?.resume();
|
|
3088
|
+
}, [grantId]);
|
|
2609
3089
|
const [recovery, setRecovery] = useState({ kind: "connected" });
|
|
2610
3090
|
const recoveryRef = useRef(recovery);
|
|
2611
3091
|
recoveryRef.current = recovery;
|
|
@@ -2651,6 +3131,7 @@ function useSessionLifecycle(input) {
|
|
|
2651
3131
|
}
|
|
2652
3132
|
}, []);
|
|
2653
3133
|
const reset = useCallback(() => {
|
|
3134
|
+
invalidateExternalPerformance("binding_changed");
|
|
2654
3135
|
clearTimer();
|
|
2655
3136
|
connectedRef.current = false;
|
|
2656
3137
|
manualReconnectPendingRef.current = false;
|
|
@@ -2665,7 +3146,7 @@ function useSessionLifecycle(input) {
|
|
|
2665
3146
|
setConnected(false);
|
|
2666
3147
|
setAgentPresentState(false);
|
|
2667
3148
|
lastActivityRef.current = Date.now();
|
|
2668
|
-
}, [clearTimer]);
|
|
3149
|
+
}, [clearTimer, invalidateExternalPerformance]);
|
|
2669
3150
|
useEffect(() => {
|
|
2670
3151
|
if (active) return;
|
|
2671
3152
|
userEndedRef.current = false;
|
|
@@ -2699,6 +3180,7 @@ function useSessionLifecycle(input) {
|
|
|
2699
3180
|
}, [grantId]);
|
|
2700
3181
|
const endIdle = useCallback(() => {
|
|
2701
3182
|
if (!connectedRef.current) return;
|
|
3183
|
+
invalidateExternalPerformance("ended");
|
|
2702
3184
|
clearTimer();
|
|
2703
3185
|
connectedRef.current = false;
|
|
2704
3186
|
setConnected(false);
|
|
@@ -2706,7 +3188,7 @@ function useSessionLifecycle(input) {
|
|
|
2706
3188
|
releaseRef.current("idle_timeout");
|
|
2707
3189
|
setRecovery({ kind: "ended", reason: "idle" });
|
|
2708
3190
|
leaveRoomRef.current?.();
|
|
2709
|
-
}, [clearTimer]);
|
|
3191
|
+
}, [clearTimer, invalidateExternalPerformance]);
|
|
2710
3192
|
const onConnected = useCallback(() => {
|
|
2711
3193
|
if (grantId === null || grantIdRef.current !== grantId) return;
|
|
2712
3194
|
if (userEndedRef.current || readinessFailureRef.current || retiredGrantRef.current !== null && retiredGrantRef.current === grantIdRef.current) return;
|
|
@@ -2724,6 +3206,7 @@ function useSessionLifecycle(input) {
|
|
|
2724
3206
|
if (grantId === null || grantIdRef.current !== grantId) return;
|
|
2725
3207
|
if (!active || userEndedRef.current || readinessFailureRef.current) return;
|
|
2726
3208
|
const action = disconnectAction(reason, connectedRef.current);
|
|
3209
|
+
invalidateExternalPerformance(action === "end" ? "ended" : "connection_lost");
|
|
2727
3210
|
if (action === "noop") return;
|
|
2728
3211
|
if (action === "reset") {
|
|
2729
3212
|
if (recoveryRef.current.kind === "refreshing" || recoveryRef.current.kind === "reconnecting") return;
|
|
@@ -2743,12 +3226,13 @@ function useSessionLifecycle(input) {
|
|
|
2743
3226
|
refreshBaselineCapacityRef.current = null;
|
|
2744
3227
|
setRecovery({ kind: "reconnecting", attempt: attemptRef.current });
|
|
2745
3228
|
},
|
|
2746
|
-
[active, reset, grantId]
|
|
3229
|
+
[active, reset, grantId, invalidateExternalPerformance]
|
|
2747
3230
|
);
|
|
2748
3231
|
const onConnectionError = useCallback(
|
|
2749
3232
|
(error) => {
|
|
2750
3233
|
if (grantId === null || grantIdRef.current !== grantId) return;
|
|
2751
3234
|
if (!active || userEndedRef.current || readinessFailureRef.current || !isRecoverableConnectionError(error)) return;
|
|
3235
|
+
invalidateExternalPerformance("connection_lost");
|
|
2752
3236
|
connectedRef.current = false;
|
|
2753
3237
|
setConnected(false);
|
|
2754
3238
|
setAgentPresentState(false);
|
|
@@ -2759,7 +3243,7 @@ function useSessionLifecycle(input) {
|
|
|
2759
3243
|
return { kind: "reconnecting", attempt: attemptRef.current };
|
|
2760
3244
|
});
|
|
2761
3245
|
},
|
|
2762
|
-
[active, grantId]
|
|
3246
|
+
[active, grantId, invalidateExternalPerformance]
|
|
2763
3247
|
);
|
|
2764
3248
|
useEffect(() => {
|
|
2765
3249
|
if (capacity.kind === "error" && recovery.kind === "connected") manualReconnectPendingRef.current = false;
|
|
@@ -2779,6 +3263,9 @@ function useSessionLifecycle(input) {
|
|
|
2779
3263
|
onConnected();
|
|
2780
3264
|
return;
|
|
2781
3265
|
}
|
|
3266
|
+
if (state === "disconnected" && connectedRef.current || state === "reconnecting" || state === "signalReconnecting") {
|
|
3267
|
+
invalidateExternalPerformance("connection_lost");
|
|
3268
|
+
}
|
|
2782
3269
|
if (state === "reconnecting" || state === "signalReconnecting") {
|
|
2783
3270
|
if (connectedRef.current) {
|
|
2784
3271
|
setConnected(false);
|
|
@@ -2788,7 +3275,7 @@ function useSessionLifecycle(input) {
|
|
|
2788
3275
|
}
|
|
2789
3276
|
}
|
|
2790
3277
|
},
|
|
2791
|
-
[active, onConnected, grantId]
|
|
3278
|
+
[active, onConnected, grantId, invalidateExternalPerformance]
|
|
2792
3279
|
);
|
|
2793
3280
|
useEffect(() => {
|
|
2794
3281
|
if (!active || userEnded || readinessFailed || !connected || !agentPresent || !grantState.grant) return;
|
|
@@ -2801,6 +3288,7 @@ function useSessionLifecycle(input) {
|
|
|
2801
3288
|
const failReadiness = useCallback((code) => {
|
|
2802
3289
|
if (userEndedRef.current || readinessFailureRef.current) return;
|
|
2803
3290
|
readinessFailureRef.current = code;
|
|
3291
|
+
invalidateExternalPerformance("ended");
|
|
2804
3292
|
clearTimer();
|
|
2805
3293
|
connectedRef.current = false;
|
|
2806
3294
|
manualReconnectPendingRef.current = false;
|
|
@@ -2813,7 +3301,7 @@ function useSessionLifecycle(input) {
|
|
|
2813
3301
|
releaseRef.current("disconnected");
|
|
2814
3302
|
setReadinessFailed(true);
|
|
2815
3303
|
leaveRoomRef.current?.();
|
|
2816
|
-
}, [clearTimer]);
|
|
3304
|
+
}, [clearTimer, invalidateExternalPerformance]);
|
|
2817
3305
|
useEffect(() => {
|
|
2818
3306
|
if (!active || userEnded || readinessFailed) return;
|
|
2819
3307
|
if (recovery.kind === "ended" || recovery.kind === "failed") return;
|
|
@@ -2894,6 +3382,7 @@ function useSessionLifecycle(input) {
|
|
|
2894
3382
|
const end = useCallback(() => {
|
|
2895
3383
|
if (!active || userEndedRef.current || phaseRef.current.kind === "ended") return;
|
|
2896
3384
|
userEndedRef.current = true;
|
|
3385
|
+
invalidateExternalPerformance("ended");
|
|
2897
3386
|
clearTimer();
|
|
2898
3387
|
connectedRef.current = false;
|
|
2899
3388
|
manualReconnectPendingRef.current = false;
|
|
@@ -2904,7 +3393,7 @@ function useSessionLifecycle(input) {
|
|
|
2904
3393
|
setRecovery({ kind: "ended", reason: "user" });
|
|
2905
3394
|
clearRef.current();
|
|
2906
3395
|
leaveRoomRef.current?.();
|
|
2907
|
-
}, [active, clearTimer]);
|
|
3396
|
+
}, [active, clearTimer, invalidateExternalPerformance]);
|
|
2908
3397
|
const reconnect = useCallback(() => {
|
|
2909
3398
|
if (!active) return;
|
|
2910
3399
|
if (readinessFailureRef.current) {
|
|
@@ -2974,16 +3463,22 @@ function useSessionLifecycle(input) {
|
|
|
2974
3463
|
const lastBehaviorRef = useRef(null);
|
|
2975
3464
|
const behaviorEnabled = Boolean(onBehaviorChange);
|
|
2976
3465
|
const onLifecycleData = useMemo(() => {
|
|
2977
|
-
if (!behaviorEnabled) return void 0;
|
|
3466
|
+
if (!behaviorEnabled && !performanceEnabled) return void 0;
|
|
2978
3467
|
return (frame) => {
|
|
2979
3468
|
const parsed = lifecycleServerFrameSchema.safeParse(frame);
|
|
2980
|
-
if (!parsed.success
|
|
3469
|
+
if (!parsed.success) return;
|
|
3470
|
+
if (parsed.data.kind === "external_performance") {
|
|
3471
|
+
if (grantId === grantIdRef.current && parsed.data.session_id === grantId) performanceRef.current?.receive(parsed.data);
|
|
3472
|
+
return;
|
|
3473
|
+
}
|
|
3474
|
+
if (parsed.data.kind === "ended") invalidateExternalPerformance("ended");
|
|
3475
|
+
if (!behaviorEnabled || parsed.data.kind !== "behavior_state") return;
|
|
2981
3476
|
const next = nextBehaviorSnapshot(lastBehaviorRef.current, parsed.data);
|
|
2982
3477
|
if (!next) return;
|
|
2983
3478
|
lastBehaviorRef.current = next;
|
|
2984
3479
|
onBehaviorChangeRef.current?.(next);
|
|
2985
3480
|
};
|
|
2986
|
-
}, [behaviorEnabled]);
|
|
3481
|
+
}, [behaviorEnabled, performanceEnabled, grantId, invalidateExternalPerformance]);
|
|
2987
3482
|
return useMemo(
|
|
2988
3483
|
() => ({
|
|
2989
3484
|
phase,
|
|
@@ -3002,7 +3497,9 @@ function useSessionLifecycle(input) {
|
|
|
3002
3497
|
setAgentPresent,
|
|
3003
3498
|
registerLeaveRoom,
|
|
3004
3499
|
reset,
|
|
3005
|
-
onLifecycleData
|
|
3500
|
+
onLifecycleData,
|
|
3501
|
+
invalidateExternalPerformance: performanceEnabled ? invalidateExternalPerformance : void 0,
|
|
3502
|
+
resumeExternalPerformance: performanceEnabled ? resumeExternalPerformance : void 0
|
|
3006
3503
|
}),
|
|
3007
3504
|
[
|
|
3008
3505
|
phase,
|
|
@@ -3021,7 +3518,10 @@ function useSessionLifecycle(input) {
|
|
|
3021
3518
|
setAgentPresent,
|
|
3022
3519
|
registerLeaveRoom,
|
|
3023
3520
|
reset,
|
|
3024
|
-
onLifecycleData
|
|
3521
|
+
onLifecycleData,
|
|
3522
|
+
invalidateExternalPerformance,
|
|
3523
|
+
resumeExternalPerformance,
|
|
3524
|
+
performanceEnabled
|
|
3025
3525
|
]
|
|
3026
3526
|
);
|
|
3027
3527
|
}
|
|
@@ -3050,9 +3550,45 @@ function SessionLifecycleRoomBridge({
|
|
|
3050
3550
|
const transcriptions = useTranscriptions();
|
|
3051
3551
|
const room = useRoomContext();
|
|
3052
3552
|
const roomMatchesGrant = lifecycle.grant === void 0 || lifecycle.grant !== null && room.name === lifecycle.grant.room_name;
|
|
3553
|
+
const correlatedRef = useRef(null);
|
|
3554
|
+
const correlatedObserver = useRef(lifecycle.correlatedAudio);
|
|
3555
|
+
correlatedObserver.current = lifecycle.correlatedAudio;
|
|
3556
|
+
const correlatedEnabled = lifecycle.correlatedAudio !== void 0;
|
|
3557
|
+
const correlatedSessionId = lifecycle.grant?.session_id;
|
|
3558
|
+
const startCorrelatedAudio = useCallback(async () => {
|
|
3559
|
+
const player = correlatedRef.current;
|
|
3560
|
+
if (!player) return false;
|
|
3561
|
+
try {
|
|
3562
|
+
await player.startAudio();
|
|
3563
|
+
return true;
|
|
3564
|
+
} catch {
|
|
3565
|
+
return false;
|
|
3566
|
+
}
|
|
3567
|
+
}, []);
|
|
3568
|
+
useEffect(() => {
|
|
3569
|
+
if (!correlatedEnabled || !correlatedSessionId || !roomMatchesGrant || !assistant.agent) return;
|
|
3570
|
+
const player = attachCorrelatedAudio(room, {
|
|
3571
|
+
sessionId: correlatedSessionId,
|
|
3572
|
+
agentIdentity: assistant.agent.identity,
|
|
3573
|
+
prepare: (header, signal) => correlatedObserver.current?.prepare?.(header, signal) ?? Promise.resolve(),
|
|
3574
|
+
onPosition: (position) => correlatedObserver.current?.onPosition(position),
|
|
3575
|
+
onInvalidated: (reason) => correlatedObserver.current?.onInvalidated(reason),
|
|
3576
|
+
onPlaybackBlocked: (unblock) => {
|
|
3577
|
+
setAudioPlayback?.(unblock ? "blocked" : "allowed");
|
|
3578
|
+
correlatedObserver.current?.onPlaybackBlocked?.(unblock);
|
|
3579
|
+
}
|
|
3580
|
+
});
|
|
3581
|
+
correlatedRef.current = player;
|
|
3582
|
+
return () => {
|
|
3583
|
+
correlatedRef.current = null;
|
|
3584
|
+
void player.dispose();
|
|
3585
|
+
setAudioPlayback?.("unknown");
|
|
3586
|
+
};
|
|
3587
|
+
}, [room, assistant.agent, correlatedEnabled, correlatedSessionId, roomMatchesGrant, setAudioPlayback]);
|
|
3053
3588
|
useCallMedia(room, {
|
|
3054
3589
|
setMicrophoneFacts,
|
|
3055
|
-
setAudioPlayback,
|
|
3590
|
+
setAudioPlayback: correlatedEnabled ? void 0 : setAudioPlayback,
|
|
3591
|
+
startAudioOverride: correlatedEnabled ? startCorrelatedAudio : void 0,
|
|
3056
3592
|
registerMediaControls,
|
|
3057
3593
|
microphoneWanted: microphone ?? publishesMicrophoneByDefault(lifecycle.grant),
|
|
3058
3594
|
connectionState,
|
|
@@ -3066,7 +3602,7 @@ function SessionLifecycleRoomBridge({
|
|
|
3066
3602
|
if (!mediaModeEnabled) return;
|
|
3067
3603
|
const agent = assistant.agent;
|
|
3068
3604
|
let previous;
|
|
3069
|
-
const
|
|
3605
|
+
const deliver2 = (mode) => {
|
|
3070
3606
|
if (mode === previous) return;
|
|
3071
3607
|
previous = mode;
|
|
3072
3608
|
try {
|
|
@@ -3077,7 +3613,7 @@ function SessionLifecycleRoomBridge({
|
|
|
3077
3613
|
};
|
|
3078
3614
|
const read = () => {
|
|
3079
3615
|
const value = agent?.attributes?.["rta.media_mode"];
|
|
3080
|
-
|
|
3616
|
+
deliver2(value === "video" ? "avatar" : value === "voice" ? "voice" : null);
|
|
3081
3617
|
};
|
|
3082
3618
|
const changed = (_attributes, participant) => {
|
|
3083
3619
|
if (participant === agent) read();
|
|
@@ -3086,7 +3622,7 @@ function SessionLifecycleRoomBridge({
|
|
|
3086
3622
|
read();
|
|
3087
3623
|
return () => {
|
|
3088
3624
|
room.off(RoomEvent.ParticipantAttributesChanged, changed);
|
|
3089
|
-
|
|
3625
|
+
deliver2(null);
|
|
3090
3626
|
};
|
|
3091
3627
|
}, [room, assistant.agent, mediaModeEnabled, mediaModeSessionId]);
|
|
3092
3628
|
const { send } = useChat();
|
|
@@ -3133,7 +3669,7 @@ function SessionLifecycleRoomBridge({
|
|
|
3133
3669
|
let closed = false;
|
|
3134
3670
|
let pending = false;
|
|
3135
3671
|
let previous;
|
|
3136
|
-
const
|
|
3672
|
+
const deliver2 = (details) => {
|
|
3137
3673
|
connectionHistoryRef.current?.enqueue(details);
|
|
3138
3674
|
try {
|
|
3139
3675
|
if (onConnectionDetailsChange) void Promise.resolve(callback?.(details)).catch(() => {
|
|
@@ -3160,7 +3696,7 @@ function SessionLifecycleRoomBridge({
|
|
|
3160
3696
|
const last = previous;
|
|
3161
3697
|
if (last?.connectionState === next.connectionState && last.localQuality === next.localQuality && ["audio", "video"].every((kind) => last[kind]?.publisherQuality === next[kind]?.publisherQuality && last[kind]?.streamState === next[kind]?.streamState)) return;
|
|
3162
3698
|
previous = next;
|
|
3163
|
-
|
|
3699
|
+
deliver2(next);
|
|
3164
3700
|
});
|
|
3165
3701
|
};
|
|
3166
3702
|
const onQuality = (_quality, participant) => {
|
|
@@ -3190,7 +3726,7 @@ function SessionLifecycleRoomBridge({
|
|
|
3190
3726
|
room.off(RoomEvent.TrackStreamStateChanged, onPublication);
|
|
3191
3727
|
room.off(RoomEvent.TrackSubscribed, onTrack);
|
|
3192
3728
|
room.off(RoomEvent.TrackUnsubscribed, onTrack);
|
|
3193
|
-
|
|
3729
|
+
deliver2(null);
|
|
3194
3730
|
};
|
|
3195
3731
|
}, [room, detailsSessionId, detailsEnabled]);
|
|
3196
3732
|
useEffect(() => {
|
|
@@ -3227,18 +3763,56 @@ function SessionLifecycleRoomBridge({
|
|
|
3227
3763
|
useEffect(() => {
|
|
3228
3764
|
if (!onLifecycleData) return;
|
|
3229
3765
|
const decoder = new TextDecoder();
|
|
3230
|
-
const handler = (payload,
|
|
3766
|
+
const handler = (payload, participant, _kind, topic) => {
|
|
3231
3767
|
if (topic !== RTA_LIFECYCLE_TOPIC) return;
|
|
3232
3768
|
try {
|
|
3233
|
-
|
|
3769
|
+
const frame = JSON.parse(decoder.decode(payload));
|
|
3770
|
+
if (typeof frame === "object" && frame !== null && "kind" in frame && (frame.kind === EXTERNAL_PERFORMANCE_KIND || frame.kind === "ended") && (!assistant.agent || participant !== assistant.agent || !roomMatchesGrant || room.state !== "connected")) return;
|
|
3771
|
+
if (typeof frame === "object" && frame !== null && "kind" in frame && frame.kind === "ended" && lifecycleServerFrameSchema.safeParse(frame).success) {
|
|
3772
|
+
void correlatedRef.current?.dispose();
|
|
3773
|
+
correlatedRef.current = null;
|
|
3774
|
+
}
|
|
3775
|
+
onLifecycleData(frame);
|
|
3234
3776
|
} catch {
|
|
3235
3777
|
}
|
|
3236
3778
|
};
|
|
3779
|
+
const invalidate = () => lifecycle.invalidateExternalPerformance?.("connection_lost");
|
|
3780
|
+
const changed = () => {
|
|
3781
|
+
if (room.state === "disconnected" || room.state === "reconnecting" || room.state === "signalReconnecting") invalidate();
|
|
3782
|
+
};
|
|
3237
3783
|
room.on(RoomEvent.DataReceived, handler);
|
|
3784
|
+
if (lifecycle.invalidateExternalPerformance) {
|
|
3785
|
+
room.on(RoomEvent.Reconnecting, invalidate);
|
|
3786
|
+
room.on(RoomEvent.SignalReconnecting, invalidate);
|
|
3787
|
+
room.on(RoomEvent.Disconnected, invalidate);
|
|
3788
|
+
room.on(RoomEvent.ConnectionStateChanged, changed);
|
|
3789
|
+
}
|
|
3238
3790
|
return () => {
|
|
3239
3791
|
room.off(RoomEvent.DataReceived, handler);
|
|
3792
|
+
room.off(RoomEvent.Reconnecting, invalidate);
|
|
3793
|
+
room.off(RoomEvent.SignalReconnecting, invalidate);
|
|
3794
|
+
room.off(RoomEvent.Disconnected, invalidate);
|
|
3795
|
+
room.off(RoomEvent.ConnectionStateChanged, changed);
|
|
3240
3796
|
};
|
|
3241
|
-
}, [room, onLifecycleData]);
|
|
3797
|
+
}, [room, onLifecycleData, assistant.agent, roomMatchesGrant, lifecycle.invalidateExternalPerformance]);
|
|
3798
|
+
useEffect(() => {
|
|
3799
|
+
if (!assistant.agent) return;
|
|
3800
|
+
return () => lifecycle.invalidateExternalPerformance?.("connection_lost");
|
|
3801
|
+
}, [assistant.agent, lifecycle.invalidateExternalPerformance]);
|
|
3802
|
+
const resumeExternalPerformance = lifecycle.resumeExternalPerformance;
|
|
3803
|
+
useEffect(() => {
|
|
3804
|
+
if (!resumeExternalPerformance) return;
|
|
3805
|
+
const resumeIfBound = () => {
|
|
3806
|
+
if (assistant.agent && roomMatchesGrant && room.state === "connected") resumeExternalPerformance();
|
|
3807
|
+
};
|
|
3808
|
+
resumeIfBound();
|
|
3809
|
+
room.on(RoomEvent.Reconnected, resumeIfBound);
|
|
3810
|
+
room.on(RoomEvent.ConnectionStateChanged, resumeIfBound);
|
|
3811
|
+
return () => {
|
|
3812
|
+
room.off(RoomEvent.Reconnected, resumeIfBound);
|
|
3813
|
+
room.off(RoomEvent.ConnectionStateChanged, resumeIfBound);
|
|
3814
|
+
};
|
|
3815
|
+
}, [room, assistant.agent, roomMatchesGrant, resumeExternalPerformance]);
|
|
3242
3816
|
useEffect(() => {
|
|
3243
3817
|
if (!registerDataPublisher) return;
|
|
3244
3818
|
const encoder = new TextEncoder();
|
|
@@ -3374,6 +3948,7 @@ function useRealtimeSession(input) {
|
|
|
3374
3948
|
onBehaviorChange,
|
|
3375
3949
|
onClipResult,
|
|
3376
3950
|
onVoiceError,
|
|
3951
|
+
correlatedAudio,
|
|
3377
3952
|
// everything else (client/active/idleSeconds/idleWarnLeadSeconds/reconnect/requestOptions…)
|
|
3378
3953
|
// flows straight into the inner SSOT hook.
|
|
3379
3954
|
...inner
|
|
@@ -3446,6 +4021,7 @@ function useRealtimeSession(input) {
|
|
|
3446
4021
|
return `t-${Date.now()}-${++turnSequenceRef.current}`;
|
|
3447
4022
|
}, []);
|
|
3448
4023
|
const onLifecycleData = useCallback((frame) => {
|
|
4024
|
+
lifecycle.onLifecycleData?.(frame);
|
|
3449
4025
|
const parsed = lifecycleServerFrameSchema.safeParse(frame);
|
|
3450
4026
|
if (!parsed.success) return;
|
|
3451
4027
|
const f = parsed.data;
|
|
@@ -3505,7 +4081,7 @@ function useRealtimeSession(input) {
|
|
|
3505
4081
|
break;
|
|
3506
4082
|
}
|
|
3507
4083
|
}
|
|
3508
|
-
}, []);
|
|
4084
|
+
}, [lifecycle.onLifecycleData]);
|
|
3509
4085
|
const registerDataPublisher = useCallback((publish) => {
|
|
3510
4086
|
dataPublisherRef.current = publish;
|
|
3511
4087
|
}, []);
|
|
@@ -3771,6 +4347,7 @@ function useRealtimeSession(input) {
|
|
|
3771
4347
|
// stayConnected/reconnect/markActivity/the sinks) → a true drop-in superset, then
|
|
3772
4348
|
// add the realtime-session surface. `reset` + `lifecycle` come AFTER so ours win.
|
|
3773
4349
|
...lifecycle,
|
|
4350
|
+
correlatedAudio,
|
|
3774
4351
|
turn,
|
|
3775
4352
|
clocks,
|
|
3776
4353
|
endsAt,
|
|
@@ -3805,6 +4382,7 @@ function useRealtimeSession(input) {
|
|
|
3805
4382
|
}),
|
|
3806
4383
|
[
|
|
3807
4384
|
lifecycle,
|
|
4385
|
+
correlatedAudio,
|
|
3808
4386
|
turn,
|
|
3809
4387
|
clocks,
|
|
3810
4388
|
endsAt,
|
|
@@ -3944,4 +4522,4 @@ function createProxyClient(options) {
|
|
|
3944
4522
|
};
|
|
3945
4523
|
}
|
|
3946
4524
|
|
|
3947
|
-
export { AdaptivePlayoutController, AvatarVideoSurface, DEFAULT_APPROACHING_END_LEAD_SECONDS, DEFAULT_AVATAR_PLAYOUT_DELAY_SECONDS, DEFAULT_CREDITS_LOW_LEAD_SECONDS, DEFAULT_FISH_TTS_MODEL, DEFAULT_GOVERNOR_CONFIG, DEFAULT_GRACE_CEILING_SECONDS, DEFAULT_GRACE_WINDOW_LEAD_SECONDS, DEFAULT_IDLE_SECONDS, DEFAULT_IDLE_WARN_LEAD_SECONDS, DEFAULT_READY_TIMEOUT_SECONDS, DEFAULT_TURN_TIMEOUT_SECONDS, FISH_DELIVERY_MODELS, MAX_RECONNECT_ATTEMPTS, MAX_SESSION_INSTRUCTIONS_CHARS, RECONNECT_BACKOFF_MS, RealtimeAvatarLiveKitRoom, SessionLifecycleRoomBridge, capacityErrorFromBusy, capacityStateFromGrant, createProxyClient, fishTtsModelSchema, isNativeLiveTrackSubscribed, knownBehaviorStates, mapTurnState, readInboundRtp, sessionBehaviorSchema, sessionClipSchema, splitCallTranscript, useAvatarAdaptivePlayoutDelay, useAvatarCamera, useAvatarPlayoutDelay, useAvatarQualityGovernor, useCallTranscript, useClearStaleInputMute, useLiveKitAvatarGrant, useLiveTrackProducing, useMicLease, useRealtimeAvatarAudioSession, useRealtimeSession, useReleaseMicLeaseOnTrackEnded, useSessionLifecycle };
|
|
4525
|
+
export { AdaptivePlayoutController, AvatarVideoSurface, DEFAULT_APPROACHING_END_LEAD_SECONDS, DEFAULT_AVATAR_PLAYOUT_DELAY_SECONDS, DEFAULT_CREDITS_LOW_LEAD_SECONDS, DEFAULT_FISH_TTS_MODEL, DEFAULT_GOVERNOR_CONFIG, DEFAULT_GRACE_CEILING_SECONDS, DEFAULT_GRACE_WINDOW_LEAD_SECONDS, DEFAULT_IDLE_SECONDS, DEFAULT_IDLE_WARN_LEAD_SECONDS, DEFAULT_READY_TIMEOUT_SECONDS, DEFAULT_TURN_TIMEOUT_SECONDS, EXTERNAL_PERFORMANCE_TIMING, FISH_DELIVERY_MODELS, KNOWN_EXTERNAL_PERFORMANCE_STATUSES, MAX_RECONNECT_ATTEMPTS, MAX_SESSION_INSTRUCTIONS_CHARS, RECONNECT_BACKOFF_MS, RealtimeAvatarLiveKitRoom, SessionLifecycleRoomBridge, capacityErrorFromBusy, capacityStateFromGrant, createProxyClient, fishTtsModelSchema, isNativeLiveTrackSubscribed, knownBehaviorStates, mapTurnState, readInboundRtp, sessionBehaviorSchema, sessionClipSchema, splitCallTranscript, useAvatarAdaptivePlayoutDelay, useAvatarCamera, useAvatarPlayoutDelay, useAvatarQualityGovernor, useCallTranscript, useClearStaleInputMute, useLiveKitAvatarGrant, useLiveTrackProducing, useMicLease, useRealtimeAvatarAudioSession, useRealtimeSession, useReleaseMicLeaseOnTrackEnded, useSessionLifecycle };
|