realtime-avatar 0.33.0 → 0.35.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +56 -1
- package/dist/browser.d.ts +4 -89
- package/dist/browser.js +362 -1
- package/dist/chunk-6KQS7CTR.js +1 -0
- package/dist/{chunk-DM5TWRC4.js → chunk-ARJEAQQT.js} +1 -13
- package/dist/{chunk-VLYUMP3I.js → chunk-WHYDSORD.js} +426 -170
- package/dist/express.d.ts +2 -2
- package/dist/express.js +2 -2
- package/dist/hono.d.ts +2 -2
- package/dist/hono.js +2 -2
- package/dist/index.d.ts +338 -3
- package/dist/index.js +2 -1
- package/dist/nextjs.d.ts +2 -2
- package/dist/nextjs.js +2 -2
- package/dist/{playout-delay-CtRZdtsj.d.ts → playout-delay-BzoM1BMN.d.ts} +135 -1
- package/dist/{proxy-client-DJhYiiKX.d.ts → proxy-client-CiTOjkg8.d.ts} +67 -3
- package/dist/react-native.d.ts +3 -3
- package/dist/react-native.js +623 -36
- package/dist/react.d.ts +12 -22
- package/dist/react.js +627 -37
- package/dist/server.d.ts +2 -2
- package/dist/server.js +2 -1
- package/dist/tanstack-start.d.ts +2 -2
- package/dist/tanstack-start.js +2 -2
- package/dist/{types-DygYBvSv.d.ts → types-CUvcV0Wq.d.ts} +1 -1
- package/dist/{types-VoejsHij.d.ts → types-CpnLuiJU.d.ts} +220 -18
- package/package.json +3 -3
package/dist/react-native.js
CHANGED
|
@@ -1251,7 +1251,7 @@ function stepNetworkQuality(previous, sample, nowMs, options) {
|
|
|
1251
1251
|
}
|
|
1252
1252
|
|
|
1253
1253
|
// ../client/src/react/network-quality-binding.ts
|
|
1254
|
-
function bindNetworkQuality(publication, inhibited,
|
|
1254
|
+
function bindNetworkQuality(publication, inhibited, notify2, tickMs) {
|
|
1255
1255
|
let disposed = false;
|
|
1256
1256
|
let running = false;
|
|
1257
1257
|
let state = initNetworkQuality(performance.now());
|
|
@@ -1265,7 +1265,7 @@ function bindNetworkQuality(publication, inhibited, notify, tickMs) {
|
|
|
1265
1265
|
};
|
|
1266
1266
|
const emit = (status) => {
|
|
1267
1267
|
try {
|
|
1268
|
-
|
|
1268
|
+
notify2(status);
|
|
1269
1269
|
} catch {
|
|
1270
1270
|
}
|
|
1271
1271
|
};
|
|
@@ -1787,15 +1787,116 @@ var schema5 = z.object({
|
|
|
1787
1787
|
"request_id": schema22.optional()
|
|
1788
1788
|
});
|
|
1789
1789
|
var voiceErrorFrameSchema = schema5;
|
|
1790
|
-
var schema04 = z.
|
|
1791
|
-
var schema13 = z.
|
|
1792
|
-
var schema23 = z.
|
|
1793
|
-
var schema33 = z.
|
|
1794
|
-
var schema43 = z.
|
|
1795
|
-
var schema52 = z.
|
|
1796
|
-
var
|
|
1790
|
+
var schema04 = z.string().min(1).max(200);
|
|
1791
|
+
var schema13 = z.literal(1);
|
|
1792
|
+
var schema23 = z.string().min(1).max(200);
|
|
1793
|
+
var schema33 = z.int().min(1).max(9007199254740991);
|
|
1794
|
+
var schema43 = z.literal(24e3);
|
|
1795
|
+
var schema52 = z.literal("pcm_s16le");
|
|
1796
|
+
var schema6 = z.object({
|
|
1797
|
+
"session_id": schema04,
|
|
1798
|
+
"generation_id": schema04,
|
|
1799
|
+
"version": schema13,
|
|
1800
|
+
"utterance_id": schema23,
|
|
1801
|
+
"stream_id": schema04,
|
|
1802
|
+
"epoch": schema33,
|
|
1803
|
+
"sample_rate": schema43,
|
|
1804
|
+
"channels": schema13,
|
|
1805
|
+
"encoding": schema52
|
|
1806
|
+
});
|
|
1807
|
+
var schema7 = z.enum(["started", "progress", "finished", "cancelled"]);
|
|
1808
|
+
var schema8 = z.int().min(0).max(144e4);
|
|
1809
|
+
z.strictObject({
|
|
1810
|
+
"session_id": schema04,
|
|
1811
|
+
"generation_id": schema04,
|
|
1812
|
+
"utterance_id": schema23,
|
|
1813
|
+
"stream_id": schema04,
|
|
1814
|
+
"epoch": schema33,
|
|
1815
|
+
"status": schema7,
|
|
1816
|
+
"samples": schema8
|
|
1817
|
+
});
|
|
1818
|
+
var schema10 = z.object({
|
|
1819
|
+
"version": schema13,
|
|
1820
|
+
"session_id": schema04,
|
|
1821
|
+
"epoch": schema33
|
|
1822
|
+
});
|
|
1823
|
+
var schema11 = z.literal("external_llm_error");
|
|
1824
|
+
var schema122 = z.enum(["external_llm_refused", "external_llm_failed"]);
|
|
1825
|
+
var schema132 = z.string();
|
|
1826
|
+
var schema14 = z.literal(true);
|
|
1827
|
+
var schema15 = z.object({
|
|
1828
|
+
"kind": schema11,
|
|
1829
|
+
"code": schema122,
|
|
1830
|
+
"message": schema132,
|
|
1831
|
+
"fatal": schema14
|
|
1832
|
+
});
|
|
1833
|
+
var correlatedAudioHeaderSchema = schema6;
|
|
1834
|
+
var correlatedAudioCancelSchema = schema10;
|
|
1835
|
+
var externalLlmErrorFrameSchema = schema15;
|
|
1836
|
+
var CORRELATED_AUDIO_CONTRACT = {
|
|
1837
|
+
"topic": "rta.audio.v1",
|
|
1838
|
+
"receiptMethod": "rta.audio.receipt.v1",
|
|
1839
|
+
"cancelMethod": "rta.audio.cancel.v1",
|
|
1840
|
+
"maxSamples": 144e4,
|
|
1841
|
+
"maxChunkBytes": 48e3,
|
|
1842
|
+
"maxBufferedSamples": 48e3
|
|
1843
|
+
};
|
|
1844
|
+
var schema05 = z.string().min(1).max(200);
|
|
1845
|
+
var schema16 = z.literal(1);
|
|
1846
|
+
var schema24 = z.enum(["respond", "recovery", "initiative"]);
|
|
1847
|
+
var schema34 = z.string().min(1).max(32);
|
|
1848
|
+
var schema44 = z.union([schema24, schema34]);
|
|
1849
|
+
var schema53 = z.enum(["draft", "accepted"]);
|
|
1850
|
+
var schema62 = z.literal("speech_segments_v1");
|
|
1851
|
+
var schema72 = z.literal("external");
|
|
1852
|
+
var schema82 = z.object({
|
|
1853
|
+
"renderer": schema72
|
|
1854
|
+
});
|
|
1855
|
+
z.object({
|
|
1856
|
+
"session_id": schema05,
|
|
1857
|
+
"generation_id": schema05,
|
|
1858
|
+
"version": schema16,
|
|
1859
|
+
"purpose": schema44,
|
|
1860
|
+
"admission": schema53,
|
|
1861
|
+
"text_output": schema62.optional(),
|
|
1862
|
+
"visible_character": schema82.optional()
|
|
1863
|
+
});
|
|
1864
|
+
var schema102 = z.string().min(1).max(200);
|
|
1865
|
+
z.object({
|
|
1866
|
+
"generation_id": schema05,
|
|
1867
|
+
"utterance_id": schema102
|
|
1868
|
+
});
|
|
1869
|
+
var schema123 = z.literal("external_performance");
|
|
1870
|
+
var schema133 = z.enum(["accepted", "dropped", "cancelled", "audio_released"]);
|
|
1871
|
+
var schema142 = z.union([schema133, schema34]);
|
|
1872
|
+
var schema152 = z.literal("unavailable");
|
|
1873
|
+
var schema162 = z.literal("unavailable");
|
|
1874
|
+
var schema17 = z.object({
|
|
1875
|
+
"session_id": schema05,
|
|
1876
|
+
"generation_id": schema05,
|
|
1877
|
+
"kind": schema123,
|
|
1878
|
+
"version": schema16,
|
|
1879
|
+
"status": schema142,
|
|
1880
|
+
"audible_checkpoint": schema152,
|
|
1881
|
+
"audio_mapping": schema162
|
|
1882
|
+
});
|
|
1883
|
+
var schema18 = z.string().regex(new RegExp("^(?:[^\\uD800-\\uDFFF]|[\\uD800-\\uDBFF][\\uDC00-\\uDFFF])*$")).min(1).max(5e3);
|
|
1884
|
+
z.strictObject({
|
|
1885
|
+
"version": schema16,
|
|
1886
|
+
"generation_id": schema05,
|
|
1887
|
+
"utterance_id": schema102,
|
|
1888
|
+
"text": schema18
|
|
1889
|
+
});
|
|
1890
|
+
var externalPerformanceFrameSchema = schema17;
|
|
1891
|
+
var schema06 = z.enum(["s1", "s2-pro", "s2.1-pro", "drama-3"]);
|
|
1892
|
+
var schema110 = z.enum(["speech-1.6", "speech-1.5", "s1-mini"]);
|
|
1893
|
+
var schema25 = z.union([schema06, schema110]);
|
|
1894
|
+
var schema35 = z.enum(["whisper"]);
|
|
1895
|
+
var schema45 = z.null();
|
|
1896
|
+
var schema54 = z.union([schema35, schema45]);
|
|
1897
|
+
var fishTtsModelSchema = schema25;
|
|
1797
1898
|
var DEFAULT_FISH_TTS_MODEL = "s2.1-pro";
|
|
1798
|
-
var fishDeliverySchema =
|
|
1899
|
+
var fishDeliverySchema = schema54;
|
|
1799
1900
|
var FISH_DELIVERY_MODELS = { "whisper": ["drama-3"] };
|
|
1800
1901
|
|
|
1801
1902
|
// ../client/src/wire.ts
|
|
@@ -2111,7 +2212,9 @@ var lifecycleServerFrameSchema = z.discriminatedUnion("kind", [
|
|
|
2111
2212
|
endedFrameSchema,
|
|
2112
2213
|
behaviorStateFrameSchema,
|
|
2113
2214
|
clipAckFrameSchema,
|
|
2114
|
-
voiceErrorFrameSchema2
|
|
2215
|
+
voiceErrorFrameSchema2,
|
|
2216
|
+
externalPerformanceFrameSchema,
|
|
2217
|
+
externalLlmErrorFrameSchema
|
|
2115
2218
|
]);
|
|
2116
2219
|
var liveKitCapacitySnapshotSchema = z.object({
|
|
2117
2220
|
// Placement identity + per-worker session ceiling. These are always present on the
|
|
@@ -2393,6 +2496,7 @@ function useCallMedia(room, options) {
|
|
|
2393
2496
|
}, [room, setAudioPlayback, connecting]);
|
|
2394
2497
|
const controls = useMemo(() => ({
|
|
2395
2498
|
startAudio: async () => {
|
|
2499
|
+
if (options.startAudioOverride) return options.startAudioOverride();
|
|
2396
2500
|
try {
|
|
2397
2501
|
await room.startAudio();
|
|
2398
2502
|
} catch {
|
|
@@ -2409,13 +2513,372 @@ function useCallMedia(room, options) {
|
|
|
2409
2513
|
setVersion((value) => value + 1);
|
|
2410
2514
|
}
|
|
2411
2515
|
}
|
|
2412
|
-
}), [room]);
|
|
2516
|
+
}), [room, options.startAudioOverride]);
|
|
2413
2517
|
useEffect(() => {
|
|
2414
2518
|
if (!registerMediaControls) return;
|
|
2415
2519
|
registerMediaControls(controls);
|
|
2416
2520
|
return () => registerMediaControls(null);
|
|
2417
2521
|
}, [controls, registerMediaControls]);
|
|
2418
2522
|
}
|
|
2523
|
+
var attachments = /* @__PURE__ */ new WeakSet();
|
|
2524
|
+
var bindings = /* @__PURE__ */ new WeakMap();
|
|
2525
|
+
function notify(callback) {
|
|
2526
|
+
try {
|
|
2527
|
+
void Promise.resolve(callback()).catch(() => {
|
|
2528
|
+
});
|
|
2529
|
+
} catch {
|
|
2530
|
+
}
|
|
2531
|
+
}
|
|
2532
|
+
function attachCorrelatedAudio(room, options) {
|
|
2533
|
+
if (attachments.has(room)) throw new Error("Correlated audio is already attached to this room");
|
|
2534
|
+
if (!options.sessionId || !options.agentIdentity) throw new Error("Correlated audio requires the held session and bound agent");
|
|
2535
|
+
attachments.add(room);
|
|
2536
|
+
let context;
|
|
2537
|
+
let disposed = false;
|
|
2538
|
+
const previous = bindings.get(room);
|
|
2539
|
+
const binding = previous?.sessionId === options.sessionId ? previous : { sessionId: options.sessionId, epoch: 1, unavailable: false, seen: /* @__PURE__ */ new Set() };
|
|
2540
|
+
bindings.set(room, binding);
|
|
2541
|
+
const sources = [];
|
|
2542
|
+
const active = (source) => sources.includes(source);
|
|
2543
|
+
const buffered = () => sources.reduce((sum, source) => sum + source.received - source.consumed, 0);
|
|
2544
|
+
let nextStart = 0;
|
|
2545
|
+
let blocked;
|
|
2546
|
+
let closing;
|
|
2547
|
+
const seen = binding.seen;
|
|
2548
|
+
const waiters = /* @__PURE__ */ new Set();
|
|
2549
|
+
const wake = () => {
|
|
2550
|
+
for (const resolve of waiters) resolve();
|
|
2551
|
+
waiters.clear();
|
|
2552
|
+
};
|
|
2553
|
+
const identity = (header) => ({
|
|
2554
|
+
session_id: header.session_id,
|
|
2555
|
+
generation_id: header.generation_id,
|
|
2556
|
+
utterance_id: header.utterance_id,
|
|
2557
|
+
stream_id: header.stream_id,
|
|
2558
|
+
epoch: header.epoch
|
|
2559
|
+
});
|
|
2560
|
+
const position = (current, state) => {
|
|
2561
|
+
notify(() => options.onPosition({ ...identity(current.header), samples: current.consumed, sampleRate: current.header.sample_rate, state }));
|
|
2562
|
+
};
|
|
2563
|
+
const receipt = (current, status) => {
|
|
2564
|
+
const payload = { ...identity(current.header), status, samples: current.consumed };
|
|
2565
|
+
if (current.rpcBusy) {
|
|
2566
|
+
current.pendingReceipt = payload;
|
|
2567
|
+
return;
|
|
2568
|
+
}
|
|
2569
|
+
const send = async (next) => {
|
|
2570
|
+
current.rpcBusy = true;
|
|
2571
|
+
try {
|
|
2572
|
+
await room.localParticipant.performRpc({
|
|
2573
|
+
destinationIdentity: options.agentIdentity,
|
|
2574
|
+
method: CORRELATED_AUDIO_CONTRACT.receiptMethod,
|
|
2575
|
+
payload: JSON.stringify(next),
|
|
2576
|
+
responseTimeout: 2e3
|
|
2577
|
+
});
|
|
2578
|
+
} catch {
|
|
2579
|
+
if (!disposed && next.epoch === binding.epoch) invalidate("connection_lost");
|
|
2580
|
+
} finally {
|
|
2581
|
+
current.rpcBusy = false;
|
|
2582
|
+
const pending = current.pendingReceipt;
|
|
2583
|
+
current.pendingReceipt = void 0;
|
|
2584
|
+
if (pending && !disposed && pending.epoch === binding.epoch && !binding.unavailable) void send(pending);
|
|
2585
|
+
}
|
|
2586
|
+
};
|
|
2587
|
+
void send(payload);
|
|
2588
|
+
};
|
|
2589
|
+
const clear = () => {
|
|
2590
|
+
for (const current of sources.splice(0)) {
|
|
2591
|
+
current.abort.abort();
|
|
2592
|
+
for (const segment of current.segments) {
|
|
2593
|
+
segment.node.onended = null;
|
|
2594
|
+
segment.node.stop();
|
|
2595
|
+
segment.node.disconnect();
|
|
2596
|
+
}
|
|
2597
|
+
current.segments.length = 0;
|
|
2598
|
+
}
|
|
2599
|
+
nextStart = context?.currentTime ?? 0;
|
|
2600
|
+
wake();
|
|
2601
|
+
};
|
|
2602
|
+
const invalidate = (reason) => {
|
|
2603
|
+
if (disposed) return;
|
|
2604
|
+
if (reason === "connection_lost") {
|
|
2605
|
+
if (binding.unavailable) return;
|
|
2606
|
+
binding.unavailable = true;
|
|
2607
|
+
}
|
|
2608
|
+
if (reason === "invalid_stream" || reason === "disposed") for (const current of sources) receipt(current, "cancelled");
|
|
2609
|
+
clear();
|
|
2610
|
+
notify(() => options.onInvalidated(reason));
|
|
2611
|
+
};
|
|
2612
|
+
const tick = (reportCredit = true) => {
|
|
2613
|
+
if (!context || context.state !== "running" || disposed || binding.unavailable) return;
|
|
2614
|
+
for (const current of [...sources]) {
|
|
2615
|
+
if (!active(current)) continue;
|
|
2616
|
+
let consumed = current.consumed;
|
|
2617
|
+
for (const segment of current.segments) {
|
|
2618
|
+
if (context.currentTime < segment.start) break;
|
|
2619
|
+
const samples = Math.min(segment.length, Math.max(0, Math.floor((context.currentTime - segment.start) * current.header.sample_rate + 1e-6)));
|
|
2620
|
+
consumed = Math.max(consumed, segment.offset + samples);
|
|
2621
|
+
if (!current.started && samples > 0) {
|
|
2622
|
+
current.started = true;
|
|
2623
|
+
receipt(current, "started");
|
|
2624
|
+
position(current, "playing");
|
|
2625
|
+
current.lastPosition = current.consumed;
|
|
2626
|
+
}
|
|
2627
|
+
}
|
|
2628
|
+
current.consumed = consumed;
|
|
2629
|
+
while (current.segments.length && current.segments[0].offset + current.segments[0].length <= consumed) {
|
|
2630
|
+
const segment = current.segments.shift();
|
|
2631
|
+
if (segment) {
|
|
2632
|
+
segment.node.onended = null;
|
|
2633
|
+
segment.node.disconnect();
|
|
2634
|
+
}
|
|
2635
|
+
}
|
|
2636
|
+
if (current.started && current.eof && consumed === current.received && consumed > 0) {
|
|
2637
|
+
position(current, "finished");
|
|
2638
|
+
if (!active(current)) continue;
|
|
2639
|
+
receipt(current, "finished");
|
|
2640
|
+
sources.splice(sources.indexOf(current), 1);
|
|
2641
|
+
wake();
|
|
2642
|
+
} else {
|
|
2643
|
+
if (current.started && consumed > current.lastPosition) {
|
|
2644
|
+
current.lastPosition = consumed;
|
|
2645
|
+
position(current, "playing");
|
|
2646
|
+
}
|
|
2647
|
+
if (!active(current)) continue;
|
|
2648
|
+
if (reportCredit && consumed > current.lastReported) {
|
|
2649
|
+
current.lastReported = consumed;
|
|
2650
|
+
receipt(current, "progress");
|
|
2651
|
+
}
|
|
2652
|
+
wake();
|
|
2653
|
+
}
|
|
2654
|
+
}
|
|
2655
|
+
};
|
|
2656
|
+
const playbackState = () => {
|
|
2657
|
+
const next = context?.state !== "running";
|
|
2658
|
+
if (next !== blocked) {
|
|
2659
|
+
blocked = next;
|
|
2660
|
+
notify(() => options.onPlaybackBlocked?.(next ? startAudio : null));
|
|
2661
|
+
}
|
|
2662
|
+
tick();
|
|
2663
|
+
};
|
|
2664
|
+
const getContext = () => {
|
|
2665
|
+
if (!context) {
|
|
2666
|
+
context = new AudioContext({ sampleRate: correlatedAudioHeaderSchema.shape.sample_rate.value });
|
|
2667
|
+
context.onstatechange = playbackState;
|
|
2668
|
+
playbackState();
|
|
2669
|
+
}
|
|
2670
|
+
return context;
|
|
2671
|
+
};
|
|
2672
|
+
async function startAudio() {
|
|
2673
|
+
if (disposed || binding.unavailable) throw new Error("Correlated audio binding is no longer available");
|
|
2674
|
+
const ctx = getContext();
|
|
2675
|
+
await ctx.resume();
|
|
2676
|
+
playbackState();
|
|
2677
|
+
if (ctx.state !== "running") throw new Error("Correlated audio playback is blocked");
|
|
2678
|
+
}
|
|
2679
|
+
async function receive(reader, participant) {
|
|
2680
|
+
if (disposed || binding.unavailable || room.state !== "connected" || participant.identity !== options.agentIdentity) return;
|
|
2681
|
+
let value;
|
|
2682
|
+
try {
|
|
2683
|
+
value = JSON.parse(reader.info.attributes?.["rta.audio"] ?? "");
|
|
2684
|
+
} catch {
|
|
2685
|
+
return;
|
|
2686
|
+
}
|
|
2687
|
+
const parsed = correlatedAudioHeaderSchema.safeParse(value);
|
|
2688
|
+
if (!parsed.success || parsed.data.session_id !== options.sessionId || parsed.data.epoch < binding.epoch) return;
|
|
2689
|
+
const header = parsed.data;
|
|
2690
|
+
if (header.stream_id !== reader.info.id) return;
|
|
2691
|
+
if (header.epoch > binding.epoch) {
|
|
2692
|
+
binding.epoch = header.epoch;
|
|
2693
|
+
invalidate("cancelled");
|
|
2694
|
+
}
|
|
2695
|
+
const key = `${header.epoch}/${header.stream_id}`;
|
|
2696
|
+
if (seen.has(key)) return;
|
|
2697
|
+
if (sources.length >= 128 || seen.size >= 1e4) {
|
|
2698
|
+
invalidate("invalid_stream");
|
|
2699
|
+
return;
|
|
2700
|
+
}
|
|
2701
|
+
seen.add(key);
|
|
2702
|
+
const previousSource = sources.at(-1);
|
|
2703
|
+
const current = { header, abort: new AbortController(), segments: [], received: 0, consumed: 0, started: false, eof: false, lastReported: 0, lastPosition: -1, rpcBusy: false };
|
|
2704
|
+
sources.push(current);
|
|
2705
|
+
let carry;
|
|
2706
|
+
try {
|
|
2707
|
+
if (options.prepare) await options.prepare(header, current.abort.signal);
|
|
2708
|
+
if (!active(current) || current.abort.signal.aborted) return;
|
|
2709
|
+
const ctx = getContext();
|
|
2710
|
+
while (active(current) && (previousSource && active(previousSource) && !previousSource.eof || buffered() > CORRELATED_AUDIO_CONTRACT.maxBufferedSamples - CORRELATED_AUDIO_CONTRACT.maxChunkBytes / 2)) {
|
|
2711
|
+
await new Promise((resolve) => waiters.add(resolve));
|
|
2712
|
+
}
|
|
2713
|
+
if (!active(current)) return;
|
|
2714
|
+
for await (const chunk of reader.withAbortSignal(current.abort.signal)) {
|
|
2715
|
+
if (!active(current)) break;
|
|
2716
|
+
if (chunk.byteLength > CORRELATED_AUDIO_CONTRACT.maxChunkBytes) throw new Error("PCM chunk exceeds contract");
|
|
2717
|
+
const count = Math.floor((chunk.byteLength + (carry === void 0 ? 0 : 1)) / 2);
|
|
2718
|
+
if (current.received + count > CORRELATED_AUDIO_CONTRACT.maxSamples) throw new Error("PCM stream exceeds contract");
|
|
2719
|
+
while (active(current) && buffered() + count > CORRELATED_AUDIO_CONTRACT.maxBufferedSamples) {
|
|
2720
|
+
await new Promise((resolve) => waiters.add(resolve));
|
|
2721
|
+
}
|
|
2722
|
+
if (!active(current)) break;
|
|
2723
|
+
if (!count) {
|
|
2724
|
+
if (chunk.length) carry = chunk[0];
|
|
2725
|
+
continue;
|
|
2726
|
+
}
|
|
2727
|
+
const buffer = ctx.createBuffer(header.channels, count, header.sample_rate);
|
|
2728
|
+
const data = buffer.getChannelData(0);
|
|
2729
|
+
let cursor = 0;
|
|
2730
|
+
for (let i = 0; i < count; i++) {
|
|
2731
|
+
const low = carry ?? chunk[cursor++];
|
|
2732
|
+
carry = void 0;
|
|
2733
|
+
const raw = low | chunk[cursor++] << 8;
|
|
2734
|
+
data[i] = (raw >= 32768 ? raw - 65536 : raw) / 32768;
|
|
2735
|
+
}
|
|
2736
|
+
if (cursor < chunk.length) carry = chunk[cursor];
|
|
2737
|
+
const node = ctx.createBufferSource();
|
|
2738
|
+
node.buffer = buffer;
|
|
2739
|
+
node.connect(ctx.destination);
|
|
2740
|
+
const start = Math.max(ctx.currentTime, nextStart);
|
|
2741
|
+
current.segments.push({ node, start, offset: current.received, length: count });
|
|
2742
|
+
current.received += count;
|
|
2743
|
+
nextStart = start + count / header.sample_rate;
|
|
2744
|
+
node.onended = () => tick();
|
|
2745
|
+
node.start(start);
|
|
2746
|
+
while (active(current) && buffered() > CORRELATED_AUDIO_CONTRACT.maxBufferedSamples - CORRELATED_AUDIO_CONTRACT.maxChunkBytes / 2) {
|
|
2747
|
+
await new Promise((resolve) => waiters.add(resolve));
|
|
2748
|
+
}
|
|
2749
|
+
}
|
|
2750
|
+
if (!active(current)) return;
|
|
2751
|
+
if (carry !== void 0 || current.received === 0) throw new Error("Incomplete PCM stream");
|
|
2752
|
+
current.eof = true;
|
|
2753
|
+
wake();
|
|
2754
|
+
tick(false);
|
|
2755
|
+
} catch {
|
|
2756
|
+
if (active(current)) invalidate("invalid_stream");
|
|
2757
|
+
}
|
|
2758
|
+
}
|
|
2759
|
+
const cancel = async (data) => {
|
|
2760
|
+
if (disposed || binding.unavailable || room.state !== "connected" || data.callerIdentity !== options.agentIdentity) throw new Error("Unbound audio cancellation");
|
|
2761
|
+
const value = JSON.parse(data.payload);
|
|
2762
|
+
const frame2 = correlatedAudioCancelSchema.parse(value);
|
|
2763
|
+
if (frame2.session_id !== options.sessionId) throw new Error("Wrong audio session");
|
|
2764
|
+
if (frame2.epoch <= binding.epoch) return "";
|
|
2765
|
+
binding.epoch = frame2.epoch;
|
|
2766
|
+
invalidate("cancelled");
|
|
2767
|
+
return "";
|
|
2768
|
+
};
|
|
2769
|
+
const lost = () => invalidate("connection_lost");
|
|
2770
|
+
const changed = () => {
|
|
2771
|
+
if (["disconnected", "reconnecting", "signalReconnecting"].includes(room.state)) lost();
|
|
2772
|
+
};
|
|
2773
|
+
const participantLeft = (participant) => {
|
|
2774
|
+
if (participant.identity === options.agentIdentity) lost();
|
|
2775
|
+
};
|
|
2776
|
+
let registered = false;
|
|
2777
|
+
try {
|
|
2778
|
+
room.registerByteStreamHandler(CORRELATED_AUDIO_CONTRACT.topic, (reader, participant) => {
|
|
2779
|
+
void receive(reader, participant);
|
|
2780
|
+
});
|
|
2781
|
+
registered = true;
|
|
2782
|
+
room.localParticipant.registerRpcMethod(CORRELATED_AUDIO_CONTRACT.cancelMethod, cancel);
|
|
2783
|
+
} catch (error) {
|
|
2784
|
+
if (registered) room.unregisterByteStreamHandler(CORRELATED_AUDIO_CONTRACT.topic);
|
|
2785
|
+
attachments.delete(room);
|
|
2786
|
+
throw error;
|
|
2787
|
+
}
|
|
2788
|
+
room.on(RoomEvent.Reconnecting, lost);
|
|
2789
|
+
room.on(RoomEvent.SignalReconnecting, lost);
|
|
2790
|
+
room.on(RoomEvent.Disconnected, lost);
|
|
2791
|
+
room.on(RoomEvent.ConnectionStateChanged, changed);
|
|
2792
|
+
room.on(RoomEvent.ParticipantDisconnected, participantLeft);
|
|
2793
|
+
const timer = setInterval(() => tick(), 100);
|
|
2794
|
+
let frame = 0;
|
|
2795
|
+
const render = () => {
|
|
2796
|
+
if (disposed) return;
|
|
2797
|
+
tick(false);
|
|
2798
|
+
frame = requestAnimationFrame(render);
|
|
2799
|
+
};
|
|
2800
|
+
frame = requestAnimationFrame(render);
|
|
2801
|
+
return {
|
|
2802
|
+
startAudio,
|
|
2803
|
+
dispose() {
|
|
2804
|
+
if (closing) return closing;
|
|
2805
|
+
invalidate("disposed");
|
|
2806
|
+
disposed = true;
|
|
2807
|
+
clearInterval(timer);
|
|
2808
|
+
cancelAnimationFrame(frame);
|
|
2809
|
+
room.unregisterByteStreamHandler(CORRELATED_AUDIO_CONTRACT.topic);
|
|
2810
|
+
room.localParticipant.unregisterRpcMethod(CORRELATED_AUDIO_CONTRACT.cancelMethod);
|
|
2811
|
+
room.off(RoomEvent.Reconnecting, lost);
|
|
2812
|
+
room.off(RoomEvent.SignalReconnecting, lost);
|
|
2813
|
+
room.off(RoomEvent.Disconnected, lost);
|
|
2814
|
+
room.off(RoomEvent.ConnectionStateChanged, changed);
|
|
2815
|
+
room.off(RoomEvent.ParticipantDisconnected, participantLeft);
|
|
2816
|
+
attachments.delete(room);
|
|
2817
|
+
if (context) context.onstatechange = null;
|
|
2818
|
+
closing = context?.close().catch(() => {
|
|
2819
|
+
}) ?? Promise.resolve();
|
|
2820
|
+
return closing;
|
|
2821
|
+
}
|
|
2822
|
+
};
|
|
2823
|
+
}
|
|
2824
|
+
|
|
2825
|
+
// ../client/src/external-performance.ts
|
|
2826
|
+
var EXTERNAL_PERFORMANCE_KIND = externalPerformanceFrameSchema.shape.kind.value;
|
|
2827
|
+
var EXTERNAL_PERFORMANCE_TIMING = Object.freeze({
|
|
2828
|
+
receiverPlayout: "unsupported",
|
|
2829
|
+
audibleCheckpoint: externalPerformanceFrameSchema.shape.audible_checkpoint.value,
|
|
2830
|
+
audioMapping: externalPerformanceFrameSchema.shape.audio_mapping.value
|
|
2831
|
+
});
|
|
2832
|
+
var statusSchema = externalPerformanceFrameSchema.shape.status;
|
|
2833
|
+
var KNOWN_EXTERNAL_PERFORMANCE_STATUSES = Object.freeze([...statusSchema.options[0].options]);
|
|
2834
|
+
var knownStatuses = new Set(KNOWN_EXTERNAL_PERFORMANCE_STATUSES);
|
|
2835
|
+
function isObservable(frame) {
|
|
2836
|
+
return knownStatuses.has(frame.status);
|
|
2837
|
+
}
|
|
2838
|
+
var NEXT = {
|
|
2839
|
+
unseen: /* @__PURE__ */ new Set(["accepted", "dropped"]),
|
|
2840
|
+
accepted: /* @__PURE__ */ new Set(["audio_released", "cancelled"]),
|
|
2841
|
+
audio_released: /* @__PURE__ */ new Set(["cancelled"]),
|
|
2842
|
+
dropped: /* @__PURE__ */ new Set(),
|
|
2843
|
+
cancelled: /* @__PURE__ */ new Set(),
|
|
2844
|
+
retired: /* @__PURE__ */ new Set()
|
|
2845
|
+
};
|
|
2846
|
+
var LEDGER_LIMIT = 4096;
|
|
2847
|
+
function deliver(callback) {
|
|
2848
|
+
try {
|
|
2849
|
+
void Promise.resolve(callback()).catch(() => {
|
|
2850
|
+
});
|
|
2851
|
+
} catch {
|
|
2852
|
+
}
|
|
2853
|
+
}
|
|
2854
|
+
function createExternalPerformanceObserver(sessionId, callbacks) {
|
|
2855
|
+
const generations = /* @__PURE__ */ new Map();
|
|
2856
|
+
let state = "live";
|
|
2857
|
+
return {
|
|
2858
|
+
receive(frame) {
|
|
2859
|
+
if (state !== "live" || frame.session_id !== sessionId || !isObservable(frame)) return;
|
|
2860
|
+
const previous = generations.get(frame.generation_id) ?? "unseen";
|
|
2861
|
+
if (!NEXT[previous].has(frame.status)) return;
|
|
2862
|
+
if (previous === "unseen" && generations.size >= LEDGER_LIMIT) {
|
|
2863
|
+
const oldest = generations.keys().next();
|
|
2864
|
+
if (!oldest.done) generations.delete(oldest.value);
|
|
2865
|
+
}
|
|
2866
|
+
generations.set(frame.generation_id, frame.status);
|
|
2867
|
+
deliver(() => callbacks.onFrame(frame));
|
|
2868
|
+
},
|
|
2869
|
+
invalidate(reason) {
|
|
2870
|
+
if (state === "closed" || state === "paused" && reason === "connection_lost") return;
|
|
2871
|
+
state = reason === "connection_lost" ? "paused" : "closed";
|
|
2872
|
+
if (state === "closed") generations.clear();
|
|
2873
|
+
else for (const id of generations.keys()) generations.set(id, "retired");
|
|
2874
|
+
deliver(() => callbacks.onInvalidated(reason));
|
|
2875
|
+
},
|
|
2876
|
+
/** The call is connected to its agent again. Only a connection loss can be resumed. */
|
|
2877
|
+
resume() {
|
|
2878
|
+
if (state === "paused") state = "live";
|
|
2879
|
+
}
|
|
2880
|
+
};
|
|
2881
|
+
}
|
|
2419
2882
|
|
|
2420
2883
|
// ../client/src/react/session-lifecycle.ts
|
|
2421
2884
|
function phaseForReason(reason) {
|
|
@@ -2606,6 +3069,32 @@ function useSessionLifecycle(input) {
|
|
|
2606
3069
|
const grantIdRef = useRef(null);
|
|
2607
3070
|
grantIdRef.current = grantId;
|
|
2608
3071
|
const retiredGrantRef = useRef(null);
|
|
3072
|
+
const performanceEnabled = input.externalPerformance !== void 0;
|
|
3073
|
+
const performanceCallbacksRef = useRef(input.externalPerformance);
|
|
3074
|
+
if (input.externalPerformance) performanceCallbacksRef.current = input.externalPerformance;
|
|
3075
|
+
const performanceRef = useRef(null);
|
|
3076
|
+
const endedPerformanceGrantRef = useRef(null);
|
|
3077
|
+
useEffect(() => {
|
|
3078
|
+
if (!grantId || !performanceEnabled || endedPerformanceGrantRef.current === grantId) return;
|
|
3079
|
+
const observer = createExternalPerformanceObserver(grantId, {
|
|
3080
|
+
onFrame: (frame) => performanceCallbacksRef.current?.onFrame(frame),
|
|
3081
|
+
onInvalidated: (reason) => {
|
|
3082
|
+
if (reason === "ended") endedPerformanceGrantRef.current = grantId;
|
|
3083
|
+
performanceCallbacksRef.current?.onInvalidated(reason);
|
|
3084
|
+
}
|
|
3085
|
+
});
|
|
3086
|
+
performanceRef.current = observer;
|
|
3087
|
+
return () => {
|
|
3088
|
+
observer.invalidate("binding_changed");
|
|
3089
|
+
if (performanceRef.current === observer) performanceRef.current = null;
|
|
3090
|
+
};
|
|
3091
|
+
}, [grantId, performanceEnabled]);
|
|
3092
|
+
const invalidateExternalPerformance = useCallback((reason) => {
|
|
3093
|
+
if (grantId === grantIdRef.current) performanceRef.current?.invalidate(reason);
|
|
3094
|
+
}, [grantId]);
|
|
3095
|
+
const resumeExternalPerformance = useCallback(() => {
|
|
3096
|
+
if (grantId === grantIdRef.current) performanceRef.current?.resume();
|
|
3097
|
+
}, [grantId]);
|
|
2609
3098
|
const [recovery, setRecovery] = useState({ kind: "connected" });
|
|
2610
3099
|
const recoveryRef = useRef(recovery);
|
|
2611
3100
|
recoveryRef.current = recovery;
|
|
@@ -2651,6 +3140,7 @@ function useSessionLifecycle(input) {
|
|
|
2651
3140
|
}
|
|
2652
3141
|
}, []);
|
|
2653
3142
|
const reset = useCallback(() => {
|
|
3143
|
+
invalidateExternalPerformance("binding_changed");
|
|
2654
3144
|
clearTimer();
|
|
2655
3145
|
connectedRef.current = false;
|
|
2656
3146
|
manualReconnectPendingRef.current = false;
|
|
@@ -2665,7 +3155,7 @@ function useSessionLifecycle(input) {
|
|
|
2665
3155
|
setConnected(false);
|
|
2666
3156
|
setAgentPresentState(false);
|
|
2667
3157
|
lastActivityRef.current = Date.now();
|
|
2668
|
-
}, [clearTimer]);
|
|
3158
|
+
}, [clearTimer, invalidateExternalPerformance]);
|
|
2669
3159
|
useEffect(() => {
|
|
2670
3160
|
if (active) return;
|
|
2671
3161
|
userEndedRef.current = false;
|
|
@@ -2699,6 +3189,7 @@ function useSessionLifecycle(input) {
|
|
|
2699
3189
|
}, [grantId]);
|
|
2700
3190
|
const endIdle = useCallback(() => {
|
|
2701
3191
|
if (!connectedRef.current) return;
|
|
3192
|
+
invalidateExternalPerformance("ended");
|
|
2702
3193
|
clearTimer();
|
|
2703
3194
|
connectedRef.current = false;
|
|
2704
3195
|
setConnected(false);
|
|
@@ -2706,7 +3197,7 @@ function useSessionLifecycle(input) {
|
|
|
2706
3197
|
releaseRef.current("idle_timeout");
|
|
2707
3198
|
setRecovery({ kind: "ended", reason: "idle" });
|
|
2708
3199
|
leaveRoomRef.current?.();
|
|
2709
|
-
}, [clearTimer]);
|
|
3200
|
+
}, [clearTimer, invalidateExternalPerformance]);
|
|
2710
3201
|
const onConnected = useCallback(() => {
|
|
2711
3202
|
if (grantId === null || grantIdRef.current !== grantId) return;
|
|
2712
3203
|
if (userEndedRef.current || readinessFailureRef.current || retiredGrantRef.current !== null && retiredGrantRef.current === grantIdRef.current) return;
|
|
@@ -2724,6 +3215,7 @@ function useSessionLifecycle(input) {
|
|
|
2724
3215
|
if (grantId === null || grantIdRef.current !== grantId) return;
|
|
2725
3216
|
if (!active || userEndedRef.current || readinessFailureRef.current) return;
|
|
2726
3217
|
const action = disconnectAction(reason, connectedRef.current);
|
|
3218
|
+
invalidateExternalPerformance(action === "end" ? "ended" : "connection_lost");
|
|
2727
3219
|
if (action === "noop") return;
|
|
2728
3220
|
if (action === "reset") {
|
|
2729
3221
|
if (recoveryRef.current.kind === "refreshing" || recoveryRef.current.kind === "reconnecting") return;
|
|
@@ -2743,12 +3235,13 @@ function useSessionLifecycle(input) {
|
|
|
2743
3235
|
refreshBaselineCapacityRef.current = null;
|
|
2744
3236
|
setRecovery({ kind: "reconnecting", attempt: attemptRef.current });
|
|
2745
3237
|
},
|
|
2746
|
-
[active, reset, grantId]
|
|
3238
|
+
[active, reset, grantId, invalidateExternalPerformance]
|
|
2747
3239
|
);
|
|
2748
3240
|
const onConnectionError = useCallback(
|
|
2749
3241
|
(error) => {
|
|
2750
3242
|
if (grantId === null || grantIdRef.current !== grantId) return;
|
|
2751
3243
|
if (!active || userEndedRef.current || readinessFailureRef.current || !isRecoverableConnectionError(error)) return;
|
|
3244
|
+
invalidateExternalPerformance("connection_lost");
|
|
2752
3245
|
connectedRef.current = false;
|
|
2753
3246
|
setConnected(false);
|
|
2754
3247
|
setAgentPresentState(false);
|
|
@@ -2759,7 +3252,7 @@ function useSessionLifecycle(input) {
|
|
|
2759
3252
|
return { kind: "reconnecting", attempt: attemptRef.current };
|
|
2760
3253
|
});
|
|
2761
3254
|
},
|
|
2762
|
-
[active, grantId]
|
|
3255
|
+
[active, grantId, invalidateExternalPerformance]
|
|
2763
3256
|
);
|
|
2764
3257
|
useEffect(() => {
|
|
2765
3258
|
if (capacity.kind === "error" && recovery.kind === "connected") manualReconnectPendingRef.current = false;
|
|
@@ -2779,6 +3272,9 @@ function useSessionLifecycle(input) {
|
|
|
2779
3272
|
onConnected();
|
|
2780
3273
|
return;
|
|
2781
3274
|
}
|
|
3275
|
+
if (state === "disconnected" && connectedRef.current || state === "reconnecting" || state === "signalReconnecting") {
|
|
3276
|
+
invalidateExternalPerformance("connection_lost");
|
|
3277
|
+
}
|
|
2782
3278
|
if (state === "reconnecting" || state === "signalReconnecting") {
|
|
2783
3279
|
if (connectedRef.current) {
|
|
2784
3280
|
setConnected(false);
|
|
@@ -2788,7 +3284,7 @@ function useSessionLifecycle(input) {
|
|
|
2788
3284
|
}
|
|
2789
3285
|
}
|
|
2790
3286
|
},
|
|
2791
|
-
[active, onConnected, grantId]
|
|
3287
|
+
[active, onConnected, grantId, invalidateExternalPerformance]
|
|
2792
3288
|
);
|
|
2793
3289
|
useEffect(() => {
|
|
2794
3290
|
if (!active || userEnded || readinessFailed || !connected || !agentPresent || !grantState.grant) return;
|
|
@@ -2801,6 +3297,7 @@ function useSessionLifecycle(input) {
|
|
|
2801
3297
|
const failReadiness = useCallback((code) => {
|
|
2802
3298
|
if (userEndedRef.current || readinessFailureRef.current) return;
|
|
2803
3299
|
readinessFailureRef.current = code;
|
|
3300
|
+
invalidateExternalPerformance("ended");
|
|
2804
3301
|
clearTimer();
|
|
2805
3302
|
connectedRef.current = false;
|
|
2806
3303
|
manualReconnectPendingRef.current = false;
|
|
@@ -2813,7 +3310,7 @@ function useSessionLifecycle(input) {
|
|
|
2813
3310
|
releaseRef.current("disconnected");
|
|
2814
3311
|
setReadinessFailed(true);
|
|
2815
3312
|
leaveRoomRef.current?.();
|
|
2816
|
-
}, [clearTimer]);
|
|
3313
|
+
}, [clearTimer, invalidateExternalPerformance]);
|
|
2817
3314
|
useEffect(() => {
|
|
2818
3315
|
if (!active || userEnded || readinessFailed) return;
|
|
2819
3316
|
if (recovery.kind === "ended" || recovery.kind === "failed") return;
|
|
@@ -2894,6 +3391,7 @@ function useSessionLifecycle(input) {
|
|
|
2894
3391
|
const end = useCallback(() => {
|
|
2895
3392
|
if (!active || userEndedRef.current || phaseRef.current.kind === "ended") return;
|
|
2896
3393
|
userEndedRef.current = true;
|
|
3394
|
+
invalidateExternalPerformance("ended");
|
|
2897
3395
|
clearTimer();
|
|
2898
3396
|
connectedRef.current = false;
|
|
2899
3397
|
manualReconnectPendingRef.current = false;
|
|
@@ -2904,7 +3402,7 @@ function useSessionLifecycle(input) {
|
|
|
2904
3402
|
setRecovery({ kind: "ended", reason: "user" });
|
|
2905
3403
|
clearRef.current();
|
|
2906
3404
|
leaveRoomRef.current?.();
|
|
2907
|
-
}, [active, clearTimer]);
|
|
3405
|
+
}, [active, clearTimer, invalidateExternalPerformance]);
|
|
2908
3406
|
const reconnect = useCallback(() => {
|
|
2909
3407
|
if (!active) return;
|
|
2910
3408
|
if (readinessFailureRef.current) {
|
|
@@ -2974,16 +3472,22 @@ function useSessionLifecycle(input) {
|
|
|
2974
3472
|
const lastBehaviorRef = useRef(null);
|
|
2975
3473
|
const behaviorEnabled = Boolean(onBehaviorChange);
|
|
2976
3474
|
const onLifecycleData = useMemo(() => {
|
|
2977
|
-
if (!behaviorEnabled) return void 0;
|
|
3475
|
+
if (!behaviorEnabled && !performanceEnabled) return void 0;
|
|
2978
3476
|
return (frame) => {
|
|
2979
3477
|
const parsed = lifecycleServerFrameSchema.safeParse(frame);
|
|
2980
|
-
if (!parsed.success
|
|
3478
|
+
if (!parsed.success) return;
|
|
3479
|
+
if (parsed.data.kind === "external_performance") {
|
|
3480
|
+
if (grantId === grantIdRef.current && parsed.data.session_id === grantId) performanceRef.current?.receive(parsed.data);
|
|
3481
|
+
return;
|
|
3482
|
+
}
|
|
3483
|
+
if (parsed.data.kind === "ended") invalidateExternalPerformance("ended");
|
|
3484
|
+
if (!behaviorEnabled || parsed.data.kind !== "behavior_state") return;
|
|
2981
3485
|
const next = nextBehaviorSnapshot(lastBehaviorRef.current, parsed.data);
|
|
2982
3486
|
if (!next) return;
|
|
2983
3487
|
lastBehaviorRef.current = next;
|
|
2984
3488
|
onBehaviorChangeRef.current?.(next);
|
|
2985
3489
|
};
|
|
2986
|
-
}, [behaviorEnabled]);
|
|
3490
|
+
}, [behaviorEnabled, performanceEnabled, grantId, invalidateExternalPerformance]);
|
|
2987
3491
|
return useMemo(
|
|
2988
3492
|
() => ({
|
|
2989
3493
|
phase,
|
|
@@ -3002,7 +3506,9 @@ function useSessionLifecycle(input) {
|
|
|
3002
3506
|
setAgentPresent,
|
|
3003
3507
|
registerLeaveRoom,
|
|
3004
3508
|
reset,
|
|
3005
|
-
onLifecycleData
|
|
3509
|
+
onLifecycleData,
|
|
3510
|
+
invalidateExternalPerformance: performanceEnabled ? invalidateExternalPerformance : void 0,
|
|
3511
|
+
resumeExternalPerformance: performanceEnabled ? resumeExternalPerformance : void 0
|
|
3006
3512
|
}),
|
|
3007
3513
|
[
|
|
3008
3514
|
phase,
|
|
@@ -3021,7 +3527,10 @@ function useSessionLifecycle(input) {
|
|
|
3021
3527
|
setAgentPresent,
|
|
3022
3528
|
registerLeaveRoom,
|
|
3023
3529
|
reset,
|
|
3024
|
-
onLifecycleData
|
|
3530
|
+
onLifecycleData,
|
|
3531
|
+
invalidateExternalPerformance,
|
|
3532
|
+
resumeExternalPerformance,
|
|
3533
|
+
performanceEnabled
|
|
3025
3534
|
]
|
|
3026
3535
|
);
|
|
3027
3536
|
}
|
|
@@ -3050,9 +3559,45 @@ function SessionLifecycleRoomBridge({
|
|
|
3050
3559
|
const transcriptions = useTranscriptions();
|
|
3051
3560
|
const room = useRoomContext();
|
|
3052
3561
|
const roomMatchesGrant = lifecycle.grant === void 0 || lifecycle.grant !== null && room.name === lifecycle.grant.room_name;
|
|
3562
|
+
const correlatedRef = useRef(null);
|
|
3563
|
+
const correlatedObserver = useRef(lifecycle.correlatedAudio);
|
|
3564
|
+
correlatedObserver.current = lifecycle.correlatedAudio;
|
|
3565
|
+
const correlatedEnabled = lifecycle.correlatedAudio !== void 0;
|
|
3566
|
+
const correlatedSessionId = lifecycle.grant?.session_id;
|
|
3567
|
+
const startCorrelatedAudio = useCallback(async () => {
|
|
3568
|
+
const player = correlatedRef.current;
|
|
3569
|
+
if (!player) return false;
|
|
3570
|
+
try {
|
|
3571
|
+
await player.startAudio();
|
|
3572
|
+
return true;
|
|
3573
|
+
} catch {
|
|
3574
|
+
return false;
|
|
3575
|
+
}
|
|
3576
|
+
}, []);
|
|
3577
|
+
useEffect(() => {
|
|
3578
|
+
if (!correlatedEnabled || !correlatedSessionId || !roomMatchesGrant || !assistant.agent) return;
|
|
3579
|
+
const player = attachCorrelatedAudio(room, {
|
|
3580
|
+
sessionId: correlatedSessionId,
|
|
3581
|
+
agentIdentity: assistant.agent.identity,
|
|
3582
|
+
prepare: (header, signal) => correlatedObserver.current?.prepare?.(header, signal) ?? Promise.resolve(),
|
|
3583
|
+
onPosition: (position) => correlatedObserver.current?.onPosition(position),
|
|
3584
|
+
onInvalidated: (reason) => correlatedObserver.current?.onInvalidated(reason),
|
|
3585
|
+
onPlaybackBlocked: (unblock) => {
|
|
3586
|
+
setAudioPlayback?.(unblock ? "blocked" : "allowed");
|
|
3587
|
+
correlatedObserver.current?.onPlaybackBlocked?.(unblock);
|
|
3588
|
+
}
|
|
3589
|
+
});
|
|
3590
|
+
correlatedRef.current = player;
|
|
3591
|
+
return () => {
|
|
3592
|
+
correlatedRef.current = null;
|
|
3593
|
+
void player.dispose();
|
|
3594
|
+
setAudioPlayback?.("unknown");
|
|
3595
|
+
};
|
|
3596
|
+
}, [room, assistant.agent, correlatedEnabled, correlatedSessionId, roomMatchesGrant, setAudioPlayback]);
|
|
3053
3597
|
useCallMedia(room, {
|
|
3054
3598
|
setMicrophoneFacts,
|
|
3055
|
-
setAudioPlayback,
|
|
3599
|
+
setAudioPlayback: correlatedEnabled ? void 0 : setAudioPlayback,
|
|
3600
|
+
startAudioOverride: correlatedEnabled ? startCorrelatedAudio : void 0,
|
|
3056
3601
|
registerMediaControls,
|
|
3057
3602
|
microphoneWanted: microphone ?? publishesMicrophoneByDefault(lifecycle.grant),
|
|
3058
3603
|
connectionState,
|
|
@@ -3066,7 +3611,7 @@ function SessionLifecycleRoomBridge({
|
|
|
3066
3611
|
if (!mediaModeEnabled) return;
|
|
3067
3612
|
const agent = assistant.agent;
|
|
3068
3613
|
let previous;
|
|
3069
|
-
const
|
|
3614
|
+
const deliver2 = (mode) => {
|
|
3070
3615
|
if (mode === previous) return;
|
|
3071
3616
|
previous = mode;
|
|
3072
3617
|
try {
|
|
@@ -3077,7 +3622,7 @@ function SessionLifecycleRoomBridge({
|
|
|
3077
3622
|
};
|
|
3078
3623
|
const read = () => {
|
|
3079
3624
|
const value = agent?.attributes?.["rta.media_mode"];
|
|
3080
|
-
|
|
3625
|
+
deliver2(value === "video" ? "avatar" : value === "voice" ? "voice" : null);
|
|
3081
3626
|
};
|
|
3082
3627
|
const changed = (_attributes, participant) => {
|
|
3083
3628
|
if (participant === agent) read();
|
|
@@ -3086,7 +3631,7 @@ function SessionLifecycleRoomBridge({
|
|
|
3086
3631
|
read();
|
|
3087
3632
|
return () => {
|
|
3088
3633
|
room.off(RoomEvent.ParticipantAttributesChanged, changed);
|
|
3089
|
-
|
|
3634
|
+
deliver2(null);
|
|
3090
3635
|
};
|
|
3091
3636
|
}, [room, assistant.agent, mediaModeEnabled, mediaModeSessionId]);
|
|
3092
3637
|
const { send } = useChat();
|
|
@@ -3133,7 +3678,7 @@ function SessionLifecycleRoomBridge({
|
|
|
3133
3678
|
let closed = false;
|
|
3134
3679
|
let pending = false;
|
|
3135
3680
|
let previous;
|
|
3136
|
-
const
|
|
3681
|
+
const deliver2 = (details) => {
|
|
3137
3682
|
connectionHistoryRef.current?.enqueue(details);
|
|
3138
3683
|
try {
|
|
3139
3684
|
if (onConnectionDetailsChange) void Promise.resolve(callback?.(details)).catch(() => {
|
|
@@ -3160,7 +3705,7 @@ function SessionLifecycleRoomBridge({
|
|
|
3160
3705
|
const last = previous;
|
|
3161
3706
|
if (last?.connectionState === next.connectionState && last.localQuality === next.localQuality && ["audio", "video"].every((kind) => last[kind]?.publisherQuality === next[kind]?.publisherQuality && last[kind]?.streamState === next[kind]?.streamState)) return;
|
|
3162
3707
|
previous = next;
|
|
3163
|
-
|
|
3708
|
+
deliver2(next);
|
|
3164
3709
|
});
|
|
3165
3710
|
};
|
|
3166
3711
|
const onQuality = (_quality, participant) => {
|
|
@@ -3190,7 +3735,7 @@ function SessionLifecycleRoomBridge({
|
|
|
3190
3735
|
room.off(RoomEvent.TrackStreamStateChanged, onPublication);
|
|
3191
3736
|
room.off(RoomEvent.TrackSubscribed, onTrack);
|
|
3192
3737
|
room.off(RoomEvent.TrackUnsubscribed, onTrack);
|
|
3193
|
-
|
|
3738
|
+
deliver2(null);
|
|
3194
3739
|
};
|
|
3195
3740
|
}, [room, detailsSessionId, detailsEnabled]);
|
|
3196
3741
|
useEffect(() => {
|
|
@@ -3227,18 +3772,56 @@ function SessionLifecycleRoomBridge({
|
|
|
3227
3772
|
useEffect(() => {
|
|
3228
3773
|
if (!onLifecycleData) return;
|
|
3229
3774
|
const decoder = new TextDecoder();
|
|
3230
|
-
const handler = (payload,
|
|
3775
|
+
const handler = (payload, participant, _kind, topic) => {
|
|
3231
3776
|
if (topic !== RTA_LIFECYCLE_TOPIC) return;
|
|
3232
3777
|
try {
|
|
3233
|
-
|
|
3778
|
+
const frame = JSON.parse(decoder.decode(payload));
|
|
3779
|
+
if (typeof frame === "object" && frame !== null && "kind" in frame && (frame.kind === EXTERNAL_PERFORMANCE_KIND || frame.kind === "ended") && (!assistant.agent || participant !== assistant.agent || !roomMatchesGrant || room.state !== "connected")) return;
|
|
3780
|
+
if (typeof frame === "object" && frame !== null && "kind" in frame && frame.kind === "ended" && lifecycleServerFrameSchema.safeParse(frame).success) {
|
|
3781
|
+
void correlatedRef.current?.dispose();
|
|
3782
|
+
correlatedRef.current = null;
|
|
3783
|
+
}
|
|
3784
|
+
onLifecycleData(frame);
|
|
3234
3785
|
} catch {
|
|
3235
3786
|
}
|
|
3236
3787
|
};
|
|
3788
|
+
const invalidate = () => lifecycle.invalidateExternalPerformance?.("connection_lost");
|
|
3789
|
+
const changed = () => {
|
|
3790
|
+
if (room.state === "disconnected" || room.state === "reconnecting" || room.state === "signalReconnecting") invalidate();
|
|
3791
|
+
};
|
|
3237
3792
|
room.on(RoomEvent.DataReceived, handler);
|
|
3793
|
+
if (lifecycle.invalidateExternalPerformance) {
|
|
3794
|
+
room.on(RoomEvent.Reconnecting, invalidate);
|
|
3795
|
+
room.on(RoomEvent.SignalReconnecting, invalidate);
|
|
3796
|
+
room.on(RoomEvent.Disconnected, invalidate);
|
|
3797
|
+
room.on(RoomEvent.ConnectionStateChanged, changed);
|
|
3798
|
+
}
|
|
3238
3799
|
return () => {
|
|
3239
3800
|
room.off(RoomEvent.DataReceived, handler);
|
|
3801
|
+
room.off(RoomEvent.Reconnecting, invalidate);
|
|
3802
|
+
room.off(RoomEvent.SignalReconnecting, invalidate);
|
|
3803
|
+
room.off(RoomEvent.Disconnected, invalidate);
|
|
3804
|
+
room.off(RoomEvent.ConnectionStateChanged, changed);
|
|
3240
3805
|
};
|
|
3241
|
-
}, [room, onLifecycleData]);
|
|
3806
|
+
}, [room, onLifecycleData, assistant.agent, roomMatchesGrant, lifecycle.invalidateExternalPerformance]);
|
|
3807
|
+
useEffect(() => {
|
|
3808
|
+
if (!assistant.agent) return;
|
|
3809
|
+
return () => lifecycle.invalidateExternalPerformance?.("connection_lost");
|
|
3810
|
+
}, [assistant.agent, lifecycle.invalidateExternalPerformance]);
|
|
3811
|
+
const resumeExternalPerformance = lifecycle.resumeExternalPerformance;
|
|
3812
|
+
useEffect(() => {
|
|
3813
|
+
if (!resumeExternalPerformance) return;
|
|
3814
|
+
const resumeIfBound = () => {
|
|
3815
|
+
if (assistant.agent && roomMatchesGrant && room.state === "connected") resumeExternalPerformance();
|
|
3816
|
+
};
|
|
3817
|
+
resumeIfBound();
|
|
3818
|
+
room.on(RoomEvent.Reconnected, resumeIfBound);
|
|
3819
|
+
room.on(RoomEvent.ConnectionStateChanged, resumeIfBound);
|
|
3820
|
+
return () => {
|
|
3821
|
+
room.off(RoomEvent.Reconnected, resumeIfBound);
|
|
3822
|
+
room.off(RoomEvent.ConnectionStateChanged, resumeIfBound);
|
|
3823
|
+
};
|
|
3824
|
+
}, [room, assistant.agent, roomMatchesGrant, resumeExternalPerformance]);
|
|
3242
3825
|
useEffect(() => {
|
|
3243
3826
|
if (!registerDataPublisher) return;
|
|
3244
3827
|
const encoder = new TextEncoder();
|
|
@@ -3374,6 +3957,7 @@ function useRealtimeSession(input) {
|
|
|
3374
3957
|
onBehaviorChange,
|
|
3375
3958
|
onClipResult,
|
|
3376
3959
|
onVoiceError,
|
|
3960
|
+
correlatedAudio,
|
|
3377
3961
|
// everything else (client/active/idleSeconds/idleWarnLeadSeconds/reconnect/requestOptions…)
|
|
3378
3962
|
// flows straight into the inner SSOT hook.
|
|
3379
3963
|
...inner
|
|
@@ -3446,6 +4030,7 @@ function useRealtimeSession(input) {
|
|
|
3446
4030
|
return `t-${Date.now()}-${++turnSequenceRef.current}`;
|
|
3447
4031
|
}, []);
|
|
3448
4032
|
const onLifecycleData = useCallback((frame) => {
|
|
4033
|
+
lifecycle.onLifecycleData?.(frame);
|
|
3449
4034
|
const parsed = lifecycleServerFrameSchema.safeParse(frame);
|
|
3450
4035
|
if (!parsed.success) return;
|
|
3451
4036
|
const f = parsed.data;
|
|
@@ -3505,7 +4090,7 @@ function useRealtimeSession(input) {
|
|
|
3505
4090
|
break;
|
|
3506
4091
|
}
|
|
3507
4092
|
}
|
|
3508
|
-
}, []);
|
|
4093
|
+
}, [lifecycle.onLifecycleData]);
|
|
3509
4094
|
const registerDataPublisher = useCallback((publish) => {
|
|
3510
4095
|
dataPublisherRef.current = publish;
|
|
3511
4096
|
}, []);
|
|
@@ -3771,6 +4356,7 @@ function useRealtimeSession(input) {
|
|
|
3771
4356
|
// stayConnected/reconnect/markActivity/the sinks) → a true drop-in superset, then
|
|
3772
4357
|
// add the realtime-session surface. `reset` + `lifecycle` come AFTER so ours win.
|
|
3773
4358
|
...lifecycle,
|
|
4359
|
+
correlatedAudio,
|
|
3774
4360
|
turn,
|
|
3775
4361
|
clocks,
|
|
3776
4362
|
endsAt,
|
|
@@ -3805,6 +4391,7 @@ function useRealtimeSession(input) {
|
|
|
3805
4391
|
}),
|
|
3806
4392
|
[
|
|
3807
4393
|
lifecycle,
|
|
4394
|
+
correlatedAudio,
|
|
3808
4395
|
turn,
|
|
3809
4396
|
clocks,
|
|
3810
4397
|
endsAt,
|
|
@@ -3944,4 +4531,4 @@ function createProxyClient(options) {
|
|
|
3944
4531
|
};
|
|
3945
4532
|
}
|
|
3946
4533
|
|
|
3947
|
-
export { AdaptivePlayoutController, AvatarVideoSurface, DEFAULT_APPROACHING_END_LEAD_SECONDS, DEFAULT_AVATAR_PLAYOUT_DELAY_SECONDS, DEFAULT_CREDITS_LOW_LEAD_SECONDS, DEFAULT_FISH_TTS_MODEL, DEFAULT_GOVERNOR_CONFIG, DEFAULT_GRACE_CEILING_SECONDS, DEFAULT_GRACE_WINDOW_LEAD_SECONDS, DEFAULT_IDLE_SECONDS, DEFAULT_IDLE_WARN_LEAD_SECONDS, DEFAULT_READY_TIMEOUT_SECONDS, DEFAULT_TURN_TIMEOUT_SECONDS, FISH_DELIVERY_MODELS, MAX_RECONNECT_ATTEMPTS, MAX_SESSION_INSTRUCTIONS_CHARS, RECONNECT_BACKOFF_MS, RealtimeAvatarLiveKitRoom, SessionLifecycleRoomBridge, capacityErrorFromBusy, capacityStateFromGrant, createProxyClient, fishTtsModelSchema, isNativeLiveTrackSubscribed, knownBehaviorStates, mapTurnState, readInboundRtp, sessionBehaviorSchema, sessionClipSchema, splitCallTranscript, useAvatarAdaptivePlayoutDelay, useAvatarCamera, useAvatarPlayoutDelay, useAvatarQualityGovernor, useCallTranscript, useClearStaleInputMute, useLiveKitAvatarGrant, useLiveTrackProducing, useMicLease, useRealtimeAvatarAudioSession, useRealtimeSession, useReleaseMicLeaseOnTrackEnded, useSessionLifecycle };
|
|
4534
|
+
export { AdaptivePlayoutController, AvatarVideoSurface, DEFAULT_APPROACHING_END_LEAD_SECONDS, DEFAULT_AVATAR_PLAYOUT_DELAY_SECONDS, DEFAULT_CREDITS_LOW_LEAD_SECONDS, DEFAULT_FISH_TTS_MODEL, DEFAULT_GOVERNOR_CONFIG, DEFAULT_GRACE_CEILING_SECONDS, DEFAULT_GRACE_WINDOW_LEAD_SECONDS, DEFAULT_IDLE_SECONDS, DEFAULT_IDLE_WARN_LEAD_SECONDS, DEFAULT_READY_TIMEOUT_SECONDS, DEFAULT_TURN_TIMEOUT_SECONDS, EXTERNAL_PERFORMANCE_TIMING, FISH_DELIVERY_MODELS, KNOWN_EXTERNAL_PERFORMANCE_STATUSES, MAX_RECONNECT_ATTEMPTS, MAX_SESSION_INSTRUCTIONS_CHARS, RECONNECT_BACKOFF_MS, RealtimeAvatarLiveKitRoom, SessionLifecycleRoomBridge, capacityErrorFromBusy, capacityStateFromGrant, createProxyClient, fishTtsModelSchema, isNativeLiveTrackSubscribed, knownBehaviorStates, mapTurnState, readInboundRtp, sessionBehaviorSchema, sessionClipSchema, splitCallTranscript, useAvatarAdaptivePlayoutDelay, useAvatarCamera, useAvatarPlayoutDelay, useAvatarQualityGovernor, useCallTranscript, useClearStaleInputMute, useLiveKitAvatarGrant, useLiveTrackProducing, useMicLease, useRealtimeAvatarAudioSession, useRealtimeSession, useReleaseMicLeaseOnTrackEnded, useSessionLifecycle };
|