realtime-avatar 0.32.0 → 0.34.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +56 -1
- package/dist/browser.d.ts +4 -89
- package/dist/browser.js +362 -1
- package/dist/chunk-6KQS7CTR.js +1 -0
- package/dist/{chunk-NKS5ZIWN.js → chunk-GPGNAQZV.js} +413 -170
- package/dist/{chunk-NDCZABOK.js → chunk-OVW4N5NH.js} +1 -13
- package/dist/express.d.ts +2 -2
- package/dist/express.js +2 -2
- package/dist/hono.d.ts +2 -2
- package/dist/hono.js +2 -2
- package/dist/index.d.ts +338 -3
- package/dist/index.js +2 -1
- package/dist/nextjs.d.ts +2 -2
- package/dist/nextjs.js +2 -2
- package/dist/{playout-delay-CtRZdtsj.d.ts → playout-delay-BzoM1BMN.d.ts} +135 -1
- package/dist/{proxy-client-BlaFxR6S.d.ts → proxy-client-CiTOjkg8.d.ts} +81 -3
- package/dist/react-native.d.ts +3 -3
- package/dist/react-native.js +624 -34
- package/dist/react.d.ts +12 -22
- package/dist/react.js +628 -35
- package/dist/server.d.ts +2 -2
- package/dist/server.js +2 -1
- package/dist/tanstack-start.d.ts +2 -2
- package/dist/tanstack-start.js +2 -2
- package/dist/{types-su27kEw0.d.ts → types-Cdi7KWWj.d.ts} +1 -1
- package/dist/{types-BSSqsu_Z.d.ts → types-Cg_i1jhk.d.ts} +209 -18
- package/package.json +3 -3
package/dist/react-native.js
CHANGED
|
@@ -1251,7 +1251,7 @@ function stepNetworkQuality(previous, sample, nowMs, options) {
|
|
|
1251
1251
|
}
|
|
1252
1252
|
|
|
1253
1253
|
// ../client/src/react/network-quality-binding.ts
|
|
1254
|
-
function bindNetworkQuality(publication, inhibited,
|
|
1254
|
+
function bindNetworkQuality(publication, inhibited, notify2, tickMs) {
|
|
1255
1255
|
let disposed = false;
|
|
1256
1256
|
let running = false;
|
|
1257
1257
|
let state = initNetworkQuality(performance.now());
|
|
@@ -1265,7 +1265,7 @@ function bindNetworkQuality(publication, inhibited, notify, tickMs) {
|
|
|
1265
1265
|
};
|
|
1266
1266
|
const emit = (status) => {
|
|
1267
1267
|
try {
|
|
1268
|
-
|
|
1268
|
+
notify2(status);
|
|
1269
1269
|
} catch {
|
|
1270
1270
|
}
|
|
1271
1271
|
};
|
|
@@ -1787,11 +1787,108 @@ var schema5 = z.object({
|
|
|
1787
1787
|
"request_id": schema22.optional()
|
|
1788
1788
|
});
|
|
1789
1789
|
var voiceErrorFrameSchema = schema5;
|
|
1790
|
-
var schema04 = z.
|
|
1791
|
-
var schema13 = z.
|
|
1792
|
-
var schema23 = z.
|
|
1793
|
-
var
|
|
1790
|
+
var schema04 = z.string().min(1).max(200);
|
|
1791
|
+
var schema13 = z.literal(1);
|
|
1792
|
+
var schema23 = z.string().min(1).max(200);
|
|
1793
|
+
var schema33 = z.int().min(1).max(9007199254740991);
|
|
1794
|
+
var schema43 = z.literal(24e3);
|
|
1795
|
+
var schema52 = z.literal("pcm_s16le");
|
|
1796
|
+
var schema6 = z.object({
|
|
1797
|
+
"session_id": schema04,
|
|
1798
|
+
"generation_id": schema04,
|
|
1799
|
+
"version": schema13,
|
|
1800
|
+
"utterance_id": schema23,
|
|
1801
|
+
"stream_id": schema04,
|
|
1802
|
+
"epoch": schema33,
|
|
1803
|
+
"sample_rate": schema43,
|
|
1804
|
+
"channels": schema13,
|
|
1805
|
+
"encoding": schema52
|
|
1806
|
+
});
|
|
1807
|
+
var schema7 = z.enum(["started", "progress", "finished", "cancelled"]);
|
|
1808
|
+
var schema8 = z.int().min(0).max(144e4);
|
|
1809
|
+
z.strictObject({
|
|
1810
|
+
"session_id": schema04,
|
|
1811
|
+
"generation_id": schema04,
|
|
1812
|
+
"utterance_id": schema23,
|
|
1813
|
+
"stream_id": schema04,
|
|
1814
|
+
"epoch": schema33,
|
|
1815
|
+
"status": schema7,
|
|
1816
|
+
"samples": schema8
|
|
1817
|
+
});
|
|
1818
|
+
var schema10 = z.object({
|
|
1819
|
+
"version": schema13,
|
|
1820
|
+
"session_id": schema04,
|
|
1821
|
+
"epoch": schema33
|
|
1822
|
+
});
|
|
1823
|
+
var schema11 = z.literal("external_llm_error");
|
|
1824
|
+
var schema122 = z.enum(["external_llm_refused", "external_llm_failed"]);
|
|
1825
|
+
var schema132 = z.string();
|
|
1826
|
+
var schema14 = z.literal(true);
|
|
1827
|
+
var schema15 = z.object({
|
|
1828
|
+
"kind": schema11,
|
|
1829
|
+
"code": schema122,
|
|
1830
|
+
"message": schema132,
|
|
1831
|
+
"fatal": schema14
|
|
1832
|
+
});
|
|
1833
|
+
var correlatedAudioHeaderSchema = schema6;
|
|
1834
|
+
var correlatedAudioCancelSchema = schema10;
|
|
1835
|
+
var externalLlmErrorFrameSchema = schema15;
|
|
1836
|
+
var CORRELATED_AUDIO_CONTRACT = {
|
|
1837
|
+
"topic": "rta.audio.v1",
|
|
1838
|
+
"receiptMethod": "rta.audio.receipt.v1",
|
|
1839
|
+
"cancelMethod": "rta.audio.cancel.v1",
|
|
1840
|
+
"maxSamples": 144e4,
|
|
1841
|
+
"maxChunkBytes": 48e3,
|
|
1842
|
+
"maxBufferedSamples": 48e3
|
|
1843
|
+
};
|
|
1844
|
+
var schema05 = z.string().min(1).max(200);
|
|
1845
|
+
var schema16 = z.literal(1);
|
|
1846
|
+
var schema24 = z.enum(["respond", "recovery", "initiative"]);
|
|
1847
|
+
var schema34 = z.string().min(1).max(32);
|
|
1848
|
+
var schema44 = z.union([schema24, schema34]);
|
|
1849
|
+
var schema53 = z.enum(["draft", "accepted"]);
|
|
1850
|
+
var schema62 = z.literal("external");
|
|
1851
|
+
var schema72 = z.object({
|
|
1852
|
+
"renderer": schema62
|
|
1853
|
+
});
|
|
1854
|
+
z.object({
|
|
1855
|
+
"session_id": schema05,
|
|
1856
|
+
"generation_id": schema05,
|
|
1857
|
+
"version": schema16,
|
|
1858
|
+
"purpose": schema44,
|
|
1859
|
+
"admission": schema53,
|
|
1860
|
+
"visible_character": schema72.optional()
|
|
1861
|
+
});
|
|
1862
|
+
var schema92 = z.string().min(1).max(200);
|
|
1863
|
+
z.object({
|
|
1864
|
+
"generation_id": schema05,
|
|
1865
|
+
"utterance_id": schema92
|
|
1866
|
+
});
|
|
1867
|
+
var schema112 = z.literal("external_performance");
|
|
1868
|
+
var schema123 = z.enum(["accepted", "dropped", "cancelled", "audio_released"]);
|
|
1869
|
+
var schema133 = z.union([schema123, schema34]);
|
|
1870
|
+
var schema142 = z.literal("unavailable");
|
|
1871
|
+
var schema152 = z.literal("unavailable");
|
|
1872
|
+
var schema162 = z.object({
|
|
1873
|
+
"session_id": schema05,
|
|
1874
|
+
"generation_id": schema05,
|
|
1875
|
+
"kind": schema112,
|
|
1876
|
+
"version": schema16,
|
|
1877
|
+
"status": schema133,
|
|
1878
|
+
"audible_checkpoint": schema142,
|
|
1879
|
+
"audio_mapping": schema152
|
|
1880
|
+
});
|
|
1881
|
+
var externalPerformanceFrameSchema = schema162;
|
|
1882
|
+
var schema06 = z.enum(["s1", "s2-pro", "s2.1-pro", "drama-3"]);
|
|
1883
|
+
var schema17 = z.enum(["speech-1.6", "speech-1.5", "s1-mini"]);
|
|
1884
|
+
var schema25 = z.union([schema06, schema17]);
|
|
1885
|
+
var schema35 = z.enum(["whisper"]);
|
|
1886
|
+
var schema45 = z.null();
|
|
1887
|
+
var schema54 = z.union([schema35, schema45]);
|
|
1888
|
+
var fishTtsModelSchema = schema25;
|
|
1794
1889
|
var DEFAULT_FISH_TTS_MODEL = "s2.1-pro";
|
|
1890
|
+
var fishDeliverySchema = schema54;
|
|
1891
|
+
var FISH_DELIVERY_MODELS = { "whisper": ["drama-3"] };
|
|
1795
1892
|
|
|
1796
1893
|
// ../client/src/wire.ts
|
|
1797
1894
|
var DEFAULT_AVATAR_ID = "maria";
|
|
@@ -1860,8 +1957,15 @@ var fishVoiceSpecSchema = z.object({
|
|
|
1860
1957
|
voice_id: z.string().min(1).max(120),
|
|
1861
1958
|
speed: z.number().min(0.5).max(2).nullable().optional(),
|
|
1862
1959
|
emotion: z.string().min(1).max(80).nullable().optional(),
|
|
1863
|
-
language: z.string().min(2).max(16).nullable().optional()
|
|
1864
|
-
|
|
1960
|
+
language: z.string().min(2).max(16).nullable().optional(),
|
|
1961
|
+
delivery: fishDeliverySchema.optional()
|
|
1962
|
+
}).strict().superRefine((voice, ctx) => {
|
|
1963
|
+
if (!voice.delivery) return;
|
|
1964
|
+
const models = FISH_DELIVERY_MODELS[voice.delivery];
|
|
1965
|
+
if (!models.includes(voice.model)) {
|
|
1966
|
+
ctx.addIssue({ code: "custom", path: ["delivery"], message: `delivery "${voice.delivery}" needs model ${models.join(" | ")}` });
|
|
1967
|
+
}
|
|
1968
|
+
});
|
|
1865
1969
|
var voiceSpecSchema = z.discriminatedUnion("provider", [
|
|
1866
1970
|
cartesiaVoiceSpecSchema,
|
|
1867
1971
|
breezeVoiceSpecSchema,
|
|
@@ -2099,7 +2203,9 @@ var lifecycleServerFrameSchema = z.discriminatedUnion("kind", [
|
|
|
2099
2203
|
endedFrameSchema,
|
|
2100
2204
|
behaviorStateFrameSchema,
|
|
2101
2205
|
clipAckFrameSchema,
|
|
2102
|
-
voiceErrorFrameSchema2
|
|
2206
|
+
voiceErrorFrameSchema2,
|
|
2207
|
+
externalPerformanceFrameSchema,
|
|
2208
|
+
externalLlmErrorFrameSchema
|
|
2103
2209
|
]);
|
|
2104
2210
|
var liveKitCapacitySnapshotSchema = z.object({
|
|
2105
2211
|
// Placement identity + per-worker session ceiling. These are always present on the
|
|
@@ -2381,6 +2487,7 @@ function useCallMedia(room, options) {
|
|
|
2381
2487
|
}, [room, setAudioPlayback, connecting]);
|
|
2382
2488
|
const controls = useMemo(() => ({
|
|
2383
2489
|
startAudio: async () => {
|
|
2490
|
+
if (options.startAudioOverride) return options.startAudioOverride();
|
|
2384
2491
|
try {
|
|
2385
2492
|
await room.startAudio();
|
|
2386
2493
|
} catch {
|
|
@@ -2397,13 +2504,372 @@ function useCallMedia(room, options) {
|
|
|
2397
2504
|
setVersion((value) => value + 1);
|
|
2398
2505
|
}
|
|
2399
2506
|
}
|
|
2400
|
-
}), [room]);
|
|
2507
|
+
}), [room, options.startAudioOverride]);
|
|
2401
2508
|
useEffect(() => {
|
|
2402
2509
|
if (!registerMediaControls) return;
|
|
2403
2510
|
registerMediaControls(controls);
|
|
2404
2511
|
return () => registerMediaControls(null);
|
|
2405
2512
|
}, [controls, registerMediaControls]);
|
|
2406
2513
|
}
|
|
2514
|
+
var attachments = /* @__PURE__ */ new WeakSet();
|
|
2515
|
+
var bindings = /* @__PURE__ */ new WeakMap();
|
|
2516
|
+
function notify(callback) {
|
|
2517
|
+
try {
|
|
2518
|
+
void Promise.resolve(callback()).catch(() => {
|
|
2519
|
+
});
|
|
2520
|
+
} catch {
|
|
2521
|
+
}
|
|
2522
|
+
}
|
|
2523
|
+
function attachCorrelatedAudio(room, options) {
|
|
2524
|
+
if (attachments.has(room)) throw new Error("Correlated audio is already attached to this room");
|
|
2525
|
+
if (!options.sessionId || !options.agentIdentity) throw new Error("Correlated audio requires the held session and bound agent");
|
|
2526
|
+
attachments.add(room);
|
|
2527
|
+
let context;
|
|
2528
|
+
let disposed = false;
|
|
2529
|
+
const previous = bindings.get(room);
|
|
2530
|
+
const binding = previous?.sessionId === options.sessionId ? previous : { sessionId: options.sessionId, epoch: 1, unavailable: false, seen: /* @__PURE__ */ new Set() };
|
|
2531
|
+
bindings.set(room, binding);
|
|
2532
|
+
const sources = [];
|
|
2533
|
+
const active = (source) => sources.includes(source);
|
|
2534
|
+
const buffered = () => sources.reduce((sum, source) => sum + source.received - source.consumed, 0);
|
|
2535
|
+
let nextStart = 0;
|
|
2536
|
+
let blocked;
|
|
2537
|
+
let closing;
|
|
2538
|
+
const seen = binding.seen;
|
|
2539
|
+
const waiters = /* @__PURE__ */ new Set();
|
|
2540
|
+
const wake = () => {
|
|
2541
|
+
for (const resolve of waiters) resolve();
|
|
2542
|
+
waiters.clear();
|
|
2543
|
+
};
|
|
2544
|
+
const identity = (header) => ({
|
|
2545
|
+
session_id: header.session_id,
|
|
2546
|
+
generation_id: header.generation_id,
|
|
2547
|
+
utterance_id: header.utterance_id,
|
|
2548
|
+
stream_id: header.stream_id,
|
|
2549
|
+
epoch: header.epoch
|
|
2550
|
+
});
|
|
2551
|
+
const position = (current, state) => {
|
|
2552
|
+
notify(() => options.onPosition({ ...identity(current.header), samples: current.consumed, sampleRate: current.header.sample_rate, state }));
|
|
2553
|
+
};
|
|
2554
|
+
const receipt = (current, status) => {
|
|
2555
|
+
const payload = { ...identity(current.header), status, samples: current.consumed };
|
|
2556
|
+
if (current.rpcBusy) {
|
|
2557
|
+
current.pendingReceipt = payload;
|
|
2558
|
+
return;
|
|
2559
|
+
}
|
|
2560
|
+
const send = async (next) => {
|
|
2561
|
+
current.rpcBusy = true;
|
|
2562
|
+
try {
|
|
2563
|
+
await room.localParticipant.performRpc({
|
|
2564
|
+
destinationIdentity: options.agentIdentity,
|
|
2565
|
+
method: CORRELATED_AUDIO_CONTRACT.receiptMethod,
|
|
2566
|
+
payload: JSON.stringify(next),
|
|
2567
|
+
responseTimeout: 2e3
|
|
2568
|
+
});
|
|
2569
|
+
} catch {
|
|
2570
|
+
if (!disposed && next.epoch === binding.epoch) invalidate("connection_lost");
|
|
2571
|
+
} finally {
|
|
2572
|
+
current.rpcBusy = false;
|
|
2573
|
+
const pending = current.pendingReceipt;
|
|
2574
|
+
current.pendingReceipt = void 0;
|
|
2575
|
+
if (pending && !disposed && pending.epoch === binding.epoch && !binding.unavailable) void send(pending);
|
|
2576
|
+
}
|
|
2577
|
+
};
|
|
2578
|
+
void send(payload);
|
|
2579
|
+
};
|
|
2580
|
+
const clear = () => {
|
|
2581
|
+
for (const current of sources.splice(0)) {
|
|
2582
|
+
current.abort.abort();
|
|
2583
|
+
for (const segment of current.segments) {
|
|
2584
|
+
segment.node.onended = null;
|
|
2585
|
+
segment.node.stop();
|
|
2586
|
+
segment.node.disconnect();
|
|
2587
|
+
}
|
|
2588
|
+
current.segments.length = 0;
|
|
2589
|
+
}
|
|
2590
|
+
nextStart = context?.currentTime ?? 0;
|
|
2591
|
+
wake();
|
|
2592
|
+
};
|
|
2593
|
+
const invalidate = (reason) => {
|
|
2594
|
+
if (disposed) return;
|
|
2595
|
+
if (reason === "connection_lost") {
|
|
2596
|
+
if (binding.unavailable) return;
|
|
2597
|
+
binding.unavailable = true;
|
|
2598
|
+
}
|
|
2599
|
+
if (reason === "invalid_stream" || reason === "disposed") for (const current of sources) receipt(current, "cancelled");
|
|
2600
|
+
clear();
|
|
2601
|
+
notify(() => options.onInvalidated(reason));
|
|
2602
|
+
};
|
|
2603
|
+
const tick = (reportCredit = true) => {
|
|
2604
|
+
if (!context || context.state !== "running" || disposed || binding.unavailable) return;
|
|
2605
|
+
for (const current of [...sources]) {
|
|
2606
|
+
if (!active(current)) continue;
|
|
2607
|
+
let consumed = current.consumed;
|
|
2608
|
+
for (const segment of current.segments) {
|
|
2609
|
+
if (context.currentTime < segment.start) break;
|
|
2610
|
+
const samples = Math.min(segment.length, Math.max(0, Math.floor((context.currentTime - segment.start) * current.header.sample_rate + 1e-6)));
|
|
2611
|
+
consumed = Math.max(consumed, segment.offset + samples);
|
|
2612
|
+
if (!current.started && samples > 0) {
|
|
2613
|
+
current.started = true;
|
|
2614
|
+
receipt(current, "started");
|
|
2615
|
+
position(current, "playing");
|
|
2616
|
+
current.lastPosition = current.consumed;
|
|
2617
|
+
}
|
|
2618
|
+
}
|
|
2619
|
+
current.consumed = consumed;
|
|
2620
|
+
while (current.segments.length && current.segments[0].offset + current.segments[0].length <= consumed) {
|
|
2621
|
+
const segment = current.segments.shift();
|
|
2622
|
+
if (segment) {
|
|
2623
|
+
segment.node.onended = null;
|
|
2624
|
+
segment.node.disconnect();
|
|
2625
|
+
}
|
|
2626
|
+
}
|
|
2627
|
+
if (current.started && current.eof && consumed === current.received && consumed > 0) {
|
|
2628
|
+
position(current, "finished");
|
|
2629
|
+
if (!active(current)) continue;
|
|
2630
|
+
receipt(current, "finished");
|
|
2631
|
+
sources.splice(sources.indexOf(current), 1);
|
|
2632
|
+
wake();
|
|
2633
|
+
} else {
|
|
2634
|
+
if (current.started && consumed > current.lastPosition) {
|
|
2635
|
+
current.lastPosition = consumed;
|
|
2636
|
+
position(current, "playing");
|
|
2637
|
+
}
|
|
2638
|
+
if (!active(current)) continue;
|
|
2639
|
+
if (reportCredit && consumed > current.lastReported) {
|
|
2640
|
+
current.lastReported = consumed;
|
|
2641
|
+
receipt(current, "progress");
|
|
2642
|
+
}
|
|
2643
|
+
wake();
|
|
2644
|
+
}
|
|
2645
|
+
}
|
|
2646
|
+
};
|
|
2647
|
+
const playbackState = () => {
|
|
2648
|
+
const next = context?.state !== "running";
|
|
2649
|
+
if (next !== blocked) {
|
|
2650
|
+
blocked = next;
|
|
2651
|
+
notify(() => options.onPlaybackBlocked?.(next ? startAudio : null));
|
|
2652
|
+
}
|
|
2653
|
+
tick();
|
|
2654
|
+
};
|
|
2655
|
+
const getContext = () => {
|
|
2656
|
+
if (!context) {
|
|
2657
|
+
context = new AudioContext({ sampleRate: correlatedAudioHeaderSchema.shape.sample_rate.value });
|
|
2658
|
+
context.onstatechange = playbackState;
|
|
2659
|
+
playbackState();
|
|
2660
|
+
}
|
|
2661
|
+
return context;
|
|
2662
|
+
};
|
|
2663
|
+
async function startAudio() {
|
|
2664
|
+
if (disposed || binding.unavailable) throw new Error("Correlated audio binding is no longer available");
|
|
2665
|
+
const ctx = getContext();
|
|
2666
|
+
await ctx.resume();
|
|
2667
|
+
playbackState();
|
|
2668
|
+
if (ctx.state !== "running") throw new Error("Correlated audio playback is blocked");
|
|
2669
|
+
}
|
|
2670
|
+
async function receive(reader, participant) {
|
|
2671
|
+
if (disposed || binding.unavailable || room.state !== "connected" || participant.identity !== options.agentIdentity) return;
|
|
2672
|
+
let value;
|
|
2673
|
+
try {
|
|
2674
|
+
value = JSON.parse(reader.info.attributes?.["rta.audio"] ?? "");
|
|
2675
|
+
} catch {
|
|
2676
|
+
return;
|
|
2677
|
+
}
|
|
2678
|
+
const parsed = correlatedAudioHeaderSchema.safeParse(value);
|
|
2679
|
+
if (!parsed.success || parsed.data.session_id !== options.sessionId || parsed.data.epoch < binding.epoch) return;
|
|
2680
|
+
const header = parsed.data;
|
|
2681
|
+
if (header.stream_id !== reader.info.id) return;
|
|
2682
|
+
if (header.epoch > binding.epoch) {
|
|
2683
|
+
binding.epoch = header.epoch;
|
|
2684
|
+
invalidate("cancelled");
|
|
2685
|
+
}
|
|
2686
|
+
const key = `${header.epoch}/${header.stream_id}`;
|
|
2687
|
+
if (seen.has(key)) return;
|
|
2688
|
+
if (sources.length >= 128 || seen.size >= 1e4) {
|
|
2689
|
+
invalidate("invalid_stream");
|
|
2690
|
+
return;
|
|
2691
|
+
}
|
|
2692
|
+
seen.add(key);
|
|
2693
|
+
const previousSource = sources.at(-1);
|
|
2694
|
+
const current = { header, abort: new AbortController(), segments: [], received: 0, consumed: 0, started: false, eof: false, lastReported: 0, lastPosition: -1, rpcBusy: false };
|
|
2695
|
+
sources.push(current);
|
|
2696
|
+
let carry;
|
|
2697
|
+
try {
|
|
2698
|
+
if (options.prepare) await options.prepare(header, current.abort.signal);
|
|
2699
|
+
if (!active(current) || current.abort.signal.aborted) return;
|
|
2700
|
+
const ctx = getContext();
|
|
2701
|
+
while (active(current) && (previousSource && active(previousSource) && !previousSource.eof || buffered() > CORRELATED_AUDIO_CONTRACT.maxBufferedSamples - CORRELATED_AUDIO_CONTRACT.maxChunkBytes / 2)) {
|
|
2702
|
+
await new Promise((resolve) => waiters.add(resolve));
|
|
2703
|
+
}
|
|
2704
|
+
if (!active(current)) return;
|
|
2705
|
+
for await (const chunk of reader.withAbortSignal(current.abort.signal)) {
|
|
2706
|
+
if (!active(current)) break;
|
|
2707
|
+
if (chunk.byteLength > CORRELATED_AUDIO_CONTRACT.maxChunkBytes) throw new Error("PCM chunk exceeds contract");
|
|
2708
|
+
const count = Math.floor((chunk.byteLength + (carry === void 0 ? 0 : 1)) / 2);
|
|
2709
|
+
if (current.received + count > CORRELATED_AUDIO_CONTRACT.maxSamples) throw new Error("PCM stream exceeds contract");
|
|
2710
|
+
while (active(current) && buffered() + count > CORRELATED_AUDIO_CONTRACT.maxBufferedSamples) {
|
|
2711
|
+
await new Promise((resolve) => waiters.add(resolve));
|
|
2712
|
+
}
|
|
2713
|
+
if (!active(current)) break;
|
|
2714
|
+
if (!count) {
|
|
2715
|
+
if (chunk.length) carry = chunk[0];
|
|
2716
|
+
continue;
|
|
2717
|
+
}
|
|
2718
|
+
const buffer = ctx.createBuffer(header.channels, count, header.sample_rate);
|
|
2719
|
+
const data = buffer.getChannelData(0);
|
|
2720
|
+
let cursor = 0;
|
|
2721
|
+
for (let i = 0; i < count; i++) {
|
|
2722
|
+
const low = carry ?? chunk[cursor++];
|
|
2723
|
+
carry = void 0;
|
|
2724
|
+
const raw = low | chunk[cursor++] << 8;
|
|
2725
|
+
data[i] = (raw >= 32768 ? raw - 65536 : raw) / 32768;
|
|
2726
|
+
}
|
|
2727
|
+
if (cursor < chunk.length) carry = chunk[cursor];
|
|
2728
|
+
const node = ctx.createBufferSource();
|
|
2729
|
+
node.buffer = buffer;
|
|
2730
|
+
node.connect(ctx.destination);
|
|
2731
|
+
const start = Math.max(ctx.currentTime, nextStart);
|
|
2732
|
+
current.segments.push({ node, start, offset: current.received, length: count });
|
|
2733
|
+
current.received += count;
|
|
2734
|
+
nextStart = start + count / header.sample_rate;
|
|
2735
|
+
node.onended = () => tick();
|
|
2736
|
+
node.start(start);
|
|
2737
|
+
while (active(current) && buffered() > CORRELATED_AUDIO_CONTRACT.maxBufferedSamples - CORRELATED_AUDIO_CONTRACT.maxChunkBytes / 2) {
|
|
2738
|
+
await new Promise((resolve) => waiters.add(resolve));
|
|
2739
|
+
}
|
|
2740
|
+
}
|
|
2741
|
+
if (!active(current)) return;
|
|
2742
|
+
if (carry !== void 0 || current.received === 0) throw new Error("Incomplete PCM stream");
|
|
2743
|
+
current.eof = true;
|
|
2744
|
+
wake();
|
|
2745
|
+
tick(false);
|
|
2746
|
+
} catch {
|
|
2747
|
+
if (active(current)) invalidate("invalid_stream");
|
|
2748
|
+
}
|
|
2749
|
+
}
|
|
2750
|
+
const cancel = async (data) => {
|
|
2751
|
+
if (disposed || binding.unavailable || room.state !== "connected" || data.callerIdentity !== options.agentIdentity) throw new Error("Unbound audio cancellation");
|
|
2752
|
+
const value = JSON.parse(data.payload);
|
|
2753
|
+
const frame2 = correlatedAudioCancelSchema.parse(value);
|
|
2754
|
+
if (frame2.session_id !== options.sessionId) throw new Error("Wrong audio session");
|
|
2755
|
+
if (frame2.epoch <= binding.epoch) return "";
|
|
2756
|
+
binding.epoch = frame2.epoch;
|
|
2757
|
+
invalidate("cancelled");
|
|
2758
|
+
return "";
|
|
2759
|
+
};
|
|
2760
|
+
const lost = () => invalidate("connection_lost");
|
|
2761
|
+
const changed = () => {
|
|
2762
|
+
if (["disconnected", "reconnecting", "signalReconnecting"].includes(room.state)) lost();
|
|
2763
|
+
};
|
|
2764
|
+
const participantLeft = (participant) => {
|
|
2765
|
+
if (participant.identity === options.agentIdentity) lost();
|
|
2766
|
+
};
|
|
2767
|
+
let registered = false;
|
|
2768
|
+
try {
|
|
2769
|
+
room.registerByteStreamHandler(CORRELATED_AUDIO_CONTRACT.topic, (reader, participant) => {
|
|
2770
|
+
void receive(reader, participant);
|
|
2771
|
+
});
|
|
2772
|
+
registered = true;
|
|
2773
|
+
room.localParticipant.registerRpcMethod(CORRELATED_AUDIO_CONTRACT.cancelMethod, cancel);
|
|
2774
|
+
} catch (error) {
|
|
2775
|
+
if (registered) room.unregisterByteStreamHandler(CORRELATED_AUDIO_CONTRACT.topic);
|
|
2776
|
+
attachments.delete(room);
|
|
2777
|
+
throw error;
|
|
2778
|
+
}
|
|
2779
|
+
room.on(RoomEvent.Reconnecting, lost);
|
|
2780
|
+
room.on(RoomEvent.SignalReconnecting, lost);
|
|
2781
|
+
room.on(RoomEvent.Disconnected, lost);
|
|
2782
|
+
room.on(RoomEvent.ConnectionStateChanged, changed);
|
|
2783
|
+
room.on(RoomEvent.ParticipantDisconnected, participantLeft);
|
|
2784
|
+
const timer = setInterval(() => tick(), 100);
|
|
2785
|
+
let frame = 0;
|
|
2786
|
+
const render = () => {
|
|
2787
|
+
if (disposed) return;
|
|
2788
|
+
tick(false);
|
|
2789
|
+
frame = requestAnimationFrame(render);
|
|
2790
|
+
};
|
|
2791
|
+
frame = requestAnimationFrame(render);
|
|
2792
|
+
return {
|
|
2793
|
+
startAudio,
|
|
2794
|
+
dispose() {
|
|
2795
|
+
if (closing) return closing;
|
|
2796
|
+
invalidate("disposed");
|
|
2797
|
+
disposed = true;
|
|
2798
|
+
clearInterval(timer);
|
|
2799
|
+
cancelAnimationFrame(frame);
|
|
2800
|
+
room.unregisterByteStreamHandler(CORRELATED_AUDIO_CONTRACT.topic);
|
|
2801
|
+
room.localParticipant.unregisterRpcMethod(CORRELATED_AUDIO_CONTRACT.cancelMethod);
|
|
2802
|
+
room.off(RoomEvent.Reconnecting, lost);
|
|
2803
|
+
room.off(RoomEvent.SignalReconnecting, lost);
|
|
2804
|
+
room.off(RoomEvent.Disconnected, lost);
|
|
2805
|
+
room.off(RoomEvent.ConnectionStateChanged, changed);
|
|
2806
|
+
room.off(RoomEvent.ParticipantDisconnected, participantLeft);
|
|
2807
|
+
attachments.delete(room);
|
|
2808
|
+
if (context) context.onstatechange = null;
|
|
2809
|
+
closing = context?.close().catch(() => {
|
|
2810
|
+
}) ?? Promise.resolve();
|
|
2811
|
+
return closing;
|
|
2812
|
+
}
|
|
2813
|
+
};
|
|
2814
|
+
}
|
|
2815
|
+
|
|
2816
|
+
// ../client/src/external-performance.ts
|
|
2817
|
+
var EXTERNAL_PERFORMANCE_KIND = externalPerformanceFrameSchema.shape.kind.value;
|
|
2818
|
+
var EXTERNAL_PERFORMANCE_TIMING = Object.freeze({
|
|
2819
|
+
receiverPlayout: "unsupported",
|
|
2820
|
+
audibleCheckpoint: externalPerformanceFrameSchema.shape.audible_checkpoint.value,
|
|
2821
|
+
audioMapping: externalPerformanceFrameSchema.shape.audio_mapping.value
|
|
2822
|
+
});
|
|
2823
|
+
var statusSchema = externalPerformanceFrameSchema.shape.status;
|
|
2824
|
+
var KNOWN_EXTERNAL_PERFORMANCE_STATUSES = Object.freeze([...statusSchema.options[0].options]);
|
|
2825
|
+
var knownStatuses = new Set(KNOWN_EXTERNAL_PERFORMANCE_STATUSES);
|
|
2826
|
+
function isObservable(frame) {
|
|
2827
|
+
return knownStatuses.has(frame.status);
|
|
2828
|
+
}
|
|
2829
|
+
var NEXT = {
|
|
2830
|
+
unseen: /* @__PURE__ */ new Set(["accepted", "dropped"]),
|
|
2831
|
+
accepted: /* @__PURE__ */ new Set(["audio_released", "cancelled"]),
|
|
2832
|
+
audio_released: /* @__PURE__ */ new Set(["cancelled"]),
|
|
2833
|
+
dropped: /* @__PURE__ */ new Set(),
|
|
2834
|
+
cancelled: /* @__PURE__ */ new Set(),
|
|
2835
|
+
retired: /* @__PURE__ */ new Set()
|
|
2836
|
+
};
|
|
2837
|
+
var LEDGER_LIMIT = 4096;
|
|
2838
|
+
function deliver(callback) {
|
|
2839
|
+
try {
|
|
2840
|
+
void Promise.resolve(callback()).catch(() => {
|
|
2841
|
+
});
|
|
2842
|
+
} catch {
|
|
2843
|
+
}
|
|
2844
|
+
}
|
|
2845
|
+
function createExternalPerformanceObserver(sessionId, callbacks) {
|
|
2846
|
+
const generations = /* @__PURE__ */ new Map();
|
|
2847
|
+
let state = "live";
|
|
2848
|
+
return {
|
|
2849
|
+
receive(frame) {
|
|
2850
|
+
if (state !== "live" || frame.session_id !== sessionId || !isObservable(frame)) return;
|
|
2851
|
+
const previous = generations.get(frame.generation_id) ?? "unseen";
|
|
2852
|
+
if (!NEXT[previous].has(frame.status)) return;
|
|
2853
|
+
if (previous === "unseen" && generations.size >= LEDGER_LIMIT) {
|
|
2854
|
+
const oldest = generations.keys().next();
|
|
2855
|
+
if (!oldest.done) generations.delete(oldest.value);
|
|
2856
|
+
}
|
|
2857
|
+
generations.set(frame.generation_id, frame.status);
|
|
2858
|
+
deliver(() => callbacks.onFrame(frame));
|
|
2859
|
+
},
|
|
2860
|
+
invalidate(reason) {
|
|
2861
|
+
if (state === "closed" || state === "paused" && reason === "connection_lost") return;
|
|
2862
|
+
state = reason === "connection_lost" ? "paused" : "closed";
|
|
2863
|
+
if (state === "closed") generations.clear();
|
|
2864
|
+
else for (const id of generations.keys()) generations.set(id, "retired");
|
|
2865
|
+
deliver(() => callbacks.onInvalidated(reason));
|
|
2866
|
+
},
|
|
2867
|
+
/** The call is connected to its agent again. Only a connection loss can be resumed. */
|
|
2868
|
+
resume() {
|
|
2869
|
+
if (state === "paused") state = "live";
|
|
2870
|
+
}
|
|
2871
|
+
};
|
|
2872
|
+
}
|
|
2407
2873
|
|
|
2408
2874
|
// ../client/src/react/session-lifecycle.ts
|
|
2409
2875
|
function phaseForReason(reason) {
|
|
@@ -2594,6 +3060,32 @@ function useSessionLifecycle(input) {
|
|
|
2594
3060
|
const grantIdRef = useRef(null);
|
|
2595
3061
|
grantIdRef.current = grantId;
|
|
2596
3062
|
const retiredGrantRef = useRef(null);
|
|
3063
|
+
const performanceEnabled = input.externalPerformance !== void 0;
|
|
3064
|
+
const performanceCallbacksRef = useRef(input.externalPerformance);
|
|
3065
|
+
if (input.externalPerformance) performanceCallbacksRef.current = input.externalPerformance;
|
|
3066
|
+
const performanceRef = useRef(null);
|
|
3067
|
+
const endedPerformanceGrantRef = useRef(null);
|
|
3068
|
+
useEffect(() => {
|
|
3069
|
+
if (!grantId || !performanceEnabled || endedPerformanceGrantRef.current === grantId) return;
|
|
3070
|
+
const observer = createExternalPerformanceObserver(grantId, {
|
|
3071
|
+
onFrame: (frame) => performanceCallbacksRef.current?.onFrame(frame),
|
|
3072
|
+
onInvalidated: (reason) => {
|
|
3073
|
+
if (reason === "ended") endedPerformanceGrantRef.current = grantId;
|
|
3074
|
+
performanceCallbacksRef.current?.onInvalidated(reason);
|
|
3075
|
+
}
|
|
3076
|
+
});
|
|
3077
|
+
performanceRef.current = observer;
|
|
3078
|
+
return () => {
|
|
3079
|
+
observer.invalidate("binding_changed");
|
|
3080
|
+
if (performanceRef.current === observer) performanceRef.current = null;
|
|
3081
|
+
};
|
|
3082
|
+
}, [grantId, performanceEnabled]);
|
|
3083
|
+
const invalidateExternalPerformance = useCallback((reason) => {
|
|
3084
|
+
if (grantId === grantIdRef.current) performanceRef.current?.invalidate(reason);
|
|
3085
|
+
}, [grantId]);
|
|
3086
|
+
const resumeExternalPerformance = useCallback(() => {
|
|
3087
|
+
if (grantId === grantIdRef.current) performanceRef.current?.resume();
|
|
3088
|
+
}, [grantId]);
|
|
2597
3089
|
const [recovery, setRecovery] = useState({ kind: "connected" });
|
|
2598
3090
|
const recoveryRef = useRef(recovery);
|
|
2599
3091
|
recoveryRef.current = recovery;
|
|
@@ -2639,6 +3131,7 @@ function useSessionLifecycle(input) {
|
|
|
2639
3131
|
}
|
|
2640
3132
|
}, []);
|
|
2641
3133
|
const reset = useCallback(() => {
|
|
3134
|
+
invalidateExternalPerformance("binding_changed");
|
|
2642
3135
|
clearTimer();
|
|
2643
3136
|
connectedRef.current = false;
|
|
2644
3137
|
manualReconnectPendingRef.current = false;
|
|
@@ -2653,7 +3146,7 @@ function useSessionLifecycle(input) {
|
|
|
2653
3146
|
setConnected(false);
|
|
2654
3147
|
setAgentPresentState(false);
|
|
2655
3148
|
lastActivityRef.current = Date.now();
|
|
2656
|
-
}, [clearTimer]);
|
|
3149
|
+
}, [clearTimer, invalidateExternalPerformance]);
|
|
2657
3150
|
useEffect(() => {
|
|
2658
3151
|
if (active) return;
|
|
2659
3152
|
userEndedRef.current = false;
|
|
@@ -2687,6 +3180,7 @@ function useSessionLifecycle(input) {
|
|
|
2687
3180
|
}, [grantId]);
|
|
2688
3181
|
const endIdle = useCallback(() => {
|
|
2689
3182
|
if (!connectedRef.current) return;
|
|
3183
|
+
invalidateExternalPerformance("ended");
|
|
2690
3184
|
clearTimer();
|
|
2691
3185
|
connectedRef.current = false;
|
|
2692
3186
|
setConnected(false);
|
|
@@ -2694,7 +3188,7 @@ function useSessionLifecycle(input) {
|
|
|
2694
3188
|
releaseRef.current("idle_timeout");
|
|
2695
3189
|
setRecovery({ kind: "ended", reason: "idle" });
|
|
2696
3190
|
leaveRoomRef.current?.();
|
|
2697
|
-
}, [clearTimer]);
|
|
3191
|
+
}, [clearTimer, invalidateExternalPerformance]);
|
|
2698
3192
|
const onConnected = useCallback(() => {
|
|
2699
3193
|
if (grantId === null || grantIdRef.current !== grantId) return;
|
|
2700
3194
|
if (userEndedRef.current || readinessFailureRef.current || retiredGrantRef.current !== null && retiredGrantRef.current === grantIdRef.current) return;
|
|
@@ -2712,6 +3206,7 @@ function useSessionLifecycle(input) {
|
|
|
2712
3206
|
if (grantId === null || grantIdRef.current !== grantId) return;
|
|
2713
3207
|
if (!active || userEndedRef.current || readinessFailureRef.current) return;
|
|
2714
3208
|
const action = disconnectAction(reason, connectedRef.current);
|
|
3209
|
+
invalidateExternalPerformance(action === "end" ? "ended" : "connection_lost");
|
|
2715
3210
|
if (action === "noop") return;
|
|
2716
3211
|
if (action === "reset") {
|
|
2717
3212
|
if (recoveryRef.current.kind === "refreshing" || recoveryRef.current.kind === "reconnecting") return;
|
|
@@ -2731,12 +3226,13 @@ function useSessionLifecycle(input) {
|
|
|
2731
3226
|
refreshBaselineCapacityRef.current = null;
|
|
2732
3227
|
setRecovery({ kind: "reconnecting", attempt: attemptRef.current });
|
|
2733
3228
|
},
|
|
2734
|
-
[active, reset, grantId]
|
|
3229
|
+
[active, reset, grantId, invalidateExternalPerformance]
|
|
2735
3230
|
);
|
|
2736
3231
|
const onConnectionError = useCallback(
|
|
2737
3232
|
(error) => {
|
|
2738
3233
|
if (grantId === null || grantIdRef.current !== grantId) return;
|
|
2739
3234
|
if (!active || userEndedRef.current || readinessFailureRef.current || !isRecoverableConnectionError(error)) return;
|
|
3235
|
+
invalidateExternalPerformance("connection_lost");
|
|
2740
3236
|
connectedRef.current = false;
|
|
2741
3237
|
setConnected(false);
|
|
2742
3238
|
setAgentPresentState(false);
|
|
@@ -2747,7 +3243,7 @@ function useSessionLifecycle(input) {
|
|
|
2747
3243
|
return { kind: "reconnecting", attempt: attemptRef.current };
|
|
2748
3244
|
});
|
|
2749
3245
|
},
|
|
2750
|
-
[active, grantId]
|
|
3246
|
+
[active, grantId, invalidateExternalPerformance]
|
|
2751
3247
|
);
|
|
2752
3248
|
useEffect(() => {
|
|
2753
3249
|
if (capacity.kind === "error" && recovery.kind === "connected") manualReconnectPendingRef.current = false;
|
|
@@ -2767,6 +3263,9 @@ function useSessionLifecycle(input) {
|
|
|
2767
3263
|
onConnected();
|
|
2768
3264
|
return;
|
|
2769
3265
|
}
|
|
3266
|
+
if (state === "disconnected" && connectedRef.current || state === "reconnecting" || state === "signalReconnecting") {
|
|
3267
|
+
invalidateExternalPerformance("connection_lost");
|
|
3268
|
+
}
|
|
2770
3269
|
if (state === "reconnecting" || state === "signalReconnecting") {
|
|
2771
3270
|
if (connectedRef.current) {
|
|
2772
3271
|
setConnected(false);
|
|
@@ -2776,7 +3275,7 @@ function useSessionLifecycle(input) {
|
|
|
2776
3275
|
}
|
|
2777
3276
|
}
|
|
2778
3277
|
},
|
|
2779
|
-
[active, onConnected, grantId]
|
|
3278
|
+
[active, onConnected, grantId, invalidateExternalPerformance]
|
|
2780
3279
|
);
|
|
2781
3280
|
useEffect(() => {
|
|
2782
3281
|
if (!active || userEnded || readinessFailed || !connected || !agentPresent || !grantState.grant) return;
|
|
@@ -2789,6 +3288,7 @@ function useSessionLifecycle(input) {
|
|
|
2789
3288
|
const failReadiness = useCallback((code) => {
|
|
2790
3289
|
if (userEndedRef.current || readinessFailureRef.current) return;
|
|
2791
3290
|
readinessFailureRef.current = code;
|
|
3291
|
+
invalidateExternalPerformance("ended");
|
|
2792
3292
|
clearTimer();
|
|
2793
3293
|
connectedRef.current = false;
|
|
2794
3294
|
manualReconnectPendingRef.current = false;
|
|
@@ -2801,7 +3301,7 @@ function useSessionLifecycle(input) {
|
|
|
2801
3301
|
releaseRef.current("disconnected");
|
|
2802
3302
|
setReadinessFailed(true);
|
|
2803
3303
|
leaveRoomRef.current?.();
|
|
2804
|
-
}, [clearTimer]);
|
|
3304
|
+
}, [clearTimer, invalidateExternalPerformance]);
|
|
2805
3305
|
useEffect(() => {
|
|
2806
3306
|
if (!active || userEnded || readinessFailed) return;
|
|
2807
3307
|
if (recovery.kind === "ended" || recovery.kind === "failed") return;
|
|
@@ -2882,6 +3382,7 @@ function useSessionLifecycle(input) {
|
|
|
2882
3382
|
const end = useCallback(() => {
|
|
2883
3383
|
if (!active || userEndedRef.current || phaseRef.current.kind === "ended") return;
|
|
2884
3384
|
userEndedRef.current = true;
|
|
3385
|
+
invalidateExternalPerformance("ended");
|
|
2885
3386
|
clearTimer();
|
|
2886
3387
|
connectedRef.current = false;
|
|
2887
3388
|
manualReconnectPendingRef.current = false;
|
|
@@ -2892,7 +3393,7 @@ function useSessionLifecycle(input) {
|
|
|
2892
3393
|
setRecovery({ kind: "ended", reason: "user" });
|
|
2893
3394
|
clearRef.current();
|
|
2894
3395
|
leaveRoomRef.current?.();
|
|
2895
|
-
}, [active, clearTimer]);
|
|
3396
|
+
}, [active, clearTimer, invalidateExternalPerformance]);
|
|
2896
3397
|
const reconnect = useCallback(() => {
|
|
2897
3398
|
if (!active) return;
|
|
2898
3399
|
if (readinessFailureRef.current) {
|
|
@@ -2962,16 +3463,22 @@ function useSessionLifecycle(input) {
|
|
|
2962
3463
|
const lastBehaviorRef = useRef(null);
|
|
2963
3464
|
const behaviorEnabled = Boolean(onBehaviorChange);
|
|
2964
3465
|
const onLifecycleData = useMemo(() => {
|
|
2965
|
-
if (!behaviorEnabled) return void 0;
|
|
3466
|
+
if (!behaviorEnabled && !performanceEnabled) return void 0;
|
|
2966
3467
|
return (frame) => {
|
|
2967
3468
|
const parsed = lifecycleServerFrameSchema.safeParse(frame);
|
|
2968
|
-
if (!parsed.success
|
|
3469
|
+
if (!parsed.success) return;
|
|
3470
|
+
if (parsed.data.kind === "external_performance") {
|
|
3471
|
+
if (grantId === grantIdRef.current && parsed.data.session_id === grantId) performanceRef.current?.receive(parsed.data);
|
|
3472
|
+
return;
|
|
3473
|
+
}
|
|
3474
|
+
if (parsed.data.kind === "ended") invalidateExternalPerformance("ended");
|
|
3475
|
+
if (!behaviorEnabled || parsed.data.kind !== "behavior_state") return;
|
|
2969
3476
|
const next = nextBehaviorSnapshot(lastBehaviorRef.current, parsed.data);
|
|
2970
3477
|
if (!next) return;
|
|
2971
3478
|
lastBehaviorRef.current = next;
|
|
2972
3479
|
onBehaviorChangeRef.current?.(next);
|
|
2973
3480
|
};
|
|
2974
|
-
}, [behaviorEnabled]);
|
|
3481
|
+
}, [behaviorEnabled, performanceEnabled, grantId, invalidateExternalPerformance]);
|
|
2975
3482
|
return useMemo(
|
|
2976
3483
|
() => ({
|
|
2977
3484
|
phase,
|
|
@@ -2990,7 +3497,9 @@ function useSessionLifecycle(input) {
|
|
|
2990
3497
|
setAgentPresent,
|
|
2991
3498
|
registerLeaveRoom,
|
|
2992
3499
|
reset,
|
|
2993
|
-
onLifecycleData
|
|
3500
|
+
onLifecycleData,
|
|
3501
|
+
invalidateExternalPerformance: performanceEnabled ? invalidateExternalPerformance : void 0,
|
|
3502
|
+
resumeExternalPerformance: performanceEnabled ? resumeExternalPerformance : void 0
|
|
2994
3503
|
}),
|
|
2995
3504
|
[
|
|
2996
3505
|
phase,
|
|
@@ -3009,7 +3518,10 @@ function useSessionLifecycle(input) {
|
|
|
3009
3518
|
setAgentPresent,
|
|
3010
3519
|
registerLeaveRoom,
|
|
3011
3520
|
reset,
|
|
3012
|
-
onLifecycleData
|
|
3521
|
+
onLifecycleData,
|
|
3522
|
+
invalidateExternalPerformance,
|
|
3523
|
+
resumeExternalPerformance,
|
|
3524
|
+
performanceEnabled
|
|
3013
3525
|
]
|
|
3014
3526
|
);
|
|
3015
3527
|
}
|
|
@@ -3038,9 +3550,45 @@ function SessionLifecycleRoomBridge({
|
|
|
3038
3550
|
const transcriptions = useTranscriptions();
|
|
3039
3551
|
const room = useRoomContext();
|
|
3040
3552
|
const roomMatchesGrant = lifecycle.grant === void 0 || lifecycle.grant !== null && room.name === lifecycle.grant.room_name;
|
|
3553
|
+
const correlatedRef = useRef(null);
|
|
3554
|
+
const correlatedObserver = useRef(lifecycle.correlatedAudio);
|
|
3555
|
+
correlatedObserver.current = lifecycle.correlatedAudio;
|
|
3556
|
+
const correlatedEnabled = lifecycle.correlatedAudio !== void 0;
|
|
3557
|
+
const correlatedSessionId = lifecycle.grant?.session_id;
|
|
3558
|
+
const startCorrelatedAudio = useCallback(async () => {
|
|
3559
|
+
const player = correlatedRef.current;
|
|
3560
|
+
if (!player) return false;
|
|
3561
|
+
try {
|
|
3562
|
+
await player.startAudio();
|
|
3563
|
+
return true;
|
|
3564
|
+
} catch {
|
|
3565
|
+
return false;
|
|
3566
|
+
}
|
|
3567
|
+
}, []);
|
|
3568
|
+
useEffect(() => {
|
|
3569
|
+
if (!correlatedEnabled || !correlatedSessionId || !roomMatchesGrant || !assistant.agent) return;
|
|
3570
|
+
const player = attachCorrelatedAudio(room, {
|
|
3571
|
+
sessionId: correlatedSessionId,
|
|
3572
|
+
agentIdentity: assistant.agent.identity,
|
|
3573
|
+
prepare: (header, signal) => correlatedObserver.current?.prepare?.(header, signal) ?? Promise.resolve(),
|
|
3574
|
+
onPosition: (position) => correlatedObserver.current?.onPosition(position),
|
|
3575
|
+
onInvalidated: (reason) => correlatedObserver.current?.onInvalidated(reason),
|
|
3576
|
+
onPlaybackBlocked: (unblock) => {
|
|
3577
|
+
setAudioPlayback?.(unblock ? "blocked" : "allowed");
|
|
3578
|
+
correlatedObserver.current?.onPlaybackBlocked?.(unblock);
|
|
3579
|
+
}
|
|
3580
|
+
});
|
|
3581
|
+
correlatedRef.current = player;
|
|
3582
|
+
return () => {
|
|
3583
|
+
correlatedRef.current = null;
|
|
3584
|
+
void player.dispose();
|
|
3585
|
+
setAudioPlayback?.("unknown");
|
|
3586
|
+
};
|
|
3587
|
+
}, [room, assistant.agent, correlatedEnabled, correlatedSessionId, roomMatchesGrant, setAudioPlayback]);
|
|
3041
3588
|
useCallMedia(room, {
|
|
3042
3589
|
setMicrophoneFacts,
|
|
3043
|
-
setAudioPlayback,
|
|
3590
|
+
setAudioPlayback: correlatedEnabled ? void 0 : setAudioPlayback,
|
|
3591
|
+
startAudioOverride: correlatedEnabled ? startCorrelatedAudio : void 0,
|
|
3044
3592
|
registerMediaControls,
|
|
3045
3593
|
microphoneWanted: microphone ?? publishesMicrophoneByDefault(lifecycle.grant),
|
|
3046
3594
|
connectionState,
|
|
@@ -3054,7 +3602,7 @@ function SessionLifecycleRoomBridge({
|
|
|
3054
3602
|
if (!mediaModeEnabled) return;
|
|
3055
3603
|
const agent = assistant.agent;
|
|
3056
3604
|
let previous;
|
|
3057
|
-
const
|
|
3605
|
+
const deliver2 = (mode) => {
|
|
3058
3606
|
if (mode === previous) return;
|
|
3059
3607
|
previous = mode;
|
|
3060
3608
|
try {
|
|
@@ -3065,7 +3613,7 @@ function SessionLifecycleRoomBridge({
|
|
|
3065
3613
|
};
|
|
3066
3614
|
const read = () => {
|
|
3067
3615
|
const value = agent?.attributes?.["rta.media_mode"];
|
|
3068
|
-
|
|
3616
|
+
deliver2(value === "video" ? "avatar" : value === "voice" ? "voice" : null);
|
|
3069
3617
|
};
|
|
3070
3618
|
const changed = (_attributes, participant) => {
|
|
3071
3619
|
if (participant === agent) read();
|
|
@@ -3074,7 +3622,7 @@ function SessionLifecycleRoomBridge({
|
|
|
3074
3622
|
read();
|
|
3075
3623
|
return () => {
|
|
3076
3624
|
room.off(RoomEvent.ParticipantAttributesChanged, changed);
|
|
3077
|
-
|
|
3625
|
+
deliver2(null);
|
|
3078
3626
|
};
|
|
3079
3627
|
}, [room, assistant.agent, mediaModeEnabled, mediaModeSessionId]);
|
|
3080
3628
|
const { send } = useChat();
|
|
@@ -3121,7 +3669,7 @@ function SessionLifecycleRoomBridge({
|
|
|
3121
3669
|
let closed = false;
|
|
3122
3670
|
let pending = false;
|
|
3123
3671
|
let previous;
|
|
3124
|
-
const
|
|
3672
|
+
const deliver2 = (details) => {
|
|
3125
3673
|
connectionHistoryRef.current?.enqueue(details);
|
|
3126
3674
|
try {
|
|
3127
3675
|
if (onConnectionDetailsChange) void Promise.resolve(callback?.(details)).catch(() => {
|
|
@@ -3148,7 +3696,7 @@ function SessionLifecycleRoomBridge({
|
|
|
3148
3696
|
const last = previous;
|
|
3149
3697
|
if (last?.connectionState === next.connectionState && last.localQuality === next.localQuality && ["audio", "video"].every((kind) => last[kind]?.publisherQuality === next[kind]?.publisherQuality && last[kind]?.streamState === next[kind]?.streamState)) return;
|
|
3150
3698
|
previous = next;
|
|
3151
|
-
|
|
3699
|
+
deliver2(next);
|
|
3152
3700
|
});
|
|
3153
3701
|
};
|
|
3154
3702
|
const onQuality = (_quality, participant) => {
|
|
@@ -3178,7 +3726,7 @@ function SessionLifecycleRoomBridge({
|
|
|
3178
3726
|
room.off(RoomEvent.TrackStreamStateChanged, onPublication);
|
|
3179
3727
|
room.off(RoomEvent.TrackSubscribed, onTrack);
|
|
3180
3728
|
room.off(RoomEvent.TrackUnsubscribed, onTrack);
|
|
3181
|
-
|
|
3729
|
+
deliver2(null);
|
|
3182
3730
|
};
|
|
3183
3731
|
}, [room, detailsSessionId, detailsEnabled]);
|
|
3184
3732
|
useEffect(() => {
|
|
@@ -3215,18 +3763,56 @@ function SessionLifecycleRoomBridge({
|
|
|
3215
3763
|
useEffect(() => {
|
|
3216
3764
|
if (!onLifecycleData) return;
|
|
3217
3765
|
const decoder = new TextDecoder();
|
|
3218
|
-
const handler = (payload,
|
|
3766
|
+
const handler = (payload, participant, _kind, topic) => {
|
|
3219
3767
|
if (topic !== RTA_LIFECYCLE_TOPIC) return;
|
|
3220
3768
|
try {
|
|
3221
|
-
|
|
3769
|
+
const frame = JSON.parse(decoder.decode(payload));
|
|
3770
|
+
if (typeof frame === "object" && frame !== null && "kind" in frame && (frame.kind === EXTERNAL_PERFORMANCE_KIND || frame.kind === "ended") && (!assistant.agent || participant !== assistant.agent || !roomMatchesGrant || room.state !== "connected")) return;
|
|
3771
|
+
if (typeof frame === "object" && frame !== null && "kind" in frame && frame.kind === "ended" && lifecycleServerFrameSchema.safeParse(frame).success) {
|
|
3772
|
+
void correlatedRef.current?.dispose();
|
|
3773
|
+
correlatedRef.current = null;
|
|
3774
|
+
}
|
|
3775
|
+
onLifecycleData(frame);
|
|
3222
3776
|
} catch {
|
|
3223
3777
|
}
|
|
3224
3778
|
};
|
|
3779
|
+
const invalidate = () => lifecycle.invalidateExternalPerformance?.("connection_lost");
|
|
3780
|
+
const changed = () => {
|
|
3781
|
+
if (room.state === "disconnected" || room.state === "reconnecting" || room.state === "signalReconnecting") invalidate();
|
|
3782
|
+
};
|
|
3225
3783
|
room.on(RoomEvent.DataReceived, handler);
|
|
3784
|
+
if (lifecycle.invalidateExternalPerformance) {
|
|
3785
|
+
room.on(RoomEvent.Reconnecting, invalidate);
|
|
3786
|
+
room.on(RoomEvent.SignalReconnecting, invalidate);
|
|
3787
|
+
room.on(RoomEvent.Disconnected, invalidate);
|
|
3788
|
+
room.on(RoomEvent.ConnectionStateChanged, changed);
|
|
3789
|
+
}
|
|
3226
3790
|
return () => {
|
|
3227
3791
|
room.off(RoomEvent.DataReceived, handler);
|
|
3792
|
+
room.off(RoomEvent.Reconnecting, invalidate);
|
|
3793
|
+
room.off(RoomEvent.SignalReconnecting, invalidate);
|
|
3794
|
+
room.off(RoomEvent.Disconnected, invalidate);
|
|
3795
|
+
room.off(RoomEvent.ConnectionStateChanged, changed);
|
|
3796
|
+
};
|
|
3797
|
+
}, [room, onLifecycleData, assistant.agent, roomMatchesGrant, lifecycle.invalidateExternalPerformance]);
|
|
3798
|
+
useEffect(() => {
|
|
3799
|
+
if (!assistant.agent) return;
|
|
3800
|
+
return () => lifecycle.invalidateExternalPerformance?.("connection_lost");
|
|
3801
|
+
}, [assistant.agent, lifecycle.invalidateExternalPerformance]);
|
|
3802
|
+
const resumeExternalPerformance = lifecycle.resumeExternalPerformance;
|
|
3803
|
+
useEffect(() => {
|
|
3804
|
+
if (!resumeExternalPerformance) return;
|
|
3805
|
+
const resumeIfBound = () => {
|
|
3806
|
+
if (assistant.agent && roomMatchesGrant && room.state === "connected") resumeExternalPerformance();
|
|
3807
|
+
};
|
|
3808
|
+
resumeIfBound();
|
|
3809
|
+
room.on(RoomEvent.Reconnected, resumeIfBound);
|
|
3810
|
+
room.on(RoomEvent.ConnectionStateChanged, resumeIfBound);
|
|
3811
|
+
return () => {
|
|
3812
|
+
room.off(RoomEvent.Reconnected, resumeIfBound);
|
|
3813
|
+
room.off(RoomEvent.ConnectionStateChanged, resumeIfBound);
|
|
3228
3814
|
};
|
|
3229
|
-
}, [room,
|
|
3815
|
+
}, [room, assistant.agent, roomMatchesGrant, resumeExternalPerformance]);
|
|
3230
3816
|
useEffect(() => {
|
|
3231
3817
|
if (!registerDataPublisher) return;
|
|
3232
3818
|
const encoder = new TextEncoder();
|
|
@@ -3362,6 +3948,7 @@ function useRealtimeSession(input) {
|
|
|
3362
3948
|
onBehaviorChange,
|
|
3363
3949
|
onClipResult,
|
|
3364
3950
|
onVoiceError,
|
|
3951
|
+
correlatedAudio,
|
|
3365
3952
|
// everything else (client/active/idleSeconds/idleWarnLeadSeconds/reconnect/requestOptions…)
|
|
3366
3953
|
// flows straight into the inner SSOT hook.
|
|
3367
3954
|
...inner
|
|
@@ -3434,6 +4021,7 @@ function useRealtimeSession(input) {
|
|
|
3434
4021
|
return `t-${Date.now()}-${++turnSequenceRef.current}`;
|
|
3435
4022
|
}, []);
|
|
3436
4023
|
const onLifecycleData = useCallback((frame) => {
|
|
4024
|
+
lifecycle.onLifecycleData?.(frame);
|
|
3437
4025
|
const parsed = lifecycleServerFrameSchema.safeParse(frame);
|
|
3438
4026
|
if (!parsed.success) return;
|
|
3439
4027
|
const f = parsed.data;
|
|
@@ -3493,7 +4081,7 @@ function useRealtimeSession(input) {
|
|
|
3493
4081
|
break;
|
|
3494
4082
|
}
|
|
3495
4083
|
}
|
|
3496
|
-
}, []);
|
|
4084
|
+
}, [lifecycle.onLifecycleData]);
|
|
3497
4085
|
const registerDataPublisher = useCallback((publish) => {
|
|
3498
4086
|
dataPublisherRef.current = publish;
|
|
3499
4087
|
}, []);
|
|
@@ -3759,6 +4347,7 @@ function useRealtimeSession(input) {
|
|
|
3759
4347
|
// stayConnected/reconnect/markActivity/the sinks) → a true drop-in superset, then
|
|
3760
4348
|
// add the realtime-session surface. `reset` + `lifecycle` come AFTER so ours win.
|
|
3761
4349
|
...lifecycle,
|
|
4350
|
+
correlatedAudio,
|
|
3762
4351
|
turn,
|
|
3763
4352
|
clocks,
|
|
3764
4353
|
endsAt,
|
|
@@ -3793,6 +4382,7 @@ function useRealtimeSession(input) {
|
|
|
3793
4382
|
}),
|
|
3794
4383
|
[
|
|
3795
4384
|
lifecycle,
|
|
4385
|
+
correlatedAudio,
|
|
3796
4386
|
turn,
|
|
3797
4387
|
clocks,
|
|
3798
4388
|
endsAt,
|
|
@@ -3932,4 +4522,4 @@ function createProxyClient(options) {
|
|
|
3932
4522
|
};
|
|
3933
4523
|
}
|
|
3934
4524
|
|
|
3935
|
-
export { AdaptivePlayoutController, AvatarVideoSurface, DEFAULT_APPROACHING_END_LEAD_SECONDS, DEFAULT_AVATAR_PLAYOUT_DELAY_SECONDS, DEFAULT_CREDITS_LOW_LEAD_SECONDS, DEFAULT_FISH_TTS_MODEL, DEFAULT_GOVERNOR_CONFIG, DEFAULT_GRACE_CEILING_SECONDS, DEFAULT_GRACE_WINDOW_LEAD_SECONDS, DEFAULT_IDLE_SECONDS, DEFAULT_IDLE_WARN_LEAD_SECONDS, DEFAULT_READY_TIMEOUT_SECONDS, DEFAULT_TURN_TIMEOUT_SECONDS, MAX_RECONNECT_ATTEMPTS, MAX_SESSION_INSTRUCTIONS_CHARS, RECONNECT_BACKOFF_MS, RealtimeAvatarLiveKitRoom, SessionLifecycleRoomBridge, capacityErrorFromBusy, capacityStateFromGrant, createProxyClient, fishTtsModelSchema, isNativeLiveTrackSubscribed, knownBehaviorStates, mapTurnState, readInboundRtp, sessionBehaviorSchema, sessionClipSchema, splitCallTranscript, useAvatarAdaptivePlayoutDelay, useAvatarCamera, useAvatarPlayoutDelay, useAvatarQualityGovernor, useCallTranscript, useClearStaleInputMute, useLiveKitAvatarGrant, useLiveTrackProducing, useMicLease, useRealtimeAvatarAudioSession, useRealtimeSession, useReleaseMicLeaseOnTrackEnded, useSessionLifecycle };
|
|
4525
|
+
export { AdaptivePlayoutController, AvatarVideoSurface, DEFAULT_APPROACHING_END_LEAD_SECONDS, DEFAULT_AVATAR_PLAYOUT_DELAY_SECONDS, DEFAULT_CREDITS_LOW_LEAD_SECONDS, DEFAULT_FISH_TTS_MODEL, DEFAULT_GOVERNOR_CONFIG, DEFAULT_GRACE_CEILING_SECONDS, DEFAULT_GRACE_WINDOW_LEAD_SECONDS, DEFAULT_IDLE_SECONDS, DEFAULT_IDLE_WARN_LEAD_SECONDS, DEFAULT_READY_TIMEOUT_SECONDS, DEFAULT_TURN_TIMEOUT_SECONDS, EXTERNAL_PERFORMANCE_TIMING, FISH_DELIVERY_MODELS, KNOWN_EXTERNAL_PERFORMANCE_STATUSES, MAX_RECONNECT_ATTEMPTS, MAX_SESSION_INSTRUCTIONS_CHARS, RECONNECT_BACKOFF_MS, RealtimeAvatarLiveKitRoom, SessionLifecycleRoomBridge, capacityErrorFromBusy, capacityStateFromGrant, createProxyClient, fishTtsModelSchema, isNativeLiveTrackSubscribed, knownBehaviorStates, mapTurnState, readInboundRtp, sessionBehaviorSchema, sessionClipSchema, splitCallTranscript, useAvatarAdaptivePlayoutDelay, useAvatarCamera, useAvatarPlayoutDelay, useAvatarQualityGovernor, useCallTranscript, useClearStaleInputMute, useLiveKitAvatarGrant, useLiveTrackProducing, useMicLease, useRealtimeAvatarAudioSession, useRealtimeSession, useReleaseMicLeaseOnTrackEnded, useSessionLifecycle };
|