realtime-avatar 0.33.0 → 0.35.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +56 -1
- package/dist/browser.d.ts +4 -89
- package/dist/browser.js +362 -1
- package/dist/chunk-6KQS7CTR.js +1 -0
- package/dist/{chunk-DM5TWRC4.js → chunk-ARJEAQQT.js} +1 -13
- package/dist/{chunk-VLYUMP3I.js → chunk-WHYDSORD.js} +426 -170
- package/dist/express.d.ts +2 -2
- package/dist/express.js +2 -2
- package/dist/hono.d.ts +2 -2
- package/dist/hono.js +2 -2
- package/dist/index.d.ts +338 -3
- package/dist/index.js +2 -1
- package/dist/nextjs.d.ts +2 -2
- package/dist/nextjs.js +2 -2
- package/dist/{playout-delay-CtRZdtsj.d.ts → playout-delay-BzoM1BMN.d.ts} +135 -1
- package/dist/{proxy-client-DJhYiiKX.d.ts → proxy-client-CiTOjkg8.d.ts} +67 -3
- package/dist/react-native.d.ts +3 -3
- package/dist/react-native.js +623 -36
- package/dist/react.d.ts +12 -22
- package/dist/react.js +627 -37
- package/dist/server.d.ts +2 -2
- package/dist/server.js +2 -1
- package/dist/tanstack-start.d.ts +2 -2
- package/dist/tanstack-start.js +2 -2
- package/dist/{types-DygYBvSv.d.ts → types-CUvcV0Wq.d.ts} +1 -1
- package/dist/{types-VoejsHij.d.ts → types-CpnLuiJU.d.ts} +220 -18
- package/package.json +3 -3
package/README.md
CHANGED
|
@@ -367,6 +367,27 @@ The legacy `lk-chat-topic` compatibility path omits attributes. Automatic RTA se
|
|
|
367
367
|
STT attribution and final transcript webhook provenance are separate work. Existing
|
|
368
368
|
webhook types and behavior are unchanged.
|
|
369
369
|
|
|
370
|
+
## Correlated browser audio (opt-in)
|
|
371
|
+
|
|
372
|
+
For an application-owned character that needs a sentence clock, set server
|
|
373
|
+
`llm.audioOutput: "correlated_pcm_v1"` with `performance: "v1"`, `failClosed: true`,
|
|
374
|
+
`mode: "voice"`, and `voiceWebhook`; do not also set `voice`. The browser uses
|
|
375
|
+
`attachCorrelatedAudio` from `realtime-avatar/browser`, or passes `correlatedAudio:
|
|
376
|
+
{ onPosition, onInvalidated, onPlaybackBlocked? }` to `useRealtimeSession`/`AvatarCall`.
|
|
377
|
+
The existing room bridge binds to the held grant and actual assistant. `AvatarCall`
|
|
378
|
+
disables ordinary audio automatically; composed rooms must use `renderRoomAudio={false}`
|
|
379
|
+
and must not mount `RoomAudioRenderer` or `attachRemoteAudio`.
|
|
380
|
+
|
|
381
|
+
The attachment returns `startAudio(): Promise<void>` and `dispose(): Promise<void>`.
|
|
382
|
+
Position callbacks contain header-derived `session_id`, `generation_id`, `utterance_id`,
|
|
383
|
+
`stream_id`, `epoch`, consumed integer `samples`, `sampleRate`, and `state:
|
|
384
|
+
"playing" | "finished"`. They report source PCM output-clock progress, not physical
|
|
385
|
+
audibility. No PCM or analysis is exposed. The player bounds buffering to two seconds,
|
|
386
|
+
returns consumed-sample credit, rejects unbound senders and stale epochs, and clears
|
|
387
|
+
pending playback on cancellation. Reconnection/agent loss fails closed; obtain a fresh
|
|
388
|
+
grant rather than replaying old audio. Default calls are unchanged. This requires the
|
|
389
|
+
corresponding platform and worker release; see the root README for the full integration.
|
|
390
|
+
|
|
370
391
|
## API
|
|
371
392
|
|
|
372
393
|
Everything is on one class. The full types are in
|
|
@@ -411,7 +432,8 @@ rta.iterateRenders({ limit? }) // every render, paging handled
|
|
|
411
432
|
rta.getRenderAccess(renderId) // { id, url, expiresAt }; signed for an hour, renew when needed
|
|
412
433
|
|
|
413
434
|
// avatars
|
|
414
|
-
rta.
|
|
435
|
+
rta.createLive2dAvatar({ displayName, voice, settings?, metadata? }) // app-owned rig, explicit voice, no assets
|
|
436
|
+
rta.createAvatarFromImage({ displayName, imageUrl, motionPrompt?, voice? }) // platform-rendered video
|
|
415
437
|
rta.createAvatarFromVideo({ displayName, videoUrl, voice? }) // DEPRECATED — closed, 422
|
|
416
438
|
rta.listAvatars()
|
|
417
439
|
rta.getAvatar(avatarId)
|
|
@@ -462,6 +484,39 @@ const voice: VoiceSpecInput = {
|
|
|
462
484
|
};
|
|
463
485
|
```
|
|
464
486
|
|
|
487
|
+
### Register an app-owned Live2D rig
|
|
488
|
+
|
|
489
|
+
```ts
|
|
490
|
+
const avatar = await rta.createLive2dAvatar({
|
|
491
|
+
displayName: "Rin",
|
|
492
|
+
voice: { provider: "fish", voice_id: "your-provider-voice-id", model: "s2-pro" },
|
|
493
|
+
metadata: { characterId: "your-character-id" },
|
|
494
|
+
});
|
|
495
|
+
// Persist avatar.id and avatar.renderer in your server-owned character mapping.
|
|
496
|
+
const call = await rta.startCall({ avatarId: avatar.id, mode: "voice" });
|
|
497
|
+
```
|
|
498
|
+
|
|
499
|
+
Requires the platform's Live2D identity support. One `POST /avatars` sends `renderer: "live2d"`,
|
|
500
|
+
`sourceKind: "image"` and an explicit voice selector; no assets, automatic casting, clips or
|
|
501
|
+
resting-loop generation are requested. Your app owns the rig. Source fields, motion prompts,
|
|
502
|
+
renderer overrides and automatic voice descriptions are refused before network I/O.
|
|
503
|
+
`settings` and `metadata` are optional records and are omitted unless provided.
|
|
504
|
+
|
|
505
|
+
The exported `CreateLive2dAvatarInput` and complete `RegisteredAvatar` response derive from
|
|
506
|
+
canonical generated schemas. The response is parsed once; malformed successful responses throw
|
|
507
|
+
`RealtimeAvatarError` without the rejected body. Existing image/video creation requests are
|
|
508
|
+
unchanged. Existing avatar methods retain `renderer`, defaulting older responses that omit it to
|
|
509
|
+
`"video"`.
|
|
510
|
+
|
|
511
|
+
### Validate persisted grants and proxy requests
|
|
512
|
+
|
|
513
|
+
The server root (`realtime-avatar` or `realtime-avatar/server`) exports the existing
|
|
514
|
+
`liveKitSessionGrantSchema`, `capacityBusyResponseSchema`, `proxyConnectRequestSchema` and
|
|
515
|
+
`proxyEndRequestSchema`. Use their `safeParse` at your unknown storage or HTTP boundary rather
|
|
516
|
+
than writing mirror schemas. Connect parsing strips client policy; end parsing requires a
|
|
517
|
+
session or queue ticket. Parsing does not authorize access: bind avatars, grants and sessions
|
|
518
|
+
to the authenticated user on your server.
|
|
519
|
+
|
|
465
520
|
### Session page validation
|
|
466
521
|
|
|
467
522
|
`listSessions` parses the complete response using the generated platform contract and
|
package/dist/browser.d.ts
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
|
-
export { D as DEFAULT_AVATAR_PLAYOUT_DELAY_SECONDS, E as EnableMicrophoneOptions, M as MicrophoneCapableRoom,
|
|
1
|
+
export { A as AttachCorrelatedAudioOptions, a as AttachRemoteAudioOptions, b as AttachableTrack, c as AudioCapableRoom, C as CorrelatedAudioAttachment, d as CorrelatedAudioInvalidation, e as CorrelatedAudioObserver, f as CorrelatedAudioPosition, D as DEFAULT_AVATAR_PLAYOUT_DELAY_SECONDS, E as EnableMicrophoneOptions, M as MicrophoneCapableRoom, g as MicrophoneFailureReason, h as MicrophoneProblem, i as MicrophoneProblemReason, j as MicrophoneResult, P as PlayoutDelayTarget, R as RemoteAudioAttachment, k as applyAvatarPlayoutDelay, l as applyPlayoutDelay, m as attachCorrelatedAudio, n as attachRemoteAudio, o as describeMicrophoneFailure, p as enableMicrophone } from './playout-delay-BzoM1BMN.js';
|
|
2
|
+
import 'livekit-client';
|
|
3
|
+
import 'zod';
|
|
2
4
|
|
|
3
5
|
type VideoSupport = {
|
|
4
6
|
supported: true;
|
|
@@ -10,93 +12,6 @@ type VideoSupport = {
|
|
|
10
12
|
/** Check before starting a billed video session; this does not prove negotiated playback. */
|
|
11
13
|
declare function checkVideoSupport(codec?: string): VideoSupport;
|
|
12
14
|
|
|
13
|
-
/**
|
|
14
|
-
* Making the character AUDIBLE — the half of "audio works" that has nothing to do with the
|
|
15
|
-
* microphone.
|
|
16
|
-
*
|
|
17
|
-
* WHY THIS EXISTS. Two failures here look identical to a developer ("she never speaks") and
|
|
18
|
-
* neither raises anything:
|
|
19
|
-
*
|
|
20
|
-
* 1. The audio element is created but never inserted into the document. `track.attach()`
|
|
21
|
-
* hands back a detached `<audio>`; a detached element is not reliably played by every
|
|
22
|
-
* engine, and there is no node on the page to fall back to. Nothing errors — the call is
|
|
23
|
-
* connected, the track is subscribed, the meter moves, and it is silent.
|
|
24
|
-
* 2. Autoplay is blocked. A page that has not yet had a user gesture may not start audio,
|
|
25
|
-
* so the first call a visitor makes is mute. `room.startAudio()` fixes it, but it must be
|
|
26
|
-
* called FROM a gesture — which means the page needs a button, which means the page has
|
|
27
|
-
* to know it is blocked. That signal is an event nobody subscribes to.
|
|
28
|
-
*
|
|
29
|
-
* So this owns both: it attaches into the DOM, and it tells you when a gesture is required
|
|
30
|
-
* and hands you the closure that spends it.
|
|
31
|
-
*
|
|
32
|
-
* Structural like its sibling — no `livekit-client` import, no version pin.
|
|
33
|
-
*/
|
|
34
|
-
/** The sliver of a livekit-client `Track` this needs. */
|
|
35
|
-
interface AttachableTrack {
|
|
36
|
-
kind: string;
|
|
37
|
-
attach(): HTMLMediaElement;
|
|
38
|
-
detach(): HTMLMediaElement[];
|
|
39
|
-
}
|
|
40
|
-
/** The sliver of a livekit-client `Room` this needs. */
|
|
41
|
-
interface AudioCapableRoom {
|
|
42
|
-
canPlaybackAudio: boolean;
|
|
43
|
-
startAudio(): Promise<void>;
|
|
44
|
-
on(event: string, listener: (...args: never[]) => void): unknown;
|
|
45
|
-
off(event: string, listener: (...args: never[]) => void): unknown;
|
|
46
|
-
}
|
|
47
|
-
interface AttachRemoteAudioOptions {
|
|
48
|
-
/**
|
|
49
|
-
* Receiver playout cushion applied to EVERY remote track the room subscribes (video
|
|
50
|
-
* included, so the pair stays lip-locked) via `prepareAvatarRoom`. Default 0.5s — the
|
|
51
|
-
* measured freeze-free depth. `false` leaves the room untouched, for a page that owns
|
|
52
|
-
* its own buffering. Not undone by `detach()`: a shallower buffer mid-call only brings
|
|
53
|
-
* the freezes back.
|
|
54
|
-
*/
|
|
55
|
-
playoutDelaySeconds?: number | false;
|
|
56
|
-
/**
|
|
57
|
-
* Where to park the audio elements. Defaults to `document.body`.
|
|
58
|
-
*
|
|
59
|
-
* They are not `display:none` — a hidden media element is still subject to the same
|
|
60
|
-
* autoplay rules, and hiding it only removes the browser's own affordance.
|
|
61
|
-
*/
|
|
62
|
-
container?: HTMLElement;
|
|
63
|
-
/**
|
|
64
|
-
* Called when the browser refuses to start audio without a gesture, and again with `null`
|
|
65
|
-
* once audio is playing. Render a button from it and call `unblock` in the click handler:
|
|
66
|
-
*
|
|
67
|
-
* ```ts
|
|
68
|
-
* onPlaybackBlocked: (unblock) => {
|
|
69
|
-
* button.hidden = unblock === null;
|
|
70
|
-
* button.onclick = () => unblock?.();
|
|
71
|
-
* }
|
|
72
|
-
* ```
|
|
73
|
-
*/
|
|
74
|
-
onPlaybackBlocked?: (unblock: (() => Promise<void>) | null) => void;
|
|
75
|
-
}
|
|
76
|
-
interface RemoteAudioAttachment {
|
|
77
|
-
/** Stop listening and remove every element this created. Safe to call twice. */
|
|
78
|
-
detach(): void;
|
|
79
|
-
}
|
|
80
|
-
/**
|
|
81
|
-
* Play every remote audio track the room subscribes to, and surface an autoplay block.
|
|
82
|
-
*
|
|
83
|
-
* Call it BEFORE `room.connect()` — a track subscribed during connect is missed otherwise,
|
|
84
|
-
* and that race is the version of this bug that only reproduces on a fast connection.
|
|
85
|
-
*
|
|
86
|
-
* ```ts
|
|
87
|
-
* const audio = attachRemoteAudio(room, {
|
|
88
|
-
* onPlaybackBlocked: (unblock) => {
|
|
89
|
-
* enableSound.hidden = unblock === null;
|
|
90
|
-
* enableSound.onclick = () => unblock?.();
|
|
91
|
-
* },
|
|
92
|
-
* });
|
|
93
|
-
* await room.connect(url, token);
|
|
94
|
-
* // …later
|
|
95
|
-
* audio.detach();
|
|
96
|
-
* ```
|
|
97
|
-
*/
|
|
98
|
-
declare function attachRemoteAudio(room: AudioCapableRoom, options?: AttachRemoteAudioOptions): RemoteAudioAttachment;
|
|
99
|
-
|
|
100
15
|
/** The sliver of a livekit-client `Room` this needs. */
|
|
101
16
|
interface PreparableRoom {
|
|
102
17
|
on(event: string, listener: (...args: never[]) => void): unknown;
|
|
@@ -166,4 +81,4 @@ type BoomerangPlayback = {
|
|
|
166
81
|
*/
|
|
167
82
|
declare function attachBoomerangPlayback(video: HTMLVideoElement, options?: BoomerangPlaybackOptions): BoomerangPlayback;
|
|
168
83
|
|
|
169
|
-
export { type
|
|
84
|
+
export { type BoomerangPlayback, type BoomerangPlaybackOptions, type PreparableParticipant, type PreparableRoom, type PrepareAvatarRoomOptions, type PreparedAvatarRoom, type VideoSupport, attachBoomerangPlayback, checkVideoSupport, prepareAvatarRoom };
|
package/dist/browser.js
CHANGED
|
@@ -1,3 +1,364 @@
|
|
|
1
|
+
import { RoomEvent } from 'livekit-client';
|
|
2
|
+
import { z } from 'zod';
|
|
3
|
+
|
|
4
|
+
// ../browser/src/correlated-audio.ts
|
|
5
|
+
var schema0 = z.string().min(1).max(200);
|
|
6
|
+
var schema1 = z.literal(1);
|
|
7
|
+
var schema2 = z.string().min(1).max(200);
|
|
8
|
+
var schema3 = z.int().min(1).max(9007199254740991);
|
|
9
|
+
var schema4 = z.literal(24e3);
|
|
10
|
+
var schema5 = z.literal("pcm_s16le");
|
|
11
|
+
var schema6 = z.object({
|
|
12
|
+
"session_id": schema0,
|
|
13
|
+
"generation_id": schema0,
|
|
14
|
+
"version": schema1,
|
|
15
|
+
"utterance_id": schema2,
|
|
16
|
+
"stream_id": schema0,
|
|
17
|
+
"epoch": schema3,
|
|
18
|
+
"sample_rate": schema4,
|
|
19
|
+
"channels": schema1,
|
|
20
|
+
"encoding": schema5
|
|
21
|
+
});
|
|
22
|
+
var schema7 = z.enum(["started", "progress", "finished", "cancelled"]);
|
|
23
|
+
var schema8 = z.int().min(0).max(144e4);
|
|
24
|
+
z.strictObject({
|
|
25
|
+
"session_id": schema0,
|
|
26
|
+
"generation_id": schema0,
|
|
27
|
+
"utterance_id": schema2,
|
|
28
|
+
"stream_id": schema0,
|
|
29
|
+
"epoch": schema3,
|
|
30
|
+
"status": schema7,
|
|
31
|
+
"samples": schema8
|
|
32
|
+
});
|
|
33
|
+
var schema10 = z.object({
|
|
34
|
+
"version": schema1,
|
|
35
|
+
"session_id": schema0,
|
|
36
|
+
"epoch": schema3
|
|
37
|
+
});
|
|
38
|
+
var schema11 = z.literal("external_llm_error");
|
|
39
|
+
var schema12 = z.enum(["external_llm_refused", "external_llm_failed"]);
|
|
40
|
+
var schema13 = z.string();
|
|
41
|
+
var schema14 = z.literal(true);
|
|
42
|
+
z.object({
|
|
43
|
+
"kind": schema11,
|
|
44
|
+
"code": schema12,
|
|
45
|
+
"message": schema13,
|
|
46
|
+
"fatal": schema14
|
|
47
|
+
});
|
|
48
|
+
var correlatedAudioHeaderSchema = schema6;
|
|
49
|
+
var correlatedAudioCancelSchema = schema10;
|
|
50
|
+
var CORRELATED_AUDIO_CONTRACT = {
|
|
51
|
+
"topic": "rta.audio.v1",
|
|
52
|
+
"receiptMethod": "rta.audio.receipt.v1",
|
|
53
|
+
"cancelMethod": "rta.audio.cancel.v1",
|
|
54
|
+
"maxSamples": 144e4,
|
|
55
|
+
"maxChunkBytes": 48e3,
|
|
56
|
+
"maxBufferedSamples": 48e3
|
|
57
|
+
};
|
|
58
|
+
|
|
59
|
+
// ../browser/src/correlated-audio.ts
|
|
60
|
+
var attachments = /* @__PURE__ */ new WeakSet();
|
|
61
|
+
var bindings = /* @__PURE__ */ new WeakMap();
|
|
62
|
+
function notify(callback) {
|
|
63
|
+
try {
|
|
64
|
+
void Promise.resolve(callback()).catch(() => {
|
|
65
|
+
});
|
|
66
|
+
} catch {
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
function attachCorrelatedAudio(room, options) {
|
|
70
|
+
if (attachments.has(room)) throw new Error("Correlated audio is already attached to this room");
|
|
71
|
+
if (!options.sessionId || !options.agentIdentity) throw new Error("Correlated audio requires the held session and bound agent");
|
|
72
|
+
attachments.add(room);
|
|
73
|
+
let context;
|
|
74
|
+
let disposed = false;
|
|
75
|
+
const previous = bindings.get(room);
|
|
76
|
+
const binding = previous?.sessionId === options.sessionId ? previous : { sessionId: options.sessionId, epoch: 1, unavailable: false, seen: /* @__PURE__ */ new Set() };
|
|
77
|
+
bindings.set(room, binding);
|
|
78
|
+
const sources = [];
|
|
79
|
+
const active = (source) => sources.includes(source);
|
|
80
|
+
const buffered = () => sources.reduce((sum, source) => sum + source.received - source.consumed, 0);
|
|
81
|
+
let nextStart = 0;
|
|
82
|
+
let blocked;
|
|
83
|
+
let closing;
|
|
84
|
+
const seen = binding.seen;
|
|
85
|
+
const waiters = /* @__PURE__ */ new Set();
|
|
86
|
+
const wake = () => {
|
|
87
|
+
for (const resolve of waiters) resolve();
|
|
88
|
+
waiters.clear();
|
|
89
|
+
};
|
|
90
|
+
const identity = (header) => ({
|
|
91
|
+
session_id: header.session_id,
|
|
92
|
+
generation_id: header.generation_id,
|
|
93
|
+
utterance_id: header.utterance_id,
|
|
94
|
+
stream_id: header.stream_id,
|
|
95
|
+
epoch: header.epoch
|
|
96
|
+
});
|
|
97
|
+
const position = (current, state) => {
|
|
98
|
+
notify(() => options.onPosition({ ...identity(current.header), samples: current.consumed, sampleRate: current.header.sample_rate, state }));
|
|
99
|
+
};
|
|
100
|
+
const receipt = (current, status) => {
|
|
101
|
+
const payload = { ...identity(current.header), status, samples: current.consumed };
|
|
102
|
+
if (current.rpcBusy) {
|
|
103
|
+
current.pendingReceipt = payload;
|
|
104
|
+
return;
|
|
105
|
+
}
|
|
106
|
+
const send = async (next) => {
|
|
107
|
+
current.rpcBusy = true;
|
|
108
|
+
try {
|
|
109
|
+
await room.localParticipant.performRpc({
|
|
110
|
+
destinationIdentity: options.agentIdentity,
|
|
111
|
+
method: CORRELATED_AUDIO_CONTRACT.receiptMethod,
|
|
112
|
+
payload: JSON.stringify(next),
|
|
113
|
+
responseTimeout: 2e3
|
|
114
|
+
});
|
|
115
|
+
} catch {
|
|
116
|
+
if (!disposed && next.epoch === binding.epoch) invalidate("connection_lost");
|
|
117
|
+
} finally {
|
|
118
|
+
current.rpcBusy = false;
|
|
119
|
+
const pending = current.pendingReceipt;
|
|
120
|
+
current.pendingReceipt = void 0;
|
|
121
|
+
if (pending && !disposed && pending.epoch === binding.epoch && !binding.unavailable) void send(pending);
|
|
122
|
+
}
|
|
123
|
+
};
|
|
124
|
+
void send(payload);
|
|
125
|
+
};
|
|
126
|
+
const clear = () => {
|
|
127
|
+
for (const current of sources.splice(0)) {
|
|
128
|
+
current.abort.abort();
|
|
129
|
+
for (const segment of current.segments) {
|
|
130
|
+
segment.node.onended = null;
|
|
131
|
+
segment.node.stop();
|
|
132
|
+
segment.node.disconnect();
|
|
133
|
+
}
|
|
134
|
+
current.segments.length = 0;
|
|
135
|
+
}
|
|
136
|
+
nextStart = context?.currentTime ?? 0;
|
|
137
|
+
wake();
|
|
138
|
+
};
|
|
139
|
+
const invalidate = (reason) => {
|
|
140
|
+
if (disposed) return;
|
|
141
|
+
if (reason === "connection_lost") {
|
|
142
|
+
if (binding.unavailable) return;
|
|
143
|
+
binding.unavailable = true;
|
|
144
|
+
}
|
|
145
|
+
if (reason === "invalid_stream" || reason === "disposed") for (const current of sources) receipt(current, "cancelled");
|
|
146
|
+
clear();
|
|
147
|
+
notify(() => options.onInvalidated(reason));
|
|
148
|
+
};
|
|
149
|
+
const tick = (reportCredit = true) => {
|
|
150
|
+
if (!context || context.state !== "running" || disposed || binding.unavailable) return;
|
|
151
|
+
for (const current of [...sources]) {
|
|
152
|
+
if (!active(current)) continue;
|
|
153
|
+
let consumed = current.consumed;
|
|
154
|
+
for (const segment of current.segments) {
|
|
155
|
+
if (context.currentTime < segment.start) break;
|
|
156
|
+
const samples = Math.min(segment.length, Math.max(0, Math.floor((context.currentTime - segment.start) * current.header.sample_rate + 1e-6)));
|
|
157
|
+
consumed = Math.max(consumed, segment.offset + samples);
|
|
158
|
+
if (!current.started && samples > 0) {
|
|
159
|
+
current.started = true;
|
|
160
|
+
receipt(current, "started");
|
|
161
|
+
position(current, "playing");
|
|
162
|
+
current.lastPosition = current.consumed;
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
current.consumed = consumed;
|
|
166
|
+
while (current.segments.length && current.segments[0].offset + current.segments[0].length <= consumed) {
|
|
167
|
+
const segment = current.segments.shift();
|
|
168
|
+
if (segment) {
|
|
169
|
+
segment.node.onended = null;
|
|
170
|
+
segment.node.disconnect();
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
if (current.started && current.eof && consumed === current.received && consumed > 0) {
|
|
174
|
+
position(current, "finished");
|
|
175
|
+
if (!active(current)) continue;
|
|
176
|
+
receipt(current, "finished");
|
|
177
|
+
sources.splice(sources.indexOf(current), 1);
|
|
178
|
+
wake();
|
|
179
|
+
} else {
|
|
180
|
+
if (current.started && consumed > current.lastPosition) {
|
|
181
|
+
current.lastPosition = consumed;
|
|
182
|
+
position(current, "playing");
|
|
183
|
+
}
|
|
184
|
+
if (!active(current)) continue;
|
|
185
|
+
if (reportCredit && consumed > current.lastReported) {
|
|
186
|
+
current.lastReported = consumed;
|
|
187
|
+
receipt(current, "progress");
|
|
188
|
+
}
|
|
189
|
+
wake();
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
};
|
|
193
|
+
const playbackState = () => {
|
|
194
|
+
const next = context?.state !== "running";
|
|
195
|
+
if (next !== blocked) {
|
|
196
|
+
blocked = next;
|
|
197
|
+
notify(() => options.onPlaybackBlocked?.(next ? startAudio : null));
|
|
198
|
+
}
|
|
199
|
+
tick();
|
|
200
|
+
};
|
|
201
|
+
const getContext = () => {
|
|
202
|
+
if (!context) {
|
|
203
|
+
context = new AudioContext({ sampleRate: correlatedAudioHeaderSchema.shape.sample_rate.value });
|
|
204
|
+
context.onstatechange = playbackState;
|
|
205
|
+
playbackState();
|
|
206
|
+
}
|
|
207
|
+
return context;
|
|
208
|
+
};
|
|
209
|
+
async function startAudio() {
|
|
210
|
+
if (disposed || binding.unavailable) throw new Error("Correlated audio binding is no longer available");
|
|
211
|
+
const ctx = getContext();
|
|
212
|
+
await ctx.resume();
|
|
213
|
+
playbackState();
|
|
214
|
+
if (ctx.state !== "running") throw new Error("Correlated audio playback is blocked");
|
|
215
|
+
}
|
|
216
|
+
async function receive(reader, participant) {
|
|
217
|
+
if (disposed || binding.unavailable || room.state !== "connected" || participant.identity !== options.agentIdentity) return;
|
|
218
|
+
let value;
|
|
219
|
+
try {
|
|
220
|
+
value = JSON.parse(reader.info.attributes?.["rta.audio"] ?? "");
|
|
221
|
+
} catch {
|
|
222
|
+
return;
|
|
223
|
+
}
|
|
224
|
+
const parsed = correlatedAudioHeaderSchema.safeParse(value);
|
|
225
|
+
if (!parsed.success || parsed.data.session_id !== options.sessionId || parsed.data.epoch < binding.epoch) return;
|
|
226
|
+
const header = parsed.data;
|
|
227
|
+
if (header.stream_id !== reader.info.id) return;
|
|
228
|
+
if (header.epoch > binding.epoch) {
|
|
229
|
+
binding.epoch = header.epoch;
|
|
230
|
+
invalidate("cancelled");
|
|
231
|
+
}
|
|
232
|
+
const key = `${header.epoch}/${header.stream_id}`;
|
|
233
|
+
if (seen.has(key)) return;
|
|
234
|
+
if (sources.length >= 128 || seen.size >= 1e4) {
|
|
235
|
+
invalidate("invalid_stream");
|
|
236
|
+
return;
|
|
237
|
+
}
|
|
238
|
+
seen.add(key);
|
|
239
|
+
const previousSource = sources.at(-1);
|
|
240
|
+
const current = { header, abort: new AbortController(), segments: [], received: 0, consumed: 0, started: false, eof: false, lastReported: 0, lastPosition: -1, rpcBusy: false };
|
|
241
|
+
sources.push(current);
|
|
242
|
+
let carry;
|
|
243
|
+
try {
|
|
244
|
+
if (options.prepare) await options.prepare(header, current.abort.signal);
|
|
245
|
+
if (!active(current) || current.abort.signal.aborted) return;
|
|
246
|
+
const ctx = getContext();
|
|
247
|
+
while (active(current) && (previousSource && active(previousSource) && !previousSource.eof || buffered() > CORRELATED_AUDIO_CONTRACT.maxBufferedSamples - CORRELATED_AUDIO_CONTRACT.maxChunkBytes / 2)) {
|
|
248
|
+
await new Promise((resolve) => waiters.add(resolve));
|
|
249
|
+
}
|
|
250
|
+
if (!active(current)) return;
|
|
251
|
+
for await (const chunk of reader.withAbortSignal(current.abort.signal)) {
|
|
252
|
+
if (!active(current)) break;
|
|
253
|
+
if (chunk.byteLength > CORRELATED_AUDIO_CONTRACT.maxChunkBytes) throw new Error("PCM chunk exceeds contract");
|
|
254
|
+
const count = Math.floor((chunk.byteLength + (carry === void 0 ? 0 : 1)) / 2);
|
|
255
|
+
if (current.received + count > CORRELATED_AUDIO_CONTRACT.maxSamples) throw new Error("PCM stream exceeds contract");
|
|
256
|
+
while (active(current) && buffered() + count > CORRELATED_AUDIO_CONTRACT.maxBufferedSamples) {
|
|
257
|
+
await new Promise((resolve) => waiters.add(resolve));
|
|
258
|
+
}
|
|
259
|
+
if (!active(current)) break;
|
|
260
|
+
if (!count) {
|
|
261
|
+
if (chunk.length) carry = chunk[0];
|
|
262
|
+
continue;
|
|
263
|
+
}
|
|
264
|
+
const buffer = ctx.createBuffer(header.channels, count, header.sample_rate);
|
|
265
|
+
const data = buffer.getChannelData(0);
|
|
266
|
+
let cursor = 0;
|
|
267
|
+
for (let i = 0; i < count; i++) {
|
|
268
|
+
const low = carry ?? chunk[cursor++];
|
|
269
|
+
carry = void 0;
|
|
270
|
+
const raw = low | chunk[cursor++] << 8;
|
|
271
|
+
data[i] = (raw >= 32768 ? raw - 65536 : raw) / 32768;
|
|
272
|
+
}
|
|
273
|
+
if (cursor < chunk.length) carry = chunk[cursor];
|
|
274
|
+
const node = ctx.createBufferSource();
|
|
275
|
+
node.buffer = buffer;
|
|
276
|
+
node.connect(ctx.destination);
|
|
277
|
+
const start = Math.max(ctx.currentTime, nextStart);
|
|
278
|
+
current.segments.push({ node, start, offset: current.received, length: count });
|
|
279
|
+
current.received += count;
|
|
280
|
+
nextStart = start + count / header.sample_rate;
|
|
281
|
+
node.onended = () => tick();
|
|
282
|
+
node.start(start);
|
|
283
|
+
while (active(current) && buffered() > CORRELATED_AUDIO_CONTRACT.maxBufferedSamples - CORRELATED_AUDIO_CONTRACT.maxChunkBytes / 2) {
|
|
284
|
+
await new Promise((resolve) => waiters.add(resolve));
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
if (!active(current)) return;
|
|
288
|
+
if (carry !== void 0 || current.received === 0) throw new Error("Incomplete PCM stream");
|
|
289
|
+
current.eof = true;
|
|
290
|
+
wake();
|
|
291
|
+
tick(false);
|
|
292
|
+
} catch {
|
|
293
|
+
if (active(current)) invalidate("invalid_stream");
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
const cancel = async (data) => {
|
|
297
|
+
if (disposed || binding.unavailable || room.state !== "connected" || data.callerIdentity !== options.agentIdentity) throw new Error("Unbound audio cancellation");
|
|
298
|
+
const value = JSON.parse(data.payload);
|
|
299
|
+
const frame2 = correlatedAudioCancelSchema.parse(value);
|
|
300
|
+
if (frame2.session_id !== options.sessionId) throw new Error("Wrong audio session");
|
|
301
|
+
if (frame2.epoch <= binding.epoch) return "";
|
|
302
|
+
binding.epoch = frame2.epoch;
|
|
303
|
+
invalidate("cancelled");
|
|
304
|
+
return "";
|
|
305
|
+
};
|
|
306
|
+
const lost = () => invalidate("connection_lost");
|
|
307
|
+
const changed = () => {
|
|
308
|
+
if (["disconnected", "reconnecting", "signalReconnecting"].includes(room.state)) lost();
|
|
309
|
+
};
|
|
310
|
+
const participantLeft = (participant) => {
|
|
311
|
+
if (participant.identity === options.agentIdentity) lost();
|
|
312
|
+
};
|
|
313
|
+
let registered = false;
|
|
314
|
+
try {
|
|
315
|
+
room.registerByteStreamHandler(CORRELATED_AUDIO_CONTRACT.topic, (reader, participant) => {
|
|
316
|
+
void receive(reader, participant);
|
|
317
|
+
});
|
|
318
|
+
registered = true;
|
|
319
|
+
room.localParticipant.registerRpcMethod(CORRELATED_AUDIO_CONTRACT.cancelMethod, cancel);
|
|
320
|
+
} catch (error) {
|
|
321
|
+
if (registered) room.unregisterByteStreamHandler(CORRELATED_AUDIO_CONTRACT.topic);
|
|
322
|
+
attachments.delete(room);
|
|
323
|
+
throw error;
|
|
324
|
+
}
|
|
325
|
+
room.on(RoomEvent.Reconnecting, lost);
|
|
326
|
+
room.on(RoomEvent.SignalReconnecting, lost);
|
|
327
|
+
room.on(RoomEvent.Disconnected, lost);
|
|
328
|
+
room.on(RoomEvent.ConnectionStateChanged, changed);
|
|
329
|
+
room.on(RoomEvent.ParticipantDisconnected, participantLeft);
|
|
330
|
+
const timer = setInterval(() => tick(), 100);
|
|
331
|
+
let frame = 0;
|
|
332
|
+
const render = () => {
|
|
333
|
+
if (disposed) return;
|
|
334
|
+
tick(false);
|
|
335
|
+
frame = requestAnimationFrame(render);
|
|
336
|
+
};
|
|
337
|
+
frame = requestAnimationFrame(render);
|
|
338
|
+
return {
|
|
339
|
+
startAudio,
|
|
340
|
+
dispose() {
|
|
341
|
+
if (closing) return closing;
|
|
342
|
+
invalidate("disposed");
|
|
343
|
+
disposed = true;
|
|
344
|
+
clearInterval(timer);
|
|
345
|
+
cancelAnimationFrame(frame);
|
|
346
|
+
room.unregisterByteStreamHandler(CORRELATED_AUDIO_CONTRACT.topic);
|
|
347
|
+
room.localParticipant.unregisterRpcMethod(CORRELATED_AUDIO_CONTRACT.cancelMethod);
|
|
348
|
+
room.off(RoomEvent.Reconnecting, lost);
|
|
349
|
+
room.off(RoomEvent.SignalReconnecting, lost);
|
|
350
|
+
room.off(RoomEvent.Disconnected, lost);
|
|
351
|
+
room.off(RoomEvent.ConnectionStateChanged, changed);
|
|
352
|
+
room.off(RoomEvent.ParticipantDisconnected, participantLeft);
|
|
353
|
+
attachments.delete(room);
|
|
354
|
+
if (context) context.onstatechange = null;
|
|
355
|
+
closing = context?.close().catch(() => {
|
|
356
|
+
}) ?? Promise.resolve();
|
|
357
|
+
return closing;
|
|
358
|
+
}
|
|
359
|
+
};
|
|
360
|
+
}
|
|
361
|
+
|
|
1
362
|
// ../browser/src/video-support.ts
|
|
2
363
|
function checkVideoSupport(codec = "h264") {
|
|
3
364
|
if (typeof RTCRtpReceiver === "undefined") {
|
|
@@ -258,4 +619,4 @@ function now() {
|
|
|
258
619
|
return typeof performance !== "undefined" ? performance.now() : Date.now();
|
|
259
620
|
}
|
|
260
621
|
|
|
261
|
-
export { DEFAULT_AVATAR_PLAYOUT_DELAY_SECONDS, applyAvatarPlayoutDelay, applyPlayoutDelay, attachBoomerangPlayback, attachRemoteAudio, checkVideoSupport, describeMicrophoneFailure, enableMicrophone, prepareAvatarRoom };
|
|
622
|
+
export { DEFAULT_AVATAR_PLAYOUT_DELAY_SECONDS, applyAvatarPlayoutDelay, applyPlayoutDelay, attachBoomerangPlayback, attachCorrelatedAudio, attachRemoteAudio, checkVideoSupport, describeMicrophoneFailure, enableMicrophone, prepareAvatarRoom };
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { z } from 'zod';
|
|
1
|
+
import { RealtimeAvatar, ROUTE_TIMEOUT_MS, proxyEndRequestSchema, proxyConnectRequestSchema, isQueued, RealtimeAvatarHttpError, RealtimeAvatarError } from './chunk-WHYDSORD.js';
|
|
3
2
|
|
|
4
3
|
// ../http-client/src/generated/error-semantics.ts
|
|
5
4
|
var ERROR_SEMANTICS = {
|
|
@@ -43,17 +42,6 @@ var MESSAGE_SHAPE_PATTERNS = { "machine": { "source": "[_=<>{}\\[\\]|\\\\$`]|\\w
|
|
|
43
42
|
Object.values(MESSAGE_SHAPE_PATTERNS).map(
|
|
44
43
|
(pattern) => new RegExp(pattern.source, pattern.flags)
|
|
45
44
|
);
|
|
46
|
-
var proxyConnectRequestSchema = z.object({
|
|
47
|
-
avatarId: z.string().min(1),
|
|
48
|
-
mode: sessionModeSchema.optional()
|
|
49
|
-
});
|
|
50
|
-
var proxyEndRequestSchema = z.object({
|
|
51
|
-
session_id: z.string().min(1).optional(),
|
|
52
|
-
queue_ticket_id: z.string().min(1).optional(),
|
|
53
|
-
reason: liveKitSessionReleaseReasonSchema.optional().catch(void 0)
|
|
54
|
-
}).strict().refine((body) => body.session_id !== void 0 || body.queue_ticket_id !== void 0, {
|
|
55
|
-
message: "session_id or queue_ticket_id is required"
|
|
56
|
-
});
|
|
57
45
|
|
|
58
46
|
// ../proxy/src/config.ts
|
|
59
47
|
var MISCONFIGURED = "internal_error";
|