realtime-avatar 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. package/AGENTS.md +132 -0
  2. package/CLAUDE.md +17 -0
  3. package/LICENSE +21 -0
  4. package/README.md +254 -0
  5. package/dist/api-keys.d.ts +26 -0
  6. package/dist/api-keys.d.ts.map +1 -0
  7. package/dist/api-keys.js +88 -0
  8. package/dist/api-keys.js.map +1 -0
  9. package/dist/browser/audio.d.ts +65 -0
  10. package/dist/browser/audio.d.ts.map +1 -0
  11. package/dist/browser/audio.js +154 -0
  12. package/dist/browser/audio.js.map +1 -0
  13. package/dist/browser/boomerang.d.ts +38 -0
  14. package/dist/browser/boomerang.d.ts.map +1 -0
  15. package/dist/browser/boomerang.js +85 -0
  16. package/dist/browser/boomerang.js.map +1 -0
  17. package/dist/browser/index.d.ts +8 -0
  18. package/dist/browser/index.d.ts.map +1 -0
  19. package/dist/browser/index.js +8 -0
  20. package/dist/browser/index.js.map +1 -0
  21. package/dist/browser/media-session.d.ts +43 -0
  22. package/dist/browser/media-session.d.ts.map +1 -0
  23. package/dist/browser/media-session.js +169 -0
  24. package/dist/browser/media-session.js.map +1 -0
  25. package/dist/browser/player.d.ts +162 -0
  26. package/dist/browser/player.d.ts.map +1 -0
  27. package/dist/browser/player.js +514 -0
  28. package/dist/browser/player.js.map +1 -0
  29. package/dist/browser/view.d.ts +47 -0
  30. package/dist/browser/view.d.ts.map +1 -0
  31. package/dist/browser/view.js +7 -0
  32. package/dist/browser/view.js.map +1 -0
  33. package/dist/browser/webrtc.d.ts +21 -0
  34. package/dist/browser/webrtc.d.ts.map +1 -0
  35. package/dist/browser/webrtc.js +149 -0
  36. package/dist/browser/webrtc.js.map +1 -0
  37. package/dist/browser/yuv-canvas.d.ts +13 -0
  38. package/dist/browser/yuv-canvas.d.ts.map +1 -0
  39. package/dist/browser/yuv-canvas.js +95 -0
  40. package/dist/browser/yuv-canvas.js.map +1 -0
  41. package/dist/client.d.ts +195 -0
  42. package/dist/client.d.ts.map +1 -0
  43. package/dist/client.js +440 -0
  44. package/dist/client.js.map +1 -0
  45. package/dist/errors.d.ts +33 -0
  46. package/dist/errors.d.ts.map +1 -0
  47. package/dist/errors.js +73 -0
  48. package/dist/errors.js.map +1 -0
  49. package/dist/generated/openapi.d.ts +1523 -0
  50. package/dist/generated/openapi.d.ts.map +1 -0
  51. package/dist/generated/openapi.js +6 -0
  52. package/dist/generated/openapi.js.map +1 -0
  53. package/dist/index.d.ts +14 -0
  54. package/dist/index.d.ts.map +1 -0
  55. package/dist/index.js +15 -0
  56. package/dist/index.js.map +1 -0
  57. package/dist/media.d.ts +40 -0
  58. package/dist/media.d.ts.map +1 -0
  59. package/dist/media.js +4 -0
  60. package/dist/media.js.map +1 -0
  61. package/dist/mux.d.ts +104 -0
  62. package/dist/mux.d.ts.map +1 -0
  63. package/dist/mux.js +290 -0
  64. package/dist/mux.js.map +1 -0
  65. package/dist/platform.d.ts +163 -0
  66. package/dist/platform.d.ts.map +1 -0
  67. package/dist/platform.js +5 -0
  68. package/dist/platform.js.map +1 -0
  69. package/dist/react/index.d.ts +5 -0
  70. package/dist/react/index.d.ts.map +1 -0
  71. package/dist/react/index.js +5 -0
  72. package/dist/react/index.js.map +1 -0
  73. package/dist/react/provider.d.ts +37 -0
  74. package/dist/react/provider.d.ts.map +1 -0
  75. package/dist/react/provider.js +33 -0
  76. package/dist/react/provider.js.map +1 -0
  77. package/dist/react/realtime.d.ts +74 -0
  78. package/dist/react/realtime.d.ts.map +1 -0
  79. package/dist/react/realtime.js +105 -0
  80. package/dist/react/realtime.js.map +1 -0
  81. package/dist/react/session.d.ts +91 -0
  82. package/dist/react/session.d.ts.map +1 -0
  83. package/dist/react/session.js +322 -0
  84. package/dist/react/session.js.map +1 -0
  85. package/dist/react/stage.d.ts +23 -0
  86. package/dist/react/stage.d.ts.map +1 -0
  87. package/dist/react/stage.js +62 -0
  88. package/dist/react/stage.js.map +1 -0
  89. package/dist/schemas.d.ts +59 -0
  90. package/dist/schemas.d.ts.map +1 -0
  91. package/dist/schemas.js +58 -0
  92. package/dist/schemas.js.map +1 -0
  93. package/dist/server.d.ts +2 -0
  94. package/dist/server.d.ts.map +1 -0
  95. package/dist/server.js +8 -0
  96. package/dist/server.js.map +1 -0
  97. package/dist/session-socket.d.ts +96 -0
  98. package/dist/session-socket.d.ts.map +1 -0
  99. package/dist/session-socket.js +299 -0
  100. package/dist/session-socket.js.map +1 -0
  101. package/dist/session.d.ts +107 -0
  102. package/dist/session.d.ts.map +1 -0
  103. package/dist/session.js +192 -0
  104. package/dist/session.js.map +1 -0
  105. package/dist/types.d.ts +24 -0
  106. package/dist/types.d.ts.map +1 -0
  107. package/dist/types.js +2 -0
  108. package/dist/types.js.map +1 -0
  109. package/package.json +94 -0
@@ -0,0 +1,96 @@
1
+ import { type AvatarMuxEvent } from "./mux";
2
+ import type { RealtimePrepareRequest, RealtimePrepareResponse, RealtimeTurnRequest } from "./types";
3
+ /** Errors surfaced by the realtime session socket (connection, close, server `error` events). */
4
+ export declare class RealtimeAvatarSessionError extends Error {
5
+ readonly detail: Record<string, unknown> | null;
6
+ constructor(message: string, detail?: Record<string, unknown> | null);
7
+ /** Machine-readable error code from the server `error` event, when present. */
8
+ readonly code?: string;
9
+ /** Mirrors RealtimeAvatarApiError so billing failures over the socket raise the same paywall handling. */
10
+ get isBillingRequired(): boolean;
11
+ }
12
+ /** Structural turn-event source — satisfied by both `RealtimeTurnStream` and {@link RealtimeSessionSocket.turn}. */
13
+ export type AvatarTurnEventSource = {
14
+ events: AsyncGenerator<AvatarMuxEvent>;
15
+ };
16
+ export type RealtimeSessionSocketOptions = {
17
+ /** Full ws(s):// URL of the realtime session endpoint. */
18
+ url: string;
19
+ /**
20
+ * Extra headers, e.g. `{ Authorization: "Bearer ..." }`. Server-side
21
+ * runtimes only (Bun/undici accept headers on `new WebSocket`); browsers
22
+ * cannot set WebSocket headers and must authenticate via a same-origin
23
+ * proxy (cookies) instead.
24
+ */
25
+ headers?: Record<string, string>;
26
+ /**
27
+ * Short-lived signed session token, appended as `?token=` — the browser
28
+ * auth path (browsers cannot set WebSocket headers). Mint one from your
29
+ * backend; the platform exposes a cookie-authenticated mint endpoint.
30
+ */
31
+ token?: string;
32
+ /** Override the WebSocket constructor (tests, Node `ws` package). */
33
+ WebSocketImpl?: typeof WebSocket;
34
+ /** Abort handshake + session. */
35
+ signal?: AbortSignal;
36
+ /**
37
+ * Idle keepalive interval in ms; `0` disables. Defaults to 25s. Pings are
38
+ * only sent while no prepare/turn is in flight, and exist so idle proxies
39
+ * never silently drop the warm session — a dead socket costs the next turn
40
+ * the full reconnect-and-prepare path.
41
+ */
42
+ keepAliveIntervalMs?: number;
43
+ };
44
+ /**
45
+ * One realtime session over one WebSocket — which on the Modal runtime means
46
+ * ONE GPU container for `prepare()` and every subsequent turn. The HTTP
47
+ * endpoints route each request independently, so a turn can land on a
48
+ * container that never saw the prepare and re-pay the avatar load; this
49
+ * socket is the low-latency path.
50
+ *
51
+ * const session = await RealtimeSessionSocket.connect({ url, headers });
52
+ * const prepared = await session.prepare({ source_kind: "source_video", ... });
53
+ * const turn = session.turn({ mode: "speak_text", text: "hi", ... });
54
+ * await player.play(turn); // AvatarPlayer accepts any { events } source
55
+ * session.close();
56
+ *
57
+ * One operation at a time: a second `prepare()`/`turn()` while one is
58
+ * streaming rejects immediately.
59
+ */
60
+ export declare class RealtimeSessionSocket {
61
+ private readonly socket;
62
+ private pending;
63
+ private turnSink;
64
+ private closed;
65
+ private keepAlive;
66
+ private constructor();
67
+ /** Ping the server on an interval while idle so intermediaries keep the session alive. */
68
+ private startKeepAlive;
69
+ private stopKeepAlive;
70
+ /** Connect and resolve once the server confirms the session (`session_ready`). */
71
+ static connect(options: RealtimeSessionSocketOptions): Promise<RealtimeSessionSocket>;
72
+ /** Warm this container for the avatar; resolves with the standard prepare response. */
73
+ prepare(request: RealtimePrepareRequest): Promise<RealtimePrepareResponse>;
74
+ /** Round-trip latency probe. Resolves with elapsed milliseconds. */
75
+ ping(): Promise<number>;
76
+ /**
77
+ * Run one turn on this session's container. Returns immediately; consume
78
+ * `.events` (ends after `done`). Pass directly to `AvatarPlayer.play()`.
79
+ */
80
+ turn(request: RealtimeTurnRequest): AvatarTurnEventSource;
81
+ /**
82
+ * Stop the in-flight turn server-side (the GPU stops rendering frames nobody
83
+ * will play) while KEEPING the session socket usable for the next turn.
84
+ * Resolves once the server confirms (`turn_cancelled`) or the turn ends on
85
+ * its own; rejects after `timeoutMs` — close the socket if that happens.
86
+ * No-op when no turn is streaming.
87
+ */
88
+ cancelTurn(timeoutMs?: number): Promise<void>;
89
+ /** True while the socket is connected and usable for prepare/turn calls. */
90
+ get open(): boolean;
91
+ close(): void;
92
+ private assertIdle;
93
+ private route;
94
+ private fail;
95
+ }
96
+ //# sourceMappingURL=session-socket.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"session-socket.d.ts","sourceRoot":"","sources":["../src/session-socket.ts"],"names":[],"mappings":"AACA,OAAO,EAA0C,KAAK,cAAc,EAAuB,MAAM,OAAO,CAAC;AACzG,OAAO,KAAK,EAAE,sBAAsB,EAAE,uBAAuB,EAAE,mBAAmB,EAAE,MAAM,SAAS,CAAC;AAEpG,iGAAiG;AACjG,qBAAa,0BAA2B,SAAQ,KAAK;IAGjD,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI;gBAD/C,OAAO,EAAE,MAAM,EACN,MAAM,GAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAW;IAOxD,+EAA+E;IAC/E,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IAEvB,0GAA0G;IAC1G,IAAI,iBAAiB,IAAI,OAAO,CAE/B;CACF;AAID,oHAAoH;AACpH,MAAM,MAAM,qBAAqB,GAAG;IAAE,MAAM,EAAE,cAAc,CAAC,cAAc,CAAC,CAAA;CAAE,CAAC;AAE/E,MAAM,MAAM,4BAA4B,GAAG;IACzC,0DAA0D;IAC1D,GAAG,EAAE,MAAM,CAAC;IACZ;;;;;OAKG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC;;;;OAIG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,qEAAqE;IACrE,aAAa,CAAC,EAAE,OAAO,SAAS,CAAC;IACjC,iCAAiC;IACjC,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB;;;;;OAKG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;CAC9B,CAAC;AAEF;;;;;;;;;;;;;;;GAeG;AACH,qBAAa,qBAAqB;IAgBZ,OAAO,CAAC,QAAQ,CAAC,MAAM;IAf3C,OAAO,CAAC,OAAO,CAIC;IAChB,OAAO,CAAC,QAAQ,CAMA;IAChB,OAAO,CAAC,MAAM,CAAsB;IACpC,OAAO,CAAC,SAAS,CAA+C;IAEhE,OAAO;IAEP,0FAA0F;IAC1F,OAAO,CAAC,cAAc;IAetB,OAAO,CAAC,aAAa;IAOrB,kFAAkF;IAClF,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,4BAA4B,GAAG,OAAO,CAAC,qBAAqB,CAAC;IA+DrF,uFAAuF;IACvF,OAAO,CAAC,OAAO,EAAE,sBAAsB,GAAG,OAAO,CAAC,uBAAuB,CAAC;IAS1E,oEAAoE;IACpE,IAAI,IAAI,OAAO,CAAC,MAAM,CAAC;IAUvB;;;OAGG;IACH,IAAI,CAAC,OAAO,EAAE,mBAAmB,GAAG,qBAAqB;IAgDzD;;;;;;OAMG;IACH,UAAU,CAAC,SAAS,SAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IAgB5C,4EAA4E;IAC5E,IAAI,IAAI,IAAI,OAAO,CAElB;IAED,KAAK,IAAI,IAAI;IAKb,OAAO,CAAC,UAAU;IAOlB,OAAO,CAAC,KAAK;IAwCb,OAAO,CAAC,IAAI;CAcb"}
@@ -0,0 +1,299 @@
1
+ import { RealtimeAvatarConfigError } from "./errors";
2
+ import { decodeAvatarMuxFrame, toAvatarMuxEvent } from "./mux";
3
+ /** Errors surfaced by the realtime session socket (connection, close, server `error` events). */
4
+ export class RealtimeAvatarSessionError extends Error {
5
+ detail;
6
+ constructor(message, detail = null) {
7
+ super(message);
8
+ this.detail = detail;
9
+ this.name = "RealtimeAvatarSessionError";
10
+ this.code = typeof detail?.code === "string" ? detail.code : undefined;
11
+ }
12
+ /** Machine-readable error code from the server `error` event, when present. */
13
+ code;
14
+ /** Mirrors RealtimeAvatarApiError so billing failures over the socket raise the same paywall handling. */
15
+ get isBillingRequired() {
16
+ return this.code === "insufficient_credits" || this.code === "spend_limit_exceeded";
17
+ }
18
+ }
19
+ /**
20
+ * One realtime session over one WebSocket — which on the Modal runtime means
21
+ * ONE GPU container for `prepare()` and every subsequent turn. The HTTP
22
+ * endpoints route each request independently, so a turn can land on a
23
+ * container that never saw the prepare and re-pay the avatar load; this
24
+ * socket is the low-latency path.
25
+ *
26
+ * const session = await RealtimeSessionSocket.connect({ url, headers });
27
+ * const prepared = await session.prepare({ source_kind: "source_video", ... });
28
+ * const turn = session.turn({ mode: "speak_text", text: "hi", ... });
29
+ * await player.play(turn); // AvatarPlayer accepts any { events } source
30
+ * session.close();
31
+ *
32
+ * One operation at a time: a second `prepare()`/`turn()` while one is
33
+ * streaming rejects immediately.
34
+ */
35
+ export class RealtimeSessionSocket {
36
+ socket;
37
+ pending = null;
38
+ turnSink = null;
39
+ closed = null;
40
+ keepAlive = null;
41
+ constructor(socket) {
42
+ this.socket = socket;
43
+ }
44
+ /** Ping the server on an interval while idle so intermediaries keep the session alive. */
45
+ startKeepAlive(intervalMs) {
46
+ if (intervalMs <= 0)
47
+ return;
48
+ const timer = setInterval(() => {
49
+ if (!this.open || this.pending || this.turnSink)
50
+ return;
51
+ try {
52
+ this.socket.send(JSON.stringify({ type: "ping" }));
53
+ }
54
+ catch {
55
+ // The close handler tears the session down; nothing to do here.
56
+ }
57
+ }, intervalMs);
58
+ // Node timers hold the event loop open; a keepalive must not.
59
+ timer.unref?.();
60
+ this.keepAlive = timer;
61
+ }
62
+ stopKeepAlive() {
63
+ if (this.keepAlive !== null) {
64
+ clearInterval(this.keepAlive);
65
+ this.keepAlive = null;
66
+ }
67
+ }
68
+ /** Connect and resolve once the server confirms the session (`session_ready`). */
69
+ static connect(options) {
70
+ const WebSocketCtor = options.WebSocketImpl ?? globalThis.WebSocket;
71
+ if (!WebSocketCtor) {
72
+ throw new RealtimeAvatarConfigError("A WebSocket implementation is required for realtime sessions");
73
+ }
74
+ const url = options.token
75
+ ? `${options.url}${options.url.includes("?") ? "&" : "?"}token=${encodeURIComponent(options.token)}`
76
+ : options.url;
77
+ const socket = options.headers
78
+ ? new WebSocketCtor(url, { headers: options.headers })
79
+ : new WebSocketCtor(url);
80
+ socket.binaryType = "arraybuffer";
81
+ const session = new RealtimeSessionSocket(socket);
82
+ return new Promise((resolve, reject) => {
83
+ let settled = false;
84
+ const settle = (fn) => {
85
+ if (!settled) {
86
+ settled = true;
87
+ fn();
88
+ }
89
+ };
90
+ options.signal?.addEventListener("abort", () => {
91
+ socket.close();
92
+ settle(() => reject(new RealtimeAvatarSessionError("Realtime session aborted")));
93
+ }, { once: true });
94
+ socket.addEventListener("message", (message) => {
95
+ let frame;
96
+ try {
97
+ frame = decodeAvatarMuxFrame(toBytes(message.data));
98
+ }
99
+ catch (error) {
100
+ session.fail(error instanceof Error ? error : new Error(String(error)));
101
+ return;
102
+ }
103
+ if (frame.type === "session_ready") {
104
+ settle(() => {
105
+ session.startKeepAlive(options.keepAliveIntervalMs ?? 25_000);
106
+ resolve(session);
107
+ });
108
+ return;
109
+ }
110
+ session.route(frame);
111
+ });
112
+ socket.addEventListener("close", (event) => {
113
+ session.fail(new RealtimeAvatarSessionError(`Realtime session closed (${event.code}${event.reason ? `: ${event.reason}` : ""})`),
114
+ /* graceful */ event.code === 1000);
115
+ settle(() => reject(new RealtimeAvatarSessionError("Realtime session closed before it was ready")));
116
+ });
117
+ socket.addEventListener("error", () => {
118
+ settle(() => reject(new RealtimeAvatarSessionError("Realtime session connection failed")));
119
+ });
120
+ });
121
+ }
122
+ /** Warm this container for the avatar; resolves with the standard prepare response. */
123
+ prepare(request) {
124
+ this.assertIdle();
125
+ const frame = new Promise((resolve, reject) => {
126
+ this.pending = { resolve, reject, kind: "prepare" };
127
+ });
128
+ this.socket.send(JSON.stringify({ type: "prepare", ...request }));
129
+ return frame.then(({ header }) => header);
130
+ }
131
+ /** Round-trip latency probe. Resolves with elapsed milliseconds. */
132
+ ping() {
133
+ this.assertIdle();
134
+ const started = nowMs();
135
+ const frame = new Promise((resolve, reject) => {
136
+ this.pending = { resolve, reject, kind: "pong" };
137
+ });
138
+ this.socket.send(JSON.stringify({ type: "ping" }));
139
+ return frame.then(() => nowMs() - started);
140
+ }
141
+ /**
142
+ * Run one turn on this session's container. Returns immediately; consume
143
+ * `.events` (ends after `done`). Pass directly to `AvatarPlayer.play()`.
144
+ */
145
+ turn(request) {
146
+ this.assertIdle();
147
+ const queue = [];
148
+ let wake = null;
149
+ let finished = false;
150
+ let failure = null;
151
+ let settle;
152
+ const settled = new Promise((resolve) => {
153
+ settle = resolve;
154
+ });
155
+ // Sink lifetime is owned by `route()`/`fail()` (cleared on done/error/
156
+ // turn_cancelled/socket loss), NOT by the consumer: a player that stops
157
+ // reading mid-turn must not orphan the cancel confirmation.
158
+ const sink = {
159
+ push: (event) => {
160
+ queue.push(event);
161
+ wake?.();
162
+ },
163
+ fail: (error) => {
164
+ failure = error;
165
+ finished = true;
166
+ settle();
167
+ wake?.();
168
+ },
169
+ end: () => {
170
+ finished = true;
171
+ settle();
172
+ wake?.();
173
+ },
174
+ settled,
175
+ };
176
+ this.turnSink = sink;
177
+ this.socket.send(JSON.stringify({ type: "turn", ...request }));
178
+ async function* events() {
179
+ while (true) {
180
+ while (queue.length > 0)
181
+ yield queue.shift();
182
+ if (failure)
183
+ throw failure;
184
+ if (finished)
185
+ return;
186
+ await new Promise((resolve) => {
187
+ wake = resolve;
188
+ });
189
+ wake = null;
190
+ }
191
+ }
192
+ return { events: events() };
193
+ }
194
+ /**
195
+ * Stop the in-flight turn server-side (the GPU stops rendering frames nobody
196
+ * will play) while KEEPING the session socket usable for the next turn.
197
+ * Resolves once the server confirms (`turn_cancelled`) or the turn ends on
198
+ * its own; rejects after `timeoutMs` — close the socket if that happens.
199
+ * No-op when no turn is streaming.
200
+ */
201
+ cancelTurn(timeoutMs = 5_000) {
202
+ const sink = this.turnSink;
203
+ if (this.closed || !sink)
204
+ return Promise.resolve();
205
+ this.socket.send(JSON.stringify({ type: "cancel" }));
206
+ return new Promise((resolve, reject) => {
207
+ const timer = setTimeout(() => reject(new RealtimeAvatarSessionError("Turn cancel was not confirmed in time")), timeoutMs);
208
+ void sink.settled.then(() => {
209
+ clearTimeout(timer);
210
+ resolve();
211
+ });
212
+ });
213
+ }
214
+ /** True while the socket is connected and usable for prepare/turn calls. */
215
+ get open() {
216
+ return this.closed === null && this.socket.readyState === 1;
217
+ }
218
+ close() {
219
+ this.stopKeepAlive();
220
+ this.socket.close(1000);
221
+ }
222
+ assertIdle() {
223
+ if (this.closed)
224
+ throw this.closed;
225
+ if (this.pending || this.turnSink) {
226
+ throw new RealtimeAvatarConfigError("The realtime session already has an operation in flight");
227
+ }
228
+ }
229
+ route(frame) {
230
+ if (frame.type === "prepared" || frame.type === "pong") {
231
+ const pending = this.pending;
232
+ // Drop strays: keepalive pongs nobody awaits, and frames whose kind
233
+ // doesn't match the pending operation (a pong must never resolve a
234
+ // prepare with the wrong header).
235
+ if (!pending || (frame.type === "prepared") !== (pending.kind === "prepare"))
236
+ return;
237
+ this.pending = null;
238
+ pending.resolve(frame);
239
+ return;
240
+ }
241
+ if (frame.type === "error" && this.pending) {
242
+ const pending = this.pending;
243
+ this.pending = null;
244
+ pending.reject(new RealtimeAvatarSessionError(String(frame.header.message ?? "Realtime session error"), frame.header));
245
+ return;
246
+ }
247
+ const sink = this.turnSink;
248
+ if (!sink)
249
+ return; // stray event outside any operation
250
+ if (frame.type === "turn_cancelled") {
251
+ sink.end();
252
+ this.turnSink = null;
253
+ return;
254
+ }
255
+ let event;
256
+ try {
257
+ event = toAvatarMuxEvent(frame);
258
+ }
259
+ catch (error) {
260
+ sink.fail(error instanceof Error ? error : new Error(String(error)));
261
+ this.turnSink = null;
262
+ return;
263
+ }
264
+ sink.push(event);
265
+ // `error` terminates the turn but keeps the session usable; `done` ends it.
266
+ if (event.header.type === "done" || event.header.type === "error") {
267
+ sink.end();
268
+ this.turnSink = null;
269
+ }
270
+ }
271
+ fail(error, graceful = false) {
272
+ if (this.closed)
273
+ return;
274
+ this.closed = error;
275
+ this.stopKeepAlive();
276
+ const pending = this.pending;
277
+ this.pending = null;
278
+ pending?.reject(error);
279
+ const sink = this.turnSink;
280
+ this.turnSink = null;
281
+ if (sink) {
282
+ if (graceful)
283
+ sink.end();
284
+ else
285
+ sink.fail(error);
286
+ }
287
+ }
288
+ }
289
+ function toBytes(data) {
290
+ if (data instanceof ArrayBuffer)
291
+ return new Uint8Array(data);
292
+ if (data instanceof Uint8Array)
293
+ return data;
294
+ throw new Error("Realtime session expected binary WebSocket messages");
295
+ }
296
+ function nowMs() {
297
+ return typeof performance !== "undefined" ? performance.now() : Date.now();
298
+ }
299
+ //# sourceMappingURL=session-socket.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"session-socket.js","sourceRoot":"","sources":["../src/session-socket.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,MAAM,UAAU,CAAC;AACrD,OAAO,EAAE,oBAAoB,EAAE,gBAAgB,EAA4C,MAAM,OAAO,CAAC;AAGzG,iGAAiG;AACjG,MAAM,OAAO,0BAA2B,SAAQ,KAAK;IAGxC;IAFX,YACE,OAAe,EACN,SAAyC,IAAI;QAEtD,KAAK,CAAC,OAAO,CAAC,CAAC;QAFN,WAAM,GAAN,MAAM,CAAuC;QAGtD,IAAI,CAAC,IAAI,GAAG,4BAA4B,CAAC;QACzC,IAAI,CAAC,IAAI,GAAG,OAAO,MAAM,EAAE,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;IACzE,CAAC;IAED,+EAA+E;IACtE,IAAI,CAAU;IAEvB,0GAA0G;IAC1G,IAAI,iBAAiB;QACnB,OAAO,IAAI,CAAC,IAAI,KAAK,sBAAsB,IAAI,IAAI,CAAC,IAAI,KAAK,sBAAsB,CAAC;IACtF,CAAC;CACF;AAoCD;;;;;;;;;;;;;;;GAeG;AACH,MAAM,OAAO,qBAAqB;IAgBK;IAf7B,OAAO,GAIJ,IAAI,CAAC;IACR,QAAQ,GAML,IAAI,CAAC;IACR,MAAM,GAAiB,IAAI,CAAC;IAC5B,SAAS,GAA0C,IAAI,CAAC;IAEhE,YAAqC,MAAiB;QAAjB,WAAM,GAAN,MAAM,CAAW;IAAG,CAAC;IAE1D,0FAA0F;IAClF,cAAc,CAAC,UAAkB;QACvC,IAAI,UAAU,IAAI,CAAC;YAAE,OAAO;QAC5B,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE;YAC7B,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,QAAQ;gBAAE,OAAO;YACxD,IAAI,CAAC;gBACH,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;YACrD,CAAC;YAAC,MAAM,CAAC;gBACP,gEAAgE;YAClE,CAAC;QACH,CAAC,EAAE,UAAU,CAAC,CAAC;QACf,8DAA8D;QAC7D,KAA2C,CAAC,KAAK,EAAE,EAAE,CAAC;QACvD,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;IACzB,CAAC;IAEO,aAAa;QACnB,IAAI,IAAI,CAAC,SAAS,KAAK,IAAI,EAAE,CAAC;YAC5B,aAAa,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC9B,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACxB,CAAC;IACH,CAAC;IAED,kFAAkF;IAClF,MAAM,CAAC,OAAO,CAAC,OAAqC;QAClD,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,UAAU,CAAC,SAAS,CAAC;QACpE,IAAI,CAAC,aAAa,EAAE,CAAC;YACnB,MAAM,IAAI,yBAAyB,CAAC,8DAA8D,CAAC,CAAC;QACtG,CAAC;QACD,MAAM,GAAG,GAAG,OAAO,CAAC,KAAK;YACvB,CAAC,CAAC,GAAG,OAAO,CAAC,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,SAAS,kBAAkB,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE;YACpG,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC;QAChB,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO;YAC5B,CAAC,CAAC,IAAK,aAAsG,CACzG,GAAG,EACH,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAC7B;YACH,CAAC,CAAC,IAAI,aAAa,CAAC,GAAG,CAAC,CAAC;QAC3B,MAAM,CAAC,UAAU,GAAG,aAAa,CAAC;QAClC,MAAM,OAAO,GAAG,IAAI,qBAAqB,CAAC,MAAM,CAAC,CAAC;QAElD,OAAO,IAAI,OAAO,CAAwB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC5D,IAAI,OAAO,GAAG,KAAK,CAAC;YACpB,MAAM,MAAM,GAAG,CAAC,EAAc,EAAE,EAAE;gBAChC,IAAI,CAAC,OAAO,EAAE,CAAC;oBACb,OAAO,GAAG,IAAI,CAAC;oBACf,EAAE,EAAE,CAAC;gBACP,CAAC;YACH,CAAC,CAAC;YACF,OAAO,CAAC,MAAM,EAAE,gBAAgB,CAC9B,OAAO,EACP,GAAG,EAAE;gBACH,MAAM,CAAC,KAAK,EAAE,CAAC;gBACf,MAAM,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,0BAA0B,CAAC,0BAA0B,CAAC,CAAC,CAAC,CAAC;YACnF,CAAC,EACD,EAAE,IAAI,EAAE,IAAI,EAAE,CACf,CAAC;YACF,MAAM,CAAC,gBAAgB,CAAC,SAAS,EAAE,CAAC,OAAqB,EAAE,EAAE;gBAC3D,IAAI,KAAqB,CAAC;gBAC1B,IAAI,CAAC;oBACH,KAAK,GAAG,oBAAoB,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;gBACtD,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,OAAO,CAAC,IAAI,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;oBACxE,OAAO;gBACT,CAAC;gBACD,IAAI,KAAK,CAAC,IAAI,KAAK,eAAe,EAAE,CAAC;oBACnC,MAAM,CAAC,GAAG,EAAE;wBACV,OAAO,CAAC,cAAc,CAAC,OAAO,CAAC,mBAAmB,IAAI,MAAM,CAAC,CAAC;wBAC9D,OAAO,CAAC,OAAO,CAAC,CAAC;oBACnB,CAAC,CAAC,CAAC;oBACH,OAAO;gBACT,CAAC;gBACD,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YACvB,CAAC,CAAC,CAAC;YACH,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,CAAC,KAAiB,EAAE,EAAE;gBACrD,OAAO,CAAC,IAAI,CACV,IAAI,0BAA0B,CAAC,4BAA4B,KAAK,CAAC,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC;gBACnH,cAAc,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CACnC,CAAC;gBACF,MAAM,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,0BAA0B,CAAC,6CAA6C,CAAC,CAAC,CAAC,CAAC;YACtG,CAAC,CAAC,CAAC;YACH,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE;gBACpC,MAAM,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,0BAA0B,CAAC,oCAAoC,CAAC,CAAC,CAAC,CAAC;YAC7F,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAED,uFAAuF;IACvF,OAAO,CAAC,OAA+B;QACrC,IAAI,CAAC,UAAU,EAAE,CAAC;QAClB,MAAM,KAAK,GAAG,IAAI,OAAO,CAAiB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC5D,IAAI,CAAC,OAAO,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC;QACtD,CAAC,CAAC,CAAC;QACH,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC,CAAC;QAClE,OAAO,KAAK,CAAC,IAAI,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,MAA4C,CAAC,CAAC;IAClF,CAAC;IAED,oEAAoE;IACpE,IAAI;QACF,IAAI,CAAC,UAAU,EAAE,CAAC;QAClB,MAAM,OAAO,GAAG,KAAK,EAAE,CAAC;QACxB,MAAM,KAAK,GAAG,IAAI,OAAO,CAAiB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC5D,IAAI,CAAC,OAAO,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;QACnD,CAAC,CAAC,CAAC;QACH,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;QACnD,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,KAAK,EAAE,GAAG,OAAO,CAAC,CAAC;IAC7C,CAAC;IAED;;;OAGG;IACH,IAAI,CAAC,OAA4B;QAC/B,IAAI,CAAC,UAAU,EAAE,CAAC;QAClB,MAAM,KAAK,GAAqB,EAAE,CAAC;QACnC,IAAI,IAAI,GAAwB,IAAI,CAAC;QACrC,IAAI,QAAQ,GAAG,KAAK,CAAC;QACrB,IAAI,OAAO,GAAiB,IAAI,CAAC;QACjC,IAAI,MAAmB,CAAC;QACxB,MAAM,OAAO,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;YAC5C,MAAM,GAAG,OAAO,CAAC;QACnB,CAAC,CAAC,CAAC;QACH,uEAAuE;QACvE,wEAAwE;QACxE,4DAA4D;QAC5D,MAAM,IAAI,GAAG;YACX,IAAI,EAAE,CAAC,KAAqB,EAAE,EAAE;gBAC9B,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAClB,IAAI,EAAE,EAAE,CAAC;YACX,CAAC;YACD,IAAI,EAAE,CAAC,KAAY,EAAE,EAAE;gBACrB,OAAO,GAAG,KAAK,CAAC;gBAChB,QAAQ,GAAG,IAAI,CAAC;gBAChB,MAAM,EAAE,CAAC;gBACT,IAAI,EAAE,EAAE,CAAC;YACX,CAAC;YACD,GAAG,EAAE,GAAG,EAAE;gBACR,QAAQ,GAAG,IAAI,CAAC;gBAChB,MAAM,EAAE,CAAC;gBACT,IAAI,EAAE,EAAE,CAAC;YACX,CAAC;YACD,OAAO;SACR,CAAC;QACF,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACrB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC,CAAC;QAE/D,KAAK,SAAS,CAAC,CAAC,MAAM;YACpB,OAAO,IAAI,EAAE,CAAC;gBACZ,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC;oBAAE,MAAM,KAAK,CAAC,KAAK,EAAoB,CAAC;gBAC/D,IAAI,OAAO;oBAAE,MAAM,OAAO,CAAC;gBAC3B,IAAI,QAAQ;oBAAE,OAAO;gBACrB,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;oBAClC,IAAI,GAAG,OAAO,CAAC;gBACjB,CAAC,CAAC,CAAC;gBACH,IAAI,GAAG,IAAI,CAAC;YACd,CAAC;QACH,CAAC;QACD,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,CAAC;IAC9B,CAAC;IAED;;;;;;OAMG;IACH,UAAU,CAAC,SAAS,GAAG,KAAK;QAC1B,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC;QAC3B,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI;YAAE,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;QACnD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC;QACrD,OAAO,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC3C,MAAM,KAAK,GAAG,UAAU,CACtB,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,0BAA0B,CAAC,uCAAuC,CAAC,CAAC,EACrF,SAAS,CACV,CAAC;YACF,KAAK,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE;gBAC1B,YAAY,CAAC,KAAK,CAAC,CAAC;gBACpB,OAAO,EAAE,CAAC;YACZ,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAED,4EAA4E;IAC5E,IAAI,IAAI;QACN,OAAO,IAAI,CAAC,MAAM,KAAK,IAAI,IAAI,IAAI,CAAC,MAAM,CAAC,UAAU,KAAK,CAAC,CAAC;IAC9D,CAAC;IAED,KAAK;QACH,IAAI,CAAC,aAAa,EAAE,CAAC;QACrB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC1B,CAAC;IAEO,UAAU;QAChB,IAAI,IAAI,CAAC,MAAM;YAAE,MAAM,IAAI,CAAC,MAAM,CAAC;QACnC,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClC,MAAM,IAAI,yBAAyB,CAAC,yDAAyD,CAAC,CAAC;QACjG,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,KAAqB;QACjC,IAAI,KAAK,CAAC,IAAI,KAAK,UAAU,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YACvD,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;YAC7B,oEAAoE;YACpE,mEAAmE;YACnE,kCAAkC;YAClC,IAAI,CAAC,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,KAAK,UAAU,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,KAAK,SAAS,CAAC;gBAAE,OAAO;YACrF,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;YACpB,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YACvB,OAAO;QACT,CAAC;QACD,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YAC3C,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;YAC7B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;YACpB,OAAO,CAAC,MAAM,CAAC,IAAI,0BAA0B,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,IAAI,wBAAwB,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;YACvH,OAAO;QACT,CAAC;QACD,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC;QAC3B,IAAI,CAAC,IAAI;YAAE,OAAO,CAAC,oCAAoC;QACvD,IAAI,KAAK,CAAC,IAAI,KAAK,gBAAgB,EAAE,CAAC;YACpC,IAAI,CAAC,GAAG,EAAE,CAAC;YACX,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;YACrB,OAAO;QACT,CAAC;QACD,IAAI,KAAqB,CAAC;QAC1B,IAAI,CAAC;YACH,KAAK,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAC;QAClC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,IAAI,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YACrE,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;YACrB,OAAO;QACT,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACjB,4EAA4E;QAC5E,IAAI,KAAK,CAAC,MAAM,CAAC,IAAI,KAAK,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;YAClE,IAAI,CAAC,GAAG,EAAE,CAAC;YACX,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACvB,CAAC;IACH,CAAC;IAEO,IAAI,CAAC,KAAY,EAAE,QAAQ,GAAG,KAAK;QACzC,IAAI,IAAI,CAAC,MAAM;YAAE,OAAO;QACxB,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;QACpB,IAAI,CAAC,aAAa,EAAE,CAAC;QACrB,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC7B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;QACvB,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC;QAC3B,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACrB,IAAI,IAAI,EAAE,CAAC;YACT,IAAI,QAAQ;gBAAE,IAAI,CAAC,GAAG,EAAE,CAAC;;gBACpB,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACxB,CAAC;IACH,CAAC;CACF;AAED,SAAS,OAAO,CAAC,IAAa;IAC5B,IAAI,IAAI,YAAY,WAAW;QAAE,OAAO,IAAI,UAAU,CAAC,IAAI,CAAC,CAAC;IAC7D,IAAI,IAAI,YAAY,UAAU;QAAE,OAAO,IAAI,CAAC;IAC5C,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAC;AACzE,CAAC;AAED,SAAS,KAAK;IACZ,OAAO,OAAO,WAAW,KAAK,WAAW,CAAC,CAAC,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;AAC7E,CAAC"}
@@ -0,0 +1,107 @@
1
+ import type { RealtimeTurnStream, RealtimeAvatarClient, RealtimeAvatarRequestOptions, RealtimeSessionGrant } from "./client";
2
+ import type { AvatarTurnEventSource } from "./session-socket";
3
+ import type { AvatarSourceKind, RealtimePrepareRequest, RealtimePrepareResponse, RealtimeTurnRequest } from "./types";
4
+ export type AvatarSessionConfig = {
5
+ avatarId: string;
6
+ /**
7
+ * Direct/web-proxy callers can pin the render source; the platform API
8
+ * resolves it server-side from the avatar id, so only `avatarId` is required
9
+ * there.
10
+ */
11
+ sourceKind?: AvatarSourceKind;
12
+ portraitUrl?: string | null;
13
+ sourceVideoUrl?: string | null;
14
+ videoCacheId?: string | null;
15
+ voiceId?: string;
16
+ backgroundId?: string;
17
+ /** Defaults merged into every turn (emotion, speed, playout_delay_ms, …). */
18
+ turnDefaults?: Partial<RealtimeTurnRequest>;
19
+ };
20
+ /**
21
+ * What `chat()`/`speak()` return: an HTTP `RealtimeTurnStream` or a socket
22
+ * turn. Both expose `.events` and both play directly via `AvatarPlayer.play()`.
23
+ */
24
+ export type AvatarTurnSource = RealtimeTurnStream | AvatarTurnEventSource;
25
+ /** How the session is currently running turns. */
26
+ export type AvatarSessionTransport = "socket" | "http";
27
+ export type AvatarSessionConnectResult = {
28
+ grant: RealtimeSessionGrant;
29
+ prepared: RealtimePrepareResponse;
30
+ };
31
+ export type AvatarSessionConnectOptions = {
32
+ signal?: AbortSignal;
33
+ WebSocketImpl?: typeof WebSocket;
34
+ /**
35
+ * Called as soon as the browser-safe session grant is minted, before the
36
+ * socket prepare starts. Media helpers use this to preconnect WHEP while the
37
+ * GPU is still preparing, hiding Cloudflare SDP/ICE latency off the turn path.
38
+ */
39
+ onGrant?: (grant: RealtimeSessionGrant) => void;
40
+ };
41
+ /**
42
+ * A reusable handle for one avatar: carries the avatar id + render source so
43
+ * callers don't repeat them on every turn, and warms the renderer via
44
+ * `prepare()` — or `connect()`, which pins prepare AND every turn to one GPU
45
+ * container over a single WebSocket (the low-latency path) with transparent
46
+ * HTTP fallback when the socket drops.
47
+ *
48
+ * const session = client.session({ avatarId });
49
+ * await session.connect().catch(() => session.prepare()); // socket, else HTTP
50
+ * const stream = await session.chat("who are you?"); // rides the socket when open
51
+ */
52
+ export declare class AvatarSession {
53
+ private readonly client;
54
+ private readonly config;
55
+ private socket;
56
+ constructor(client: RealtimeAvatarClient, config: AvatarSessionConfig);
57
+ get avatarId(): string;
58
+ /** True while a live session socket is open; turns will ride it. */
59
+ get connected(): boolean;
60
+ /** Transport the NEXT turn will use. */
61
+ get transport(): AvatarSessionTransport;
62
+ /**
63
+ * Open a single-socket realtime session and prepare the avatar on it: every
64
+ * subsequent `chat()`/`speak()` runs on the SAME GPU container until
65
+ * `disconnect()`. Throws when the socket cannot be established — callers that
66
+ * want graceful degradation just fall back to `prepare()` (HTTP turns).
67
+ */
68
+ connect(options?: AvatarSessionConnectOptions): Promise<RealtimePrepareResponse>;
69
+ /**
70
+ * Same as `connect()`, but also exposes the session grant minted by the
71
+ * platform. The grant may contain an advisory media-plane plan (for example
72
+ * Cloudflare WHEP playback) that the browser can attach without another API call.
73
+ */
74
+ connectWithGrant(options?: AvatarSessionConnectOptions): Promise<AvatarSessionConnectResult>;
75
+ /** Close the session socket (if any); turns fall back to HTTP. */
76
+ disconnect(): void;
77
+ /**
78
+ * Stop the in-flight socket turn server-side (the GPU stops rendering frames
79
+ * nobody will play) while KEEPING the socket warm for the next turn. The
80
+ * socket is dropped only if the server never confirms. No-op on the HTTP
81
+ * path — abort the fetch via its `signal` instead.
82
+ */
83
+ cancel(): Promise<void>;
84
+ prepare(options?: RealtimeAvatarRequestOptions): Promise<RealtimePrepareResponse>;
85
+ /** The prepare payload this session sends — reusable over a `RealtimeSessionSocket`. */
86
+ prepareRequest(): RealtimePrepareRequest;
87
+ chat(text: string, options?: RealtimeAvatarRequestOptions & {
88
+ history?: RealtimeTurnRequest["messages"];
89
+ /** Per-turn request overrides (e.g. `source_start_frame` for seamless idle handoff). */
90
+ turn?: Partial<RealtimeTurnRequest>;
91
+ }): Promise<AvatarTurnSource>;
92
+ speak(text: string, options?: RealtimeAvatarRequestOptions & {
93
+ turn?: Partial<RealtimeTurnRequest>;
94
+ }): Promise<AvatarTurnSource>;
95
+ /** The chat turn payload — reusable over a `RealtimeSessionSocket`. */
96
+ chatRequest(text: string, history?: RealtimeTurnRequest["messages"]): RealtimeTurnRequest;
97
+ /** The speak-text turn payload — reusable over a `RealtimeSessionSocket`. */
98
+ speakRequest(text: string): RealtimeTurnRequest;
99
+ /**
100
+ * Run the turn on the live socket when one is open. Waits out a previous
101
+ * turn that is still draining its cancel (no-op when idle); a socket that
102
+ * fails the drain is dropped so the caller's turn falls back to HTTP.
103
+ */
104
+ private socketTurn;
105
+ private turnBase;
106
+ }
107
+ //# sourceMappingURL=session.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"session.d.ts","sourceRoot":"","sources":["../src/session.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,4BAA4B,EAAE,oBAAoB,EAAE,MAAM,UAAU,CAAC;AAC7H,OAAO,KAAK,EAAE,qBAAqB,EAAyB,MAAM,kBAAkB,CAAC;AACrF,OAAO,KAAK,EAAE,gBAAgB,EAAE,sBAAsB,EAAE,uBAAuB,EAAE,mBAAmB,EAAE,MAAM,SAAS,CAAC;AAEtH,MAAM,MAAM,mBAAmB,GAAG;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB;;;;OAIG;IACH,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,6EAA6E;IAC7E,YAAY,CAAC,EAAE,OAAO,CAAC,mBAAmB,CAAC,CAAC;CAC7C,CAAC;AAEF;;;GAGG;AACH,MAAM,MAAM,gBAAgB,GAAG,kBAAkB,GAAG,qBAAqB,CAAC;AAE1E,kDAAkD;AAClD,MAAM,MAAM,sBAAsB,GAAG,QAAQ,GAAG,MAAM,CAAC;AAEvD,MAAM,MAAM,0BAA0B,GAAG;IACvC,KAAK,EAAE,oBAAoB,CAAC;IAC5B,QAAQ,EAAE,uBAAuB,CAAC;CACnC,CAAC;AAEF,MAAM,MAAM,2BAA2B,GAAG;IACxC,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,aAAa,CAAC,EAAE,OAAO,SAAS,CAAC;IACjC;;;;OAIG;IACH,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,oBAAoB,KAAK,IAAI,CAAC;CACjD,CAAC;AAEF;;;;;;;;;;GAUG;AACH,qBAAa,aAAa;IAItB,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,MAAM;IAJzB,OAAO,CAAC,MAAM,CAAsC;gBAGjC,MAAM,EAAE,oBAAoB,EAC5B,MAAM,EAAE,mBAAmB;IAG9C,IAAI,QAAQ,IAAI,MAAM,CAErB;IAED,oEAAoE;IACpE,IAAI,SAAS,IAAI,OAAO,CAEvB;IAED,wCAAwC;IACxC,IAAI,SAAS,IAAI,sBAAsB,CAEtC;IAED;;;;;OAKG;IACG,OAAO,CACX,OAAO,GAAE,2BAAgC,GACxC,OAAO,CAAC,uBAAuB,CAAC;IAInC;;;;OAIG;IACG,gBAAgB,CACpB,OAAO,GAAE,2BAAgC,GACxC,OAAO,CAAC,0BAA0B,CAAC;IAmBtC,kEAAkE;IAClE,UAAU,IAAI,IAAI;IAKlB;;;;;OAKG;IACG,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;IAW7B,OAAO,CAAC,OAAO,GAAE,4BAAiC,GAAG,OAAO,CAAC,uBAAuB,CAAC;IAKrF,wFAAwF;IACxF,cAAc,IAAI,sBAAsB;IAUlC,IAAI,CACR,IAAI,EAAE,MAAM,EACZ,OAAO,GAAE,4BAA4B,GAAG;QACtC,OAAO,CAAC,EAAE,mBAAmB,CAAC,UAAU,CAAC,CAAC;QAC1C,wFAAwF;QACxF,IAAI,CAAC,EAAE,OAAO,CAAC,mBAAmB,CAAC,CAAC;KAChC,GACL,OAAO,CAAC,gBAAgB,CAAC;IAOtB,KAAK,CACT,IAAI,EAAE,MAAM,EACZ,OAAO,GAAE,4BAA4B,GAAG;QAAE,IAAI,CAAC,EAAE,OAAO,CAAC,mBAAmB,CAAC,CAAA;KAAO,GACnF,OAAO,CAAC,gBAAgB,CAAC;IAO5B,uEAAuE;IACvE,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,mBAAmB,CAAC,UAAU,CAAC,GAAG,mBAAmB;IASzF,6EAA6E;IAC7E,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,mBAAmB;IAS/C;;;;OAIG;YACW,UAAU;IAaxB,OAAO,CAAC,QAAQ;CAgBjB"}