realtime-avatar 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +132 -0
- package/CLAUDE.md +17 -0
- package/LICENSE +21 -0
- package/README.md +254 -0
- package/dist/api-keys.d.ts +26 -0
- package/dist/api-keys.d.ts.map +1 -0
- package/dist/api-keys.js +88 -0
- package/dist/api-keys.js.map +1 -0
- package/dist/browser/audio.d.ts +65 -0
- package/dist/browser/audio.d.ts.map +1 -0
- package/dist/browser/audio.js +154 -0
- package/dist/browser/audio.js.map +1 -0
- package/dist/browser/boomerang.d.ts +38 -0
- package/dist/browser/boomerang.d.ts.map +1 -0
- package/dist/browser/boomerang.js +85 -0
- package/dist/browser/boomerang.js.map +1 -0
- package/dist/browser/index.d.ts +8 -0
- package/dist/browser/index.d.ts.map +1 -0
- package/dist/browser/index.js +8 -0
- package/dist/browser/index.js.map +1 -0
- package/dist/browser/media-session.d.ts +43 -0
- package/dist/browser/media-session.d.ts.map +1 -0
- package/dist/browser/media-session.js +169 -0
- package/dist/browser/media-session.js.map +1 -0
- package/dist/browser/player.d.ts +162 -0
- package/dist/browser/player.d.ts.map +1 -0
- package/dist/browser/player.js +514 -0
- package/dist/browser/player.js.map +1 -0
- package/dist/browser/view.d.ts +47 -0
- package/dist/browser/view.d.ts.map +1 -0
- package/dist/browser/view.js +7 -0
- package/dist/browser/view.js.map +1 -0
- package/dist/browser/webrtc.d.ts +21 -0
- package/dist/browser/webrtc.d.ts.map +1 -0
- package/dist/browser/webrtc.js +149 -0
- package/dist/browser/webrtc.js.map +1 -0
- package/dist/browser/yuv-canvas.d.ts +13 -0
- package/dist/browser/yuv-canvas.d.ts.map +1 -0
- package/dist/browser/yuv-canvas.js +95 -0
- package/dist/browser/yuv-canvas.js.map +1 -0
- package/dist/client.d.ts +195 -0
- package/dist/client.d.ts.map +1 -0
- package/dist/client.js +440 -0
- package/dist/client.js.map +1 -0
- package/dist/errors.d.ts +33 -0
- package/dist/errors.d.ts.map +1 -0
- package/dist/errors.js +73 -0
- package/dist/errors.js.map +1 -0
- package/dist/generated/openapi.d.ts +1523 -0
- package/dist/generated/openapi.d.ts.map +1 -0
- package/dist/generated/openapi.js +6 -0
- package/dist/generated/openapi.js.map +1 -0
- package/dist/index.d.ts +14 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +15 -0
- package/dist/index.js.map +1 -0
- package/dist/media.d.ts +40 -0
- package/dist/media.d.ts.map +1 -0
- package/dist/media.js +4 -0
- package/dist/media.js.map +1 -0
- package/dist/mux.d.ts +104 -0
- package/dist/mux.d.ts.map +1 -0
- package/dist/mux.js +290 -0
- package/dist/mux.js.map +1 -0
- package/dist/platform.d.ts +163 -0
- package/dist/platform.d.ts.map +1 -0
- package/dist/platform.js +5 -0
- package/dist/platform.js.map +1 -0
- package/dist/react/index.d.ts +5 -0
- package/dist/react/index.d.ts.map +1 -0
- package/dist/react/index.js +5 -0
- package/dist/react/index.js.map +1 -0
- package/dist/react/provider.d.ts +37 -0
- package/dist/react/provider.d.ts.map +1 -0
- package/dist/react/provider.js +33 -0
- package/dist/react/provider.js.map +1 -0
- package/dist/react/realtime.d.ts +74 -0
- package/dist/react/realtime.d.ts.map +1 -0
- package/dist/react/realtime.js +105 -0
- package/dist/react/realtime.js.map +1 -0
- package/dist/react/session.d.ts +91 -0
- package/dist/react/session.d.ts.map +1 -0
- package/dist/react/session.js +322 -0
- package/dist/react/session.js.map +1 -0
- package/dist/react/stage.d.ts +23 -0
- package/dist/react/stage.d.ts.map +1 -0
- package/dist/react/stage.js +62 -0
- package/dist/react/stage.js.map +1 -0
- package/dist/schemas.d.ts +59 -0
- package/dist/schemas.d.ts.map +1 -0
- package/dist/schemas.js +58 -0
- package/dist/schemas.js.map +1 -0
- package/dist/server.d.ts +2 -0
- package/dist/server.d.ts.map +1 -0
- package/dist/server.js +8 -0
- package/dist/server.js.map +1 -0
- package/dist/session-socket.d.ts +96 -0
- package/dist/session-socket.d.ts.map +1 -0
- package/dist/session-socket.js +299 -0
- package/dist/session-socket.js.map +1 -0
- package/dist/session.d.ts +107 -0
- package/dist/session.d.ts.map +1 -0
- package/dist/session.js +192 -0
- package/dist/session.js.map +1 -0
- package/dist/types.d.ts +24 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +2 -0
- package/dist/types.js.map +1 -0
- package/package.json +94 -0
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
import { type AvatarMuxEvent } from "./mux";
|
|
2
|
+
import type { RealtimePrepareRequest, RealtimePrepareResponse, RealtimeTurnRequest } from "./types";
|
|
3
|
+
/** Errors surfaced by the realtime session socket (connection, close, server `error` events). */
|
|
4
|
+
export declare class RealtimeAvatarSessionError extends Error {
|
|
5
|
+
readonly detail: Record<string, unknown> | null;
|
|
6
|
+
constructor(message: string, detail?: Record<string, unknown> | null);
|
|
7
|
+
/** Machine-readable error code from the server `error` event, when present. */
|
|
8
|
+
readonly code?: string;
|
|
9
|
+
/** Mirrors RealtimeAvatarApiError so billing failures over the socket raise the same paywall handling. */
|
|
10
|
+
get isBillingRequired(): boolean;
|
|
11
|
+
}
|
|
12
|
+
/** Structural turn-event source — satisfied by both `RealtimeTurnStream` and {@link RealtimeSessionSocket.turn}. */
|
|
13
|
+
export type AvatarTurnEventSource = {
|
|
14
|
+
events: AsyncGenerator<AvatarMuxEvent>;
|
|
15
|
+
};
|
|
16
|
+
export type RealtimeSessionSocketOptions = {
|
|
17
|
+
/** Full ws(s):// URL of the realtime session endpoint. */
|
|
18
|
+
url: string;
|
|
19
|
+
/**
|
|
20
|
+
* Extra headers, e.g. `{ Authorization: "Bearer ..." }`. Server-side
|
|
21
|
+
* runtimes only (Bun/undici accept headers on `new WebSocket`); browsers
|
|
22
|
+
* cannot set WebSocket headers and must authenticate via a same-origin
|
|
23
|
+
* proxy (cookies) instead.
|
|
24
|
+
*/
|
|
25
|
+
headers?: Record<string, string>;
|
|
26
|
+
/**
|
|
27
|
+
* Short-lived signed session token, appended as `?token=` — the browser
|
|
28
|
+
* auth path (browsers cannot set WebSocket headers). Mint one from your
|
|
29
|
+
* backend; the platform exposes a cookie-authenticated mint endpoint.
|
|
30
|
+
*/
|
|
31
|
+
token?: string;
|
|
32
|
+
/** Override the WebSocket constructor (tests, Node `ws` package). */
|
|
33
|
+
WebSocketImpl?: typeof WebSocket;
|
|
34
|
+
/** Abort handshake + session. */
|
|
35
|
+
signal?: AbortSignal;
|
|
36
|
+
/**
|
|
37
|
+
* Idle keepalive interval in ms; `0` disables. Defaults to 25s. Pings are
|
|
38
|
+
* only sent while no prepare/turn is in flight, and exist so idle proxies
|
|
39
|
+
* never silently drop the warm session — a dead socket costs the next turn
|
|
40
|
+
* the full reconnect-and-prepare path.
|
|
41
|
+
*/
|
|
42
|
+
keepAliveIntervalMs?: number;
|
|
43
|
+
};
|
|
44
|
+
/**
|
|
45
|
+
* One realtime session over one WebSocket — which on the Modal runtime means
|
|
46
|
+
* ONE GPU container for `prepare()` and every subsequent turn. The HTTP
|
|
47
|
+
* endpoints route each request independently, so a turn can land on a
|
|
48
|
+
* container that never saw the prepare and re-pay the avatar load; this
|
|
49
|
+
* socket is the low-latency path.
|
|
50
|
+
*
|
|
51
|
+
* const session = await RealtimeSessionSocket.connect({ url, headers });
|
|
52
|
+
* const prepared = await session.prepare({ source_kind: "source_video", ... });
|
|
53
|
+
* const turn = session.turn({ mode: "speak_text", text: "hi", ... });
|
|
54
|
+
* await player.play(turn); // AvatarPlayer accepts any { events } source
|
|
55
|
+
* session.close();
|
|
56
|
+
*
|
|
57
|
+
* One operation at a time: a second `prepare()`/`turn()` while one is
|
|
58
|
+
* streaming rejects immediately.
|
|
59
|
+
*/
|
|
60
|
+
export declare class RealtimeSessionSocket {
|
|
61
|
+
private readonly socket;
|
|
62
|
+
private pending;
|
|
63
|
+
private turnSink;
|
|
64
|
+
private closed;
|
|
65
|
+
private keepAlive;
|
|
66
|
+
private constructor();
|
|
67
|
+
/** Ping the server on an interval while idle so intermediaries keep the session alive. */
|
|
68
|
+
private startKeepAlive;
|
|
69
|
+
private stopKeepAlive;
|
|
70
|
+
/** Connect and resolve once the server confirms the session (`session_ready`). */
|
|
71
|
+
static connect(options: RealtimeSessionSocketOptions): Promise<RealtimeSessionSocket>;
|
|
72
|
+
/** Warm this container for the avatar; resolves with the standard prepare response. */
|
|
73
|
+
prepare(request: RealtimePrepareRequest): Promise<RealtimePrepareResponse>;
|
|
74
|
+
/** Round-trip latency probe. Resolves with elapsed milliseconds. */
|
|
75
|
+
ping(): Promise<number>;
|
|
76
|
+
/**
|
|
77
|
+
* Run one turn on this session's container. Returns immediately; consume
|
|
78
|
+
* `.events` (ends after `done`). Pass directly to `AvatarPlayer.play()`.
|
|
79
|
+
*/
|
|
80
|
+
turn(request: RealtimeTurnRequest): AvatarTurnEventSource;
|
|
81
|
+
/**
|
|
82
|
+
* Stop the in-flight turn server-side (the GPU stops rendering frames nobody
|
|
83
|
+
* will play) while KEEPING the session socket usable for the next turn.
|
|
84
|
+
* Resolves once the server confirms (`turn_cancelled`) or the turn ends on
|
|
85
|
+
* its own; rejects after `timeoutMs` — close the socket if that happens.
|
|
86
|
+
* No-op when no turn is streaming.
|
|
87
|
+
*/
|
|
88
|
+
cancelTurn(timeoutMs?: number): Promise<void>;
|
|
89
|
+
/** True while the socket is connected and usable for prepare/turn calls. */
|
|
90
|
+
get open(): boolean;
|
|
91
|
+
close(): void;
|
|
92
|
+
private assertIdle;
|
|
93
|
+
private route;
|
|
94
|
+
private fail;
|
|
95
|
+
}
|
|
96
|
+
//# sourceMappingURL=session-socket.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"session-socket.d.ts","sourceRoot":"","sources":["../src/session-socket.ts"],"names":[],"mappings":"AACA,OAAO,EAA0C,KAAK,cAAc,EAAuB,MAAM,OAAO,CAAC;AACzG,OAAO,KAAK,EAAE,sBAAsB,EAAE,uBAAuB,EAAE,mBAAmB,EAAE,MAAM,SAAS,CAAC;AAEpG,iGAAiG;AACjG,qBAAa,0BAA2B,SAAQ,KAAK;IAGjD,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI;gBAD/C,OAAO,EAAE,MAAM,EACN,MAAM,GAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAW;IAOxD,+EAA+E;IAC/E,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IAEvB,0GAA0G;IAC1G,IAAI,iBAAiB,IAAI,OAAO,CAE/B;CACF;AAID,oHAAoH;AACpH,MAAM,MAAM,qBAAqB,GAAG;IAAE,MAAM,EAAE,cAAc,CAAC,cAAc,CAAC,CAAA;CAAE,CAAC;AAE/E,MAAM,MAAM,4BAA4B,GAAG;IACzC,0DAA0D;IAC1D,GAAG,EAAE,MAAM,CAAC;IACZ;;;;;OAKG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC;;;;OAIG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,qEAAqE;IACrE,aAAa,CAAC,EAAE,OAAO,SAAS,CAAC;IACjC,iCAAiC;IACjC,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB;;;;;OAKG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;CAC9B,CAAC;AAEF;;;;;;;;;;;;;;;GAeG;AACH,qBAAa,qBAAqB;IAgBZ,OAAO,CAAC,QAAQ,CAAC,MAAM;IAf3C,OAAO,CAAC,OAAO,CAIC;IAChB,OAAO,CAAC,QAAQ,CAMA;IAChB,OAAO,CAAC,MAAM,CAAsB;IACpC,OAAO,CAAC,SAAS,CAA+C;IAEhE,OAAO;IAEP,0FAA0F;IAC1F,OAAO,CAAC,cAAc;IAetB,OAAO,CAAC,aAAa;IAOrB,kFAAkF;IAClF,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,4BAA4B,GAAG,OAAO,CAAC,qBAAqB,CAAC;IA+DrF,uFAAuF;IACvF,OAAO,CAAC,OAAO,EAAE,sBAAsB,GAAG,OAAO,CAAC,uBAAuB,CAAC;IAS1E,oEAAoE;IACpE,IAAI,IAAI,OAAO,CAAC,MAAM,CAAC;IAUvB;;;OAGG;IACH,IAAI,CAAC,OAAO,EAAE,mBAAmB,GAAG,qBAAqB;IAgDzD;;;;;;OAMG;IACH,UAAU,CAAC,SAAS,SAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IAgB5C,4EAA4E;IAC5E,IAAI,IAAI,IAAI,OAAO,CAElB;IAED,KAAK,IAAI,IAAI;IAKb,OAAO,CAAC,UAAU;IAOlB,OAAO,CAAC,KAAK;IAwCb,OAAO,CAAC,IAAI;CAcb"}
|
|
@@ -0,0 +1,299 @@
|
|
|
1
|
+
import { RealtimeAvatarConfigError } from "./errors";
|
|
2
|
+
import { decodeAvatarMuxFrame, toAvatarMuxEvent } from "./mux";
|
|
3
|
+
/** Errors surfaced by the realtime session socket (connection, close, server `error` events). */
|
|
4
|
+
export class RealtimeAvatarSessionError extends Error {
|
|
5
|
+
detail;
|
|
6
|
+
constructor(message, detail = null) {
|
|
7
|
+
super(message);
|
|
8
|
+
this.detail = detail;
|
|
9
|
+
this.name = "RealtimeAvatarSessionError";
|
|
10
|
+
this.code = typeof detail?.code === "string" ? detail.code : undefined;
|
|
11
|
+
}
|
|
12
|
+
/** Machine-readable error code from the server `error` event, when present. */
|
|
13
|
+
code;
|
|
14
|
+
/** Mirrors RealtimeAvatarApiError so billing failures over the socket raise the same paywall handling. */
|
|
15
|
+
get isBillingRequired() {
|
|
16
|
+
return this.code === "insufficient_credits" || this.code === "spend_limit_exceeded";
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* One realtime session over one WebSocket — which on the Modal runtime means
|
|
21
|
+
* ONE GPU container for `prepare()` and every subsequent turn. The HTTP
|
|
22
|
+
* endpoints route each request independently, so a turn can land on a
|
|
23
|
+
* container that never saw the prepare and re-pay the avatar load; this
|
|
24
|
+
* socket is the low-latency path.
|
|
25
|
+
*
|
|
26
|
+
* const session = await RealtimeSessionSocket.connect({ url, headers });
|
|
27
|
+
* const prepared = await session.prepare({ source_kind: "source_video", ... });
|
|
28
|
+
* const turn = session.turn({ mode: "speak_text", text: "hi", ... });
|
|
29
|
+
* await player.play(turn); // AvatarPlayer accepts any { events } source
|
|
30
|
+
* session.close();
|
|
31
|
+
*
|
|
32
|
+
* One operation at a time: a second `prepare()`/`turn()` while one is
|
|
33
|
+
* streaming rejects immediately.
|
|
34
|
+
*/
|
|
35
|
+
export class RealtimeSessionSocket {
|
|
36
|
+
socket;
|
|
37
|
+
pending = null;
|
|
38
|
+
turnSink = null;
|
|
39
|
+
closed = null;
|
|
40
|
+
keepAlive = null;
|
|
41
|
+
constructor(socket) {
|
|
42
|
+
this.socket = socket;
|
|
43
|
+
}
|
|
44
|
+
/** Ping the server on an interval while idle so intermediaries keep the session alive. */
|
|
45
|
+
startKeepAlive(intervalMs) {
|
|
46
|
+
if (intervalMs <= 0)
|
|
47
|
+
return;
|
|
48
|
+
const timer = setInterval(() => {
|
|
49
|
+
if (!this.open || this.pending || this.turnSink)
|
|
50
|
+
return;
|
|
51
|
+
try {
|
|
52
|
+
this.socket.send(JSON.stringify({ type: "ping" }));
|
|
53
|
+
}
|
|
54
|
+
catch {
|
|
55
|
+
// The close handler tears the session down; nothing to do here.
|
|
56
|
+
}
|
|
57
|
+
}, intervalMs);
|
|
58
|
+
// Node timers hold the event loop open; a keepalive must not.
|
|
59
|
+
timer.unref?.();
|
|
60
|
+
this.keepAlive = timer;
|
|
61
|
+
}
|
|
62
|
+
stopKeepAlive() {
|
|
63
|
+
if (this.keepAlive !== null) {
|
|
64
|
+
clearInterval(this.keepAlive);
|
|
65
|
+
this.keepAlive = null;
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
/** Connect and resolve once the server confirms the session (`session_ready`). */
|
|
69
|
+
static connect(options) {
|
|
70
|
+
const WebSocketCtor = options.WebSocketImpl ?? globalThis.WebSocket;
|
|
71
|
+
if (!WebSocketCtor) {
|
|
72
|
+
throw new RealtimeAvatarConfigError("A WebSocket implementation is required for realtime sessions");
|
|
73
|
+
}
|
|
74
|
+
const url = options.token
|
|
75
|
+
? `${options.url}${options.url.includes("?") ? "&" : "?"}token=${encodeURIComponent(options.token)}`
|
|
76
|
+
: options.url;
|
|
77
|
+
const socket = options.headers
|
|
78
|
+
? new WebSocketCtor(url, { headers: options.headers })
|
|
79
|
+
: new WebSocketCtor(url);
|
|
80
|
+
socket.binaryType = "arraybuffer";
|
|
81
|
+
const session = new RealtimeSessionSocket(socket);
|
|
82
|
+
return new Promise((resolve, reject) => {
|
|
83
|
+
let settled = false;
|
|
84
|
+
const settle = (fn) => {
|
|
85
|
+
if (!settled) {
|
|
86
|
+
settled = true;
|
|
87
|
+
fn();
|
|
88
|
+
}
|
|
89
|
+
};
|
|
90
|
+
options.signal?.addEventListener("abort", () => {
|
|
91
|
+
socket.close();
|
|
92
|
+
settle(() => reject(new RealtimeAvatarSessionError("Realtime session aborted")));
|
|
93
|
+
}, { once: true });
|
|
94
|
+
socket.addEventListener("message", (message) => {
|
|
95
|
+
let frame;
|
|
96
|
+
try {
|
|
97
|
+
frame = decodeAvatarMuxFrame(toBytes(message.data));
|
|
98
|
+
}
|
|
99
|
+
catch (error) {
|
|
100
|
+
session.fail(error instanceof Error ? error : new Error(String(error)));
|
|
101
|
+
return;
|
|
102
|
+
}
|
|
103
|
+
if (frame.type === "session_ready") {
|
|
104
|
+
settle(() => {
|
|
105
|
+
session.startKeepAlive(options.keepAliveIntervalMs ?? 25_000);
|
|
106
|
+
resolve(session);
|
|
107
|
+
});
|
|
108
|
+
return;
|
|
109
|
+
}
|
|
110
|
+
session.route(frame);
|
|
111
|
+
});
|
|
112
|
+
socket.addEventListener("close", (event) => {
|
|
113
|
+
session.fail(new RealtimeAvatarSessionError(`Realtime session closed (${event.code}${event.reason ? `: ${event.reason}` : ""})`),
|
|
114
|
+
/* graceful */ event.code === 1000);
|
|
115
|
+
settle(() => reject(new RealtimeAvatarSessionError("Realtime session closed before it was ready")));
|
|
116
|
+
});
|
|
117
|
+
socket.addEventListener("error", () => {
|
|
118
|
+
settle(() => reject(new RealtimeAvatarSessionError("Realtime session connection failed")));
|
|
119
|
+
});
|
|
120
|
+
});
|
|
121
|
+
}
|
|
122
|
+
/** Warm this container for the avatar; resolves with the standard prepare response. */
|
|
123
|
+
prepare(request) {
|
|
124
|
+
this.assertIdle();
|
|
125
|
+
const frame = new Promise((resolve, reject) => {
|
|
126
|
+
this.pending = { resolve, reject, kind: "prepare" };
|
|
127
|
+
});
|
|
128
|
+
this.socket.send(JSON.stringify({ type: "prepare", ...request }));
|
|
129
|
+
return frame.then(({ header }) => header);
|
|
130
|
+
}
|
|
131
|
+
/** Round-trip latency probe. Resolves with elapsed milliseconds. */
|
|
132
|
+
ping() {
|
|
133
|
+
this.assertIdle();
|
|
134
|
+
const started = nowMs();
|
|
135
|
+
const frame = new Promise((resolve, reject) => {
|
|
136
|
+
this.pending = { resolve, reject, kind: "pong" };
|
|
137
|
+
});
|
|
138
|
+
this.socket.send(JSON.stringify({ type: "ping" }));
|
|
139
|
+
return frame.then(() => nowMs() - started);
|
|
140
|
+
}
|
|
141
|
+
/**
|
|
142
|
+
* Run one turn on this session's container. Returns immediately; consume
|
|
143
|
+
* `.events` (ends after `done`). Pass directly to `AvatarPlayer.play()`.
|
|
144
|
+
*/
|
|
145
|
+
turn(request) {
|
|
146
|
+
this.assertIdle();
|
|
147
|
+
const queue = [];
|
|
148
|
+
let wake = null;
|
|
149
|
+
let finished = false;
|
|
150
|
+
let failure = null;
|
|
151
|
+
let settle;
|
|
152
|
+
const settled = new Promise((resolve) => {
|
|
153
|
+
settle = resolve;
|
|
154
|
+
});
|
|
155
|
+
// Sink lifetime is owned by `route()`/`fail()` (cleared on done/error/
|
|
156
|
+
// turn_cancelled/socket loss), NOT by the consumer: a player that stops
|
|
157
|
+
// reading mid-turn must not orphan the cancel confirmation.
|
|
158
|
+
const sink = {
|
|
159
|
+
push: (event) => {
|
|
160
|
+
queue.push(event);
|
|
161
|
+
wake?.();
|
|
162
|
+
},
|
|
163
|
+
fail: (error) => {
|
|
164
|
+
failure = error;
|
|
165
|
+
finished = true;
|
|
166
|
+
settle();
|
|
167
|
+
wake?.();
|
|
168
|
+
},
|
|
169
|
+
end: () => {
|
|
170
|
+
finished = true;
|
|
171
|
+
settle();
|
|
172
|
+
wake?.();
|
|
173
|
+
},
|
|
174
|
+
settled,
|
|
175
|
+
};
|
|
176
|
+
this.turnSink = sink;
|
|
177
|
+
this.socket.send(JSON.stringify({ type: "turn", ...request }));
|
|
178
|
+
async function* events() {
|
|
179
|
+
while (true) {
|
|
180
|
+
while (queue.length > 0)
|
|
181
|
+
yield queue.shift();
|
|
182
|
+
if (failure)
|
|
183
|
+
throw failure;
|
|
184
|
+
if (finished)
|
|
185
|
+
return;
|
|
186
|
+
await new Promise((resolve) => {
|
|
187
|
+
wake = resolve;
|
|
188
|
+
});
|
|
189
|
+
wake = null;
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
return { events: events() };
|
|
193
|
+
}
|
|
194
|
+
/**
|
|
195
|
+
* Stop the in-flight turn server-side (the GPU stops rendering frames nobody
|
|
196
|
+
* will play) while KEEPING the session socket usable for the next turn.
|
|
197
|
+
* Resolves once the server confirms (`turn_cancelled`) or the turn ends on
|
|
198
|
+
* its own; rejects after `timeoutMs` — close the socket if that happens.
|
|
199
|
+
* No-op when no turn is streaming.
|
|
200
|
+
*/
|
|
201
|
+
cancelTurn(timeoutMs = 5_000) {
|
|
202
|
+
const sink = this.turnSink;
|
|
203
|
+
if (this.closed || !sink)
|
|
204
|
+
return Promise.resolve();
|
|
205
|
+
this.socket.send(JSON.stringify({ type: "cancel" }));
|
|
206
|
+
return new Promise((resolve, reject) => {
|
|
207
|
+
const timer = setTimeout(() => reject(new RealtimeAvatarSessionError("Turn cancel was not confirmed in time")), timeoutMs);
|
|
208
|
+
void sink.settled.then(() => {
|
|
209
|
+
clearTimeout(timer);
|
|
210
|
+
resolve();
|
|
211
|
+
});
|
|
212
|
+
});
|
|
213
|
+
}
|
|
214
|
+
/** True while the socket is connected and usable for prepare/turn calls. */
|
|
215
|
+
get open() {
|
|
216
|
+
return this.closed === null && this.socket.readyState === 1;
|
|
217
|
+
}
|
|
218
|
+
close() {
|
|
219
|
+
this.stopKeepAlive();
|
|
220
|
+
this.socket.close(1000);
|
|
221
|
+
}
|
|
222
|
+
assertIdle() {
|
|
223
|
+
if (this.closed)
|
|
224
|
+
throw this.closed;
|
|
225
|
+
if (this.pending || this.turnSink) {
|
|
226
|
+
throw new RealtimeAvatarConfigError("The realtime session already has an operation in flight");
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
route(frame) {
|
|
230
|
+
if (frame.type === "prepared" || frame.type === "pong") {
|
|
231
|
+
const pending = this.pending;
|
|
232
|
+
// Drop strays: keepalive pongs nobody awaits, and frames whose kind
|
|
233
|
+
// doesn't match the pending operation (a pong must never resolve a
|
|
234
|
+
// prepare with the wrong header).
|
|
235
|
+
if (!pending || (frame.type === "prepared") !== (pending.kind === "prepare"))
|
|
236
|
+
return;
|
|
237
|
+
this.pending = null;
|
|
238
|
+
pending.resolve(frame);
|
|
239
|
+
return;
|
|
240
|
+
}
|
|
241
|
+
if (frame.type === "error" && this.pending) {
|
|
242
|
+
const pending = this.pending;
|
|
243
|
+
this.pending = null;
|
|
244
|
+
pending.reject(new RealtimeAvatarSessionError(String(frame.header.message ?? "Realtime session error"), frame.header));
|
|
245
|
+
return;
|
|
246
|
+
}
|
|
247
|
+
const sink = this.turnSink;
|
|
248
|
+
if (!sink)
|
|
249
|
+
return; // stray event outside any operation
|
|
250
|
+
if (frame.type === "turn_cancelled") {
|
|
251
|
+
sink.end();
|
|
252
|
+
this.turnSink = null;
|
|
253
|
+
return;
|
|
254
|
+
}
|
|
255
|
+
let event;
|
|
256
|
+
try {
|
|
257
|
+
event = toAvatarMuxEvent(frame);
|
|
258
|
+
}
|
|
259
|
+
catch (error) {
|
|
260
|
+
sink.fail(error instanceof Error ? error : new Error(String(error)));
|
|
261
|
+
this.turnSink = null;
|
|
262
|
+
return;
|
|
263
|
+
}
|
|
264
|
+
sink.push(event);
|
|
265
|
+
// `error` terminates the turn but keeps the session usable; `done` ends it.
|
|
266
|
+
if (event.header.type === "done" || event.header.type === "error") {
|
|
267
|
+
sink.end();
|
|
268
|
+
this.turnSink = null;
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
fail(error, graceful = false) {
|
|
272
|
+
if (this.closed)
|
|
273
|
+
return;
|
|
274
|
+
this.closed = error;
|
|
275
|
+
this.stopKeepAlive();
|
|
276
|
+
const pending = this.pending;
|
|
277
|
+
this.pending = null;
|
|
278
|
+
pending?.reject(error);
|
|
279
|
+
const sink = this.turnSink;
|
|
280
|
+
this.turnSink = null;
|
|
281
|
+
if (sink) {
|
|
282
|
+
if (graceful)
|
|
283
|
+
sink.end();
|
|
284
|
+
else
|
|
285
|
+
sink.fail(error);
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
function toBytes(data) {
|
|
290
|
+
if (data instanceof ArrayBuffer)
|
|
291
|
+
return new Uint8Array(data);
|
|
292
|
+
if (data instanceof Uint8Array)
|
|
293
|
+
return data;
|
|
294
|
+
throw new Error("Realtime session expected binary WebSocket messages");
|
|
295
|
+
}
|
|
296
|
+
function nowMs() {
|
|
297
|
+
return typeof performance !== "undefined" ? performance.now() : Date.now();
|
|
298
|
+
}
|
|
299
|
+
//# sourceMappingURL=session-socket.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"session-socket.js","sourceRoot":"","sources":["../src/session-socket.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,MAAM,UAAU,CAAC;AACrD,OAAO,EAAE,oBAAoB,EAAE,gBAAgB,EAA4C,MAAM,OAAO,CAAC;AAGzG,iGAAiG;AACjG,MAAM,OAAO,0BAA2B,SAAQ,KAAK;IAGxC;IAFX,YACE,OAAe,EACN,SAAyC,IAAI;QAEtD,KAAK,CAAC,OAAO,CAAC,CAAC;QAFN,WAAM,GAAN,MAAM,CAAuC;QAGtD,IAAI,CAAC,IAAI,GAAG,4BAA4B,CAAC;QACzC,IAAI,CAAC,IAAI,GAAG,OAAO,MAAM,EAAE,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;IACzE,CAAC;IAED,+EAA+E;IACtE,IAAI,CAAU;IAEvB,0GAA0G;IAC1G,IAAI,iBAAiB;QACnB,OAAO,IAAI,CAAC,IAAI,KAAK,sBAAsB,IAAI,IAAI,CAAC,IAAI,KAAK,sBAAsB,CAAC;IACtF,CAAC;CACF;AAoCD;;;;;;;;;;;;;;;GAeG;AACH,MAAM,OAAO,qBAAqB;IAgBK;IAf7B,OAAO,GAIJ,IAAI,CAAC;IACR,QAAQ,GAML,IAAI,CAAC;IACR,MAAM,GAAiB,IAAI,CAAC;IAC5B,SAAS,GAA0C,IAAI,CAAC;IAEhE,YAAqC,MAAiB;QAAjB,WAAM,GAAN,MAAM,CAAW;IAAG,CAAC;IAE1D,0FAA0F;IAClF,cAAc,CAAC,UAAkB;QACvC,IAAI,UAAU,IAAI,CAAC;YAAE,OAAO;QAC5B,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE;YAC7B,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,QAAQ;gBAAE,OAAO;YACxD,IAAI,CAAC;gBACH,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;YACrD,CAAC;YAAC,MAAM,CAAC;gBACP,gEAAgE;YAClE,CAAC;QACH,CAAC,EAAE,UAAU,CAAC,CAAC;QACf,8DAA8D;QAC7D,KAA2C,CAAC,KAAK,EAAE,EAAE,CAAC;QACvD,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;IACzB,CAAC;IAEO,aAAa;QACnB,IAAI,IAAI,CAAC,SAAS,KAAK,IAAI,EAAE,CAAC;YAC5B,aAAa,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC9B,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACxB,CAAC;IACH,CAAC;IAED,kFAAkF;IAClF,MAAM,CAAC,OAAO,CAAC,OAAqC;QAClD,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,UAAU,CAAC,SAAS,CAAC;QACpE,IAAI,CAAC,aAAa,EAAE,CAAC;YACnB,MAAM,IAAI,yBAAyB,CAAC,8DAA8D,CAAC,CAAC;QACtG,CAAC;QACD,MAAM,GAAG,GAAG,OAAO,CAAC,KAAK;YACvB,CAAC,CAAC,GAAG,OAAO,CAAC,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,SAAS,kBAAkB,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE;YACpG,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC;QAChB,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO;YAC5B,CAAC,CAAC,IAAK,aAAsG,CACzG,GAAG,EACH,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAC7B;YACH,CAAC,CAAC,IAAI,aAAa,CAAC,GAAG,CAAC,CAAC;QAC3B,MAAM,CAAC,UAAU,GAAG,aAAa,CAAC;QAClC,MAAM,OAAO,GAAG,IAAI,qBAAqB,CAAC,MAAM,CAAC,CAAC;QAElD,OAAO,IAAI,OAAO,CAAwB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC5D,IAAI,OAAO,GAAG,KAAK,CAAC;YACpB,MAAM,MAAM,GAAG,CAAC,EAAc,EAAE,EAAE;gBAChC,IAAI,CAAC,OAAO,EAAE,CAAC;oBACb,OAAO,GAAG,IAAI,CAAC;oBACf,EAAE,EAAE,CAAC;gBACP,CAAC;YACH,CAAC,CAAC;YACF,OAAO,CAAC,MAAM,EAAE,gBAAgB,CAC9B,OAAO,EACP,GAAG,EAAE;gBACH,MAAM,CAAC,KAAK,EAAE,CAAC;gBACf,MAAM,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,0BAA0B,CAAC,0BAA0B,CAAC,CAAC,CAAC,CAAC;YACnF,CAAC,EACD,EAAE,IAAI,EAAE,IAAI,EAAE,CACf,CAAC;YACF,MAAM,CAAC,gBAAgB,CAAC,SAAS,EAAE,CAAC,OAAqB,EAAE,EAAE;gBAC3D,IAAI,KAAqB,CAAC;gBAC1B,IAAI,CAAC;oBACH,KAAK,GAAG,oBAAoB,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;gBACtD,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,OAAO,CAAC,IAAI,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;oBACxE,OAAO;gBACT,CAAC;gBACD,IAAI,KAAK,CAAC,IAAI,KAAK,eAAe,EAAE,CAAC;oBACnC,MAAM,CAAC,GAAG,EAAE;wBACV,OAAO,CAAC,cAAc,CAAC,OAAO,CAAC,mBAAmB,IAAI,MAAM,CAAC,CAAC;wBAC9D,OAAO,CAAC,OAAO,CAAC,CAAC;oBACnB,CAAC,CAAC,CAAC;oBACH,OAAO;gBACT,CAAC;gBACD,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YACvB,CAAC,CAAC,CAAC;YACH,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,CAAC,KAAiB,EAAE,EAAE;gBACrD,OAAO,CAAC,IAAI,CACV,IAAI,0BAA0B,CAAC,4BAA4B,KAAK,CAAC,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC;gBACnH,cAAc,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CACnC,CAAC;gBACF,MAAM,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,0BAA0B,CAAC,6CAA6C,CAAC,CAAC,CAAC,CAAC;YACtG,CAAC,CAAC,CAAC;YACH,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE;gBACpC,MAAM,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,0BAA0B,CAAC,oCAAoC,CAAC,CAAC,CAAC,CAAC;YAC7F,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAED,uFAAuF;IACvF,OAAO,CAAC,OAA+B;QACrC,IAAI,CAAC,UAAU,EAAE,CAAC;QAClB,MAAM,KAAK,GAAG,IAAI,OAAO,CAAiB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC5D,IAAI,CAAC,OAAO,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC;QACtD,CAAC,CAAC,CAAC;QACH,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC,CAAC;QAClE,OAAO,KAAK,CAAC,IAAI,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,MAA4C,CAAC,CAAC;IAClF,CAAC;IAED,oEAAoE;IACpE,IAAI;QACF,IAAI,CAAC,UAAU,EAAE,CAAC;QAClB,MAAM,OAAO,GAAG,KAAK,EAAE,CAAC;QACxB,MAAM,KAAK,GAAG,IAAI,OAAO,CAAiB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC5D,IAAI,CAAC,OAAO,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;QACnD,CAAC,CAAC,CAAC;QACH,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;QACnD,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,KAAK,EAAE,GAAG,OAAO,CAAC,CAAC;IAC7C,CAAC;IAED;;;OAGG;IACH,IAAI,CAAC,OAA4B;QAC/B,IAAI,CAAC,UAAU,EAAE,CAAC;QAClB,MAAM,KAAK,GAAqB,EAAE,CAAC;QACnC,IAAI,IAAI,GAAwB,IAAI,CAAC;QACrC,IAAI,QAAQ,GAAG,KAAK,CAAC;QACrB,IAAI,OAAO,GAAiB,IAAI,CAAC;QACjC,IAAI,MAAmB,CAAC;QACxB,MAAM,OAAO,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;YAC5C,MAAM,GAAG,OAAO,CAAC;QACnB,CAAC,CAAC,CAAC;QACH,uEAAuE;QACvE,wEAAwE;QACxE,4DAA4D;QAC5D,MAAM,IAAI,GAAG;YACX,IAAI,EAAE,CAAC,KAAqB,EAAE,EAAE;gBAC9B,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAClB,IAAI,EAAE,EAAE,CAAC;YACX,CAAC;YACD,IAAI,EAAE,CAAC,KAAY,EAAE,EAAE;gBACrB,OAAO,GAAG,KAAK,CAAC;gBAChB,QAAQ,GAAG,IAAI,CAAC;gBAChB,MAAM,EAAE,CAAC;gBACT,IAAI,EAAE,EAAE,CAAC;YACX,CAAC;YACD,GAAG,EAAE,GAAG,EAAE;gBACR,QAAQ,GAAG,IAAI,CAAC;gBAChB,MAAM,EAAE,CAAC;gBACT,IAAI,EAAE,EAAE,CAAC;YACX,CAAC;YACD,OAAO;SACR,CAAC;QACF,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACrB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC,CAAC;QAE/D,KAAK,SAAS,CAAC,CAAC,MAAM;YACpB,OAAO,IAAI,EAAE,CAAC;gBACZ,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC;oBAAE,MAAM,KAAK,CAAC,KAAK,EAAoB,CAAC;gBAC/D,IAAI,OAAO;oBAAE,MAAM,OAAO,CAAC;gBAC3B,IAAI,QAAQ;oBAAE,OAAO;gBACrB,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;oBAClC,IAAI,GAAG,OAAO,CAAC;gBACjB,CAAC,CAAC,CAAC;gBACH,IAAI,GAAG,IAAI,CAAC;YACd,CAAC;QACH,CAAC;QACD,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,CAAC;IAC9B,CAAC;IAED;;;;;;OAMG;IACH,UAAU,CAAC,SAAS,GAAG,KAAK;QAC1B,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC;QAC3B,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI;YAAE,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;QACnD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC;QACrD,OAAO,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC3C,MAAM,KAAK,GAAG,UAAU,CACtB,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,0BAA0B,CAAC,uCAAuC,CAAC,CAAC,EACrF,SAAS,CACV,CAAC;YACF,KAAK,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE;gBAC1B,YAAY,CAAC,KAAK,CAAC,CAAC;gBACpB,OAAO,EAAE,CAAC;YACZ,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAED,4EAA4E;IAC5E,IAAI,IAAI;QACN,OAAO,IAAI,CAAC,MAAM,KAAK,IAAI,IAAI,IAAI,CAAC,MAAM,CAAC,UAAU,KAAK,CAAC,CAAC;IAC9D,CAAC;IAED,KAAK;QACH,IAAI,CAAC,aAAa,EAAE,CAAC;QACrB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC1B,CAAC;IAEO,UAAU;QAChB,IAAI,IAAI,CAAC,MAAM;YAAE,MAAM,IAAI,CAAC,MAAM,CAAC;QACnC,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClC,MAAM,IAAI,yBAAyB,CAAC,yDAAyD,CAAC,CAAC;QACjG,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,KAAqB;QACjC,IAAI,KAAK,CAAC,IAAI,KAAK,UAAU,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YACvD,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;YAC7B,oEAAoE;YACpE,mEAAmE;YACnE,kCAAkC;YAClC,IAAI,CAAC,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,KAAK,UAAU,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,KAAK,SAAS,CAAC;gBAAE,OAAO;YACrF,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;YACpB,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YACvB,OAAO;QACT,CAAC;QACD,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YAC3C,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;YAC7B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;YACpB,OAAO,CAAC,MAAM,CAAC,IAAI,0BAA0B,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,IAAI,wBAAwB,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;YACvH,OAAO;QACT,CAAC;QACD,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC;QAC3B,IAAI,CAAC,IAAI;YAAE,OAAO,CAAC,oCAAoC;QACvD,IAAI,KAAK,CAAC,IAAI,KAAK,gBAAgB,EAAE,CAAC;YACpC,IAAI,CAAC,GAAG,EAAE,CAAC;YACX,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;YACrB,OAAO;QACT,CAAC;QACD,IAAI,KAAqB,CAAC;QAC1B,IAAI,CAAC;YACH,KAAK,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAC;QAClC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,IAAI,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YACrE,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;YACrB,OAAO;QACT,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACjB,4EAA4E;QAC5E,IAAI,KAAK,CAAC,MAAM,CAAC,IAAI,KAAK,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;YAClE,IAAI,CAAC,GAAG,EAAE,CAAC;YACX,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACvB,CAAC;IACH,CAAC;IAEO,IAAI,CAAC,KAAY,EAAE,QAAQ,GAAG,KAAK;QACzC,IAAI,IAAI,CAAC,MAAM;YAAE,OAAO;QACxB,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;QACpB,IAAI,CAAC,aAAa,EAAE,CAAC;QACrB,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC7B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;QACvB,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC;QAC3B,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACrB,IAAI,IAAI,EAAE,CAAC;YACT,IAAI,QAAQ;gBAAE,IAAI,CAAC,GAAG,EAAE,CAAC;;gBACpB,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACxB,CAAC;IACH,CAAC;CACF;AAED,SAAS,OAAO,CAAC,IAAa;IAC5B,IAAI,IAAI,YAAY,WAAW;QAAE,OAAO,IAAI,UAAU,CAAC,IAAI,CAAC,CAAC;IAC7D,IAAI,IAAI,YAAY,UAAU;QAAE,OAAO,IAAI,CAAC;IAC5C,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAC;AACzE,CAAC;AAED,SAAS,KAAK;IACZ,OAAO,OAAO,WAAW,KAAK,WAAW,CAAC,CAAC,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;AAC7E,CAAC"}
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
import type { RealtimeTurnStream, RealtimeAvatarClient, RealtimeAvatarRequestOptions, RealtimeSessionGrant } from "./client";
|
|
2
|
+
import type { AvatarTurnEventSource } from "./session-socket";
|
|
3
|
+
import type { AvatarSourceKind, RealtimePrepareRequest, RealtimePrepareResponse, RealtimeTurnRequest } from "./types";
|
|
4
|
+
export type AvatarSessionConfig = {
|
|
5
|
+
avatarId: string;
|
|
6
|
+
/**
|
|
7
|
+
* Direct/web-proxy callers can pin the render source; the platform API
|
|
8
|
+
* resolves it server-side from the avatar id, so only `avatarId` is required
|
|
9
|
+
* there.
|
|
10
|
+
*/
|
|
11
|
+
sourceKind?: AvatarSourceKind;
|
|
12
|
+
portraitUrl?: string | null;
|
|
13
|
+
sourceVideoUrl?: string | null;
|
|
14
|
+
videoCacheId?: string | null;
|
|
15
|
+
voiceId?: string;
|
|
16
|
+
backgroundId?: string;
|
|
17
|
+
/** Defaults merged into every turn (emotion, speed, playout_delay_ms, …). */
|
|
18
|
+
turnDefaults?: Partial<RealtimeTurnRequest>;
|
|
19
|
+
};
|
|
20
|
+
/**
|
|
21
|
+
* What `chat()`/`speak()` return: an HTTP `RealtimeTurnStream` or a socket
|
|
22
|
+
* turn. Both expose `.events` and both play directly via `AvatarPlayer.play()`.
|
|
23
|
+
*/
|
|
24
|
+
export type AvatarTurnSource = RealtimeTurnStream | AvatarTurnEventSource;
|
|
25
|
+
/** How the session is currently running turns. */
|
|
26
|
+
export type AvatarSessionTransport = "socket" | "http";
|
|
27
|
+
export type AvatarSessionConnectResult = {
|
|
28
|
+
grant: RealtimeSessionGrant;
|
|
29
|
+
prepared: RealtimePrepareResponse;
|
|
30
|
+
};
|
|
31
|
+
export type AvatarSessionConnectOptions = {
|
|
32
|
+
signal?: AbortSignal;
|
|
33
|
+
WebSocketImpl?: typeof WebSocket;
|
|
34
|
+
/**
|
|
35
|
+
* Called as soon as the browser-safe session grant is minted, before the
|
|
36
|
+
* socket prepare starts. Media helpers use this to preconnect WHEP while the
|
|
37
|
+
* GPU is still preparing, hiding Cloudflare SDP/ICE latency off the turn path.
|
|
38
|
+
*/
|
|
39
|
+
onGrant?: (grant: RealtimeSessionGrant) => void;
|
|
40
|
+
};
|
|
41
|
+
/**
|
|
42
|
+
* A reusable handle for one avatar: carries the avatar id + render source so
|
|
43
|
+
* callers don't repeat them on every turn, and warms the renderer via
|
|
44
|
+
* `prepare()` — or `connect()`, which pins prepare AND every turn to one GPU
|
|
45
|
+
* container over a single WebSocket (the low-latency path) with transparent
|
|
46
|
+
* HTTP fallback when the socket drops.
|
|
47
|
+
*
|
|
48
|
+
* const session = client.session({ avatarId });
|
|
49
|
+
* await session.connect().catch(() => session.prepare()); // socket, else HTTP
|
|
50
|
+
* const stream = await session.chat("who are you?"); // rides the socket when open
|
|
51
|
+
*/
|
|
52
|
+
export declare class AvatarSession {
|
|
53
|
+
private readonly client;
|
|
54
|
+
private readonly config;
|
|
55
|
+
private socket;
|
|
56
|
+
constructor(client: RealtimeAvatarClient, config: AvatarSessionConfig);
|
|
57
|
+
get avatarId(): string;
|
|
58
|
+
/** True while a live session socket is open; turns will ride it. */
|
|
59
|
+
get connected(): boolean;
|
|
60
|
+
/** Transport the NEXT turn will use. */
|
|
61
|
+
get transport(): AvatarSessionTransport;
|
|
62
|
+
/**
|
|
63
|
+
* Open a single-socket realtime session and prepare the avatar on it: every
|
|
64
|
+
* subsequent `chat()`/`speak()` runs on the SAME GPU container until
|
|
65
|
+
* `disconnect()`. Throws when the socket cannot be established — callers that
|
|
66
|
+
* want graceful degradation just fall back to `prepare()` (HTTP turns).
|
|
67
|
+
*/
|
|
68
|
+
connect(options?: AvatarSessionConnectOptions): Promise<RealtimePrepareResponse>;
|
|
69
|
+
/**
|
|
70
|
+
* Same as `connect()`, but also exposes the session grant minted by the
|
|
71
|
+
* platform. The grant may contain an advisory media-plane plan (for example
|
|
72
|
+
* Cloudflare WHEP playback) that the browser can attach without another API call.
|
|
73
|
+
*/
|
|
74
|
+
connectWithGrant(options?: AvatarSessionConnectOptions): Promise<AvatarSessionConnectResult>;
|
|
75
|
+
/** Close the session socket (if any); turns fall back to HTTP. */
|
|
76
|
+
disconnect(): void;
|
|
77
|
+
/**
|
|
78
|
+
* Stop the in-flight socket turn server-side (the GPU stops rendering frames
|
|
79
|
+
* nobody will play) while KEEPING the socket warm for the next turn. The
|
|
80
|
+
* socket is dropped only if the server never confirms. No-op on the HTTP
|
|
81
|
+
* path — abort the fetch via its `signal` instead.
|
|
82
|
+
*/
|
|
83
|
+
cancel(): Promise<void>;
|
|
84
|
+
prepare(options?: RealtimeAvatarRequestOptions): Promise<RealtimePrepareResponse>;
|
|
85
|
+
/** The prepare payload this session sends — reusable over a `RealtimeSessionSocket`. */
|
|
86
|
+
prepareRequest(): RealtimePrepareRequest;
|
|
87
|
+
chat(text: string, options?: RealtimeAvatarRequestOptions & {
|
|
88
|
+
history?: RealtimeTurnRequest["messages"];
|
|
89
|
+
/** Per-turn request overrides (e.g. `source_start_frame` for seamless idle handoff). */
|
|
90
|
+
turn?: Partial<RealtimeTurnRequest>;
|
|
91
|
+
}): Promise<AvatarTurnSource>;
|
|
92
|
+
speak(text: string, options?: RealtimeAvatarRequestOptions & {
|
|
93
|
+
turn?: Partial<RealtimeTurnRequest>;
|
|
94
|
+
}): Promise<AvatarTurnSource>;
|
|
95
|
+
/** The chat turn payload — reusable over a `RealtimeSessionSocket`. */
|
|
96
|
+
chatRequest(text: string, history?: RealtimeTurnRequest["messages"]): RealtimeTurnRequest;
|
|
97
|
+
/** The speak-text turn payload — reusable over a `RealtimeSessionSocket`. */
|
|
98
|
+
speakRequest(text: string): RealtimeTurnRequest;
|
|
99
|
+
/**
|
|
100
|
+
* Run the turn on the live socket when one is open. Waits out a previous
|
|
101
|
+
* turn that is still draining its cancel (no-op when idle); a socket that
|
|
102
|
+
* fails the drain is dropped so the caller's turn falls back to HTTP.
|
|
103
|
+
*/
|
|
104
|
+
private socketTurn;
|
|
105
|
+
private turnBase;
|
|
106
|
+
}
|
|
107
|
+
//# sourceMappingURL=session.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"session.d.ts","sourceRoot":"","sources":["../src/session.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,4BAA4B,EAAE,oBAAoB,EAAE,MAAM,UAAU,CAAC;AAC7H,OAAO,KAAK,EAAE,qBAAqB,EAAyB,MAAM,kBAAkB,CAAC;AACrF,OAAO,KAAK,EAAE,gBAAgB,EAAE,sBAAsB,EAAE,uBAAuB,EAAE,mBAAmB,EAAE,MAAM,SAAS,CAAC;AAEtH,MAAM,MAAM,mBAAmB,GAAG;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB;;;;OAIG;IACH,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,6EAA6E;IAC7E,YAAY,CAAC,EAAE,OAAO,CAAC,mBAAmB,CAAC,CAAC;CAC7C,CAAC;AAEF;;;GAGG;AACH,MAAM,MAAM,gBAAgB,GAAG,kBAAkB,GAAG,qBAAqB,CAAC;AAE1E,kDAAkD;AAClD,MAAM,MAAM,sBAAsB,GAAG,QAAQ,GAAG,MAAM,CAAC;AAEvD,MAAM,MAAM,0BAA0B,GAAG;IACvC,KAAK,EAAE,oBAAoB,CAAC;IAC5B,QAAQ,EAAE,uBAAuB,CAAC;CACnC,CAAC;AAEF,MAAM,MAAM,2BAA2B,GAAG;IACxC,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,aAAa,CAAC,EAAE,OAAO,SAAS,CAAC;IACjC;;;;OAIG;IACH,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,oBAAoB,KAAK,IAAI,CAAC;CACjD,CAAC;AAEF;;;;;;;;;;GAUG;AACH,qBAAa,aAAa;IAItB,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,MAAM;IAJzB,OAAO,CAAC,MAAM,CAAsC;gBAGjC,MAAM,EAAE,oBAAoB,EAC5B,MAAM,EAAE,mBAAmB;IAG9C,IAAI,QAAQ,IAAI,MAAM,CAErB;IAED,oEAAoE;IACpE,IAAI,SAAS,IAAI,OAAO,CAEvB;IAED,wCAAwC;IACxC,IAAI,SAAS,IAAI,sBAAsB,CAEtC;IAED;;;;;OAKG;IACG,OAAO,CACX,OAAO,GAAE,2BAAgC,GACxC,OAAO,CAAC,uBAAuB,CAAC;IAInC;;;;OAIG;IACG,gBAAgB,CACpB,OAAO,GAAE,2BAAgC,GACxC,OAAO,CAAC,0BAA0B,CAAC;IAmBtC,kEAAkE;IAClE,UAAU,IAAI,IAAI;IAKlB;;;;;OAKG;IACG,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;IAW7B,OAAO,CAAC,OAAO,GAAE,4BAAiC,GAAG,OAAO,CAAC,uBAAuB,CAAC;IAKrF,wFAAwF;IACxF,cAAc,IAAI,sBAAsB;IAUlC,IAAI,CACR,IAAI,EAAE,MAAM,EACZ,OAAO,GAAE,4BAA4B,GAAG;QACtC,OAAO,CAAC,EAAE,mBAAmB,CAAC,UAAU,CAAC,CAAC;QAC1C,wFAAwF;QACxF,IAAI,CAAC,EAAE,OAAO,CAAC,mBAAmB,CAAC,CAAC;KAChC,GACL,OAAO,CAAC,gBAAgB,CAAC;IAOtB,KAAK,CACT,IAAI,EAAE,MAAM,EACZ,OAAO,GAAE,4BAA4B,GAAG;QAAE,IAAI,CAAC,EAAE,OAAO,CAAC,mBAAmB,CAAC,CAAA;KAAO,GACnF,OAAO,CAAC,gBAAgB,CAAC;IAO5B,uEAAuE;IACvE,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,mBAAmB,CAAC,UAAU,CAAC,GAAG,mBAAmB;IASzF,6EAA6E;IAC7E,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,mBAAmB;IAS/C;;;;OAIG;YACW,UAAU;IAaxB,OAAO,CAAC,QAAQ;CAgBjB"}
|