realtime-voice-agents 2.0.0

package/dist/session-config-c8sOw1XL.cjs

@@ -0,0 +1,85 @@
+const require_BaseRealtimeProvider = require("./BaseRealtimeProvider-DI4pKtOb.cjs");
+//#region src/providers/openai-compatible/session-config.ts
+/**
+* session.update payload construction — GA Realtime API only.
+*
+* The single most fragile spot in Twilio↔OpenAI bridges: a rejected
+* session.update leaves the session on 24 kHz PCM defaults, which Twilio
+* renders as loud static. Keeping every field in one builder makes the wire
+* shape a single point of truth (and of fixing).
+*
+* GA shape notes (vs the old beta many tutorials still show):
+* - audio formats are MIME-style objects (`{ type: 'audio/pcmu' }`), nested
+*   under `session.audio.input/output` — not flat `input_audio_format: 'g711_ulaw'`
+* - assistant audio arrives as `response.output_audio.delta` — not
+*   `response.audio.delta`
+* - `turn_detection: null` explicitly disables server VAD
+*/
+function buildTurnDetection(vad) {
+	if (vad === null) return null;
+	const config = vad ?? { type: "server" };
+	const shared = {
+		...config.interruptResponse !== void 0 ? { interrupt_response: config.interruptResponse } : {},
+		...config.createResponse !== void 0 ? { create_response: config.createResponse } : {}
+	};
+	if (config.type === "semantic") return {
+		type: "semantic_vad",
+		...config.eagerness ? { eagerness: config.eagerness } : {},
+		...shared
+	};
+	return {
+		type: "server_vad",
+		...config.threshold !== void 0 ? { threshold: config.threshold } : {},
+		...config.silenceDurationMs !== void 0 ? { silence_duration_ms: config.silenceDurationMs } : {},
+		...config.prefixPaddingMs !== void 0 ? { prefix_padding_ms: config.prefixPaddingMs } : {},
+		...shared
+	};
+}
+function buildSessionUpdate(init, config = {}) {
+	const voice = init.voice ?? config.defaultVoice;
+	const transcription = init.transcription === false ? void 0 : init.transcription;
+	let session = {
+		type: "realtime",
+		instructions: init.instructions,
+		tools: (init.tools ?? []).map((tool) => ({
+			type: "function",
+			name: tool.name,
+			description: tool.description ?? "",
+			parameters: tool.parameters
+		})),
+		tool_choice: "auto",
+		audio: {
+			input: {
+				format: { type: "audio/pcmu" },
+				turn_detection: buildTurnDetection(init.vad),
+				...transcription ? { transcription: {
+					model: transcription.model ?? "gpt-4o-mini-transcribe",
+					...transcription.language ? { language: transcription.language } : {}
+				} } : {}
+			},
+			output: {
+				format: { type: "audio/pcmu" },
+				...voice ? { voice } : {}
+			}
+		}
+	};
+	session = require_BaseRealtimeProvider.deepMerge(session, init.providerOptions);
+	session = require_BaseRealtimeProvider.deepMerge(session, config.extraSessionOptions);
+	return {
+		type: "session.update",
+		session
+	};
+}
+//#endregion
+Object.defineProperty(exports, "buildSessionUpdate", {
+	enumerable: true,
+	get: function() {
+		return buildSessionUpdate;
+	}
+});
+Object.defineProperty(exports, "buildTurnDetection", {
+	enumerable: true,
+	get: function() {
+		return buildTurnDetection;
+	}
+});

package/dist/store.cjs

@@ -0,0 +1,3 @@
+Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
+const require_InMemorySessionStore = require("./InMemorySessionStore-B0_i-DOU.cjs");
+exports.InMemorySessionStore = require_InMemorySessionStore.InMemorySessionStore;

package/dist/store.d.cts

@@ -0,0 +1,2 @@
+import { i as UsageInfo, n as SessionStore, o as TranscriptEntry, r as CallSnapshot, t as InMemorySessionStore } from "./InMemorySessionStore-B5_rq61L.cjs";
+export { type CallSnapshot, InMemorySessionStore, type SessionStore, type TranscriptEntry, type UsageInfo };

package/dist/store.d.mts

@@ -0,0 +1,2 @@
+import { i as UsageInfo, n as SessionStore, o as TranscriptEntry, r as CallSnapshot, t as InMemorySessionStore } from "./InMemorySessionStore-B5_rq61L.mjs";
+export { type CallSnapshot, InMemorySessionStore, type SessionStore, type TranscriptEntry, type UsageInfo };

package/dist/store.mjs

@@ -0,0 +1,2 @@
+import { t as InMemorySessionStore } from "./InMemorySessionStore-DYParOJO.mjs";
+export { InMemorySessionStore };

package/dist/testing.cjs

@@ -0,0 +1,544 @@
+Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
+const require_mulaw = require("./mulaw-DLUObjdP.cjs");
+let ws = require("ws");
+let node_timers_promises = require("node:timers/promises");
+//#region src/testing/FakeTwilioMediaStream.ts
+/**
+* FakeTwilioMediaStream — a deterministic stand-in for Twilio's side of a
+* Media Stream WebSocket.
+*
+* It exposes the ws-shaped surface the bridge consumes (send/close/on), and a
+* test-driver surface for the "caller": connect(), sendMedia(), stop(), plus a
+* manual playout clock (`advancePlayback`) that emulates Twilio's buffer-and-
+* play-in-order behavior:
+*
+* - outbound `media` frames queue with duration = μ-law bytes / 8 per ms
+* - outbound `mark` frames queue in order and echo back once playout reaches
+*   them
+* - outbound `clear` drops everything buffered and immediately echoes every
+*   pending mark (exactly what real Twilio does with interrupted marks)
+*
+* No wall-clock timers: tests advance playback explicitly, so scenarios are
+* fully deterministic.
+*/
+/** Base64 μ-law silence of the given duration (whole milliseconds at 8 kHz). */
+function mulawSilenceBase64(ms) {
+	return Buffer.alloc(Math.round(ms * 8), 255).toString("base64");
+}
+var FakeTwilioMediaStream = class {
+	readyState = 1;
+	callSid;
+	streamSid;
+	/** Everything the bridge sent, parsed, in order. */
+	outbound = [];
+	/** Base64 payloads of outbound media frames, in order. */
+	sentMediaPayloads = [];
+	/** Names of marks echoed back to the bridge (played or discarded). */
+	echoedMarks = [];
+	/** Number of `clear` frames received from the bridge. */
+	clearCount = 0;
+	messageListeners = [];
+	closeListeners = [];
+	errorListeners = [];
+	playoutQueue = [];
+	playedMsTotal = 0;
+	callerClockMs = 0;
+	mediaSequence = 0;
+	closedByBridge = false;
+	constructor(options = {}) {
+		this.callSid = options.callSid ?? "CA" + "f".repeat(32);
+		this.streamSid = options.streamSid ?? "MZ" + "f".repeat(32);
+	}
+	send(data) {
+		let frame;
+		try {
+			frame = JSON.parse(data);
+		} catch {
+			return;
+		}
+		this.outbound.push(frame);
+		switch (frame.event) {
+			case "media": {
+				const payload = frame.media?.payload ?? "";
+				this.sentMediaPayloads.push(payload);
+				const ms = require_mulaw.base64ByteLength(payload) / 8;
+				this.playoutQueue.push({
+					kind: "media",
+					remainingMs: ms,
+					totalMs: ms
+				});
+				break;
+			}
+			case "mark":
+				this.playoutQueue.push({
+					kind: "mark",
+					name: frame.mark?.name ?? ""
+				});
+				break;
+			case "clear": {
+				this.clearCount++;
+				const pendingMarks = this.playoutQueue.filter((i) => i.kind === "mark");
+				this.playoutQueue.length = 0;
+				for (const mark of pendingMarks) this.echoMark(mark.name);
+				break;
+			}
+		}
+	}
+	close(code, reason) {
+		if (this.readyState === 3) return;
+		this.readyState = 3;
+		this.closedByBridge = true;
+		for (const listener of this.closeListeners) listener(code ?? 1e3, reason);
+	}
+	on(event, listener) {
+		if (event === "message") this.messageListeners.push(listener);
+		else if (event === "close") this.closeListeners.push(listener);
+		else if (event === "error") this.errorListeners.push(listener);
+		return this;
+	}
+	/** Send the `connected` preamble and `start` frame. */
+	connect(options = {}) {
+		if (options.sendConnectedFrame !== false) this.deliver({
+			event: "connected",
+			protocol: "Call",
+			version: "1.0.0"
+		});
+		this.deliver({
+			event: "start",
+			sequenceNumber: "1",
+			streamSid: options.streamSid ?? this.streamSid,
+			start: {
+				streamSid: options.streamSid ?? this.streamSid,
+				accountSid: options.accountSid ?? "AC" + "f".repeat(32),
+				callSid: options.callSid ?? this.callSid,
+				tracks: ["inbound"],
+				mediaFormat: {
+					encoding: "audio/x-mulaw",
+					sampleRate: 8e3,
+					channels: 1
+				},
+				customParameters: options.customParameters ?? {}
+			}
+		});
+	}
+	/** Deliver one caller media frame (base64 μ-law) to the bridge. */
+	sendMedia(payload) {
+		this.mediaSequence++;
+		const ms = require_mulaw.base64ByteLength(payload) / 8;
+		this.deliver({
+			event: "media",
+			sequenceNumber: String(this.mediaSequence + 1),
+			streamSid: this.streamSid,
+			media: {
+				track: "inbound",
+				chunk: String(this.mediaSequence),
+				timestamp: String(Math.round(this.callerClockMs)),
+				payload
+			}
+		});
+		this.callerClockMs += ms;
+	}
+	/** Deliver `ms` of caller silence as 20 ms μ-law frames. */
+	sendSilence(ms) {
+		let remaining = ms;
+		while (remaining > 0) {
+			const frameMs = Math.min(20, remaining);
+			this.sendMedia(mulawSilenceBase64(frameMs));
+			remaining -= frameMs;
+		}
+	}
+	sendDtmf(digit) {
+		this.deliver({
+			event: "dtmf",
+			streamSid: this.streamSid,
+			dtmf: {
+				track: "inbound_track",
+				digit
+			}
+		});
+	}
+	/** Deliver the `stop` frame (Twilio ended the stream). */
+	stop() {
+		this.deliver({
+			event: "stop",
+			streamSid: this.streamSid,
+			stop: { callSid: this.callSid }
+		});
+	}
+	/** Simulate the socket dropping from Twilio's side. */
+	disconnect(code = 1006, reason = "connection lost") {
+		if (this.readyState === 3) return;
+		this.readyState = 3;
+		for (const listener of this.closeListeners) listener(code, reason);
+	}
+	/** Total milliseconds of agent audio actually "played" to the caller. */
+	get playedMs() {
+		return this.playedMsTotal;
+	}
+	/** Milliseconds of agent audio still buffered. */
+	get queuedMs() {
+		return this.playoutQueue.reduce((sum, item) => sum + (item.kind === "media" ? item.remainingMs : 0), 0);
+	}
+	get wasClosedByBridge() {
+		return this.closedByBridge;
+	}
+	/**
+	* Advance the playout clock: consume buffered media in order and echo each
+	* mark as playout reaches it.
+	*/
+	advancePlayback(ms) {
+		let budget = ms;
+		while (this.playoutQueue.length > 0) {
+			const head = this.playoutQueue[0];
+			if (head.kind === "mark") {
+				this.playoutQueue.shift();
+				this.echoMark(head.name);
+				continue;
+			}
+			if (budget <= 0) break;
+			const consumed = Math.min(budget, head.remainingMs);
+			head.remainingMs -= consumed;
+			budget -= consumed;
+			this.playedMsTotal += consumed;
+			if (head.remainingMs <= 1e-9) this.playoutQueue.shift();
+		}
+		while (this.playoutQueue[0]?.kind === "mark") {
+			const mark = this.playoutQueue.shift();
+			this.echoMark(mark.name);
+		}
+	}
+	/** Play out everything currently buffered. */
+	playAll() {
+		this.advancePlayback(Number.MAX_SAFE_INTEGER);
+	}
+	echoMark(name) {
+		this.echoedMarks.push(name);
+		this.deliver({
+			event: "mark",
+			streamSid: this.streamSid,
+			mark: { name }
+		});
+	}
+	deliver(frame) {
+		if (this.readyState === 3) return;
+		const data = JSON.stringify(frame);
+		for (const listener of this.messageListeners) listener(data);
+	}
+};
+//#endregion
+//#region src/testing/FakeOpenAIServer.ts
+let responseCounter = 0;
+var FakeOpenAIConnection = class {
+	socket;
+	/** Every parsed client → server frame, in order. */
+	received = [];
+	/** HTTP headers of the client's upgrade request (auth, extensions…). */
+	upgradeHeaders;
+	server;
+	constructor(server, socket, upgradeHeaders = {}) {
+		this.server = server;
+		this.socket = socket;
+		this.upgradeHeaders = upgradeHeaders;
+	}
+	/** Base64 payloads from input_audio_buffer.append frames. */
+	get appendedAudio() {
+		return this.received.filter((f) => f.type === "input_audio_buffer.append").map((f) => f.audio);
+	}
+	eventsOfType(type) {
+		return this.received.filter((f) => f.type === type);
+	}
+	async waitForEvent(predicate, timeoutMs = 2e3) {
+		const matches = typeof predicate === "string" ? (frame) => frame.type === predicate : predicate;
+		const deadline = Date.now() + timeoutMs;
+		let seen = 0;
+		while (Date.now() < deadline) {
+			for (; seen < this.received.length; seen++) {
+				const frame = this.received[seen];
+				if (matches(frame)) return frame;
+			}
+			await (0, node_timers_promises.setTimeout)(5);
+		}
+		throw new Error(`timed out waiting for client event; got: ${this.received.map((f) => f.type).join(", ")}`);
+	}
+	send(frame) {
+		this.socket.send(JSON.stringify(frame));
+	}
+	/** Script a complete assistant audio response (created → deltas → done). */
+	sendAudioResponse(options) {
+		const responseId = options.responseId ?? `resp_${++responseCounter}`;
+		const itemId = options.itemId ?? `item_${responseCounter}`;
+		this.send({
+			type: "response.created",
+			response: { id: responseId }
+		});
+		this.send({
+			type: "response.output_item.added",
+			response_id: responseId,
+			item: {
+				id: itemId,
+				type: "message"
+			}
+		});
+		for (const chunk of options.chunks) this.send({
+			type: "response.output_audio.delta",
+			response_id: responseId,
+			item_id: itemId,
+			delta: chunk
+		});
+		if (options.transcript) {
+			this.send({
+				type: "response.output_audio_transcript.delta",
+				response_id: responseId,
+				delta: options.transcript
+			});
+			this.send({
+				type: "response.output_audio_transcript.done",
+				response_id: responseId,
+				transcript: options.transcript
+			});
+		}
+		if (options.complete !== false) this.send({
+			type: "response.done",
+			response: {
+				id: responseId,
+				status: "completed",
+				usage: options.usage
+			}
+		});
+		return {
+			responseId,
+			itemId
+		};
+	}
+	sendToolCall(options) {
+		const callId = options.callId ?? `call_${++responseCounter}`;
+		const responseId = options.responseId ?? `resp_${responseCounter}`;
+		this.send({
+			type: "response.created",
+			response: { id: responseId }
+		});
+		this.send({
+			type: "response.function_call_arguments.done",
+			response_id: responseId,
+			item_id: `item_${responseCounter}`,
+			call_id: callId,
+			name: options.name,
+			arguments: options.argumentsJson ?? "{}"
+		});
+		this.send({
+			type: "response.done",
+			response: {
+				id: responseId,
+				status: "completed"
+			}
+		});
+		return callId;
+	}
+	sendSpeechStarted() {
+		this.send({ type: "input_audio_buffer.speech_started" });
+	}
+	sendSpeechStopped() {
+		this.send({ type: "input_audio_buffer.speech_stopped" });
+	}
+	sendUserTranscript(text) {
+		this.send({
+			type: "conversation.item.input_audio_transcription.completed",
+			transcript: text
+		});
+	}
+	sendError(error) {
+		this.send({
+			type: "error",
+			error
+		});
+	}
+	drop(code = 1011, reason = "fake server drop") {
+		this.socket.close(code, reason);
+	}
+};
+var FakeOpenAIServer = class FakeOpenAIServer {
+	connections = [];
+	wss;
+	options;
+	url;
+	constructor(wss, url, options) {
+		this.wss = wss;
+		this.url = url;
+		this.options = options;
+		wss.on("connection", (socket, request) => {
+			const connection = new FakeOpenAIConnection(this, socket, request.headers);
+			this.connections.push(connection);
+			socket.on("message", (raw) => {
+				let frame;
+				try {
+					frame = JSON.parse(raw.toString());
+				} catch {
+					return;
+				}
+				connection.received.push(frame);
+				if (frame.type === "session.update" && this.options.autoAckSessionUpdate !== false) connection.send({
+					type: "session.updated",
+					session: frame.session
+				});
+			});
+			socket.send(JSON.stringify({
+				type: "session.created",
+				session: {}
+			}));
+		});
+	}
+	static async start(options = {}) {
+		const wss = new ws.WebSocketServer({
+			port: 0,
+			host: "127.0.0.1"
+		});
+		await new Promise((resolve, reject) => {
+			wss.once("listening", resolve);
+			wss.once("error", reject);
+		});
+		const address = wss.address();
+		if (typeof address === "string" || address === null) throw new Error("no server address");
+		return new FakeOpenAIServer(wss, `ws://127.0.0.1:${address.port}`, options);
+	}
+	get latest() {
+		const connection = this.connections[this.connections.length - 1];
+		if (!connection) throw new Error("no connections yet");
+		return connection;
+	}
+	async waitForConnection(timeoutMs = 2e3) {
+		const count = this.connections.length;
+		const deadline = Date.now() + timeoutMs;
+		while (Date.now() < deadline) {
+			if (this.connections.length > count) return this.connections[this.connections.length - 1];
+			await (0, node_timers_promises.setTimeout)(5);
+		}
+		throw new Error("timed out waiting for a provider connection");
+	}
+	async close() {
+		for (const connection of this.connections) try {
+			connection.socket.terminate();
+		} catch {}
+		await new Promise((resolve) => this.wss.close(() => resolve()));
+	}
+};
+//#endregion
+//#region src/testing/FakeGeminiLive.ts
+/**
+* FakeGeminiLive — a scripted stand-in for @google/genai's live connection.
+*
+* Instead of a WebSocket server, this fakes the SDK seam: pass
+* `fake.connector` as the Gemini provider's `connector`. Each connect yields a
+* FakeGeminiSession that records client messages and lets tests push server
+* messages (audio parts, tool calls, transcriptions, resumption updates,
+* goAway) through the provider's callbacks.
+*/
+var FakeGeminiSession = class {
+	params;
+	realtimeInputs = [];
+	clientContents = [];
+	toolResponses = [];
+	closed = false;
+	constructor(params) {
+		this.params = params;
+	}
+	sendRealtimeInput(input) {
+		this.realtimeInputs.push(input.audio);
+	}
+	sendClientContent(content) {
+		this.clientContents.push(content);
+	}
+	sendToolResponse(response) {
+		this.toolResponses.push(response);
+	}
+	close() {
+		if (this.closed) return;
+		this.closed = true;
+		this.params.callbacks.onclose?.({
+			code: 1e3,
+			reason: "client close"
+		});
+	}
+	serverMessage(message) {
+		this.params.callbacks.onmessage(message);
+	}
+	completeSetup() {
+		this.serverMessage({ setupComplete: {} });
+	}
+	/** Stream a model turn: PCM24k audio parts (+ transcript), then turnComplete. */
+	sendAudioTurn(options) {
+		for (const chunk of options.pcm24kBase64Chunks) this.serverMessage({ serverContent: { modelTurn: { parts: [{ inlineData: {
+			data: chunk,
+			mimeType: "audio/pcm;rate=24000"
+		} }] } } });
+		if (options.transcript) this.serverMessage({ serverContent: { outputTranscription: { text: options.transcript } } });
+		this.serverMessage({ serverContent: { turnComplete: true } });
+	}
+	sendToolCall(options) {
+		const id = options.id ?? `fn_${Math.random().toString(36).slice(2, 8)}`;
+		this.serverMessage({ toolCall: { functionCalls: [{
+			id,
+			name: options.name,
+			args: options.args ?? {}
+		}] } });
+		return id;
+	}
+	sendInterrupted() {
+		this.serverMessage({ serverContent: { interrupted: true } });
+	}
+	sendInputTranscription(text) {
+		this.serverMessage({ serverContent: { inputTranscription: { text } } });
+	}
+	sendResumptionUpdate(handle) {
+		this.serverMessage({ sessionResumptionUpdate: {
+			resumable: true,
+			newHandle: handle
+		} });
+	}
+	sendGoAway(timeLeft) {
+		this.serverMessage({ goAway: { timeLeft } });
+	}
+	sendUsage(usage) {
+		this.serverMessage({ usageMetadata: usage });
+	}
+	/** Drop the connection from the server side. */
+	drop(code = 1011, reason = "fake drop") {
+		if (this.closed) return;
+		this.closed = true;
+		this.params.callbacks.onclose?.({
+			code,
+			reason
+		});
+	}
+};
+var FakeGeminiLive = class {
+	sessions = [];
+	/** Auto-send setupComplete on connect. Default true. */
+	autoCompleteSetup = true;
+	connector = async (params) => {
+		const session = new FakeGeminiSession(params);
+		this.sessions.push(session);
+		params.callbacks.onopen?.();
+		if (this.autoCompleteSetup) queueMicrotask(() => session.completeSetup());
+		return session;
+	};
+	get latest() {
+		const session = this.sessions[this.sessions.length - 1];
+		if (!session) throw new Error("no gemini sessions yet");
+		return session;
+	}
+	async waitForSessions(count, timeoutMs = 2e3) {
+		const deadline = Date.now() + timeoutMs;
+		while (Date.now() < deadline) {
+			if (this.sessions.length >= count) return this.sessions[count - 1];
+			await (0, node_timers_promises.setTimeout)(5);
+		}
+		throw new Error(`timed out waiting for ${count} gemini sessions (have ${this.sessions.length})`);
+	}
+};
+//#endregion
+exports.FakeGeminiLive = FakeGeminiLive;
+exports.FakeGeminiSession = FakeGeminiSession;
+exports.FakeOpenAIConnection = FakeOpenAIConnection;
+exports.FakeOpenAIServer = FakeOpenAIServer;
+exports.FakeTwilioMediaStream = FakeTwilioMediaStream;
+exports.mulawSilenceBase64 = mulawSilenceBase64;