realtime-voice-agents 2.0.0

package/dist/OpenAICompatibleProvider-NS4cKVQj.cjs

@@ -0,0 +1,409 @@
+const require_rolldown_runtime = require("./rolldown-runtime-VH7oDXx4.cjs");
+const require_BaseRealtimeProvider = require("./BaseRealtimeProvider-DI4pKtOb.cjs");
+const require_session_config = require("./session-config-c8sOw1XL.cjs");
+let ws = require("ws");
+ws = require_rolldown_runtime.__toESM(ws, 1);
+//#region src/providers/openai-compatible/OpenAICompatibleProvider.ts
+/**
+* Provider for the OpenAI Realtime API (GA protocol) and API-compatible
+* services (xAI Grok Voice Agent). Speaks `audio/pcmu` in both directions, so
+* Twilio payloads pass through verbatim — zero transcoding on this path.
+*
+* Connect handshake: open WS → receive `session.created` → send
+* `session.update` (audio/pcmu, VAD, tools, instructions) → resolve on
+* `session.updated`. A handshake that hangs rejects after `connectTimeoutMs`
+* so a dead upstream can't leak half-open calls.
+*/
+const DEFAULT_BASE_URL = "wss://api.openai.com/v1/realtime";
+/** WS close codes where retrying cannot help (protocol/auth/policy failures). */
+const NON_RETRIABLE_CLOSE_CODES = /* @__PURE__ */ new Set([
+	1002,
+	1003,
+	1007,
+	1008
+]);
+var OpenAICompatibleProvider = class extends require_BaseRealtimeProvider.BaseRealtimeProvider {
+	name;
+	capabilities;
+	config;
+	logger;
+	ws = null;
+	sessionInit = null;
+	ready = false;
+	intentionalClose = false;
+	currentResponseId = null;
+	/**
+	* response.create serialization. The GA API rejects a `response.create`
+	* issued while another response is in flight
+	* (`conversation_already_has_active_response`), so creates requested
+	* mid-response wait in a single pending slot and fire on `response.done`.
+	* Later requests coalesce into that slot — one response reads the whole
+	* conversation state, so only the instructions payload is worth keeping.
+	*/
+	responseActive = false;
+	pendingCreate = null;
+	lastCreateSent = null;
+	constructor(config, logger = require_BaseRealtimeProvider.noopLogger) {
+		super();
+		this.config = config;
+		this.logger = logger;
+		this.name = config.providerName ?? "openai";
+		this.capabilities = {
+			truncate: true,
+			sessionUpdate: true,
+			voiceChangeMidSession: false,
+			transcodeRequired: false,
+			resumption: false,
+			agentTranscriptDeltas: true,
+			vadInterruptControl: true,
+			...config.capabilityOverrides
+		};
+	}
+	get isConnected() {
+		return this.ready && this.ws?.readyState === ws.default.OPEN;
+	}
+	async connect(init) {
+		if (this.ws) await this.close();
+		let vad = init.vad !== void 0 ? init.vad : this.config.defaultVad;
+		if (init.bridgeOwnsInterruptions && this.capabilities.vadInterruptControl && vad !== null) vad = {
+			interruptResponse: false,
+			...vad ?? { type: "server" }
+		};
+		this.sessionInit = {
+			...init,
+			vad,
+			transcription: init.transcription !== void 0 ? init.transcription : this.config.defaultTranscription
+		};
+		this.intentionalClose = false;
+		this.ready = false;
+		this.responseActive = false;
+		this.pendingCreate = null;
+		this.lastCreateSent = null;
+		this.currentResponseId = null;
+		const url = `${this.config.baseUrl ?? DEFAULT_BASE_URL}?model=${encodeURIComponent(this.config.model)}`;
+		const ws$1 = new ws.default(url, {
+			headers: {
+				Authorization: `Bearer ${this.config.apiKey}`,
+				...this.config.headers
+			},
+			perMessageDeflate: false
+		});
+		this.ws = ws$1;
+		await new Promise((resolve, reject) => {
+			const timeoutMs = this.config.connectTimeoutMs ?? 1e4;
+			const timer = setTimeout(() => {
+				fail(/* @__PURE__ */ new Error(`${this.name} session not ready within ${timeoutMs}ms`));
+			}, timeoutMs);
+			timer.unref?.();
+			let settled = false;
+			const fail = (error) => {
+				if (settled) return;
+				settled = true;
+				clearTimeout(timer);
+				try {
+					ws$1.close();
+				} catch {}
+				reject(error);
+			};
+			ws$1.on("unexpected-response", (_request, response) => {
+				let body = "";
+				response.setEncoding("utf8");
+				response.on("data", (chunk) => {
+					if (body.length < 512) body += chunk;
+				});
+				response.on("end", () => {
+					const detail = body.trim().slice(0, 500);
+					this.logger.error("provider rejected the WebSocket upgrade", {
+						status: response.statusCode,
+						body: detail
+					});
+					fail(/* @__PURE__ */ new Error(`${this.name} rejected the WebSocket upgrade: HTTP ${response.statusCode}${detail ? ` — ${detail}` : ""}`));
+				});
+			});
+			ws$1.on("open", () => this.emit("open"));
+			ws$1.on("message", (raw) => {
+				const event = this.parseEvent(raw);
+				if (!event) return;
+				if (!settled) {
+					if (event.type === "session.created") {
+						this.send(this.buildSessionPayload());
+						return;
+					}
+					if (event.type === "session.updated") {
+						settled = true;
+						clearTimeout(timer);
+						this.ready = true;
+						resolve();
+						return;
+					}
+					if (event.type === "error") {
+						fail(/* @__PURE__ */ new Error(`${this.name} session setup error: ${JSON.stringify(event.error ?? event)}`));
+						return;
+					}
+				}
+				this.handleEvent(event);
+			});
+			ws$1.on("error", (error) => {
+				if (!settled) fail(error instanceof Error ? error : new Error(String(error)));
+				else this.emit("error", error instanceof Error ? error : new Error(String(error)));
+			});
+			ws$1.on("close", (code, reasonBuf) => {
+				const reason = reasonBuf?.toString();
+				this.ready = false;
+				if (!settled) {
+					fail(/* @__PURE__ */ new Error(`${this.name} socket closed during setup (${code} ${reason ?? ""})`));
+					return;
+				}
+				this.emit("close", {
+					code,
+					reason,
+					retriable: !this.intentionalClose && !NON_RETRIABLE_CLOSE_CODES.has(code)
+				});
+			});
+		});
+	}
+	async close() {
+		this.intentionalClose = true;
+		this.ready = false;
+		const ws$2 = this.ws;
+		this.ws = null;
+		if (!ws$2) return;
+		if (ws$2.readyState === ws.default.CLOSED) return;
+		await new Promise((resolve) => {
+			const timer = setTimeout(() => {
+				ws$2.terminate();
+				resolve();
+			}, 1e3);
+			timer.unref?.();
+			ws$2.once("close", () => {
+				clearTimeout(timer);
+				resolve();
+			});
+			try {
+				ws$2.close(1e3);
+			} catch {
+				clearTimeout(timer);
+				ws$2.terminate();
+				resolve();
+			}
+		});
+	}
+	sendAudio(base64Mulaw) {
+		if (!this.isConnected) return;
+		this.send({
+			type: "input_audio_buffer.append",
+			audio: base64Mulaw
+		});
+	}
+	sendText(text, options = {}) {
+		const role = options.role ?? "user";
+		this.send({
+			type: "conversation.item.create",
+			item: {
+				type: "message",
+				role,
+				content: [{
+					type: role === "assistant" ? "output_text" : "input_text",
+					text
+				}]
+			}
+		});
+		if (options.triggerResponse !== false) this.createResponse();
+	}
+	sendToolResult(callId, output, options = {}) {
+		this.send({
+			type: "conversation.item.create",
+			item: {
+				type: "function_call_output",
+				call_id: callId,
+				output: typeof output === "string" ? output : JSON.stringify(output ?? null)
+			}
+		});
+		if (options.triggerResponse !== false) this.createResponse();
+	}
+	createResponse(options = {}) {
+		if (this.responseActive) {
+			this.pendingCreate = {
+				instructions: options.instructions ?? this.pendingCreate?.instructions,
+				attempts: 0
+			};
+			return;
+		}
+		this.sendCreate({
+			instructions: options.instructions,
+			attempts: 0
+		});
+	}
+	sendCreate(create) {
+		this.lastCreateSent = create;
+		this.send({
+			type: "response.create",
+			...create.instructions ? { response: { instructions: create.instructions } } : {}
+		});
+	}
+	async updateSession(patch) {
+		if (!this.sessionInit) throw new Error("updateSession before connect");
+		this.sessionInit = {
+			...this.sessionInit,
+			...patch
+		};
+		this.send(this.buildSessionPayload());
+	}
+	buildSessionPayload() {
+		return (this.config.buildSession ?? require_session_config.buildSessionUpdate)(this.sessionInit, {
+			defaultVoice: this.config.voice,
+			extraSessionOptions: this.config.extraSessionOptions
+		});
+	}
+	cancelResponse() {
+		if (this.pendingCreate && !this.pendingCreate.instructions) this.pendingCreate = null;
+		this.send({ type: "response.cancel" });
+	}
+	truncatePlayback(itemId, audioEndMs) {
+		this.send({
+			type: "conversation.item.truncate",
+			item_id: itemId,
+			content_index: 0,
+			audio_end_ms: Math.max(0, Math.round(audioEndMs))
+		});
+	}
+	handleEvent(event) {
+		switch (event.type) {
+			case "response.created": {
+				const responseId = event.response?.id ?? `resp_${Date.now()}`;
+				this.currentResponseId = responseId;
+				this.responseActive = true;
+				this.emit("responseStarted", { responseId });
+				break;
+			}
+			case "response.output_item.added": {
+				const itemId = event.item?.id;
+				if (itemId) this.emit("outputItemAdded", {
+					itemId,
+					responseId: event.response_id ?? this.currentResponseId ?? ""
+				});
+				break;
+			}
+			case "response.output_audio.delta":
+				if (typeof event.delta === "string" && event.delta.length > 0) this.emit("audio", {
+					base64Mulaw: event.delta,
+					responseId: event.response_id ?? this.currentResponseId ?? "",
+					itemId: event.item_id
+				});
+				break;
+			case "response.output_audio_transcript.delta":
+				if (typeof event.delta === "string") this.emit("agentTranscriptDelta", {
+					responseId: event.response_id ?? this.currentResponseId ?? "",
+					delta: event.delta
+				});
+				break;
+			case "response.output_audio_transcript.done":
+				if (typeof event.transcript === "string") this.emit("agentTranscript", {
+					responseId: event.response_id ?? this.currentResponseId ?? "",
+					text: event.transcript
+				});
+				break;
+			case "conversation.item.input_audio_transcription.completed":
+				if (typeof event.transcript === "string" && event.transcript.trim().length > 0) this.emit("userTranscript", { text: event.transcript.trim() });
+				break;
+			case "input_audio_buffer.speech_started":
+				if (this.pendingCreate && !this.pendingCreate.instructions) this.pendingCreate = null;
+				this.emit("userSpeechStarted");
+				break;
+			case "input_audio_buffer.speech_stopped":
+				this.emit("userSpeechStopped");
+				break;
+			case "response.done": {
+				const responseId = event.response?.id ?? this.currentResponseId ?? "";
+				const usage = normalizeUsage(event.response?.usage);
+				this.responseActive = false;
+				const pending = this.pendingCreate;
+				if (pending) {
+					this.pendingCreate = null;
+					this.sendCreate(pending);
+				}
+				if (usage) this.emit("usage", usage);
+				this.emit("responseDone", {
+					responseId,
+					usage: usage ?? void 0
+				});
+				break;
+			}
+			case "response.function_call_arguments.done":
+				this.emit("toolCall", {
+					id: event.call_id ?? event.item_id ?? `call_${Date.now()}`,
+					name: event.name,
+					argumentsJson: typeof event.arguments === "string" ? event.arguments : "{}",
+					responseId: event.response_id ?? this.currentResponseId ?? void 0,
+					itemId: event.item_id
+				});
+				break;
+			case "error": {
+				const code = event.error?.code;
+				if (code === "conversation_already_has_active_response") {
+					this.responseActive = true;
+					const last = this.lastCreateSent;
+					if (last && last.attempts < 2 && !this.pendingCreate) this.pendingCreate = {
+						...last,
+						attempts: last.attempts + 1
+					};
+					this.logger.warn("response.create raced an active response; deferred until it completes", { error: event.error });
+					break;
+				}
+				{
+					const message = typeof event.error?.message === "string" ? event.error.message : "";
+					if (code === "response_cancel_not_active" || /cancellation failed: no active response/i.test(message)) {
+						this.logger.warn("response.cancel raced completion (ignored)", { error: event.error });
+						break;
+					}
+				}
+				this.logger.warn("provider error event", { error: event.error });
+				this.emit("error", /* @__PURE__ */ new Error(`${this.name} error: ${JSON.stringify(event.error ?? event)}`));
+				break;
+			}
+		}
+	}
+	parseEvent(raw) {
+		try {
+			return JSON.parse(raw.toString());
+		} catch {
+			this.logger.warn("unparseable provider frame");
+			return null;
+		}
+	}
+	send(payload) {
+		if (this.ws?.readyState !== ws.default.OPEN) return;
+		try {
+			this.ws.send(JSON.stringify(payload));
+		} catch (error) {
+			this.emit("error", error instanceof Error ? error : new Error(String(error)));
+		}
+	}
+};
+function normalizeUsage(raw) {
+	if (!raw || typeof raw !== "object") return null;
+	const inputDetails = raw.input_token_details ?? raw.input_tokens_details;
+	const outputDetails = raw.output_token_details ?? raw.output_tokens_details;
+	return {
+		inputTokens: raw.input_tokens ?? 0,
+		outputTokens: raw.output_tokens ?? 0,
+		totalTokens: raw.total_tokens ?? (raw.input_tokens ?? 0) + (raw.output_tokens ?? 0),
+		...inputDetails ? { inputTokenDetails: {
+			textTokens: inputDetails.text_tokens,
+			audioTokens: inputDetails.audio_tokens,
+			cachedTokens: inputDetails.cached_tokens
+		} } : {},
+		...outputDetails ? { outputTokenDetails: {
+			textTokens: outputDetails.text_tokens,
+			audioTokens: outputDetails.audio_tokens
+		} } : {},
+		raw
+	};
+}
+//#endregion
+Object.defineProperty(exports, "OpenAICompatibleProvider", {
+	enumerable: true,
+	get: function() {
+		return OpenAICompatibleProvider;
+	}
+});

package/dist/audio.cjs

@@ -0,0 +1,21 @@
+Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
+const require_mulaw = require("./mulaw-DLUObjdP.cjs");
+const require_BackgroundAudioPlayer = require("./BackgroundAudioPlayer-jfRULWKC.cjs");
+const require_transcode = require("./transcode-C9aJG7_W.cjs");
+exports.BackgroundAudioPlayer = require_BackgroundAudioPlayer.BackgroundAudioPlayer;
+exports.InboundTranscoder = require_transcode.InboundTranscoder;
+exports.MULAW_FRAME_BYTES_20MS = require_mulaw.MULAW_FRAME_BYTES_20MS;
+exports.MULAW_SAMPLE_RATE = require_mulaw.MULAW_SAMPLE_RATE;
+exports.MULAW_SILENCE_BYTE = require_mulaw.MULAW_SILENCE_BYTE;
+exports.OutboundTranscoder = require_transcode.OutboundTranscoder;
+exports.Resampler = require_transcode.Resampler;
+exports.base64ByteLength = require_mulaw.base64ByteLength;
+exports.fadeMulaw = require_BackgroundAudioPlayer.fadeMulaw;
+exports.isPresetName = require_BackgroundAudioPlayer.isPresetName;
+exports.loadBackgroundAudio = require_BackgroundAudioPlayer.loadBackgroundAudio;
+exports.mulawBytesToMs = require_mulaw.mulawBytesToMs;
+exports.mulawDecodeSample = require_mulaw.mulawDecodeSample;
+exports.mulawEncodeSample = require_mulaw.mulawEncodeSample;
+exports.mulawToPcm16 = require_mulaw.mulawToPcm16;
+exports.pcm16ToMulaw = require_mulaw.pcm16ToMulaw;
+exports.scaleMulaw = require_BackgroundAudioPlayer.scaleMulaw;

package/dist/audio.d.cts

@@ -0,0 +1,165 @@
+import { n as BackgroundAudioPreset, r as BackgroundAudioSpec, t as BackgroundAudioOptions } from "./presets-Bf75YXs5.cjs";
+//#region src/audio/mulaw.d.ts
+/**
+ * G.711 μ-law codec (the format Twilio Media Streams speak: 8 kHz, 1 byte/sample).
+ *
+ * Decode uses a 256-entry lookup table; encode uses the canonical Sun g711.c
+ * algorithm (bias 0x84, clip 32635). μ-law silence is 0xFF.
+ */
+/** One byte of μ-law silence (encodes linear 0). */
+declare const MULAW_SILENCE_BYTE = 255;
+/** Samples per second on the Twilio media stream. μ-law is 1 byte per sample. */
+declare const MULAW_SAMPLE_RATE = 8000;
+/** Bytes per 20 ms Twilio media frame at 8 kHz μ-law. */
+declare const MULAW_FRAME_BYTES_20MS = 160;
+/** Decode a single μ-law byte to a linear PCM16 sample. */
+declare function mulawDecodeSample(byte: number): number;
+/** Encode a single linear PCM16 sample to a μ-law byte. */
+declare function mulawEncodeSample(sample: number): number;
+/** Decode a μ-law buffer to linear PCM16 samples. */
+declare function mulawToPcm16(mulaw: Uint8Array): Int16Array;
+/** Encode linear PCM16 samples to a μ-law buffer. */
+declare function pcm16ToMulaw(pcm: Int16Array): Uint8Array;
+/** Duration in milliseconds of a μ-law byte count at 8 kHz (1 byte = 125 µs). */
+declare function mulawBytesToMs(bytes: number): number;
+/** Byte length of a base64 string without decoding it. */
+declare function base64ByteLength(base64: string): number;
+//#endregion
+//#region src/audio/resampler.d.ts
+/**
+ * Stateful polyphase FIR sample-rate converter.
+ *
+ * Built for streaming telephony audio: the instance carries the FIR history
+ * across process() calls, so chunk boundaries produce bit-identical output to
+ * single-pass processing — no per-chunk zero-padding, no boundary clicks.
+ * Coefficients are computed once per (from, to, quality) in a module cache.
+ *
+ * Rational L/M design: the prototype lowpass is a Kaiser-windowed sinc at the
+ * upsampled rate (from × L), decomposed into L phases of N/L taps. Each phase
+ * is normalized to unit DC gain so constant signals pass through exactly.
+ */
+interface ResamplerOptions {
+  /**
+   * Prototype filter length multiplier. The prototype has
+   * `quality × max(L, M)` taps (rounded up to a multiple of L); higher is
+   * cleaner and slower. Default 48 (≈70–80 dB alias rejection for 24k→8k).
+   */
+  quality?: number;
+  /** Kaiser window beta. Default 8. */
+  kaiserBeta?: number;
+  /** Passband edge as a fraction of the tighter Nyquist. Default 0.9. */
+  rolloff?: number;
+}
+declare class Resampler {
+  readonly fromRate: number;
+  readonly toRate: number;
+  private readonly L;
+  private readonly M;
+  private readonly bank;
+  /** Trailing input samples carried between chunks (tapsPerPhase − 1 long). */
+  private hist;
+  /** Phase index of the next output sample. */
+  private phase;
+  /**
+   * Input index (relative to the start of the next [hist + chunk] buffer) that
+   * the next output sample is anchored on. Always ≥ tapsPerPhase − 1.
+   */
+  private kRel;
+  constructor(fromRate: number, toRate: number, options?: ResamplerOptions);
+  /** True when no rate conversion is needed (input is passed through). */
+  get isPassthrough(): boolean;
+  process(input: Int16Array): Int16Array;
+  /** Drain the filter tail by pushing one phase-length of silence. */
+  flush(): Int16Array;
+  /** Clear all carried state (history, phase) for reuse on a fresh stream. */
+  reset(): void;
+}
+//#endregion
+//#region src/audio/transcode.d.ts
+/** Twilio → provider: base64 μ-law 8 kHz in, base64 PCM16LE at `outRate` out. */
+declare class InboundTranscoder {
+  private readonly resampler;
+  constructor(outRate?: number, options?: ResamplerOptions);
+  process(base64Mulaw: string): string;
+  reset(): void;
+}
+/** Provider → Twilio: base64 PCM16LE at `inRate` in, base64 μ-law 8 kHz out. */
+declare class OutboundTranscoder {
+  private readonly resampler;
+  /** Dangling byte from a chunk that split a 16-bit sample. */
+  private carry;
+  constructor(inRate?: number, options?: ResamplerOptions);
+  process(base64Pcm16: string): string;
+  reset(): void;
+}
+//#endregion
+//#region src/audio/gain.d.ts
+/**
+ * Gain application on μ-law frames, used for background-audio fades and volume.
+ * μ-law is logarithmic, so gain must ramp in the linear domain:
+ * decode → scale → re-encode.
+ */
+/** Scale an entire μ-law frame by a constant linear gain. */
+declare function scaleMulaw(frame: Uint8Array, gain: number): Uint8Array;
+/**
+ * Apply a linearly interpolated gain ramp across a μ-law frame
+ * (`startGain` at the first sample → `endGain` at the last).
+ */
+declare function fadeMulaw(frame: Uint8Array, startGain: number, endGain: number): Uint8Array;
+//#endregion
+//#region src/audio/background/BackgroundAudioPlayer.d.ts
+interface BackgroundAudioPlayerDeps {
+  sendMedia: (base64Payload: string) => void;
+  onStarted?: (info: {
+    preset?: string;
+  }) => void;
+  onStopped?: (info: {
+    preset?: string;
+  }) => void;
+  now?: () => number;
+}
+declare class BackgroundAudioPlayer {
+  private readonly deps;
+  private readonly now;
+  private generation;
+  private active;
+  private startDelayTimer;
+  /** Tool-call ids currently holding the loop. */
+  private readonly holders;
+  private pendingSpec;
+  constructor(deps: BackgroundAudioPlayerDeps);
+  get isPlaying(): boolean;
+  /**
+   * Refcounted acquisition (one holder per tool call). The loop starts after
+   * `startDelayMs` unless every holder releases first.
+   */
+  acquire(holderId: string, spec: BackgroundAudioSpec, options?: BackgroundAudioOptions): void;
+  /** Release one holder; the loop stops when the last holder releases. */
+  release(holderId: string, opts?: {
+    immediate?: boolean;
+  }): void;
+  /** Direct start (facade `playBackgroundAudio`). Bypasses refcounting. */
+  start(spec: BackgroundAudioSpec, options?: BackgroundAudioOptions): void;
+  /**
+   * Stop the loop. Fade-out is applied when `immediate` is false and a
+   * fadeOutMs was configured; agent-audio preemption should pass
+   * `immediate: true` (a fade would delay real speech behind it).
+   */
+  stop(opts?: {
+    immediate?: boolean;
+  }): void;
+  /** Real agent audio arrived — kill the loop instantly, no fade, no clear. */
+  notifyAgentAudio(): void;
+  private cancelDelay;
+  private startNow;
+  private tick;
+  /** Wrap-around frame extraction for seamless looping. */
+  private extractFrame;
+}
+//#endregion
+//#region src/audio/background/loader.d.ts
+declare function isPresetName(spec: BackgroundAudioSpec): spec is BackgroundAudioPreset;
+/** Resolve a spec to raw μ-law bytes (cached per preset/path). */
+declare function loadBackgroundAudio(spec: BackgroundAudioSpec): Buffer;
+//#endregion
+export { type BackgroundAudioOptions, BackgroundAudioPlayer, type BackgroundAudioPlayerDeps, type BackgroundAudioPreset, type BackgroundAudioSpec, InboundTranscoder, MULAW_FRAME_BYTES_20MS, MULAW_SAMPLE_RATE, MULAW_SILENCE_BYTE, OutboundTranscoder, Resampler, type ResamplerOptions, base64ByteLength, fadeMulaw, isPresetName, loadBackgroundAudio, mulawBytesToMs, mulawDecodeSample, mulawEncodeSample, mulawToPcm16, pcm16ToMulaw, scaleMulaw };