realtime-voice-agents 2.0.1 → 2.2.0

package/dist/index.cjs

@@ -1,12 +1,12 @@
 Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
 const require_rolldown_runtime = require("./rolldown-runtime-VH7oDXx4.cjs");
 const require_events = require("./events-BxDTIKKq.cjs");
-const require_BaseRealtimeProvider = require("./BaseRealtimeProvider-DI4pKtOb.cjs");
+const require_BaseRealtimeProvider = require("./BaseRealtimeProvider-C2mRn1V_.cjs");
 const require_InMemorySessionStore = require("./InMemorySessionStore-B0_i-DOU.cjs");
 const require_rest = require("./rest-BvUKut_k.cjs");
 const require_mulaw = require("./mulaw-DLUObjdP.cjs");
-const require_BackgroundAudioPlayer = require("./BackgroundAudioPlayer-jfRULWKC.cjs");
-const require_session_config = require("./session-config-c8sOw1XL.cjs");
+const require_noise = require("./noise-D9jKottW.cjs");
+const require_session_config = require("./session-config-CqJm2Kxz.cjs");
 let node_fs = require("node:fs");
 let zod = require("zod");
 zod = require_rolldown_runtime.__toESM(zod, 1);
@@ -499,6 +499,190 @@ var PlaybackTracker = class {
 	}
 };
 //#endregion
+//#region src/vad/NoiseAdaptiveVadController.ts
+/**
+* Noise-adaptive VAD policy: watches inbound caller audio through a
+* NoiseFloorEstimator and, on sustained pervasive noise, proposes a bounded
+* turn-detection escalation (server VAD threshold up stepwise; semantic VAD
+* and sensitivity-only providers get suggestion events instead).
+*
+* Modeled on InterruptionController: pure decision objects, zero timers —
+* every clock is measured in milliseconds of inbound audio, so behavior is
+* deterministic in tests and matches wall clock on a live call.
+*
+* Lifecycle: onInboundFrame() PROPOSES without mutating ladder state — the
+* identical proposal recurs until the caller settles it:
+* - commitApplied(): the provider ACKNOWLEDGED the change (auto mode). Only
+*   this and rebase() move the effective config — an emitted-but-unapplied
+*   suggestion is never treated as reality.
+* - markSuggested(): the suggestion was surfaced (suggest mode, or a rung
+*   that is never auto-applied). Latched until rebase() so it fires once.
+* - defer(): the apply failed or timed out — retry after cooldownMs.
+* - rebase(): the app changed VAD via session.updateVad() — new effective
+*   base, fresh ladder. rebase(null) disables the controller entirely (turn
+*   detection off — noise adaptation must never re-enable it on its own).
+*/
+function round2(value) {
+	return Math.round(value * 100) / 100;
+}
+var NoiseAdaptiveVadController = class {
+	windowMs;
+	noiseFloorDb;
+	sustainMs;
+	cooldownMs;
+	maxSteps;
+	thresholdStep;
+	maxThreshold;
+	profile;
+	estimator;
+	/** Current EFFECTIVE config — moved only by commitApplied()/rebase(). */
+	effective;
+	disabled;
+	stepsTaken = 0;
+	suggestionLatched = false;
+	sustainedMs = 0;
+	/** Infinity = no step yet, so the first proposal is never cooldown-blocked. */
+	msSinceLastStep = Number.POSITIVE_INFINITY;
+	elapsedMs = 0;
+	analyzedMs = 0;
+	/** Memoized next rung; `undefined` = dirty, `null` = ladder exhausted. */
+	cachedNext = void 0;
+	constructor(options, effectiveVad, profile) {
+		this.windowMs = options.windowMs ?? 5e3;
+		this.noiseFloorDb = options.noiseFloorDb ?? -45;
+		this.sustainMs = options.sustainMs ?? 3e3;
+		this.cooldownMs = options.cooldownMs ?? 15e3;
+		this.maxSteps = options.maxSteps ?? 1;
+		this.thresholdStep = options.thresholdStep ?? .1;
+		this.maxThreshold = options.maxThreshold ?? .9;
+		this.profile = profile;
+		this.estimator = new require_noise.NoiseFloorEstimator({ windowMs: this.windowMs });
+		this.disabled = effectiveVad === null;
+		this.effective = effectiveVad ?? { type: "server" };
+	}
+	/** No escalation room at all (disabled, or the ladder has no next rung). */
+	get exhausted() {
+		return this.disabled || this.nextRung() === null;
+	}
+	/**
+	* Feed one inbound Twilio frame (base64 μ-law). `excluded` frames (user
+	* speech, agent playback, pregreeting) advance the cooldown clock but are
+	* kept out of the floor estimate — they freeze, not reset, the sustain
+	* counter. Pure w.r.t. ladder state: the same proposal recurs until
+	* commitApplied/markSuggested/defer/rebase.
+	*/
+	onInboundFrame(base64Mulaw, excluded) {
+		if (this.disabled || this.suggestionLatched) return null;
+		const next = this.nextRung();
+		if (!next) return null;
+		const bytes = Buffer.from(base64Mulaw, "base64");
+		if (bytes.length === 0) return null;
+		const frameMs = bytes.length / 8;
+		this.elapsedMs += frameMs;
+		this.msSinceLastStep += frameMs;
+		if (excluded) return null;
+		this.estimator.addAudio(bytes);
+		this.analyzedMs += frameMs;
+		if (!this.estimator.isWarm) return null;
+		const floor = this.estimator.floorDb();
+		if (floor >= this.noiseFloorDb) this.sustainedMs += frameMs;
+		else this.sustainedMs = 0;
+		if (this.sustainedMs < this.sustainMs) return null;
+		if (this.msSinceLastStep < this.cooldownMs) return null;
+		return {
+			previous: { ...this.effective },
+			suggested: next.suggested,
+			step: this.stepsTaken + 1,
+			maxSteps: this.maxSteps,
+			autoApplicable: next.autoApplicable,
+			metrics: {
+				noiseFloorDb: floor,
+				windowMs: this.windowMs,
+				sustainedMs: this.sustainedMs,
+				analyzedMs: this.analyzedMs,
+				elapsedMs: this.elapsedMs
+			}
+		};
+	}
+	/** The proposal was applied AND acknowledged: it is the new effective truth. */
+	commitApplied(adjustment) {
+		this.effective = { ...adjustment.suggested };
+		this.stepsTaken++;
+		this.sustainedMs = 0;
+		this.msSinceLastStep = 0;
+		this.cachedNext = void 0;
+	}
+	/**
+	* The suggestion was emitted without being applied: latch so it fires once
+	* per effective state. The effective config is deliberately NOT moved — an
+	* unapplied suggestion is not reality.
+	*/
+	markSuggested(_adjustment) {
+		this.suggestionLatched = true;
+	}
+	/** The apply failed/timed out/was superseded: retry after cooldownMs. */
+	defer() {
+		this.msSinceLastStep = 0;
+	}
+	/**
+	* App-driven updateVad(): the declared config is the new effective base —
+	* a fresh ladder starts from it (never below it). `null` disables the
+	* controller; a later non-null rebase re-enables it. A full cooldown
+	* applies before the next adaptive step.
+	*/
+	rebase(vad) {
+		this.disabled = vad === null;
+		this.effective = vad ? { ...vad } : { type: "server" };
+		this.stepsTaken = 0;
+		this.suggestionLatched = false;
+		this.sustainedMs = 0;
+		this.msSinceLastStep = 0;
+		this.cachedNext = void 0;
+	}
+	nextRung() {
+		if (this.cachedNext === void 0) this.cachedNext = this.computeNextRung();
+		return this.cachedNext;
+	}
+	computeNextRung() {
+		const effective = this.effective;
+		if (effective.type === "semantic") {
+			if (effective.eagerness === "low") return null;
+			return {
+				suggested: {
+					...effective,
+					eagerness: "low"
+				},
+				autoApplicable: false
+			};
+		}
+		if (this.stepsTaken < this.maxSteps) {
+			const baseline = effective.threshold ?? this.profile?.defaultServerThreshold;
+			if (baseline !== void 0) {
+				const ceiling = Math.min(this.maxThreshold, this.profile?.maxServerThreshold ?? 1);
+				const bottom = this.profile?.minServerThreshold ?? 0;
+				const target = round2(Math.min(Math.max(baseline + this.thresholdStep, bottom), ceiling));
+				if (target > baseline + 1e-9) return {
+					suggested: {
+						...effective,
+						type: "server",
+						threshold: target,
+						startSensitivity: "low"
+					},
+					autoApplicable: true
+				};
+			}
+		}
+		if (effective.startSensitivity !== "low") return {
+			suggested: {
+				...effective,
+				startSensitivity: "low"
+			},
+			autoApplicable: false
+		};
+		return null;
+	}
+};
+//#endregion
 //#region src/providers/base/reconnect.ts
 const DEFAULT_RECONNECT_POLICY = {
 	maxAttempts: 5,
@@ -698,6 +882,12 @@ function createTransferCallTool(options = {}) {
 * socket is released exactly once.
 */
 const MAX_BUFFERED_INBOUND_FRAMES = 250;
+/**
+* Cap on the noise meter's caller-speech exclusion: Gemini never emits a
+* speech-stopped event, and phantom speech starts on a noisy line must not
+* latch the meter off forever.
+*/
+const MAX_SPEECH_EXCLUSION_MS = 15e3;
 var CallSession = class extends require_events.TypedEmitter {
 	callSid;
 	streamSid;
@@ -717,6 +907,8 @@ var CallSession = class extends require_events.TypedEmitter {
 	runningTools = /* @__PURE__ */ new Map();
 	timers = /* @__PURE__ */ new Set();
 	provider = null;
+	/** Primary + fallbacks, in try-order. Only walked while connecting. */
+	providerChain;
 	activeAgentValue;
 	generating = false;
 	currentResponseId = null;
@@ -754,12 +946,28 @@ var CallSession = class extends require_events.TypedEmitter {
 	handoffInProgress = false;
 	/** Pre-synthesized greeting playout state. */
 	pregreeting = null;
+	/** Noise-adaptive VAD (opt-in); built after connect from the provider's ACKed config. */
+	noiseVad = null;
+	/**
+	* DESIRED turn-detection override — drives reconnect/handoff rebuilds via
+	* buildProviderInit. Distinct from the provider's ACKed effective state:
+	* manual updateVad() promotes it immediately (user intent survives network
+	* failures); adaptive applies promote it only after the provider's ack.
+	*/
+	vadOverride = void 0;
+	/** VAD mutations in flight (manual AND adaptive); proposals drop while > 0. */
+	vadOpsInFlight = 0;
+	/** Bumped by every updateVad(); a stale revision means an adaptive apply was superseded. */
+	vadRevision = 0;
+	userSpeechActive = false;
+	userSpeechStartedAtMs = 0;
 	constructor(deps) {
 		super();
 		this.deps = deps;
 		this.callSid = deps.start.start.callSid;
 		this.streamSid = deps.start.start.streamSid ?? deps.start.streamSid;
 		this.log = require_BaseRealtimeProvider.childLogger(deps.logger, { callSid: this.callSid });
+		this.providerChain = [deps.providerFactory, ...deps.fallbacks ?? []];
 		this.activeAgentValue = deps.agent;
 		this.context = new SessionContext(deps.options.context);
 		this.interruptions = new InterruptionController(deps.options.interruptions);
@@ -773,7 +981,7 @@ var CallSession = class extends require_events.TypedEmitter {
 		this.answered = deps.answeredEarly === true || this.callInfo.direction === "inbound";
 		this.agents = collectAgentGraph(deps.agent);
 		this.middlewares = deps.middlewares ?? [];
-		this.bgAudio = new require_BackgroundAudioPlayer.BackgroundAudioPlayer({
+		this.bgAudio = new require_noise.BackgroundAudioPlayer({
 			sendMedia: (payload) => {
 				if (this.stateValue === "active" && this.deps.transport.isOpen) this.deps.transport.sendMedia(payload);
 			},
@@ -798,17 +1006,7 @@ var CallSession = class extends require_events.TypedEmitter {
 	/** Connect the provider and activate the call. Called by the bridge. */
 	async begin() {
 		this.playPreGreeting();
-		try {
-			this.provider = this.deps.providerFactory({
-				logger: this.log,
-				callSid: this.callSid
-			});
-			this.wireProvider(this.provider);
-			await this.provider.connect(this.buildProviderInit());
-		} catch (error) {
-			this.fail(error instanceof Error ? error : new Error(String(error)), "provider-failed");
-			return;
-		}
+		if (!await this.connectInitialProvider()) return;
 		if (this.stateValue !== "connecting") return;
 		this.stateValue = "active";
 		this.emit("provider.connected");
@@ -820,6 +1018,10 @@ var CallSession = class extends require_events.TypedEmitter {
 			to: this.callInfo.to,
 			customParameters: this.callInfo.customParameters
 		});
+		if (this.deps.options.noiseAdaptiveVad && this.deps.options.vad !== null) {
+			this.noiseVad = new NoiseAdaptiveVadController(this.deps.options.noiseAdaptiveVad, this.provider.getEffectiveVad(), this.provider.capabilities.vadTuning);
+			if (this.noiseVad.exhausted) this.log.warn("noiseAdaptiveVad enabled but the effective VAD config leaves no escalation room");
+		}
 		if (this.pregreeting) this.provider.sendText(this.pregreeting.text, {
 			role: "assistant",
 			triggerResponse: false
@@ -830,6 +1032,55 @@ var CallSession = class extends require_events.TypedEmitter {
 		this.saveSnapshot();
 		this.maybeGreet();
 	}
+	/**
+	* Walk primary + fallbacks until one connects. A provider that fails to
+	* come up is detached BEFORE the chain advances, so a half-open socket's
+	* late events can never reach the session once the next provider owns the
+	* call. Connect-time only by design: once a provider has answered, the
+	* call stays with it. Returns false after failing the call (chain
+	* exhausted) or when the call was torn down while connecting.
+	*/
+	async connectInitialProvider() {
+		let lastFrom = "provider";
+		let lastError = null;
+		for (const factory of this.providerChain) {
+			let provider;
+			try {
+				provider = factory({
+					logger: this.log,
+					callSid: this.callSid
+				});
+			} catch (error) {
+				lastError = toError(error);
+				this.log.warn("provider factory threw — trying the next fallback", { error: String(lastError) });
+				continue;
+			}
+			if (lastError) this.emit("provider.fallback", {
+				from: lastFrom,
+				to: provider.name,
+				error: lastError
+			});
+			this.provider = provider;
+			this.wireProvider(provider);
+			try {
+				await provider.connect(this.buildProviderInit());
+				return true;
+			} catch (error) {
+				provider.removeAllListeners();
+				provider.close().catch(() => {});
+				this.provider = null;
+				if (this.stateValue !== "connecting") return false;
+				lastFrom = provider.name;
+				lastError = toError(error);
+				this.log.warn("provider failed to connect", {
+					provider: provider.name,
+					error: String(lastError)
+				});
+			}
+		}
+		this.fail(lastError ?? /* @__PURE__ */ new Error("provider connect failed"), "provider-failed");
+		return false;
+	}
 	/** The outbound leg was answered (host's Twilio status callback). */
 	notifyAnswered() {
 		if (this.answered) return;
@@ -842,6 +1093,36 @@ var CallSession = class extends require_events.TypedEmitter {
 	async updateInstructions(instructions) {
 		await this.provider?.updateSession({ instructions });
 	}
+	/**
+	* Replace the session's turn-detection config mid-call. The value is the
+	* new DESIRED state: it survives reconnects and handoffs, and the
+	* noise-adaptive ladder (when enabled) rebases onto it — future escalation
+	* is relative to this config, never below it. `null` disables turn
+	* detection AND suspends noise adaptation (adaptation never re-enables VAD
+	* on its own); a later non-null call re-enables both.
+	*
+	* Returns true when the live session applied it — provider-acknowledged
+	* when `noiseAdaptiveVad` is configured (serialized updates), best-effort
+	* "sent" otherwise. False when it could not be applied live (no provider,
+	* call ended, no sessionUpdate capability, or the ack timed out — the
+	* resulting reconnect then applies the stored config).
+	*/
+	async updateVad(vad) {
+		this.vadRevision++;
+		this.vadOpsInFlight++;
+		try {
+			this.vadOverride = vad;
+			if (!this.provider || this.stateValue === "ended") return false;
+			if (!this.provider.capabilities.sessionUpdate) {
+				this.log.warn("updateVad: provider has no mid-session update — stored for the next reconnect");
+				return false;
+			}
+			return await this.provider.updateSession({ vad }, { awaitAck: true }) === true;
+		} finally {
+			this.noiseVad?.rebase(vad);
+			this.vadOpsInFlight--;
+		}
+	}
 	/** Manual barge-in: stop the agent mid-sentence. */
 	interrupt() {
 		if (!this.generating && !this.tracker.isPlaybackActive()) return;
@@ -935,8 +1216,9 @@ var CallSession = class extends require_events.TypedEmitter {
 		return {
 			instructions: this.activeAgentValue.resolveInstructions(this.context) + (this.pregreeting ? `\n\nYou already opened the call by saying: "${this.pregreeting.text}". Do not greet again — continue the conversation from there.` : ""),
 			voice: this.activeAgentValue.voice,
-			vad: this.deps.options.vad,
+			vad: this.vadOverride !== void 0 ? this.vadOverride : this.deps.options.vad,
 			bridgeOwnsInterruptions: true,
+			serializedSessionUpdates: this.deps.options.noiseAdaptiveVad !== void 0,
 			tools: [...this.toolset.values()].map((tool) => {
 				const decorated = decorateTool(tool, this.middlewares);
 				return {
@@ -1027,12 +1309,15 @@ var CallSession = class extends require_events.TypedEmitter {
 			this.emit("transcript.user", entry);
 		});
 		provider.on("userSpeechStarted", () => {
+			this.userSpeechActive = true;
+			this.userSpeechStartedAtMs = Date.now();
 			this.clearIdleTimer();
 			this.nudgeCount = 0;
 			this.emit("user.speech.started");
 			this.handleBargeIn();
 		});
 		provider.on("userSpeechStopped", () => {
+			this.userSpeechActive = false;
 			if (this.blockedUserTurn === "speaking") this.blockedUserTurn = "committed";
 			this.emit("user.speech.ended");
 		});
@@ -1054,6 +1339,11 @@ var CallSession = class extends require_events.TypedEmitter {
 	}
 	handleInboundMedia(payload) {
 		if (this.stateValue === "ended" || this.stateValue === "ending") return;
+		if (this.noiseVad) {
+			const excluded = this.isUserSpeechExcluded() || this.tracker.isPlaybackActive() || this.pregreeting !== null && !this.pregreeting.played;
+			const proposal = this.noiseVad.onInboundFrame(payload, excluded);
+			if (proposal) this.handleVadProposal(proposal);
+		}
 		if (this.pregreeting && !this.pregreeting.played) return;
 		if (this.deps.options.deafness.ignoreUserAudioUntilFirstTurnDone && !this.firstTurnDone) return;
 		if (this.deps.options.deafness.muteDuringToolExecution && this.runningTools.size > 0) return;
@@ -1070,6 +1360,51 @@ var CallSession = class extends require_events.TypedEmitter {
 		for (const payload of this.inboundBuffer) this.provider.sendAudio(payload);
 		this.inboundBuffer = [];
 	}
+	/** Caller speech is excluded from the noise floor, capped so a missing
+	* speech-stopped event (Gemini) can't latch the meter off forever. */
+	isUserSpeechExcluded() {
+		return this.userSpeechActive && Date.now() - this.userSpeechStartedAtMs < MAX_SPEECH_EXCLUSION_MS;
+	}
+	handleVadProposal(proposal) {
+		if (this.vadOpsInFlight > 0) return;
+		const willAutoApply = (this.deps.options.noiseAdaptiveVad?.mode ?? "auto") === "auto" && proposal.autoApplicable && this.provider?.capabilities.sessionUpdate === true;
+		this.emit("vad.suggestion", {
+			...proposal,
+			willAutoApply
+		});
+		if (!willAutoApply) {
+			this.noiseVad?.markSuggested(proposal);
+			return;
+		}
+		this.vadOpsInFlight++;
+		const revisionAtPropose = this.vadRevision;
+		this.applyAdaptiveVadUpdate(proposal, revisionAtPropose).catch((error) => {
+			this.noiseVad?.defer();
+			this.log.warn("noise-adaptive VAD apply failed", { error: String(error) });
+		}).finally(() => {
+			this.vadOpsInFlight--;
+		});
+	}
+	/**
+	* Adaptive applies promote NOTHING until the provider ACKs: a timed-out
+	* update must not leak into vadOverride, or the desync-reconnect would
+	* apply a config the controller never committed (dual truth). Contrast
+	* with updateVad(), where the user's declared intent persists regardless.
+	*/
+	async applyAdaptiveVadUpdate(proposal, revisionAtPropose) {
+		const provider = this.provider;
+		if (!provider || this.stateValue === "ended") {
+			this.noiseVad?.defer();
+			return;
+		}
+		const acked = await provider.updateSession({ vad: proposal.suggested }, { awaitAck: true });
+		if (revisionAtPropose !== this.vadRevision) return;
+		if (acked === true) {
+			this.vadOverride = proposal.suggested;
+			this.noiseVad?.commitApplied(proposal);
+			this.emit("vad.adjusted", proposal);
+		} else this.noiseVad?.defer();
+	}
 	handleMarkEcho(name) {
 		if (name === PREGREETING_MARK) {
 			if (this.pregreeting && !this.pregreeting.played) {
@@ -1866,6 +2201,9 @@ var CallSession = class extends require_events.TypedEmitter {
 const PREGREETING_MARK = "pre:greeting";
 /** 400ms per frame — matches production burst-write implementations. */
 const PREGREETING_CHUNK_BYTES = 3200;
+function toError(value) {
+	return value instanceof Error ? value : new Error(String(value));
+}
 function safeJsonStringify(value) {
 	try {
 		return JSON.stringify(value) ?? "null";
@@ -2000,6 +2338,7 @@ var TwilioRealtimeBridge = class extends require_events.TypedEmitter {
 				transport,
 				start,
 				providerFactory: this.config.provider,
+				fallbacks: this.config.fallbacks,
 				agent,
 				options,
 				store: this.store,
@@ -2121,6 +2460,7 @@ exports.DEFAULT_RECONNECT_POLICY = DEFAULT_RECONNECT_POLICY;
 exports.DEFAULT_SESSION_OPTIONS = DEFAULT_SESSION_OPTIONS;
 exports.InMemorySessionStore = require_InMemorySessionStore.InMemorySessionStore;
 exports.InterruptionController = InterruptionController;
+exports.NoiseAdaptiveVadController = NoiseAdaptiveVadController;
 exports.PlaybackTracker = PlaybackTracker;
 exports.SessionContext = SessionContext;
 exports.TwilioRealtimeBridge = TwilioRealtimeBridge;