realtime-voice-agents 2.0.1 → 2.2.0

package/dist/index.d.cts

@@ -1,6 +1,6 @@
 import { n as BackgroundAudioPreset, r as BackgroundAudioSpec, t as BackgroundAudioOptions } from "./presets-Bf75YXs5.cjs";
 import { t as TypedEmitter } from "./events-BUMYdETO.cjs";
-import { _ as noopLogger, a as ProviderToolSchema, c as VadConfig, d as ProviderEvents, f as ProviderToolCall, g as consoleLogger, h as Logger, i as ProviderSessionInit, l as ProviderAudioDelta, m as ProviderCapabilities, n as ProviderFactory, o as SendTextOptions, p as ProviderUsage, r as ProviderFactoryContext, s as SendToolResultOptions, t as BaseRealtimeProvider, u as ProviderCloseInfo } from "./BaseRealtimeProvider-BehPNT1r.cjs";
+import { _ as Logger, a as ProviderToolSchema, c as SessionUpdateOptions, d as ProviderCloseInfo, f as ProviderEvents, g as VadTuningProfile, h as ProviderCapabilities, i as ProviderSessionInit, l as VadConfig, m as ProviderUsage, n as ProviderFactory, o as SendTextOptions, p as ProviderToolCall, r as ProviderFactoryContext, s as SendToolResultOptions, t as BaseRealtimeProvider, u as ProviderAudioDelta, v as consoleLogger, y as noopLogger } from "./BaseRealtimeProvider-BL75_HHh.cjs";
 import { i as WebSocketLike, m as TwilioStartEvent, r as TwilioMediaTransport } from "./transport-B_PJFIVd.cjs";
 import { a as emptyUsage, i as UsageInfo, n as SessionStore, o as TranscriptEntry, r as CallSnapshot, t as InMemorySessionStore } from "./InMemorySessionStore-B5_rq61L.cjs";
 import { i as TwilioRestClient, n as connectStreamTwiml } from "./twiml-z9LjoF4_.cjs";
@@ -262,6 +262,112 @@ declare class InterruptionController {
   private guardActive;
 }
 //#endregion
+//#region src/vad/NoiseAdaptiveVadController.d.ts
+interface NoiseAdaptiveVadOptions {
+  /**
+   * 'auto' applies each escalation via the provider's mid-session update
+   * (ack-gated) and emits `vad.adjusted`; 'suggest' only emits
+   * `vad.suggestion` and leaves applying to the app. Default 'auto'.
+   */
+  mode?: 'auto' | 'suggest';
+  /** Sliding analysis window, in ms of ANALYZED (non-excluded) audio. Default 5000. */
+  windowMs?: number;
+  /** Noise floor (dBFS re PCM16 full scale) at/above which the line counts as noisy. Default -45. */
+  noiseFloorDb?: number;
+  /** How long the floor must hold, in ms of analyzed audio, before a step. Default 3000. */
+  sustainMs?: number;
+  /** Minimum inbound audio between steps (and between apply retries). Default 15000. */
+  cooldownMs?: number;
+  /** Maximum committed escalation steps per call. Default 1. */
+  maxSteps?: number;
+  /** Server-VAD threshold increase per step. Default 0.1. */
+  thresholdStep?: number;
+  /** Server-VAD threshold ceiling (intersected with the provider profile range). Default 0.9. */
+  maxThreshold?: number;
+}
+interface VadNoiseMetrics {
+  /** Estimated noise floor at decision time, dBFS. */
+  noiseFloorDb: number;
+  /** The analysis window the floor was computed over. */
+  windowMs: number;
+  /** Analyzed audio with the floor continuously at/above the trigger. */
+  sustainedMs: number;
+  /** Non-excluded audio metered so far. */
+  analyzedMs: number;
+  /** ALL inbound audio observed (deterministic wall-clock proxy) — field data for tuning windowMs. */
+  elapsedMs: number;
+}
+interface VadAdjustment {
+  /** Effective config before the decision (ACKed truth; `{ type: 'server' }` synthesized when unset). */
+  previous: VadConfig;
+  /** Recommended (suggest) or applied (auto) config. Escalation only. */
+  suggested: VadConfig;
+  /** 1-based step number this proposal would commit. */
+  step: number;
+  maxSteps: number;
+  /**
+   * POLICY: is this kind of change safe to apply automatically? Server-VAD
+   * threshold rungs are; semantic eagerness (an end-of-turn latency control)
+   * and sensitivity-only rungs are surfaced as suggestions instead. Runtime
+   * gating (mode/capability) is reported separately as `willAutoApply`.
+   */
+  autoApplicable: boolean;
+  metrics: VadNoiseMetrics;
+}
+declare class NoiseAdaptiveVadController {
+  private readonly windowMs;
+  private readonly noiseFloorDb;
+  private readonly sustainMs;
+  private readonly cooldownMs;
+  private readonly maxSteps;
+  private readonly thresholdStep;
+  private readonly maxThreshold;
+  private readonly profile;
+  private readonly estimator;
+  /** Current EFFECTIVE config — moved only by commitApplied()/rebase(). */
+  private effective;
+  private disabled;
+  private stepsTaken;
+  private suggestionLatched;
+  private sustainedMs;
+  /** Infinity = no step yet, so the first proposal is never cooldown-blocked. */
+  private msSinceLastStep;
+  private elapsedMs;
+  private analyzedMs;
+  /** Memoized next rung; `undefined` = dirty, `null` = ladder exhausted. */
+  private cachedNext;
+  constructor(options: NoiseAdaptiveVadOptions, effectiveVad: VadConfig | null | undefined, profile: VadTuningProfile | undefined);
+  /** No escalation room at all (disabled, or the ladder has no next rung). */
+  get exhausted(): boolean;
+  /**
+   * Feed one inbound Twilio frame (base64 μ-law). `excluded` frames (user
+   * speech, agent playback, pregreeting) advance the cooldown clock but are
+   * kept out of the floor estimate — they freeze, not reset, the sustain
+   * counter. Pure w.r.t. ladder state: the same proposal recurs until
+   * commitApplied/markSuggested/defer/rebase.
+   */
+  onInboundFrame(base64Mulaw: string, excluded: boolean): VadAdjustment | null;
+  /** The proposal was applied AND acknowledged: it is the new effective truth. */
+  commitApplied(adjustment: VadAdjustment): void;
+  /**
+   * The suggestion was emitted without being applied: latch so it fires once
+   * per effective state. The effective config is deliberately NOT moved — an
+   * unapplied suggestion is not reality.
+   */
+  markSuggested(_adjustment: VadAdjustment): void;
+  /** The apply failed/timed out/was superseded: retry after cooldownMs. */
+  defer(): void;
+  /**
+   * App-driven updateVad(): the declared config is the new effective base —
+   * a fresh ladder starts from it (never below it). `null` disables the
+   * controller; a later non-null rebase re-enables it. A full cooldown
+   * applies before the next adaptive step.
+   */
+  rebase(vad: VadConfig | null): void;
+  private nextRung;
+  private computeNextRung;
+}
+//#endregion
 //#region src/providers/base/reconnect.d.ts
 /** Exponential backoff with optional full jitter, shared by all providers. */
 interface ReconnectPolicy {
@@ -348,6 +454,14 @@ interface SessionOptions {
   hangup: HangupOptions;
   /** Normalized VAD, mapped to the provider's native config. */
   vad?: VadConfig | null;
+  /**
+   * Opt-in noise-adaptive VAD escalation: analyze inbound caller audio and,
+   * on sustained background noise, raise turn-detection settings mid-call
+   * (auto-applied where the provider supports acknowledged session updates;
+   * suggestion events otherwise). Set `{}` to enable with defaults. Enabling
+   * this also serializes ALL mid-call session updates for the session.
+   */
+  noiseAdaptiveVad?: NoiseAdaptiveVadOptions;
   /**
    * `afterPlayback` (default): tool results wait until current agent audio
    * finishes playing. `immediate`: send as soon as the tool completes.
@@ -421,6 +535,19 @@ interface BridgeConfig {
   /** The (root) agent, or a resolver for multi-tenant routing per call. */
   agent: Agent | ((start: TwilioStartEvent) => Agent | Promise<Agent>);
   provider: ProviderFactory;
+  /**
+   * Backup providers tried in order when `provider` fails to come up at call
+   * start (bad key, exhausted quota, outage, connect timeout). Each attempt
+   * emits `provider.fallback`; the call fails only when every fallback is
+   * exhausted. Connect-time only: once a provider answers, the call stays
+   * with it (mid-call reconnects keep using the same provider).
+   *
+   * ```ts
+   * provider: openaiRealtime(),
+   * fallbacks: [xaiRealtime(), geminiLive()],
+   * ```
+   */
+  fallbacks?: ProviderFactory[];
   session?: Partial<SessionOptions> | ((start: TwilioStartEvent) => Partial<SessionOptions> | Promise<Partial<SessionOptions>>);
   /** Twilio REST credentials — enables clean hangup, transfer, SMS/WhatsApp. */
   twilio?: TwilioRestOptions;
@@ -470,6 +597,28 @@ interface ApprovalRequestInfo {
   agentId: string;
   expiresAtMs: number;
 }
+/**
+ * A noise-adaptive VAD suggestion. `autoApplicable` is policy ("is this kind
+ * of change safe to auto-apply?"); `willAutoApply` is runtime — false in
+ * suggest mode or when the provider has no mid-session update, even for an
+ * auto-applicable change.
+ */
+type VadSuggestionInfo = VadAdjustment & {
+  willAutoApply: boolean;
+};
+/**
+ * While ESTABLISHING the call, a provider failed to come up and the session
+ * is trying the next factory in the fallback chain. Fires only before the
+ * call is active — once a provider has connected, the call stays with it.
+ */
+interface ProviderFallbackInfo {
+  /** Name of the provider whose connect failed. */
+  from: string;
+  /** Name of the provider being tried instead. */
+  to: string;
+  /** The connect error that triggered the fallback. */
+  error: Error;
+}
 interface SessionEventMap {
   'call.started': (info: CallStartedInfo) => void;
   'call.ended': (info: {
@@ -488,6 +637,8 @@ interface SessionEventMap {
     code?: number;
     reason?: string;
   }) => void;
+  /** Connect-time fallback: trying the next provider in the configured chain. */
+  'provider.fallback': (info: ProviderFallbackInfo) => void;
   /** Generation-side: the model started/finished producing a response. */
   'agent.speech.started': (info: {
     responseId: string;
@@ -527,6 +678,10 @@ interface SessionEventMap {
   'interruption.blocked': (info: {
     cause: string;
   }) => void;
+  /** Sustained background noise detected; an escalation is recommended (fires in both modes). */
+  'vad.suggestion': (info: VadSuggestionInfo) => void;
+  /** The escalation was applied AND acknowledged by the provider (auto mode only). */
+  'vad.adjusted': (info: VadAdjustment) => void;
   'background_audio.started': (info: {
     preset?: string;
   }) => void;
@@ -556,6 +711,8 @@ interface CallSessionDeps {
   transport: TwilioMediaTransport;
   start: TwilioStartEvent;
   providerFactory: ProviderFactory;
+  /** Backup factories tried in order when `providerFactory` fails to connect. */
+  fallbacks?: readonly ProviderFactory[];
   agent: Agent;
   options: SessionOptions;
   store: SessionStore;
@@ -586,6 +743,8 @@ declare class CallSession extends TypedEmitter<SessionEventMap> {
   private readonly runningTools;
   private readonly timers;
   private provider;
+  /** Primary + fallbacks, in try-order. Only walked while connecting. */
+  private readonly providerChain;
   private activeAgentValue;
   private generating;
   private currentResponseId;
@@ -623,6 +782,21 @@ declare class CallSession extends TypedEmitter<SessionEventMap> {
   private handoffInProgress;
   /** Pre-synthesized greeting playout state. */
   private pregreeting;
+  /** Noise-adaptive VAD (opt-in); built after connect from the provider's ACKed config. */
+  private noiseVad;
+  /**
+   * DESIRED turn-detection override — drives reconnect/handoff rebuilds via
+   * buildProviderInit. Distinct from the provider's ACKed effective state:
+   * manual updateVad() promotes it immediately (user intent survives network
+   * failures); adaptive applies promote it only after the provider's ack.
+   */
+  private vadOverride;
+  /** VAD mutations in flight (manual AND adaptive); proposals drop while > 0. */
+  private vadOpsInFlight;
+  /** Bumped by every updateVad(); a stale revision means an adaptive apply was superseded. */
+  private vadRevision;
+  private userSpeechActive;
+  private userSpeechStartedAtMs;
   constructor(deps: CallSessionDeps);
   get state(): CallState;
   get activeAgent(): Agent;
@@ -630,6 +804,15 @@ declare class CallSession extends TypedEmitter<SessionEventMap> {
   get transcript(): readonly TranscriptEntry[];
   /** Connect the provider and activate the call. Called by the bridge. */
   begin(): Promise<void>;
+  /**
+   * Walk primary + fallbacks until one connects. A provider that fails to
+   * come up is detached BEFORE the chain advances, so a half-open socket's
+   * late events can never reach the session once the next provider owns the
+   * call. Connect-time only by design: once a provider has answered, the
+   * call stays with it. Returns false after failing the call (chain
+   * exhausted) or when the call was torn down while connecting.
+   */
+  private connectInitialProvider;
   /** The outbound leg was answered (host's Twilio status callback). */
   notifyAnswered(): void;
   sendText(text: string, options?: {
@@ -637,6 +820,21 @@ declare class CallSession extends TypedEmitter<SessionEventMap> {
     triggerResponse?: boolean;
   }): void;
   updateInstructions(instructions: string): Promise<void>;
+  /**
+   * Replace the session's turn-detection config mid-call. The value is the
+   * new DESIRED state: it survives reconnects and handoffs, and the
+   * noise-adaptive ladder (when enabled) rebases onto it — future escalation
+   * is relative to this config, never below it. `null` disables turn
+   * detection AND suspends noise adaptation (adaptation never re-enables VAD
+   * on its own); a later non-null call re-enables both.
+   *
+   * Returns true when the live session applied it — provider-acknowledged
+   * when `noiseAdaptiveVad` is configured (serialized updates), best-effort
+   * "sent" otherwise. False when it could not be applied live (no provider,
+   * call ended, no sessionUpdate capability, or the ack timed out — the
+   * resulting reconnect then applies the stored config).
+   */
+  updateVad(vad: VadConfig | null): Promise<boolean>;
   /** Manual barge-in: stop the agent mid-sentence. */
   interrupt(): void;
   /**
@@ -672,6 +870,17 @@ declare class CallSession extends TypedEmitter<SessionEventMap> {
   private wireProvider;
   private handleInboundMedia;
   private flushInboundBuffer;
+  /** Caller speech is excluded from the noise floor, capped so a missing
+   * speech-stopped event (Gemini) can't latch the meter off forever. */
+  private isUserSpeechExcluded;
+  private handleVadProposal;
+  /**
+   * Adaptive applies promote NOTHING until the provider ACKs: a timed-out
+   * update must not leak into vadOverride, or the desync-reconnect would
+   * apply a config the controller never committed (dual truth). Contrast
+   * with updateVad(), where the user's declared intent persists regardless.
+   */
+  private applyAdaptiveVadUpdate;
   private handleMarkEcho;
   private onPlaybackFinished;
   private handleBargeIn;
@@ -917,4 +1126,4 @@ declare class PlaybackTracker {
   private maybeForget;
 }
 //#endregion
-export { Agent, type AgentDefinition, type ApprovalRequestInfo, type BackgroundAudioOptions, type BackgroundAudioPreset, type BackgroundAudioSpec, BaseRealtimeProvider, type BridgeConfig, type BridgeEventMap, type BuiltinToolsConfig, type CallEndReason, CallSession, type CallSessionFacade, type CallSnapshot, type CallStartedInfo, type CallState, type CaptureGreetingOptions, type CapturedGreeting, DEFAULT_RECONNECT_POLICY, DEFAULT_SESSION_OPTIONS, type DeafnessOptions, type FinishCallToolOptions, type GreetingOptions, type HandoffDirective, type HangupOptions, type IdleOptions, InMemorySessionStore, type InferSchema, type InterruptionBlockCause, InterruptionController, type InterruptionDecision, type InterruptionRateLimit, type InterruptionSettings, type Logger, type MarkEchoResult, PlaybackTracker, type ProviderAudioDelta, type ProviderCapabilities, type ProviderCloseInfo, type ProviderEvents, type ProviderFactory, type ProviderFactoryContext, type ProviderSessionInit, type ProviderToolCall, type ProviderToolSchema, type ProviderUsage, type ReconnectPolicy, type SendTextOptions, type SendToolResultOptions, SessionContext, type SessionEventMap, type SessionOptions, type SessionStore, type Tool, type ToolCallInfo, type ToolContext, type ToolDecoration, type ToolDefinition, type ToolMiddleware, type ToolRunInfo, type ToolStrategy, type TranscriptEntry, type TransferCallToolOptions, TwilioRealtimeBridge, type TwilioRestOptions, type TwilioStartEvent, type UsageInfo, type VadConfig, type WebSocketLike, type ZodSchemaLike, captureGreetingAudio, collectAgentGraph, composeExecution, connectStreamTwiml, consoleLogger, createFinishCallTool, createHandoffTool, createTransferCallTool, decorateTool, emptyUsage, handoffToolName, isHandoffDirective, noopLogger, resolveSessionOptions, tool, zodToJsonSchema };
+export { Agent, type AgentDefinition, type ApprovalRequestInfo, type BackgroundAudioOptions, type BackgroundAudioPreset, type BackgroundAudioSpec, BaseRealtimeProvider, type BridgeConfig, type BridgeEventMap, type BuiltinToolsConfig, type CallEndReason, CallSession, type CallSessionFacade, type CallSnapshot, type CallStartedInfo, type CallState, type CaptureGreetingOptions, type CapturedGreeting, DEFAULT_RECONNECT_POLICY, DEFAULT_SESSION_OPTIONS, type DeafnessOptions, type FinishCallToolOptions, type GreetingOptions, type HandoffDirective, type HangupOptions, type IdleOptions, InMemorySessionStore, type InferSchema, type InterruptionBlockCause, InterruptionController, type InterruptionDecision, type InterruptionRateLimit, type InterruptionSettings, type Logger, type MarkEchoResult, NoiseAdaptiveVadController, type NoiseAdaptiveVadOptions, PlaybackTracker, type ProviderAudioDelta, type ProviderCapabilities, type ProviderCloseInfo, type ProviderEvents, type ProviderFactory, type ProviderFactoryContext, type ProviderFallbackInfo, type ProviderSessionInit, type ProviderToolCall, type ProviderToolSchema, type ProviderUsage, type ReconnectPolicy, type SendTextOptions, type SendToolResultOptions, SessionContext, type SessionEventMap, type SessionOptions, type SessionStore, type SessionUpdateOptions, type Tool, type ToolCallInfo, type ToolContext, type ToolDecoration, type ToolDefinition, type ToolMiddleware, type ToolRunInfo, type ToolStrategy, type TranscriptEntry, type TransferCallToolOptions, TwilioRealtimeBridge, type TwilioRestOptions, type TwilioStartEvent, type UsageInfo, type VadAdjustment, type VadConfig, type VadNoiseMetrics, type VadSuggestionInfo, type VadTuningProfile, type WebSocketLike, type ZodSchemaLike, captureGreetingAudio, collectAgentGraph, composeExecution, connectStreamTwiml, consoleLogger, createFinishCallTool, createHandoffTool, createTransferCallTool, decorateTool, emptyUsage, handoffToolName, isHandoffDirective, noopLogger, resolveSessionOptions, tool, zodToJsonSchema };

package/dist/index.d.mts

@@ -1,6 +1,6 @@
 import { n as BackgroundAudioPreset, r as BackgroundAudioSpec, t as BackgroundAudioOptions } from "./presets-Bf75YXs5.mjs";
 import { t as TypedEmitter } from "./events-BUMYdETO.mjs";
-import { _ as noopLogger, a as ProviderToolSchema, c as VadConfig, d as ProviderEvents, f as ProviderToolCall, g as consoleLogger, h as Logger, i as ProviderSessionInit, l as ProviderAudioDelta, m as ProviderCapabilities, n as ProviderFactory, o as SendTextOptions, p as ProviderUsage, r as ProviderFactoryContext, s as SendToolResultOptions, t as BaseRealtimeProvider, u as ProviderCloseInfo } from "./BaseRealtimeProvider-ClP8Wx1X.mjs";
+import { _ as Logger, a as ProviderToolSchema, c as SessionUpdateOptions, d as ProviderCloseInfo, f as ProviderEvents, g as VadTuningProfile, h as ProviderCapabilities, i as ProviderSessionInit, l as VadConfig, m as ProviderUsage, n as ProviderFactory, o as SendTextOptions, p as ProviderToolCall, r as ProviderFactoryContext, s as SendToolResultOptions, t as BaseRealtimeProvider, u as ProviderAudioDelta, v as consoleLogger, y as noopLogger } from "./BaseRealtimeProvider-CWJ81HIt.mjs";
 import { i as WebSocketLike, m as TwilioStartEvent, r as TwilioMediaTransport } from "./transport-CEaLFV4E.mjs";
 import { a as emptyUsage, i as UsageInfo, n as SessionStore, o as TranscriptEntry, r as CallSnapshot, t as InMemorySessionStore } from "./InMemorySessionStore-B5_rq61L.mjs";
 import { i as TwilioRestClient, n as connectStreamTwiml } from "./twiml-z9LjoF4_.mjs";
@@ -262,6 +262,112 @@ declare class InterruptionController {
   private guardActive;
 }
 //#endregion
+//#region src/vad/NoiseAdaptiveVadController.d.ts
+interface NoiseAdaptiveVadOptions {
+  /**
+   * 'auto' applies each escalation via the provider's mid-session update
+   * (ack-gated) and emits `vad.adjusted`; 'suggest' only emits
+   * `vad.suggestion` and leaves applying to the app. Default 'auto'.
+   */
+  mode?: 'auto' | 'suggest';
+  /** Sliding analysis window, in ms of ANALYZED (non-excluded) audio. Default 5000. */
+  windowMs?: number;
+  /** Noise floor (dBFS re PCM16 full scale) at/above which the line counts as noisy. Default -45. */
+  noiseFloorDb?: number;
+  /** How long the floor must hold, in ms of analyzed audio, before a step. Default 3000. */
+  sustainMs?: number;
+  /** Minimum inbound audio between steps (and between apply retries). Default 15000. */
+  cooldownMs?: number;
+  /** Maximum committed escalation steps per call. Default 1. */
+  maxSteps?: number;
+  /** Server-VAD threshold increase per step. Default 0.1. */
+  thresholdStep?: number;
+  /** Server-VAD threshold ceiling (intersected with the provider profile range). Default 0.9. */
+  maxThreshold?: number;
+}
+interface VadNoiseMetrics {
+  /** Estimated noise floor at decision time, dBFS. */
+  noiseFloorDb: number;
+  /** The analysis window the floor was computed over. */
+  windowMs: number;
+  /** Analyzed audio with the floor continuously at/above the trigger. */
+  sustainedMs: number;
+  /** Non-excluded audio metered so far. */
+  analyzedMs: number;
+  /** ALL inbound audio observed (deterministic wall-clock proxy) — field data for tuning windowMs. */
+  elapsedMs: number;
+}
+interface VadAdjustment {
+  /** Effective config before the decision (ACKed truth; `{ type: 'server' }` synthesized when unset). */
+  previous: VadConfig;
+  /** Recommended (suggest) or applied (auto) config. Escalation only. */
+  suggested: VadConfig;
+  /** 1-based step number this proposal would commit. */
+  step: number;
+  maxSteps: number;
+  /**
+   * POLICY: is this kind of change safe to apply automatically? Server-VAD
+   * threshold rungs are; semantic eagerness (an end-of-turn latency control)
+   * and sensitivity-only rungs are surfaced as suggestions instead. Runtime
+   * gating (mode/capability) is reported separately as `willAutoApply`.
+   */
+  autoApplicable: boolean;
+  metrics: VadNoiseMetrics;
+}
+declare class NoiseAdaptiveVadController {
+  private readonly windowMs;
+  private readonly noiseFloorDb;
+  private readonly sustainMs;
+  private readonly cooldownMs;
+  private readonly maxSteps;
+  private readonly thresholdStep;
+  private readonly maxThreshold;
+  private readonly profile;
+  private readonly estimator;
+  /** Current EFFECTIVE config — moved only by commitApplied()/rebase(). */
+  private effective;
+  private disabled;
+  private stepsTaken;
+  private suggestionLatched;
+  private sustainedMs;
+  /** Infinity = no step yet, so the first proposal is never cooldown-blocked. */
+  private msSinceLastStep;
+  private elapsedMs;
+  private analyzedMs;
+  /** Memoized next rung; `undefined` = dirty, `null` = ladder exhausted. */
+  private cachedNext;
+  constructor(options: NoiseAdaptiveVadOptions, effectiveVad: VadConfig | null | undefined, profile: VadTuningProfile | undefined);
+  /** No escalation room at all (disabled, or the ladder has no next rung). */
+  get exhausted(): boolean;
+  /**
+   * Feed one inbound Twilio frame (base64 μ-law). `excluded` frames (user
+   * speech, agent playback, pregreeting) advance the cooldown clock but are
+   * kept out of the floor estimate — they freeze, not reset, the sustain
+   * counter. Pure w.r.t. ladder state: the same proposal recurs until
+   * commitApplied/markSuggested/defer/rebase.
+   */
+  onInboundFrame(base64Mulaw: string, excluded: boolean): VadAdjustment | null;
+  /** The proposal was applied AND acknowledged: it is the new effective truth. */
+  commitApplied(adjustment: VadAdjustment): void;
+  /**
+   * The suggestion was emitted without being applied: latch so it fires once
+   * per effective state. The effective config is deliberately NOT moved — an
+   * unapplied suggestion is not reality.
+   */
+  markSuggested(_adjustment: VadAdjustment): void;
+  /** The apply failed/timed out/was superseded: retry after cooldownMs. */
+  defer(): void;
+  /**
+   * App-driven updateVad(): the declared config is the new effective base —
+   * a fresh ladder starts from it (never below it). `null` disables the
+   * controller; a later non-null rebase re-enables it. A full cooldown
+   * applies before the next adaptive step.
+   */
+  rebase(vad: VadConfig | null): void;
+  private nextRung;
+  private computeNextRung;
+}
+//#endregion
 //#region src/providers/base/reconnect.d.ts
 /** Exponential backoff with optional full jitter, shared by all providers. */
 interface ReconnectPolicy {
@@ -348,6 +454,14 @@ interface SessionOptions {
   hangup: HangupOptions;
   /** Normalized VAD, mapped to the provider's native config. */
   vad?: VadConfig | null;
+  /**
+   * Opt-in noise-adaptive VAD escalation: analyze inbound caller audio and,
+   * on sustained background noise, raise turn-detection settings mid-call
+   * (auto-applied where the provider supports acknowledged session updates;
+   * suggestion events otherwise). Set `{}` to enable with defaults. Enabling
+   * this also serializes ALL mid-call session updates for the session.
+   */
+  noiseAdaptiveVad?: NoiseAdaptiveVadOptions;
   /**
    * `afterPlayback` (default): tool results wait until current agent audio
    * finishes playing. `immediate`: send as soon as the tool completes.
@@ -421,6 +535,19 @@ interface BridgeConfig {
   /** The (root) agent, or a resolver for multi-tenant routing per call. */
   agent: Agent | ((start: TwilioStartEvent) => Agent | Promise<Agent>);
   provider: ProviderFactory;
+  /**
+   * Backup providers tried in order when `provider` fails to come up at call
+   * start (bad key, exhausted quota, outage, connect timeout). Each attempt
+   * emits `provider.fallback`; the call fails only when every fallback is
+   * exhausted. Connect-time only: once a provider answers, the call stays
+   * with it (mid-call reconnects keep using the same provider).
+   *
+   * ```ts
+   * provider: openaiRealtime(),
+   * fallbacks: [xaiRealtime(), geminiLive()],
+   * ```
+   */
+  fallbacks?: ProviderFactory[];
   session?: Partial<SessionOptions> | ((start: TwilioStartEvent) => Partial<SessionOptions> | Promise<Partial<SessionOptions>>);
   /** Twilio REST credentials — enables clean hangup, transfer, SMS/WhatsApp. */
   twilio?: TwilioRestOptions;
@@ -470,6 +597,28 @@ interface ApprovalRequestInfo {
   agentId: string;
   expiresAtMs: number;
 }
+/**
+ * A noise-adaptive VAD suggestion. `autoApplicable` is policy ("is this kind
+ * of change safe to auto-apply?"); `willAutoApply` is runtime — false in
+ * suggest mode or when the provider has no mid-session update, even for an
+ * auto-applicable change.
+ */
+type VadSuggestionInfo = VadAdjustment & {
+  willAutoApply: boolean;
+};
+/**
+ * While ESTABLISHING the call, a provider failed to come up and the session
+ * is trying the next factory in the fallback chain. Fires only before the
+ * call is active — once a provider has connected, the call stays with it.
+ */
+interface ProviderFallbackInfo {
+  /** Name of the provider whose connect failed. */
+  from: string;
+  /** Name of the provider being tried instead. */
+  to: string;
+  /** The connect error that triggered the fallback. */
+  error: Error;
+}
 interface SessionEventMap {
   'call.started': (info: CallStartedInfo) => void;
   'call.ended': (info: {
@@ -488,6 +637,8 @@ interface SessionEventMap {
     code?: number;
     reason?: string;
   }) => void;
+  /** Connect-time fallback: trying the next provider in the configured chain. */
+  'provider.fallback': (info: ProviderFallbackInfo) => void;
   /** Generation-side: the model started/finished producing a response. */
   'agent.speech.started': (info: {
     responseId: string;
@@ -527,6 +678,10 @@ interface SessionEventMap {
   'interruption.blocked': (info: {
     cause: string;
   }) => void;
+  /** Sustained background noise detected; an escalation is recommended (fires in both modes). */
+  'vad.suggestion': (info: VadSuggestionInfo) => void;
+  /** The escalation was applied AND acknowledged by the provider (auto mode only). */
+  'vad.adjusted': (info: VadAdjustment) => void;
   'background_audio.started': (info: {
     preset?: string;
   }) => void;
@@ -556,6 +711,8 @@ interface CallSessionDeps {
   transport: TwilioMediaTransport;
   start: TwilioStartEvent;
   providerFactory: ProviderFactory;
+  /** Backup factories tried in order when `providerFactory` fails to connect. */
+  fallbacks?: readonly ProviderFactory[];
   agent: Agent;
   options: SessionOptions;
   store: SessionStore;
@@ -586,6 +743,8 @@ declare class CallSession extends TypedEmitter<SessionEventMap> {
   private readonly runningTools;
   private readonly timers;
   private provider;
+  /** Primary + fallbacks, in try-order. Only walked while connecting. */
+  private readonly providerChain;
   private activeAgentValue;
   private generating;
   private currentResponseId;
@@ -623,6 +782,21 @@ declare class CallSession extends TypedEmitter<SessionEventMap> {
   private handoffInProgress;
   /** Pre-synthesized greeting playout state. */
   private pregreeting;
+  /** Noise-adaptive VAD (opt-in); built after connect from the provider's ACKed config. */
+  private noiseVad;
+  /**
+   * DESIRED turn-detection override — drives reconnect/handoff rebuilds via
+   * buildProviderInit. Distinct from the provider's ACKed effective state:
+   * manual updateVad() promotes it immediately (user intent survives network
+   * failures); adaptive applies promote it only after the provider's ack.
+   */
+  private vadOverride;
+  /** VAD mutations in flight (manual AND adaptive); proposals drop while > 0. */
+  private vadOpsInFlight;
+  /** Bumped by every updateVad(); a stale revision means an adaptive apply was superseded. */
+  private vadRevision;
+  private userSpeechActive;
+  private userSpeechStartedAtMs;
   constructor(deps: CallSessionDeps);
   get state(): CallState;
   get activeAgent(): Agent;
@@ -630,6 +804,15 @@ declare class CallSession extends TypedEmitter<SessionEventMap> {
   get transcript(): readonly TranscriptEntry[];
   /** Connect the provider and activate the call. Called by the bridge. */
   begin(): Promise<void>;
+  /**
+   * Walk primary + fallbacks until one connects. A provider that fails to
+   * come up is detached BEFORE the chain advances, so a half-open socket's
+   * late events can never reach the session once the next provider owns the
+   * call. Connect-time only by design: once a provider has answered, the
+   * call stays with it. Returns false after failing the call (chain
+   * exhausted) or when the call was torn down while connecting.
+   */
+  private connectInitialProvider;
   /** The outbound leg was answered (host's Twilio status callback). */
   notifyAnswered(): void;
   sendText(text: string, options?: {
@@ -637,6 +820,21 @@ declare class CallSession extends TypedEmitter<SessionEventMap> {
     triggerResponse?: boolean;
   }): void;
   updateInstructions(instructions: string): Promise<void>;
+  /**
+   * Replace the session's turn-detection config mid-call. The value is the
+   * new DESIRED state: it survives reconnects and handoffs, and the
+   * noise-adaptive ladder (when enabled) rebases onto it — future escalation
+   * is relative to this config, never below it. `null` disables turn
+   * detection AND suspends noise adaptation (adaptation never re-enables VAD
+   * on its own); a later non-null call re-enables both.
+   *
+   * Returns true when the live session applied it — provider-acknowledged
+   * when `noiseAdaptiveVad` is configured (serialized updates), best-effort
+   * "sent" otherwise. False when it could not be applied live (no provider,
+   * call ended, no sessionUpdate capability, or the ack timed out — the
+   * resulting reconnect then applies the stored config).
+   */
+  updateVad(vad: VadConfig | null): Promise<boolean>;
   /** Manual barge-in: stop the agent mid-sentence. */
   interrupt(): void;
   /**
@@ -672,6 +870,17 @@ declare class CallSession extends TypedEmitter<SessionEventMap> {
   private wireProvider;
   private handleInboundMedia;
   private flushInboundBuffer;
+  /** Caller speech is excluded from the noise floor, capped so a missing
+   * speech-stopped event (Gemini) can't latch the meter off forever. */
+  private isUserSpeechExcluded;
+  private handleVadProposal;
+  /**
+   * Adaptive applies promote NOTHING until the provider ACKs: a timed-out
+   * update must not leak into vadOverride, or the desync-reconnect would
+   * apply a config the controller never committed (dual truth). Contrast
+   * with updateVad(), where the user's declared intent persists regardless.
+   */
+  private applyAdaptiveVadUpdate;
   private handleMarkEcho;
   private onPlaybackFinished;
   private handleBargeIn;
@@ -917,4 +1126,4 @@ declare class PlaybackTracker {
   private maybeForget;
 }
 //#endregion
-export { Agent, type AgentDefinition, type ApprovalRequestInfo, type BackgroundAudioOptions, type BackgroundAudioPreset, type BackgroundAudioSpec, BaseRealtimeProvider, type BridgeConfig, type BridgeEventMap, type BuiltinToolsConfig, type CallEndReason, CallSession, type CallSessionFacade, type CallSnapshot, type CallStartedInfo, type CallState, type CaptureGreetingOptions, type CapturedGreeting, DEFAULT_RECONNECT_POLICY, DEFAULT_SESSION_OPTIONS, type DeafnessOptions, type FinishCallToolOptions, type GreetingOptions, type HandoffDirective, type HangupOptions, type IdleOptions, InMemorySessionStore, type InferSchema, type InterruptionBlockCause, InterruptionController, type InterruptionDecision, type InterruptionRateLimit, type InterruptionSettings, type Logger, type MarkEchoResult, PlaybackTracker, type ProviderAudioDelta, type ProviderCapabilities, type ProviderCloseInfo, type ProviderEvents, type ProviderFactory, type ProviderFactoryContext, type ProviderSessionInit, type ProviderToolCall, type ProviderToolSchema, type ProviderUsage, type ReconnectPolicy, type SendTextOptions, type SendToolResultOptions, SessionContext, type SessionEventMap, type SessionOptions, type SessionStore, type Tool, type ToolCallInfo, type ToolContext, type ToolDecoration, type ToolDefinition, type ToolMiddleware, type ToolRunInfo, type ToolStrategy, type TranscriptEntry, type TransferCallToolOptions, TwilioRealtimeBridge, type TwilioRestOptions, type TwilioStartEvent, type UsageInfo, type VadConfig, type WebSocketLike, type ZodSchemaLike, captureGreetingAudio, collectAgentGraph, composeExecution, connectStreamTwiml, consoleLogger, createFinishCallTool, createHandoffTool, createTransferCallTool, decorateTool, emptyUsage, handoffToolName, isHandoffDirective, noopLogger, resolveSessionOptions, tool, zodToJsonSchema };
+export { Agent, type AgentDefinition, type ApprovalRequestInfo, type BackgroundAudioOptions, type BackgroundAudioPreset, type BackgroundAudioSpec, BaseRealtimeProvider, type BridgeConfig, type BridgeEventMap, type BuiltinToolsConfig, type CallEndReason, CallSession, type CallSessionFacade, type CallSnapshot, type CallStartedInfo, type CallState, type CaptureGreetingOptions, type CapturedGreeting, DEFAULT_RECONNECT_POLICY, DEFAULT_SESSION_OPTIONS, type DeafnessOptions, type FinishCallToolOptions, type GreetingOptions, type HandoffDirective, type HangupOptions, type IdleOptions, InMemorySessionStore, type InferSchema, type InterruptionBlockCause, InterruptionController, type InterruptionDecision, type InterruptionRateLimit, type InterruptionSettings, type Logger, type MarkEchoResult, NoiseAdaptiveVadController, type NoiseAdaptiveVadOptions, PlaybackTracker, type ProviderAudioDelta, type ProviderCapabilities, type ProviderCloseInfo, type ProviderEvents, type ProviderFactory, type ProviderFactoryContext, type ProviderFallbackInfo, type ProviderSessionInit, type ProviderToolCall, type ProviderToolSchema, type ProviderUsage, type ReconnectPolicy, type SendTextOptions, type SendToolResultOptions, SessionContext, type SessionEventMap, type SessionOptions, type SessionStore, type SessionUpdateOptions, type Tool, type ToolCallInfo, type ToolContext, type ToolDecoration, type ToolDefinition, type ToolMiddleware, type ToolRunInfo, type ToolStrategy, type TranscriptEntry, type TransferCallToolOptions, TwilioRealtimeBridge, type TwilioRestOptions, type TwilioStartEvent, type UsageInfo, type VadAdjustment, type VadConfig, type VadNoiseMetrics, type VadSuggestionInfo, type VadTuningProfile, type WebSocketLike, type ZodSchemaLike, captureGreetingAudio, collectAgentGraph, composeExecution, connectStreamTwiml, consoleLogger, createFinishCallTool, createHandoffTool, createTransferCallTool, decorateTool, emptyUsage, handoffToolName, isHandoffDirective, noopLogger, resolveSessionOptions, tool, zodToJsonSchema };