realtime-voice-agents 2.0.1 → 2.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. package/README.md +58 -2
  2. package/dist/{BaseRealtimeProvider-BehPNT1r.d.cts → BaseRealtimeProvider-BL75_HHh.d.cts} +53 -3
  3. package/dist/{BaseRealtimeProvider-DI4pKtOb.cjs → BaseRealtimeProvider-C2mRn1V_.cjs} +10 -0
  4. package/dist/{BaseRealtimeProvider-BQigr5mB.mjs → BaseRealtimeProvider-C9C3s8jx.mjs} +10 -0
  5. package/dist/{BaseRealtimeProvider-ClP8Wx1X.d.mts → BaseRealtimeProvider-CWJ81HIt.d.mts} +53 -3
  6. package/dist/{GeminiLiveProvider-DvkTgzjG.d.cts → GeminiLiveProvider-Cc1dkxW6.d.cts} +1 -1
  7. package/dist/{GeminiLiveProvider-x2nyx5aO.d.mts → GeminiLiveProvider-TvY_cQQZ.d.mts} +1 -1
  8. package/dist/{OpenAICompatibleProvider-Bdtl-UXH.mjs → OpenAICompatibleProvider-D-2OOVBU.mjs} +98 -11
  9. package/dist/{OpenAICompatibleProvider-NS4cKVQj.cjs → OpenAICompatibleProvider-Mp0Mefbh.cjs} +98 -11
  10. package/dist/audio.cjs +9 -6
  11. package/dist/audio.d.cts +51 -1
  12. package/dist/audio.d.mts +51 -1
  13. package/dist/audio.mjs +2 -2
  14. package/dist/gemini.cjs +16 -6
  15. package/dist/gemini.d.cts +8 -3
  16. package/dist/gemini.d.mts +8 -3
  17. package/dist/gemini.mjs +16 -6
  18. package/dist/index.cjs +356 -16
  19. package/dist/index.d.cts +211 -2
  20. package/dist/index.d.mts +211 -2
  21. package/dist/index.mjs +355 -16
  22. package/dist/{BackgroundAudioPlayer-iMcivjis.mjs → noise-CJ789zzj.mjs} +94 -1
  23. package/dist/{BackgroundAudioPlayer-jfRULWKC.cjs → noise-D9jKottW.cjs} +111 -0
  24. package/dist/openai.cjs +22 -8
  25. package/dist/openai.d.cts +31 -3
  26. package/dist/openai.d.mts +31 -3
  27. package/dist/openai.mjs +22 -8
  28. package/dist/{session-config-BVLl7-ha.mjs → session-config-CbifLlkV.mjs} +1 -1
  29. package/dist/{session-config-c8sOw1XL.cjs → session-config-CqJm2Kxz.cjs} +1 -1
  30. package/dist/testing.cjs +29 -2
  31. package/dist/testing.d.cts +31 -1
  32. package/dist/testing.d.mts +31 -1
  33. package/dist/testing.mjs +29 -2
  34. package/dist/xai.cjs +23 -9
  35. package/dist/xai.d.cts +7 -2
  36. package/dist/xai.d.mts +7 -2
  37. package/dist/xai.mjs +23 -9
  38. package/package.json +1 -1
package/dist/index.mjs CHANGED
@@ -1,10 +1,10 @@
1
1
  import { t as TypedEmitter } from "./events-BylBSBW-.mjs";
2
- import { a as noopLogger, i as consoleLogger, n as deepMerge, r as childLogger, t as BaseRealtimeProvider } from "./BaseRealtimeProvider-BQigr5mB.mjs";
2
+ import { a as noopLogger, i as consoleLogger, n as deepMerge, r as childLogger, t as BaseRealtimeProvider } from "./BaseRealtimeProvider-C9C3s8jx.mjs";
3
3
  import { t as InMemorySessionStore } from "./InMemorySessionStore-DYParOJO.mjs";
4
4
  import { a as TwilioMediaTransport, r as connectStreamTwiml, t as TwilioRestClient } from "./rest-BYqiVOhe.mjs";
5
5
  import { a as mulawBytesToMs, i as base64ByteLength } from "./mulaw--cwU2c9L.mjs";
6
- import { t as BackgroundAudioPlayer } from "./BackgroundAudioPlayer-iMcivjis.mjs";
7
- import { t as buildSessionUpdate } from "./session-config-BVLl7-ha.mjs";
6
+ import { i as BackgroundAudioPlayer, n as NoiseFloorEstimator } from "./noise-CJ789zzj.mjs";
7
+ import { t as buildSessionUpdate } from "./session-config-CbifLlkV.mjs";
8
8
  import { readFileSync } from "node:fs";
9
9
  import * as z from "zod";
10
10
  import WebSocket$1 from "ws";
@@ -495,6 +495,190 @@ var PlaybackTracker = class {
495
495
  }
496
496
  };
497
497
  //#endregion
498
+ //#region src/vad/NoiseAdaptiveVadController.ts
499
+ /**
500
+ * Noise-adaptive VAD policy: watches inbound caller audio through a
501
+ * NoiseFloorEstimator and, on sustained pervasive noise, proposes a bounded
502
+ * turn-detection escalation (server VAD threshold up stepwise; semantic VAD
503
+ * and sensitivity-only providers get suggestion events instead).
504
+ *
505
+ * Modeled on InterruptionController: pure decision objects, zero timers —
506
+ * every clock is measured in milliseconds of inbound audio, so behavior is
507
+ * deterministic in tests and matches wall clock on a live call.
508
+ *
509
+ * Lifecycle: onInboundFrame() PROPOSES without mutating ladder state — the
510
+ * identical proposal recurs until the caller settles it:
511
+ * - commitApplied(): the provider ACKNOWLEDGED the change (auto mode). Only
512
+ * this and rebase() move the effective config — an emitted-but-unapplied
513
+ * suggestion is never treated as reality.
514
+ * - markSuggested(): the suggestion was surfaced (suggest mode, or a rung
515
+ * that is never auto-applied). Latched until rebase() so it fires once.
516
+ * - defer(): the apply failed or timed out — retry after cooldownMs.
517
+ * - rebase(): the app changed VAD via session.updateVad() — new effective
518
+ * base, fresh ladder. rebase(null) disables the controller entirely (turn
519
+ * detection off — noise adaptation must never re-enable it on its own).
520
+ */
521
+ function round2(value) {
522
+ return Math.round(value * 100) / 100;
523
+ }
524
+ var NoiseAdaptiveVadController = class {
525
+ windowMs;
526
+ noiseFloorDb;
527
+ sustainMs;
528
+ cooldownMs;
529
+ maxSteps;
530
+ thresholdStep;
531
+ maxThreshold;
532
+ profile;
533
+ estimator;
534
+ /** Current EFFECTIVE config — moved only by commitApplied()/rebase(). */
535
+ effective;
536
+ disabled;
537
+ stepsTaken = 0;
538
+ suggestionLatched = false;
539
+ sustainedMs = 0;
540
+ /** Infinity = no step yet, so the first proposal is never cooldown-blocked. */
541
+ msSinceLastStep = Number.POSITIVE_INFINITY;
542
+ elapsedMs = 0;
543
+ analyzedMs = 0;
544
+ /** Memoized next rung; `undefined` = dirty, `null` = ladder exhausted. */
545
+ cachedNext = void 0;
546
+ constructor(options, effectiveVad, profile) {
547
+ this.windowMs = options.windowMs ?? 5e3;
548
+ this.noiseFloorDb = options.noiseFloorDb ?? -45;
549
+ this.sustainMs = options.sustainMs ?? 3e3;
550
+ this.cooldownMs = options.cooldownMs ?? 15e3;
551
+ this.maxSteps = options.maxSteps ?? 1;
552
+ this.thresholdStep = options.thresholdStep ?? .1;
553
+ this.maxThreshold = options.maxThreshold ?? .9;
554
+ this.profile = profile;
555
+ this.estimator = new NoiseFloorEstimator({ windowMs: this.windowMs });
556
+ this.disabled = effectiveVad === null;
557
+ this.effective = effectiveVad ?? { type: "server" };
558
+ }
559
+ /** No escalation room at all (disabled, or the ladder has no next rung). */
560
+ get exhausted() {
561
+ return this.disabled || this.nextRung() === null;
562
+ }
563
+ /**
564
+ * Feed one inbound Twilio frame (base64 μ-law). `excluded` frames (user
565
+ * speech, agent playback, pregreeting) advance the cooldown clock but are
566
+ * kept out of the floor estimate — they freeze, not reset, the sustain
567
+ * counter. Pure w.r.t. ladder state: the same proposal recurs until
568
+ * commitApplied/markSuggested/defer/rebase.
569
+ */
570
+ onInboundFrame(base64Mulaw, excluded) {
571
+ if (this.disabled || this.suggestionLatched) return null;
572
+ const next = this.nextRung();
573
+ if (!next) return null;
574
+ const bytes = Buffer.from(base64Mulaw, "base64");
575
+ if (bytes.length === 0) return null;
576
+ const frameMs = bytes.length / 8;
577
+ this.elapsedMs += frameMs;
578
+ this.msSinceLastStep += frameMs;
579
+ if (excluded) return null;
580
+ this.estimator.addAudio(bytes);
581
+ this.analyzedMs += frameMs;
582
+ if (!this.estimator.isWarm) return null;
583
+ const floor = this.estimator.floorDb();
584
+ if (floor >= this.noiseFloorDb) this.sustainedMs += frameMs;
585
+ else this.sustainedMs = 0;
586
+ if (this.sustainedMs < this.sustainMs) return null;
587
+ if (this.msSinceLastStep < this.cooldownMs) return null;
588
+ return {
589
+ previous: { ...this.effective },
590
+ suggested: next.suggested,
591
+ step: this.stepsTaken + 1,
592
+ maxSteps: this.maxSteps,
593
+ autoApplicable: next.autoApplicable,
594
+ metrics: {
595
+ noiseFloorDb: floor,
596
+ windowMs: this.windowMs,
597
+ sustainedMs: this.sustainedMs,
598
+ analyzedMs: this.analyzedMs,
599
+ elapsedMs: this.elapsedMs
600
+ }
601
+ };
602
+ }
603
+ /** The proposal was applied AND acknowledged: it is the new effective truth. */
604
+ commitApplied(adjustment) {
605
+ this.effective = { ...adjustment.suggested };
606
+ this.stepsTaken++;
607
+ this.sustainedMs = 0;
608
+ this.msSinceLastStep = 0;
609
+ this.cachedNext = void 0;
610
+ }
611
+ /**
612
+ * The suggestion was emitted without being applied: latch so it fires once
613
+ * per effective state. The effective config is deliberately NOT moved — an
614
+ * unapplied suggestion is not reality.
615
+ */
616
+ markSuggested(_adjustment) {
617
+ this.suggestionLatched = true;
618
+ }
619
+ /** The apply failed/timed out/was superseded: retry after cooldownMs. */
620
+ defer() {
621
+ this.msSinceLastStep = 0;
622
+ }
623
+ /**
624
+ * App-driven updateVad(): the declared config is the new effective base —
625
+ * a fresh ladder starts from it (never below it). `null` disables the
626
+ * controller; a later non-null rebase re-enables it. A full cooldown
627
+ * applies before the next adaptive step.
628
+ */
629
+ rebase(vad) {
630
+ this.disabled = vad === null;
631
+ this.effective = vad ? { ...vad } : { type: "server" };
632
+ this.stepsTaken = 0;
633
+ this.suggestionLatched = false;
634
+ this.sustainedMs = 0;
635
+ this.msSinceLastStep = 0;
636
+ this.cachedNext = void 0;
637
+ }
638
+ nextRung() {
639
+ if (this.cachedNext === void 0) this.cachedNext = this.computeNextRung();
640
+ return this.cachedNext;
641
+ }
642
+ computeNextRung() {
643
+ const effective = this.effective;
644
+ if (effective.type === "semantic") {
645
+ if (effective.eagerness === "low") return null;
646
+ return {
647
+ suggested: {
648
+ ...effective,
649
+ eagerness: "low"
650
+ },
651
+ autoApplicable: false
652
+ };
653
+ }
654
+ if (this.stepsTaken < this.maxSteps) {
655
+ const baseline = effective.threshold ?? this.profile?.defaultServerThreshold;
656
+ if (baseline !== void 0) {
657
+ const ceiling = Math.min(this.maxThreshold, this.profile?.maxServerThreshold ?? 1);
658
+ const bottom = this.profile?.minServerThreshold ?? 0;
659
+ const target = round2(Math.min(Math.max(baseline + this.thresholdStep, bottom), ceiling));
660
+ if (target > baseline + 1e-9) return {
661
+ suggested: {
662
+ ...effective,
663
+ type: "server",
664
+ threshold: target,
665
+ startSensitivity: "low"
666
+ },
667
+ autoApplicable: true
668
+ };
669
+ }
670
+ }
671
+ if (effective.startSensitivity !== "low") return {
672
+ suggested: {
673
+ ...effective,
674
+ startSensitivity: "low"
675
+ },
676
+ autoApplicable: false
677
+ };
678
+ return null;
679
+ }
680
+ };
681
+ //#endregion
498
682
  //#region src/providers/base/reconnect.ts
499
683
  const DEFAULT_RECONNECT_POLICY = {
500
684
  maxAttempts: 5,
@@ -694,6 +878,12 @@ function createTransferCallTool(options = {}) {
694
878
  * socket is released exactly once.
695
879
  */
696
880
  const MAX_BUFFERED_INBOUND_FRAMES = 250;
881
+ /**
882
+ * Cap on the noise meter's caller-speech exclusion: Gemini never emits a
883
+ * speech-stopped event, and phantom speech starts on a noisy line must not
884
+ * latch the meter off forever.
885
+ */
886
+ const MAX_SPEECH_EXCLUSION_MS = 15e3;
697
887
  var CallSession = class extends TypedEmitter {
698
888
  callSid;
699
889
  streamSid;
@@ -713,6 +903,8 @@ var CallSession = class extends TypedEmitter {
713
903
  runningTools = /* @__PURE__ */ new Map();
714
904
  timers = /* @__PURE__ */ new Set();
715
905
  provider = null;
906
+ /** Primary + fallbacks, in try-order. Only walked while connecting. */
907
+ providerChain;
716
908
  activeAgentValue;
717
909
  generating = false;
718
910
  currentResponseId = null;
@@ -750,12 +942,28 @@ var CallSession = class extends TypedEmitter {
750
942
  handoffInProgress = false;
751
943
  /** Pre-synthesized greeting playout state. */
752
944
  pregreeting = null;
945
+ /** Noise-adaptive VAD (opt-in); built after connect from the provider's ACKed config. */
946
+ noiseVad = null;
947
+ /**
948
+ * DESIRED turn-detection override — drives reconnect/handoff rebuilds via
949
+ * buildProviderInit. Distinct from the provider's ACKed effective state:
950
+ * manual updateVad() promotes it immediately (user intent survives network
951
+ * failures); adaptive applies promote it only after the provider's ack.
952
+ */
953
+ vadOverride = void 0;
954
+ /** VAD mutations in flight (manual AND adaptive); proposals drop while > 0. */
955
+ vadOpsInFlight = 0;
956
+ /** Bumped by every updateVad(); a stale revision means an adaptive apply was superseded. */
957
+ vadRevision = 0;
958
+ userSpeechActive = false;
959
+ userSpeechStartedAtMs = 0;
753
960
  constructor(deps) {
754
961
  super();
755
962
  this.deps = deps;
756
963
  this.callSid = deps.start.start.callSid;
757
964
  this.streamSid = deps.start.start.streamSid ?? deps.start.streamSid;
758
965
  this.log = childLogger(deps.logger, { callSid: this.callSid });
966
+ this.providerChain = [deps.providerFactory, ...deps.fallbacks ?? []];
759
967
  this.activeAgentValue = deps.agent;
760
968
  this.context = new SessionContext(deps.options.context);
761
969
  this.interruptions = new InterruptionController(deps.options.interruptions);
@@ -794,17 +1002,7 @@ var CallSession = class extends TypedEmitter {
794
1002
  /** Connect the provider and activate the call. Called by the bridge. */
795
1003
  async begin() {
796
1004
  this.playPreGreeting();
797
- try {
798
- this.provider = this.deps.providerFactory({
799
- logger: this.log,
800
- callSid: this.callSid
801
- });
802
- this.wireProvider(this.provider);
803
- await this.provider.connect(this.buildProviderInit());
804
- } catch (error) {
805
- this.fail(error instanceof Error ? error : new Error(String(error)), "provider-failed");
806
- return;
807
- }
1005
+ if (!await this.connectInitialProvider()) return;
808
1006
  if (this.stateValue !== "connecting") return;
809
1007
  this.stateValue = "active";
810
1008
  this.emit("provider.connected");
@@ -816,6 +1014,10 @@ var CallSession = class extends TypedEmitter {
816
1014
  to: this.callInfo.to,
817
1015
  customParameters: this.callInfo.customParameters
818
1016
  });
1017
+ if (this.deps.options.noiseAdaptiveVad && this.deps.options.vad !== null) {
1018
+ this.noiseVad = new NoiseAdaptiveVadController(this.deps.options.noiseAdaptiveVad, this.provider.getEffectiveVad(), this.provider.capabilities.vadTuning);
1019
+ if (this.noiseVad.exhausted) this.log.warn("noiseAdaptiveVad enabled but the effective VAD config leaves no escalation room");
1020
+ }
819
1021
  if (this.pregreeting) this.provider.sendText(this.pregreeting.text, {
820
1022
  role: "assistant",
821
1023
  triggerResponse: false
@@ -826,6 +1028,55 @@ var CallSession = class extends TypedEmitter {
826
1028
  this.saveSnapshot();
827
1029
  this.maybeGreet();
828
1030
  }
1031
+ /**
1032
+ * Walk primary + fallbacks until one connects. A provider that fails to
1033
+ * come up is detached BEFORE the chain advances, so a half-open socket's
1034
+ * late events can never reach the session once the next provider owns the
1035
+ * call. Connect-time only by design: once a provider has answered, the
1036
+ * call stays with it. Returns false after failing the call (chain
1037
+ * exhausted) or when the call was torn down while connecting.
1038
+ */
1039
+ async connectInitialProvider() {
1040
+ let lastFrom = "provider";
1041
+ let lastError = null;
1042
+ for (const factory of this.providerChain) {
1043
+ let provider;
1044
+ try {
1045
+ provider = factory({
1046
+ logger: this.log,
1047
+ callSid: this.callSid
1048
+ });
1049
+ } catch (error) {
1050
+ lastError = toError(error);
1051
+ this.log.warn("provider factory threw — trying the next fallback", { error: String(lastError) });
1052
+ continue;
1053
+ }
1054
+ if (lastError) this.emit("provider.fallback", {
1055
+ from: lastFrom,
1056
+ to: provider.name,
1057
+ error: lastError
1058
+ });
1059
+ this.provider = provider;
1060
+ this.wireProvider(provider);
1061
+ try {
1062
+ await provider.connect(this.buildProviderInit());
1063
+ return true;
1064
+ } catch (error) {
1065
+ provider.removeAllListeners();
1066
+ provider.close().catch(() => {});
1067
+ this.provider = null;
1068
+ if (this.stateValue !== "connecting") return false;
1069
+ lastFrom = provider.name;
1070
+ lastError = toError(error);
1071
+ this.log.warn("provider failed to connect", {
1072
+ provider: provider.name,
1073
+ error: String(lastError)
1074
+ });
1075
+ }
1076
+ }
1077
+ this.fail(lastError ?? /* @__PURE__ */ new Error("provider connect failed"), "provider-failed");
1078
+ return false;
1079
+ }
829
1080
  /** The outbound leg was answered (host's Twilio status callback). */
830
1081
  notifyAnswered() {
831
1082
  if (this.answered) return;
@@ -838,6 +1089,36 @@ var CallSession = class extends TypedEmitter {
838
1089
  async updateInstructions(instructions) {
839
1090
  await this.provider?.updateSession({ instructions });
840
1091
  }
1092
+ /**
1093
+ * Replace the session's turn-detection config mid-call. The value is the
1094
+ * new DESIRED state: it survives reconnects and handoffs, and the
1095
+ * noise-adaptive ladder (when enabled) rebases onto it — future escalation
1096
+ * is relative to this config, never below it. `null` disables turn
1097
+ * detection AND suspends noise adaptation (adaptation never re-enables VAD
1098
+ * on its own); a later non-null call re-enables both.
1099
+ *
1100
+ * Returns true when the live session applied it — provider-acknowledged
1101
+ * when `noiseAdaptiveVad` is configured (serialized updates), best-effort
1102
+ * "sent" otherwise. False when it could not be applied live (no provider,
1103
+ * call ended, no sessionUpdate capability, or the ack timed out — the
1104
+ * resulting reconnect then applies the stored config).
1105
+ */
1106
+ async updateVad(vad) {
1107
+ this.vadRevision++;
1108
+ this.vadOpsInFlight++;
1109
+ try {
1110
+ this.vadOverride = vad;
1111
+ if (!this.provider || this.stateValue === "ended") return false;
1112
+ if (!this.provider.capabilities.sessionUpdate) {
1113
+ this.log.warn("updateVad: provider has no mid-session update — stored for the next reconnect");
1114
+ return false;
1115
+ }
1116
+ return await this.provider.updateSession({ vad }, { awaitAck: true }) === true;
1117
+ } finally {
1118
+ this.noiseVad?.rebase(vad);
1119
+ this.vadOpsInFlight--;
1120
+ }
1121
+ }
841
1122
  /** Manual barge-in: stop the agent mid-sentence. */
842
1123
  interrupt() {
843
1124
  if (!this.generating && !this.tracker.isPlaybackActive()) return;
@@ -931,8 +1212,9 @@ var CallSession = class extends TypedEmitter {
931
1212
  return {
932
1213
  instructions: this.activeAgentValue.resolveInstructions(this.context) + (this.pregreeting ? `\n\nYou already opened the call by saying: "${this.pregreeting.text}". Do not greet again — continue the conversation from there.` : ""),
933
1214
  voice: this.activeAgentValue.voice,
934
- vad: this.deps.options.vad,
1215
+ vad: this.vadOverride !== void 0 ? this.vadOverride : this.deps.options.vad,
935
1216
  bridgeOwnsInterruptions: true,
1217
+ serializedSessionUpdates: this.deps.options.noiseAdaptiveVad !== void 0,
936
1218
  tools: [...this.toolset.values()].map((tool) => {
937
1219
  const decorated = decorateTool(tool, this.middlewares);
938
1220
  return {
@@ -1023,12 +1305,15 @@ var CallSession = class extends TypedEmitter {
1023
1305
  this.emit("transcript.user", entry);
1024
1306
  });
1025
1307
  provider.on("userSpeechStarted", () => {
1308
+ this.userSpeechActive = true;
1309
+ this.userSpeechStartedAtMs = Date.now();
1026
1310
  this.clearIdleTimer();
1027
1311
  this.nudgeCount = 0;
1028
1312
  this.emit("user.speech.started");
1029
1313
  this.handleBargeIn();
1030
1314
  });
1031
1315
  provider.on("userSpeechStopped", () => {
1316
+ this.userSpeechActive = false;
1032
1317
  if (this.blockedUserTurn === "speaking") this.blockedUserTurn = "committed";
1033
1318
  this.emit("user.speech.ended");
1034
1319
  });
@@ -1050,6 +1335,11 @@ var CallSession = class extends TypedEmitter {
1050
1335
  }
1051
1336
  handleInboundMedia(payload) {
1052
1337
  if (this.stateValue === "ended" || this.stateValue === "ending") return;
1338
+ if (this.noiseVad) {
1339
+ const excluded = this.isUserSpeechExcluded() || this.tracker.isPlaybackActive() || this.pregreeting !== null && !this.pregreeting.played;
1340
+ const proposal = this.noiseVad.onInboundFrame(payload, excluded);
1341
+ if (proposal) this.handleVadProposal(proposal);
1342
+ }
1053
1343
  if (this.pregreeting && !this.pregreeting.played) return;
1054
1344
  if (this.deps.options.deafness.ignoreUserAudioUntilFirstTurnDone && !this.firstTurnDone) return;
1055
1345
  if (this.deps.options.deafness.muteDuringToolExecution && this.runningTools.size > 0) return;
@@ -1066,6 +1356,51 @@ var CallSession = class extends TypedEmitter {
1066
1356
  for (const payload of this.inboundBuffer) this.provider.sendAudio(payload);
1067
1357
  this.inboundBuffer = [];
1068
1358
  }
1359
+ /** Caller speech is excluded from the noise floor, capped so a missing
1360
+ * speech-stopped event (Gemini) can't latch the meter off forever. */
1361
+ isUserSpeechExcluded() {
1362
+ return this.userSpeechActive && Date.now() - this.userSpeechStartedAtMs < MAX_SPEECH_EXCLUSION_MS;
1363
+ }
1364
+ handleVadProposal(proposal) {
1365
+ if (this.vadOpsInFlight > 0) return;
1366
+ const willAutoApply = (this.deps.options.noiseAdaptiveVad?.mode ?? "auto") === "auto" && proposal.autoApplicable && this.provider?.capabilities.sessionUpdate === true;
1367
+ this.emit("vad.suggestion", {
1368
+ ...proposal,
1369
+ willAutoApply
1370
+ });
1371
+ if (!willAutoApply) {
1372
+ this.noiseVad?.markSuggested(proposal);
1373
+ return;
1374
+ }
1375
+ this.vadOpsInFlight++;
1376
+ const revisionAtPropose = this.vadRevision;
1377
+ this.applyAdaptiveVadUpdate(proposal, revisionAtPropose).catch((error) => {
1378
+ this.noiseVad?.defer();
1379
+ this.log.warn("noise-adaptive VAD apply failed", { error: String(error) });
1380
+ }).finally(() => {
1381
+ this.vadOpsInFlight--;
1382
+ });
1383
+ }
1384
+ /**
1385
+ * Adaptive applies promote NOTHING until the provider ACKs: a timed-out
1386
+ * update must not leak into vadOverride, or the desync-reconnect would
1387
+ * apply a config the controller never committed (dual truth). Contrast
1388
+ * with updateVad(), where the user's declared intent persists regardless.
1389
+ */
1390
+ async applyAdaptiveVadUpdate(proposal, revisionAtPropose) {
1391
+ const provider = this.provider;
1392
+ if (!provider || this.stateValue === "ended") {
1393
+ this.noiseVad?.defer();
1394
+ return;
1395
+ }
1396
+ const acked = await provider.updateSession({ vad: proposal.suggested }, { awaitAck: true });
1397
+ if (revisionAtPropose !== this.vadRevision) return;
1398
+ if (acked === true) {
1399
+ this.vadOverride = proposal.suggested;
1400
+ this.noiseVad?.commitApplied(proposal);
1401
+ this.emit("vad.adjusted", proposal);
1402
+ } else this.noiseVad?.defer();
1403
+ }
1069
1404
  handleMarkEcho(name) {
1070
1405
  if (name === PREGREETING_MARK) {
1071
1406
  if (this.pregreeting && !this.pregreeting.played) {
@@ -1862,6 +2197,9 @@ var CallSession = class extends TypedEmitter {
1862
2197
  const PREGREETING_MARK = "pre:greeting";
1863
2198
  /** 400ms per frame — matches production burst-write implementations. */
1864
2199
  const PREGREETING_CHUNK_BYTES = 3200;
2200
+ function toError(value) {
2201
+ return value instanceof Error ? value : new Error(String(value));
2202
+ }
1865
2203
  function safeJsonStringify(value) {
1866
2204
  try {
1867
2205
  return JSON.stringify(value) ?? "null";
@@ -1996,6 +2334,7 @@ var TwilioRealtimeBridge = class extends TypedEmitter {
1996
2334
  transport,
1997
2335
  start,
1998
2336
  providerFactory: this.config.provider,
2337
+ fallbacks: this.config.fallbacks,
1999
2338
  agent,
2000
2339
  options,
2001
2340
  store: this.store,
@@ -2110,4 +2449,4 @@ async function captureGreetingAudio(options) {
2110
2449
  });
2111
2450
  }
2112
2451
  //#endregion
2113
- export { Agent, BaseRealtimeProvider, CallSession, DEFAULT_RECONNECT_POLICY, DEFAULT_SESSION_OPTIONS, InMemorySessionStore, InterruptionController, PlaybackTracker, SessionContext, TwilioRealtimeBridge, captureGreetingAudio, collectAgentGraph, composeExecution, connectStreamTwiml, consoleLogger, createFinishCallTool, createHandoffTool, createTransferCallTool, decorateTool, emptyUsage, handoffToolName, isHandoffDirective, noopLogger, resolveSessionOptions, tool, zodToJsonSchema };
2452
+ export { Agent, BaseRealtimeProvider, CallSession, DEFAULT_RECONNECT_POLICY, DEFAULT_SESSION_OPTIONS, InMemorySessionStore, InterruptionController, NoiseAdaptiveVadController, PlaybackTracker, SessionContext, TwilioRealtimeBridge, captureGreetingAudio, collectAgentGraph, composeExecution, connectStreamTwiml, consoleLogger, createFinishCallTool, createHandoffTool, createTransferCallTool, decorateTool, emptyUsage, handoffToolName, isHandoffDirective, noopLogger, resolveSessionOptions, tool, zodToJsonSchema };
@@ -279,4 +279,97 @@ var BackgroundAudioPlayer = class {
279
279
  }
280
280
  };
281
281
  //#endregion
282
- export { scaleMulaw as a, fadeMulaw as i, isPresetName as n, loadBackgroundAudio as r, BackgroundAudioPlayer as t };
282
+ //#region src/audio/noise.ts
283
+ /**
284
+ * Inbound noise metering: μ-law frames → dBFS levels → a sliding noise-floor
285
+ * estimate. Pure DSP — no timers, no audio-path buffering, no base64. The
286
+ * caller decides which frames to feed (exclusion policy lives upstream), so
287
+ * the estimator stays reusable outside the bridge.
288
+ */
289
+ /** Clamp for silent/near-silent frames (all-0xFF μ-law decodes to 0). */
290
+ const NOISE_DBFS_FLOOR = -100;
291
+ const SQUARED = /* @__PURE__ */ new Float64Array(256);
292
+ for (let i = 0; i < 256; i++) SQUARED[i] = mulawDecodeSample(i) ** 2;
293
+ /**
294
+ * RMS level of a μ-law frame in dBFS relative to PCM16 full scale (32768).
295
+ * μ-law's max decoded magnitude is 32124, so a full-scale μ-law tone reads
296
+ * ≈ −0.17 dBFS instead of 0 — negligible, and the numbers stay comparable to
297
+ * any standard PCM16 meter.
298
+ */
299
+ function mulawFrameDbfs(mulaw) {
300
+ if (mulaw.length === 0) return NOISE_DBFS_FLOOR;
301
+ let sum = 0;
302
+ for (let i = 0; i < mulaw.length; i++) sum += SQUARED[mulaw[i]];
303
+ if (sum === 0) return NOISE_DBFS_FLOOR;
304
+ const db = 20 * Math.log10(Math.sqrt(sum / mulaw.length) / 32768);
305
+ return Math.min(0, Math.max(NOISE_DBFS_FLOOR, db));
306
+ }
307
+ const BUCKET_COUNT = 101;
308
+ /**
309
+ * Sliding-window noise-floor estimator over 20 ms μ-law slots.
310
+ *
311
+ * The floor is a low percentile of per-slot RMS: speech bursts sit above it
312
+ * without inflating it, and genuine quiet pauses pull it down. With the
313
+ * default p20, `floorDb() ≥ T` means at least 80% of the window sits at or
314
+ * above T — pervasive line noise, not a loud caller. O(1) per slot: a 1 dB
315
+ * histogram plus a ring buffer of bucket indices.
316
+ */
317
+ var NoiseFloorEstimator = class {
318
+ windowFrames;
319
+ percentile;
320
+ histogram = new Uint32Array(BUCKET_COUNT);
321
+ ring;
322
+ ringIndex = 0;
323
+ count = 0;
324
+ /** <20 ms tail carried between addAudio calls (real Twilio frames are exactly 160 bytes). */
325
+ residual = null;
326
+ constructor(options = {}) {
327
+ this.windowFrames = Math.max(1, Math.round((options.windowMs ?? 5e3) / 20));
328
+ this.percentile = options.percentile ?? .2;
329
+ this.ring = new Uint8Array(this.windowFrames);
330
+ }
331
+ /** Feed raw μ-law bytes (any length; 20 ms slotting + residual carry inside). */
332
+ addAudio(mulaw) {
333
+ let input = mulaw;
334
+ if (this.residual && this.residual.length > 0) {
335
+ const merged = new Uint8Array(this.residual.length + mulaw.length);
336
+ merged.set(this.residual, 0);
337
+ merged.set(mulaw, this.residual.length);
338
+ input = merged;
339
+ this.residual = null;
340
+ }
341
+ let offset = 0;
342
+ while (input.length - offset >= 160) {
343
+ this.addSlot(input.subarray(offset, offset + 160));
344
+ offset += 160;
345
+ }
346
+ if (offset < input.length) this.residual = input.slice(offset);
347
+ }
348
+ /** True once a full window of analyzed audio has been observed. */
349
+ get isWarm() {
350
+ return this.count >= this.windowFrames;
351
+ }
352
+ /** Noise floor over the window, dBFS in [NOISE_DBFS_FLOOR, 0]. */
353
+ floorDb() {
354
+ if (this.count === 0) return NOISE_DBFS_FLOOR;
355
+ const rank = Math.max(1, Math.ceil(this.percentile * this.count));
356
+ let cumulative = 0;
357
+ for (let bucket = 0; bucket < BUCKET_COUNT; bucket++) {
358
+ cumulative += this.histogram[bucket];
359
+ if (cumulative >= rank) return bucket + NOISE_DBFS_FLOOR;
360
+ }
361
+ return 0;
362
+ }
363
+ addSlot(slot) {
364
+ const bucket = Math.round(mulawFrameDbfs(slot)) - NOISE_DBFS_FLOOR;
365
+ if (this.count >= this.windowFrames) {
366
+ const evicted = this.ring[this.ringIndex];
367
+ this.histogram[evicted] = Math.max(0, this.histogram[evicted] - 1);
368
+ } else this.count++;
369
+ this.ring[this.ringIndex] = bucket;
370
+ this.histogram[bucket] = this.histogram[bucket] + 1;
371
+ this.ringIndex = (this.ringIndex + 1) % this.windowFrames;
372
+ }
373
+ };
374
+ //#endregion
375
+ export { isPresetName as a, scaleMulaw as c, BackgroundAudioPlayer as i, NoiseFloorEstimator as n, loadBackgroundAudio as o, mulawFrameDbfs as r, fadeMulaw as s, NOISE_DBFS_FLOOR as t };