realtime-voice-agents 2.0.1 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +58 -2
- package/dist/{BaseRealtimeProvider-BehPNT1r.d.cts → BaseRealtimeProvider-BL75_HHh.d.cts} +53 -3
- package/dist/{BaseRealtimeProvider-DI4pKtOb.cjs → BaseRealtimeProvider-C2mRn1V_.cjs} +10 -0
- package/dist/{BaseRealtimeProvider-BQigr5mB.mjs → BaseRealtimeProvider-C9C3s8jx.mjs} +10 -0
- package/dist/{BaseRealtimeProvider-ClP8Wx1X.d.mts → BaseRealtimeProvider-CWJ81HIt.d.mts} +53 -3
- package/dist/{GeminiLiveProvider-DvkTgzjG.d.cts → GeminiLiveProvider-Cc1dkxW6.d.cts} +1 -1
- package/dist/{GeminiLiveProvider-x2nyx5aO.d.mts → GeminiLiveProvider-TvY_cQQZ.d.mts} +1 -1
- package/dist/{OpenAICompatibleProvider-Bdtl-UXH.mjs → OpenAICompatibleProvider-D-2OOVBU.mjs} +98 -11
- package/dist/{OpenAICompatibleProvider-NS4cKVQj.cjs → OpenAICompatibleProvider-Mp0Mefbh.cjs} +98 -11
- package/dist/audio.cjs +9 -6
- package/dist/audio.d.cts +51 -1
- package/dist/audio.d.mts +51 -1
- package/dist/audio.mjs +2 -2
- package/dist/gemini.cjs +16 -6
- package/dist/gemini.d.cts +8 -3
- package/dist/gemini.d.mts +8 -3
- package/dist/gemini.mjs +16 -6
- package/dist/index.cjs +356 -16
- package/dist/index.d.cts +211 -2
- package/dist/index.d.mts +211 -2
- package/dist/index.mjs +355 -16
- package/dist/{BackgroundAudioPlayer-iMcivjis.mjs → noise-CJ789zzj.mjs} +94 -1
- package/dist/{BackgroundAudioPlayer-jfRULWKC.cjs → noise-D9jKottW.cjs} +111 -0
- package/dist/openai.cjs +22 -8
- package/dist/openai.d.cts +31 -3
- package/dist/openai.d.mts +31 -3
- package/dist/openai.mjs +22 -8
- package/dist/{session-config-BVLl7-ha.mjs → session-config-CbifLlkV.mjs} +1 -1
- package/dist/{session-config-c8sOw1XL.cjs → session-config-CqJm2Kxz.cjs} +1 -1
- package/dist/testing.cjs +29 -2
- package/dist/testing.d.cts +31 -1
- package/dist/testing.d.mts +31 -1
- package/dist/testing.mjs +29 -2
- package/dist/xai.cjs +23 -9
- package/dist/xai.d.cts +7 -2
- package/dist/xai.d.mts +7 -2
- package/dist/xai.mjs +23 -9
- package/package.json +1 -1
package/dist/index.mjs
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { t as TypedEmitter } from "./events-BylBSBW-.mjs";
|
|
2
|
-
import { a as noopLogger, i as consoleLogger, n as deepMerge, r as childLogger, t as BaseRealtimeProvider } from "./BaseRealtimeProvider-
|
|
2
|
+
import { a as noopLogger, i as consoleLogger, n as deepMerge, r as childLogger, t as BaseRealtimeProvider } from "./BaseRealtimeProvider-C9C3s8jx.mjs";
|
|
3
3
|
import { t as InMemorySessionStore } from "./InMemorySessionStore-DYParOJO.mjs";
|
|
4
4
|
import { a as TwilioMediaTransport, r as connectStreamTwiml, t as TwilioRestClient } from "./rest-BYqiVOhe.mjs";
|
|
5
5
|
import { a as mulawBytesToMs, i as base64ByteLength } from "./mulaw--cwU2c9L.mjs";
|
|
6
|
-
import {
|
|
7
|
-
import { t as buildSessionUpdate } from "./session-config-
|
|
6
|
+
import { i as BackgroundAudioPlayer, n as NoiseFloorEstimator } from "./noise-CJ789zzj.mjs";
|
|
7
|
+
import { t as buildSessionUpdate } from "./session-config-CbifLlkV.mjs";
|
|
8
8
|
import { readFileSync } from "node:fs";
|
|
9
9
|
import * as z from "zod";
|
|
10
10
|
import WebSocket$1 from "ws";
|
|
@@ -495,6 +495,190 @@ var PlaybackTracker = class {
|
|
|
495
495
|
}
|
|
496
496
|
};
|
|
497
497
|
//#endregion
|
|
498
|
+
//#region src/vad/NoiseAdaptiveVadController.ts
|
|
499
|
+
/**
|
|
500
|
+
* Noise-adaptive VAD policy: watches inbound caller audio through a
|
|
501
|
+
* NoiseFloorEstimator and, on sustained pervasive noise, proposes a bounded
|
|
502
|
+
* turn-detection escalation (server VAD threshold up stepwise; semantic VAD
|
|
503
|
+
* and sensitivity-only providers get suggestion events instead).
|
|
504
|
+
*
|
|
505
|
+
* Modeled on InterruptionController: pure decision objects, zero timers —
|
|
506
|
+
* every clock is measured in milliseconds of inbound audio, so behavior is
|
|
507
|
+
* deterministic in tests and matches wall clock on a live call.
|
|
508
|
+
*
|
|
509
|
+
* Lifecycle: onInboundFrame() PROPOSES without mutating ladder state — the
|
|
510
|
+
* identical proposal recurs until the caller settles it:
|
|
511
|
+
* - commitApplied(): the provider ACKNOWLEDGED the change (auto mode). Only
|
|
512
|
+
* this and rebase() move the effective config — an emitted-but-unapplied
|
|
513
|
+
* suggestion is never treated as reality.
|
|
514
|
+
* - markSuggested(): the suggestion was surfaced (suggest mode, or a rung
|
|
515
|
+
* that is never auto-applied). Latched until rebase() so it fires once.
|
|
516
|
+
* - defer(): the apply failed or timed out — retry after cooldownMs.
|
|
517
|
+
* - rebase(): the app changed VAD via session.updateVad() — new effective
|
|
518
|
+
* base, fresh ladder. rebase(null) disables the controller entirely (turn
|
|
519
|
+
* detection off — noise adaptation must never re-enable it on its own).
|
|
520
|
+
*/
|
|
521
|
+
function round2(value) {
|
|
522
|
+
return Math.round(value * 100) / 100;
|
|
523
|
+
}
|
|
524
|
+
var NoiseAdaptiveVadController = class {
|
|
525
|
+
windowMs;
|
|
526
|
+
noiseFloorDb;
|
|
527
|
+
sustainMs;
|
|
528
|
+
cooldownMs;
|
|
529
|
+
maxSteps;
|
|
530
|
+
thresholdStep;
|
|
531
|
+
maxThreshold;
|
|
532
|
+
profile;
|
|
533
|
+
estimator;
|
|
534
|
+
/** Current EFFECTIVE config — moved only by commitApplied()/rebase(). */
|
|
535
|
+
effective;
|
|
536
|
+
disabled;
|
|
537
|
+
stepsTaken = 0;
|
|
538
|
+
suggestionLatched = false;
|
|
539
|
+
sustainedMs = 0;
|
|
540
|
+
/** Infinity = no step yet, so the first proposal is never cooldown-blocked. */
|
|
541
|
+
msSinceLastStep = Number.POSITIVE_INFINITY;
|
|
542
|
+
elapsedMs = 0;
|
|
543
|
+
analyzedMs = 0;
|
|
544
|
+
/** Memoized next rung; `undefined` = dirty, `null` = ladder exhausted. */
|
|
545
|
+
cachedNext = void 0;
|
|
546
|
+
constructor(options, effectiveVad, profile) {
|
|
547
|
+
this.windowMs = options.windowMs ?? 5e3;
|
|
548
|
+
this.noiseFloorDb = options.noiseFloorDb ?? -45;
|
|
549
|
+
this.sustainMs = options.sustainMs ?? 3e3;
|
|
550
|
+
this.cooldownMs = options.cooldownMs ?? 15e3;
|
|
551
|
+
this.maxSteps = options.maxSteps ?? 1;
|
|
552
|
+
this.thresholdStep = options.thresholdStep ?? .1;
|
|
553
|
+
this.maxThreshold = options.maxThreshold ?? .9;
|
|
554
|
+
this.profile = profile;
|
|
555
|
+
this.estimator = new NoiseFloorEstimator({ windowMs: this.windowMs });
|
|
556
|
+
this.disabled = effectiveVad === null;
|
|
557
|
+
this.effective = effectiveVad ?? { type: "server" };
|
|
558
|
+
}
|
|
559
|
+
/** No escalation room at all (disabled, or the ladder has no next rung). */
|
|
560
|
+
get exhausted() {
|
|
561
|
+
return this.disabled || this.nextRung() === null;
|
|
562
|
+
}
|
|
563
|
+
/**
|
|
564
|
+
* Feed one inbound Twilio frame (base64 μ-law). `excluded` frames (user
|
|
565
|
+
* speech, agent playback, pregreeting) advance the cooldown clock but are
|
|
566
|
+
* kept out of the floor estimate — they freeze, not reset, the sustain
|
|
567
|
+
* counter. Pure w.r.t. ladder state: the same proposal recurs until
|
|
568
|
+
* commitApplied/markSuggested/defer/rebase.
|
|
569
|
+
*/
|
|
570
|
+
onInboundFrame(base64Mulaw, excluded) {
|
|
571
|
+
if (this.disabled || this.suggestionLatched) return null;
|
|
572
|
+
const next = this.nextRung();
|
|
573
|
+
if (!next) return null;
|
|
574
|
+
const bytes = Buffer.from(base64Mulaw, "base64");
|
|
575
|
+
if (bytes.length === 0) return null;
|
|
576
|
+
const frameMs = bytes.length / 8;
|
|
577
|
+
this.elapsedMs += frameMs;
|
|
578
|
+
this.msSinceLastStep += frameMs;
|
|
579
|
+
if (excluded) return null;
|
|
580
|
+
this.estimator.addAudio(bytes);
|
|
581
|
+
this.analyzedMs += frameMs;
|
|
582
|
+
if (!this.estimator.isWarm) return null;
|
|
583
|
+
const floor = this.estimator.floorDb();
|
|
584
|
+
if (floor >= this.noiseFloorDb) this.sustainedMs += frameMs;
|
|
585
|
+
else this.sustainedMs = 0;
|
|
586
|
+
if (this.sustainedMs < this.sustainMs) return null;
|
|
587
|
+
if (this.msSinceLastStep < this.cooldownMs) return null;
|
|
588
|
+
return {
|
|
589
|
+
previous: { ...this.effective },
|
|
590
|
+
suggested: next.suggested,
|
|
591
|
+
step: this.stepsTaken + 1,
|
|
592
|
+
maxSteps: this.maxSteps,
|
|
593
|
+
autoApplicable: next.autoApplicable,
|
|
594
|
+
metrics: {
|
|
595
|
+
noiseFloorDb: floor,
|
|
596
|
+
windowMs: this.windowMs,
|
|
597
|
+
sustainedMs: this.sustainedMs,
|
|
598
|
+
analyzedMs: this.analyzedMs,
|
|
599
|
+
elapsedMs: this.elapsedMs
|
|
600
|
+
}
|
|
601
|
+
};
|
|
602
|
+
}
|
|
603
|
+
/** The proposal was applied AND acknowledged: it is the new effective truth. */
|
|
604
|
+
commitApplied(adjustment) {
|
|
605
|
+
this.effective = { ...adjustment.suggested };
|
|
606
|
+
this.stepsTaken++;
|
|
607
|
+
this.sustainedMs = 0;
|
|
608
|
+
this.msSinceLastStep = 0;
|
|
609
|
+
this.cachedNext = void 0;
|
|
610
|
+
}
|
|
611
|
+
/**
|
|
612
|
+
* The suggestion was emitted without being applied: latch so it fires once
|
|
613
|
+
* per effective state. The effective config is deliberately NOT moved — an
|
|
614
|
+
* unapplied suggestion is not reality.
|
|
615
|
+
*/
|
|
616
|
+
markSuggested(_adjustment) {
|
|
617
|
+
this.suggestionLatched = true;
|
|
618
|
+
}
|
|
619
|
+
/** The apply failed/timed out/was superseded: retry after cooldownMs. */
|
|
620
|
+
defer() {
|
|
621
|
+
this.msSinceLastStep = 0;
|
|
622
|
+
}
|
|
623
|
+
/**
|
|
624
|
+
* App-driven updateVad(): the declared config is the new effective base —
|
|
625
|
+
* a fresh ladder starts from it (never below it). `null` disables the
|
|
626
|
+
* controller; a later non-null rebase re-enables it. A full cooldown
|
|
627
|
+
* applies before the next adaptive step.
|
|
628
|
+
*/
|
|
629
|
+
rebase(vad) {
|
|
630
|
+
this.disabled = vad === null;
|
|
631
|
+
this.effective = vad ? { ...vad } : { type: "server" };
|
|
632
|
+
this.stepsTaken = 0;
|
|
633
|
+
this.suggestionLatched = false;
|
|
634
|
+
this.sustainedMs = 0;
|
|
635
|
+
this.msSinceLastStep = 0;
|
|
636
|
+
this.cachedNext = void 0;
|
|
637
|
+
}
|
|
638
|
+
nextRung() {
|
|
639
|
+
if (this.cachedNext === void 0) this.cachedNext = this.computeNextRung();
|
|
640
|
+
return this.cachedNext;
|
|
641
|
+
}
|
|
642
|
+
computeNextRung() {
|
|
643
|
+
const effective = this.effective;
|
|
644
|
+
if (effective.type === "semantic") {
|
|
645
|
+
if (effective.eagerness === "low") return null;
|
|
646
|
+
return {
|
|
647
|
+
suggested: {
|
|
648
|
+
...effective,
|
|
649
|
+
eagerness: "low"
|
|
650
|
+
},
|
|
651
|
+
autoApplicable: false
|
|
652
|
+
};
|
|
653
|
+
}
|
|
654
|
+
if (this.stepsTaken < this.maxSteps) {
|
|
655
|
+
const baseline = effective.threshold ?? this.profile?.defaultServerThreshold;
|
|
656
|
+
if (baseline !== void 0) {
|
|
657
|
+
const ceiling = Math.min(this.maxThreshold, this.profile?.maxServerThreshold ?? 1);
|
|
658
|
+
const bottom = this.profile?.minServerThreshold ?? 0;
|
|
659
|
+
const target = round2(Math.min(Math.max(baseline + this.thresholdStep, bottom), ceiling));
|
|
660
|
+
if (target > baseline + 1e-9) return {
|
|
661
|
+
suggested: {
|
|
662
|
+
...effective,
|
|
663
|
+
type: "server",
|
|
664
|
+
threshold: target,
|
|
665
|
+
startSensitivity: "low"
|
|
666
|
+
},
|
|
667
|
+
autoApplicable: true
|
|
668
|
+
};
|
|
669
|
+
}
|
|
670
|
+
}
|
|
671
|
+
if (effective.startSensitivity !== "low") return {
|
|
672
|
+
suggested: {
|
|
673
|
+
...effective,
|
|
674
|
+
startSensitivity: "low"
|
|
675
|
+
},
|
|
676
|
+
autoApplicable: false
|
|
677
|
+
};
|
|
678
|
+
return null;
|
|
679
|
+
}
|
|
680
|
+
};
|
|
681
|
+
//#endregion
|
|
498
682
|
//#region src/providers/base/reconnect.ts
|
|
499
683
|
const DEFAULT_RECONNECT_POLICY = {
|
|
500
684
|
maxAttempts: 5,
|
|
@@ -694,6 +878,12 @@ function createTransferCallTool(options = {}) {
|
|
|
694
878
|
* socket is released exactly once.
|
|
695
879
|
*/
|
|
696
880
|
const MAX_BUFFERED_INBOUND_FRAMES = 250;
|
|
881
|
+
/**
|
|
882
|
+
* Cap on the noise meter's caller-speech exclusion: Gemini never emits a
|
|
883
|
+
* speech-stopped event, and phantom speech starts on a noisy line must not
|
|
884
|
+
* latch the meter off forever.
|
|
885
|
+
*/
|
|
886
|
+
const MAX_SPEECH_EXCLUSION_MS = 15e3;
|
|
697
887
|
var CallSession = class extends TypedEmitter {
|
|
698
888
|
callSid;
|
|
699
889
|
streamSid;
|
|
@@ -713,6 +903,8 @@ var CallSession = class extends TypedEmitter {
|
|
|
713
903
|
runningTools = /* @__PURE__ */ new Map();
|
|
714
904
|
timers = /* @__PURE__ */ new Set();
|
|
715
905
|
provider = null;
|
|
906
|
+
/** Primary + fallbacks, in try-order. Only walked while connecting. */
|
|
907
|
+
providerChain;
|
|
716
908
|
activeAgentValue;
|
|
717
909
|
generating = false;
|
|
718
910
|
currentResponseId = null;
|
|
@@ -750,12 +942,28 @@ var CallSession = class extends TypedEmitter {
|
|
|
750
942
|
handoffInProgress = false;
|
|
751
943
|
/** Pre-synthesized greeting playout state. */
|
|
752
944
|
pregreeting = null;
|
|
945
|
+
/** Noise-adaptive VAD (opt-in); built after connect from the provider's ACKed config. */
|
|
946
|
+
noiseVad = null;
|
|
947
|
+
/**
|
|
948
|
+
* DESIRED turn-detection override — drives reconnect/handoff rebuilds via
|
|
949
|
+
* buildProviderInit. Distinct from the provider's ACKed effective state:
|
|
950
|
+
* manual updateVad() promotes it immediately (user intent survives network
|
|
951
|
+
* failures); adaptive applies promote it only after the provider's ack.
|
|
952
|
+
*/
|
|
953
|
+
vadOverride = void 0;
|
|
954
|
+
/** VAD mutations in flight (manual AND adaptive); proposals drop while > 0. */
|
|
955
|
+
vadOpsInFlight = 0;
|
|
956
|
+
/** Bumped by every updateVad(); a stale revision means an adaptive apply was superseded. */
|
|
957
|
+
vadRevision = 0;
|
|
958
|
+
userSpeechActive = false;
|
|
959
|
+
userSpeechStartedAtMs = 0;
|
|
753
960
|
constructor(deps) {
|
|
754
961
|
super();
|
|
755
962
|
this.deps = deps;
|
|
756
963
|
this.callSid = deps.start.start.callSid;
|
|
757
964
|
this.streamSid = deps.start.start.streamSid ?? deps.start.streamSid;
|
|
758
965
|
this.log = childLogger(deps.logger, { callSid: this.callSid });
|
|
966
|
+
this.providerChain = [deps.providerFactory, ...deps.fallbacks ?? []];
|
|
759
967
|
this.activeAgentValue = deps.agent;
|
|
760
968
|
this.context = new SessionContext(deps.options.context);
|
|
761
969
|
this.interruptions = new InterruptionController(deps.options.interruptions);
|
|
@@ -794,17 +1002,7 @@ var CallSession = class extends TypedEmitter {
|
|
|
794
1002
|
/** Connect the provider and activate the call. Called by the bridge. */
|
|
795
1003
|
async begin() {
|
|
796
1004
|
this.playPreGreeting();
|
|
797
|
-
|
|
798
|
-
this.provider = this.deps.providerFactory({
|
|
799
|
-
logger: this.log,
|
|
800
|
-
callSid: this.callSid
|
|
801
|
-
});
|
|
802
|
-
this.wireProvider(this.provider);
|
|
803
|
-
await this.provider.connect(this.buildProviderInit());
|
|
804
|
-
} catch (error) {
|
|
805
|
-
this.fail(error instanceof Error ? error : new Error(String(error)), "provider-failed");
|
|
806
|
-
return;
|
|
807
|
-
}
|
|
1005
|
+
if (!await this.connectInitialProvider()) return;
|
|
808
1006
|
if (this.stateValue !== "connecting") return;
|
|
809
1007
|
this.stateValue = "active";
|
|
810
1008
|
this.emit("provider.connected");
|
|
@@ -816,6 +1014,10 @@ var CallSession = class extends TypedEmitter {
|
|
|
816
1014
|
to: this.callInfo.to,
|
|
817
1015
|
customParameters: this.callInfo.customParameters
|
|
818
1016
|
});
|
|
1017
|
+
if (this.deps.options.noiseAdaptiveVad && this.deps.options.vad !== null) {
|
|
1018
|
+
this.noiseVad = new NoiseAdaptiveVadController(this.deps.options.noiseAdaptiveVad, this.provider.getEffectiveVad(), this.provider.capabilities.vadTuning);
|
|
1019
|
+
if (this.noiseVad.exhausted) this.log.warn("noiseAdaptiveVad enabled but the effective VAD config leaves no escalation room");
|
|
1020
|
+
}
|
|
819
1021
|
if (this.pregreeting) this.provider.sendText(this.pregreeting.text, {
|
|
820
1022
|
role: "assistant",
|
|
821
1023
|
triggerResponse: false
|
|
@@ -826,6 +1028,55 @@ var CallSession = class extends TypedEmitter {
|
|
|
826
1028
|
this.saveSnapshot();
|
|
827
1029
|
this.maybeGreet();
|
|
828
1030
|
}
|
|
1031
|
+
/**
|
|
1032
|
+
* Walk primary + fallbacks until one connects. A provider that fails to
|
|
1033
|
+
* come up is detached BEFORE the chain advances, so a half-open socket's
|
|
1034
|
+
* late events can never reach the session once the next provider owns the
|
|
1035
|
+
* call. Connect-time only by design: once a provider has answered, the
|
|
1036
|
+
* call stays with it. Returns false after failing the call (chain
|
|
1037
|
+
* exhausted) or when the call was torn down while connecting.
|
|
1038
|
+
*/
|
|
1039
|
+
async connectInitialProvider() {
|
|
1040
|
+
let lastFrom = "provider";
|
|
1041
|
+
let lastError = null;
|
|
1042
|
+
for (const factory of this.providerChain) {
|
|
1043
|
+
let provider;
|
|
1044
|
+
try {
|
|
1045
|
+
provider = factory({
|
|
1046
|
+
logger: this.log,
|
|
1047
|
+
callSid: this.callSid
|
|
1048
|
+
});
|
|
1049
|
+
} catch (error) {
|
|
1050
|
+
lastError = toError(error);
|
|
1051
|
+
this.log.warn("provider factory threw — trying the next fallback", { error: String(lastError) });
|
|
1052
|
+
continue;
|
|
1053
|
+
}
|
|
1054
|
+
if (lastError) this.emit("provider.fallback", {
|
|
1055
|
+
from: lastFrom,
|
|
1056
|
+
to: provider.name,
|
|
1057
|
+
error: lastError
|
|
1058
|
+
});
|
|
1059
|
+
this.provider = provider;
|
|
1060
|
+
this.wireProvider(provider);
|
|
1061
|
+
try {
|
|
1062
|
+
await provider.connect(this.buildProviderInit());
|
|
1063
|
+
return true;
|
|
1064
|
+
} catch (error) {
|
|
1065
|
+
provider.removeAllListeners();
|
|
1066
|
+
provider.close().catch(() => {});
|
|
1067
|
+
this.provider = null;
|
|
1068
|
+
if (this.stateValue !== "connecting") return false;
|
|
1069
|
+
lastFrom = provider.name;
|
|
1070
|
+
lastError = toError(error);
|
|
1071
|
+
this.log.warn("provider failed to connect", {
|
|
1072
|
+
provider: provider.name,
|
|
1073
|
+
error: String(lastError)
|
|
1074
|
+
});
|
|
1075
|
+
}
|
|
1076
|
+
}
|
|
1077
|
+
this.fail(lastError ?? /* @__PURE__ */ new Error("provider connect failed"), "provider-failed");
|
|
1078
|
+
return false;
|
|
1079
|
+
}
|
|
829
1080
|
/** The outbound leg was answered (host's Twilio status callback). */
|
|
830
1081
|
notifyAnswered() {
|
|
831
1082
|
if (this.answered) return;
|
|
@@ -838,6 +1089,36 @@ var CallSession = class extends TypedEmitter {
|
|
|
838
1089
|
async updateInstructions(instructions) {
|
|
839
1090
|
await this.provider?.updateSession({ instructions });
|
|
840
1091
|
}
|
|
1092
|
+
/**
|
|
1093
|
+
* Replace the session's turn-detection config mid-call. The value is the
|
|
1094
|
+
* new DESIRED state: it survives reconnects and handoffs, and the
|
|
1095
|
+
* noise-adaptive ladder (when enabled) rebases onto it — future escalation
|
|
1096
|
+
* is relative to this config, never below it. `null` disables turn
|
|
1097
|
+
* detection AND suspends noise adaptation (adaptation never re-enables VAD
|
|
1098
|
+
* on its own); a later non-null call re-enables both.
|
|
1099
|
+
*
|
|
1100
|
+
* Returns true when the live session applied it — provider-acknowledged
|
|
1101
|
+
* when `noiseAdaptiveVad` is configured (serialized updates), best-effort
|
|
1102
|
+
* "sent" otherwise. False when it could not be applied live (no provider,
|
|
1103
|
+
* call ended, no sessionUpdate capability, or the ack timed out — the
|
|
1104
|
+
* resulting reconnect then applies the stored config).
|
|
1105
|
+
*/
|
|
1106
|
+
async updateVad(vad) {
|
|
1107
|
+
this.vadRevision++;
|
|
1108
|
+
this.vadOpsInFlight++;
|
|
1109
|
+
try {
|
|
1110
|
+
this.vadOverride = vad;
|
|
1111
|
+
if (!this.provider || this.stateValue === "ended") return false;
|
|
1112
|
+
if (!this.provider.capabilities.sessionUpdate) {
|
|
1113
|
+
this.log.warn("updateVad: provider has no mid-session update — stored for the next reconnect");
|
|
1114
|
+
return false;
|
|
1115
|
+
}
|
|
1116
|
+
return await this.provider.updateSession({ vad }, { awaitAck: true }) === true;
|
|
1117
|
+
} finally {
|
|
1118
|
+
this.noiseVad?.rebase(vad);
|
|
1119
|
+
this.vadOpsInFlight--;
|
|
1120
|
+
}
|
|
1121
|
+
}
|
|
841
1122
|
/** Manual barge-in: stop the agent mid-sentence. */
|
|
842
1123
|
interrupt() {
|
|
843
1124
|
if (!this.generating && !this.tracker.isPlaybackActive()) return;
|
|
@@ -931,8 +1212,9 @@ var CallSession = class extends TypedEmitter {
|
|
|
931
1212
|
return {
|
|
932
1213
|
instructions: this.activeAgentValue.resolveInstructions(this.context) + (this.pregreeting ? `\n\nYou already opened the call by saying: "${this.pregreeting.text}". Do not greet again — continue the conversation from there.` : ""),
|
|
933
1214
|
voice: this.activeAgentValue.voice,
|
|
934
|
-
vad: this.deps.options.vad,
|
|
1215
|
+
vad: this.vadOverride !== void 0 ? this.vadOverride : this.deps.options.vad,
|
|
935
1216
|
bridgeOwnsInterruptions: true,
|
|
1217
|
+
serializedSessionUpdates: this.deps.options.noiseAdaptiveVad !== void 0,
|
|
936
1218
|
tools: [...this.toolset.values()].map((tool) => {
|
|
937
1219
|
const decorated = decorateTool(tool, this.middlewares);
|
|
938
1220
|
return {
|
|
@@ -1023,12 +1305,15 @@ var CallSession = class extends TypedEmitter {
|
|
|
1023
1305
|
this.emit("transcript.user", entry);
|
|
1024
1306
|
});
|
|
1025
1307
|
provider.on("userSpeechStarted", () => {
|
|
1308
|
+
this.userSpeechActive = true;
|
|
1309
|
+
this.userSpeechStartedAtMs = Date.now();
|
|
1026
1310
|
this.clearIdleTimer();
|
|
1027
1311
|
this.nudgeCount = 0;
|
|
1028
1312
|
this.emit("user.speech.started");
|
|
1029
1313
|
this.handleBargeIn();
|
|
1030
1314
|
});
|
|
1031
1315
|
provider.on("userSpeechStopped", () => {
|
|
1316
|
+
this.userSpeechActive = false;
|
|
1032
1317
|
if (this.blockedUserTurn === "speaking") this.blockedUserTurn = "committed";
|
|
1033
1318
|
this.emit("user.speech.ended");
|
|
1034
1319
|
});
|
|
@@ -1050,6 +1335,11 @@ var CallSession = class extends TypedEmitter {
|
|
|
1050
1335
|
}
|
|
1051
1336
|
handleInboundMedia(payload) {
|
|
1052
1337
|
if (this.stateValue === "ended" || this.stateValue === "ending") return;
|
|
1338
|
+
if (this.noiseVad) {
|
|
1339
|
+
const excluded = this.isUserSpeechExcluded() || this.tracker.isPlaybackActive() || this.pregreeting !== null && !this.pregreeting.played;
|
|
1340
|
+
const proposal = this.noiseVad.onInboundFrame(payload, excluded);
|
|
1341
|
+
if (proposal) this.handleVadProposal(proposal);
|
|
1342
|
+
}
|
|
1053
1343
|
if (this.pregreeting && !this.pregreeting.played) return;
|
|
1054
1344
|
if (this.deps.options.deafness.ignoreUserAudioUntilFirstTurnDone && !this.firstTurnDone) return;
|
|
1055
1345
|
if (this.deps.options.deafness.muteDuringToolExecution && this.runningTools.size > 0) return;
|
|
@@ -1066,6 +1356,51 @@ var CallSession = class extends TypedEmitter {
|
|
|
1066
1356
|
for (const payload of this.inboundBuffer) this.provider.sendAudio(payload);
|
|
1067
1357
|
this.inboundBuffer = [];
|
|
1068
1358
|
}
|
|
1359
|
+
/** Caller speech is excluded from the noise floor, capped so a missing
|
|
1360
|
+
* speech-stopped event (Gemini) can't latch the meter off forever. */
|
|
1361
|
+
isUserSpeechExcluded() {
|
|
1362
|
+
return this.userSpeechActive && Date.now() - this.userSpeechStartedAtMs < MAX_SPEECH_EXCLUSION_MS;
|
|
1363
|
+
}
|
|
1364
|
+
handleVadProposal(proposal) {
|
|
1365
|
+
if (this.vadOpsInFlight > 0) return;
|
|
1366
|
+
const willAutoApply = (this.deps.options.noiseAdaptiveVad?.mode ?? "auto") === "auto" && proposal.autoApplicable && this.provider?.capabilities.sessionUpdate === true;
|
|
1367
|
+
this.emit("vad.suggestion", {
|
|
1368
|
+
...proposal,
|
|
1369
|
+
willAutoApply
|
|
1370
|
+
});
|
|
1371
|
+
if (!willAutoApply) {
|
|
1372
|
+
this.noiseVad?.markSuggested(proposal);
|
|
1373
|
+
return;
|
|
1374
|
+
}
|
|
1375
|
+
this.vadOpsInFlight++;
|
|
1376
|
+
const revisionAtPropose = this.vadRevision;
|
|
1377
|
+
this.applyAdaptiveVadUpdate(proposal, revisionAtPropose).catch((error) => {
|
|
1378
|
+
this.noiseVad?.defer();
|
|
1379
|
+
this.log.warn("noise-adaptive VAD apply failed", { error: String(error) });
|
|
1380
|
+
}).finally(() => {
|
|
1381
|
+
this.vadOpsInFlight--;
|
|
1382
|
+
});
|
|
1383
|
+
}
|
|
1384
|
+
/**
|
|
1385
|
+
* Adaptive applies promote NOTHING until the provider ACKs: a timed-out
|
|
1386
|
+
* update must not leak into vadOverride, or the desync-reconnect would
|
|
1387
|
+
* apply a config the controller never committed (dual truth). Contrast
|
|
1388
|
+
* with updateVad(), where the user's declared intent persists regardless.
|
|
1389
|
+
*/
|
|
1390
|
+
async applyAdaptiveVadUpdate(proposal, revisionAtPropose) {
|
|
1391
|
+
const provider = this.provider;
|
|
1392
|
+
if (!provider || this.stateValue === "ended") {
|
|
1393
|
+
this.noiseVad?.defer();
|
|
1394
|
+
return;
|
|
1395
|
+
}
|
|
1396
|
+
const acked = await provider.updateSession({ vad: proposal.suggested }, { awaitAck: true });
|
|
1397
|
+
if (revisionAtPropose !== this.vadRevision) return;
|
|
1398
|
+
if (acked === true) {
|
|
1399
|
+
this.vadOverride = proposal.suggested;
|
|
1400
|
+
this.noiseVad?.commitApplied(proposal);
|
|
1401
|
+
this.emit("vad.adjusted", proposal);
|
|
1402
|
+
} else this.noiseVad?.defer();
|
|
1403
|
+
}
|
|
1069
1404
|
handleMarkEcho(name) {
|
|
1070
1405
|
if (name === PREGREETING_MARK) {
|
|
1071
1406
|
if (this.pregreeting && !this.pregreeting.played) {
|
|
@@ -1862,6 +2197,9 @@ var CallSession = class extends TypedEmitter {
|
|
|
1862
2197
|
const PREGREETING_MARK = "pre:greeting";
|
|
1863
2198
|
/** 400ms per frame — matches production burst-write implementations. */
|
|
1864
2199
|
const PREGREETING_CHUNK_BYTES = 3200;
|
|
2200
|
+
function toError(value) {
|
|
2201
|
+
return value instanceof Error ? value : new Error(String(value));
|
|
2202
|
+
}
|
|
1865
2203
|
function safeJsonStringify(value) {
|
|
1866
2204
|
try {
|
|
1867
2205
|
return JSON.stringify(value) ?? "null";
|
|
@@ -1996,6 +2334,7 @@ var TwilioRealtimeBridge = class extends TypedEmitter {
|
|
|
1996
2334
|
transport,
|
|
1997
2335
|
start,
|
|
1998
2336
|
providerFactory: this.config.provider,
|
|
2337
|
+
fallbacks: this.config.fallbacks,
|
|
1999
2338
|
agent,
|
|
2000
2339
|
options,
|
|
2001
2340
|
store: this.store,
|
|
@@ -2110,4 +2449,4 @@ async function captureGreetingAudio(options) {
|
|
|
2110
2449
|
});
|
|
2111
2450
|
}
|
|
2112
2451
|
//#endregion
|
|
2113
|
-
export { Agent, BaseRealtimeProvider, CallSession, DEFAULT_RECONNECT_POLICY, DEFAULT_SESSION_OPTIONS, InMemorySessionStore, InterruptionController, PlaybackTracker, SessionContext, TwilioRealtimeBridge, captureGreetingAudio, collectAgentGraph, composeExecution, connectStreamTwiml, consoleLogger, createFinishCallTool, createHandoffTool, createTransferCallTool, decorateTool, emptyUsage, handoffToolName, isHandoffDirective, noopLogger, resolveSessionOptions, tool, zodToJsonSchema };
|
|
2452
|
+
export { Agent, BaseRealtimeProvider, CallSession, DEFAULT_RECONNECT_POLICY, DEFAULT_SESSION_OPTIONS, InMemorySessionStore, InterruptionController, NoiseAdaptiveVadController, PlaybackTracker, SessionContext, TwilioRealtimeBridge, captureGreetingAudio, collectAgentGraph, composeExecution, connectStreamTwiml, consoleLogger, createFinishCallTool, createHandoffTool, createTransferCallTool, decorateTool, emptyUsage, handoffToolName, isHandoffDirective, noopLogger, resolveSessionOptions, tool, zodToJsonSchema };
|
|
@@ -279,4 +279,97 @@ var BackgroundAudioPlayer = class {
|
|
|
279
279
|
}
|
|
280
280
|
};
|
|
281
281
|
//#endregion
|
|
282
|
-
|
|
282
|
+
//#region src/audio/noise.ts
|
|
283
|
+
/**
|
|
284
|
+
* Inbound noise metering: μ-law frames → dBFS levels → a sliding noise-floor
|
|
285
|
+
* estimate. Pure DSP — no timers, no audio-path buffering, no base64. The
|
|
286
|
+
* caller decides which frames to feed (exclusion policy lives upstream), so
|
|
287
|
+
* the estimator stays reusable outside the bridge.
|
|
288
|
+
*/
|
|
289
|
+
/** Clamp for silent/near-silent frames (all-0xFF μ-law decodes to 0). */
|
|
290
|
+
const NOISE_DBFS_FLOOR = -100;
|
|
291
|
+
const SQUARED = /* @__PURE__ */ new Float64Array(256);
|
|
292
|
+
for (let i = 0; i < 256; i++) SQUARED[i] = mulawDecodeSample(i) ** 2;
|
|
293
|
+
/**
|
|
294
|
+
* RMS level of a μ-law frame in dBFS relative to PCM16 full scale (32768).
|
|
295
|
+
* μ-law's max decoded magnitude is 32124, so a full-scale μ-law tone reads
|
|
296
|
+
* ≈ −0.17 dBFS instead of 0 — negligible, and the numbers stay comparable to
|
|
297
|
+
* any standard PCM16 meter.
|
|
298
|
+
*/
|
|
299
|
+
function mulawFrameDbfs(mulaw) {
|
|
300
|
+
if (mulaw.length === 0) return NOISE_DBFS_FLOOR;
|
|
301
|
+
let sum = 0;
|
|
302
|
+
for (let i = 0; i < mulaw.length; i++) sum += SQUARED[mulaw[i]];
|
|
303
|
+
if (sum === 0) return NOISE_DBFS_FLOOR;
|
|
304
|
+
const db = 20 * Math.log10(Math.sqrt(sum / mulaw.length) / 32768);
|
|
305
|
+
return Math.min(0, Math.max(NOISE_DBFS_FLOOR, db));
|
|
306
|
+
}
|
|
307
|
+
const BUCKET_COUNT = 101;
|
|
308
|
+
/**
|
|
309
|
+
* Sliding-window noise-floor estimator over 20 ms μ-law slots.
|
|
310
|
+
*
|
|
311
|
+
* The floor is a low percentile of per-slot RMS: speech bursts sit above it
|
|
312
|
+
* without inflating it, and genuine quiet pauses pull it down. With the
|
|
313
|
+
* default p20, `floorDb() ≥ T` means at least 80% of the window sits at or
|
|
314
|
+
* above T — pervasive line noise, not a loud caller. O(1) per slot: a 1 dB
|
|
315
|
+
* histogram plus a ring buffer of bucket indices.
|
|
316
|
+
*/
|
|
317
|
+
var NoiseFloorEstimator = class {
|
|
318
|
+
windowFrames;
|
|
319
|
+
percentile;
|
|
320
|
+
histogram = new Uint32Array(BUCKET_COUNT);
|
|
321
|
+
ring;
|
|
322
|
+
ringIndex = 0;
|
|
323
|
+
count = 0;
|
|
324
|
+
/** <20 ms tail carried between addAudio calls (real Twilio frames are exactly 160 bytes). */
|
|
325
|
+
residual = null;
|
|
326
|
+
constructor(options = {}) {
|
|
327
|
+
this.windowFrames = Math.max(1, Math.round((options.windowMs ?? 5e3) / 20));
|
|
328
|
+
this.percentile = options.percentile ?? .2;
|
|
329
|
+
this.ring = new Uint8Array(this.windowFrames);
|
|
330
|
+
}
|
|
331
|
+
/** Feed raw μ-law bytes (any length; 20 ms slotting + residual carry inside). */
|
|
332
|
+
addAudio(mulaw) {
|
|
333
|
+
let input = mulaw;
|
|
334
|
+
if (this.residual && this.residual.length > 0) {
|
|
335
|
+
const merged = new Uint8Array(this.residual.length + mulaw.length);
|
|
336
|
+
merged.set(this.residual, 0);
|
|
337
|
+
merged.set(mulaw, this.residual.length);
|
|
338
|
+
input = merged;
|
|
339
|
+
this.residual = null;
|
|
340
|
+
}
|
|
341
|
+
let offset = 0;
|
|
342
|
+
while (input.length - offset >= 160) {
|
|
343
|
+
this.addSlot(input.subarray(offset, offset + 160));
|
|
344
|
+
offset += 160;
|
|
345
|
+
}
|
|
346
|
+
if (offset < input.length) this.residual = input.slice(offset);
|
|
347
|
+
}
|
|
348
|
+
/** True once a full window of analyzed audio has been observed. */
|
|
349
|
+
get isWarm() {
|
|
350
|
+
return this.count >= this.windowFrames;
|
|
351
|
+
}
|
|
352
|
+
/** Noise floor over the window, dBFS in [NOISE_DBFS_FLOOR, 0]. */
|
|
353
|
+
floorDb() {
|
|
354
|
+
if (this.count === 0) return NOISE_DBFS_FLOOR;
|
|
355
|
+
const rank = Math.max(1, Math.ceil(this.percentile * this.count));
|
|
356
|
+
let cumulative = 0;
|
|
357
|
+
for (let bucket = 0; bucket < BUCKET_COUNT; bucket++) {
|
|
358
|
+
cumulative += this.histogram[bucket];
|
|
359
|
+
if (cumulative >= rank) return bucket + NOISE_DBFS_FLOOR;
|
|
360
|
+
}
|
|
361
|
+
return 0;
|
|
362
|
+
}
|
|
363
|
+
addSlot(slot) {
|
|
364
|
+
const bucket = Math.round(mulawFrameDbfs(slot)) - NOISE_DBFS_FLOOR;
|
|
365
|
+
if (this.count >= this.windowFrames) {
|
|
366
|
+
const evicted = this.ring[this.ringIndex];
|
|
367
|
+
this.histogram[evicted] = Math.max(0, this.histogram[evicted] - 1);
|
|
368
|
+
} else this.count++;
|
|
369
|
+
this.ring[this.ringIndex] = bucket;
|
|
370
|
+
this.histogram[bucket] = this.histogram[bucket] + 1;
|
|
371
|
+
this.ringIndex = (this.ringIndex + 1) % this.windowFrames;
|
|
372
|
+
}
|
|
373
|
+
};
|
|
374
|
+
//#endregion
|
|
375
|
+
export { isPresetName as a, scaleMulaw as c, BackgroundAudioPlayer as i, NoiseFloorEstimator as n, loadBackgroundAudio as o, mulawFrameDbfs as r, fadeMulaw as s, NOISE_DBFS_FLOOR as t };
|