realtime-voice-agents 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +298 -0
- package/assets/elevator-jazz.ulaw +1 -0
- package/assets/keyboard-typing.ulaw +1 -0
- package/assets/lofi.ulaw +1 -0
- package/assets/ringing.ulaw +1 -0
- package/assets/thinking-hum.ulaw +1 -0
- package/dist/BackgroundAudioPlayer-iMcivjis.mjs +282 -0
- package/dist/BackgroundAudioPlayer-jfRULWKC.cjs +313 -0
- package/dist/BaseRealtimeProvider-BQigr5mB.mjs +62 -0
- package/dist/BaseRealtimeProvider-BehPNT1r.d.cts +239 -0
- package/dist/BaseRealtimeProvider-ClP8Wx1X.d.mts +239 -0
- package/dist/BaseRealtimeProvider-DI4pKtOb.cjs +91 -0
- package/dist/GeminiLiveProvider-DvkTgzjG.d.cts +101 -0
- package/dist/GeminiLiveProvider-x2nyx5aO.d.mts +101 -0
- package/dist/InMemorySessionStore-B0_i-DOU.cjs +34 -0
- package/dist/InMemorySessionStore-B5_rq61L.d.cts +83 -0
- package/dist/InMemorySessionStore-B5_rq61L.d.mts +83 -0
- package/dist/InMemorySessionStore-DYParOJO.mjs +29 -0
- package/dist/OpenAICompatibleProvider-Bdtl-UXH.mjs +402 -0
- package/dist/OpenAICompatibleProvider-NS4cKVQj.cjs +409 -0
- package/dist/audio.cjs +21 -0
- package/dist/audio.d.cts +165 -0
- package/dist/audio.d.mts +165 -0
- package/dist/audio.mjs +4 -0
- package/dist/env-DSnGaERV.cjs +19 -0
- package/dist/env-DUwUWTsg.mjs +14 -0
- package/dist/events-BUMYdETO.d.cts +24 -0
- package/dist/events-BUMYdETO.d.mts +24 -0
- package/dist/events-BxDTIKKq.cjs +48 -0
- package/dist/events-BylBSBW-.mjs +43 -0
- package/dist/gemini.cjs +401 -0
- package/dist/gemini.d.cts +41 -0
- package/dist/gemini.d.mts +41 -0
- package/dist/gemini.mjs +395 -0
- package/dist/index.cjs +2142 -0
- package/dist/index.d.cts +920 -0
- package/dist/index.d.mts +920 -0
- package/dist/index.mjs +2113 -0
- package/dist/mulaw--cwU2c9L.mjs +64 -0
- package/dist/mulaw-DLUObjdP.cjs +117 -0
- package/dist/openai.cjs +61 -0
- package/dist/openai.d.cts +121 -0
- package/dist/openai.d.mts +121 -0
- package/dist/openai.mjs +53 -0
- package/dist/presets-Bf75YXs5.d.cts +24 -0
- package/dist/presets-Bf75YXs5.d.mts +24 -0
- package/dist/rest-BYqiVOhe.mjs +265 -0
- package/dist/rest-BvUKut_k.cjs +300 -0
- package/dist/rolldown-runtime-VH7oDXx4.cjs +28 -0
- package/dist/session-config-BVLl7-ha.mjs +74 -0
- package/dist/session-config-c8sOw1XL.cjs +85 -0
- package/dist/store.cjs +3 -0
- package/dist/store.d.cts +2 -0
- package/dist/store.d.mts +2 -0
- package/dist/store.mjs +2 -0
- package/dist/testing.cjs +544 -0
- package/dist/testing.d.cts +191 -0
- package/dist/testing.d.mts +191 -0
- package/dist/testing.mjs +538 -0
- package/dist/transcode-C9aJG7_W.cjs +215 -0
- package/dist/transcode-CsAp97G9.mjs +198 -0
- package/dist/transport-B_PJFIVd.d.cts +155 -0
- package/dist/transport-CEaLFV4E.d.mts +155 -0
- package/dist/twilio.cjs +8 -0
- package/dist/twilio.d.cts +3 -0
- package/dist/twilio.d.mts +3 -0
- package/dist/twilio.mjs +2 -0
- package/dist/twiml-z9LjoF4_.d.cts +67 -0
- package/dist/twiml-z9LjoF4_.d.mts +67 -0
- package/dist/xai.cjs +92 -0
- package/dist/xai.d.cts +35 -0
- package/dist/xai.d.mts +35 -0
- package/dist/xai.mjs +86 -0
- package/package.json +183 -0
package/dist/index.mjs
ADDED
|
@@ -0,0 +1,2113 @@
|
|
|
1
|
+
import { t as TypedEmitter } from "./events-BylBSBW-.mjs";
|
|
2
|
+
import { a as noopLogger, i as consoleLogger, n as deepMerge, r as childLogger, t as BaseRealtimeProvider } from "./BaseRealtimeProvider-BQigr5mB.mjs";
|
|
3
|
+
import { t as InMemorySessionStore } from "./InMemorySessionStore-DYParOJO.mjs";
|
|
4
|
+
import { a as TwilioMediaTransport, r as connectStreamTwiml, t as TwilioRestClient } from "./rest-BYqiVOhe.mjs";
|
|
5
|
+
import { a as mulawBytesToMs, i as base64ByteLength } from "./mulaw--cwU2c9L.mjs";
|
|
6
|
+
import { t as BackgroundAudioPlayer } from "./BackgroundAudioPlayer-iMcivjis.mjs";
|
|
7
|
+
import { t as buildSessionUpdate } from "./session-config-BVLl7-ha.mjs";
|
|
8
|
+
import { readFileSync } from "node:fs";
|
|
9
|
+
import * as z from "zod";
|
|
10
|
+
import WebSocket$1 from "ws";
|
|
11
|
+
//#region src/agents/Agent.ts
|
|
12
|
+
function slugify(name) {
|
|
13
|
+
const slug = name.toLowerCase().replace(/[^a-z0-9]+/g, "_").replace(/^_+|_+$/g, "");
|
|
14
|
+
if (!slug) throw new Error(`cannot derive an agent id from name "${name}"`);
|
|
15
|
+
return slug;
|
|
16
|
+
}
|
|
17
|
+
var Agent = class {
|
|
18
|
+
id;
|
|
19
|
+
name;
|
|
20
|
+
voice;
|
|
21
|
+
model;
|
|
22
|
+
tools;
|
|
23
|
+
handoffs;
|
|
24
|
+
handoffDescription;
|
|
25
|
+
providerOptions;
|
|
26
|
+
instructionsSource;
|
|
27
|
+
constructor(definition) {
|
|
28
|
+
this.id = definition.id ?? slugify(definition.name);
|
|
29
|
+
this.name = definition.name;
|
|
30
|
+
this.voice = definition.voice;
|
|
31
|
+
this.model = definition.model;
|
|
32
|
+
this.tools = definition.tools ?? [];
|
|
33
|
+
this.handoffs = definition.handoffs ?? [];
|
|
34
|
+
this.handoffDescription = definition.handoffDescription;
|
|
35
|
+
this.providerOptions = definition.providerOptions;
|
|
36
|
+
this.instructionsSource = definition.instructions;
|
|
37
|
+
}
|
|
38
|
+
resolveInstructions(context) {
|
|
39
|
+
return typeof this.instructionsSource === "function" ? this.instructionsSource(context) : this.instructionsSource;
|
|
40
|
+
}
|
|
41
|
+
};
|
|
42
|
+
/**
|
|
43
|
+
* Walk the handoff graph (BFS) from a root agent. Cycles are fine — swarms
|
|
44
|
+
* commonly hand back and forth. Two different Agent instances claiming the
|
|
45
|
+
* same id is a configuration error caught here, at session build.
|
|
46
|
+
*/
|
|
47
|
+
function collectAgentGraph(root) {
|
|
48
|
+
const agents = /* @__PURE__ */ new Map();
|
|
49
|
+
const queue = [root];
|
|
50
|
+
while (queue.length > 0) {
|
|
51
|
+
const agent = queue.shift();
|
|
52
|
+
const existing = agents.get(agent.id);
|
|
53
|
+
if (existing === agent) continue;
|
|
54
|
+
if (existing) throw new Error(`duplicate agent id "${agent.id}" (two different Agent instances)`);
|
|
55
|
+
agents.set(agent.id, agent);
|
|
56
|
+
queue.push(...agent.handoffs);
|
|
57
|
+
}
|
|
58
|
+
return agents;
|
|
59
|
+
}
|
|
60
|
+
//#endregion
|
|
61
|
+
//#region src/tools/json-schema.ts
|
|
62
|
+
/**
|
|
63
|
+
* Zod → JSON Schema for provider tool declarations.
|
|
64
|
+
*
|
|
65
|
+
* Zod 4 ships `z.toJSONSchema()` — used when available (zod is a required
|
|
66
|
+
* peer). For Zod 3 (≥3.25) a minimal structural converter covers the subset
|
|
67
|
+
* that makes sense for voice-agent tool parameters: objects, strings,
|
|
68
|
+
* numbers, booleans, enums, literals, arrays, records, unions,
|
|
69
|
+
* optional/nullable/default, descriptions.
|
|
70
|
+
*/
|
|
71
|
+
function zodToJsonSchema(schema) {
|
|
72
|
+
let result;
|
|
73
|
+
const native = z.toJSONSchema;
|
|
74
|
+
if (typeof native === "function" && isZod4Schema(schema)) try {
|
|
75
|
+
result = { ...native(schema, {
|
|
76
|
+
target: "draft-7",
|
|
77
|
+
io: "input"
|
|
78
|
+
}) };
|
|
79
|
+
} catch {
|
|
80
|
+
result = { type: "object" };
|
|
81
|
+
}
|
|
82
|
+
else result = convertZod3(schema);
|
|
83
|
+
delete result.$schema;
|
|
84
|
+
if (result.type === void 0 && !result.anyOf && !result.oneOf && !result.enum) result.type = "object";
|
|
85
|
+
return result;
|
|
86
|
+
}
|
|
87
|
+
/** Zod 4 schemas carry an internal `_zod` bag; Zod 3 uses `_def.typeName`. */
|
|
88
|
+
function isZod4Schema(schema) {
|
|
89
|
+
return schema != null && typeof schema === "object" && "_zod" in schema;
|
|
90
|
+
}
|
|
91
|
+
function convertZod3(schema) {
|
|
92
|
+
const def = schema?._def;
|
|
93
|
+
if (!def) return { type: "object" };
|
|
94
|
+
const typeName = def.typeName ?? "";
|
|
95
|
+
const description = def.description;
|
|
96
|
+
const withDescription = (node) => description ? {
|
|
97
|
+
...node,
|
|
98
|
+
description
|
|
99
|
+
} : node;
|
|
100
|
+
switch (typeName) {
|
|
101
|
+
case "ZodObject": {
|
|
102
|
+
const shape = typeof def.shape === "function" ? def.shape() : def.shape;
|
|
103
|
+
const properties = {};
|
|
104
|
+
const required = [];
|
|
105
|
+
for (const [key, value] of Object.entries(shape ?? {})) {
|
|
106
|
+
const child = value;
|
|
107
|
+
properties[key] = convertZod3(child);
|
|
108
|
+
if (!isOptionalZod3(child)) required.push(key);
|
|
109
|
+
}
|
|
110
|
+
return withDescription({
|
|
111
|
+
type: "object",
|
|
112
|
+
properties,
|
|
113
|
+
...required.length ? { required } : {},
|
|
114
|
+
additionalProperties: false
|
|
115
|
+
});
|
|
116
|
+
}
|
|
117
|
+
case "ZodString": return withDescription({ type: "string" });
|
|
118
|
+
case "ZodNumber": return withDescription({ type: "number" });
|
|
119
|
+
case "ZodBoolean": return withDescription({ type: "boolean" });
|
|
120
|
+
case "ZodEnum": return withDescription({
|
|
121
|
+
type: "string",
|
|
122
|
+
enum: def.values
|
|
123
|
+
});
|
|
124
|
+
case "ZodNativeEnum": return withDescription({ enum: Object.values(def.values ?? {}) });
|
|
125
|
+
case "ZodLiteral": return withDescription({ const: def.value });
|
|
126
|
+
case "ZodArray": return withDescription({
|
|
127
|
+
type: "array",
|
|
128
|
+
items: convertZod3(def.type)
|
|
129
|
+
});
|
|
130
|
+
case "ZodRecord": return withDescription({
|
|
131
|
+
type: "object",
|
|
132
|
+
additionalProperties: def.valueType ? convertZod3(def.valueType) : true
|
|
133
|
+
});
|
|
134
|
+
case "ZodUnion": return withDescription({ anyOf: (def.options ?? []).map(convertZod3) });
|
|
135
|
+
case "ZodOptional": return withDescription(convertZod3(def.innerType));
|
|
136
|
+
case "ZodNullable": return withDescription({ anyOf: [convertZod3(def.innerType), { type: "null" }] });
|
|
137
|
+
case "ZodDefault": return withDescription({
|
|
138
|
+
...convertZod3(def.innerType),
|
|
139
|
+
default: def.defaultValue?.()
|
|
140
|
+
});
|
|
141
|
+
case "ZodEffects": return withDescription(convertZod3(def.schema));
|
|
142
|
+
default: return withDescription({});
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
function isOptionalZod3(schema) {
|
|
146
|
+
const typeName = schema?._def?.typeName;
|
|
147
|
+
if (typeName === "ZodOptional" || typeName === "ZodDefault") return true;
|
|
148
|
+
if (typeName === "ZodEffects") return isOptionalZod3(schema._def.schema);
|
|
149
|
+
return false;
|
|
150
|
+
}
|
|
151
|
+
//#endregion
|
|
152
|
+
//#region src/tools/tool.ts
|
|
153
|
+
const NAME_PATTERN = /^[a-zA-Z0-9_-]{1,64}$/;
|
|
154
|
+
/** Define a provider-agnostic tool with a Zod parameters schema. */
|
|
155
|
+
function tool(definition) {
|
|
156
|
+
if (!NAME_PATTERN.test(definition.name)) throw new Error(`tool name "${definition.name}" is invalid (allowed: letters, digits, _ and -, max 64 chars)`);
|
|
157
|
+
return {
|
|
158
|
+
...definition,
|
|
159
|
+
strategy: definition.strategy ?? "sync",
|
|
160
|
+
parametersJsonSchema: zodToJsonSchema(definition.parameters)
|
|
161
|
+
};
|
|
162
|
+
}
|
|
163
|
+
//#endregion
|
|
164
|
+
//#region src/agents/handoff.ts
|
|
165
|
+
/**
|
|
166
|
+
* Swarm-style handoffs (LangGraph-inspired): every agent listed in
|
|
167
|
+
* `handoffs` gets an auto-generated `transfer_to_<id>` tool. The tool's
|
|
168
|
+
* execute returns a Command-like directive; the session intercepts it,
|
|
169
|
+
* delivers the tool result, and performs the swap — so tool-initiated and
|
|
170
|
+
* programmatic (`session.handoffTo`) handoffs share one code path.
|
|
171
|
+
*/
|
|
172
|
+
const HANDOFF_MARKER = "__twilioRealtimeAgentsHandoff";
|
|
173
|
+
function isHandoffDirective(value) {
|
|
174
|
+
return typeof value === "object" && value !== null && value[HANDOFF_MARKER] === true;
|
|
175
|
+
}
|
|
176
|
+
function handoffToolName(agentId) {
|
|
177
|
+
return `transfer_to_${agentId}`;
|
|
178
|
+
}
|
|
179
|
+
function createHandoffTool(target) {
|
|
180
|
+
return tool({
|
|
181
|
+
name: handoffToolName(target.id),
|
|
182
|
+
description: target.handoffDescription ?? `Transfer the conversation to ${target.name}. Use when the caller's request is better handled by ${target.name}.`,
|
|
183
|
+
parameters: z.object({ reason: z.string().optional().describe("Brief context for the receiving agent about why the caller was transferred") }),
|
|
184
|
+
execute: async (input) => ({
|
|
185
|
+
[HANDOFF_MARKER]: true,
|
|
186
|
+
targetAgentId: target.id,
|
|
187
|
+
reason: input.reason
|
|
188
|
+
})
|
|
189
|
+
});
|
|
190
|
+
}
|
|
191
|
+
//#endregion
|
|
192
|
+
//#region src/interruption/InterruptionController.ts
|
|
193
|
+
var InterruptionController = class {
|
|
194
|
+
settings;
|
|
195
|
+
now;
|
|
196
|
+
responseIndex = 0;
|
|
197
|
+
guardStartedAt = null;
|
|
198
|
+
guardResponseId = null;
|
|
199
|
+
/** The guarded response's audio has not finished (or been flushed) yet. */
|
|
200
|
+
guardPlaybackOpen = false;
|
|
201
|
+
/** Responses that began while the guarded one was still playing. */
|
|
202
|
+
deferredStarts = 0;
|
|
203
|
+
suspended = false;
|
|
204
|
+
bargeInTimestamps = [];
|
|
205
|
+
constructor(settings = {}, now = Date.now) {
|
|
206
|
+
this.settings = settings;
|
|
207
|
+
this.now = now;
|
|
208
|
+
}
|
|
209
|
+
get isSuspended() {
|
|
210
|
+
return this.suspended;
|
|
211
|
+
}
|
|
212
|
+
/** A new response started generating: rotate the guard to it. */
|
|
213
|
+
onResponseStarted(responseId) {
|
|
214
|
+
if (this.guardPlaybackOpen && this.guardActive()) {
|
|
215
|
+
this.deferredStarts++;
|
|
216
|
+
return;
|
|
217
|
+
}
|
|
218
|
+
this.responseIndex += 1 + this.deferredStarts;
|
|
219
|
+
this.deferredStarts = 0;
|
|
220
|
+
this.guardResponseId = responseId;
|
|
221
|
+
this.guardStartedAt = null;
|
|
222
|
+
this.guardPlaybackOpen = true;
|
|
223
|
+
}
|
|
224
|
+
/** Playback of a response reached the caller: the guard clock starts now. */
|
|
225
|
+
onPlaybackStarted(responseId) {
|
|
226
|
+
if (responseId === this.guardResponseId) this.guardStartedAt = this.now();
|
|
227
|
+
}
|
|
228
|
+
/** Current playback finished or was cleared: cooldown suspension ends. */
|
|
229
|
+
onPlaybackEnded() {
|
|
230
|
+
this.suspended = false;
|
|
231
|
+
this.guardPlaybackOpen = false;
|
|
232
|
+
if (this.deferredStarts > 0) {
|
|
233
|
+
this.responseIndex += this.deferredStarts;
|
|
234
|
+
this.deferredStarts = 0;
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
/**
|
|
238
|
+
* Evaluate a barge-in attempt (user speech while the agent is audible).
|
|
239
|
+
* Every attempt counts toward the rate limiter, including blocked ones.
|
|
240
|
+
*/
|
|
241
|
+
evaluate(context) {
|
|
242
|
+
if (this.settings.enabled === false) return {
|
|
243
|
+
allow: false,
|
|
244
|
+
cause: "disabled"
|
|
245
|
+
};
|
|
246
|
+
if (this.suspended) return {
|
|
247
|
+
allow: false,
|
|
248
|
+
cause: "suspended"
|
|
249
|
+
};
|
|
250
|
+
if (context.toolRunning) return {
|
|
251
|
+
allow: false,
|
|
252
|
+
cause: "tool_running"
|
|
253
|
+
};
|
|
254
|
+
const rateLimit = this.settings.rateLimit;
|
|
255
|
+
if (rateLimit) {
|
|
256
|
+
const now = this.now();
|
|
257
|
+
this.bargeInTimestamps = this.bargeInTimestamps.filter((ts) => now - ts <= rateLimit.windowMs);
|
|
258
|
+
this.bargeInTimestamps.push(now);
|
|
259
|
+
if (this.bargeInTimestamps.length >= rateLimit.threshold) {
|
|
260
|
+
this.suspended = true;
|
|
261
|
+
this.bargeInTimestamps = [];
|
|
262
|
+
return {
|
|
263
|
+
allow: false,
|
|
264
|
+
cause: "rate_limit",
|
|
265
|
+
instruction: rateLimit.instruction
|
|
266
|
+
};
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
if (this.guardActive()) return {
|
|
270
|
+
allow: false,
|
|
271
|
+
cause: "guard"
|
|
272
|
+
};
|
|
273
|
+
return { allow: true };
|
|
274
|
+
}
|
|
275
|
+
guardActive() {
|
|
276
|
+
const { guardDurationMs, firstResponseOnly } = this.settings;
|
|
277
|
+
if (!guardDurationMs) return false;
|
|
278
|
+
if (firstResponseOnly && this.responseIndex > 1) return false;
|
|
279
|
+
if (this.guardStartedAt === null) return true;
|
|
280
|
+
return this.now() - this.guardStartedAt < guardDurationMs;
|
|
281
|
+
}
|
|
282
|
+
};
|
|
283
|
+
//#endregion
|
|
284
|
+
//#region src/playback/PlaybackTracker.ts
|
|
285
|
+
/**
|
|
286
|
+
* PlaybackTracker — the honest playback clock.
|
|
287
|
+
*
|
|
288
|
+
* Marks are interleaved into the outbound media stream at checkpoints — after
|
|
289
|
+
* the FIRST audio chunk of a response (exact `playback.started`), then after
|
|
290
|
+
* every ~1s of accumulated audio, and after the final chunk once generation
|
|
291
|
+
* completes (exact `playback.finished`). Twilio plays media strictly in order
|
|
292
|
+
* and echoes each mark when playout reaches it, so a mark echo is
|
|
293
|
+
* hardware-level confirmation of what the caller has heard — unlike
|
|
294
|
+
* generation-time events, which run seconds ahead of the phone line.
|
|
295
|
+
*
|
|
296
|
+
* Why checkpoints and not a mark per delta: field testing showed per-delta
|
|
297
|
+
* marks (which double the message count on the Twilio socket) audibly degrade
|
|
298
|
+
* playback smoothness, while sparse marks lose nothing — between echoes,
|
|
299
|
+
* `estimatePlayedMs` interpolates with the wall clock, so truncate accuracy
|
|
300
|
+
* is bounded by echo jitter, not by the checkpoint interval.
|
|
301
|
+
*
|
|
302
|
+
* Clear-epoch: `clear` flushes Twilio's buffer, and Twilio echoes the
|
|
303
|
+
* discarded marks immediately. Bumping the epoch before sending `clear` lets
|
|
304
|
+
* those echoes be classified as flushed instead of played, so `playedMs`
|
|
305
|
+
* stays truthful — this number feeds `conversation.item.truncate`.
|
|
306
|
+
*/
|
|
307
|
+
const TRACKED_MARK_PREFIX = "tra:";
|
|
308
|
+
const DEFAULT_CHECKPOINT_INTERVAL_MS = 1e3;
|
|
309
|
+
var PlaybackTracker = class {
|
|
310
|
+
epoch = 0;
|
|
311
|
+
markSeq = 0;
|
|
312
|
+
marks = /* @__PURE__ */ new Map();
|
|
313
|
+
responses = /* @__PURE__ */ new Map();
|
|
314
|
+
now;
|
|
315
|
+
checkpointIntervalMs;
|
|
316
|
+
constructor(now = Date.now, options = {}) {
|
|
317
|
+
this.now = now;
|
|
318
|
+
this.checkpointIntervalMs = options.checkpointIntervalMs ?? DEFAULT_CHECKPOINT_INTERVAL_MS;
|
|
319
|
+
}
|
|
320
|
+
/** Is `name` one of ours (as opposed to a host-app or background mark)? */
|
|
321
|
+
static isTrackedMark(name) {
|
|
322
|
+
return name.startsWith(TRACKED_MARK_PREFIX);
|
|
323
|
+
}
|
|
324
|
+
/**
|
|
325
|
+
* Record an outgoing audio chunk. Returns a mark name to interleave after
|
|
326
|
+
* it when a checkpoint is due (first chunk of the response, or
|
|
327
|
+
* checkpointIntervalMs of audio accumulated since the last mark) — null
|
|
328
|
+
* otherwise.
|
|
329
|
+
*/
|
|
330
|
+
onAudioSent(responseId, chunkMs, itemId) {
|
|
331
|
+
const track = this.ensureTrack(responseId);
|
|
332
|
+
if (itemId && !track.itemId) track.itemId = itemId;
|
|
333
|
+
track.totalMs += chunkMs;
|
|
334
|
+
const firstChunk = track.markedMs === 0 && track.sentMarks === 0;
|
|
335
|
+
const intervalDue = track.totalMs - track.markedMs >= this.checkpointIntervalMs;
|
|
336
|
+
if (!firstChunk && !intervalDue) return null;
|
|
337
|
+
return this.createMark(track, false);
|
|
338
|
+
}
|
|
339
|
+
/**
|
|
340
|
+
* The response finished generating. If audio accumulated past the last
|
|
341
|
+
* checkpoint, returns a final tail mark that MUST be sent to Twilio (it is
|
|
342
|
+
* what makes `playback.finished` fire); otherwise flags the pending mark
|
|
343
|
+
* covering the full total as final and returns null.
|
|
344
|
+
*/
|
|
345
|
+
onGenerationDone(responseId) {
|
|
346
|
+
const track = this.responses.get(responseId);
|
|
347
|
+
if (!track) return null;
|
|
348
|
+
track.generationDone = true;
|
|
349
|
+
if (track.totalMs === 0) {
|
|
350
|
+
track.finished = true;
|
|
351
|
+
return null;
|
|
352
|
+
}
|
|
353
|
+
if (track.finished || track.flushed) return null;
|
|
354
|
+
let covered = false;
|
|
355
|
+
for (const record of this.marks.values()) if (record.responseId === responseId && record.cumulativeMs === track.totalMs) {
|
|
356
|
+
record.isFinal = true;
|
|
357
|
+
covered = true;
|
|
358
|
+
}
|
|
359
|
+
if (covered) return null;
|
|
360
|
+
if (track.playedMs >= track.totalMs) {
|
|
361
|
+
track.finished = true;
|
|
362
|
+
this.maybeForget(track);
|
|
363
|
+
return null;
|
|
364
|
+
}
|
|
365
|
+
return this.createMark(track, true);
|
|
366
|
+
}
|
|
367
|
+
createMark(track, isFinal) {
|
|
368
|
+
track.sentMarks++;
|
|
369
|
+
track.markedMs = track.totalMs;
|
|
370
|
+
const name = `${TRACKED_MARK_PREFIX}${++this.markSeq}`;
|
|
371
|
+
this.marks.set(name, {
|
|
372
|
+
responseId: track.responseId,
|
|
373
|
+
cumulativeMs: track.totalMs,
|
|
374
|
+
epoch: this.epoch,
|
|
375
|
+
isFinal
|
|
376
|
+
});
|
|
377
|
+
return name;
|
|
378
|
+
}
|
|
379
|
+
/** Process a mark echo from Twilio. Returns null for unknown marks. */
|
|
380
|
+
onMarkEcho(name) {
|
|
381
|
+
const record = this.marks.get(name);
|
|
382
|
+
if (!record) return null;
|
|
383
|
+
this.marks.delete(name);
|
|
384
|
+
const track = this.responses.get(record.responseId);
|
|
385
|
+
if (!track) return null;
|
|
386
|
+
track.echoedMarks++;
|
|
387
|
+
if (record.epoch !== this.epoch || track.flushed) {
|
|
388
|
+
this.maybeForget(track);
|
|
389
|
+
return {
|
|
390
|
+
kind: "flushed",
|
|
391
|
+
responseId: record.responseId,
|
|
392
|
+
playedMs: track.playedMs,
|
|
393
|
+
playbackStarted: false,
|
|
394
|
+
playbackFinished: false
|
|
395
|
+
};
|
|
396
|
+
}
|
|
397
|
+
track.playedMs = Math.max(track.playedMs, record.cumulativeMs);
|
|
398
|
+
track.lastEchoAtMs = this.now();
|
|
399
|
+
const playbackStarted = !track.playbackStarted;
|
|
400
|
+
track.playbackStarted = true;
|
|
401
|
+
let playbackFinished = false;
|
|
402
|
+
if (record.isFinal && track.generationDone && !track.finished) {
|
|
403
|
+
track.finished = true;
|
|
404
|
+
playbackFinished = true;
|
|
405
|
+
}
|
|
406
|
+
this.maybeForget(track);
|
|
407
|
+
return {
|
|
408
|
+
kind: "played",
|
|
409
|
+
responseId: record.responseId,
|
|
410
|
+
playedMs: track.playedMs,
|
|
411
|
+
playbackStarted,
|
|
412
|
+
playbackFinished
|
|
413
|
+
};
|
|
414
|
+
}
|
|
415
|
+
/**
|
|
416
|
+
* Bump the clear-epoch (call this immediately BEFORE sending `clear`).
|
|
417
|
+
* Every un-echoed mark becomes flushed; responses with unplayed audio are
|
|
418
|
+
* finalized at their current playedMs. Returns those interrupted responses.
|
|
419
|
+
*/
|
|
420
|
+
onClear() {
|
|
421
|
+
this.epoch++;
|
|
422
|
+
const interrupted = [];
|
|
423
|
+
for (const track of this.responses.values()) if (!track.finished && track.playedMs < track.totalMs) {
|
|
424
|
+
track.flushed = true;
|
|
425
|
+
track.finished = true;
|
|
426
|
+
interrupted.push({
|
|
427
|
+
responseId: track.responseId,
|
|
428
|
+
playedMs: track.playedMs,
|
|
429
|
+
itemId: track.itemId
|
|
430
|
+
});
|
|
431
|
+
}
|
|
432
|
+
return interrupted;
|
|
433
|
+
}
|
|
434
|
+
/**
|
|
435
|
+
* Best-estimate of what the caller has heard of `responseId` right now:
|
|
436
|
+
* last confirmed mark plus wall-clock elapsed since, clamped to the total.
|
|
437
|
+
*/
|
|
438
|
+
estimatePlayedMs(responseId) {
|
|
439
|
+
const track = this.responses.get(responseId);
|
|
440
|
+
if (!track) return 0;
|
|
441
|
+
if (track.finished || !track.playbackStarted || track.lastEchoAtMs === null) return track.playedMs;
|
|
442
|
+
const elapsed = this.now() - track.lastEchoAtMs;
|
|
443
|
+
return Math.min(track.playedMs + Math.max(0, elapsed), track.totalMs);
|
|
444
|
+
}
|
|
445
|
+
/**
|
|
446
|
+
* Live responses with unplayed audio, with their current played-ms
|
|
447
|
+
* estimates — captured BEFORE onClear() freezes them, to feed truncation.
|
|
448
|
+
*/
|
|
449
|
+
snapshotActive() {
|
|
450
|
+
const active = [];
|
|
451
|
+
for (const track of this.responses.values()) if (!track.finished && track.totalMs > 0) active.push({
|
|
452
|
+
responseId: track.responseId,
|
|
453
|
+
itemId: track.itemId,
|
|
454
|
+
estimatedPlayedMs: this.estimatePlayedMs(track.responseId)
|
|
455
|
+
});
|
|
456
|
+
return active;
|
|
457
|
+
}
|
|
458
|
+
/** Any response with audio still unplayed (and not flushed)? */
|
|
459
|
+
isPlaybackActive() {
|
|
460
|
+
for (const track of this.responses.values()) if (!track.finished && track.totalMs > 0) return true;
|
|
461
|
+
return false;
|
|
462
|
+
}
|
|
463
|
+
itemIdFor(responseId) {
|
|
464
|
+
return this.responses.get(responseId)?.itemId;
|
|
465
|
+
}
|
|
466
|
+
playedMsFor(responseId) {
|
|
467
|
+
return this.responses.get(responseId)?.playedMs ?? 0;
|
|
468
|
+
}
|
|
469
|
+
totalMsFor(responseId) {
|
|
470
|
+
return this.responses.get(responseId)?.totalMs ?? 0;
|
|
471
|
+
}
|
|
472
|
+
ensureTrack(responseId) {
|
|
473
|
+
let track = this.responses.get(responseId);
|
|
474
|
+
if (!track) {
|
|
475
|
+
track = {
|
|
476
|
+
responseId,
|
|
477
|
+
totalMs: 0,
|
|
478
|
+
markedMs: 0,
|
|
479
|
+
playedMs: 0,
|
|
480
|
+
sentMarks: 0,
|
|
481
|
+
echoedMarks: 0,
|
|
482
|
+
generationDone: false,
|
|
483
|
+
playbackStarted: false,
|
|
484
|
+
finished: false,
|
|
485
|
+
flushed: false,
|
|
486
|
+
lastEchoAtMs: null
|
|
487
|
+
};
|
|
488
|
+
this.responses.set(responseId, track);
|
|
489
|
+
}
|
|
490
|
+
return track;
|
|
491
|
+
}
|
|
492
|
+
/** Drop bookkeeping once a response is finished and every mark came home. */
|
|
493
|
+
maybeForget(track) {
|
|
494
|
+
if (track.finished && track.echoedMarks >= track.sentMarks) this.responses.delete(track.responseId);
|
|
495
|
+
}
|
|
496
|
+
};
|
|
497
|
+
//#endregion
|
|
498
|
+
//#region src/providers/base/reconnect.ts
|
|
499
|
+
const DEFAULT_RECONNECT_POLICY = {
|
|
500
|
+
maxAttempts: 5,
|
|
501
|
+
initialDelayMs: 250,
|
|
502
|
+
maxDelayMs: 8e3,
|
|
503
|
+
jitter: true
|
|
504
|
+
};
|
|
505
|
+
/** Deterministic (pre-jitter) delay for a 1-based attempt number. */
|
|
506
|
+
function baseDelayForAttempt(policy, attempt) {
|
|
507
|
+
const exp = policy.initialDelayMs * 2 ** Math.max(0, attempt - 1);
|
|
508
|
+
return Math.min(exp, policy.maxDelayMs);
|
|
509
|
+
}
|
|
510
|
+
function delayForAttempt(policy, attempt, random = Math.random) {
|
|
511
|
+
const base = baseDelayForAttempt(policy, attempt);
|
|
512
|
+
return policy.jitter ? Math.floor(random() * base) : base;
|
|
513
|
+
}
|
|
514
|
+
//#endregion
|
|
515
|
+
//#region src/session/transcript.ts
|
|
516
|
+
/** Render a transcript for history re-injection after reconnect/handoff. */
|
|
517
|
+
function formatTranscriptForInjection(entries, maxTurns = 30) {
|
|
518
|
+
return entries.slice(-maxTurns).map((entry) => `${entry.role === "user" ? "Caller" : "Agent"}: ${entry.text}`).join("\n");
|
|
519
|
+
}
|
|
520
|
+
//#endregion
|
|
521
|
+
//#region src/session/usage.ts
|
|
522
|
+
function emptyUsage() {
|
|
523
|
+
return {
|
|
524
|
+
inputTokens: 0,
|
|
525
|
+
outputTokens: 0,
|
|
526
|
+
totalTokens: 0,
|
|
527
|
+
inputTokenDetails: {
|
|
528
|
+
textTokens: 0,
|
|
529
|
+
audioTokens: 0,
|
|
530
|
+
cachedTokens: 0
|
|
531
|
+
},
|
|
532
|
+
outputTokenDetails: {
|
|
533
|
+
textTokens: 0,
|
|
534
|
+
audioTokens: 0
|
|
535
|
+
},
|
|
536
|
+
responses: 0
|
|
537
|
+
};
|
|
538
|
+
}
|
|
539
|
+
/** Sums per-response provider usage into call totals. */
|
|
540
|
+
var UsageAccumulator = class {
|
|
541
|
+
usage = emptyUsage();
|
|
542
|
+
add(providerUsage) {
|
|
543
|
+
this.usage.inputTokens += providerUsage.inputTokens;
|
|
544
|
+
this.usage.outputTokens += providerUsage.outputTokens;
|
|
545
|
+
this.usage.totalTokens += providerUsage.totalTokens;
|
|
546
|
+
this.usage.inputTokenDetails.textTokens += providerUsage.inputTokenDetails?.textTokens ?? 0;
|
|
547
|
+
this.usage.inputTokenDetails.audioTokens += providerUsage.inputTokenDetails?.audioTokens ?? 0;
|
|
548
|
+
this.usage.inputTokenDetails.cachedTokens += providerUsage.inputTokenDetails?.cachedTokens ?? 0;
|
|
549
|
+
this.usage.outputTokenDetails.textTokens += providerUsage.outputTokenDetails?.textTokens ?? 0;
|
|
550
|
+
this.usage.outputTokenDetails.audioTokens += providerUsage.outputTokenDetails?.audioTokens ?? 0;
|
|
551
|
+
this.usage.responses += 1;
|
|
552
|
+
return this.snapshot();
|
|
553
|
+
}
|
|
554
|
+
snapshot() {
|
|
555
|
+
return structuredClone(this.usage);
|
|
556
|
+
}
|
|
557
|
+
};
|
|
558
|
+
//#endregion
|
|
559
|
+
//#region src/tools/context.ts
|
|
560
|
+
/**
|
|
561
|
+
* Session-scoped key/value context: seeded at call start (tenant id, caller
|
|
562
|
+
* profile, …), readable and writable from any tool during the call, carried
|
|
563
|
+
* across agent handoffs, and included in store snapshots.
|
|
564
|
+
*/
|
|
565
|
+
var SessionContext = class {
|
|
566
|
+
data = /* @__PURE__ */ new Map();
|
|
567
|
+
constructor(initial) {
|
|
568
|
+
for (const [key, value] of Object.entries(initial ?? {})) this.data.set(key, value);
|
|
569
|
+
}
|
|
570
|
+
get(key) {
|
|
571
|
+
return this.data.get(key);
|
|
572
|
+
}
|
|
573
|
+
set(key, value) {
|
|
574
|
+
this.data.set(key, value);
|
|
575
|
+
}
|
|
576
|
+
has(key) {
|
|
577
|
+
return this.data.has(key);
|
|
578
|
+
}
|
|
579
|
+
delete(key) {
|
|
580
|
+
return this.data.delete(key);
|
|
581
|
+
}
|
|
582
|
+
toJSON() {
|
|
583
|
+
return Object.fromEntries(this.data);
|
|
584
|
+
}
|
|
585
|
+
};
|
|
586
|
+
//#endregion
|
|
587
|
+
//#region src/tools/middleware.ts
|
|
588
|
+
/** Apply every middleware's decoration, first-registered first. */
|
|
589
|
+
function decorateTool(tool, middlewares) {
|
|
590
|
+
let description = tool.description;
|
|
591
|
+
let parameters = tool.parametersJsonSchema;
|
|
592
|
+
for (const middleware of middlewares) {
|
|
593
|
+
const decoration = middleware.decorate?.(tool);
|
|
594
|
+
if (decoration?.description !== void 0) description = decoration.description;
|
|
595
|
+
if (decoration?.parameters !== void 0) parameters = decoration.parameters;
|
|
596
|
+
}
|
|
597
|
+
return {
|
|
598
|
+
description,
|
|
599
|
+
parameters
|
|
600
|
+
};
|
|
601
|
+
}
|
|
602
|
+
/** Compose wrapExecute chains around an innermost executor. */
|
|
603
|
+
function composeExecution(tool, input, ctx, middlewares, innermost) {
|
|
604
|
+
let chain = innermost;
|
|
605
|
+
for (const middleware of [...middlewares].reverse()) {
|
|
606
|
+
if (!middleware.wrapExecute) continue;
|
|
607
|
+
const next = chain;
|
|
608
|
+
chain = () => middleware.wrapExecute(tool, input, ctx, next);
|
|
609
|
+
}
|
|
610
|
+
return chain();
|
|
611
|
+
}
|
|
612
|
+
//#endregion
|
|
613
|
+
//#region src/tools/result-queue.ts
|
|
614
|
+
var ToolResultQueue = class {
|
|
615
|
+
items = [];
|
|
616
|
+
get size() {
|
|
617
|
+
return this.items.length;
|
|
618
|
+
}
|
|
619
|
+
enqueue(item) {
|
|
620
|
+
this.items.push(item);
|
|
621
|
+
}
|
|
622
|
+
/** Remove and return everything, in arrival order. */
|
|
623
|
+
drain() {
|
|
624
|
+
const drained = this.items;
|
|
625
|
+
this.items = [];
|
|
626
|
+
return drained;
|
|
627
|
+
}
|
|
628
|
+
clear() {
|
|
629
|
+
this.items = [];
|
|
630
|
+
}
|
|
631
|
+
};
|
|
632
|
+
//#endregion
|
|
633
|
+
//#region src/tools/builtins/finishCall.ts
|
|
634
|
+
const DEFAULT_DESCRIPTION$1 = "End the phone call when the conversation is complete, the caller says goodbye, or they ask to hang up. After calling this you will be asked to say one brief closing line; say it and then stop speaking.";
|
|
635
|
+
const DEFAULT_FAREWELL_INSTRUCTION = "The call is now ending. Say one brief, warm goodbye line to the caller — in the language of the conversation — and then stop speaking entirely.";
|
|
636
|
+
/**
|
|
637
|
+
* Graceful hangup: the tool result *instructs* the model to say goodbye (so
|
|
638
|
+
* the farewell is the model's own, in context and language), the session
|
|
639
|
+
* watches the goodbye's playout via marks, and only then completes the leg.
|
|
640
|
+
* A watchdog forces completion if the goodbye never materializes.
|
|
641
|
+
*/
|
|
642
|
+
function createFinishCallTool(options = {}) {
|
|
643
|
+
return tool({
|
|
644
|
+
name: "finish_call",
|
|
645
|
+
description: options.description ?? DEFAULT_DESCRIPTION$1,
|
|
646
|
+
parameters: z.object({ reason: z.string().optional().describe("Why the call is ending, for the call log") }),
|
|
647
|
+
execute: async (input, ctx) => {
|
|
648
|
+
ctx.logger.info("finish_call invoked", { reason: input.reason });
|
|
649
|
+
ctx.session.finishCall();
|
|
650
|
+
return {
|
|
651
|
+
status: "ending_call",
|
|
652
|
+
instruction: options.farewellInstruction ?? DEFAULT_FAREWELL_INSTRUCTION
|
|
653
|
+
};
|
|
654
|
+
}
|
|
655
|
+
});
|
|
656
|
+
}
|
|
657
|
+
//#endregion
|
|
658
|
+
//#region src/tools/builtins/transferCall.ts
|
|
659
|
+
const DEFAULT_DESCRIPTION = "Transfer this call to a human agent or an external phone number. Use when the caller asks for a person, or the request is beyond what you can handle.";
|
|
660
|
+
function createTransferCallTool(options = {}) {
|
|
661
|
+
return tool({
|
|
662
|
+
name: "transfer_call",
|
|
663
|
+
description: options.description ?? DEFAULT_DESCRIPTION,
|
|
664
|
+
parameters: z.object({
|
|
665
|
+
phoneNumber: z.string().optional().describe("Destination in E.164 format (+15551234567). Omit to use the default."),
|
|
666
|
+
reason: z.string().optional().describe("Why the caller is being transferred")
|
|
667
|
+
}),
|
|
668
|
+
execute: async (input, ctx) => {
|
|
669
|
+
const target = input.phoneNumber ?? options.defaultPhoneNumber;
|
|
670
|
+
if (!target) return {
|
|
671
|
+
error: "no destination number available",
|
|
672
|
+
hint: "Ask the caller to hold and take a message instead."
|
|
673
|
+
};
|
|
674
|
+
ctx.session.transferTo(target, {
|
|
675
|
+
callerId: options.callerId,
|
|
676
|
+
announcement: options.announcement
|
|
677
|
+
});
|
|
678
|
+
return {
|
|
679
|
+
status: "transferring",
|
|
680
|
+
to: target
|
|
681
|
+
};
|
|
682
|
+
}
|
|
683
|
+
});
|
|
684
|
+
}
|
|
685
|
+
//#endregion
|
|
686
|
+
//#region src/bridge/CallSession.ts
|
|
687
|
+
/**
|
|
688
|
+
* CallSession — the per-call orchestrator.
|
|
689
|
+
*
|
|
690
|
+
* One mutable object owns all live state for a call (no split-brain between a
|
|
691
|
+
* store and hot data). The SessionStore only receives immutable snapshots at
|
|
692
|
+
* checkpoints, never reads on the audio path. Teardown is centralized and
|
|
693
|
+
* idempotent: every timer, tool AbortController, provider socket, and Twilio
|
|
694
|
+
* socket is released exactly once.
|
|
695
|
+
*/
|
|
696
|
+
const MAX_BUFFERED_INBOUND_FRAMES = 250;
|
|
697
|
+
var CallSession = class extends TypedEmitter {
|
|
698
|
+
callSid;
|
|
699
|
+
streamSid;
|
|
700
|
+
callInfo;
|
|
701
|
+
context;
|
|
702
|
+
stateValue = "connecting";
|
|
703
|
+
deps;
|
|
704
|
+
log;
|
|
705
|
+
tracker = new PlaybackTracker();
|
|
706
|
+
interruptions;
|
|
707
|
+
usageAccumulator = new UsageAccumulator();
|
|
708
|
+
toolQueue = new ToolResultQueue();
|
|
709
|
+
transcriptEntries = [];
|
|
710
|
+
toolset;
|
|
711
|
+
startedAtMs = Date.now();
|
|
712
|
+
interruptedResponses = /* @__PURE__ */ new Set();
|
|
713
|
+
runningTools = /* @__PURE__ */ new Map();
|
|
714
|
+
timers = /* @__PURE__ */ new Set();
|
|
715
|
+
provider = null;
|
|
716
|
+
activeAgentValue;
|
|
717
|
+
generating = false;
|
|
718
|
+
currentResponseId = null;
|
|
719
|
+
firstTurnDone = false;
|
|
720
|
+
/**
|
|
721
|
+
* A caller turn that a blocked barge-in swallowed. With bridge-owned
|
|
722
|
+
* interruptions the server no longer auto-responds while the protected
|
|
723
|
+
* response is active, so the bridge answers it after protected playback:
|
|
724
|
+
* 'speaking' → user started during a block; 'committed' → their turn ended
|
|
725
|
+
* (VAD committed it) and deserves a response once playback finishes.
|
|
726
|
+
*/
|
|
727
|
+
blockedUserTurn = "idle";
|
|
728
|
+
greeted = false;
|
|
729
|
+
answered;
|
|
730
|
+
inboundBuffer = [];
|
|
731
|
+
reconnectAttempt = 0;
|
|
732
|
+
reconnecting = false;
|
|
733
|
+
pendingHangup = null;
|
|
734
|
+
pendingTransfer = null;
|
|
735
|
+
endedReason = null;
|
|
736
|
+
hangupReason = "agent-hangup";
|
|
737
|
+
middlewares;
|
|
738
|
+
bgAudio;
|
|
739
|
+
/** Deferred tool calls awaiting a real result (execute() or submitToolResult). */
|
|
740
|
+
deferredPending = /* @__PURE__ */ new Map();
|
|
741
|
+
/** Human-in-the-loop calls awaiting approve/reject. */
|
|
742
|
+
approvals = /* @__PURE__ */ new Map();
|
|
743
|
+
/** Text turns to inject once the agent finishes speaking (deferred results). */
|
|
744
|
+
pendingInjections = [];
|
|
745
|
+
idleTimer = null;
|
|
746
|
+
nudgeCount = 0;
|
|
747
|
+
/** All agents reachable from the root via handoffs, by id. */
|
|
748
|
+
agents;
|
|
749
|
+
handoffHistory = [];
|
|
750
|
+
handoffInProgress = false;
|
|
751
|
+
/** Pre-synthesized greeting playout state. */
|
|
752
|
+
pregreeting = null;
|
|
753
|
+
constructor(deps) {
|
|
754
|
+
super();
|
|
755
|
+
this.deps = deps;
|
|
756
|
+
this.callSid = deps.start.start.callSid;
|
|
757
|
+
this.streamSid = deps.start.start.streamSid ?? deps.start.streamSid;
|
|
758
|
+
this.log = childLogger(deps.logger, { callSid: this.callSid });
|
|
759
|
+
this.activeAgentValue = deps.agent;
|
|
760
|
+
this.context = new SessionContext(deps.options.context);
|
|
761
|
+
this.interruptions = new InterruptionController(deps.options.interruptions);
|
|
762
|
+
const params = deps.start.start.customParameters ?? {};
|
|
763
|
+
this.callInfo = {
|
|
764
|
+
direction: params.direction === "outbound" ? "outbound" : "inbound",
|
|
765
|
+
from: params.from,
|
|
766
|
+
to: params.to,
|
|
767
|
+
customParameters: params
|
|
768
|
+
};
|
|
769
|
+
this.answered = deps.answeredEarly === true || this.callInfo.direction === "inbound";
|
|
770
|
+
this.agents = collectAgentGraph(deps.agent);
|
|
771
|
+
this.middlewares = deps.middlewares ?? [];
|
|
772
|
+
this.bgAudio = new BackgroundAudioPlayer({
|
|
773
|
+
sendMedia: (payload) => {
|
|
774
|
+
if (this.stateValue === "active" && this.deps.transport.isOpen) this.deps.transport.sendMedia(payload);
|
|
775
|
+
},
|
|
776
|
+
onStarted: (info) => this.emit("background_audio.started", info),
|
|
777
|
+
onStopped: (info) => this.emit("background_audio.stopped", info)
|
|
778
|
+
});
|
|
779
|
+
this.toolset = this.buildToolset();
|
|
780
|
+
this.wireTransport();
|
|
781
|
+
}
|
|
782
|
+
get state() {
|
|
783
|
+
return this.stateValue;
|
|
784
|
+
}
|
|
785
|
+
get activeAgent() {
|
|
786
|
+
return this.activeAgentValue;
|
|
787
|
+
}
|
|
788
|
+
get usage() {
|
|
789
|
+
return this.usageAccumulator.snapshot();
|
|
790
|
+
}
|
|
791
|
+
get transcript() {
|
|
792
|
+
return this.transcriptEntries;
|
|
793
|
+
}
|
|
794
|
+
/** Connect the provider and activate the call. Called by the bridge. */
|
|
795
|
+
async begin() {
|
|
796
|
+
this.playPreGreeting();
|
|
797
|
+
try {
|
|
798
|
+
this.provider = this.deps.providerFactory({
|
|
799
|
+
logger: this.log,
|
|
800
|
+
callSid: this.callSid
|
|
801
|
+
});
|
|
802
|
+
this.wireProvider(this.provider);
|
|
803
|
+
await this.provider.connect(this.buildProviderInit());
|
|
804
|
+
} catch (error) {
|
|
805
|
+
this.fail(error instanceof Error ? error : new Error(String(error)), "provider-failed");
|
|
806
|
+
return;
|
|
807
|
+
}
|
|
808
|
+
if (this.stateValue !== "connecting") return;
|
|
809
|
+
this.stateValue = "active";
|
|
810
|
+
this.emit("provider.connected");
|
|
811
|
+
this.emit("call.started", {
|
|
812
|
+
callSid: this.callSid,
|
|
813
|
+
streamSid: this.streamSid,
|
|
814
|
+
direction: this.callInfo.direction,
|
|
815
|
+
from: this.callInfo.from,
|
|
816
|
+
to: this.callInfo.to,
|
|
817
|
+
customParameters: this.callInfo.customParameters
|
|
818
|
+
});
|
|
819
|
+
if (this.pregreeting) this.provider.sendText(this.pregreeting.text, {
|
|
820
|
+
role: "assistant",
|
|
821
|
+
triggerResponse: false
|
|
822
|
+
});
|
|
823
|
+
this.flushInboundBuffer();
|
|
824
|
+
this.startMaxDurationWatchdog();
|
|
825
|
+
this.armIdleTimer();
|
|
826
|
+
this.saveSnapshot();
|
|
827
|
+
this.maybeGreet();
|
|
828
|
+
}
|
|
829
|
+
/** The outbound leg was answered (host's Twilio status callback). */
|
|
830
|
+
notifyAnswered() {
|
|
831
|
+
if (this.answered) return;
|
|
832
|
+
this.answered = true;
|
|
833
|
+
this.maybeGreet();
|
|
834
|
+
}
|
|
835
|
+
sendText(text, options = {}) {
|
|
836
|
+
this.provider?.sendText(text, options);
|
|
837
|
+
}
|
|
838
|
+
async updateInstructions(instructions) {
|
|
839
|
+
await this.provider?.updateSession({ instructions });
|
|
840
|
+
}
|
|
841
|
+
/** Manual barge-in: stop the agent mid-sentence. */
|
|
842
|
+
interrupt() {
|
|
843
|
+
if (!this.generating && !this.tracker.isPlaybackActive()) return;
|
|
844
|
+
this.provider?.cancelResponse();
|
|
845
|
+
this.performInterrupt();
|
|
846
|
+
}
|
|
847
|
+
/**
|
|
848
|
+
* Gracefully end the call. With `finalMessage`, the agent speaks it first.
|
|
849
|
+
* With `immediate: true`, skip the goodbye and complete now. Resolves when
|
|
850
|
+
* the call has actually ended.
|
|
851
|
+
*/
|
|
852
|
+
finishCall(options = {}) {
|
|
853
|
+
if (this.stateValue === "ended") return Promise.resolve();
|
|
854
|
+
if (options.immediate) {
|
|
855
|
+
this.completeHangup();
|
|
856
|
+
return Promise.resolve();
|
|
857
|
+
}
|
|
858
|
+
if (options.finalMessage) this.provider?.createResponse({ instructions: `Say exactly this to the caller, then stop speaking: "${options.finalMessage}"` });
|
|
859
|
+
return this.armHangup();
|
|
860
|
+
}
|
|
861
|
+
/** Transfer the PSTN leg. Waits for current playback (and announcement). */
|
|
862
|
+
async transferTo(phoneNumber, options = {}) {
|
|
863
|
+
if (!this.deps.rest) throw new Error("transferTo requires Twilio REST credentials (BridgeConfig.twilio)");
|
|
864
|
+
if (this.stateValue === "ended" || this.stateValue === "ending") return;
|
|
865
|
+
this.pendingTransfer = {
|
|
866
|
+
phoneNumber,
|
|
867
|
+
callerId: options.callerId ?? this.deps.restCallerId
|
|
868
|
+
};
|
|
869
|
+
if (options.announcement) {
|
|
870
|
+
this.provider?.createResponse({ instructions: `Tell the caller now, briefly: "${options.announcement}". Then stop speaking.` });
|
|
871
|
+
return;
|
|
872
|
+
}
|
|
873
|
+
if (this.generating || this.tracker.isPlaybackActive()) return;
|
|
874
|
+
await this.executePendingTransfer();
|
|
875
|
+
}
|
|
876
|
+
/** Swap the active agent (swarm handoff). Accepts an Agent or its id. */
|
|
877
|
+
async handoffTo(agent) {
|
|
878
|
+
const targetId = typeof agent === "string" ? agent : agent.id;
|
|
879
|
+
const target = this.agents.get(targetId);
|
|
880
|
+
if (!target) throw new Error(`unknown agent "${targetId}" — agents must be reachable from the root agent's handoffs`);
|
|
881
|
+
await this.performHandoff(target);
|
|
882
|
+
}
|
|
883
|
+
/** Complete a `deferred` tool call from the host side. */
|
|
884
|
+
submitToolResult(toolCallId, result) {
|
|
885
|
+
this.completeDeferredFromHost(toolCallId, result);
|
|
886
|
+
}
|
|
887
|
+
/** Start background audio manually (independent of tools). */
|
|
888
|
+
async playBackgroundAudio(spec, options = {}) {
|
|
889
|
+
this.bgAudio.start(spec, {
|
|
890
|
+
startDelayMs: 0,
|
|
891
|
+
...options
|
|
892
|
+
});
|
|
893
|
+
}
|
|
894
|
+
async stopBackgroundAudio(options = {}) {
|
|
895
|
+
this.bgAudio.stop({ immediate: !options.fadeOutMs });
|
|
896
|
+
}
|
|
897
|
+
/** Immediate teardown (no goodbye). */
|
|
898
|
+
async end(reason = "agent-hangup") {
|
|
899
|
+
await this.teardown(reason);
|
|
900
|
+
}
|
|
901
|
+
buildToolset() {
|
|
902
|
+
const tools = /* @__PURE__ */ new Map();
|
|
903
|
+
const builtin = this.deps.builtinTools ?? {};
|
|
904
|
+
if (builtin.finishCall) {
|
|
905
|
+
const opts = typeof builtin.finishCall === "object" ? builtin.finishCall : {};
|
|
906
|
+
tools.set("finish_call", createFinishCallTool({
|
|
907
|
+
description: opts.description,
|
|
908
|
+
farewellInstruction: opts.farewellInstruction
|
|
909
|
+
}));
|
|
910
|
+
}
|
|
911
|
+
if (builtin.transferCall) {
|
|
912
|
+
const opts = builtin.transferCall;
|
|
913
|
+
tools.set("transfer_call", createTransferCallTool({
|
|
914
|
+
defaultPhoneNumber: opts.defaultPhoneNumber,
|
|
915
|
+
callerId: opts.callerId,
|
|
916
|
+
description: opts.description,
|
|
917
|
+
announcement: opts.announcement
|
|
918
|
+
}));
|
|
919
|
+
}
|
|
920
|
+
for (const target of this.activeAgentValue.handoffs) {
|
|
921
|
+
const handoffTool = createHandoffTool(target);
|
|
922
|
+
tools.set(handoffTool.name, handoffTool);
|
|
923
|
+
}
|
|
924
|
+
for (const tool of this.activeAgentValue.tools) {
|
|
925
|
+
if (tools.has(tool.name)) this.log.warn(`agent tool "${tool.name}" overrides a builtin of the same name`);
|
|
926
|
+
tools.set(tool.name, tool);
|
|
927
|
+
}
|
|
928
|
+
return tools;
|
|
929
|
+
}
|
|
930
|
+
buildProviderInit() {
|
|
931
|
+
return {
|
|
932
|
+
instructions: this.activeAgentValue.resolveInstructions(this.context) + (this.pregreeting ? `\n\nYou already opened the call by saying: "${this.pregreeting.text}". Do not greet again — continue the conversation from there.` : ""),
|
|
933
|
+
voice: this.activeAgentValue.voice,
|
|
934
|
+
vad: this.deps.options.vad,
|
|
935
|
+
bridgeOwnsInterruptions: true,
|
|
936
|
+
tools: [...this.toolset.values()].map((tool) => {
|
|
937
|
+
const decorated = decorateTool(tool, this.middlewares);
|
|
938
|
+
return {
|
|
939
|
+
name: tool.name,
|
|
940
|
+
description: decorated.description,
|
|
941
|
+
parameters: decorated.parameters
|
|
942
|
+
};
|
|
943
|
+
}),
|
|
944
|
+
providerOptions: this.activeAgentValue.providerOptions
|
|
945
|
+
};
|
|
946
|
+
}
|
|
947
|
+
wireTransport() {
|
|
948
|
+
const { transport } = this.deps;
|
|
949
|
+
transport.on("media", (event) => this.handleInboundMedia(event.media.payload));
|
|
950
|
+
transport.on("mark", (event) => this.handleMarkEcho(event.mark.name));
|
|
951
|
+
transport.on("dtmf", (event) => {
|
|
952
|
+
this.clearIdleTimer();
|
|
953
|
+
this.nudgeCount = 0;
|
|
954
|
+
this.emit("dtmf", { digit: event.dtmf.digit });
|
|
955
|
+
});
|
|
956
|
+
transport.on("stop", () => void this.teardown("caller-hangup"));
|
|
957
|
+
transport.on("close", () => void this.teardown("caller-hangup"));
|
|
958
|
+
transport.on("error", (error) => this.emitError(error));
|
|
959
|
+
}
|
|
960
|
+
/** Emit 'error'; when the host attached no listener, log instead of losing it. */
|
|
961
|
+
emitError(error) {
|
|
962
|
+
if (this.listenerCount("error") === 0) this.log.error("session error (no \"error\" listener attached)", { error: String(error) });
|
|
963
|
+
this.emit("error", error);
|
|
964
|
+
}
|
|
965
|
+
wireProvider(provider) {
|
|
966
|
+
provider.on("audio", (delta) => {
|
|
967
|
+
if (this.stateValue !== "active" && this.stateValue !== "ending") return;
|
|
968
|
+
if (!this.deps.transport.isOpen) return;
|
|
969
|
+
if (this.interruptedResponses.has(delta.responseId)) return;
|
|
970
|
+
this.bgAudio.notifyAgentAudio();
|
|
971
|
+
this.deps.transport.sendMedia(delta.base64Mulaw);
|
|
972
|
+
this.noteHangupProgress();
|
|
973
|
+
const chunkMs = base64ByteLength(delta.base64Mulaw) / 8;
|
|
974
|
+
const markName = this.tracker.onAudioSent(delta.responseId, chunkMs, delta.itemId);
|
|
975
|
+
if (markName) this.deps.transport.sendMark(markName);
|
|
976
|
+
});
|
|
977
|
+
provider.on("responseStarted", ({ responseId }) => {
|
|
978
|
+
this.generating = true;
|
|
979
|
+
this.currentResponseId = responseId;
|
|
980
|
+
this.blockedUserTurn = "idle";
|
|
981
|
+
if (this.pendingHangup) this.pendingHangup.sawResponse = true;
|
|
982
|
+
this.noteHangupProgress();
|
|
983
|
+
this.clearIdleTimer();
|
|
984
|
+
this.interruptions.onResponseStarted(responseId);
|
|
985
|
+
this.emit("agent.speech.started", { responseId });
|
|
986
|
+
});
|
|
987
|
+
provider.on("responseDone", ({ responseId, usage }) => {
|
|
988
|
+
this.generating = false;
|
|
989
|
+
const tailMark = this.tracker.onGenerationDone(responseId);
|
|
990
|
+
if (tailMark && this.deps.transport.isOpen) this.deps.transport.sendMark(tailMark);
|
|
991
|
+
this.emit("agent.speech.ended", { responseId });
|
|
992
|
+
if (usage) {
|
|
993
|
+
const total = this.usageAccumulator.add(usage);
|
|
994
|
+
this.emit("usage.updated", total, usage);
|
|
995
|
+
}
|
|
996
|
+
if (!this.tracker.isPlaybackActive()) {
|
|
997
|
+
if (this.runningTools.size === 0 && this.toolQueue.size === 0) this.firstTurnDone = true;
|
|
998
|
+
this.interruptions.onPlaybackEnded();
|
|
999
|
+
this.flushToolQueue();
|
|
1000
|
+
this.executePendingTransfer();
|
|
1001
|
+
this.maybeCompleteHangup();
|
|
1002
|
+
}
|
|
1003
|
+
});
|
|
1004
|
+
provider.on("agentTranscript", ({ responseId, text }) => {
|
|
1005
|
+
const entry = {
|
|
1006
|
+
role: "agent",
|
|
1007
|
+
text,
|
|
1008
|
+
timestampMs: Date.now() - this.startedAtMs,
|
|
1009
|
+
agentId: this.activeAgentValue.id,
|
|
1010
|
+
...this.interruptedResponses.has(responseId) ? { interrupted: true } : {}
|
|
1011
|
+
};
|
|
1012
|
+
this.transcriptEntries.push(entry);
|
|
1013
|
+
this.emit("transcript.agent", entry);
|
|
1014
|
+
});
|
|
1015
|
+
provider.on("userTranscript", ({ text }) => {
|
|
1016
|
+
const entry = {
|
|
1017
|
+
role: "user",
|
|
1018
|
+
text,
|
|
1019
|
+
timestampMs: Date.now() - this.startedAtMs
|
|
1020
|
+
};
|
|
1021
|
+
this.transcriptEntries.push(entry);
|
|
1022
|
+
this.nudgeCount = 0;
|
|
1023
|
+
this.emit("transcript.user", entry);
|
|
1024
|
+
});
|
|
1025
|
+
provider.on("userSpeechStarted", () => {
|
|
1026
|
+
this.clearIdleTimer();
|
|
1027
|
+
this.nudgeCount = 0;
|
|
1028
|
+
this.emit("user.speech.started");
|
|
1029
|
+
this.handleBargeIn();
|
|
1030
|
+
});
|
|
1031
|
+
provider.on("userSpeechStopped", () => {
|
|
1032
|
+
if (this.blockedUserTurn === "speaking") this.blockedUserTurn = "committed";
|
|
1033
|
+
this.emit("user.speech.ended");
|
|
1034
|
+
});
|
|
1035
|
+
provider.on("toolCall", (call) => void this.handleToolCall(call));
|
|
1036
|
+
provider.on("error", (error) => this.emitError(error));
|
|
1037
|
+
provider.on("close", (info) => {
|
|
1038
|
+
if (this.stateValue === "ended" || this.stateValue === "ending") return;
|
|
1039
|
+
if (this.handoffInProgress) return;
|
|
1040
|
+
this.emit("provider.closed", {
|
|
1041
|
+
code: info.code,
|
|
1042
|
+
reason: info.reason
|
|
1043
|
+
});
|
|
1044
|
+
if (!info.retriable) {
|
|
1045
|
+
this.fail(/* @__PURE__ */ new Error(`provider closed (${info.code} ${info.reason ?? ""})`), "provider-failed");
|
|
1046
|
+
return;
|
|
1047
|
+
}
|
|
1048
|
+
this.scheduleReconnect();
|
|
1049
|
+
});
|
|
1050
|
+
}
|
|
1051
|
+
handleInboundMedia(payload) {
|
|
1052
|
+
if (this.stateValue === "ended" || this.stateValue === "ending") return;
|
|
1053
|
+
if (this.pregreeting && !this.pregreeting.played) return;
|
|
1054
|
+
if (this.deps.options.deafness.ignoreUserAudioUntilFirstTurnDone && !this.firstTurnDone) return;
|
|
1055
|
+
if (this.deps.options.deafness.muteDuringToolExecution && this.runningTools.size > 0) return;
|
|
1056
|
+
if (this.deps.options.deafness.muteWhileAgentSpeaking && this.tracker.isPlaybackActive()) return;
|
|
1057
|
+
if (this.interruptions.isSuspended) return;
|
|
1058
|
+
if (this.provider?.isConnected && !this.reconnecting) this.provider.sendAudio(payload);
|
|
1059
|
+
else {
|
|
1060
|
+
this.inboundBuffer.push(payload);
|
|
1061
|
+
if (this.inboundBuffer.length > MAX_BUFFERED_INBOUND_FRAMES) this.inboundBuffer.shift();
|
|
1062
|
+
}
|
|
1063
|
+
}
|
|
1064
|
+
flushInboundBuffer() {
|
|
1065
|
+
if (!this.provider?.isConnected) return;
|
|
1066
|
+
for (const payload of this.inboundBuffer) this.provider.sendAudio(payload);
|
|
1067
|
+
this.inboundBuffer = [];
|
|
1068
|
+
}
|
|
1069
|
+
handleMarkEcho(name) {
|
|
1070
|
+
if (name === PREGREETING_MARK) {
|
|
1071
|
+
if (this.pregreeting && !this.pregreeting.played) {
|
|
1072
|
+
this.pregreeting.played = true;
|
|
1073
|
+
this.firstTurnDone = true;
|
|
1074
|
+
this.emit("playback.finished", {
|
|
1075
|
+
responseId: "pregreeting",
|
|
1076
|
+
playedMs: this.pregreeting.durationMs
|
|
1077
|
+
});
|
|
1078
|
+
this.armIdleTimer();
|
|
1079
|
+
}
|
|
1080
|
+
return;
|
|
1081
|
+
}
|
|
1082
|
+
if (!PlaybackTracker.isTrackedMark(name)) return;
|
|
1083
|
+
const result = this.tracker.onMarkEcho(name);
|
|
1084
|
+
if (!result) return;
|
|
1085
|
+
this.noteHangupProgress();
|
|
1086
|
+
if (result.kind === "flushed") return;
|
|
1087
|
+
if (result.playbackStarted) {
|
|
1088
|
+
this.emit("playback.started", { responseId: result.responseId });
|
|
1089
|
+
this.interruptions.onPlaybackStarted(result.responseId);
|
|
1090
|
+
}
|
|
1091
|
+
if (result.playbackFinished) this.onPlaybackFinished(result.responseId, result.playedMs);
|
|
1092
|
+
}
|
|
1093
|
+
onPlaybackFinished(responseId, playedMs) {
|
|
1094
|
+
this.emit("playback.finished", {
|
|
1095
|
+
responseId,
|
|
1096
|
+
playedMs
|
|
1097
|
+
});
|
|
1098
|
+
this.firstTurnDone = true;
|
|
1099
|
+
this.interruptions.onPlaybackEnded();
|
|
1100
|
+
const hadQueuedToolResults = this.toolQueue.size > 0;
|
|
1101
|
+
this.flushToolQueue();
|
|
1102
|
+
this.executePendingTransfer();
|
|
1103
|
+
this.maybeCompleteHangup();
|
|
1104
|
+
if (this.blockedUserTurn === "committed") {
|
|
1105
|
+
this.blockedUserTurn = "idle";
|
|
1106
|
+
if (!hadQueuedToolResults && this.runningTools.size === 0) this.provider?.createResponse();
|
|
1107
|
+
}
|
|
1108
|
+
this.armIdleTimer();
|
|
1109
|
+
}
|
|
1110
|
+
handleBargeIn() {
|
|
1111
|
+
if (!this.generating && !this.tracker.isPlaybackActive()) return;
|
|
1112
|
+
const decision = this.interruptions.evaluate({ toolRunning: this.runningTools.size > 0 });
|
|
1113
|
+
if (!decision.allow) {
|
|
1114
|
+
if (decision.cause === "guard" || decision.cause === "disabled") this.blockedUserTurn = "speaking";
|
|
1115
|
+
this.emit("interruption.blocked", { cause: decision.cause });
|
|
1116
|
+
if (decision.instruction) this.provider?.sendText(decision.instruction, {
|
|
1117
|
+
role: "system",
|
|
1118
|
+
triggerResponse: true
|
|
1119
|
+
});
|
|
1120
|
+
return;
|
|
1121
|
+
}
|
|
1122
|
+
if (this.generating && this.provider?.capabilities.vadInterruptControl) this.provider.cancelResponse();
|
|
1123
|
+
this.performInterrupt();
|
|
1124
|
+
}
|
|
1125
|
+
performInterrupt() {
|
|
1126
|
+
const active = this.tracker.snapshotActive();
|
|
1127
|
+
this.tracker.onClear();
|
|
1128
|
+
this.interruptions.onPlaybackEnded();
|
|
1129
|
+
if (this.deps.transport.isOpen) this.deps.transport.sendClear();
|
|
1130
|
+
for (const response of active) {
|
|
1131
|
+
this.interruptedResponses.add(response.responseId);
|
|
1132
|
+
if (this.provider?.capabilities.truncate && response.itemId) this.provider.truncatePlayback(response.itemId, response.estimatedPlayedMs);
|
|
1133
|
+
this.emit("playback.interrupted", {
|
|
1134
|
+
responseId: response.responseId,
|
|
1135
|
+
playedMs: response.estimatedPlayedMs
|
|
1136
|
+
});
|
|
1137
|
+
this.emit("interruption", {
|
|
1138
|
+
responseId: response.responseId,
|
|
1139
|
+
playedMs: response.estimatedPlayedMs
|
|
1140
|
+
});
|
|
1141
|
+
}
|
|
1142
|
+
this.flushToolQueue();
|
|
1143
|
+
if (this.pendingHangup?.sawResponse) this.completeHangup();
|
|
1144
|
+
}
|
|
1145
|
+
async handleToolCall(call) {
|
|
1146
|
+
const tool = this.toolset.get(call.name);
|
|
1147
|
+
const started = Date.now();
|
|
1148
|
+
const baseInfo = {
|
|
1149
|
+
toolCallId: call.id,
|
|
1150
|
+
toolName: call.name,
|
|
1151
|
+
agentId: this.activeAgentValue.id
|
|
1152
|
+
};
|
|
1153
|
+
if (!tool) {
|
|
1154
|
+
this.log.warn(`model called unknown tool "${call.name}"`);
|
|
1155
|
+
this.emit("tool.failed", {
|
|
1156
|
+
...baseInfo,
|
|
1157
|
+
strategy: "unknown",
|
|
1158
|
+
error: "unknown tool"
|
|
1159
|
+
});
|
|
1160
|
+
this.deliverToolResult(call.id, { error: `Unknown tool: ${call.name}` });
|
|
1161
|
+
return;
|
|
1162
|
+
}
|
|
1163
|
+
let rawInput;
|
|
1164
|
+
try {
|
|
1165
|
+
rawInput = call.argumentsJson.trim() === "" ? {} : JSON.parse(call.argumentsJson);
|
|
1166
|
+
} catch {
|
|
1167
|
+
this.deliverToolResult(call.id, {
|
|
1168
|
+
error: "invalid_arguments",
|
|
1169
|
+
message: "Tool arguments were not valid JSON. Retry with corrected arguments."
|
|
1170
|
+
});
|
|
1171
|
+
this.emit("tool.failed", {
|
|
1172
|
+
...baseInfo,
|
|
1173
|
+
strategy: tool.strategy,
|
|
1174
|
+
error: "invalid JSON arguments"
|
|
1175
|
+
});
|
|
1176
|
+
return;
|
|
1177
|
+
}
|
|
1178
|
+
const parsed = tool.parameters.safeParse(rawInput);
|
|
1179
|
+
if (!parsed.success) {
|
|
1180
|
+
this.deliverToolResult(call.id, {
|
|
1181
|
+
error: "validation_failed",
|
|
1182
|
+
message: "Tool arguments failed validation. Fix them and retry.",
|
|
1183
|
+
issues: parsed.error?.issues ?? []
|
|
1184
|
+
});
|
|
1185
|
+
this.emit("tool.failed", {
|
|
1186
|
+
...baseInfo,
|
|
1187
|
+
strategy: tool.strategy,
|
|
1188
|
+
error: "argument validation failed"
|
|
1189
|
+
});
|
|
1190
|
+
return;
|
|
1191
|
+
}
|
|
1192
|
+
const input = parsed.data;
|
|
1193
|
+
this.emit("tool.started", {
|
|
1194
|
+
...baseInfo,
|
|
1195
|
+
strategy: tool.strategy,
|
|
1196
|
+
input
|
|
1197
|
+
});
|
|
1198
|
+
switch (tool.strategy) {
|
|
1199
|
+
case "sync":
|
|
1200
|
+
await this.runForegroundTool(tool, call, input, started);
|
|
1201
|
+
break;
|
|
1202
|
+
case "dispatch":
|
|
1203
|
+
this.deliverToolResult(call.id, {
|
|
1204
|
+
status: "queued",
|
|
1205
|
+
note: "The task was dispatched and runs in the background. Continue the conversation naturally; do not mention internal processing."
|
|
1206
|
+
});
|
|
1207
|
+
this.runDetachedTool(tool, call, input, started, (outcome) => {
|
|
1208
|
+
if (outcome.ok) this.emit("tool.completed", {
|
|
1209
|
+
...baseInfo,
|
|
1210
|
+
strategy: tool.strategy,
|
|
1211
|
+
input,
|
|
1212
|
+
result: outcome.result,
|
|
1213
|
+
durationMs: Date.now() - started
|
|
1214
|
+
});
|
|
1215
|
+
else this.emit("tool.failed", {
|
|
1216
|
+
...baseInfo,
|
|
1217
|
+
strategy: tool.strategy,
|
|
1218
|
+
error: outcome.message,
|
|
1219
|
+
durationMs: Date.now() - started
|
|
1220
|
+
});
|
|
1221
|
+
});
|
|
1222
|
+
break;
|
|
1223
|
+
case "deferred":
|
|
1224
|
+
this.deferredPending.set(call.id, { toolName: call.name });
|
|
1225
|
+
this.deliverToolResult(call.id, {
|
|
1226
|
+
status: "pending",
|
|
1227
|
+
note: "The result is being prepared and will arrive shortly as a system message. Tell the caller you are looking into it and continue naturally."
|
|
1228
|
+
});
|
|
1229
|
+
this.runDetachedTool(tool, call, input, started, (outcome) => {
|
|
1230
|
+
if (outcome.ok) {
|
|
1231
|
+
this.completeDeferred(call.id, call.name, outcome.result);
|
|
1232
|
+
this.emit("tool.completed", {
|
|
1233
|
+
...baseInfo,
|
|
1234
|
+
strategy: tool.strategy,
|
|
1235
|
+
input,
|
|
1236
|
+
result: outcome.result,
|
|
1237
|
+
durationMs: Date.now() - started
|
|
1238
|
+
});
|
|
1239
|
+
} else {
|
|
1240
|
+
this.completeDeferred(call.id, call.name, { error: outcome.message });
|
|
1241
|
+
this.emit("tool.failed", {
|
|
1242
|
+
...baseInfo,
|
|
1243
|
+
strategy: tool.strategy,
|
|
1244
|
+
error: outcome.message,
|
|
1245
|
+
durationMs: Date.now() - started
|
|
1246
|
+
});
|
|
1247
|
+
}
|
|
1248
|
+
});
|
|
1249
|
+
break;
|
|
1250
|
+
case "humanInTheLoop": this.requestApproval(tool, call, input);
|
|
1251
|
+
}
|
|
1252
|
+
}
|
|
1253
|
+
/** sync path (also the approved HITL path): hold audio, execute, deliver. */
|
|
1254
|
+
async runForegroundTool(tool, call, input, started) {
|
|
1255
|
+
const baseInfo = {
|
|
1256
|
+
toolCallId: call.id,
|
|
1257
|
+
toolName: call.name,
|
|
1258
|
+
agentId: this.activeAgentValue.id
|
|
1259
|
+
};
|
|
1260
|
+
this.acquireHoldAudio(call.id, tool);
|
|
1261
|
+
try {
|
|
1262
|
+
const outcome = await this.executeToolBody(tool, call, input);
|
|
1263
|
+
if (outcome.ok && isHandoffDirective(outcome.result)) {
|
|
1264
|
+
const directive = outcome.result;
|
|
1265
|
+
const target = this.agents.get(directive.targetAgentId);
|
|
1266
|
+
if (!target) {
|
|
1267
|
+
this.deliverToolResult(call.id, { error: `unknown agent "${directive.targetAgentId}"` });
|
|
1268
|
+
return;
|
|
1269
|
+
}
|
|
1270
|
+
this.deliverToolResult(call.id, {
|
|
1271
|
+
status: "transferring_conversation",
|
|
1272
|
+
to: target.name
|
|
1273
|
+
}, false);
|
|
1274
|
+
this.emit("tool.completed", {
|
|
1275
|
+
...baseInfo,
|
|
1276
|
+
strategy: tool.strategy,
|
|
1277
|
+
input,
|
|
1278
|
+
result: { handoffTo: target.id },
|
|
1279
|
+
durationMs: Date.now() - started
|
|
1280
|
+
});
|
|
1281
|
+
await this.performHandoff(target, directive.reason);
|
|
1282
|
+
return;
|
|
1283
|
+
}
|
|
1284
|
+
if (outcome.ok) {
|
|
1285
|
+
this.deliverToolResult(call.id, outcome.result ?? { ok: true });
|
|
1286
|
+
this.emit("tool.completed", {
|
|
1287
|
+
...baseInfo,
|
|
1288
|
+
strategy: tool.strategy,
|
|
1289
|
+
input,
|
|
1290
|
+
result: outcome.result,
|
|
1291
|
+
durationMs: Date.now() - started
|
|
1292
|
+
});
|
|
1293
|
+
this.saveSnapshot();
|
|
1294
|
+
} else {
|
|
1295
|
+
this.deliverToolResult(call.id, outcome.payload);
|
|
1296
|
+
this.emit("tool.failed", {
|
|
1297
|
+
...baseInfo,
|
|
1298
|
+
strategy: tool.strategy,
|
|
1299
|
+
error: outcome.message,
|
|
1300
|
+
durationMs: Date.now() - started
|
|
1301
|
+
});
|
|
1302
|
+
}
|
|
1303
|
+
} finally {
|
|
1304
|
+
this.bgAudio.release(call.id);
|
|
1305
|
+
}
|
|
1306
|
+
}
|
|
1307
|
+
/** dispatch/deferred body — no hold audio (the model keeps talking). */
|
|
1308
|
+
async runDetachedTool(tool, call, input, _started, onOutcome) {
|
|
1309
|
+
onOutcome(await this.executeToolBody(tool, call, input));
|
|
1310
|
+
}
|
|
1311
|
+
/** Shared execution core: abort/timeout, middleware onion, per-tool hooks. */
|
|
1312
|
+
async executeToolBody(tool, call, initialInput) {
|
|
1313
|
+
const controller = new AbortController();
|
|
1314
|
+
this.runningTools.set(call.id, controller);
|
|
1315
|
+
const timeoutMs = tool.timeoutMs ?? 15e3;
|
|
1316
|
+
const timeout = setTimeout(() => controller.abort(/* @__PURE__ */ new Error("tool timed out")), timeoutMs);
|
|
1317
|
+
this.timers.add(timeout);
|
|
1318
|
+
const ctx = this.buildToolContext(call.id, controller.signal);
|
|
1319
|
+
try {
|
|
1320
|
+
let input = initialInput;
|
|
1321
|
+
const innermost = async () => {
|
|
1322
|
+
if (tool.onBeforeExecute) {
|
|
1323
|
+
const replaced = await tool.onBeforeExecute(input, ctx);
|
|
1324
|
+
if (replaced !== void 0) input = replaced;
|
|
1325
|
+
}
|
|
1326
|
+
let result = await this.raceAbort(Promise.resolve(tool.execute(input, ctx)), controller.signal);
|
|
1327
|
+
if (tool.onAfterExecute) {
|
|
1328
|
+
const replaced = await tool.onAfterExecute(result, ctx);
|
|
1329
|
+
if (replaced !== void 0) result = replaced;
|
|
1330
|
+
}
|
|
1331
|
+
return result;
|
|
1332
|
+
};
|
|
1333
|
+
return {
|
|
1334
|
+
ok: true,
|
|
1335
|
+
result: await composeExecution(tool, input, ctx, this.middlewares, innermost)
|
|
1336
|
+
};
|
|
1337
|
+
} catch (error) {
|
|
1338
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1339
|
+
let payload = {
|
|
1340
|
+
error: "tool_failed",
|
|
1341
|
+
message
|
|
1342
|
+
};
|
|
1343
|
+
if (tool.onError) try {
|
|
1344
|
+
const replaced = await tool.onError(error, ctx);
|
|
1345
|
+
if (replaced !== void 0) payload = replaced;
|
|
1346
|
+
} catch {}
|
|
1347
|
+
return {
|
|
1348
|
+
ok: false,
|
|
1349
|
+
payload,
|
|
1350
|
+
message
|
|
1351
|
+
};
|
|
1352
|
+
} finally {
|
|
1353
|
+
clearTimeout(timeout);
|
|
1354
|
+
this.timers.delete(timeout);
|
|
1355
|
+
this.runningTools.delete(call.id);
|
|
1356
|
+
}
|
|
1357
|
+
}
|
|
1358
|
+
/** Inject a deferred result as a conversation turn (idempotent per call). */
|
|
1359
|
+
completeDeferred(callId, toolName, result) {
|
|
1360
|
+
if (!this.deferredPending.delete(callId)) return;
|
|
1361
|
+
if (this.stateValue === "ended") return;
|
|
1362
|
+
const text = `[Tool "${toolName}" finished] Result: ${safeJsonStringify(result)}. Share what is relevant with the caller now.`;
|
|
1363
|
+
this.injectOrQueueText(text, true);
|
|
1364
|
+
}
|
|
1365
|
+
/** Complete a deferred tool from the host (webhook, operator console…). */
|
|
1366
|
+
completeDeferredFromHost(toolCallId, result) {
|
|
1367
|
+
const pending = this.deferredPending.get(toolCallId);
|
|
1368
|
+
if (!pending) {
|
|
1369
|
+
this.log.warn("submitToolResult for unknown/settled tool call", { toolCallId });
|
|
1370
|
+
return;
|
|
1371
|
+
}
|
|
1372
|
+
this.completeDeferred(toolCallId, pending.toolName, result);
|
|
1373
|
+
}
|
|
1374
|
+
requestApproval(tool, call, input) {
|
|
1375
|
+
const timeoutMs = tool.approvalTimeoutMs ?? 3e4;
|
|
1376
|
+
const timer = setTimeout(() => {
|
|
1377
|
+
this.rejectTool(call.id, "approval timed out");
|
|
1378
|
+
}, timeoutMs);
|
|
1379
|
+
timer.unref?.();
|
|
1380
|
+
this.timers.add(timer);
|
|
1381
|
+
this.approvals.set(call.id, {
|
|
1382
|
+
call,
|
|
1383
|
+
tool,
|
|
1384
|
+
input,
|
|
1385
|
+
timer
|
|
1386
|
+
});
|
|
1387
|
+
this.acquireHoldAudio(call.id, tool);
|
|
1388
|
+
this.emit("tool.approval.required", {
|
|
1389
|
+
approvalId: call.id,
|
|
1390
|
+
toolCallId: call.id,
|
|
1391
|
+
toolName: call.name,
|
|
1392
|
+
input,
|
|
1393
|
+
agentId: this.activeAgentValue.id,
|
|
1394
|
+
expiresAtMs: Date.now() + timeoutMs
|
|
1395
|
+
});
|
|
1396
|
+
}
|
|
1397
|
+
/** Approve a pending humanInTheLoop tool call (optionally editing input). */
|
|
1398
|
+
approveTool(approvalId, editedInput) {
|
|
1399
|
+
const entry = this.approvals.get(approvalId);
|
|
1400
|
+
if (!entry) {
|
|
1401
|
+
this.log.warn("approveTool for unknown approval", { approvalId });
|
|
1402
|
+
return;
|
|
1403
|
+
}
|
|
1404
|
+
this.approvals.delete(approvalId);
|
|
1405
|
+
clearTimeout(entry.timer);
|
|
1406
|
+
this.timers.delete(entry.timer);
|
|
1407
|
+
this.runForegroundTool(entry.tool, entry.call, editedInput !== void 0 ? editedInput : entry.input, Date.now());
|
|
1408
|
+
}
|
|
1409
|
+
/** Reject a pending humanInTheLoop tool call. */
|
|
1410
|
+
rejectTool(approvalId, reason) {
|
|
1411
|
+
const entry = this.approvals.get(approvalId);
|
|
1412
|
+
if (!entry) return;
|
|
1413
|
+
this.approvals.delete(approvalId);
|
|
1414
|
+
clearTimeout(entry.timer);
|
|
1415
|
+
this.timers.delete(entry.timer);
|
|
1416
|
+
this.bgAudio.release(approvalId);
|
|
1417
|
+
this.deliverToolResult(approvalId, {
|
|
1418
|
+
error: "rejected",
|
|
1419
|
+
message: reason ?? "A human operator declined this action. Tell the caller it cannot be done right now."
|
|
1420
|
+
});
|
|
1421
|
+
this.emit("tool.failed", {
|
|
1422
|
+
toolCallId: approvalId,
|
|
1423
|
+
toolName: entry.call.name,
|
|
1424
|
+
strategy: entry.tool.strategy,
|
|
1425
|
+
agentId: this.activeAgentValue.id,
|
|
1426
|
+
error: reason ?? "rejected by operator"
|
|
1427
|
+
});
|
|
1428
|
+
}
|
|
1429
|
+
acquireHoldAudio(holderId, tool) {
|
|
1430
|
+
if (tool.backgroundAudio === false) return;
|
|
1431
|
+
const sessionDefault = this.deps.options.toolBackgroundAudio;
|
|
1432
|
+
const spec = tool.backgroundAudio ?? sessionDefault?.spec;
|
|
1433
|
+
if (!spec) return;
|
|
1434
|
+
this.bgAudio.acquire(holderId, spec, {
|
|
1435
|
+
volume: sessionDefault?.volume,
|
|
1436
|
+
fadeInMs: sessionDefault?.fadeInMs,
|
|
1437
|
+
fadeOutMs: sessionDefault?.fadeOutMs,
|
|
1438
|
+
startDelayMs: sessionDefault?.startDelayMs,
|
|
1439
|
+
maxDurationMs: sessionDefault?.maxDurationMs
|
|
1440
|
+
});
|
|
1441
|
+
}
|
|
1442
|
+
/** Inject a text turn now, or after the agent finishes speaking. */
|
|
1443
|
+
injectOrQueueText(text, triggerResponse) {
|
|
1444
|
+
if (this.generating || this.tracker.isPlaybackActive()) {
|
|
1445
|
+
this.pendingInjections.push({
|
|
1446
|
+
text,
|
|
1447
|
+
triggerResponse
|
|
1448
|
+
});
|
|
1449
|
+
return;
|
|
1450
|
+
}
|
|
1451
|
+
this.provider?.sendText(text, {
|
|
1452
|
+
role: "system",
|
|
1453
|
+
triggerResponse
|
|
1454
|
+
});
|
|
1455
|
+
}
|
|
1456
|
+
raceAbort(promise, signal) {
|
|
1457
|
+
if (signal.aborted) return Promise.reject(signal.reason ?? /* @__PURE__ */ new Error("aborted"));
|
|
1458
|
+
return new Promise((resolve, reject) => {
|
|
1459
|
+
const onAbort = () => reject(signal.reason ?? /* @__PURE__ */ new Error("aborted"));
|
|
1460
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
1461
|
+
promise.then((value) => {
|
|
1462
|
+
signal.removeEventListener("abort", onAbort);
|
|
1463
|
+
resolve(value);
|
|
1464
|
+
}, (error) => {
|
|
1465
|
+
signal.removeEventListener("abort", onAbort);
|
|
1466
|
+
reject(error);
|
|
1467
|
+
});
|
|
1468
|
+
});
|
|
1469
|
+
}
|
|
1470
|
+
deliverToolResult(callId, payload, triggerResponse = true) {
|
|
1471
|
+
if (this.stateValue === "ended") return;
|
|
1472
|
+
if (this.deps.options.toolResultDelivery === "afterPlayback" && (this.generating || this.tracker.isPlaybackActive())) {
|
|
1473
|
+
this.toolQueue.enqueue({
|
|
1474
|
+
callId,
|
|
1475
|
+
toolName: "",
|
|
1476
|
+
payload,
|
|
1477
|
+
triggerResponse
|
|
1478
|
+
});
|
|
1479
|
+
return;
|
|
1480
|
+
}
|
|
1481
|
+
this.provider?.sendToolResult(callId, payload, { triggerResponse });
|
|
1482
|
+
}
|
|
1483
|
+
flushToolQueue() {
|
|
1484
|
+
if (this.stateValue === "ended" || !this.provider?.isConnected) return;
|
|
1485
|
+
const items = this.toolQueue.drain();
|
|
1486
|
+
const wantTrigger = items.some((item) => item.triggerResponse);
|
|
1487
|
+
for (const [index, item] of items.entries()) this.provider.sendToolResult(item.callId, item.payload, { triggerResponse: wantTrigger && index === items.length - 1 });
|
|
1488
|
+
const injections = this.pendingInjections;
|
|
1489
|
+
this.pendingInjections = [];
|
|
1490
|
+
for (const injection of injections) this.provider.sendText(injection.text, {
|
|
1491
|
+
role: "system",
|
|
1492
|
+
triggerResponse: injection.triggerResponse
|
|
1493
|
+
});
|
|
1494
|
+
}
|
|
1495
|
+
buildToolContext(toolCallId, signal) {
|
|
1496
|
+
const session = {
|
|
1497
|
+
callSid: this.callSid,
|
|
1498
|
+
sendText: (text, options) => this.sendText(text, options),
|
|
1499
|
+
finishCall: (options) => this.finishCall(options),
|
|
1500
|
+
transferTo: (phoneNumber, options) => this.transferTo(phoneNumber, options),
|
|
1501
|
+
handoffTo: (agent) => this.handoffTo(agent),
|
|
1502
|
+
playBackgroundAudio: (spec, options) => this.playBackgroundAudio(spec, options),
|
|
1503
|
+
stopBackgroundAudio: (options) => this.stopBackgroundAudio(options),
|
|
1504
|
+
submitToolResult: (id, result) => this.submitToolResult(id, result)
|
|
1505
|
+
};
|
|
1506
|
+
return {
|
|
1507
|
+
callSid: this.callSid,
|
|
1508
|
+
agent: this.activeAgentValue,
|
|
1509
|
+
session,
|
|
1510
|
+
context: this.context,
|
|
1511
|
+
callInfo: this.callInfo,
|
|
1512
|
+
logger: this.log,
|
|
1513
|
+
signal,
|
|
1514
|
+
toolCallId
|
|
1515
|
+
};
|
|
1516
|
+
}
|
|
1517
|
+
maybeGreet() {
|
|
1518
|
+
if (this.greeted) return;
|
|
1519
|
+
if (this.pregreeting) {
|
|
1520
|
+
this.greeted = true;
|
|
1521
|
+
return;
|
|
1522
|
+
}
|
|
1523
|
+
if (this.deps.options.greeting.mode !== "agent-initiates") return;
|
|
1524
|
+
if (!this.provider?.isConnected) return;
|
|
1525
|
+
if (this.callInfo.direction === "outbound" && !this.answered) return;
|
|
1526
|
+
this.greeted = true;
|
|
1527
|
+
this.provider.createResponse(this.deps.options.greeting.instructions ? { instructions: this.deps.options.greeting.instructions } : {});
|
|
1528
|
+
}
|
|
1529
|
+
armHangup() {
|
|
1530
|
+
if (this.pendingHangup) return new Promise((resolve) => this.pendingHangup.resolvers.push(resolve));
|
|
1531
|
+
this.pendingHangup = {
|
|
1532
|
+
resolvers: [],
|
|
1533
|
+
watchdog: this.armHangupWatchdog(),
|
|
1534
|
+
sawResponse: false,
|
|
1535
|
+
progress: 0,
|
|
1536
|
+
progressAtCheck: 0
|
|
1537
|
+
};
|
|
1538
|
+
return new Promise((resolve) => this.pendingHangup.resolvers.push(resolve));
|
|
1539
|
+
}
|
|
1540
|
+
/** Evidence the goodbye is alive (generating or playing) — feeds the watchdog. */
|
|
1541
|
+
noteHangupProgress() {
|
|
1542
|
+
if (this.pendingHangup) this.pendingHangup.progress++;
|
|
1543
|
+
}
|
|
1544
|
+
/**
|
|
1545
|
+
* Watchdog: forces completion only after a full quiet window — no deltas, no
|
|
1546
|
+
* response start, no mark echoes for `markTimeoutMs`. Evidence re-arms it, so
|
|
1547
|
+
* a slow or long goodbye is NEVER truncated mid-playout (wall-clock must not
|
|
1548
|
+
* override live mark evidence); a dead socket or a model that never says
|
|
1549
|
+
* goodbye still completes within one or two quiet windows.
|
|
1550
|
+
*/
|
|
1551
|
+
armHangupWatchdog() {
|
|
1552
|
+
const timer = setTimeout(() => {
|
|
1553
|
+
this.timers.delete(timer);
|
|
1554
|
+
const pending = this.pendingHangup;
|
|
1555
|
+
if (!pending) return;
|
|
1556
|
+
if (pending.progress !== pending.progressAtCheck) {
|
|
1557
|
+
pending.progressAtCheck = pending.progress;
|
|
1558
|
+
pending.watchdog = this.armHangupWatchdog();
|
|
1559
|
+
return;
|
|
1560
|
+
}
|
|
1561
|
+
this.log.warn("hangup watchdog fired — no goodbye progress for a full window");
|
|
1562
|
+
this.completeHangup();
|
|
1563
|
+
}, this.deps.options.hangup.markTimeoutMs);
|
|
1564
|
+
timer.unref?.();
|
|
1565
|
+
this.timers.add(timer);
|
|
1566
|
+
return timer;
|
|
1567
|
+
}
|
|
1568
|
+
maybeCompleteHangup() {
|
|
1569
|
+
if (!this.pendingHangup) return;
|
|
1570
|
+
if (!this.pendingHangup.sawResponse) return;
|
|
1571
|
+
if (this.generating || this.tracker.isPlaybackActive()) return;
|
|
1572
|
+
this.completeHangup();
|
|
1573
|
+
}
|
|
1574
|
+
completeHangup() {
|
|
1575
|
+
const pending = this.pendingHangup;
|
|
1576
|
+
if (pending) {
|
|
1577
|
+
clearTimeout(pending.watchdog);
|
|
1578
|
+
this.timers.delete(pending.watchdog);
|
|
1579
|
+
}
|
|
1580
|
+
this.pendingHangup = null;
|
|
1581
|
+
if (this.stateValue === "ended") {
|
|
1582
|
+
pending?.resolvers.forEach((resolve) => resolve());
|
|
1583
|
+
return;
|
|
1584
|
+
}
|
|
1585
|
+
this.stateValue = "ending";
|
|
1586
|
+
const rest = this.deps.rest;
|
|
1587
|
+
const reason = this.hangupReason;
|
|
1588
|
+
const finish = () => void this.teardown(reason).then(() => {
|
|
1589
|
+
pending?.resolvers.forEach((resolve) => resolve());
|
|
1590
|
+
});
|
|
1591
|
+
if (rest) rest.completeCall(this.callSid).catch((error) => this.log.warn("REST hangup failed; closing stream only", { error: String(error) })).finally(finish);
|
|
1592
|
+
else finish();
|
|
1593
|
+
}
|
|
1594
|
+
async executePendingTransfer() {
|
|
1595
|
+
const transfer = this.pendingTransfer;
|
|
1596
|
+
if (!transfer || !this.deps.rest) return;
|
|
1597
|
+
if (this.generating || this.tracker.isPlaybackActive()) return;
|
|
1598
|
+
if (this.stateValue !== "active") return;
|
|
1599
|
+
this.pendingTransfer = null;
|
|
1600
|
+
this.stateValue = "ending";
|
|
1601
|
+
try {
|
|
1602
|
+
await this.deps.rest.transferCall(this.callSid, transfer.phoneNumber, { callerId: transfer.callerId });
|
|
1603
|
+
await this.teardown("transferred");
|
|
1604
|
+
} catch (error) {
|
|
1605
|
+
this.stateValue = "active";
|
|
1606
|
+
this.emitError(error instanceof Error ? error : new Error(String(error)));
|
|
1607
|
+
this.log.error("transfer failed; call stays with the agent", { error: String(error) });
|
|
1608
|
+
}
|
|
1609
|
+
}
|
|
1610
|
+
scheduleReconnect() {
|
|
1611
|
+
const policy = this.deps.options.reconnect;
|
|
1612
|
+
this.reconnectAttempt++;
|
|
1613
|
+
if (this.reconnectAttempt > policy.maxAttempts) {
|
|
1614
|
+
this.fail(/* @__PURE__ */ new Error(`provider reconnect exhausted after ${policy.maxAttempts} attempts`), "provider-failed");
|
|
1615
|
+
return;
|
|
1616
|
+
}
|
|
1617
|
+
this.reconnecting = true;
|
|
1618
|
+
const delayMs = delayForAttempt(policy, this.reconnectAttempt);
|
|
1619
|
+
this.emit("provider.reconnecting", {
|
|
1620
|
+
attempt: this.reconnectAttempt,
|
|
1621
|
+
delayMs
|
|
1622
|
+
});
|
|
1623
|
+
const timer = setTimeout(() => {
|
|
1624
|
+
this.timers.delete(timer);
|
|
1625
|
+
this.reconnectProvider();
|
|
1626
|
+
}, delayMs);
|
|
1627
|
+
this.timers.add(timer);
|
|
1628
|
+
}
|
|
1629
|
+
async reconnectProvider() {
|
|
1630
|
+
if (this.stateValue !== "active") return;
|
|
1631
|
+
const provider = this.provider;
|
|
1632
|
+
if (!provider) return;
|
|
1633
|
+
try {
|
|
1634
|
+
await provider.connect(this.buildProviderInit());
|
|
1635
|
+
} catch (error) {
|
|
1636
|
+
this.log.warn("reconnect attempt failed", {
|
|
1637
|
+
attempt: this.reconnectAttempt,
|
|
1638
|
+
error: String(error)
|
|
1639
|
+
});
|
|
1640
|
+
this.scheduleReconnect();
|
|
1641
|
+
return;
|
|
1642
|
+
}
|
|
1643
|
+
if (this.stateValue !== "active") return;
|
|
1644
|
+
this.reconnectAttempt = 0;
|
|
1645
|
+
this.reconnecting = false;
|
|
1646
|
+
if (!provider.didResume) this.reinjectHistory(provider);
|
|
1647
|
+
this.flushInboundBuffer();
|
|
1648
|
+
this.emit("provider.reconnected");
|
|
1649
|
+
}
|
|
1650
|
+
/** After a reconnect the provider session is blank — restore conversational context. */
|
|
1651
|
+
reinjectHistory(provider) {
|
|
1652
|
+
if (this.transcriptEntries.length === 0) return;
|
|
1653
|
+
const summary = formatTranscriptForInjection(this.transcriptEntries);
|
|
1654
|
+
provider.sendText(`Context: this phone call reconnected mid-conversation. Transcript so far:\n${summary}\nContinue naturally from where it left off; do not greet again.`, {
|
|
1655
|
+
role: "system",
|
|
1656
|
+
triggerResponse: false
|
|
1657
|
+
});
|
|
1658
|
+
}
|
|
1659
|
+
async performHandoff(target, reason) {
|
|
1660
|
+
if (this.stateValue !== "active" || !this.provider) return;
|
|
1661
|
+
if (target.id === this.activeAgentValue.id) return;
|
|
1662
|
+
const from = this.activeAgentValue;
|
|
1663
|
+
this.activeAgentValue = target;
|
|
1664
|
+
this.toolset = this.buildToolset();
|
|
1665
|
+
this.handoffHistory.push({
|
|
1666
|
+
from: from.id,
|
|
1667
|
+
to: target.id,
|
|
1668
|
+
atMs: Date.now() - this.startedAtMs
|
|
1669
|
+
});
|
|
1670
|
+
const continueInstruction = `You are now ${target.name}. Continue the SAME phone conversation naturally — acknowledge the caller and take over; do not restart with a cold greeting.` + (reason ? ` Transfer context: ${reason}.` : "");
|
|
1671
|
+
const voiceChanges = target.voice !== void 0 && target.voice !== from.voice;
|
|
1672
|
+
if (!(!this.provider.capabilities.sessionUpdate || voiceChanges && !this.provider.capabilities.voiceChangeMidSession && this.deps.options.handoffVoicePolicy === "reconnect")) {
|
|
1673
|
+
if (voiceChanges && !this.provider.capabilities.voiceChangeMidSession) this.log.warn(`agent "${target.id}" declares voice "${target.voice}" but the provider cannot change voice mid-session — keeping the current voice (set handoffVoicePolicy: 'reconnect' to switch)`);
|
|
1674
|
+
await this.provider.updateSession({
|
|
1675
|
+
instructions: this.activeAgentValue.resolveInstructions(this.context),
|
|
1676
|
+
tools: this.buildProviderInit().tools,
|
|
1677
|
+
providerOptions: target.providerOptions
|
|
1678
|
+
});
|
|
1679
|
+
this.provider.createResponse({ instructions: continueInstruction });
|
|
1680
|
+
} else {
|
|
1681
|
+
this.handoffInProgress = true;
|
|
1682
|
+
this.reconnecting = true;
|
|
1683
|
+
const hold = this.deps.options.handoffHold;
|
|
1684
|
+
if (hold) this.bgAudio.acquire("__handoff__", hold.spec, {
|
|
1685
|
+
...hold,
|
|
1686
|
+
startDelayMs: hold.startDelayMs ?? 300
|
|
1687
|
+
});
|
|
1688
|
+
try {
|
|
1689
|
+
await this.provider.close();
|
|
1690
|
+
await this.provider.connect({
|
|
1691
|
+
...this.buildProviderInit(),
|
|
1692
|
+
freshSession: true
|
|
1693
|
+
});
|
|
1694
|
+
this.reinjectHistory(this.provider);
|
|
1695
|
+
this.provider.createResponse({ instructions: continueInstruction });
|
|
1696
|
+
} catch (error) {
|
|
1697
|
+
this.handoffInProgress = false;
|
|
1698
|
+
this.reconnecting = false;
|
|
1699
|
+
this.fail(error instanceof Error ? error : new Error(String(error)), "provider-failed");
|
|
1700
|
+
return;
|
|
1701
|
+
} finally {
|
|
1702
|
+
this.bgAudio.release("__handoff__", { immediate: false });
|
|
1703
|
+
}
|
|
1704
|
+
this.handoffInProgress = false;
|
|
1705
|
+
this.reconnecting = false;
|
|
1706
|
+
this.flushInboundBuffer();
|
|
1707
|
+
}
|
|
1708
|
+
this.emit("agent.handoff", {
|
|
1709
|
+
from,
|
|
1710
|
+
to: target,
|
|
1711
|
+
reason
|
|
1712
|
+
});
|
|
1713
|
+
this.saveSnapshot();
|
|
1714
|
+
}
|
|
1715
|
+
/**
|
|
1716
|
+
* Burst-write a stored μ-law greeting straight onto the Twilio socket —
|
|
1717
|
+
* no pacing loop (Twilio buffers and plays at line rate), so playback
|
|
1718
|
+
* starts immediately while the provider session is still being built.
|
|
1719
|
+
*/
|
|
1720
|
+
playPreGreeting() {
|
|
1721
|
+
const preSynthesized = this.deps.options.greeting.preSynthesized;
|
|
1722
|
+
if (!preSynthesized || !this.deps.transport.isOpen) return;
|
|
1723
|
+
let audio;
|
|
1724
|
+
try {
|
|
1725
|
+
audio = Buffer.isBuffer(preSynthesized.audio) ? preSynthesized.audio : readFileSync(preSynthesized.audio);
|
|
1726
|
+
} catch (error) {
|
|
1727
|
+
this.log.warn("pre-synthesized greeting unavailable — falling back to model greeting", { error: String(error) });
|
|
1728
|
+
return;
|
|
1729
|
+
}
|
|
1730
|
+
if (audio.length === 0) return;
|
|
1731
|
+
const durationMs = mulawBytesToMs(audio.length);
|
|
1732
|
+
this.pregreeting = {
|
|
1733
|
+
text: preSynthesized.text,
|
|
1734
|
+
durationMs,
|
|
1735
|
+
played: false
|
|
1736
|
+
};
|
|
1737
|
+
for (let offset = 0; offset < audio.length; offset += PREGREETING_CHUNK_BYTES) {
|
|
1738
|
+
const chunk = audio.subarray(offset, Math.min(offset + PREGREETING_CHUNK_BYTES, audio.length));
|
|
1739
|
+
this.deps.transport.sendMedia(chunk.toString("base64"));
|
|
1740
|
+
}
|
|
1741
|
+
this.deps.transport.sendMark(PREGREETING_MARK);
|
|
1742
|
+
const entry = {
|
|
1743
|
+
role: "agent",
|
|
1744
|
+
text: preSynthesized.text,
|
|
1745
|
+
timestampMs: 0,
|
|
1746
|
+
agentId: this.activeAgentValue.id
|
|
1747
|
+
};
|
|
1748
|
+
this.transcriptEntries.push(entry);
|
|
1749
|
+
this.emit("transcript.agent", entry);
|
|
1750
|
+
}
|
|
1751
|
+
/** Armed whenever the agent goes quiet and we're waiting on the caller. */
|
|
1752
|
+
armIdleTimer() {
|
|
1753
|
+
const idle = this.deps.options.idle;
|
|
1754
|
+
if (!idle || this.stateValue !== "active" || this.pendingHangup) return;
|
|
1755
|
+
this.clearIdleTimer();
|
|
1756
|
+
this.idleTimer = setTimeout(() => {
|
|
1757
|
+
this.idleTimer = null;
|
|
1758
|
+
this.onIdleTimeout();
|
|
1759
|
+
}, idle.timeoutSeconds * 1e3);
|
|
1760
|
+
this.idleTimer.unref?.();
|
|
1761
|
+
}
|
|
1762
|
+
clearIdleTimer() {
|
|
1763
|
+
if (this.idleTimer) {
|
|
1764
|
+
clearTimeout(this.idleTimer);
|
|
1765
|
+
this.idleTimer = null;
|
|
1766
|
+
}
|
|
1767
|
+
}
|
|
1768
|
+
onIdleTimeout() {
|
|
1769
|
+
const idle = this.deps.options.idle;
|
|
1770
|
+
if (!idle || this.stateValue !== "active" || !this.provider?.isConnected) return;
|
|
1771
|
+
if (this.generating || this.tracker.isPlaybackActive() || this.runningTools.size > 0) {
|
|
1772
|
+
this.armIdleTimer();
|
|
1773
|
+
return;
|
|
1774
|
+
}
|
|
1775
|
+
const prompts = idle.prompts?.length ? idle.prompts : ["The caller has gone quiet. Gently check if they are still there."];
|
|
1776
|
+
const maxNudges = idle.maxNudges ?? prompts.length;
|
|
1777
|
+
if (this.nudgeCount < maxNudges) {
|
|
1778
|
+
const prompt = prompts[Math.min(this.nudgeCount, prompts.length - 1)];
|
|
1779
|
+
this.nudgeCount++;
|
|
1780
|
+
this.provider.createResponse({ instructions: prompt });
|
|
1781
|
+
return;
|
|
1782
|
+
}
|
|
1783
|
+
this.log.info("caller idle beyond nudges — ending call");
|
|
1784
|
+
this.hangupReason = "idle-timeout";
|
|
1785
|
+
this.provider.createResponse({ instructions: idle.goodbye ?? "You could not hear the caller anymore. Say a brief goodbye and that they are welcome to call back, then stop speaking." });
|
|
1786
|
+
this.armHangup();
|
|
1787
|
+
}
|
|
1788
|
+
startMaxDurationWatchdog() {
|
|
1789
|
+
const seconds = this.deps.options.maxCallDurationSeconds;
|
|
1790
|
+
if (!seconds) return;
|
|
1791
|
+
const timer = setTimeout(() => {
|
|
1792
|
+
this.timers.delete(timer);
|
|
1793
|
+
this.log.info("max call duration reached");
|
|
1794
|
+
this.teardown("max-duration");
|
|
1795
|
+
}, seconds * 1e3);
|
|
1796
|
+
timer.unref?.();
|
|
1797
|
+
this.timers.add(timer);
|
|
1798
|
+
}
|
|
1799
|
+
fail(error, reason) {
|
|
1800
|
+
this.emit("call.failed", error);
|
|
1801
|
+
this.teardown(reason);
|
|
1802
|
+
}
|
|
1803
|
+
async teardown(reason) {
|
|
1804
|
+
if (this.stateValue === "ended") return;
|
|
1805
|
+
this.stateValue = "ended";
|
|
1806
|
+
this.endedReason = reason;
|
|
1807
|
+
for (const timer of this.timers) clearTimeout(timer);
|
|
1808
|
+
this.timers.clear();
|
|
1809
|
+
this.clearIdleTimer();
|
|
1810
|
+
this.bgAudio.stop({ immediate: true });
|
|
1811
|
+
for (const controller of this.runningTools.values()) controller.abort(/* @__PURE__ */ new Error("call ended"));
|
|
1812
|
+
this.runningTools.clear();
|
|
1813
|
+
for (const approval of this.approvals.values()) clearTimeout(approval.timer);
|
|
1814
|
+
this.approvals.clear();
|
|
1815
|
+
this.deferredPending.clear();
|
|
1816
|
+
this.pendingInjections = [];
|
|
1817
|
+
this.toolQueue.clear();
|
|
1818
|
+
this.inboundBuffer = [];
|
|
1819
|
+
const pending = this.pendingHangup;
|
|
1820
|
+
this.pendingHangup = null;
|
|
1821
|
+
if (pending) {
|
|
1822
|
+
clearTimeout(pending.watchdog);
|
|
1823
|
+
pending.resolvers.forEach((resolve) => resolve());
|
|
1824
|
+
}
|
|
1825
|
+
const provider = this.provider;
|
|
1826
|
+
this.provider = null;
|
|
1827
|
+
if (provider) provider.removeAllListeners();
|
|
1828
|
+
this.deps.transport.close();
|
|
1829
|
+
const durationMs = Date.now() - this.startedAtMs;
|
|
1830
|
+
this.emit("call.ended", {
|
|
1831
|
+
reason,
|
|
1832
|
+
durationMs,
|
|
1833
|
+
usage: this.usage
|
|
1834
|
+
});
|
|
1835
|
+
if (provider) await provider.close().catch(() => {});
|
|
1836
|
+
await this.saveSnapshot();
|
|
1837
|
+
this.deps.onEnded?.(this.callSid, reason);
|
|
1838
|
+
}
|
|
1839
|
+
async saveSnapshot() {
|
|
1840
|
+
const snapshot = {
|
|
1841
|
+
callSid: this.callSid,
|
|
1842
|
+
streamSid: this.streamSid,
|
|
1843
|
+
state: this.stateValue,
|
|
1844
|
+
activeAgentId: this.activeAgentValue.id,
|
|
1845
|
+
transcript: [...this.transcriptEntries],
|
|
1846
|
+
usage: this.usage,
|
|
1847
|
+
context: this.context.toJSON(),
|
|
1848
|
+
handoffHistory: [...this.handoffHistory],
|
|
1849
|
+
startedAtMs: this.startedAtMs,
|
|
1850
|
+
...this.stateValue === "ended" ? {
|
|
1851
|
+
endedAtMs: Date.now(),
|
|
1852
|
+
endReason: this.endedReason ?? void 0
|
|
1853
|
+
} : {}
|
|
1854
|
+
};
|
|
1855
|
+
try {
|
|
1856
|
+
await this.deps.store.save(snapshot);
|
|
1857
|
+
} catch (error) {
|
|
1858
|
+
this.log.warn("session store save failed", { error: String(error) });
|
|
1859
|
+
}
|
|
1860
|
+
}
|
|
1861
|
+
};
|
|
1862
|
+
const PREGREETING_MARK = "pre:greeting";
|
|
1863
|
+
/** 400ms per frame — matches production burst-write implementations. */
|
|
1864
|
+
const PREGREETING_CHUNK_BYTES = 3200;
|
|
1865
|
+
function safeJsonStringify(value) {
|
|
1866
|
+
try {
|
|
1867
|
+
return JSON.stringify(value) ?? "null";
|
|
1868
|
+
} catch {
|
|
1869
|
+
return String(value);
|
|
1870
|
+
}
|
|
1871
|
+
}
|
|
1872
|
+
//#endregion
|
|
1873
|
+
//#region src/bridge/config.ts
|
|
1874
|
+
const DEFAULT_SESSION_OPTIONS = {
|
|
1875
|
+
greeting: { mode: "agent-initiates" },
|
|
1876
|
+
interruptions: { enabled: true },
|
|
1877
|
+
deafness: {
|
|
1878
|
+
ignoreUserAudioUntilFirstTurnDone: true,
|
|
1879
|
+
muteDuringToolExecution: true,
|
|
1880
|
+
muteWhileAgentSpeaking: false
|
|
1881
|
+
},
|
|
1882
|
+
reconnect: DEFAULT_RECONNECT_POLICY,
|
|
1883
|
+
hangup: { markTimeoutMs: 7e3 },
|
|
1884
|
+
toolResultDelivery: "afterPlayback"
|
|
1885
|
+
};
|
|
1886
|
+
function resolveSessionOptions(partial) {
|
|
1887
|
+
const resolved = deepMerge(structuredClone(DEFAULT_SESSION_OPTIONS), partial);
|
|
1888
|
+
if (resolved.greeting.mode === "user-initiates" && partial?.deafness?.ignoreUserAudioUntilFirstTurnDone === void 0) resolved.deafness.ignoreUserAudioUntilFirstTurnDone = false;
|
|
1889
|
+
return resolved;
|
|
1890
|
+
}
|
|
1891
|
+
//#endregion
|
|
1892
|
+
//#region src/bridge/TwilioRealtimeBridge.ts
|
|
1893
|
+
const MAX_ANSWERED_EARLY_ENTRIES = 1e3;
|
|
1894
|
+
var TwilioRealtimeBridge = class extends TypedEmitter {
|
|
1895
|
+
config;
|
|
1896
|
+
logger;
|
|
1897
|
+
store;
|
|
1898
|
+
rest;
|
|
1899
|
+
sessionsBySid = /* @__PURE__ */ new Map();
|
|
1900
|
+
/** Status callbacks can beat the media stream; remember early answers. */
|
|
1901
|
+
answeredEarly = /* @__PURE__ */ new Set();
|
|
1902
|
+
middlewares = [];
|
|
1903
|
+
closed = false;
|
|
1904
|
+
constructor(config) {
|
|
1905
|
+
super();
|
|
1906
|
+
this.config = config;
|
|
1907
|
+
this.logger = config.logger ?? noopLogger;
|
|
1908
|
+
this.store = config.store ?? new InMemorySessionStore();
|
|
1909
|
+
if (config.twilio) this.rest = new TwilioRestClient({
|
|
1910
|
+
accountSid: config.twilio.accountSid,
|
|
1911
|
+
authToken: config.twilio.authToken,
|
|
1912
|
+
callerId: config.twilio.callerId
|
|
1913
|
+
});
|
|
1914
|
+
}
|
|
1915
|
+
/** Attach one incoming Twilio Media Stream WebSocket. */
|
|
1916
|
+
handleConnection(ws, request) {
|
|
1917
|
+
if (this.closed) {
|
|
1918
|
+
try {
|
|
1919
|
+
ws.close(1013, "bridge closed");
|
|
1920
|
+
} catch {}
|
|
1921
|
+
return;
|
|
1922
|
+
}
|
|
1923
|
+
const transport = new TwilioMediaTransport(ws);
|
|
1924
|
+
this.handshake(transport, request);
|
|
1925
|
+
}
|
|
1926
|
+
/**
|
|
1927
|
+
* Register cross-cutting tool middleware (first registered wraps
|
|
1928
|
+
* outermost). Applies to sessions created after the call.
|
|
1929
|
+
*/
|
|
1930
|
+
use(middleware) {
|
|
1931
|
+
this.middlewares.push(middleware);
|
|
1932
|
+
return this;
|
|
1933
|
+
}
|
|
1934
|
+
getSession(callSid) {
|
|
1935
|
+
return this.sessionsBySid.get(callSid);
|
|
1936
|
+
}
|
|
1937
|
+
sessions() {
|
|
1938
|
+
return this.sessionsBySid;
|
|
1939
|
+
}
|
|
1940
|
+
/**
|
|
1941
|
+
* Feed the outbound `answered` signal from your Twilio status callback so
|
|
1942
|
+
* agent-initiated greetings fire when the human actually picks up.
|
|
1943
|
+
*/
|
|
1944
|
+
notifyAnswered(callSid) {
|
|
1945
|
+
const session = this.sessionsBySid.get(callSid);
|
|
1946
|
+
if (session) {
|
|
1947
|
+
session.notifyAnswered();
|
|
1948
|
+
return;
|
|
1949
|
+
}
|
|
1950
|
+
this.answeredEarly.add(callSid);
|
|
1951
|
+
if (this.answeredEarly.size > MAX_ANSWERED_EARLY_ENTRIES) {
|
|
1952
|
+
const oldest = this.answeredEarly.values().next().value;
|
|
1953
|
+
if (oldest !== void 0) this.answeredEarly.delete(oldest);
|
|
1954
|
+
}
|
|
1955
|
+
}
|
|
1956
|
+
/** Deterministically end every active session and stop accepting new ones. */
|
|
1957
|
+
async close() {
|
|
1958
|
+
this.closed = true;
|
|
1959
|
+
const active = [...this.sessionsBySid.values()];
|
|
1960
|
+
await Promise.allSettled(active.map((session) => session.end("bridge-closed")));
|
|
1961
|
+
this.sessionsBySid.clear();
|
|
1962
|
+
}
|
|
1963
|
+
async handshake(transport, request) {
|
|
1964
|
+
const handshakeOptions = typeof this.config.session === "object" ? this.config.session.handshake : void 0;
|
|
1965
|
+
let start;
|
|
1966
|
+
try {
|
|
1967
|
+
start = await transport.awaitStart({
|
|
1968
|
+
timeoutMs: handshakeOptions?.timeoutMs,
|
|
1969
|
+
maxPreStartMessages: handshakeOptions?.maxPreStartMessages
|
|
1970
|
+
});
|
|
1971
|
+
} catch (error) {
|
|
1972
|
+
this.logger.warn("media stream handshake failed", { error: String(error) });
|
|
1973
|
+
this.emit("connection.rejected", { reason: "no-start-frame" });
|
|
1974
|
+
transport.close(1008, "no start frame");
|
|
1975
|
+
return;
|
|
1976
|
+
}
|
|
1977
|
+
const callSid = start.start.callSid;
|
|
1978
|
+
try {
|
|
1979
|
+
if (this.config.validateConnection) {
|
|
1980
|
+
if (!await this.config.validateConnection(start, request)) {
|
|
1981
|
+
this.emit("connection.rejected", { reason: "unauthorized" });
|
|
1982
|
+
transport.close(1008, "unauthorized");
|
|
1983
|
+
return;
|
|
1984
|
+
}
|
|
1985
|
+
}
|
|
1986
|
+
if (this.sessionsBySid.has(callSid)) {
|
|
1987
|
+
this.emit("connection.rejected", { reason: "duplicate-call" });
|
|
1988
|
+
transport.close(1008, "duplicate stream for call");
|
|
1989
|
+
return;
|
|
1990
|
+
}
|
|
1991
|
+
const agent = typeof this.config.agent === "function" ? await this.config.agent(start) : this.config.agent;
|
|
1992
|
+
if (!(agent instanceof Agent)) throw new Error("BridgeConfig.agent must resolve to an Agent instance");
|
|
1993
|
+
const options = resolveSessionOptions(typeof this.config.session === "function" ? await this.config.session(start) : this.config.session);
|
|
1994
|
+
const answeredEarly = this.answeredEarly.delete(callSid);
|
|
1995
|
+
const session = new CallSession({
|
|
1996
|
+
transport,
|
|
1997
|
+
start,
|
|
1998
|
+
providerFactory: this.config.provider,
|
|
1999
|
+
agent,
|
|
2000
|
+
options,
|
|
2001
|
+
store: this.store,
|
|
2002
|
+
logger: this.logger,
|
|
2003
|
+
builtinTools: this.config.builtinTools,
|
|
2004
|
+
rest: this.rest,
|
|
2005
|
+
restCallerId: this.config.twilio?.callerId,
|
|
2006
|
+
middlewares: this.middlewares,
|
|
2007
|
+
answeredEarly,
|
|
2008
|
+
onEnded: (sid, reason) => this.onSessionEnded(sid, reason)
|
|
2009
|
+
});
|
|
2010
|
+
this.sessionsBySid.set(callSid, session);
|
|
2011
|
+
this.emit("session.started", session);
|
|
2012
|
+
await session.begin();
|
|
2013
|
+
} catch (error) {
|
|
2014
|
+
this.logger.error("failed to establish call session", {
|
|
2015
|
+
callSid,
|
|
2016
|
+
error: String(error)
|
|
2017
|
+
});
|
|
2018
|
+
this.emit("error", error instanceof Error ? error : new Error(String(error)));
|
|
2019
|
+
this.sessionsBySid.delete(callSid);
|
|
2020
|
+
transport.close(1011, "session setup failed");
|
|
2021
|
+
}
|
|
2022
|
+
}
|
|
2023
|
+
onSessionEnded(callSid, reason) {
|
|
2024
|
+
this.sessionsBySid.delete(callSid);
|
|
2025
|
+
this.emit("session.ended", {
|
|
2026
|
+
callSid,
|
|
2027
|
+
reason
|
|
2028
|
+
});
|
|
2029
|
+
}
|
|
2030
|
+
};
|
|
2031
|
+
//#endregion
|
|
2032
|
+
//#region src/greeting/capture.ts
|
|
2033
|
+
/**
|
|
2034
|
+
* Capture a pre-synthesized greeting from a throwaway realtime session.
|
|
2035
|
+
*
|
|
2036
|
+
* Why not a TTS API? The realtime voice sets don't fully overlap the TTS
|
|
2037
|
+
* voice sets — capturing from a real session guarantees the stored greeting
|
|
2038
|
+
* sounds exactly like the live agent. Run this at deploy/config time and
|
|
2039
|
+
* store the returned μ-law buffer (it is Twilio wire format; pass it to
|
|
2040
|
+
* `greeting.preSynthesized.audio`).
|
|
2041
|
+
*/
|
|
2042
|
+
async function captureGreetingAudio(options) {
|
|
2043
|
+
const url = `${options.baseUrl ?? "wss://api.openai.com/v1/realtime"}?model=${encodeURIComponent(options.model ?? "gpt-realtime")}`;
|
|
2044
|
+
const ws = new WebSocket$1(url, { headers: {
|
|
2045
|
+
Authorization: `Bearer ${options.apiKey}`,
|
|
2046
|
+
...options.headers
|
|
2047
|
+
} });
|
|
2048
|
+
const chunks = [];
|
|
2049
|
+
return new Promise((resolve, reject) => {
|
|
2050
|
+
const timeoutMs = options.timeoutMs ?? 3e4;
|
|
2051
|
+
const timer = setTimeout(() => {
|
|
2052
|
+
try {
|
|
2053
|
+
ws.close();
|
|
2054
|
+
} catch {}
|
|
2055
|
+
reject(/* @__PURE__ */ new Error(`greeting capture timed out after ${timeoutMs}ms`));
|
|
2056
|
+
}, timeoutMs);
|
|
2057
|
+
timer.unref?.();
|
|
2058
|
+
const finish = (error) => {
|
|
2059
|
+
clearTimeout(timer);
|
|
2060
|
+
try {
|
|
2061
|
+
ws.close();
|
|
2062
|
+
} catch {}
|
|
2063
|
+
if (error) {
|
|
2064
|
+
reject(error);
|
|
2065
|
+
return;
|
|
2066
|
+
}
|
|
2067
|
+
const audio = Buffer.concat(chunks);
|
|
2068
|
+
resolve({
|
|
2069
|
+
audio,
|
|
2070
|
+
text: options.text,
|
|
2071
|
+
durationMs: audio.length / 8
|
|
2072
|
+
});
|
|
2073
|
+
};
|
|
2074
|
+
ws.on("message", (raw) => {
|
|
2075
|
+
let event;
|
|
2076
|
+
try {
|
|
2077
|
+
event = JSON.parse(raw.toString());
|
|
2078
|
+
} catch {
|
|
2079
|
+
return;
|
|
2080
|
+
}
|
|
2081
|
+
switch (event.type) {
|
|
2082
|
+
case "session.created":
|
|
2083
|
+
ws.send(JSON.stringify(buildSessionUpdate({
|
|
2084
|
+
instructions: "You produce exact voice lines for a phone system.",
|
|
2085
|
+
voice: options.voice,
|
|
2086
|
+
vad: null,
|
|
2087
|
+
transcription: false
|
|
2088
|
+
}, { defaultVoice: options.voice })));
|
|
2089
|
+
break;
|
|
2090
|
+
case "session.updated":
|
|
2091
|
+
ws.send(JSON.stringify({
|
|
2092
|
+
type: "response.create",
|
|
2093
|
+
response: { instructions: `Say exactly this, and nothing else: "${options.text}"` }
|
|
2094
|
+
}));
|
|
2095
|
+
break;
|
|
2096
|
+
case "response.output_audio.delta":
|
|
2097
|
+
if (typeof event.delta === "string") chunks.push(Buffer.from(event.delta, "base64"));
|
|
2098
|
+
break;
|
|
2099
|
+
case "response.done":
|
|
2100
|
+
finish();
|
|
2101
|
+
break;
|
|
2102
|
+
case "error": finish(/* @__PURE__ */ new Error(`greeting capture failed: ${JSON.stringify(event.error ?? event)}`));
|
|
2103
|
+
}
|
|
2104
|
+
});
|
|
2105
|
+
ws.on("error", (error) => finish(error instanceof Error ? error : new Error(String(error))));
|
|
2106
|
+
ws.on("close", () => {
|
|
2107
|
+
if (chunks.length > 0) finish();
|
|
2108
|
+
else finish(/* @__PURE__ */ new Error("greeting capture connection closed before audio arrived"));
|
|
2109
|
+
});
|
|
2110
|
+
});
|
|
2111
|
+
}
|
|
2112
|
+
//#endregion
|
|
2113
|
+
export { Agent, BaseRealtimeProvider, CallSession, DEFAULT_RECONNECT_POLICY, DEFAULT_SESSION_OPTIONS, InMemorySessionStore, InterruptionController, PlaybackTracker, SessionContext, TwilioRealtimeBridge, captureGreetingAudio, collectAgentGraph, composeExecution, connectStreamTwiml, consoleLogger, createFinishCallTool, createHandoffTool, createTransferCallTool, decorateTool, emptyUsage, handoffToolName, isHandoffDirective, noopLogger, resolveSessionOptions, tool, zodToJsonSchema };
|