twilio-agent-connect 2.2.0 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -0
- package/dist/index.d.ts +2353 -619
- package/dist/index.js +3529 -746
- package/dist/index.js.map +1 -1
- package/package.json +6 -5
package/dist/index.js
CHANGED
|
@@ -3,9 +3,10 @@ export { z } from 'zod';
|
|
|
3
3
|
import pino from 'pino';
|
|
4
4
|
import axios, { AxiosError } from 'axios';
|
|
5
5
|
import axiosRetry from 'axios-retry';
|
|
6
|
+
import { Analytics } from '@segment/analytics-node';
|
|
7
|
+
import twilio from 'twilio';
|
|
6
8
|
import { WebSocket } from 'ws';
|
|
7
9
|
import VoiceResponse from 'twilio/lib/twiml/VoiceResponse.js';
|
|
8
|
-
import twilio from 'twilio';
|
|
9
10
|
import Fastify from 'fastify';
|
|
10
11
|
import formbody from '@fastify/formbody';
|
|
11
12
|
import websocket from '@fastify/websocket';
|
|
@@ -769,10 +770,16 @@ var InterruptMessageSchema = z.object({
|
|
|
769
770
|
utteranceUntilInterrupt: z.string().optional(),
|
|
770
771
|
durationUntilInterruptMs: z.number().int().nonnegative().optional()
|
|
771
772
|
});
|
|
773
|
+
var DtmfMessageSchema = z.object({
|
|
774
|
+
type: z.literal("dtmf"),
|
|
775
|
+
/** The key pressed: `0`-`9`, `*`, `#`, or `A`-`D`. */
|
|
776
|
+
digit: z.string()
|
|
777
|
+
});
|
|
772
778
|
var WebSocketMessageSchema = z.union([
|
|
773
779
|
SetupMessageSchema,
|
|
774
780
|
PromptMessageSchema,
|
|
775
|
-
InterruptMessageSchema
|
|
781
|
+
InterruptMessageSchema,
|
|
782
|
+
DtmfMessageSchema
|
|
776
783
|
]);
|
|
777
784
|
var TextTokenMessageSchema = z.object({
|
|
778
785
|
type: z.literal("text"),
|
|
@@ -799,37 +806,43 @@ var LanguageConfigSchema = z.object({
|
|
|
799
806
|
/** Speech model for STT. Choices vary by transcriptionProvider. */
|
|
800
807
|
speechModel: z.string().optional()
|
|
801
808
|
});
|
|
802
|
-
var
|
|
803
|
-
/** Custom parameters to pass to ConversationRelay as `<Parameter>` children */
|
|
804
|
-
customParameters: CustomParametersSchema.optional(),
|
|
805
|
-
/** Initial greeting message for the caller */
|
|
806
|
-
welcomeGreeting: z.string().optional(),
|
|
809
|
+
var VoiceTwiMLOptionsBase = z.object({
|
|
807
810
|
/**
|
|
808
|
-
*
|
|
809
|
-
*
|
|
811
|
+
* Custom parameters to pass to the provider's TwiML element as `<Parameter>`
|
|
812
|
+
* children.
|
|
810
813
|
*/
|
|
811
|
-
|
|
814
|
+
customParameters: CustomParametersSchema.optional(),
|
|
812
815
|
/**
|
|
813
816
|
* URL for Twilio to request when the call ends (`<Connect action>`). Set to
|
|
814
817
|
* a non-empty URL, or leave unset. An explicit `undefined` suppresses the
|
|
815
|
-
* action entirely (see
|
|
818
|
+
* action entirely (see the TwiML builder's actionUrl resolution); an empty
|
|
816
819
|
* string is rejected so it can't silently drop the action.
|
|
817
820
|
*/
|
|
818
821
|
actionUrl: z.string().min(1, "actionUrl must not be empty").optional(),
|
|
819
822
|
/**
|
|
820
|
-
*
|
|
821
|
-
*
|
|
822
|
-
*/
|
|
823
|
-
conversationConfiguration: z.string().optional(),
|
|
824
|
-
/**
|
|
825
|
-
* ConversationRelay WebSocket URL (the `<ConversationRelay url=...>`
|
|
826
|
-
* attribute). Leave unset (the default) to use the URL the channel derives
|
|
823
|
+
* WebSocket URL for the provider's TwiML element (`<ConversationRelay url=...>`
|
|
824
|
+
* today). Leave unset (the default) to use the URL the channel derives
|
|
827
825
|
* from `TACConfig.voicePublicDomain` + `voiceWebsocketPath`. Set it only for
|
|
828
826
|
* a per-call URL — e.g. an affinity-routed host that appends a token to the
|
|
829
827
|
* upgrade URL — typically from an `onInboundCallTwiml` customizer. Layers
|
|
830
828
|
* per-field like every other field. Must be non-empty when set.
|
|
831
829
|
*/
|
|
832
|
-
websocketUrl: z.string().min(1, "websocketUrl must not be empty").optional()
|
|
830
|
+
websocketUrl: z.string().min(1, "websocketUrl must not be empty").optional()
|
|
831
|
+
});
|
|
832
|
+
var VoiceTwiMLOptionsSchema = VoiceTwiMLOptionsBase.strict();
|
|
833
|
+
var VoiceTwiMLOptionsConversationRelaySchema = VoiceTwiMLOptionsBase.extend({
|
|
834
|
+
/** Initial greeting message for the caller */
|
|
835
|
+
welcomeGreeting: z.string().optional(),
|
|
836
|
+
/**
|
|
837
|
+
* What caller input can interrupt the welcome greeting.
|
|
838
|
+
* Defaults to 'any' on Twilio.
|
|
839
|
+
*/
|
|
840
|
+
welcomeGreetingInterruptible: InterruptModeSchema.optional(),
|
|
841
|
+
/**
|
|
842
|
+
* Conversation Service SID. When set, ConversationRelay will manage
|
|
843
|
+
* conversation creation and participants.
|
|
844
|
+
*/
|
|
845
|
+
conversationConfiguration: z.string().optional(),
|
|
833
846
|
// Language, TTS, STT
|
|
834
847
|
/**
|
|
835
848
|
* Language for both STT and TTS, e.g. 'en-US'. Equivalent to setting both
|
|
@@ -937,17 +950,19 @@ var TwiMLOptionsSchema = z.object({
|
|
|
937
950
|
extra: z.record(z.string(), z.union([z.string(), z.boolean(), z.number()])).optional()
|
|
938
951
|
}).strict().superRefine((value, ctx) => {
|
|
939
952
|
if (!value.extra) return;
|
|
940
|
-
const typed = new Set(
|
|
953
|
+
const typed = new Set(
|
|
954
|
+
Object.keys(VoiceTwiMLOptionsConversationRelayShape).filter((k) => k !== "extra")
|
|
955
|
+
);
|
|
941
956
|
const shadowed = Object.keys(value.extra).filter((k) => typed.has(k)).sort();
|
|
942
957
|
if (shadowed.length > 0) {
|
|
943
958
|
ctx.addIssue({
|
|
944
959
|
code: "custom",
|
|
945
960
|
path: ["extra"],
|
|
946
|
-
message: `
|
|
961
|
+
message: `VoiceTwiMLOptionsConversationRelay.extra keys [${shadowed.join(", ")}] shadow typed fields. Set the typed field directly instead of using \`extra\`.`
|
|
947
962
|
});
|
|
948
963
|
}
|
|
949
964
|
});
|
|
950
|
-
var
|
|
965
|
+
var VoiceTwiMLOptionsConversationRelayShape = {
|
|
951
966
|
customParameters: true,
|
|
952
967
|
welcomeGreeting: true,
|
|
953
968
|
welcomeGreetingInterruptible: true,
|
|
@@ -979,6 +994,28 @@ var TwiMLOptionsShape = {
|
|
|
979
994
|
languages: true,
|
|
980
995
|
extra: true
|
|
981
996
|
};
|
|
997
|
+
var VoiceTwiMLOptionsMediaStreamsSchema = VoiceTwiMLOptionsBase.extend({
|
|
998
|
+
/**
|
|
999
|
+
* Friendly name for the stream (`<Stream name=...>`). Must be unique per
|
|
1000
|
+
* call; it arrives back in the WebSocket `start` event.
|
|
1001
|
+
*/
|
|
1002
|
+
name: z.string().optional(),
|
|
1003
|
+
/**
|
|
1004
|
+
* Absolute URL Twilio posts to when the stream starts, stops, or errors
|
|
1005
|
+
* (StreamSid/StreamName/StreamEvent/StreamError/Timestamp params). Must be
|
|
1006
|
+
* non-empty when set, like the other URL fields, so an empty string can't
|
|
1007
|
+
* silently emit a broken attribute.
|
|
1008
|
+
*/
|
|
1009
|
+
statusCallback: z.string().min(1, "statusCallback must not be empty").optional(),
|
|
1010
|
+
/** HTTP method for `statusCallback`. Defaults to POST on Twilio. */
|
|
1011
|
+
statusCallbackMethod: z.enum(["GET", "POST"]).optional(),
|
|
1012
|
+
/**
|
|
1013
|
+
* HTTP method for `actionUrl`. Defaults to POST on Twilio. Lives here rather
|
|
1014
|
+
* than on the shared base because no other provider's TwiML exposes it.
|
|
1015
|
+
*/
|
|
1016
|
+
actionMethod: z.enum(["GET", "POST"]).optional()
|
|
1017
|
+
}).strict();
|
|
1018
|
+
var TwiMLOptionsSchema = VoiceTwiMLOptionsConversationRelaySchema;
|
|
982
1019
|
var TwiMLRequestSchema = z.object({
|
|
983
1020
|
from: z.string().optional(),
|
|
984
1021
|
to: z.string().optional(),
|
|
@@ -990,8 +1027,8 @@ var TwiMLRequestSchema = z.object({
|
|
|
990
1027
|
/**
|
|
991
1028
|
* Any other fields from the Twilio webhook not captured above. Values are
|
|
992
1029
|
* always strings here (webhook form fields are url-encoded), unlike
|
|
993
|
-
*
|
|
994
|
-
* TwiML attributes.
|
|
1030
|
+
* VoiceTwiMLOptionsConversationRelay.extra which accepts string | boolean |
|
|
1031
|
+
* number for emitted TwiML attributes.
|
|
995
1032
|
*/
|
|
996
1033
|
extra: z.record(z.string(), z.string()).default({})
|
|
997
1034
|
});
|
|
@@ -1231,12 +1268,52 @@ function callOptionsToCreateParams(options) {
|
|
|
1231
1268
|
}
|
|
1232
1269
|
return params;
|
|
1233
1270
|
}
|
|
1234
|
-
var
|
|
1271
|
+
var InitiateVoiceConversationOptionsBase = z.object({
|
|
1235
1272
|
to: z.string().min(1, "Recipient phone number is required"),
|
|
1236
1273
|
websocketUrl: z.url().optional(),
|
|
1237
|
-
twimlOptions:
|
|
1274
|
+
twimlOptions: VoiceTwiMLOptionsConversationRelaySchema.optional(),
|
|
1238
1275
|
callOptions: CallOptionsSchema.optional()
|
|
1276
|
+
});
|
|
1277
|
+
var InitiateVoiceConversationOptionsSchema = InitiateVoiceConversationOptionsBase.strict();
|
|
1278
|
+
var InitiateVoiceConversationOptionsOpenAIRealtimeSchema = InitiateVoiceConversationOptionsBase.extend({
|
|
1279
|
+
// Overridden to the Media Streams subtype: the inherited ConversationRelay
|
|
1280
|
+
// schema is `.strict()` and would reject `name` / `statusCallback` outright,
|
|
1281
|
+
// so this provider's TwiML options could never survive parsing.
|
|
1282
|
+
twimlOptions: VoiceTwiMLOptionsMediaStreamsSchema.optional(),
|
|
1283
|
+
/**
|
|
1284
|
+
* Used verbatim in place of `OpenAIRealtimeProviderConfig.defaultSessionConfig`
|
|
1285
|
+
* for this call.
|
|
1286
|
+
*/
|
|
1287
|
+
sessionConfig: z.record(z.string(), z.unknown()).nullable().optional()
|
|
1239
1288
|
}).strict();
|
|
1289
|
+
var InitiateVoiceConversationOptionsGPTLiveSchema = InitiateVoiceConversationOptionsBase.extend({
|
|
1290
|
+
// Overridden to the Media Streams subtype: the inherited ConversationRelay
|
|
1291
|
+
// schema is `.strict()` and would reject `name` / `statusCallback` outright,
|
|
1292
|
+
// so this provider's TwiML options could never survive parsing.
|
|
1293
|
+
twimlOptions: VoiceTwiMLOptionsMediaStreamsSchema.optional(),
|
|
1294
|
+
/**
|
|
1295
|
+
* Used verbatim in place of `GPTLiveProviderConfig.defaultSessionConfig`
|
|
1296
|
+
* for this call.
|
|
1297
|
+
*/
|
|
1298
|
+
sessionConfig: z.record(z.string(), z.unknown()).nullable().optional()
|
|
1299
|
+
}).strict();
|
|
1300
|
+
var StreamStartMessageSchema = z.object({
|
|
1301
|
+
/** SID of the Media Stream itself (`MZ...`), used to address media back to Twilio. */
|
|
1302
|
+
streamSid: z.string(),
|
|
1303
|
+
/** SID of the call the stream is attached to — TAC's conversation identifier. */
|
|
1304
|
+
callSid: z.string(),
|
|
1305
|
+
/**
|
|
1306
|
+
* Negotiated audio format (encoding, sample rate, channels). Left untyped
|
|
1307
|
+
* because Twilio may add fields here, and the provider only reads it for
|
|
1308
|
+
* diagnostics.
|
|
1309
|
+
*/
|
|
1310
|
+
mediaFormat: z.record(z.string(), z.unknown()).nullable().optional(),
|
|
1311
|
+
/**
|
|
1312
|
+
* Values of the `<Parameter>` children emitted on `<Stream>`. Defaults to an
|
|
1313
|
+
* empty object so callers never have to null-check it.
|
|
1314
|
+
*/
|
|
1315
|
+
customParameters: z.record(z.string(), z.string()).default({})
|
|
1316
|
+
});
|
|
1240
1317
|
var JSONSchemaSchema = z.object({
|
|
1241
1318
|
type: z.enum(["object", "string", "number", "boolean", "array"]),
|
|
1242
1319
|
properties: z.record(z.string(), z.any()).optional(),
|
|
@@ -1258,6 +1335,12 @@ var AnthropicToolSchema = z.object({
|
|
|
1258
1335
|
description: z.string(),
|
|
1259
1336
|
input_schema: JSONSchemaSchema
|
|
1260
1337
|
});
|
|
1338
|
+
var OpenAIRealtimeToolSchema = z.object({
|
|
1339
|
+
type: z.literal("function"),
|
|
1340
|
+
name: z.string(),
|
|
1341
|
+
description: z.string(),
|
|
1342
|
+
parameters: JSONSchemaSchema
|
|
1343
|
+
});
|
|
1261
1344
|
var ToolExecutionResultSchema = z.object({
|
|
1262
1345
|
success: z.boolean(),
|
|
1263
1346
|
data: z.any().optional(),
|
|
@@ -1681,7 +1764,97 @@ function createLogger(options) {
|
|
|
1681
1764
|
|
|
1682
1765
|
// package.json
|
|
1683
1766
|
var package_default = {
|
|
1684
|
-
|
|
1767
|
+
name: "twilio-agent-connect",
|
|
1768
|
+
version: "2.4.0",
|
|
1769
|
+
description: "Twilio Agent Connect - A TypeScript framework for building intelligent agents",
|
|
1770
|
+
type: "module",
|
|
1771
|
+
main: "./dist/index.js",
|
|
1772
|
+
module: "./dist/index.js",
|
|
1773
|
+
types: "./dist/index.d.ts",
|
|
1774
|
+
exports: {
|
|
1775
|
+
".": {
|
|
1776
|
+
types: "./dist/index.d.ts",
|
|
1777
|
+
import: "./dist/index.js",
|
|
1778
|
+
default: "./dist/index.js"
|
|
1779
|
+
}
|
|
1780
|
+
},
|
|
1781
|
+
files: [
|
|
1782
|
+
"dist"
|
|
1783
|
+
],
|
|
1784
|
+
scripts: {
|
|
1785
|
+
build: "tsup",
|
|
1786
|
+
clean: "rimraf dist packages/*/dist",
|
|
1787
|
+
test: "vitest --run",
|
|
1788
|
+
"test:watch": "vitest",
|
|
1789
|
+
"test:coverage": "vitest --coverage",
|
|
1790
|
+
lint: "eslint 'packages/*/src/**/*.ts' 'src/**/*.ts'",
|
|
1791
|
+
"lint:fix": "eslint 'packages/*/src/**/*.ts' 'src/**/*.ts' --fix",
|
|
1792
|
+
format: "prettier --write 'packages/*/src/**/*.ts' 'src/**/*.ts' 'getting_started/**/*.ts'",
|
|
1793
|
+
"format:check": "prettier --check 'packages/*/src/**/*.ts' 'src/**/*.ts' 'getting_started/**/*.ts'",
|
|
1794
|
+
typecheck: "tsc --noEmit",
|
|
1795
|
+
docs: "typedoc",
|
|
1796
|
+
"docs:watch": "typedoc --watch",
|
|
1797
|
+
"example:getting-started": "cd getting_started/examples/openai && npm install && npm run dev",
|
|
1798
|
+
"example:chat": "cd getting_started/examples/chat && npm install && npm run dev"
|
|
1799
|
+
},
|
|
1800
|
+
keywords: [
|
|
1801
|
+
"twilio",
|
|
1802
|
+
"ai",
|
|
1803
|
+
"agents",
|
|
1804
|
+
"framework",
|
|
1805
|
+
"typescript"
|
|
1806
|
+
],
|
|
1807
|
+
author: "Twilio",
|
|
1808
|
+
license: "MIT",
|
|
1809
|
+
repository: {
|
|
1810
|
+
type: "git",
|
|
1811
|
+
url: "git+https://github.com/twilio/twilio-agent-connect-typescript.git"
|
|
1812
|
+
},
|
|
1813
|
+
devDependencies: {
|
|
1814
|
+
"@eslint/js": "^9.0.0",
|
|
1815
|
+
"@shipgirl/typedoc-plugin-versions": "^0.3.2",
|
|
1816
|
+
"@types/node": "^22.0.0",
|
|
1817
|
+
"@types/ws": "^8.18.1",
|
|
1818
|
+
"@vitest/coverage-v8": "^4.1.2",
|
|
1819
|
+
"axios-mock-adapter": "^2.1.0",
|
|
1820
|
+
eslint: "^9.0.0",
|
|
1821
|
+
globals: "^15.0.0",
|
|
1822
|
+
prettier: "^3.0.0",
|
|
1823
|
+
rimraf: "^5.0.0",
|
|
1824
|
+
tsup: "^8.5.1",
|
|
1825
|
+
typedoc: "^0.28.20",
|
|
1826
|
+
"typedoc-material-theme": "^1.4.1",
|
|
1827
|
+
typescript: "^5.0.0",
|
|
1828
|
+
"typescript-eslint": "^8.0.0",
|
|
1829
|
+
vitest: "^4.1.2"
|
|
1830
|
+
},
|
|
1831
|
+
dependencies: {
|
|
1832
|
+
"@fastify/formbody": "^8.0.2",
|
|
1833
|
+
"@fastify/websocket": "^11.2.0",
|
|
1834
|
+
"@segment/analytics-node": "^3.1.0",
|
|
1835
|
+
axios: "^1.15.2",
|
|
1836
|
+
"axios-retry": "^4.5.0",
|
|
1837
|
+
dotenv: "^16.4.7",
|
|
1838
|
+
fastify: "^5.8.5",
|
|
1839
|
+
"fastify-graceful-shutdown": "^5.0.0",
|
|
1840
|
+
pino: "^9.0.0",
|
|
1841
|
+
twilio: "^5.10.7",
|
|
1842
|
+
ws: "^8.19.0",
|
|
1843
|
+
zod: "^4.0.0"
|
|
1844
|
+
},
|
|
1845
|
+
peerDependencies: {
|
|
1846
|
+
"@openai/agents": "^0.8.0"
|
|
1847
|
+
},
|
|
1848
|
+
peerDependenciesMeta: {
|
|
1849
|
+
"@openai/agents": {
|
|
1850
|
+
optional: true
|
|
1851
|
+
}
|
|
1852
|
+
},
|
|
1853
|
+
engines: {
|
|
1854
|
+
node: ">=22.13.0",
|
|
1855
|
+
npm: ">=9.0.0"
|
|
1856
|
+
}
|
|
1857
|
+
};
|
|
1685
1858
|
function buildUserAgent() {
|
|
1686
1859
|
return `twilio-agent-connect-typescript/${package_default.version}`;
|
|
1687
1860
|
}
|
|
@@ -2425,6 +2598,57 @@ var KnowledgeClient = class extends BaseClient {
|
|
|
2425
2598
|
}
|
|
2426
2599
|
}
|
|
2427
2600
|
};
|
|
2601
|
+
var SDK_PACKAGE = "twilio-agent-connect-typescript";
|
|
2602
|
+
var client = null;
|
|
2603
|
+
var log = null;
|
|
2604
|
+
var disabled = null;
|
|
2605
|
+
function getLog() {
|
|
2606
|
+
return log ??= createLogger({ name: "analytics" });
|
|
2607
|
+
}
|
|
2608
|
+
function isDisabled() {
|
|
2609
|
+
if (disabled === null) {
|
|
2610
|
+
disabled = process.env.TAC_ANALYTICS_DISABLED === "true";
|
|
2611
|
+
}
|
|
2612
|
+
return disabled;
|
|
2613
|
+
}
|
|
2614
|
+
function getClient() {
|
|
2615
|
+
if (isDisabled()) return null;
|
|
2616
|
+
if (client) return client;
|
|
2617
|
+
client = new Analytics({
|
|
2618
|
+
writeKey: "oH5gLNxB4NEg60y81mBxHWZn4RAoXQTN",
|
|
2619
|
+
flushAt: 20,
|
|
2620
|
+
flushInterval: 1e4
|
|
2621
|
+
});
|
|
2622
|
+
client.on("error", (err) => {
|
|
2623
|
+
getLog().debug({ err }, "Segment analytics error");
|
|
2624
|
+
});
|
|
2625
|
+
return client;
|
|
2626
|
+
}
|
|
2627
|
+
function trackEvent(event, properties) {
|
|
2628
|
+
try {
|
|
2629
|
+
const analytics = getClient();
|
|
2630
|
+
if (!analytics) return;
|
|
2631
|
+
analytics.track({
|
|
2632
|
+
anonymousId: properties.account_sid,
|
|
2633
|
+
event,
|
|
2634
|
+
properties: {
|
|
2635
|
+
...properties,
|
|
2636
|
+
sdk_version: package_default.version,
|
|
2637
|
+
sdk_package: SDK_PACKAGE
|
|
2638
|
+
}
|
|
2639
|
+
});
|
|
2640
|
+
getLog().debug({ event }, "Analytics event tracked");
|
|
2641
|
+
} catch (err) {
|
|
2642
|
+
getLog().debug({ err, event }, "Analytics event failed");
|
|
2643
|
+
}
|
|
2644
|
+
}
|
|
2645
|
+
async function shutdownAnalytics() {
|
|
2646
|
+
if (client) {
|
|
2647
|
+
await client.closeAndFlush({ timeout: 5e3 }).catch(() => {
|
|
2648
|
+
});
|
|
2649
|
+
client = null;
|
|
2650
|
+
}
|
|
2651
|
+
}
|
|
2428
2652
|
|
|
2429
2653
|
// packages/core/src/lib/operator-result-processor.ts
|
|
2430
2654
|
function extractProfileIds(operatorResult) {
|
|
@@ -3164,6 +3388,8 @@ var TAC = class _TAC {
|
|
|
3164
3388
|
channel.shutdown();
|
|
3165
3389
|
}
|
|
3166
3390
|
this.channels.clear();
|
|
3391
|
+
shutdownAnalytics().catch(() => {
|
|
3392
|
+
});
|
|
3167
3393
|
this.logger.info("TAC shutdown complete");
|
|
3168
3394
|
}
|
|
3169
3395
|
};
|
|
@@ -3255,6 +3481,12 @@ var BaseChannel = class {
|
|
|
3255
3481
|
if (this.callbacks.onConversationStarted) {
|
|
3256
3482
|
this.callbacks.onConversationStarted({ session });
|
|
3257
3483
|
}
|
|
3484
|
+
trackEvent("Conversation Started", {
|
|
3485
|
+
account_sid: this.config.accountSid,
|
|
3486
|
+
channel: this.channelType,
|
|
3487
|
+
conversation_id: conversationId,
|
|
3488
|
+
has_profile_id: !!profileId
|
|
3489
|
+
});
|
|
3258
3490
|
return session;
|
|
3259
3491
|
}
|
|
3260
3492
|
/**
|
|
@@ -3277,6 +3509,12 @@ var BaseChannel = class {
|
|
|
3277
3509
|
);
|
|
3278
3510
|
}
|
|
3279
3511
|
}
|
|
3512
|
+
trackEvent("Conversation Ended", {
|
|
3513
|
+
account_sid: this.config.accountSid,
|
|
3514
|
+
channel: this.channelType,
|
|
3515
|
+
conversation_id: conversationId,
|
|
3516
|
+
duration_ms: Date.now() - session.startedAt.getTime()
|
|
3517
|
+
});
|
|
3280
3518
|
this.activeConversations.delete(conversationId);
|
|
3281
3519
|
this.logger.debug(
|
|
3282
3520
|
{
|
|
@@ -3543,6 +3781,21 @@ var MessagingChannel = class extends BaseChannel {
|
|
|
3543
3781
|
this.conversationClient = tac.getConversationClient();
|
|
3544
3782
|
this.messagingCallbacks = {};
|
|
3545
3783
|
}
|
|
3784
|
+
/**
|
|
3785
|
+
* Report a delivered response.
|
|
3786
|
+
*
|
|
3787
|
+
* Each channel calls this from its own `sendResponse` rather than the base
|
|
3788
|
+
* wrapping the call: `sendResponse` is the public extension point, so making
|
|
3789
|
+
* it a template method would break subclasses defined outside this package.
|
|
3790
|
+
*/
|
|
3791
|
+
trackResponseSent(conversationId) {
|
|
3792
|
+
trackEvent("Response Sent", {
|
|
3793
|
+
account_sid: this.config.accountSid,
|
|
3794
|
+
channel: this.channelType,
|
|
3795
|
+
conversation_id: conversationId,
|
|
3796
|
+
response_type: "full"
|
|
3797
|
+
});
|
|
3798
|
+
}
|
|
3546
3799
|
/**
|
|
3547
3800
|
* Check if a message is from the bot itself (2-tier).
|
|
3548
3801
|
*
|
|
@@ -3777,6 +4030,11 @@ var MessagingChannel = class extends BaseChannel {
|
|
|
3777
4030
|
userMemory
|
|
3778
4031
|
});
|
|
3779
4032
|
}
|
|
4033
|
+
trackEvent("Message Received", {
|
|
4034
|
+
account_sid: this.config.accountSid,
|
|
4035
|
+
channel: this.channelType,
|
|
4036
|
+
conversation_id: conversationId
|
|
4037
|
+
});
|
|
3780
4038
|
}
|
|
3781
4039
|
/**
|
|
3782
4040
|
* Handle conversation updated event
|
|
@@ -4253,6 +4511,7 @@ var SMSChannel = class extends MessagingChannel {
|
|
|
4253
4511
|
});
|
|
4254
4512
|
throw error;
|
|
4255
4513
|
}
|
|
4514
|
+
this.trackResponseSent(conversationId);
|
|
4256
4515
|
}
|
|
4257
4516
|
/**
|
|
4258
4517
|
* Initiate an outbound SMS conversation
|
|
@@ -4375,6 +4634,7 @@ var RCSChannel = class extends MessagingChannel {
|
|
|
4375
4634
|
});
|
|
4376
4635
|
throw error;
|
|
4377
4636
|
}
|
|
4637
|
+
this.trackResponseSent(conversationId);
|
|
4378
4638
|
}
|
|
4379
4639
|
/**
|
|
4380
4640
|
* Initiate an outbound RCS conversation
|
|
@@ -4499,6 +4759,7 @@ var WhatsAppChannel = class extends MessagingChannel {
|
|
|
4499
4759
|
});
|
|
4500
4760
|
throw error;
|
|
4501
4761
|
}
|
|
4762
|
+
this.trackResponseSent(conversationId);
|
|
4502
4763
|
}
|
|
4503
4764
|
/**
|
|
4504
4765
|
* Initiate an outbound WhatsApp conversation
|
|
@@ -4639,6 +4900,7 @@ var ChatChannel = class extends MessagingChannel {
|
|
|
4639
4900
|
});
|
|
4640
4901
|
throw error;
|
|
4641
4902
|
}
|
|
4903
|
+
this.trackResponseSent(conversationId);
|
|
4642
4904
|
}
|
|
4643
4905
|
/**
|
|
4644
4906
|
* Initiate an outbound chat conversation
|
|
@@ -4667,271 +4929,489 @@ var ChatChannel = class extends MessagingChannel {
|
|
|
4667
4929
|
}
|
|
4668
4930
|
};
|
|
4669
4931
|
|
|
4670
|
-
// packages/core/src/
|
|
4671
|
-
|
|
4672
|
-
|
|
4673
|
-
|
|
4674
|
-
|
|
4675
|
-
|
|
4932
|
+
// packages/core/src/lib/deprecation.ts
|
|
4933
|
+
var warned = /* @__PURE__ */ new Set();
|
|
4934
|
+
function warnDeprecated(oldName, replacement) {
|
|
4935
|
+
if (warned.has(oldName)) return;
|
|
4936
|
+
warned.add(oldName);
|
|
4937
|
+
console.warn(`${oldName} is deprecated and will be removed in 3.0 \u2014 use ${replacement} instead.`);
|
|
4676
4938
|
}
|
|
4677
4939
|
|
|
4678
|
-
// packages/core/src/channels/voice.ts
|
|
4679
|
-
var
|
|
4680
|
-
|
|
4681
|
-
|
|
4682
|
-
|
|
4683
|
-
|
|
4684
|
-
|
|
4685
|
-
|
|
4940
|
+
// packages/core/src/channels/voice/provider.ts
|
|
4941
|
+
var VoiceProvider = class {
|
|
4942
|
+
/** The `VoiceChannel` that owns this provider. */
|
|
4943
|
+
channel;
|
|
4944
|
+
/** Logger named after the concrete provider class. */
|
|
4945
|
+
logger;
|
|
4946
|
+
constructor(channel) {
|
|
4947
|
+
this.channel = channel;
|
|
4948
|
+
this.logger = createLogger({ name: this.constructor.name });
|
|
4686
4949
|
}
|
|
4687
|
-
|
|
4688
|
-
|
|
4689
|
-
|
|
4690
|
-
|
|
4691
|
-
|
|
4692
|
-
|
|
4693
|
-
|
|
4694
|
-
|
|
4695
|
-
|
|
4696
|
-
|
|
4697
|
-
twilioClient;
|
|
4698
|
-
voiceConfig;
|
|
4699
|
-
onInboundCallTwimlHandler;
|
|
4700
|
-
onCallStatusHandler;
|
|
4701
|
-
onAmdHandler;
|
|
4702
|
-
onRecordingHandler;
|
|
4703
|
-
constructor(tac, options) {
|
|
4704
|
-
super(tac, options);
|
|
4705
|
-
this.voiceConfig = options ?? {};
|
|
4706
|
-
this.webSocketConnections = /* @__PURE__ */ new Map();
|
|
4707
|
-
this.voiceCallbacks = {};
|
|
4708
|
-
this.streamTasks = /* @__PURE__ */ new Map();
|
|
4709
|
-
this.promptQueues = /* @__PURE__ */ new Map();
|
|
4710
|
-
this.initializationRetries = /* @__PURE__ */ new Map();
|
|
4711
|
-
this.callSidToConversationId = /* @__PURE__ */ new Map();
|
|
4950
|
+
/**
|
|
4951
|
+
* Stable snake_case identifier for this provider, reported on voice
|
|
4952
|
+
* telemetry events so emissions from different transports are
|
|
4953
|
+
* distinguishable. Built-in providers override it; a provider defined outside
|
|
4954
|
+
* the SDK inherits `"custom"`.
|
|
4955
|
+
*
|
|
4956
|
+
* @internal
|
|
4957
|
+
*/
|
|
4958
|
+
get providerId() {
|
|
4959
|
+
return "custom";
|
|
4712
4960
|
}
|
|
4713
4961
|
/**
|
|
4714
|
-
*
|
|
4715
|
-
* `<ConversationRelay>` on inbound calls.
|
|
4962
|
+
* Channel name identifier, e.g. `"VOICE"`.
|
|
4716
4963
|
*
|
|
4717
|
-
*
|
|
4718
|
-
*
|
|
4719
|
-
|
|
4720
|
-
|
|
4964
|
+
* A label for this provider's transport; it does not change how
|
|
4965
|
+
* `VoiceChannel` reports its `ChannelType`.
|
|
4966
|
+
*/
|
|
4967
|
+
get channelName() {
|
|
4968
|
+
return "VOICE";
|
|
4969
|
+
}
|
|
4970
|
+
/**
|
|
4971
|
+
* Build the response for an inbound call. Default: not supported.
|
|
4721
4972
|
*
|
|
4722
|
-
* @
|
|
4723
|
-
*
|
|
4724
|
-
*
|
|
4725
|
-
*
|
|
4726
|
-
|
|
4727
|
-
|
|
4728
|
-
|
|
4729
|
-
|
|
4730
|
-
|
|
4973
|
+
* @param _twimlRequest - Parsed Twilio webhook fields for the inbound call.
|
|
4974
|
+
* @param _options - Additional per-call inputs.
|
|
4975
|
+
* @param _options.hostTwimlOptions - Per-call TwiML supplied by a custom
|
|
4976
|
+
* in-process host.
|
|
4977
|
+
*/
|
|
4978
|
+
// eslint-disable-next-line @typescript-eslint/require-await -- Default rejects without awaiting, but stays `async` so callers always get a Promise
|
|
4979
|
+
async handleIncomingCall(_twimlRequest, _options) {
|
|
4980
|
+
throw new Error(`${this.constructor.name} does not support inbound calls.`);
|
|
4981
|
+
}
|
|
4982
|
+
/**
|
|
4983
|
+
* Handle this provider's own out-of-band lifecycle webhook, if it has one.
|
|
4731
4984
|
*
|
|
4732
|
-
*
|
|
4733
|
-
* `
|
|
4985
|
+
* Not every provider has an equivalent — Twilio's ConversationRelay posts to
|
|
4986
|
+
* `<Connect action=...>` when the session ends (`ConversationRelayProvider`
|
|
4987
|
+
* uses this as a WebSocket-disconnect backup); Media Streams instead has its
|
|
4988
|
+
* own independent `statusCallback` (`stream-started` / `stream-stopped` /
|
|
4989
|
+
* `stream-error`), which is purely informational and doesn't gate call flow.
|
|
4990
|
+
* Default acknowledges with an empty 200 for providers with nothing to do
|
|
4991
|
+
* here.
|
|
4734
4992
|
*/
|
|
4735
|
-
|
|
4736
|
-
|
|
4993
|
+
// eslint-disable-next-line @typescript-eslint/require-await -- Default acknowledges without awaiting, but stays `async` so callers always get a Promise
|
|
4994
|
+
async handleTwilioProviderCallback(_payload) {
|
|
4995
|
+
return { status: 200, content: "", contentType: "text/plain" };
|
|
4737
4996
|
}
|
|
4738
4997
|
/**
|
|
4739
|
-
*
|
|
4998
|
+
* Drive one WebSocket connection from accept to disconnect.
|
|
4740
4999
|
*
|
|
4741
|
-
*
|
|
4742
|
-
*
|
|
4743
|
-
*
|
|
5000
|
+
* Implementations may be synchronous or asynchronous: a provider that only
|
|
5001
|
+
* attaches event handlers can return `void`, while one that awaits an
|
|
5002
|
+
* upstream handshake before serving traffic returns a `Promise<void>`. The
|
|
5003
|
+
* owning `VoiceChannel` is responsible for handling a returned promise's
|
|
5004
|
+
* rejection, so an async override never produces an unhandled rejection.
|
|
5005
|
+
*/
|
|
5006
|
+
handleWebSocket(_websocket) {
|
|
5007
|
+
throw new Error(`${this.constructor.name} does not support WebSocket connections.`);
|
|
5008
|
+
}
|
|
5009
|
+
/** Place an outbound call. Default: not supported. */
|
|
5010
|
+
// eslint-disable-next-line @typescript-eslint/require-await -- Default rejects without awaiting, but stays `async` so callers always get a Promise
|
|
5011
|
+
async initiateOutboundConversation(_options) {
|
|
5012
|
+
throw new Error(`${this.constructor.name} does not support outbound calls.`);
|
|
5013
|
+
}
|
|
5014
|
+
/** Send a text response back through this provider's transport, if supported. */
|
|
5015
|
+
// eslint-disable-next-line @typescript-eslint/require-await -- Default rejects without awaiting, but stays `async` so callers always get a Promise
|
|
5016
|
+
async sendResponse(_conversationId, _message, _metadata) {
|
|
5017
|
+
throw new Error(`${this.constructor.name} does not support sendResponse.`);
|
|
5018
|
+
}
|
|
5019
|
+
/**
|
|
5020
|
+
* Stream a text response back through this provider's transport, token by
|
|
5021
|
+
* token, if supported. Default: not supported.
|
|
4744
5022
|
*
|
|
4745
|
-
*
|
|
4746
|
-
*
|
|
4747
|
-
*
|
|
4748
|
-
*
|
|
5023
|
+
* @param _conversationId - Conversation whose transport receives the tokens.
|
|
5024
|
+
* @param _stream - Async iterable of text chunks to relay as they arrive.
|
|
5025
|
+
* @param _options - Additional per-call inputs.
|
|
5026
|
+
* @param _options.signal - Aborts the stream mid-flight, e.g. when the caller
|
|
5027
|
+
* interrupts.
|
|
5028
|
+
* @returns The accumulated response text.
|
|
5029
|
+
*/
|
|
5030
|
+
// eslint-disable-next-line @typescript-eslint/require-await -- Default rejects without awaiting, but stays `async` so callers always get a Promise
|
|
5031
|
+
async sendStreamingResponse(_conversationId, _stream, _options) {
|
|
5032
|
+
throw new Error(`${this.constructor.name} does not support sendStreamingResponse.`);
|
|
5033
|
+
}
|
|
5034
|
+
/** Return the Twilio-facing WebSocket for a conversation, if tracked. */
|
|
5035
|
+
getWebSocket(_conversationId) {
|
|
5036
|
+
return null;
|
|
5037
|
+
}
|
|
5038
|
+
/**
|
|
5039
|
+
* Drop this provider's transport state on channel shutdown.
|
|
4749
5040
|
*
|
|
4750
|
-
*
|
|
4751
|
-
*
|
|
5041
|
+
* Called by {@link VoiceChannel.shutdown} before the channel clears its own
|
|
5042
|
+
* conversation bookkeeping. Default no-op — providers override this to drop
|
|
5043
|
+
* whatever transport state they track. Live WebSocket connections are owned
|
|
5044
|
+
* and closed by the server, so an override only clears in-process tracking.
|
|
5045
|
+
*/
|
|
5046
|
+
shutdown() {
|
|
5047
|
+
return void 0;
|
|
5048
|
+
}
|
|
5049
|
+
/**
|
|
5050
|
+
* Set callback URLs on `callParams` for every registered call-event handler.
|
|
4752
5051
|
*
|
|
4753
|
-
*
|
|
4754
|
-
*
|
|
4755
|
-
*
|
|
4756
|
-
*
|
|
4757
|
-
*
|
|
4758
|
-
*
|
|
4759
|
-
* });
|
|
4760
|
-
* ```
|
|
5052
|
+
* A URL is derived only when its handler is registered — an unwanted
|
|
5053
|
+
* call-event URL would otherwise surface as silent 11200 alerts for a
|
|
5054
|
+
* feature nobody asked for. If TAC isn't serving these routes, set the URLs
|
|
5055
|
+
* explicitly via `CallOptions` (or the provider config's default call
|
|
5056
|
+
* options, where available). An explicit URL from either options layer is
|
|
5057
|
+
* never overwritten.
|
|
4761
5058
|
*/
|
|
4762
|
-
|
|
4763
|
-
|
|
5059
|
+
applyCallEventCallbacks(callParams) {
|
|
5060
|
+
const handlers = this.channel.getCallEventHandlers();
|
|
5061
|
+
const wiring = [
|
|
5062
|
+
["status", "statusCallback", handlers.status],
|
|
5063
|
+
["amd", "asyncAmdStatusCallback", handlers.amd],
|
|
5064
|
+
["recording", "recordingStatusCallback", handlers.recording]
|
|
5065
|
+
];
|
|
5066
|
+
for (const [kind, param, handler] of wiring) {
|
|
5067
|
+
if (!handler) continue;
|
|
5068
|
+
const url = this.channel.getTacConfig().callEventUrl(kind);
|
|
5069
|
+
if (url !== void 0 && callParams[param] === void 0) {
|
|
5070
|
+
callParams[param] = url;
|
|
5071
|
+
}
|
|
5072
|
+
}
|
|
5073
|
+
return callParams;
|
|
4764
5074
|
}
|
|
5075
|
+
};
|
|
5076
|
+
var VoiceProviderConfig = class {
|
|
5077
|
+
/** Memory retrieval mode for this channel. Defaults to `'never'`. */
|
|
5078
|
+
memoryMode;
|
|
4765
5079
|
/**
|
|
4766
|
-
*
|
|
5080
|
+
* The {@link BaseChannelOptions} this config was constructed with, retained
|
|
5081
|
+
* verbatim so `VoiceChannel` can hand them to `BaseChannel`.
|
|
4767
5082
|
*
|
|
4768
|
-
*
|
|
4769
|
-
*
|
|
4770
|
-
*
|
|
4771
|
-
*
|
|
4772
|
-
* for this to fire (at most once per call).
|
|
5083
|
+
* A provider config carries channel-level options (`dedupCapacity` and
|
|
5084
|
+
* friends) alongside its own provider settings; without this they would be
|
|
5085
|
+
* dropped on the config path while still applying on the plain-object path,
|
|
5086
|
+
* which is the same channel configured two ways.
|
|
4773
5087
|
*
|
|
4774
|
-
* @
|
|
4775
|
-
* ```typescript
|
|
4776
|
-
* voiceChannel.onAmd(async event => {
|
|
4777
|
-
* if (event.isMachine) {
|
|
4778
|
-
* await voiceChannel.endCall(event.callSid); // voicemail → hang up
|
|
4779
|
-
* }
|
|
4780
|
-
* });
|
|
4781
|
-
* ```
|
|
5088
|
+
* @internal
|
|
4782
5089
|
*/
|
|
4783
|
-
|
|
4784
|
-
|
|
5090
|
+
channelOptions;
|
|
5091
|
+
constructor(options) {
|
|
5092
|
+
this.memoryMode = options?.memoryMode ?? "never";
|
|
5093
|
+
this.channelOptions = { ...options };
|
|
4785
5094
|
}
|
|
4786
5095
|
/**
|
|
4787
|
-
*
|
|
4788
|
-
*
|
|
4789
|
-
* Registering makes later outbound calls pass `recordingStatusCallback` to
|
|
4790
|
-
* `calls.create`; without a handler TAC omits it and Twilio has nowhere to
|
|
4791
|
-
* post. It does not start recording — that's `CallOptions.record`, which is
|
|
4792
|
-
* required for this to fire.
|
|
5096
|
+
* Build the {@link VoiceProvider} this config configures.
|
|
4793
5097
|
*
|
|
4794
|
-
* @
|
|
4795
|
-
*
|
|
4796
|
-
*
|
|
4797
|
-
* if (event.recordingStatus === 'completed') {
|
|
4798
|
-
* // store event.recordingUrl
|
|
4799
|
-
* }
|
|
4800
|
-
* });
|
|
4801
|
-
* ```
|
|
5098
|
+
* @param _channel - The owning `VoiceChannel`.
|
|
5099
|
+
* @param _tacConfig - `TACConfig` — providers that talk TwiML need it to
|
|
5100
|
+
* derive default URLs (`voicePublicDomain` etc.).
|
|
4802
5101
|
*/
|
|
4803
|
-
|
|
4804
|
-
|
|
5102
|
+
createProvider(_channel, _tacConfig) {
|
|
5103
|
+
throw new Error(
|
|
5104
|
+
`${this.constructor.name} must implement createProvider() to be usable as a VoiceChannel config.`
|
|
5105
|
+
);
|
|
5106
|
+
}
|
|
5107
|
+
};
|
|
5108
|
+
|
|
5109
|
+
// packages/core/src/channels/voice/twiml.ts
|
|
5110
|
+
function filterUnsetValues(config) {
|
|
5111
|
+
const filtered = {};
|
|
5112
|
+
for (const [key, value] of Object.entries(config)) {
|
|
5113
|
+
if (value !== void 0) {
|
|
5114
|
+
filtered[key] = value;
|
|
5115
|
+
}
|
|
5116
|
+
}
|
|
5117
|
+
return filtered;
|
|
5118
|
+
}
|
|
5119
|
+
function stringifyParameterValue(value) {
|
|
5120
|
+
if (typeof value === "object") {
|
|
5121
|
+
return JSON.stringify(value);
|
|
5122
|
+
}
|
|
5123
|
+
return String(value);
|
|
5124
|
+
}
|
|
5125
|
+
var TwiMLBuilderBase = class {
|
|
5126
|
+
tacConfig;
|
|
5127
|
+
channelConfig;
|
|
5128
|
+
logger;
|
|
5129
|
+
constructor(tacConfig, channelConfig, logger) {
|
|
5130
|
+
this.tacConfig = tacConfig;
|
|
5131
|
+
this.channelConfig = channelConfig;
|
|
5132
|
+
this.logger = logger;
|
|
4805
5133
|
}
|
|
4806
5134
|
/**
|
|
4807
|
-
*
|
|
4808
|
-
*
|
|
5135
|
+
* Apply fields explicitly present on `source` onto `target`, except those
|
|
5136
|
+
* named in `skip`.
|
|
5137
|
+
*
|
|
5138
|
+
* Nested objects, arrays, and dicts replace wholesale — there's no per-key
|
|
5139
|
+
* merging.
|
|
5140
|
+
*
|
|
5141
|
+
* "Explicitly present" is detected via key presence (`Object.keys`), which
|
|
5142
|
+
* mirrors Python's `model_fields_set`: a key set to `undefined` is still
|
|
5143
|
+
* "present" and overrides lower layers, while an absent key falls through.
|
|
4809
5144
|
*/
|
|
4810
|
-
|
|
4811
|
-
|
|
4812
|
-
|
|
5145
|
+
overlayFields(target, source, skip = []) {
|
|
5146
|
+
for (const key of Object.keys(source)) {
|
|
5147
|
+
if (skip.includes(key)) {
|
|
5148
|
+
continue;
|
|
5149
|
+
}
|
|
5150
|
+
target[key] = source[key];
|
|
4813
5151
|
}
|
|
4814
|
-
|
|
4815
|
-
|
|
5152
|
+
}
|
|
5153
|
+
/**
|
|
5154
|
+
* The error thrown when no layer and no `TACConfig`-derived default supplies
|
|
5155
|
+
* a WebSocket URL. `caller` names the API the developer actually called.
|
|
5156
|
+
*/
|
|
5157
|
+
missingWebsocketUrlError(caller) {
|
|
5158
|
+
return new Error(
|
|
5159
|
+
`${caller} needs a WebSocket URL. Set TWILIO_VOICE_PUBLIC_DOMAIN (or TACConfig.voicePublicDomain).`
|
|
4816
5160
|
);
|
|
4817
5161
|
}
|
|
4818
5162
|
/**
|
|
4819
|
-
*
|
|
5163
|
+
* The WebSocket URL derived from `TACConfig.voicePublicDomain` +
|
|
5164
|
+
* `TACConfig.voiceWebsocketPath`, or undefined when `voicePublicDomain` is
|
|
5165
|
+
* unset.
|
|
5166
|
+
*/
|
|
5167
|
+
defaultWebsocketUrl() {
|
|
5168
|
+
if (!this.tacConfig.voicePublicDomain) {
|
|
5169
|
+
return void 0;
|
|
5170
|
+
}
|
|
5171
|
+
return `wss://${this.tacConfig.voicePublicDomain}${this.tacConfig.voiceWebsocketPath}`;
|
|
5172
|
+
}
|
|
5173
|
+
/**
|
|
5174
|
+
* Resolve the default `<Connect action=...>` cleanup URL from
|
|
5175
|
+
* `TACConfig.voicePublicDomain` + `TACConfig.voiceActionPath`.
|
|
4820
5176
|
*
|
|
4821
5177
|
* Returns undefined if `voicePublicDomain` isn't set; that's fine because
|
|
4822
5178
|
* actionUrl has higher-priority layers (customizer, twimlOptions, Studio
|
|
4823
5179
|
* handoff) above this fallback.
|
|
4824
5180
|
*/
|
|
4825
|
-
|
|
4826
|
-
if (this.
|
|
4827
|
-
return `https://${this.
|
|
5181
|
+
defaultActionUrl() {
|
|
5182
|
+
if (this.tacConfig.voicePublicDomain) {
|
|
5183
|
+
return `https://${this.tacConfig.voicePublicDomain}${this.tacConfig.voiceActionPath}`;
|
|
4828
5184
|
}
|
|
4829
5185
|
return void 0;
|
|
4830
5186
|
}
|
|
4831
|
-
|
|
4832
|
-
|
|
4833
|
-
|
|
4834
|
-
|
|
4835
|
-
|
|
5187
|
+
};
|
|
5188
|
+
|
|
5189
|
+
// packages/core/src/util/handoff-urls.ts
|
|
5190
|
+
function studioExecutionsUrl(flowSid) {
|
|
5191
|
+
return `https://studio.twilio.com/v2/Flows/${flowSid}/Executions`;
|
|
5192
|
+
}
|
|
5193
|
+
function studioVoiceHandoffUrl(accountSid, flowSid) {
|
|
5194
|
+
return `https://webhooks.twilio.com/v1/Accounts/${accountSid}/Flows/${flowSid}?Trigger=incomingCall`;
|
|
5195
|
+
}
|
|
5196
|
+
|
|
5197
|
+
// packages/core/src/channels/voice/conversation-relay/twiml.ts
|
|
5198
|
+
var DEFAULT_WELCOME_GREETING = "Hello! How can I assist you today?";
|
|
5199
|
+
var SKIP_ACTION_URL = ["actionUrl"];
|
|
5200
|
+
var TwiMLBuilderConversationRelay = class _TwiMLBuilderConversationRelay extends TwiMLBuilderBase {
|
|
5201
|
+
/**
|
|
5202
|
+
* Field names on {@link VoiceTwiMLOptionsConversationRelay} that map directly to `<ConversationRelay>`
|
|
5203
|
+
* attributes (camelCase, emitted as-is). Excludes the fields handled specially
|
|
5204
|
+
* by {@link generateTwiml}: websocketUrl (resolved through the layered merge and
|
|
5205
|
+
* emitted as the `url` attribute), actionUrl, languages, customParameters, extra.
|
|
5206
|
+
*/
|
|
5207
|
+
static RELAY_ATTR_FIELDS = [
|
|
5208
|
+
"welcomeGreeting",
|
|
5209
|
+
"welcomeGreetingInterruptible",
|
|
5210
|
+
"conversationConfiguration",
|
|
5211
|
+
"language",
|
|
5212
|
+
"ttsLanguage",
|
|
5213
|
+
"transcriptionLanguage",
|
|
5214
|
+
"voice",
|
|
5215
|
+
"ttsProvider",
|
|
5216
|
+
"transcriptionProvider",
|
|
5217
|
+
"speechModel",
|
|
5218
|
+
"elevenlabsTextNormalization",
|
|
5219
|
+
"eotThreshold",
|
|
5220
|
+
"partialPrompts",
|
|
5221
|
+
"deepgramSmartFormat",
|
|
5222
|
+
"speechTimeout",
|
|
5223
|
+
"interruptible",
|
|
5224
|
+
"interruptSensitivity",
|
|
5225
|
+
"reportInputDuringAgentSpeech",
|
|
5226
|
+
"ignoreBackchannel",
|
|
5227
|
+
"preemptible",
|
|
5228
|
+
"dtmfDetection",
|
|
5229
|
+
"hints",
|
|
5230
|
+
"events",
|
|
5231
|
+
"debug",
|
|
5232
|
+
"intelligenceService"
|
|
5233
|
+
];
|
|
5234
|
+
/**
|
|
5235
|
+
* Build the TwiML XML for one call.
|
|
5236
|
+
*
|
|
5237
|
+
* @param caller - Name of the calling method, used in the "no WebSocket URL"
|
|
5238
|
+
* error so it points at the API the developer actually called.
|
|
5239
|
+
* @param options - Per-call option layers and WebSocket override.
|
|
5240
|
+
* @throws {Error} if no layer and no `TACConfig`-derived default supplies a
|
|
5241
|
+
* WebSocket URL.
|
|
5242
|
+
*/
|
|
5243
|
+
build(caller, options) {
|
|
5244
|
+
const merged = this.buildTwimlOptions(options?.host, options?.perCall);
|
|
5245
|
+
const resolvedWebsocketUrl = options?.websocketUrl ?? merged.websocketUrl ?? this.defaultWebsocketUrl();
|
|
5246
|
+
if (!resolvedWebsocketUrl) {
|
|
5247
|
+
throw this.missingWebsocketUrlError(caller);
|
|
4836
5248
|
}
|
|
4837
|
-
return this.
|
|
4838
|
-
}
|
|
4839
|
-
get channelType() {
|
|
4840
|
-
return "voice";
|
|
5249
|
+
return this.generateTwiml(resolvedWebsocketUrl, merged);
|
|
4841
5250
|
}
|
|
4842
5251
|
/**
|
|
4843
|
-
*
|
|
5252
|
+
* Layer TwiML options, lowest precedence first: TAC defaults → `host`
|
|
5253
|
+
* (calling host's per-call values) → channel `defaultTwimlOptions` → `perCall`
|
|
5254
|
+
* (application customizer output for inbound, or
|
|
5255
|
+
* `InitiateVoiceConversationOptions.twimlOptions` for outbound).
|
|
5256
|
+
*
|
|
5257
|
+
* `actionUrl` is skipped by the overlays on purpose — it's resolved once via
|
|
5258
|
+
* {@link resolveActionUrl} looking at every layer at once, and that resolved
|
|
5259
|
+
* value is written into `merged` before the overlays run. Letting it through
|
|
5260
|
+
* would let a higher-priority layer that didn't set actionUrl silently clobber
|
|
5261
|
+
* a lower layer that did.
|
|
4844
5262
|
*/
|
|
4845
|
-
|
|
4846
|
-
|
|
4847
|
-
|
|
4848
|
-
|
|
4849
|
-
|
|
4850
|
-
|
|
4851
|
-
|
|
4852
|
-
|
|
4853
|
-
|
|
4854
|
-
|
|
4855
|
-
|
|
4856
|
-
break;
|
|
4857
|
-
case "webSocketConnected":
|
|
4858
|
-
this.voiceCallbacks.onWebSocketConnected = callback;
|
|
4859
|
-
break;
|
|
4860
|
-
case "webSocketDisconnected":
|
|
4861
|
-
this.voiceCallbacks.onWebSocketDisconnected = callback;
|
|
4862
|
-
break;
|
|
4863
|
-
default:
|
|
4864
|
-
super.on(event, callback);
|
|
4865
|
-
break;
|
|
5263
|
+
buildTwimlOptions(host, perCall) {
|
|
5264
|
+
const merged = {
|
|
5265
|
+
welcomeGreeting: DEFAULT_WELCOME_GREETING,
|
|
5266
|
+
...this.tacConfig.isOrchestratorEnabled() && this.tacConfig.conversationConfigurationId !== void 0 ? { conversationConfiguration: this.tacConfig.conversationConfigurationId } : {}
|
|
5267
|
+
};
|
|
5268
|
+
const resolvedActionUrl = this.resolveActionUrl(host, perCall);
|
|
5269
|
+
if (resolvedActionUrl !== void 0) {
|
|
5270
|
+
merged.actionUrl = resolvedActionUrl;
|
|
5271
|
+
}
|
|
5272
|
+
if (host) {
|
|
5273
|
+
this.overlayFields(merged, host, SKIP_ACTION_URL);
|
|
4866
5274
|
}
|
|
5275
|
+
if (this.channelConfig.defaultTwimlOptions) {
|
|
5276
|
+
this.overlayFields(merged, this.channelConfig.defaultTwimlOptions, SKIP_ACTION_URL);
|
|
5277
|
+
}
|
|
5278
|
+
if (perCall) {
|
|
5279
|
+
this.overlayFields(merged, perCall, SKIP_ACTION_URL);
|
|
5280
|
+
}
|
|
5281
|
+
return merged;
|
|
4867
5282
|
}
|
|
4868
5283
|
/**
|
|
4869
|
-
*
|
|
5284
|
+
* Resolve the TwiML `<Connect action=...>` URL.
|
|
4870
5285
|
*
|
|
4871
|
-
*
|
|
4872
|
-
*
|
|
5286
|
+
* Precedence (highest to lowest):
|
|
5287
|
+
* 1. application customizer
|
|
5288
|
+
* 2. channel `defaultTwimlOptions`
|
|
5289
|
+
* 3. `host` (calling host's per-call options)
|
|
5290
|
+
* 4. Studio handoff (when `studioHandoffFlowSid` is configured)
|
|
5291
|
+
* 5. Channel default — derived from `TACConfig.voicePublicDomain` +
|
|
5292
|
+
* `TACConfig.voiceActionPath`.
|
|
4873
5293
|
*
|
|
4874
|
-
*
|
|
4875
|
-
*
|
|
5294
|
+
* User-expressed intent (Studio handoff is configured explicitly on
|
|
5295
|
+
* `TACConfig`) beats the SDK's generated cleanup default.
|
|
4876
5296
|
*
|
|
4877
|
-
*
|
|
4878
|
-
*
|
|
5297
|
+
* Explicit `actionUrl: undefined` on a layer (key present, value undefined)
|
|
5298
|
+
* suppresses `<Connect action=...>` entirely — all lower layers are skipped.
|
|
5299
|
+
* `actionUrl` left absent (key not present) falls through to the next layer.
|
|
4879
5300
|
*/
|
|
4880
|
-
|
|
4881
|
-
|
|
4882
|
-
|
|
4883
|
-
|
|
4884
|
-
|
|
5301
|
+
resolveActionUrl(host, customized) {
|
|
5302
|
+
if (customized && "actionUrl" in customized) {
|
|
5303
|
+
return customized.actionUrl;
|
|
5304
|
+
}
|
|
5305
|
+
const defaults = this.channelConfig.defaultTwimlOptions;
|
|
5306
|
+
if (defaults && "actionUrl" in defaults) {
|
|
5307
|
+
return defaults.actionUrl;
|
|
5308
|
+
}
|
|
5309
|
+
if (host && "actionUrl" in host) {
|
|
5310
|
+
return host.actionUrl;
|
|
5311
|
+
}
|
|
5312
|
+
if (this.tacConfig.studioHandoffFlowSid) {
|
|
5313
|
+
return studioVoiceHandoffUrl(this.tacConfig.accountSid, this.tacConfig.studioHandoffFlowSid);
|
|
5314
|
+
}
|
|
5315
|
+
return this.defaultActionUrl();
|
|
5316
|
+
}
|
|
5317
|
+
/**
|
|
5318
|
+
* Generate TwiML XML for ConversationRelay from a merged {@link VoiceTwiMLOptionsConversationRelay}.
|
|
5319
|
+
*
|
|
5320
|
+
* This is the low-level emitter used by {@link build} after layering. It
|
|
5321
|
+
* mirrors the Python SDK's `generate_twiml`.
|
|
5322
|
+
*
|
|
5323
|
+
* @param websocketUrl - Public WebSocket URL (e.g. 'wss://example.ngrok.app/ws').
|
|
5324
|
+
* @param options - Merged VoiceTwiMLOptionsConversationRelay to emit.
|
|
5325
|
+
* @returns TwiML XML string ready to return to Twilio.
|
|
5326
|
+
*/
|
|
5327
|
+
generateTwiml(websocketUrl, options) {
|
|
5328
|
+
const response = new VoiceResponse();
|
|
5329
|
+
const connect = response.connect(options.actionUrl ? { action: options.actionUrl } : {});
|
|
5330
|
+
const relayAttrs = { url: websocketUrl };
|
|
5331
|
+
for (const field of _TwiMLBuilderConversationRelay.RELAY_ATTR_FIELDS) {
|
|
5332
|
+
let value = options[field];
|
|
5333
|
+
if (value === void 0) {
|
|
5334
|
+
continue;
|
|
4885
5335
|
}
|
|
4886
|
-
|
|
4887
|
-
|
|
4888
|
-
|
|
4889
|
-
|
|
4890
|
-
|
|
4891
|
-
|
|
4892
|
-
|
|
4893
|
-
|
|
4894
|
-
|
|
4895
|
-
|
|
4896
|
-
this.logger.debug(
|
|
4897
|
-
{
|
|
4898
|
-
event_type: eventType,
|
|
4899
|
-
raw_event_type: webhookData.eventType,
|
|
4900
|
-
conversation_id: conversationId
|
|
4901
|
-
},
|
|
4902
|
-
"Unhandled event type - this event will be ignored"
|
|
5336
|
+
if (field === "interruptible" && typeof value === "boolean") {
|
|
5337
|
+
value = value ? "any" : "none";
|
|
5338
|
+
}
|
|
5339
|
+
relayAttrs[field] = value;
|
|
5340
|
+
}
|
|
5341
|
+
if (options.extra) {
|
|
5342
|
+
for (const [key, value] of Object.entries(options.extra)) {
|
|
5343
|
+
if (key === "url") {
|
|
5344
|
+
this.logger.warn(
|
|
5345
|
+
"Ignoring `url` in VoiceTwiMLOptionsConversationRelay.extra; set `websocketUrl` to override the ConversationRelay URL."
|
|
4903
5346
|
);
|
|
5347
|
+
continue;
|
|
5348
|
+
}
|
|
5349
|
+
relayAttrs[key] = value;
|
|
4904
5350
|
}
|
|
4905
|
-
|
|
4906
|
-
|
|
4907
|
-
|
|
4908
|
-
|
|
5351
|
+
}
|
|
5352
|
+
const relay = connect.conversationRelay(
|
|
5353
|
+
relayAttrs
|
|
5354
|
+
);
|
|
5355
|
+
if (options.languages && options.languages.length > 0) {
|
|
5356
|
+
for (const lang of options.languages) {
|
|
5357
|
+
const langAttrs = filterUnsetValues(lang);
|
|
5358
|
+
relay.language(langAttrs);
|
|
4909
5359
|
}
|
|
4910
|
-
this.handleError(error instanceof Error ? error : new Error(String(error)), { payload });
|
|
4911
5360
|
}
|
|
5361
|
+
if (options.customParameters) {
|
|
5362
|
+
for (const [name, value] of Object.entries(options.customParameters)) {
|
|
5363
|
+
if (value !== null && value !== void 0) {
|
|
5364
|
+
relay.parameter({ name, value: stringifyParameterValue(value) });
|
|
5365
|
+
}
|
|
5366
|
+
}
|
|
5367
|
+
}
|
|
5368
|
+
return response.toString();
|
|
5369
|
+
}
|
|
5370
|
+
};
|
|
5371
|
+
|
|
5372
|
+
// packages/core/src/channels/voice/conversation-relay/provider.ts
|
|
5373
|
+
var POLL_ATTEMPTS = 10;
|
|
5374
|
+
var POLL_BASE_DELAY_MS = 250;
|
|
5375
|
+
var POLL_MAX_DELAY_MS = 1500;
|
|
5376
|
+
var ConversationRelayProvider = class extends VoiceProvider {
|
|
5377
|
+
/** @internal */
|
|
5378
|
+
get providerId() {
|
|
5379
|
+
return "conversation_relay";
|
|
4912
5380
|
}
|
|
4913
5381
|
/**
|
|
4914
|
-
*
|
|
5382
|
+
* The owning channel's logger, so relocated ConversationRelay logic keeps
|
|
5383
|
+
* logging exactly as it did when it lived on `VoiceChannel`.
|
|
4915
5384
|
*/
|
|
4916
|
-
|
|
4917
|
-
|
|
4918
|
-
|
|
4919
|
-
|
|
4920
|
-
|
|
4921
|
-
|
|
4922
|
-
|
|
4923
|
-
|
|
4924
|
-
|
|
4925
|
-
|
|
4926
|
-
|
|
4927
|
-
|
|
4928
|
-
|
|
4929
|
-
|
|
5385
|
+
logger;
|
|
5386
|
+
/** In-flight streaming responses, keyed by conversation. */
|
|
5387
|
+
streamTasks;
|
|
5388
|
+
config;
|
|
5389
|
+
tacConfig;
|
|
5390
|
+
twimlBuilder;
|
|
5391
|
+
webSocketConnections;
|
|
5392
|
+
promptQueues;
|
|
5393
|
+
initializationRetries;
|
|
5394
|
+
callSidToConversationId;
|
|
5395
|
+
MAX_INITIALIZATION_RETRIES = 3;
|
|
5396
|
+
constructor(channel, tacConfig, config) {
|
|
5397
|
+
super(channel);
|
|
5398
|
+
this.logger = channel.getLoggerInternal();
|
|
5399
|
+
this.config = config;
|
|
5400
|
+
this.tacConfig = tacConfig;
|
|
5401
|
+
this.twimlBuilder = new TwiMLBuilderConversationRelay(tacConfig, config, this.logger);
|
|
5402
|
+
this.streamTasks = /* @__PURE__ */ new Map();
|
|
5403
|
+
this.webSocketConnections = /* @__PURE__ */ new Map();
|
|
5404
|
+
this.promptQueues = /* @__PURE__ */ new Map();
|
|
5405
|
+
this.initializationRetries = /* @__PURE__ */ new Map();
|
|
5406
|
+
this.callSidToConversationId = /* @__PURE__ */ new Map();
|
|
5407
|
+
}
|
|
5408
|
+
get channelName() {
|
|
5409
|
+
return "VOICE";
|
|
4930
5410
|
}
|
|
4931
5411
|
/**
|
|
4932
5412
|
* Get active WebSocket connection for a conversation
|
|
4933
5413
|
*/
|
|
4934
|
-
|
|
5414
|
+
getWebSocket(conversationId) {
|
|
4935
5415
|
return this.webSocketConnections.get(conversationId) || null;
|
|
4936
5416
|
}
|
|
4937
5417
|
/**
|
|
@@ -4941,12 +5421,13 @@ var VoiceChannel = class _VoiceChannel extends BaseChannel {
|
|
|
4941
5421
|
* caller speaks.
|
|
4942
5422
|
*/
|
|
4943
5423
|
async initializeOrchestratedConversation(callSid, fromNumber, ws) {
|
|
4944
|
-
|
|
5424
|
+
const conversationClient = this.channel.getConversationClientInternal();
|
|
5425
|
+
if (!conversationClient) {
|
|
4945
5426
|
throw new Error("Conversation client is required in orchestrated mode");
|
|
4946
5427
|
}
|
|
4947
5428
|
let conversations = [];
|
|
4948
5429
|
for (let attempt = 0; attempt < POLL_ATTEMPTS; attempt++) {
|
|
4949
|
-
conversations = await
|
|
5430
|
+
conversations = await conversationClient.listConversations({
|
|
4950
5431
|
channelId: callSid,
|
|
4951
5432
|
status: ["ACTIVE"]
|
|
4952
5433
|
});
|
|
@@ -4967,33 +5448,100 @@ var VoiceChannel = class _VoiceChannel extends BaseChannel {
|
|
|
4967
5448
|
}
|
|
4968
5449
|
const conversation = conversations[0];
|
|
4969
5450
|
const conversationId = conversation.id;
|
|
4970
|
-
const participants = await
|
|
5451
|
+
const participants = await conversationClient.listParticipants(conversationId);
|
|
4971
5452
|
const customerParticipant = participants.find((p) => p.type === "CUSTOMER");
|
|
4972
5453
|
const customerAddress = customerParticipant?.addresses?.find((a) => a.channel === "VOICE")?.address ?? fromNumber ?? void 0;
|
|
4973
5454
|
const profileId = customerParticipant?.profileId ? customerParticipant.profileId : void 0;
|
|
4974
5455
|
this.webSocketConnections.set(conversationId, ws);
|
|
4975
5456
|
this.callSidToConversationId.set(callSid, conversationId);
|
|
4976
|
-
const session = this.
|
|
5457
|
+
const session = this.channel.startConversationInternal(conversationId, profileId);
|
|
4977
5458
|
session.callSid = callSid;
|
|
4978
5459
|
if (customerAddress) {
|
|
4979
5460
|
session.authorInfo = {
|
|
4980
5461
|
address: customerAddress
|
|
4981
5462
|
};
|
|
4982
5463
|
}
|
|
4983
|
-
|
|
4984
|
-
|
|
5464
|
+
const voiceCallbacks = this.channel.getVoiceCallbacks();
|
|
5465
|
+
if (voiceCallbacks.onWebSocketConnected) {
|
|
5466
|
+
voiceCallbacks.onWebSocketConnected({ conversationId });
|
|
4985
5467
|
}
|
|
4986
5468
|
return conversationId;
|
|
4987
5469
|
}
|
|
4988
5470
|
/**
|
|
4989
5471
|
* Handle WebSocket connection from ConversationRelay
|
|
4990
5472
|
*/
|
|
4991
|
-
|
|
5473
|
+
handleWebSocket(ws) {
|
|
4992
5474
|
let conversationId = null;
|
|
4993
5475
|
let callSid = null;
|
|
4994
5476
|
let fromNumber = null;
|
|
4995
5477
|
let initializationFailed = false;
|
|
4996
5478
|
let initPromise = null;
|
|
5479
|
+
const ensureConversation = async () => {
|
|
5480
|
+
if (conversationId) {
|
|
5481
|
+
return conversationId;
|
|
5482
|
+
}
|
|
5483
|
+
const sid = callSid;
|
|
5484
|
+
if (!sid) {
|
|
5485
|
+
return null;
|
|
5486
|
+
}
|
|
5487
|
+
const retryCount = this.initializationRetries.get(sid) ?? 0;
|
|
5488
|
+
if (retryCount >= this.MAX_INITIALIZATION_RETRIES) {
|
|
5489
|
+
throw new Error(
|
|
5490
|
+
`Cannot process message - conversation initialization failed after ${retryCount} attempts for callSid ${sid}`
|
|
5491
|
+
);
|
|
5492
|
+
}
|
|
5493
|
+
try {
|
|
5494
|
+
if (initializationFailed) {
|
|
5495
|
+
this.logger.info(
|
|
5496
|
+
{ call_sid: sid, retry_count: retryCount },
|
|
5497
|
+
"Retrying conversation initialization after previous failure"
|
|
5498
|
+
);
|
|
5499
|
+
}
|
|
5500
|
+
if (!this.channel.isOrchestratorEnabledInternal()) {
|
|
5501
|
+
conversationId = sid;
|
|
5502
|
+
this.webSocketConnections.set(conversationId, ws);
|
|
5503
|
+
this.callSidToConversationId.set(sid, conversationId);
|
|
5504
|
+
const session = this.channel.startConversationInternal(conversationId);
|
|
5505
|
+
session.callSid = sid;
|
|
5506
|
+
if (fromNumber) {
|
|
5507
|
+
session.authorInfo = { address: fromNumber };
|
|
5508
|
+
}
|
|
5509
|
+
const voiceCallbacks = this.channel.getVoiceCallbacks();
|
|
5510
|
+
if (voiceCallbacks.onWebSocketConnected) {
|
|
5511
|
+
voiceCallbacks.onWebSocketConnected({ conversationId });
|
|
5512
|
+
}
|
|
5513
|
+
} else {
|
|
5514
|
+
initPromise ??= this.initializeOrchestratedConversation(sid, fromNumber, ws);
|
|
5515
|
+
try {
|
|
5516
|
+
conversationId = await initPromise;
|
|
5517
|
+
} finally {
|
|
5518
|
+
initPromise = null;
|
|
5519
|
+
}
|
|
5520
|
+
}
|
|
5521
|
+
initializationFailed = false;
|
|
5522
|
+
this.initializationRetries.delete(sid);
|
|
5523
|
+
this.logger.info(
|
|
5524
|
+
{ conversation_id: conversationId, call_sid: sid },
|
|
5525
|
+
"Conversation initialization succeeded"
|
|
5526
|
+
);
|
|
5527
|
+
trackEvent("Conversation Initialized", {
|
|
5528
|
+
account_sid: this.tacConfig.accountSid,
|
|
5529
|
+
channel: "voice",
|
|
5530
|
+
conversation_id: conversationId,
|
|
5531
|
+
provider: this.providerId,
|
|
5532
|
+
orchestrator_enabled: this.tacConfig.isOrchestratorEnabled()
|
|
5533
|
+
});
|
|
5534
|
+
return conversationId;
|
|
5535
|
+
} catch (err) {
|
|
5536
|
+
initializationFailed = true;
|
|
5537
|
+
this.initializationRetries.set(sid, retryCount + 1);
|
|
5538
|
+
this.logger.error(
|
|
5539
|
+
{ err, call_sid: sid, retry_count: retryCount + 1 },
|
|
5540
|
+
"Conversation initialization failed"
|
|
5541
|
+
);
|
|
5542
|
+
throw err;
|
|
5543
|
+
}
|
|
5544
|
+
};
|
|
4997
5545
|
ws.on("message", (data) => {
|
|
4998
5546
|
(async () => {
|
|
4999
5547
|
try {
|
|
@@ -5016,7 +5564,7 @@ var VoiceChannel = class _VoiceChannel extends BaseChannel {
|
|
|
5016
5564
|
case "setup":
|
|
5017
5565
|
callSid = message.callSid;
|
|
5018
5566
|
fromNumber = message.from;
|
|
5019
|
-
if (this.
|
|
5567
|
+
if (this.channel.isOrchestratorEnabledInternal()) {
|
|
5020
5568
|
this.logger.debug(
|
|
5021
5569
|
{ call_sid: callSid },
|
|
5022
5570
|
"Starting background conversation initialization"
|
|
@@ -5024,77 +5572,30 @@ var VoiceChannel = class _VoiceChannel extends BaseChannel {
|
|
|
5024
5572
|
initPromise = this.initializeOrchestratedConversation(callSid, fromNumber, ws);
|
|
5025
5573
|
void initPromise.catch(() => void 0);
|
|
5026
5574
|
}
|
|
5027
|
-
|
|
5028
|
-
this.
|
|
5029
|
-
|
|
5030
|
-
|
|
5031
|
-
|
|
5032
|
-
|
|
5033
|
-
|
|
5575
|
+
{
|
|
5576
|
+
const voiceCallbacks = this.channel.getVoiceCallbacks();
|
|
5577
|
+
if (voiceCallbacks.onSetup) {
|
|
5578
|
+
voiceCallbacks.onSetup({
|
|
5579
|
+
callSid,
|
|
5580
|
+
from: message.from,
|
|
5581
|
+
to: message.to,
|
|
5582
|
+
customParameters: message.customParameters
|
|
5583
|
+
});
|
|
5584
|
+
}
|
|
5034
5585
|
}
|
|
5035
5586
|
break;
|
|
5036
5587
|
case "prompt":
|
|
5037
|
-
|
|
5038
|
-
const retryCount = this.initializationRetries.get(callSid) ?? 0;
|
|
5039
|
-
if (retryCount >= this.MAX_INITIALIZATION_RETRIES) {
|
|
5040
|
-
throw new Error(
|
|
5041
|
-
`Cannot process prompt - conversation initialization failed after ${retryCount} attempts for callSid ${callSid}`
|
|
5042
|
-
);
|
|
5043
|
-
}
|
|
5044
|
-
try {
|
|
5045
|
-
if (initializationFailed) {
|
|
5046
|
-
this.logger.info(
|
|
5047
|
-
{ call_sid: callSid, retry_count: retryCount },
|
|
5048
|
-
"Retrying conversation initialization after previous failure"
|
|
5049
|
-
);
|
|
5050
|
-
}
|
|
5051
|
-
if (!this.tac.isOrchestratorEnabled()) {
|
|
5052
|
-
conversationId = callSid;
|
|
5053
|
-
this.webSocketConnections.set(conversationId, ws);
|
|
5054
|
-
this.callSidToConversationId.set(callSid, conversationId);
|
|
5055
|
-
const session = this.startConversation(conversationId);
|
|
5056
|
-
session.callSid = callSid;
|
|
5057
|
-
if (fromNumber) {
|
|
5058
|
-
session.authorInfo = { address: fromNumber };
|
|
5059
|
-
}
|
|
5060
|
-
if (this.voiceCallbacks.onWebSocketConnected) {
|
|
5061
|
-
this.voiceCallbacks.onWebSocketConnected({ conversationId });
|
|
5062
|
-
}
|
|
5063
|
-
} else {
|
|
5064
|
-
initPromise ??= this.initializeOrchestratedConversation(
|
|
5065
|
-
callSid,
|
|
5066
|
-
fromNumber,
|
|
5067
|
-
ws
|
|
5068
|
-
);
|
|
5069
|
-
try {
|
|
5070
|
-
conversationId = await initPromise;
|
|
5071
|
-
} finally {
|
|
5072
|
-
initPromise = null;
|
|
5073
|
-
}
|
|
5074
|
-
}
|
|
5075
|
-
initializationFailed = false;
|
|
5076
|
-
this.initializationRetries.delete(callSid);
|
|
5077
|
-
this.logger.info(
|
|
5078
|
-
{ conversation_id: conversationId, call_sid: callSid },
|
|
5079
|
-
"Conversation initialization succeeded"
|
|
5080
|
-
);
|
|
5081
|
-
} catch (err) {
|
|
5082
|
-
initializationFailed = true;
|
|
5083
|
-
this.initializationRetries.set(callSid, retryCount + 1);
|
|
5084
|
-
this.logger.error(
|
|
5085
|
-
{ err, call_sid: callSid, retry_count: retryCount + 1 },
|
|
5086
|
-
"Conversation initialization failed"
|
|
5087
|
-
);
|
|
5088
|
-
throw err;
|
|
5089
|
-
}
|
|
5090
|
-
}
|
|
5588
|
+
await ensureConversation();
|
|
5091
5589
|
if (conversationId) {
|
|
5092
5590
|
const previousPrompt = this.promptQueues.get(conversationId) ?? Promise.resolve();
|
|
5093
5591
|
const currentPrompt = previousPrompt.then(() => this.handlePromptMessage(conversationId, message)).catch((err) => {
|
|
5094
|
-
this.
|
|
5095
|
-
|
|
5096
|
-
|
|
5097
|
-
|
|
5592
|
+
this.channel.handleErrorInternal(
|
|
5593
|
+
err instanceof Error ? err : new Error(String(err)),
|
|
5594
|
+
{
|
|
5595
|
+
conversationId,
|
|
5596
|
+
message: data.toString()
|
|
5597
|
+
}
|
|
5598
|
+
);
|
|
5098
5599
|
});
|
|
5099
5600
|
this.promptQueues.set(conversationId, currentPrompt);
|
|
5100
5601
|
} else {
|
|
@@ -5106,6 +5607,17 @@ var VoiceChannel = class _VoiceChannel extends BaseChannel {
|
|
|
5106
5607
|
this.handleInterruptMessage(conversationId, message);
|
|
5107
5608
|
}
|
|
5108
5609
|
break;
|
|
5610
|
+
case "dtmf":
|
|
5611
|
+
try {
|
|
5612
|
+
await ensureConversation();
|
|
5613
|
+
} catch (err) {
|
|
5614
|
+
this.logger.warn(
|
|
5615
|
+
{ err, call_sid: callSid },
|
|
5616
|
+
"Conversation initialization failed on DTMF keypress, delivering digit without a conversation"
|
|
5617
|
+
);
|
|
5618
|
+
}
|
|
5619
|
+
await this.handleDtmfMessage(conversationId, callSid, message);
|
|
5620
|
+
break;
|
|
5109
5621
|
default:
|
|
5110
5622
|
this.logger.debug(
|
|
5111
5623
|
{
|
|
@@ -5117,11 +5629,14 @@ var VoiceChannel = class _VoiceChannel extends BaseChannel {
|
|
|
5117
5629
|
break;
|
|
5118
5630
|
}
|
|
5119
5631
|
} catch (error) {
|
|
5120
|
-
this.
|
|
5121
|
-
|
|
5122
|
-
|
|
5123
|
-
|
|
5124
|
-
|
|
5632
|
+
this.channel.handleErrorInternal(
|
|
5633
|
+
error instanceof Error ? error : new Error(String(error)),
|
|
5634
|
+
{
|
|
5635
|
+
conversationId,
|
|
5636
|
+
callSid,
|
|
5637
|
+
message: data.toString()
|
|
5638
|
+
}
|
|
5639
|
+
);
|
|
5125
5640
|
}
|
|
5126
5641
|
})().catch((err) => {
|
|
5127
5642
|
this.logger.error({ err }, "Unhandled error in WebSocket message handler");
|
|
@@ -5155,7 +5670,7 @@ var VoiceChannel = class _VoiceChannel extends BaseChannel {
|
|
|
5155
5670
|
}
|
|
5156
5671
|
});
|
|
5157
5672
|
ws.on("error", (error) => {
|
|
5158
|
-
this.
|
|
5673
|
+
this.channel.handleErrorInternal(error, { conversationId });
|
|
5159
5674
|
});
|
|
5160
5675
|
}
|
|
5161
5676
|
/**
|
|
@@ -5164,10 +5679,11 @@ var VoiceChannel = class _VoiceChannel extends BaseChannel {
|
|
|
5164
5679
|
async handlePromptMessage(conversationId, message) {
|
|
5165
5680
|
const transcript = message.voicePrompt;
|
|
5166
5681
|
const streamTask = this.startStreamTask(conversationId);
|
|
5167
|
-
const session = this.getConversationSession(conversationId);
|
|
5168
|
-
const userMemory = session ? await this.
|
|
5169
|
-
|
|
5170
|
-
|
|
5682
|
+
const session = this.channel.getConversationSession(conversationId);
|
|
5683
|
+
const userMemory = session ? await this.channel.retrieveMemoryInternal(session, transcript) : void 0;
|
|
5684
|
+
const voiceCallbacks = this.channel.getVoiceCallbacks();
|
|
5685
|
+
if (voiceCallbacks.onPrompt) {
|
|
5686
|
+
await voiceCallbacks.onPrompt({
|
|
5171
5687
|
conversationId,
|
|
5172
5688
|
transcript,
|
|
5173
5689
|
abortSignal: streamTask.controller.signal,
|
|
@@ -5203,13 +5719,42 @@ var VoiceChannel = class _VoiceChannel extends BaseChannel {
|
|
|
5203
5719
|
}
|
|
5204
5720
|
}
|
|
5205
5721
|
}
|
|
5206
|
-
|
|
5207
|
-
|
|
5722
|
+
const voiceCallbacks = this.channel.getVoiceCallbacks();
|
|
5723
|
+
if (voiceCallbacks.onInterrupt) {
|
|
5724
|
+
voiceCallbacks.onInterrupt({
|
|
5208
5725
|
conversationId,
|
|
5209
5726
|
utteranceUntilInterrupt,
|
|
5210
5727
|
durationUntilInterruptMs
|
|
5211
5728
|
});
|
|
5212
5729
|
}
|
|
5730
|
+
trackEvent("Voice Interrupt", {
|
|
5731
|
+
account_sid: this.tacConfig.accountSid,
|
|
5732
|
+
channel: "voice",
|
|
5733
|
+
conversation_id: conversationId,
|
|
5734
|
+
...durationUntilInterruptMs !== void 0 && {
|
|
5735
|
+
duration_until_interrupt_ms: durationUntilInterruptMs
|
|
5736
|
+
},
|
|
5737
|
+
provider: this.providerId,
|
|
5738
|
+
orchestrator_enabled: this.tacConfig.isOrchestratorEnabled()
|
|
5739
|
+
});
|
|
5740
|
+
}
|
|
5741
|
+
/**
|
|
5742
|
+
* Handle WebSocket DTMF message (caller keypress)
|
|
5743
|
+
*/
|
|
5744
|
+
async handleDtmfMessage(conversationId, callSid, message) {
|
|
5745
|
+
const { digit } = message;
|
|
5746
|
+
this.logger.debug({ conversation_id: conversationId, call_sid: callSid }, "DTMF keypress");
|
|
5747
|
+
const voiceCallbacks = this.channel.getVoiceCallbacks();
|
|
5748
|
+
if (!voiceCallbacks.onDtmf) {
|
|
5749
|
+
return;
|
|
5750
|
+
}
|
|
5751
|
+
const session = conversationId ? this.channel.getConversationSession(conversationId) : void 0;
|
|
5752
|
+
await voiceCallbacks.onDtmf({
|
|
5753
|
+
conversationId: conversationId ?? void 0,
|
|
5754
|
+
callSid: callSid ?? void 0,
|
|
5755
|
+
digit,
|
|
5756
|
+
...session !== void 0 && { session }
|
|
5757
|
+
});
|
|
5213
5758
|
}
|
|
5214
5759
|
/**
|
|
5215
5760
|
* Handle WebSocket disconnection. In orchestrated mode the conversation stays
|
|
@@ -5220,11 +5765,19 @@ var VoiceChannel = class _VoiceChannel extends BaseChannel {
|
|
|
5220
5765
|
this.cancelStreamTask(conversationId);
|
|
5221
5766
|
this.webSocketConnections.delete(conversationId);
|
|
5222
5767
|
this.promptQueues.delete(conversationId);
|
|
5223
|
-
|
|
5224
|
-
|
|
5225
|
-
|
|
5226
|
-
|
|
5227
|
-
|
|
5768
|
+
const voiceCallbacks = this.channel.getVoiceCallbacks();
|
|
5769
|
+
if (voiceCallbacks.onWebSocketDisconnected) {
|
|
5770
|
+
voiceCallbacks.onWebSocketDisconnected({ conversationId });
|
|
5771
|
+
}
|
|
5772
|
+
trackEvent("Websocket Disconnected", {
|
|
5773
|
+
account_sid: this.tacConfig.accountSid,
|
|
5774
|
+
channel: "voice",
|
|
5775
|
+
conversation_id: conversationId,
|
|
5776
|
+
provider: this.providerId,
|
|
5777
|
+
orchestrator_enabled: this.tacConfig.isOrchestratorEnabled()
|
|
5778
|
+
});
|
|
5779
|
+
if (!this.channel.isOrchestratorEnabledInternal()) {
|
|
5780
|
+
await this.channel.endConversationInternal(conversationId);
|
|
5228
5781
|
}
|
|
5229
5782
|
}
|
|
5230
5783
|
/**
|
|
@@ -5242,7 +5795,7 @@ var VoiceChannel = class _VoiceChannel extends BaseChannel {
|
|
|
5242
5795
|
last: true
|
|
5243
5796
|
};
|
|
5244
5797
|
ws.send(JSON.stringify(response));
|
|
5245
|
-
const session = this.getConversationSession(conversationId);
|
|
5798
|
+
const session = this.channel.getConversationSession(conversationId);
|
|
5246
5799
|
if (session?.pendingHandoffData) {
|
|
5247
5800
|
try {
|
|
5248
5801
|
ws.send(JSON.stringify(session.pendingHandoffData));
|
|
@@ -5256,7 +5809,7 @@ var VoiceChannel = class _VoiceChannel extends BaseChannel {
|
|
|
5256
5809
|
}
|
|
5257
5810
|
return Promise.resolve();
|
|
5258
5811
|
} catch (error) {
|
|
5259
|
-
this.
|
|
5812
|
+
this.channel.handleErrorInternal(error instanceof Error ? error : new Error(String(error)), {
|
|
5260
5813
|
conversationId,
|
|
5261
5814
|
message,
|
|
5262
5815
|
metadata
|
|
@@ -5315,7 +5868,7 @@ var VoiceChannel = class _VoiceChannel extends BaseChannel {
|
|
|
5315
5868
|
ws.send(JSON.stringify({ type: "text", token: "", last: true }));
|
|
5316
5869
|
}
|
|
5317
5870
|
} catch (error) {
|
|
5318
|
-
this.
|
|
5871
|
+
this.channel.handleErrorInternal(error instanceof Error ? error : new Error(String(error)), {
|
|
5319
5872
|
conversationId
|
|
5320
5873
|
});
|
|
5321
5874
|
throw error;
|
|
@@ -5327,6 +5880,61 @@ var VoiceChannel = class _VoiceChannel extends BaseChannel {
|
|
|
5327
5880
|
return fullResponse;
|
|
5328
5881
|
}
|
|
5329
5882
|
// =========================================================================
|
|
5883
|
+
// Stream Task Management
|
|
5884
|
+
//
|
|
5885
|
+
// ConversationRelay-specific: these track the AbortController for a text
|
|
5886
|
+
// token stream. Full-duplex audio transports have no equivalent, so this
|
|
5887
|
+
// stays off `VoiceProvider`.
|
|
5888
|
+
// =========================================================================
|
|
5889
|
+
/**
|
|
5890
|
+
* Start tracking a streaming task for a conversation
|
|
5891
|
+
*
|
|
5892
|
+
* @param conversationId - The conversation ID
|
|
5893
|
+
* @returns The stream task with its AbortController
|
|
5894
|
+
*/
|
|
5895
|
+
startStreamTask(conversationId) {
|
|
5896
|
+
this.cancelStreamTask(conversationId);
|
|
5897
|
+
const task = { controller: new AbortController(), hasSentTokens: false };
|
|
5898
|
+
this.streamTasks.set(conversationId, task);
|
|
5899
|
+
this.logger.debug({ conversation_id: conversationId }, "Started stream task");
|
|
5900
|
+
return task;
|
|
5901
|
+
}
|
|
5902
|
+
/**
|
|
5903
|
+
* Cancel an active streaming task
|
|
5904
|
+
*
|
|
5905
|
+
* @param conversationId - The conversation ID
|
|
5906
|
+
* @returns true if a task was cancelled, false otherwise
|
|
5907
|
+
*/
|
|
5908
|
+
cancelStreamTask(conversationId) {
|
|
5909
|
+
const task = this.streamTasks.get(conversationId);
|
|
5910
|
+
if (task) {
|
|
5911
|
+
task.controller.abort();
|
|
5912
|
+
this.streamTasks.delete(conversationId);
|
|
5913
|
+
this.logger.debug({ conversation_id: conversationId }, "Cancelled stream task");
|
|
5914
|
+
return true;
|
|
5915
|
+
}
|
|
5916
|
+
return false;
|
|
5917
|
+
}
|
|
5918
|
+
/**
|
|
5919
|
+
* Complete a streaming task (remove from tracking)
|
|
5920
|
+
*
|
|
5921
|
+
* @param conversationId - The conversation ID
|
|
5922
|
+
*/
|
|
5923
|
+
completeStreamTask(conversationId) {
|
|
5924
|
+
this.streamTasks.delete(conversationId);
|
|
5925
|
+
this.logger.debug({ conversation_id: conversationId }, "Completed stream task");
|
|
5926
|
+
}
|
|
5927
|
+
/**
|
|
5928
|
+
* Check if a stream task is active
|
|
5929
|
+
*
|
|
5930
|
+
* @param conversationId - The conversation ID
|
|
5931
|
+
* @returns true if an active task exists
|
|
5932
|
+
*/
|
|
5933
|
+
hasActiveStreamTask(conversationId) {
|
|
5934
|
+
const task = this.streamTasks.get(conversationId);
|
|
5935
|
+
return task !== void 0 && !task.controller.signal.aborted;
|
|
5936
|
+
}
|
|
5937
|
+
// =========================================================================
|
|
5330
5938
|
// Incoming Call Handling
|
|
5331
5939
|
// =========================================================================
|
|
5332
5940
|
/**
|
|
@@ -5343,7 +5951,8 @@ var VoiceChannel = class _VoiceChannel extends BaseChannel {
|
|
|
5343
5951
|
* 1. Output of the customizer registered via
|
|
5344
5952
|
* `VoiceChannel.onInboundCallTwiml(...)` if configured and `twimlRequest`
|
|
5345
5953
|
* is given. (Application-owned.)
|
|
5346
|
-
* 2. `
|
|
5954
|
+
* 2. `ConversationRelayProviderConfig.defaultTwimlOptions` — per-channel
|
|
5955
|
+
* defaults.
|
|
5347
5956
|
* 3. `hostTwimlOptions` — per-call transport facts supplied by the host (the
|
|
5348
5957
|
* code owning the route), e.g. a per-call `websocketUrl` with an affinity
|
|
5349
5958
|
* token.
|
|
@@ -5366,117 +5975,64 @@ var VoiceChannel = class _VoiceChannel extends BaseChannel {
|
|
|
5366
5975
|
* `websocketUrl`), layered below `defaultTwimlOptions` and the application
|
|
5367
5976
|
* customizer but above the TAC defaults.
|
|
5368
5977
|
* @returns TwiML XML string for call connection.
|
|
5978
|
+
* @throws {Error} if either options layer isn't a
|
|
5979
|
+
* {@link VoiceTwiMLOptionsConversationRelay}.
|
|
5369
5980
|
*/
|
|
5370
5981
|
async handleIncomingCall(twimlRequest, options) {
|
|
5982
|
+
const host = this.narrowTwimlOptions(options?.hostTwimlOptions, "options.hostTwimlOptions");
|
|
5983
|
+
const onInboundCallTwimlHandler = this.channel.getInboundCallTwimlHandler();
|
|
5371
5984
|
let customized;
|
|
5372
|
-
if (
|
|
5373
|
-
customized =
|
|
5374
|
-
|
|
5375
|
-
|
|
5376
|
-
|
|
5377
|
-
return this.generateTwiml(websocketUrl, merged);
|
|
5378
|
-
}
|
|
5379
|
-
/**
|
|
5380
|
-
* Layer TwiML options, lowest precedence first: TAC defaults → `host`
|
|
5381
|
-
* (calling host's per-call values) → channel `defaultTwimlOptions` → `perCall`
|
|
5382
|
-
* (application customizer output for inbound, or
|
|
5383
|
-
* `InitiateVoiceConversationOptions.twimlOptions` for outbound).
|
|
5384
|
-
*/
|
|
5385
|
-
buildTwimlOptions(host, perCall) {
|
|
5386
|
-
const merged = {
|
|
5387
|
-
welcomeGreeting: DEFAULT_WELCOME_GREETING,
|
|
5388
|
-
...this.tac.isOrchestratorEnabled() && this.config.conversationConfigurationId !== void 0 ? { conversationConfiguration: this.config.conversationConfigurationId } : {}
|
|
5389
|
-
};
|
|
5390
|
-
const resolvedActionUrl = this.resolveActionUrl(host, perCall);
|
|
5391
|
-
if (resolvedActionUrl !== void 0) {
|
|
5392
|
-
merged.actionUrl = resolvedActionUrl;
|
|
5393
|
-
}
|
|
5394
|
-
if (host) {
|
|
5395
|
-
this.overlayFields(merged, host);
|
|
5396
|
-
}
|
|
5397
|
-
if (this.voiceConfig.defaultTwimlOptions) {
|
|
5398
|
-
this.overlayFields(merged, this.voiceConfig.defaultTwimlOptions);
|
|
5399
|
-
}
|
|
5400
|
-
if (perCall) {
|
|
5401
|
-
this.overlayFields(merged, perCall);
|
|
5402
|
-
}
|
|
5403
|
-
return merged;
|
|
5404
|
-
}
|
|
5405
|
-
/**
|
|
5406
|
-
* Apply fields explicitly present on `source` onto `target`.
|
|
5407
|
-
*
|
|
5408
|
-
* Nested objects (`customParameters`), arrays (`languages`), and dicts
|
|
5409
|
-
* (`extra`) replace wholesale — there's no per-key merging.
|
|
5410
|
-
*
|
|
5411
|
-
* `actionUrl` is skipped here on purpose — it's resolved once via
|
|
5412
|
-
* `resolveActionUrl` looking at every layer at once, and that resolved value
|
|
5413
|
-
* is written into `target` before this overlay runs. Letting it through here
|
|
5414
|
-
* would let a higher-priority layer that didn't set actionUrl silently clobber
|
|
5415
|
-
* a lower layer that did.
|
|
5416
|
-
*
|
|
5417
|
-
* "Explicitly present" is detected via key presence (`key in source`), which
|
|
5418
|
-
* mirrors Python's `model_fields_set`: a key set to `undefined` is still
|
|
5419
|
-
* "present" and overrides lower layers, while an absent key falls through.
|
|
5420
|
-
*/
|
|
5421
|
-
overlayFields(target, source) {
|
|
5422
|
-
for (const key of Object.keys(source)) {
|
|
5423
|
-
if (key === "actionUrl") {
|
|
5424
|
-
continue;
|
|
5425
|
-
}
|
|
5426
|
-
target[key] = source[key];
|
|
5985
|
+
if (onInboundCallTwimlHandler && twimlRequest) {
|
|
5986
|
+
customized = this.narrowTwimlOptions(
|
|
5987
|
+
await onInboundCallTwimlHandler(twimlRequest),
|
|
5988
|
+
"the onInboundCallTwiml customizer output"
|
|
5989
|
+
);
|
|
5427
5990
|
}
|
|
5991
|
+
return this.twimlBuilder.build("handleIncomingCall", {
|
|
5992
|
+
host,
|
|
5993
|
+
perCall: customized
|
|
5994
|
+
});
|
|
5428
5995
|
}
|
|
5429
5996
|
/**
|
|
5430
|
-
*
|
|
5431
|
-
*
|
|
5432
|
-
*
|
|
5433
|
-
*
|
|
5434
|
-
* 2. channel `defaultTwimlOptions`
|
|
5435
|
-
* 3. `host` (calling host's per-call options)
|
|
5436
|
-
* 4. Studio handoff (when `studioHandoffFlowSid` is configured)
|
|
5437
|
-
* 5. Channel default — derived from `TACConfig.voicePublicDomain` +
|
|
5438
|
-
* `TACConfig.voiceActionPath`.
|
|
5439
|
-
*
|
|
5440
|
-
* User-expressed intent (Studio handoff is configured explicitly on
|
|
5441
|
-
* `TACConfig`) beats the SDK's generated cleanup default.
|
|
5997
|
+
* Narrow provider-agnostic {@link VoiceTwiMLOptions} to this provider's
|
|
5998
|
+
* concrete shape. `VoiceProvider.handleIncomingCall` is typed against the
|
|
5999
|
+
* base so every provider can accept its own TwiML options, so the
|
|
6000
|
+
* ConversationRelay shape has to be established at runtime.
|
|
5442
6001
|
*
|
|
5443
|
-
*
|
|
5444
|
-
*
|
|
5445
|
-
* `actionUrl` left absent (key not present) falls through to the next layer.
|
|
6002
|
+
* @param value - Options from a caller or the application customizer.
|
|
6003
|
+
* @param label - What produced `value`, for the error message.
|
|
5446
6004
|
*/
|
|
5447
|
-
|
|
5448
|
-
if (
|
|
5449
|
-
return
|
|
5450
|
-
}
|
|
5451
|
-
if (this.voiceConfig.defaultTwimlOptions && "actionUrl" in this.voiceConfig.defaultTwimlOptions) {
|
|
5452
|
-
return this.voiceConfig.defaultTwimlOptions.actionUrl;
|
|
5453
|
-
}
|
|
5454
|
-
if (host && "actionUrl" in host) {
|
|
5455
|
-
return host.actionUrl;
|
|
6005
|
+
narrowTwimlOptions(value, label) {
|
|
6006
|
+
if (value === void 0) {
|
|
6007
|
+
return void 0;
|
|
5456
6008
|
}
|
|
5457
|
-
|
|
5458
|
-
|
|
6009
|
+
const parsed = VoiceTwiMLOptionsConversationRelaySchema.safeParse(value);
|
|
6010
|
+
if (!parsed.success) {
|
|
6011
|
+
const errorMessage = parsed.error.issues.map((issue) => `${issue.path.join(".")}: ${issue.message}`).join(", ");
|
|
6012
|
+
throw new Error(
|
|
6013
|
+
`ConversationRelayProvider.handleIncomingCall requires ${label} to be a VoiceTwiMLOptionsConversationRelay: ${errorMessage}`
|
|
6014
|
+
);
|
|
5459
6015
|
}
|
|
5460
|
-
return
|
|
6016
|
+
return parsed.data;
|
|
5461
6017
|
}
|
|
5462
6018
|
// =========================================================================
|
|
5463
6019
|
// Outbound Call Handling
|
|
5464
6020
|
// =========================================================================
|
|
5465
6021
|
/**
|
|
5466
|
-
* Overlay `perCall` onto `
|
|
6022
|
+
* Overlay `perCall` onto `ConversationRelayProviderConfig.defaultCallOptions`.
|
|
5467
6023
|
*
|
|
5468
|
-
* Per-field via key presence, the same convention
|
|
5469
|
-
* for TwiML options — so a per-call `{ machineDetection: undefined }`
|
|
6024
|
+
* Per-field via key presence, the same convention `TwiMLBuilderBase.overlayFields`
|
|
6025
|
+
* uses for TwiML options — so a per-call `{ machineDetection: undefined }`
|
|
5470
6026
|
* explicitly clears the channel default rather than falling through to it.
|
|
5471
6027
|
*
|
|
5472
6028
|
* The result is always validated, for two reasons: a combination only
|
|
5473
6029
|
* reachable by layering — per-call clearing `machineDetection` while the
|
|
5474
6030
|
* default set `asyncAmd` — must still fail instead of reaching Twilio, and
|
|
5475
|
-
* `
|
|
5476
|
-
* no runtime validation of its own.
|
|
6031
|
+
* `ConversationRelayProviderConfigOptions` is a plain interface, so
|
|
6032
|
+
* `defaultCallOptions` has had no runtime validation of its own.
|
|
5477
6033
|
*/
|
|
5478
6034
|
mergeCallOptions(perCall) {
|
|
5479
|
-
const defaults = this.
|
|
6035
|
+
const defaults = this.config.defaultCallOptions;
|
|
5480
6036
|
if (!defaults && !perCall) {
|
|
5481
6037
|
return void 0;
|
|
5482
6038
|
}
|
|
@@ -5495,8 +6051,8 @@ var VoiceChannel = class _VoiceChannel extends BaseChannel {
|
|
|
5495
6051
|
* Build the extra arguments for `client.calls.create`.
|
|
5496
6052
|
*
|
|
5497
6053
|
* Layers, highest precedence first: this call's `callOptions`,
|
|
5498
|
-
* `
|
|
5499
|
-
* `voicePublicDomain` + `voiceCallEventPath`.
|
|
6054
|
+
* `ConversationRelayProviderConfig.defaultCallOptions`, then callback URLs
|
|
6055
|
+
* derived from `voicePublicDomain` + `voiceCallEventPath`.
|
|
5500
6056
|
*
|
|
5501
6057
|
* A URL is derived only when its handler is registered. That's a deliberate
|
|
5502
6058
|
* deviation from `websocketUrl` / `actionUrl`, which derive unconditionally:
|
|
@@ -5508,19 +6064,7 @@ var VoiceChannel = class _VoiceChannel extends BaseChannel {
|
|
|
5508
6064
|
buildCallParams(callOptions) {
|
|
5509
6065
|
const merged = this.mergeCallOptions(callOptions);
|
|
5510
6066
|
const params = merged ? callOptionsToCreateParams(merged) : {};
|
|
5511
|
-
|
|
5512
|
-
["status", "statusCallback", this.onCallStatusHandler],
|
|
5513
|
-
["amd", "asyncAmdStatusCallback", this.onAmdHandler],
|
|
5514
|
-
["recording", "recordingStatusCallback", this.onRecordingHandler]
|
|
5515
|
-
];
|
|
5516
|
-
for (const [kind, param, handler] of wiring) {
|
|
5517
|
-
if (!handler) continue;
|
|
5518
|
-
const url = this.config.callEventUrl(kind);
|
|
5519
|
-
if (url !== void 0 && params[param] === void 0) {
|
|
5520
|
-
params[param] = url;
|
|
5521
|
-
}
|
|
5522
|
-
}
|
|
5523
|
-
return params;
|
|
6067
|
+
return this.applyCallEventCallbacks(params);
|
|
5524
6068
|
}
|
|
5525
6069
|
/**
|
|
5526
6070
|
* Initiate an outbound voice conversation
|
|
@@ -5532,14 +6076,16 @@ var VoiceChannel = class _VoiceChannel extends BaseChannel {
|
|
|
5532
6076
|
*
|
|
5533
6077
|
* TwiML fields are merged per-field, highest precedence first:
|
|
5534
6078
|
* 1. `options.twimlOptions` — per-call overrides
|
|
5535
|
-
* 2. `
|
|
6079
|
+
* 2. `ConversationRelayProviderConfig.defaultTwimlOptions` — channel-wide
|
|
6080
|
+
* defaults
|
|
5536
6081
|
* 3. TAC defaults: welcome greeting, `conversationConfiguration` from
|
|
5537
6082
|
* `TACConfig`, and `actionUrl` from Studio handoff (if configured), else
|
|
5538
6083
|
* derived from `TACConfig.voicePublicDomain` + `voiceActionPath`.
|
|
5539
6084
|
*
|
|
5540
6085
|
* Calls-API parameters merge the same way:
|
|
5541
6086
|
* 1. `options.callOptions` — per-call overrides
|
|
5542
|
-
* 2. `
|
|
6087
|
+
* 2. `ConversationRelayProviderConfig.defaultCallOptions` — channel-wide
|
|
6088
|
+
* defaults
|
|
5543
6089
|
* 3. Callback URLs derived from `TACConfig.voicePublicDomain` +
|
|
5544
6090
|
* `voiceCallEventPath`, for handlers that are registered
|
|
5545
6091
|
*
|
|
@@ -5549,22 +6095,23 @@ var VoiceChannel = class _VoiceChannel extends BaseChannel {
|
|
|
5549
6095
|
*/
|
|
5550
6096
|
async initiateOutboundConversation(options) {
|
|
5551
6097
|
const validated = InitiateVoiceConversationOptionsSchema.parse(options);
|
|
5552
|
-
const fromNumber = this.
|
|
6098
|
+
const fromNumber = this.tacConfig.phoneNumber;
|
|
5553
6099
|
this.logger.info(
|
|
5554
6100
|
{ to: validated.to, from: fromNumber },
|
|
5555
6101
|
"Initiating outbound voice conversation"
|
|
5556
6102
|
);
|
|
5557
6103
|
try {
|
|
5558
|
-
const
|
|
5559
|
-
|
|
5560
|
-
|
|
6104
|
+
const twiml = this.twimlBuilder.build("initiateOutboundConversation", {
|
|
6105
|
+
perCall: validated.twimlOptions,
|
|
6106
|
+
websocketUrl: validated.websocketUrl
|
|
6107
|
+
});
|
|
5561
6108
|
const callParams = this.buildCallParams(validated.callOptions);
|
|
5562
6109
|
this.logger.debug(
|
|
5563
6110
|
{ twiml: redactTwimlParameters(twiml), to: maskAddress(validated.to) },
|
|
5564
6111
|
"Outbound call TwiML"
|
|
5565
6112
|
);
|
|
5566
|
-
const
|
|
5567
|
-
const call = await
|
|
6113
|
+
const client2 = this.channel.getTwilioClientInternal();
|
|
6114
|
+
const call = await client2.calls.create({
|
|
5568
6115
|
to: validated.to,
|
|
5569
6116
|
from: fromNumber,
|
|
5570
6117
|
twiml,
|
|
@@ -5580,7 +6127,7 @@ var VoiceChannel = class _VoiceChannel extends BaseChannel {
|
|
|
5580
6127
|
{ err: error, to: maskAddress(validated.to) },
|
|
5581
6128
|
"Failed to initiate outbound call"
|
|
5582
6129
|
);
|
|
5583
|
-
this.
|
|
6130
|
+
this.channel.handleErrorInternal(error instanceof Error ? error : new Error(String(error)), {
|
|
5584
6131
|
to: validated.to
|
|
5585
6132
|
});
|
|
5586
6133
|
throw error;
|
|
@@ -5593,412 +6140,2642 @@ var VoiceChannel = class _VoiceChannel extends BaseChannel {
|
|
|
5593
6140
|
* Handle ConversationRelay callback from Twilio. Cleans up on call completion
|
|
5594
6141
|
* in voice-only mode; in orchestrated mode the CO webhook owns cleanup.
|
|
5595
6142
|
*
|
|
5596
|
-
* @param
|
|
6143
|
+
* @param rawPayload - Callback payload from Twilio
|
|
5597
6144
|
* @returns Response with status, content, and content type
|
|
5598
6145
|
*/
|
|
5599
|
-
async
|
|
6146
|
+
async handleTwilioProviderCallback(rawPayload) {
|
|
6147
|
+
const parsed = ConversationRelayCallbackPayloadSchema.safeParse(rawPayload);
|
|
6148
|
+
if (!parsed.success) {
|
|
6149
|
+
this.logger.warn(
|
|
6150
|
+
{ errors: parsed.error.issues },
|
|
6151
|
+
"Invalid ConversationRelay callback payload"
|
|
6152
|
+
);
|
|
6153
|
+
return { status: 400, content: "Invalid payload", contentType: "text/plain" };
|
|
6154
|
+
}
|
|
6155
|
+
const payload = parsed.data;
|
|
5600
6156
|
this.logger.debug(
|
|
5601
6157
|
{ call_sid: payload.CallSid, call_status: payload.CallStatus },
|
|
5602
6158
|
"ConversationRelay callback received"
|
|
5603
6159
|
);
|
|
5604
|
-
if (payload.AccountSid !== this.
|
|
6160
|
+
if (payload.AccountSid !== this.tacConfig.accountSid) {
|
|
5605
6161
|
this.logger.warn(
|
|
5606
|
-
{ expected: this.
|
|
6162
|
+
{ expected: this.tacConfig.accountSid, received: payload.AccountSid },
|
|
5607
6163
|
"ConversationRelay callback AccountSid mismatch, ignoring"
|
|
5608
6164
|
);
|
|
5609
6165
|
return { status: 403, content: "Forbidden", contentType: "text/plain" };
|
|
5610
6166
|
}
|
|
5611
|
-
if (payload.CallStatus === "completed" && !this.
|
|
6167
|
+
if (payload.CallStatus === "completed" && !this.channel.isOrchestratorEnabledInternal()) {
|
|
5612
6168
|
const conversationId = this.callSidToConversationId.get(payload.CallSid);
|
|
5613
6169
|
if (conversationId) {
|
|
5614
6170
|
this.callSidToConversationId.delete(payload.CallSid);
|
|
5615
|
-
await this.
|
|
6171
|
+
await this.channel.endConversationInternal(conversationId);
|
|
5616
6172
|
}
|
|
5617
6173
|
}
|
|
5618
6174
|
return { status: 200, content: "OK", contentType: "text/plain" };
|
|
5619
6175
|
}
|
|
5620
6176
|
// =========================================================================
|
|
5621
|
-
//
|
|
6177
|
+
// ConversationRelay TwiML Generation
|
|
5622
6178
|
// =========================================================================
|
|
5623
6179
|
/**
|
|
5624
|
-
*
|
|
5625
|
-
*
|
|
5626
|
-
* Twilio signature validation already gates the route; this is defense in
|
|
5627
|
-
* depth. A payload with no `AccountSid` is allowed through.
|
|
6180
|
+
* Generate TwiML to connect a call to ConversationRelay.
|
|
6181
|
+
* Validates configuration with Zod before generating TwiML.
|
|
5628
6182
|
*
|
|
5629
|
-
*
|
|
5630
|
-
*
|
|
6183
|
+
* @param config - ConversationRelay configuration (url, transcription, TTS, etc.)
|
|
6184
|
+
* @param options - Optional settings for parameters and the Connect verb
|
|
6185
|
+
* @returns TwiML XML string
|
|
6186
|
+
* @throws {Error} if config validation fails
|
|
5631
6187
|
*/
|
|
5632
|
-
|
|
5633
|
-
const
|
|
5634
|
-
if (
|
|
5635
|
-
|
|
5636
|
-
|
|
5637
|
-
"Call event AccountSid mismatch, ignoring"
|
|
5638
|
-
);
|
|
5639
|
-
return false;
|
|
6188
|
+
connectConversationRelay(config, options) {
|
|
6189
|
+
const validationResult = ConversationRelayConfigSchema.safeParse(config);
|
|
6190
|
+
if (!validationResult.success) {
|
|
6191
|
+
const errorMessage = validationResult.error.issues.map((issue) => `${issue.path.join(".")}: ${issue.message}`).join(", ");
|
|
6192
|
+
throw new Error(`Invalid ConversationRelay configuration: ${errorMessage}`);
|
|
5640
6193
|
}
|
|
5641
|
-
|
|
5642
|
-
|
|
5643
|
-
|
|
5644
|
-
|
|
5645
|
-
|
|
5646
|
-
|
|
5647
|
-
|
|
5648
|
-
|
|
5649
|
-
|
|
5650
|
-
|
|
5651
|
-
|
|
5652
|
-
const ok = { status: 200, content: "OK", contentType: "text/plain" };
|
|
5653
|
-
if (!handler || !this.callEventAccountOk(form)) {
|
|
5654
|
-
return ok;
|
|
6194
|
+
const validatedConfig = validationResult.data;
|
|
6195
|
+
const { languages, ...conversationRelayAttributes } = validatedConfig;
|
|
6196
|
+
const filteredConfig = filterUnsetValues(conversationRelayAttributes);
|
|
6197
|
+
const response = new VoiceResponse();
|
|
6198
|
+
const connect = response.connect(options?.actionUrl ? { action: options.actionUrl } : {});
|
|
6199
|
+
const relay = connect.conversationRelay(filteredConfig);
|
|
6200
|
+
if (languages && languages.length > 0) {
|
|
6201
|
+
for (const lang of languages) {
|
|
6202
|
+
const filteredLang = filterUnsetValues(lang);
|
|
6203
|
+
relay.language(filteredLang);
|
|
6204
|
+
}
|
|
5655
6205
|
}
|
|
5656
|
-
|
|
5657
|
-
const
|
|
5658
|
-
|
|
5659
|
-
|
|
5660
|
-
} catch (error) {
|
|
5661
|
-
this.logger.error({ err: error, kind }, "Failed to process call event callback");
|
|
5662
|
-
return { status: 400, content: "Bad Request", contentType: "text/plain" };
|
|
6206
|
+
if (options?.parameters) {
|
|
6207
|
+
for (const [name, value] of Object.entries(options.parameters)) {
|
|
6208
|
+
relay.parameter({ name, value: String(value) });
|
|
6209
|
+
}
|
|
5663
6210
|
}
|
|
5664
|
-
return
|
|
6211
|
+
return response.toString();
|
|
5665
6212
|
}
|
|
5666
6213
|
/**
|
|
5667
|
-
*
|
|
6214
|
+
* Drop this provider's ConversationRelay transport state on channel shutdown.
|
|
5668
6215
|
*
|
|
5669
|
-
*
|
|
5670
|
-
*
|
|
5671
|
-
* and dispatched to the {@link onCallStatus} handler. No-op if no handler is
|
|
5672
|
-
* registered.
|
|
5673
|
-
*
|
|
5674
|
-
* @param form - Raw form data from the webhook request.
|
|
6216
|
+
* Note: WebSocket connections are managed by the server and closed there.
|
|
6217
|
+
* This method only cleans up internal provider state.
|
|
5675
6218
|
*/
|
|
5676
|
-
|
|
5677
|
-
|
|
5678
|
-
|
|
5679
|
-
|
|
5680
|
-
|
|
5681
|
-
|
|
5682
|
-
|
|
5683
|
-
|
|
6219
|
+
shutdown() {
|
|
6220
|
+
super.shutdown();
|
|
6221
|
+
this.streamTasks.clear();
|
|
6222
|
+
this.webSocketConnections.clear();
|
|
6223
|
+
this.promptQueues.clear();
|
|
6224
|
+
this.initializationRetries.clear();
|
|
6225
|
+
this.callSidToConversationId.clear();
|
|
6226
|
+
}
|
|
6227
|
+
};
|
|
6228
|
+
|
|
6229
|
+
// packages/core/src/channels/voice/conversation-relay/config.ts
|
|
6230
|
+
var ConversationRelayProviderConfig = class extends VoiceProviderConfig {
|
|
6231
|
+
/**
|
|
6232
|
+
* Static `VoiceTwiMLOptionsConversationRelay` for the TwiML inside `<ConversationRelay>`, applied
|
|
6233
|
+
* to every call (inbound and outbound).
|
|
6234
|
+
*/
|
|
6235
|
+
defaultTwimlOptions;
|
|
6236
|
+
/** Static {@link CallOptions} applied to every outbound call. */
|
|
6237
|
+
defaultCallOptions;
|
|
6238
|
+
constructor(options) {
|
|
6239
|
+
super(options);
|
|
6240
|
+
if (options?.defaultTwimlOptions !== void 0) {
|
|
6241
|
+
this.defaultTwimlOptions = options.defaultTwimlOptions;
|
|
6242
|
+
}
|
|
6243
|
+
if (options?.defaultCallOptions !== void 0) {
|
|
6244
|
+
this.defaultCallOptions = options.defaultCallOptions;
|
|
6245
|
+
}
|
|
6246
|
+
}
|
|
6247
|
+
createProvider(channel, tacConfig) {
|
|
6248
|
+
return new ConversationRelayProvider(channel, tacConfig, this);
|
|
6249
|
+
}
|
|
6250
|
+
};
|
|
6251
|
+
|
|
6252
|
+
// packages/core/src/channels/voice/channel.ts
|
|
6253
|
+
var VoiceChannel = class _VoiceChannel extends BaseChannel {
|
|
6254
|
+
provider;
|
|
6255
|
+
voiceCallbacks;
|
|
6256
|
+
twilioClient;
|
|
6257
|
+
onInboundCallTwimlHandler;
|
|
6258
|
+
onCallStatusHandler;
|
|
6259
|
+
onAmdHandler;
|
|
6260
|
+
onRecordingHandler;
|
|
6261
|
+
/**
|
|
6262
|
+
* @param tac - The owning {@link TAC} instance.
|
|
6263
|
+
* @param options - Either a {@link VoiceProviderConfig} selecting the media
|
|
6264
|
+
* provider, or a plain object — shorthand for
|
|
6265
|
+
* {@link ConversationRelayProviderConfig}, which is what TAC builds when no
|
|
6266
|
+
* provider config is given.
|
|
6267
|
+
*/
|
|
6268
|
+
constructor(tac, options) {
|
|
6269
|
+
super(tac, _VoiceChannel.toBaseOptions(options));
|
|
6270
|
+
this.voiceCallbacks = {};
|
|
6271
|
+
this.provider = _VoiceChannel.toProviderConfig(options).createProvider(this, this.config);
|
|
6272
|
+
}
|
|
6273
|
+
/**
|
|
6274
|
+
* The {@link BaseChannelOptions} to hand `BaseChannel`. A provider config
|
|
6275
|
+
* replays the options it retained (with its resolved `memoryMode`, which is
|
|
6276
|
+
* writable after construction); a plain options object is passed through. Both
|
|
6277
|
+
* paths therefore honour `dedupCapacity` and friends identically.
|
|
6278
|
+
*/
|
|
6279
|
+
static toBaseOptions(options) {
|
|
6280
|
+
if (options === void 0) {
|
|
6281
|
+
return void 0;
|
|
6282
|
+
}
|
|
6283
|
+
if (options instanceof VoiceProviderConfig) {
|
|
6284
|
+
return { ...options.channelOptions, memoryMode: options.memoryMode };
|
|
6285
|
+
}
|
|
6286
|
+
return options;
|
|
6287
|
+
}
|
|
6288
|
+
/**
|
|
6289
|
+
* Resolve the provider config: an explicit {@link VoiceProviderConfig} as-is,
|
|
6290
|
+
* anything else wrapped as a {@link ConversationRelayProviderConfig}.
|
|
6291
|
+
*/
|
|
6292
|
+
static toProviderConfig(options) {
|
|
6293
|
+
return options instanceof VoiceProviderConfig ? options : new ConversationRelayProviderConfig(options);
|
|
6294
|
+
}
|
|
6295
|
+
/**
|
|
6296
|
+
* Register a callback that produces per-call overrides for the TwiML inside
|
|
6297
|
+
* `<ConversationRelay>` on inbound calls.
|
|
6298
|
+
*
|
|
6299
|
+
* The callback receives a framework-neutral {@link TwiMLRequest} (parsed from
|
|
6300
|
+
* the Twilio webhook form) and returns
|
|
6301
|
+
* {@link VoiceTwiMLOptionsConversationRelay}. Fields the
|
|
6302
|
+
* callback explicitly sets override `defaultTwimlOptions` and TAC defaults;
|
|
6303
|
+
* unset fields fall through.
|
|
6304
|
+
*
|
|
6305
|
+
* @example
|
|
6306
|
+
* ```typescript
|
|
6307
|
+
* voiceChannel.onInboundCallTwiml(async req => {
|
|
6308
|
+
* if (req.callerCountry === 'MX') {
|
|
6309
|
+
* return { language: 'es-MX', welcomeGreeting: '¡Hola!' };
|
|
6310
|
+
* }
|
|
6311
|
+
* return {};
|
|
6312
|
+
* });
|
|
6313
|
+
* ```
|
|
6314
|
+
*
|
|
6315
|
+
* Outbound calls don't use this — pass per-call TwiML via
|
|
6316
|
+
* `InitiateVoiceConversationOptions.twimlOptions` directly.
|
|
6317
|
+
*/
|
|
6318
|
+
onInboundCallTwiml(callback) {
|
|
6319
|
+
this.onInboundCallTwimlHandler = callback;
|
|
6320
|
+
}
|
|
6321
|
+
/**
|
|
6322
|
+
* Register a handler for Twilio `statusCallback` webhooks.
|
|
6323
|
+
*
|
|
6324
|
+
* This is the Calls-API status callback (call disposition), not the
|
|
6325
|
+
* ConversationRelay session callback — see
|
|
6326
|
+
* {@link handleTwilioProviderCallback}.
|
|
6327
|
+
*
|
|
6328
|
+
* Registering does two things: it stores the handler, and it makes later
|
|
6329
|
+
* outbound calls pass `statusCallback` to `calls.create`. With no handler
|
|
6330
|
+
* registered TAC omits that parameter, so Twilio has nowhere to post and the
|
|
6331
|
+
* event never arrives.
|
|
6332
|
+
*
|
|
6333
|
+
* Twilio reports only the terminal event by default, which covers every
|
|
6334
|
+
* disposition; set `CallOptions.statusCallbackEvent` for ringing/answered.
|
|
6335
|
+
*
|
|
6336
|
+
* @example
|
|
6337
|
+
* ```typescript
|
|
6338
|
+
* voiceChannel.onCallStatus(async event => {
|
|
6339
|
+
* if (event.isUnreached) {
|
|
6340
|
+
* // queue a retry
|
|
6341
|
+
* }
|
|
6342
|
+
* });
|
|
6343
|
+
* ```
|
|
6344
|
+
*/
|
|
6345
|
+
onCallStatus(callback) {
|
|
6346
|
+
this.onCallStatusHandler = callback;
|
|
6347
|
+
}
|
|
6348
|
+
/**
|
|
6349
|
+
* Register a handler for Twilio `asyncAmdStatusCallback` webhooks.
|
|
6350
|
+
*
|
|
6351
|
+
* Registering makes later outbound calls pass `asyncAmdStatusCallback` to
|
|
6352
|
+
* `calls.create`; without a handler TAC omits it and Twilio has nowhere to
|
|
6353
|
+
* post the result. It does not enable detection — that's per-call, via
|
|
6354
|
+
* `CallOptions.machineDetection` and `asyncAmd`, both of which are required
|
|
6355
|
+
* for this to fire (at most once per call).
|
|
6356
|
+
*
|
|
6357
|
+
* @example
|
|
6358
|
+
* ```typescript
|
|
6359
|
+
* voiceChannel.onAmd(async event => {
|
|
6360
|
+
* if (event.isMachine) {
|
|
6361
|
+
* await voiceChannel.endCall(event.callSid); // voicemail → hang up
|
|
6362
|
+
* }
|
|
6363
|
+
* });
|
|
6364
|
+
* ```
|
|
6365
|
+
*/
|
|
6366
|
+
onAmd(callback) {
|
|
6367
|
+
this.onAmdHandler = callback;
|
|
6368
|
+
}
|
|
6369
|
+
/**
|
|
6370
|
+
* Register a handler for Twilio `recordingStatusCallback` webhooks.
|
|
6371
|
+
*
|
|
6372
|
+
* Registering makes later outbound calls pass `recordingStatusCallback` to
|
|
6373
|
+
* `calls.create`; without a handler TAC omits it and Twilio has nowhere to
|
|
6374
|
+
* post. It does not start recording — that's `CallOptions.record`, which is
|
|
6375
|
+
* required for this to fire.
|
|
6376
|
+
*
|
|
6377
|
+
* @example
|
|
6378
|
+
* ```typescript
|
|
6379
|
+
* voiceChannel.onRecording(async event => {
|
|
6380
|
+
* if (event.recordingStatus === 'completed') {
|
|
6381
|
+
* // store event.recordingUrl
|
|
6382
|
+
* }
|
|
6383
|
+
* });
|
|
6384
|
+
* ```
|
|
6385
|
+
*/
|
|
6386
|
+
onRecording(callback) {
|
|
6387
|
+
this.onRecordingHandler = callback;
|
|
6388
|
+
}
|
|
6389
|
+
/**
|
|
6390
|
+
* Register a handler for DTMF keypresses, called once per key in order.
|
|
6391
|
+
*
|
|
6392
|
+
* Requires `dtmfDetection: true` on the ConversationRelay config — without it
|
|
6393
|
+
* Twilio sends nothing and this never fires. Digits aren't buffered, so
|
|
6394
|
+
* accumulating a multi-digit entry is the handler's job.
|
|
6395
|
+
*
|
|
6396
|
+
* A keypress initializes the conversation just as a prompt does, since a
|
|
6397
|
+
* caller can type without ever speaking; if that fails the digit still
|
|
6398
|
+
* arrives, with `conversationId` and `session` undefined. Keypresses don't
|
|
6399
|
+
* cancel in-flight streaming on their own — that's a separate `interrupt`
|
|
6400
|
+
* message, sent when `interruptible` includes `dtmf`.
|
|
6401
|
+
*
|
|
6402
|
+
* @example
|
|
6403
|
+
* ```typescript
|
|
6404
|
+
* const digits = new Map<string, string>();
|
|
6405
|
+
*
|
|
6406
|
+
* voiceChannel.onDtmf(({ conversationId, digit }) => {
|
|
6407
|
+
* if (!conversationId) return;
|
|
6408
|
+
* digits.set(conversationId, (digits.get(conversationId) ?? '') + digit);
|
|
6409
|
+
* });
|
|
6410
|
+
* ```
|
|
6411
|
+
*/
|
|
6412
|
+
onDtmf(callback) {
|
|
6413
|
+
this.voiceCallbacks.onDtmf = callback;
|
|
6414
|
+
}
|
|
6415
|
+
// =========================================================================
|
|
6416
|
+
// Provider-facing surface
|
|
6417
|
+
//
|
|
6418
|
+
// `BaseChannel`'s state is `protected`, and a `VoiceProvider` is not a
|
|
6419
|
+
// subclass of `VoiceChannel`, so relocated transport logic genuinely cannot
|
|
6420
|
+
// reach it. These forwarders open exactly what a provider needs and nothing
|
|
6421
|
+
// more; they are `@internal` and are not exported from the package root.
|
|
6422
|
+
// =========================================================================
|
|
6423
|
+
/**
|
|
6424
|
+
* The registered call-event handlers, for a `VoiceProvider` deciding which
|
|
6425
|
+
* callback URLs to derive. A provider is not a subclass of `VoiceChannel`,
|
|
6426
|
+
* so the `private` fields are genuinely out of reach without this.
|
|
6427
|
+
*
|
|
6428
|
+
* @internal
|
|
6429
|
+
*/
|
|
6430
|
+
getCallEventHandlers() {
|
|
6431
|
+
return {
|
|
6432
|
+
status: this.onCallStatusHandler,
|
|
6433
|
+
amd: this.onAmdHandler,
|
|
6434
|
+
recording: this.onRecordingHandler
|
|
6435
|
+
};
|
|
6436
|
+
}
|
|
6437
|
+
/**
|
|
6438
|
+
* This channel's `TACConfig`, for a `VoiceProvider` deriving default URLs.
|
|
6439
|
+
* `BaseChannel.config` is `protected`, and a provider is not a subclass.
|
|
6440
|
+
*
|
|
6441
|
+
* @internal
|
|
6442
|
+
*/
|
|
6443
|
+
getTacConfig() {
|
|
6444
|
+
return this.config;
|
|
6445
|
+
}
|
|
6446
|
+
/**
|
|
6447
|
+
* This channel's logger, so a provider's relocated logic keeps logging under
|
|
6448
|
+
* the same name it did when it lived on `VoiceChannel`.
|
|
6449
|
+
*
|
|
6450
|
+
* @internal
|
|
6451
|
+
*/
|
|
6452
|
+
getLoggerInternal() {
|
|
6453
|
+
return this.logger;
|
|
6454
|
+
}
|
|
6455
|
+
/**
|
|
6456
|
+
* The Conversation Orchestrator client, or `null` in ConversationRelay-only
|
|
6457
|
+
* mode.
|
|
6458
|
+
*
|
|
6459
|
+
* @internal
|
|
6460
|
+
*/
|
|
6461
|
+
getConversationClientInternal() {
|
|
6462
|
+
return this.conversationClient;
|
|
6463
|
+
}
|
|
6464
|
+
/**
|
|
6465
|
+
* The lazily built Twilio REST client, for a provider placing outbound calls.
|
|
6466
|
+
*
|
|
6467
|
+
* @internal
|
|
6468
|
+
*/
|
|
6469
|
+
getTwilioClientInternal() {
|
|
6470
|
+
return this.getTwilioClient();
|
|
6471
|
+
}
|
|
6472
|
+
/**
|
|
6473
|
+
* Whether Conversation Orchestrator is configured. Providers branch on this
|
|
6474
|
+
* to decide who owns conversation cleanup.
|
|
6475
|
+
*
|
|
6476
|
+
* @internal
|
|
6477
|
+
*/
|
|
6478
|
+
isOrchestratorEnabledInternal() {
|
|
6479
|
+
return this.tac.isOrchestratorEnabled();
|
|
6480
|
+
}
|
|
6481
|
+
/**
|
|
6482
|
+
* Voice event callbacks registered via {@link on}, for a provider to fire.
|
|
6483
|
+
*
|
|
6484
|
+
* @internal
|
|
6485
|
+
*/
|
|
6486
|
+
getVoiceCallbacks() {
|
|
6487
|
+
return this.voiceCallbacks;
|
|
6488
|
+
}
|
|
6489
|
+
/**
|
|
6490
|
+
* The inbound-TwiML customizer registered via {@link onInboundCallTwiml}, for
|
|
6491
|
+
* a provider building the inbound response.
|
|
6492
|
+
*
|
|
6493
|
+
* @internal
|
|
6494
|
+
*/
|
|
6495
|
+
getInboundCallTwimlHandler() {
|
|
6496
|
+
return this.onInboundCallTwimlHandler;
|
|
6497
|
+
}
|
|
6498
|
+
/**
|
|
6499
|
+
* Start tracking a conversation session. Forwards to
|
|
6500
|
+
* `BaseChannel.startConversation`.
|
|
6501
|
+
*
|
|
6502
|
+
* @internal
|
|
6503
|
+
*/
|
|
6504
|
+
startConversationInternal(conversationId, profileId) {
|
|
6505
|
+
return this.startConversation(conversationId, profileId);
|
|
6506
|
+
}
|
|
6507
|
+
/**
|
|
6508
|
+
* End a tracked conversation session. Forwards to
|
|
6509
|
+
* `BaseChannel.endConversation`.
|
|
6510
|
+
*
|
|
6511
|
+
* @internal
|
|
6512
|
+
*/
|
|
6513
|
+
endConversationInternal(conversationId) {
|
|
6514
|
+
return this.endConversation(conversationId);
|
|
6515
|
+
}
|
|
6516
|
+
/**
|
|
6517
|
+
* Retrieve memory when `memoryMode` calls for it. Forwards to
|
|
6518
|
+
* `BaseChannel.retrieveMemoryIfEnabled`.
|
|
6519
|
+
*
|
|
6520
|
+
* @internal
|
|
6521
|
+
*/
|
|
6522
|
+
retrieveMemoryInternal(session, query) {
|
|
6523
|
+
return this.retrieveMemoryIfEnabled(session, query);
|
|
6524
|
+
}
|
|
6525
|
+
/**
|
|
6526
|
+
* Report an error through the channel's `onError` callback and logger.
|
|
6527
|
+
* Forwards to `BaseChannel.handleError`.
|
|
6528
|
+
*
|
|
6529
|
+
* @internal
|
|
6530
|
+
*/
|
|
6531
|
+
handleErrorInternal(error, context) {
|
|
6532
|
+
this.handleError(error, context);
|
|
6533
|
+
}
|
|
6534
|
+
getTwilioClient() {
|
|
6535
|
+
if (!this.twilioClient) {
|
|
6536
|
+
this.twilioClient = twilio(this.config.apiKey, this.config.apiSecret, {
|
|
6537
|
+
accountSid: this.config.accountSid
|
|
6538
|
+
});
|
|
6539
|
+
}
|
|
6540
|
+
return this.twilioClient;
|
|
6541
|
+
}
|
|
6542
|
+
get channelType() {
|
|
6543
|
+
return "voice";
|
|
6544
|
+
}
|
|
6545
|
+
/**
|
|
6546
|
+
* Register event callbacks (override for Voice-specific events)
|
|
6547
|
+
*/
|
|
6548
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- Generic event callback needs to accept any args
|
|
6549
|
+
on(event, callback) {
|
|
6550
|
+
switch (event) {
|
|
6551
|
+
case "setup":
|
|
6552
|
+
this.voiceCallbacks.onSetup = callback;
|
|
6553
|
+
break;
|
|
6554
|
+
case "prompt":
|
|
6555
|
+
this.voiceCallbacks.onPrompt = callback;
|
|
6556
|
+
break;
|
|
6557
|
+
case "interrupt":
|
|
6558
|
+
this.voiceCallbacks.onInterrupt = callback;
|
|
6559
|
+
break;
|
|
6560
|
+
case "dtmf":
|
|
6561
|
+
this.voiceCallbacks.onDtmf = callback;
|
|
6562
|
+
break;
|
|
6563
|
+
case "webSocketConnected":
|
|
6564
|
+
this.voiceCallbacks.onWebSocketConnected = callback;
|
|
6565
|
+
break;
|
|
6566
|
+
case "webSocketDisconnected":
|
|
6567
|
+
this.voiceCallbacks.onWebSocketDisconnected = callback;
|
|
6568
|
+
break;
|
|
6569
|
+
default:
|
|
6570
|
+
super.on(event, callback);
|
|
6571
|
+
break;
|
|
6572
|
+
}
|
|
6573
|
+
}
|
|
6574
|
+
/**
|
|
6575
|
+
* Process conversation webhooks for cleanup.
|
|
6576
|
+
*
|
|
6577
|
+
* Voice channel processes CONVERSATION_UPDATED events:
|
|
6578
|
+
* - CLOSED status: Clean up local session state
|
|
6579
|
+
*
|
|
6580
|
+
* Note: Conversation tracking uses instance-local memory. In multi-instance
|
|
6581
|
+
* deployments, webhooks may route to a different instance, preventing cleanup.
|
|
6582
|
+
*
|
|
6583
|
+
* @param payload - Raw webhook event data from Twilio
|
|
6584
|
+
* @param idempotencyToken - Optional Twilio idempotency token from request headers
|
|
6585
|
+
*/
|
|
6586
|
+
async processWebhook(payload, idempotencyToken) {
|
|
6587
|
+
try {
|
|
6588
|
+
const result = this.preprocessWebhook(payload, idempotencyToken);
|
|
6589
|
+
if (!result) {
|
|
6590
|
+
return;
|
|
6591
|
+
}
|
|
6592
|
+
const { webhookData, eventType, conversationId } = result;
|
|
6593
|
+
switch (eventType) {
|
|
6594
|
+
case "CONVERSATION_UPDATED":
|
|
6595
|
+
this.logger.debug(
|
|
6596
|
+
{ conversation_id: conversationId, status: webhookData.data?.status },
|
|
6597
|
+
"Handling CONVERSATION_UPDATED"
|
|
6598
|
+
);
|
|
6599
|
+
await this.handleConversationUpdated(webhookData);
|
|
6600
|
+
break;
|
|
6601
|
+
default:
|
|
6602
|
+
this.logger.debug(
|
|
6603
|
+
{
|
|
6604
|
+
event_type: eventType,
|
|
6605
|
+
raw_event_type: webhookData.eventType,
|
|
6606
|
+
conversation_id: conversationId
|
|
6607
|
+
},
|
|
6608
|
+
"Unhandled event type - this event will be ignored"
|
|
6609
|
+
);
|
|
6610
|
+
}
|
|
6611
|
+
this.logger.debug({ event_type: eventType }, "Webhook processing completed");
|
|
6612
|
+
} catch (error) {
|
|
6613
|
+
if (idempotencyToken) {
|
|
6614
|
+
this.removeWebhookToken(idempotencyToken);
|
|
6615
|
+
}
|
|
6616
|
+
this.handleError(error instanceof Error ? error : new Error(String(error)), { payload });
|
|
6617
|
+
}
|
|
6618
|
+
}
|
|
6619
|
+
/**
|
|
6620
|
+
* Handle conversation updated event
|
|
6621
|
+
*/
|
|
6622
|
+
async handleConversationUpdated(payload) {
|
|
6623
|
+
const conversationId = this.extractConversationId(payload);
|
|
6624
|
+
if (!conversationId) {
|
|
6625
|
+
throw new Error("Missing conversation ID in conversation.updated event");
|
|
6626
|
+
}
|
|
6627
|
+
if (payload.data?.status === "CLOSED") {
|
|
6628
|
+
this.logger.debug(
|
|
6629
|
+
{ conversation_id: conversationId, status: payload.data.status },
|
|
6630
|
+
"Conversation closed, cleaning up"
|
|
6631
|
+
);
|
|
6632
|
+
await this.endConversation(conversationId);
|
|
6633
|
+
} else if (payload.data?.status === "INACTIVE") {
|
|
6634
|
+
this.invalidateCachedMemory(conversationId);
|
|
6635
|
+
}
|
|
6636
|
+
}
|
|
6637
|
+
/**
|
|
6638
|
+
* Get active WebSocket connection for a conversation
|
|
6639
|
+
*/
|
|
6640
|
+
getWebsocket(conversationId) {
|
|
6641
|
+
return this.provider.getWebSocket(conversationId);
|
|
6642
|
+
}
|
|
6643
|
+
/**
|
|
6644
|
+
* Hand one WebSocket connection to the active provider, which drives its
|
|
6645
|
+
* lifecycle from accept to disconnect.
|
|
6646
|
+
*
|
|
6647
|
+
* @param ws - The accepted WebSocket, from ConversationRelay or whatever
|
|
6648
|
+
* transport the active provider serves.
|
|
6649
|
+
*/
|
|
6650
|
+
handleWebSocketConnection(ws) {
|
|
6651
|
+
trackEvent("Websocket Connected", {
|
|
6652
|
+
account_sid: this.config.accountSid,
|
|
6653
|
+
channel: "voice",
|
|
6654
|
+
provider: this.provider.providerId,
|
|
6655
|
+
orchestrator_enabled: this.config.isOrchestratorEnabled()
|
|
6656
|
+
});
|
|
6657
|
+
const result = this.provider.handleWebSocket(ws);
|
|
6658
|
+
if (result instanceof Promise) {
|
|
6659
|
+
void result.catch((err) => {
|
|
6660
|
+
this.logger.error({ err }, "WebSocket handler error");
|
|
6661
|
+
});
|
|
6662
|
+
}
|
|
6663
|
+
}
|
|
6664
|
+
/**
|
|
6665
|
+
* Send voice response via WebSocket
|
|
6666
|
+
*/
|
|
6667
|
+
async sendResponse(conversationId, message, metadata) {
|
|
6668
|
+
await this.provider.sendResponse(conversationId, message, metadata);
|
|
6669
|
+
trackEvent("Response Sent", {
|
|
6670
|
+
account_sid: this.config.accountSid,
|
|
6671
|
+
channel: "voice",
|
|
6672
|
+
conversation_id: conversationId,
|
|
6673
|
+
response_type: "full",
|
|
6674
|
+
provider: this.provider.providerId,
|
|
6675
|
+
orchestrator_enabled: this.config.isOrchestratorEnabled()
|
|
6676
|
+
});
|
|
6677
|
+
}
|
|
6678
|
+
/**
|
|
6679
|
+
* Send a streaming voice response through the active provider's transport,
|
|
6680
|
+
* token by token. Delegates to the active provider — see
|
|
6681
|
+
* {@link ConversationRelayProvider.sendStreamingResponse} for the token
|
|
6682
|
+
* protocol and abort semantics.
|
|
6683
|
+
*
|
|
6684
|
+
* @param conversationId - Conversation whose transport receives the tokens.
|
|
6685
|
+
* @param stream - Async iterable of text chunks to relay as they arrive.
|
|
6686
|
+
* @param options - Additional per-call inputs.
|
|
6687
|
+
* @param options.signal - Aborts the stream mid-flight, e.g. when the caller
|
|
6688
|
+
* interrupts.
|
|
6689
|
+
* @returns The accumulated full response text.
|
|
6690
|
+
*/
|
|
6691
|
+
async sendStreamingResponse(conversationId, stream, options) {
|
|
6692
|
+
const fullResponse = await this.provider.sendStreamingResponse(conversationId, stream, options);
|
|
6693
|
+
if (fullResponse) {
|
|
6694
|
+
trackEvent("Response Sent", {
|
|
6695
|
+
account_sid: this.config.accountSid,
|
|
6696
|
+
channel: "voice",
|
|
6697
|
+
conversation_id: conversationId,
|
|
6698
|
+
response_type: "streaming",
|
|
6699
|
+
provider: this.provider.providerId,
|
|
6700
|
+
orchestrator_enabled: this.config.isOrchestratorEnabled()
|
|
6701
|
+
});
|
|
6702
|
+
}
|
|
6703
|
+
return fullResponse;
|
|
6704
|
+
}
|
|
6705
|
+
// =========================================================================
|
|
6706
|
+
// Incoming Call Handling
|
|
6707
|
+
// =========================================================================
|
|
6708
|
+
/**
|
|
6709
|
+
* Generate the response for an incoming voice call. Delegates to the active
|
|
6710
|
+
* provider — see {@link ConversationRelayProvider.handleIncomingCall} for the
|
|
6711
|
+
* full TwiML merge/precedence rules (only meaningful for that provider; a
|
|
6712
|
+
* provider with no inbound story declines instead).
|
|
6713
|
+
*
|
|
6714
|
+
* @param twimlRequest - Parsed Twilio webhook fields. Passed to the customizer
|
|
6715
|
+
* registered via {@link onInboundCallTwiml}, if one is configured.
|
|
6716
|
+
* @param options - Additional per-call inputs.
|
|
6717
|
+
* @param options.hostTwimlOptions - Per-call TwiML supplied by a custom
|
|
6718
|
+
* in-process host (e.g. an affinity-routed deployment injecting a per-call
|
|
6719
|
+
* `websocketUrl`). Typed against the ConversationRelay subtype for the same
|
|
6720
|
+
* reason as {@link InboundCallTwimlHandler}.
|
|
6721
|
+
* @returns TwiML XML string for call connection.
|
|
6722
|
+
*/
|
|
6723
|
+
async handleIncomingCall(twimlRequest, options) {
|
|
6724
|
+
return this.provider.handleIncomingCall(twimlRequest, options);
|
|
6725
|
+
}
|
|
6726
|
+
// =========================================================================
|
|
6727
|
+
// Outbound Call Handling
|
|
6728
|
+
// =========================================================================
|
|
6729
|
+
/**
|
|
6730
|
+
* Initiate an outbound voice conversation. Delegates to the active provider —
|
|
6731
|
+
* see {@link ConversationRelayProvider.initiateOutboundConversation} for the
|
|
6732
|
+
* TwiML and Calls-API merge/precedence rules.
|
|
6733
|
+
*
|
|
6734
|
+
* {@link ConversationRelayProvider} and `OpenAIRealtimeProvider` place
|
|
6735
|
+
* outbound calls; any other provider declines.
|
|
6736
|
+
*
|
|
6737
|
+
* @param options - Destination, per-call TwiML and Calls-API overrides.
|
|
6738
|
+
* `OpenAIRealtimeProvider` additionally accepts a per-call `sessionConfig`.
|
|
6739
|
+
* Each provider validates against its own schema and rejects the other's.
|
|
6740
|
+
* @returns The placed call's `callSid`.
|
|
6741
|
+
*/
|
|
6742
|
+
async initiateOutboundConversation(options) {
|
|
6743
|
+
return this.provider.initiateOutboundConversation(options);
|
|
6744
|
+
}
|
|
6745
|
+
// =========================================================================
|
|
6746
|
+
// Provider Callback Handling
|
|
6747
|
+
// =========================================================================
|
|
6748
|
+
/**
|
|
6749
|
+
* Handle the provider's own out-of-band lifecycle webhook from Twilio.
|
|
6750
|
+
*
|
|
6751
|
+
* Not every provider has one; those that don't inherit a plain 200
|
|
6752
|
+
* acknowledgement. ConversationRelay posts here when a session ends, and
|
|
6753
|
+
* cleans up on call completion in voice-only mode — in orchestrated mode the
|
|
6754
|
+
* CO webhook owns cleanup.
|
|
6755
|
+
*
|
|
6756
|
+
* @param payload - Raw callback payload from Twilio; the provider validates it.
|
|
6757
|
+
* @returns Response with status, content, and content type
|
|
6758
|
+
*/
|
|
6759
|
+
async handleTwilioProviderCallback(payload) {
|
|
6760
|
+
return this.provider.handleTwilioProviderCallback(payload);
|
|
6761
|
+
}
|
|
6762
|
+
/**
|
|
6763
|
+
* @deprecated Use {@link VoiceChannel.handleTwilioProviderCallback} instead.
|
|
6764
|
+
*/
|
|
6765
|
+
async handleConversationRelayCallback(payload) {
|
|
6766
|
+
warnDeprecated("handleConversationRelayCallback", "handleTwilioProviderCallback");
|
|
6767
|
+
return this.handleTwilioProviderCallback(payload);
|
|
6768
|
+
}
|
|
6769
|
+
// =========================================================================
|
|
6770
|
+
// Call Event Handling (status callback, async AMD, recording)
|
|
6771
|
+
// =========================================================================
|
|
6772
|
+
/**
|
|
6773
|
+
* Whether a call-webhook payload belongs to the configured account.
|
|
6774
|
+
*
|
|
6775
|
+
* Twilio signature validation already gates the route; this is defense in
|
|
6776
|
+
* depth. A payload with no `AccountSid` is allowed through.
|
|
6777
|
+
*
|
|
6778
|
+
* Subaccounts: events carry the SID the call was placed on, so configure TAC
|
|
6779
|
+
* with that account or its events get dropped here.
|
|
6780
|
+
*/
|
|
6781
|
+
callEventAccountOk(form) {
|
|
6782
|
+
const accountSid = form["AccountSid"];
|
|
6783
|
+
if (accountSid && accountSid !== this.config.accountSid) {
|
|
6784
|
+
this.logger.warn(
|
|
6785
|
+
{ expected: this.config.accountSid, received: accountSid },
|
|
6786
|
+
"Call event AccountSid mismatch, ignoring"
|
|
6787
|
+
);
|
|
6788
|
+
return false;
|
|
6789
|
+
}
|
|
6790
|
+
return true;
|
|
6791
|
+
}
|
|
6792
|
+
/**
|
|
6793
|
+
* Parse a call-event webhook form and dispatch it to its handler.
|
|
6794
|
+
*
|
|
6795
|
+
* Returns 400 when the payload can't be parsed (no `CallSid`) or the handler
|
|
6796
|
+
* throws — better than handing Twilio a 200 for an event that wasn't
|
|
6797
|
+
* processed. Everything else, including no handler registered and an
|
|
6798
|
+
* account mismatch, is a 200 no-op.
|
|
6799
|
+
*/
|
|
6800
|
+
async dispatchCallEvent(kind, form, handler, parse, logFields) {
|
|
6801
|
+
const ok = { status: 200, content: "OK", contentType: "text/plain" };
|
|
6802
|
+
if (!handler || !this.callEventAccountOk(form)) {
|
|
6803
|
+
return ok;
|
|
6804
|
+
}
|
|
6805
|
+
try {
|
|
6806
|
+
const event = parse(form);
|
|
6807
|
+
this.logger.debug(logFields(event), `Call ${kind} event received`);
|
|
6808
|
+
await handler(event);
|
|
6809
|
+
} catch (error) {
|
|
6810
|
+
this.logger.error({ err: error, kind }, "Failed to process call event callback");
|
|
6811
|
+
return { status: 400, content: "Bad Request", contentType: "text/plain" };
|
|
6812
|
+
}
|
|
6813
|
+
return ok;
|
|
6814
|
+
}
|
|
6815
|
+
/**
|
|
6816
|
+
* Handle a Twilio `statusCallback` webhook.
|
|
6817
|
+
*
|
|
6818
|
+
* The developer routes the request here (`TACServer` does this automatically
|
|
6819
|
+
* for its `/status` call-event route). Parsed into a {@link CallStatusEvent}
|
|
6820
|
+
* and dispatched to the {@link onCallStatus} handler. No-op if no handler is
|
|
6821
|
+
* registered.
|
|
6822
|
+
*
|
|
6823
|
+
* @param form - Raw form data from the webhook request.
|
|
6824
|
+
*/
|
|
6825
|
+
async handleCallStatusEvent(form) {
|
|
6826
|
+
return this.dispatchCallEvent(
|
|
6827
|
+
"status",
|
|
6828
|
+
form,
|
|
6829
|
+
this.onCallStatusHandler,
|
|
6830
|
+
callStatusEventFromForm,
|
|
6831
|
+
(event) => ({ call_sid: event.callSid, call_status: event.callStatus })
|
|
6832
|
+
);
|
|
5684
6833
|
}
|
|
5685
6834
|
/**
|
|
5686
6835
|
* Handle a Twilio `asyncAmdStatusCallback` webhook.
|
|
5687
6836
|
*
|
|
5688
|
-
* The developer routes the request here (`TACServer` does this automatically
|
|
5689
|
-
* for its `/amd` call-event route). Parsed into an {@link AmdEvent} and
|
|
5690
|
-
* dispatched to the {@link onAmd} handler. No-op if no handler is registered.
|
|
6837
|
+
* The developer routes the request here (`TACServer` does this automatically
|
|
6838
|
+
* for its `/amd` call-event route). Parsed into an {@link AmdEvent} and
|
|
6839
|
+
* dispatched to the {@link onAmd} handler. No-op if no handler is registered.
|
|
6840
|
+
*
|
|
6841
|
+
* @param form - Raw form data from the webhook request.
|
|
6842
|
+
*/
|
|
6843
|
+
async handleAmdEvent(form) {
|
|
6844
|
+
return this.dispatchCallEvent("amd", form, this.onAmdHandler, amdEventFromForm, (event) => ({
|
|
6845
|
+
call_sid: event.callSid,
|
|
6846
|
+
answered_by: event.answeredBy
|
|
6847
|
+
}));
|
|
6848
|
+
}
|
|
6849
|
+
/**
|
|
6850
|
+
* Handle a Twilio `recordingStatusCallback` webhook.
|
|
6851
|
+
*
|
|
6852
|
+
* The developer routes the request here (`TACServer` does this automatically
|
|
6853
|
+
* for its `/recording` call-event route). Parsed into a
|
|
6854
|
+
* {@link RecordingEvent} and dispatched to the {@link onRecording} handler.
|
|
6855
|
+
* No-op if no handler is registered.
|
|
6856
|
+
*
|
|
6857
|
+
* @param form - Raw form data from the webhook request.
|
|
6858
|
+
*/
|
|
6859
|
+
async handleRecordingEvent(form) {
|
|
6860
|
+
return this.dispatchCallEvent(
|
|
6861
|
+
"recording",
|
|
6862
|
+
form,
|
|
6863
|
+
this.onRecordingHandler,
|
|
6864
|
+
recordingEventFromForm,
|
|
6865
|
+
(event) => ({ call_sid: event.callSid, recording_status: event.recordingStatus })
|
|
6866
|
+
);
|
|
6867
|
+
}
|
|
6868
|
+
/**
|
|
6869
|
+
* Hang up a call and clean up its ConversationRelay session.
|
|
6870
|
+
*
|
|
6871
|
+
* Works on `callSid` alone, whether or not a session exists yet. No-ops the
|
|
6872
|
+
* session cleanup if none is tracked.
|
|
6873
|
+
*
|
|
6874
|
+
* Does not throw — hanging up an already-ended call is routine (the callee
|
|
6875
|
+
* hangs up while AMD is still resolving), and handlers shouldn't have to
|
|
6876
|
+
* guard against it.
|
|
6877
|
+
*
|
|
6878
|
+
* @param callSid - Twilio Call SID (from a call event, the outbound result, or
|
|
6879
|
+
* `ConversationSession.callSid`).
|
|
6880
|
+
* @returns True if Twilio accepted the hangup, false if it failed (logged).
|
|
6881
|
+
* Session cleanup runs either way.
|
|
6882
|
+
*/
|
|
6883
|
+
async endCall(callSid) {
|
|
6884
|
+
const client2 = this.getTwilioClient();
|
|
6885
|
+
let hungUp = true;
|
|
6886
|
+
try {
|
|
6887
|
+
await client2.calls(callSid).update({ status: "completed" });
|
|
6888
|
+
} catch (error) {
|
|
6889
|
+
hungUp = false;
|
|
6890
|
+
this.logger.error({ err: error, call_sid: callSid }, "Failed to hang up call");
|
|
6891
|
+
}
|
|
6892
|
+
const session = this.getConversationSessionByCallSid(callSid);
|
|
6893
|
+
if (session) {
|
|
6894
|
+
await this.endConversation(session.conversationId);
|
|
6895
|
+
}
|
|
6896
|
+
return hungUp;
|
|
6897
|
+
}
|
|
6898
|
+
/**
|
|
6899
|
+
* Look up the active voice session for a Twilio Call SID.
|
|
6900
|
+
*
|
|
6901
|
+
* Out-of-band code holding a CallSid — a dashboard route, an operator action,
|
|
6902
|
+
* a call-event handler — can't reach the session-facing methods, which are
|
|
6903
|
+
* keyed by conversation id: the Orchestrator conversation id in orchestrator
|
|
6904
|
+
* mode, the CallSid only in ConversationRelay-only mode.
|
|
6905
|
+
*
|
|
6906
|
+
* Relay-only mode creates the session on the first prompt; orchestrated
|
|
6907
|
+
* mode creates it when the lookup started at setup finishes, so it may
|
|
6908
|
+
* exist before the caller speaks — including before `onAmd` fires. Treat it
|
|
6909
|
+
* as racy and hang up with {@link endCall}, which needs no session.
|
|
6910
|
+
*
|
|
6911
|
+
* At the other end, orchestrator mode keeps the session until Conversation
|
|
6912
|
+
* Orchestrator's CLOSED webhook, so it outlives the call and `onCallStatus` /
|
|
6913
|
+
* `onRecording` do resolve. Relay-only mode tears down on the
|
|
6914
|
+
* ConversationRelay callback instead, which races them.
|
|
6915
|
+
*
|
|
6916
|
+
* @example
|
|
6917
|
+
* ```typescript
|
|
6918
|
+
* async function nudge(callSid: string): Promise<void> {
|
|
6919
|
+
* const session = voiceChannel.getConversationSessionByCallSid(callSid);
|
|
6920
|
+
* if (session) {
|
|
6921
|
+
* await voiceChannel.sendResponse(session.conversationId, 'Still there?');
|
|
6922
|
+
* }
|
|
6923
|
+
* }
|
|
6924
|
+
* ```
|
|
6925
|
+
*
|
|
6926
|
+
* @param callSid - Twilio Call SID, e.g. from
|
|
6927
|
+
* `InitiateVoiceConversationResult.callSid` or a call event.
|
|
6928
|
+
* @returns The session, or `undefined` — not created yet, the call ended, or
|
|
6929
|
+
* it landed on another instance (see the horizontal-scaling note in
|
|
6930
|
+
* CLAUDE.md).
|
|
6931
|
+
*/
|
|
6932
|
+
getConversationSessionByCallSid(callSid) {
|
|
6933
|
+
for (const session of this.activeConversations.values()) {
|
|
6934
|
+
if (session.callSid === callSid) {
|
|
6935
|
+
return session;
|
|
6936
|
+
}
|
|
6937
|
+
}
|
|
6938
|
+
return void 0;
|
|
6939
|
+
}
|
|
6940
|
+
// =========================================================================
|
|
6941
|
+
// Stream Task Management
|
|
6942
|
+
//
|
|
6943
|
+
// Stream tasks are ConversationRelay's text-token machinery and live on
|
|
6944
|
+
// `ConversationRelayProvider`, alongside the transport that consumes their
|
|
6945
|
+
// AbortSignal. These forwarders keep the channel-level API intact.
|
|
6946
|
+
// =========================================================================
|
|
6947
|
+
/**
|
|
6948
|
+
* Narrow `provider` to the ConversationRelay implementation for the
|
|
6949
|
+
* ConversationRelay-only forwarders on this channel.
|
|
6950
|
+
*
|
|
6951
|
+
* Throws whenever a consumer supplies a non-ConversationRelay provider, which
|
|
6952
|
+
* is the point of the guard. The provider callback used to narrow here too; it
|
|
6953
|
+
* now rides {@link VoiceProvider.handleTwilioProviderCallback}'s
|
|
6954
|
+
* base-compatible signature instead, which is the eventual shape for these
|
|
6955
|
+
* forwarders.
|
|
6956
|
+
*
|
|
6957
|
+
* @throws {Error} if this channel's provider is not ConversationRelay-based.
|
|
6958
|
+
*/
|
|
6959
|
+
requireConversationRelayProvider(capability) {
|
|
6960
|
+
if (!(this.provider instanceof ConversationRelayProvider)) {
|
|
6961
|
+
throw new Error(`${this.provider.constructor.name} does not support ${capability}.`);
|
|
6962
|
+
}
|
|
6963
|
+
return this.provider;
|
|
6964
|
+
}
|
|
6965
|
+
/**
|
|
6966
|
+
* Start tracking a streaming task for a conversation
|
|
6967
|
+
*
|
|
6968
|
+
* @param conversationId - The conversation ID
|
|
6969
|
+
* @returns The stream task with its AbortController
|
|
6970
|
+
*/
|
|
6971
|
+
startStreamTask(conversationId) {
|
|
6972
|
+
return this.requireConversationRelayProvider("stream tasks").startStreamTask(conversationId);
|
|
6973
|
+
}
|
|
6974
|
+
/**
|
|
6975
|
+
* Cancel an active streaming task
|
|
6976
|
+
*
|
|
6977
|
+
* @param conversationId - The conversation ID
|
|
6978
|
+
* @returns true if a task was cancelled, false otherwise
|
|
6979
|
+
*/
|
|
6980
|
+
cancelStreamTask(conversationId) {
|
|
6981
|
+
return this.requireConversationRelayProvider("stream tasks").cancelStreamTask(conversationId);
|
|
6982
|
+
}
|
|
6983
|
+
/**
|
|
6984
|
+
* Complete a streaming task (remove from tracking)
|
|
6985
|
+
*
|
|
6986
|
+
* @param conversationId - The conversation ID
|
|
6987
|
+
*/
|
|
6988
|
+
completeStreamTask(conversationId) {
|
|
6989
|
+
this.requireConversationRelayProvider("stream tasks").completeStreamTask(conversationId);
|
|
6990
|
+
}
|
|
6991
|
+
/**
|
|
6992
|
+
* Check if a stream task is active
|
|
6993
|
+
*
|
|
6994
|
+
* @param conversationId - The conversation ID
|
|
6995
|
+
* @returns true if an active task exists
|
|
6996
|
+
*/
|
|
6997
|
+
hasActiveStreamTask(conversationId) {
|
|
6998
|
+
return this.requireConversationRelayProvider("stream tasks").hasActiveStreamTask(
|
|
6999
|
+
conversationId
|
|
7000
|
+
);
|
|
7001
|
+
}
|
|
7002
|
+
// =========================================================================
|
|
7003
|
+
// ConversationRelay TwiML Generation
|
|
7004
|
+
// =========================================================================
|
|
7005
|
+
/**
|
|
7006
|
+
* Generate TwiML to connect a call to ConversationRelay. Delegates to
|
|
7007
|
+
* {@link ConversationRelayProvider.connectConversationRelay}, which validates
|
|
7008
|
+
* the configuration with Zod before generating TwiML.
|
|
7009
|
+
*
|
|
7010
|
+
* @param config - ConversationRelay configuration (url, transcription, TTS, etc.)
|
|
7011
|
+
* @param options - Optional settings for parameters and the Connect verb
|
|
7012
|
+
* @returns TwiML XML string
|
|
7013
|
+
* @throws {Error} if config validation fails, or if this channel's provider is
|
|
7014
|
+
* not ConversationRelay-based.
|
|
7015
|
+
*/
|
|
7016
|
+
connectConversationRelay(config, options) {
|
|
7017
|
+
return this.requireConversationRelayProvider(
|
|
7018
|
+
"ConversationRelay TwiML generation"
|
|
7019
|
+
).connectConversationRelay(config, options);
|
|
7020
|
+
}
|
|
7021
|
+
/**
|
|
7022
|
+
* Cleanup channel state on shutdown
|
|
7023
|
+
*
|
|
7024
|
+
* Note: WebSocket connections are managed by the server and closed there.
|
|
7025
|
+
* This method only cleans up internal channel state.
|
|
7026
|
+
*/
|
|
7027
|
+
shutdown() {
|
|
7028
|
+
this.provider.shutdown();
|
|
7029
|
+
super.shutdown();
|
|
7030
|
+
}
|
|
7031
|
+
};
|
|
7032
|
+
function generateStreamTwiml(websocketUrl, options) {
|
|
7033
|
+
const resolved = websocketUrl ?? options?.websocketUrl;
|
|
7034
|
+
if (!resolved || !resolved.trim()) {
|
|
7035
|
+
throw new Error(
|
|
7036
|
+
"generateStreamTwiml requires a WebSocket URL \u2014 pass it positionally or set options.websocketUrl."
|
|
7037
|
+
);
|
|
7038
|
+
}
|
|
7039
|
+
const response = new VoiceResponse();
|
|
7040
|
+
const connectAttrs = {};
|
|
7041
|
+
if (options?.actionUrl) {
|
|
7042
|
+
connectAttrs.action = options.actionUrl;
|
|
7043
|
+
}
|
|
7044
|
+
if (options?.actionMethod) {
|
|
7045
|
+
connectAttrs.method = options.actionMethod;
|
|
7046
|
+
}
|
|
7047
|
+
const connect = response.connect(connectAttrs);
|
|
7048
|
+
const streamAttrs = { url: resolved };
|
|
7049
|
+
if (options?.name) {
|
|
7050
|
+
streamAttrs.name = options.name;
|
|
7051
|
+
}
|
|
7052
|
+
if (options?.statusCallback) {
|
|
7053
|
+
streamAttrs.statusCallback = options.statusCallback;
|
|
7054
|
+
}
|
|
7055
|
+
if (options?.statusCallbackMethod) {
|
|
7056
|
+
streamAttrs.statusCallbackMethod = options.statusCallbackMethod;
|
|
7057
|
+
}
|
|
7058
|
+
const stream = connect.stream(streamAttrs);
|
|
7059
|
+
if (options?.customParameters) {
|
|
7060
|
+
for (const [name, value] of Object.entries(options.customParameters)) {
|
|
7061
|
+
if (value !== null && value !== void 0) {
|
|
7062
|
+
stream.parameter({ name, value: stringifyParameterValue(value) });
|
|
7063
|
+
}
|
|
7064
|
+
}
|
|
7065
|
+
}
|
|
7066
|
+
return response.toString();
|
|
7067
|
+
}
|
|
7068
|
+
var TwiMLBuilderMediaStreams = class extends TwiMLBuilderBase {
|
|
7069
|
+
/**
|
|
7070
|
+
* Build the TwiML XML for one call.
|
|
7071
|
+
*
|
|
7072
|
+
* TwiML fields are merged per-field, highest precedence first:
|
|
7073
|
+
* 1. `perCall` — the `onInboundCallTwiml` customizer's output for inbound,
|
|
7074
|
+
* or `InitiateVoiceConversationOptions.twimlOptions` for outbound
|
|
7075
|
+
* 2. the provider config's `defaultTwimlOptions` — channel-wide defaults
|
|
7076
|
+
* 3. `host` — per-call transport facts supplied by the host
|
|
7077
|
+
* 4. TAC defaults: the WebSocket URL derived from
|
|
7078
|
+
* `TACConfig.voicePublicDomain` + `voiceWebsocketPath`
|
|
7079
|
+
*
|
|
7080
|
+
* @param caller - Name of the calling method, used in the "no WebSocket URL"
|
|
7081
|
+
* error so it points at the API the developer actually called.
|
|
7082
|
+
* @param options - Per-call option layers and WebSocket override.
|
|
7083
|
+
* @throws {Error} if no layer and no `TACConfig`-derived default supplies a
|
|
7084
|
+
* WebSocket URL.
|
|
7085
|
+
*/
|
|
7086
|
+
build(caller, options) {
|
|
7087
|
+
const merged = this.buildTwimlOptions(options?.host, options?.perCall);
|
|
7088
|
+
const resolvedWebsocketUrl = options?.websocketUrl || merged.websocketUrl || this.defaultWebsocketUrl();
|
|
7089
|
+
if (!resolvedWebsocketUrl) {
|
|
7090
|
+
throw this.missingWebsocketUrlError(caller);
|
|
7091
|
+
}
|
|
7092
|
+
return generateStreamTwiml(resolvedWebsocketUrl, merged);
|
|
7093
|
+
}
|
|
7094
|
+
/**
|
|
7095
|
+
* Layer TwiML options, lowest precedence first: `host` →
|
|
7096
|
+
* `defaultTwimlOptions` → `perCall`.
|
|
7097
|
+
*
|
|
7098
|
+
* `customParameters` replaces wholesale when set at a higher-priority layer —
|
|
7099
|
+
* there's no per-key merging.
|
|
7100
|
+
*/
|
|
7101
|
+
buildTwimlOptions(host, perCall) {
|
|
7102
|
+
const merged = {};
|
|
7103
|
+
if (host) {
|
|
7104
|
+
this.overlayFields(merged, host);
|
|
7105
|
+
}
|
|
7106
|
+
if (this.channelConfig.defaultTwimlOptions) {
|
|
7107
|
+
this.overlayFields(merged, this.channelConfig.defaultTwimlOptions);
|
|
7108
|
+
}
|
|
7109
|
+
if (perCall) {
|
|
7110
|
+
this.overlayFields(merged, perCall);
|
|
7111
|
+
}
|
|
7112
|
+
return merged;
|
|
7113
|
+
}
|
|
7114
|
+
};
|
|
7115
|
+
|
|
7116
|
+
// packages/core/src/channels/voice/media-streams/config.ts
|
|
7117
|
+
var MediaStreamsProviderConfig = class extends VoiceProviderConfig {
|
|
7118
|
+
/**
|
|
7119
|
+
* Static `VoiceTwiMLOptionsMediaStreams` applied to every inbound call.
|
|
7120
|
+
* Per-call customization is registered via
|
|
7121
|
+
* `VoiceChannel.onInboundCallTwiml(...)`, which takes precedence over this.
|
|
7122
|
+
*/
|
|
7123
|
+
defaultTwimlOptions;
|
|
7124
|
+
constructor(options) {
|
|
7125
|
+
super(options);
|
|
7126
|
+
if (options?.defaultTwimlOptions !== void 0) {
|
|
7127
|
+
this.defaultTwimlOptions = options.defaultTwimlOptions;
|
|
7128
|
+
}
|
|
7129
|
+
}
|
|
7130
|
+
};
|
|
7131
|
+
|
|
7132
|
+
// packages/core/src/channels/voice/media-streams/shared/config.ts
|
|
7133
|
+
var MediaStreamsOpenAIProviderConfig = class extends MediaStreamsProviderConfig {
|
|
7134
|
+
/** OpenAI API key. Defaults to the `OPENAI_API_KEY` environment variable. */
|
|
7135
|
+
openaiApiKey;
|
|
7136
|
+
/**
|
|
7137
|
+
* Executable `TACTool` implementations, looked up by name to run mid-call
|
|
7138
|
+
* tool requests. This alone does not tell the model these tools exist — the
|
|
7139
|
+
* session config sent to OpenAI must separately declare each tool's schema.
|
|
7140
|
+
*/
|
|
7141
|
+
// See the `never` note on `MediaStreamsOpenAIProviderConfigOptions.tools`.
|
|
7142
|
+
tools;
|
|
7143
|
+
/**
|
|
7144
|
+
* Session configuration sent to OpenAI once the model connects — used for any
|
|
7145
|
+
* call that doesn't supply its own via
|
|
7146
|
+
* {@link MediaStreamsOpenAIProviderConfig.onInboundCallSessionConfig} or
|
|
7147
|
+
* per-call outbound options.
|
|
7148
|
+
*
|
|
7149
|
+
* If using {@link MediaStreamsOpenAIProviderConfig.tools}, this config must
|
|
7150
|
+
* separately list each tool's schema — it is passed to OpenAI as-is, with no
|
|
7151
|
+
* tool schemas merged in.
|
|
7152
|
+
*/
|
|
7153
|
+
defaultSessionConfig;
|
|
7154
|
+
/**
|
|
7155
|
+
* Per-inbound-call override for
|
|
7156
|
+
* {@link MediaStreamsOpenAIProviderConfig.defaultSessionConfig}, called with
|
|
7157
|
+
* the `TwiMLRequest`. Its return value is used verbatim (not merged with
|
|
7158
|
+
* `defaultSessionConfig`); return `null` to fall back to it. Outbound calls
|
|
7159
|
+
* don't use this — they pass their session config per call.
|
|
7160
|
+
*/
|
|
7161
|
+
onInboundCallSessionConfig;
|
|
7162
|
+
constructor(options) {
|
|
7163
|
+
super(options);
|
|
7164
|
+
this.openaiApiKey = options?.openaiApiKey ?? process.env.OPENAI_API_KEY ?? "";
|
|
7165
|
+
this.tools = options?.tools ?? [];
|
|
7166
|
+
if (options?.defaultSessionConfig !== void 0) {
|
|
7167
|
+
this.defaultSessionConfig = options.defaultSessionConfig;
|
|
7168
|
+
}
|
|
7169
|
+
if (options?.onInboundCallSessionConfig !== void 0) {
|
|
7170
|
+
this.onInboundCallSessionConfig = options.onInboundCallSessionConfig;
|
|
7171
|
+
}
|
|
7172
|
+
if (!this.openaiApiKey) {
|
|
7173
|
+
throw new Error(
|
|
7174
|
+
`openaiApiKey is required. Set the OPENAI_API_KEY environment variable or provide openaiApiKey in ${this.constructor.name}.`
|
|
7175
|
+
);
|
|
7176
|
+
}
|
|
7177
|
+
}
|
|
7178
|
+
};
|
|
7179
|
+
|
|
7180
|
+
// packages/core/src/channels/voice/media-streams/shared/state.ts
|
|
7181
|
+
var MediaStreamsOpenAICallState = class {
|
|
7182
|
+
twilioWs = null;
|
|
7183
|
+
modelWs = null;
|
|
7184
|
+
/**
|
|
7185
|
+
* Resolves `true` once this call's model socket is open and its session
|
|
7186
|
+
* config has been sent, or `false` if that handshake failed. `null` until the
|
|
7187
|
+
* handshake has been started.
|
|
7188
|
+
*
|
|
7189
|
+
* Twilio begins streaming caller audio as soon as the media stream opens,
|
|
7190
|
+
* which is well before the OpenAI handshake completes. Caller audio waits on
|
|
7191
|
+
* this promise rather than being written to a socket that does not exist
|
|
7192
|
+
* yet, so a caller who speaks the instant the call connects is not clipped.
|
|
7193
|
+
* Every frame awaits this same promise, so the frames resume in the order
|
|
7194
|
+
* they arrived.
|
|
7195
|
+
*
|
|
7196
|
+
* It resolves rather than rejects: a failed handshake is reported once, by
|
|
7197
|
+
* the code that opened the socket, not once per waiting frame.
|
|
7198
|
+
*/
|
|
7199
|
+
modelReady = null;
|
|
7200
|
+
};
|
|
7201
|
+
var OPENAI_USER_AGENT = `twilio-agent-connect/TypeScript ${package_default.version}`;
|
|
7202
|
+
var INBOUND_SESSION_CONFIG_TTL_MS = 12e4;
|
|
7203
|
+
function describeIssues(issues) {
|
|
7204
|
+
return issues.map((issue) => `${issue.path.join(".") || "(root)"}: ${issue.message}`).join(", ");
|
|
7205
|
+
}
|
|
7206
|
+
var MediaStreamsOpenAIProvider = class extends VoiceProvider {
|
|
7207
|
+
/**
|
|
7208
|
+
* The owning channel's logger, so this provider logs under the same name the
|
|
7209
|
+
* rest of the voice channel does.
|
|
7210
|
+
*/
|
|
7211
|
+
logger;
|
|
7212
|
+
/** Executable tools from the config, looked up by the name the model sends. */
|
|
7213
|
+
toolsByName;
|
|
7214
|
+
/** Per-call transport state, keyed by conversation id. */
|
|
7215
|
+
calls;
|
|
7216
|
+
config;
|
|
7217
|
+
tacConfig;
|
|
7218
|
+
twimlBuilder;
|
|
7219
|
+
/**
|
|
7220
|
+
* Session config overrides awaiting the call they belong to.
|
|
7221
|
+
*
|
|
7222
|
+
* Inbound entries are keyed by call SID (known when the TwiML webhook is
|
|
7223
|
+
* answered); a subclass keys its outbound entries by whatever token it round
|
|
7224
|
+
* trips through the stream's custom parameters.
|
|
7225
|
+
*/
|
|
7226
|
+
pendingSessionConfigs;
|
|
7227
|
+
/**
|
|
7228
|
+
* Expiry timers for the inbound {@link pendingSessionConfigs} entries, keyed
|
|
7229
|
+
* by the same call SID. Each fires once to purge a stash whose Media Stream
|
|
7230
|
+
* never connected; a normal connect cancels it in the subclass's
|
|
7231
|
+
* `registerCall`. Outbound entries are keyed by token instead and are not
|
|
7232
|
+
* tracked here.
|
|
7233
|
+
*/
|
|
7234
|
+
pendingInboundExpiries;
|
|
7235
|
+
constructor(channel, tacConfig, config) {
|
|
7236
|
+
super(channel);
|
|
7237
|
+
this.logger = channel.getLoggerInternal();
|
|
7238
|
+
this.config = config;
|
|
7239
|
+
this.tacConfig = tacConfig;
|
|
7240
|
+
this.toolsByName = new Map(config.tools.map((tool) => [tool.name, tool]));
|
|
7241
|
+
this.calls = /* @__PURE__ */ new Map();
|
|
7242
|
+
this.twimlBuilder = new TwiMLBuilderMediaStreams(tacConfig, config, this.logger);
|
|
7243
|
+
this.pendingSessionConfigs = /* @__PURE__ */ new Map();
|
|
7244
|
+
this.pendingInboundExpiries = /* @__PURE__ */ new Map();
|
|
7245
|
+
}
|
|
7246
|
+
/** The Twilio-facing WebSocket for a conversation, if one is tracked. */
|
|
7247
|
+
getWebSocket(conversationId) {
|
|
7248
|
+
return this.calls.get(conversationId)?.twilioWs ?? null;
|
|
7249
|
+
}
|
|
7250
|
+
/**
|
|
7251
|
+
* Open a WebSocket and resolve once it is ready to carry traffic.
|
|
7252
|
+
*
|
|
7253
|
+
* Isolated from each subclass's `connectModel` so tests can substitute a
|
|
7254
|
+
* socket without reaching the network.
|
|
7255
|
+
*
|
|
7256
|
+
* A resolved socket always carries at least one `'error'` listener, whatever
|
|
7257
|
+
* the caller does with it next.
|
|
7258
|
+
*
|
|
7259
|
+
* @internal
|
|
7260
|
+
*/
|
|
7261
|
+
openModelSocket(url, headers) {
|
|
7262
|
+
return new Promise((resolve, reject) => {
|
|
7263
|
+
const ws = new WebSocket(url, { headers });
|
|
7264
|
+
const onOpen = () => {
|
|
7265
|
+
ws.off("error", onError);
|
|
7266
|
+
ws.on("error", () => {
|
|
7267
|
+
});
|
|
7268
|
+
resolve(ws);
|
|
7269
|
+
};
|
|
7270
|
+
const onError = (error) => {
|
|
7271
|
+
ws.off("open", onOpen);
|
|
7272
|
+
reject(error);
|
|
7273
|
+
};
|
|
7274
|
+
ws.once("open", onOpen);
|
|
7275
|
+
ws.once("error", onError);
|
|
7276
|
+
});
|
|
7277
|
+
}
|
|
7278
|
+
/**
|
|
7279
|
+
* The transcript captured so far for an in-progress call.
|
|
7280
|
+
*
|
|
7281
|
+
* It lives on `ConversationSession.metadata.transcript`, so once the call
|
|
7282
|
+
* ends and the session is dropped it is no longer reachable here — read it
|
|
7283
|
+
* from the session an `onConversationEnded` handler receives instead.
|
|
7284
|
+
*/
|
|
7285
|
+
getTranscript(conversationId) {
|
|
7286
|
+
const transcript = this.channel.getConversationSession(conversationId)?.metadata.transcript;
|
|
7287
|
+
return Array.isArray(transcript) ? [...transcript] : [];
|
|
7288
|
+
}
|
|
7289
|
+
/**
|
|
7290
|
+
* The session config stashed for `key` — a call SID for inbound calls, a
|
|
7291
|
+
* token for outbound ones.
|
|
7292
|
+
*
|
|
7293
|
+
* @internal
|
|
7294
|
+
*/
|
|
7295
|
+
peekPendingSessionConfig(key) {
|
|
7296
|
+
return this.pendingSessionConfigs.get(key);
|
|
7297
|
+
}
|
|
7298
|
+
/**
|
|
7299
|
+
* How many session config overrides are waiting for their call.
|
|
7300
|
+
*
|
|
7301
|
+
* @internal
|
|
7302
|
+
*/
|
|
7303
|
+
pendingSessionConfigCount() {
|
|
7304
|
+
return this.pendingSessionConfigs.size;
|
|
7305
|
+
}
|
|
7306
|
+
/**
|
|
7307
|
+
* Start the clock on an inbound stash, so a call whose Media Stream never
|
|
7308
|
+
* connects cannot strand its override in {@link pendingSessionConfigs} until
|
|
7309
|
+
* shutdown.
|
|
7310
|
+
*
|
|
7311
|
+
* Unref'd: a two-minute timer must not be what keeps the process alive after
|
|
7312
|
+
* the call it belongs to is long over. Re-arming replaces any prior timer for
|
|
7313
|
+
* the same call SID, so a duplicate inbound webhook can't orphan one.
|
|
7314
|
+
*/
|
|
7315
|
+
armInboundConfigExpiry(callSid) {
|
|
7316
|
+
this.cancelInboundConfigExpiry(callSid);
|
|
7317
|
+
const timer = setTimeout(() => {
|
|
7318
|
+
this.pendingInboundExpiries.delete(callSid);
|
|
7319
|
+
this.pendingSessionConfigs.delete(callSid);
|
|
7320
|
+
}, INBOUND_SESSION_CONFIG_TTL_MS);
|
|
7321
|
+
timer.unref();
|
|
7322
|
+
this.pendingInboundExpiries.set(callSid, timer);
|
|
7323
|
+
}
|
|
7324
|
+
/**
|
|
7325
|
+
* Stop the clock on an inbound stash, once the call it belongs to has
|
|
7326
|
+
* connected and is about to consume the entry. A no-op for outbound calls,
|
|
7327
|
+
* which key their stash by token and never arm one — the subclass calls this
|
|
7328
|
+
* with the call SID for every `start`, inbound or not.
|
|
7329
|
+
*
|
|
7330
|
+
* @internal
|
|
7331
|
+
*/
|
|
7332
|
+
cancelInboundConfigExpiry(callSid) {
|
|
7333
|
+
const timer = this.pendingInboundExpiries.get(callSid);
|
|
7334
|
+
if (timer !== void 0) {
|
|
7335
|
+
clearTimeout(timer);
|
|
7336
|
+
this.pendingInboundExpiries.delete(callSid);
|
|
7337
|
+
}
|
|
7338
|
+
}
|
|
7339
|
+
/**
|
|
7340
|
+
* The executable tool the model would run for `name`, if the config supplied
|
|
7341
|
+
* one.
|
|
7342
|
+
*
|
|
7343
|
+
* @internal
|
|
7344
|
+
*/
|
|
7345
|
+
peekTool(name) {
|
|
7346
|
+
return this.toolsByName.get(name);
|
|
7347
|
+
}
|
|
7348
|
+
// =========================================================================
|
|
7349
|
+
// Inbound Call Handling
|
|
7350
|
+
// =========================================================================
|
|
7351
|
+
/**
|
|
7352
|
+
* Build the `<Connect><Stream>` TwiML for an inbound call.
|
|
7353
|
+
*
|
|
7354
|
+
* TwiML fields are merged per-field, highest precedence first:
|
|
7355
|
+
* 1. Output of the customizer registered via
|
|
7356
|
+
* `VoiceChannel.onInboundCallTwiml(...)`, if configured and
|
|
7357
|
+
* `twimlRequest` is given
|
|
7358
|
+
* 2. `MediaStreamsProviderConfig.defaultTwimlOptions` — channel-wide
|
|
7359
|
+
* defaults
|
|
7360
|
+
* 3. `options.hostTwimlOptions` — per-call transport facts supplied by the
|
|
7361
|
+
* host
|
|
7362
|
+
* 4. TAC defaults: the WebSocket URL derived from
|
|
7363
|
+
* `TACConfig.voicePublicDomain` + `voiceWebsocketPath`
|
|
7364
|
+
*
|
|
7365
|
+
* Also runs `MediaStreamsOpenAIProviderConfig.onInboundCallSessionConfig`, if
|
|
7366
|
+
* set, and stashes its result for the call to pick up once it connects. The
|
|
7367
|
+
* hook runs only after the TwiML builds, so a call that never connects
|
|
7368
|
+
* leaves nothing stashed behind it.
|
|
7369
|
+
*
|
|
7370
|
+
* @param twimlRequest - Parsed Twilio webhook fields for the inbound call.
|
|
7371
|
+
* @param options - Additional per-call inputs.
|
|
7372
|
+
* @param options.hostTwimlOptions - Per-call TwiML supplied by a custom
|
|
7373
|
+
* in-process host.
|
|
7374
|
+
* @throws {TypeError} if either the host options or the customizer's output
|
|
7375
|
+
* is not a `VoiceTwiMLOptionsMediaStreams`.
|
|
7376
|
+
* @throws {Error} if no WebSocket URL can be resolved — none of the TwiML
|
|
7377
|
+
* layers set one and `TACConfig.voicePublicDomain` is unset.
|
|
7378
|
+
*/
|
|
7379
|
+
async handleIncomingCall(twimlRequest, options) {
|
|
7380
|
+
const host = this.narrowTwimlOptions(
|
|
7381
|
+
options?.hostTwimlOptions,
|
|
7382
|
+
"handleIncomingCall",
|
|
7383
|
+
"options.hostTwimlOptions"
|
|
7384
|
+
);
|
|
7385
|
+
const onInboundCallTwimlHandler = this.channel.getInboundCallTwimlHandler();
|
|
7386
|
+
let customized;
|
|
7387
|
+
if (onInboundCallTwimlHandler && twimlRequest) {
|
|
7388
|
+
customized = this.narrowTwimlOptions(
|
|
7389
|
+
await onInboundCallTwimlHandler(twimlRequest),
|
|
7390
|
+
"handleIncomingCall",
|
|
7391
|
+
"the onInboundCallTwiml customizer output"
|
|
7392
|
+
);
|
|
7393
|
+
}
|
|
7394
|
+
const twiml = this.twimlBuilder.build("handleIncomingCall", { host, perCall: customized });
|
|
7395
|
+
if (this.config.onInboundCallSessionConfig && twimlRequest?.callSid) {
|
|
7396
|
+
const sessionConfig = await this.config.onInboundCallSessionConfig(twimlRequest);
|
|
7397
|
+
if (sessionConfig !== null) {
|
|
7398
|
+
this.pendingSessionConfigs.set(twimlRequest.callSid, sessionConfig);
|
|
7399
|
+
this.armInboundConfigExpiry(twimlRequest.callSid);
|
|
7400
|
+
}
|
|
7401
|
+
}
|
|
7402
|
+
return twiml;
|
|
7403
|
+
}
|
|
7404
|
+
/**
|
|
7405
|
+
* Narrow provider-agnostic {@link VoiceTwiMLOptions} to this provider's
|
|
7406
|
+
* concrete shape. `VoiceProvider.handleIncomingCall` is typed against the
|
|
7407
|
+
* base so every provider can accept its own TwiML options, so the Media
|
|
7408
|
+
* Streams shape has to be established at runtime.
|
|
7409
|
+
*
|
|
7410
|
+
* @param value - Options from a caller or the application customizer.
|
|
7411
|
+
* @param caller - Name of the calling method, for the error message.
|
|
7412
|
+
* @param label - What produced `value`, for the error message.
|
|
7413
|
+
*/
|
|
7414
|
+
narrowTwimlOptions(value, caller, label) {
|
|
7415
|
+
if (value === void 0) {
|
|
7416
|
+
return void 0;
|
|
7417
|
+
}
|
|
7418
|
+
const parsed = VoiceTwiMLOptionsMediaStreamsSchema.safeParse(value);
|
|
7419
|
+
if (!parsed.success) {
|
|
7420
|
+
throw new TypeError(
|
|
7421
|
+
`MediaStreamsOpenAIProvider.${caller} requires ${label} to be a VoiceTwiMLOptionsMediaStreams: ${describeIssues(parsed.error.issues)}`
|
|
7422
|
+
);
|
|
7423
|
+
}
|
|
7424
|
+
return parsed.data;
|
|
7425
|
+
}
|
|
7426
|
+
// =========================================================================
|
|
7427
|
+
// Audio Bridge
|
|
7428
|
+
// =========================================================================
|
|
7429
|
+
/**
|
|
7430
|
+
* Parse one frame off the model socket and hand it to
|
|
7431
|
+
* {@link dispatchModelEvent}.
|
|
7432
|
+
*
|
|
7433
|
+
* A failure here is logged and skipped rather than ending the call: one
|
|
7434
|
+
* malformed delta must not hang up on the caller.
|
|
7435
|
+
*
|
|
7436
|
+
* @internal
|
|
7437
|
+
*/
|
|
7438
|
+
async handleModelMessage(conversationId, raw) {
|
|
7439
|
+
try {
|
|
7440
|
+
const event = JSON.parse(typeof raw === "string" ? raw : raw.toString("utf8"));
|
|
7441
|
+
const session = this.channel.getConversationSession(conversationId);
|
|
7442
|
+
if (session === void 0) {
|
|
7443
|
+
return;
|
|
7444
|
+
}
|
|
7445
|
+
await this.dispatchModelEvent(conversationId, session, event);
|
|
7446
|
+
} catch (err) {
|
|
7447
|
+
this.logger.error({ err, conversation_id: conversationId }, "Error handling model event");
|
|
7448
|
+
}
|
|
7449
|
+
}
|
|
7450
|
+
/**
|
|
7451
|
+
* Look up a model-requested tool by name, run it, and return its output.
|
|
7452
|
+
*
|
|
7453
|
+
* Errors are returned as part of the output rather than thrown, so a bad
|
|
7454
|
+
* tool call does not kill the call.
|
|
7455
|
+
*
|
|
7456
|
+
* @internal
|
|
7457
|
+
*/
|
|
7458
|
+
async runToolCall(conversationId, name, argumentsJson) {
|
|
7459
|
+
this.logger.debug({ conversation_id: conversationId, tool_name: name }, "Tool call");
|
|
7460
|
+
const tool = this.toolsByName.get(name);
|
|
7461
|
+
if (tool === void 0) {
|
|
7462
|
+
return { error: `Unknown tool '${name}'` };
|
|
7463
|
+
}
|
|
7464
|
+
try {
|
|
7465
|
+
const parsedArguments = JSON.parse(
|
|
7466
|
+
typeof argumentsJson === "string" && argumentsJson ? argumentsJson : "{}"
|
|
7467
|
+
);
|
|
7468
|
+
const output = await tool.implementation(parsedArguments);
|
|
7469
|
+
this.logger.debug({ conversation_id: conversationId, tool_name: name }, "Tool result");
|
|
7470
|
+
return output;
|
|
7471
|
+
} catch (err) {
|
|
7472
|
+
this.logger.error({ err, conversation_id: conversationId, tool_name: name }, "Tool failed");
|
|
7473
|
+
return { error: `Tool '${name}' failed to execute.` };
|
|
7474
|
+
}
|
|
7475
|
+
}
|
|
7476
|
+
/** Write one event to this call's model socket, if it still has one. */
|
|
7477
|
+
modelSend(conversationId, payload) {
|
|
7478
|
+
const modelWs = this.calls.get(conversationId)?.modelWs;
|
|
7479
|
+
if (!modelWs) {
|
|
7480
|
+
return;
|
|
7481
|
+
}
|
|
7482
|
+
try {
|
|
7483
|
+
modelWs.send(JSON.stringify(payload));
|
|
7484
|
+
} catch (err) {
|
|
7485
|
+
this.logger.debug({ err, conversation_id: conversationId }, "Failed to send to model");
|
|
7486
|
+
}
|
|
7487
|
+
}
|
|
7488
|
+
/** Write one message to this call's Twilio socket, if it still has one. */
|
|
7489
|
+
twilioSend(conversationId, payload) {
|
|
7490
|
+
const twilioWs = this.calls.get(conversationId)?.twilioWs;
|
|
7491
|
+
if (!twilioWs) {
|
|
7492
|
+
return;
|
|
7493
|
+
}
|
|
7494
|
+
try {
|
|
7495
|
+
twilioWs.send(JSON.stringify(payload));
|
|
7496
|
+
} catch (err) {
|
|
7497
|
+
this.logger.debug({ err, conversation_id: conversationId }, "Failed to send to Twilio");
|
|
7498
|
+
}
|
|
7499
|
+
}
|
|
7500
|
+
/**
|
|
7501
|
+
* Always throws: the model streams its reply as audio straight to Twilio, so
|
|
7502
|
+
* this transport has no text response to send.
|
|
7503
|
+
*/
|
|
7504
|
+
// eslint-disable-next-line @typescript-eslint/require-await -- Rejects without awaiting, but stays `async` so callers always get a Promise
|
|
7505
|
+
async sendResponse(_conversationId, _message, _metadata) {
|
|
7506
|
+
throw new Error(
|
|
7507
|
+
`${this.constructor.name} produces audio via the model; it has no text sendResponse.`
|
|
7508
|
+
);
|
|
7509
|
+
}
|
|
7510
|
+
/**
|
|
7511
|
+
* Drop this provider's Media Streams transport state on channel shutdown.
|
|
7512
|
+
*
|
|
7513
|
+
* Note: WebSocket connections are managed by the server and closed there.
|
|
7514
|
+
* This method only cleans up internal provider state — including session
|
|
7515
|
+
* config overrides stashed for calls that were placed but never connected.
|
|
7516
|
+
*/
|
|
7517
|
+
shutdown() {
|
|
7518
|
+
super.shutdown();
|
|
7519
|
+
for (const timer of this.pendingInboundExpiries.values()) {
|
|
7520
|
+
clearTimeout(timer);
|
|
7521
|
+
}
|
|
7522
|
+
this.pendingInboundExpiries.clear();
|
|
7523
|
+
this.calls.clear();
|
|
7524
|
+
this.pendingSessionConfigs.clear();
|
|
7525
|
+
}
|
|
7526
|
+
};
|
|
7527
|
+
|
|
7528
|
+
// packages/core/src/channels/voice/media-streams/openai-realtime/state.ts
|
|
7529
|
+
var BargeInState = class {
|
|
7530
|
+
lastAssistantItem = null;
|
|
7531
|
+
currentItemAudioMs = 0;
|
|
7532
|
+
mutedItemId = null;
|
|
7533
|
+
responseActive = false;
|
|
7534
|
+
};
|
|
7535
|
+
var CallState = class extends MediaStreamsOpenAICallState {
|
|
7536
|
+
/**
|
|
7537
|
+
* Tail of this call's model-event chain: each incoming OpenAI Realtime event
|
|
7538
|
+
* is appended to it rather than dispatched on arrival.
|
|
7539
|
+
*
|
|
7540
|
+
* The Python SDK reads model events in a sequential loop, so event N is fully
|
|
7541
|
+
* handled before N+1 is even read. `ws` delivers each event on its own
|
|
7542
|
+
* `'message'` emission with nothing serializing them, so without this chain a
|
|
7543
|
+
* `response.output_audio.delta` could advance the barge-in bookkeeping while
|
|
7544
|
+
* dispatch is suspended on the `handleFunctionCall` await, producing a
|
|
7545
|
+
* truncate that overruns the item it names.
|
|
7546
|
+
*/
|
|
7547
|
+
modelEvents = Promise.resolve();
|
|
7548
|
+
bargeIn = new BargeInState();
|
|
7549
|
+
};
|
|
7550
|
+
|
|
7551
|
+
// packages/core/src/channels/voice/media-streams/openai-realtime/provider.ts
|
|
7552
|
+
var SESSION_CONFIG_TOKEN_PARAM = "_tac_session_config_token";
|
|
7553
|
+
var TWILIO_AUDIO_FORMAT_FOR_REALTIME = { type: "audio/pcmu" };
|
|
7554
|
+
var PCMU_BYTES_PER_MS = 8;
|
|
7555
|
+
function isTwilioMediaStreamAudioFormat(value) {
|
|
7556
|
+
if (typeof value !== "object" || value === null) {
|
|
7557
|
+
return false;
|
|
7558
|
+
}
|
|
7559
|
+
const expected = TWILIO_AUDIO_FORMAT_FOR_REALTIME;
|
|
7560
|
+
const actual = value;
|
|
7561
|
+
const keys = Object.keys(actual);
|
|
7562
|
+
return keys.length === Object.keys(expected).length && keys.every((key) => actual[key] === expected[key]);
|
|
7563
|
+
}
|
|
7564
|
+
var OpenAIRealtimeProvider = class extends MediaStreamsOpenAIProvider {
|
|
7565
|
+
/** @internal */
|
|
7566
|
+
get providerId() {
|
|
7567
|
+
return "openai_realtime";
|
|
7568
|
+
}
|
|
7569
|
+
constructor(channel, tacConfig, config) {
|
|
7570
|
+
super(channel, tacConfig, config);
|
|
7571
|
+
}
|
|
7572
|
+
get channelName() {
|
|
7573
|
+
return "VOICE_MEDIA_STREAM_OPENAI_REALTIME";
|
|
7574
|
+
}
|
|
7575
|
+
// =========================================================================
|
|
7576
|
+
// Outbound Call Handling
|
|
7577
|
+
// =========================================================================
|
|
7578
|
+
/**
|
|
7579
|
+
* Initiate an outbound voice conversation.
|
|
7580
|
+
*
|
|
7581
|
+
* Places an outbound call with inline TwiML that connects to a Media Stream.
|
|
7582
|
+
* Unlike inbound, there is no local session yet at this point — one is
|
|
7583
|
+
* created when Twilio's WebSocket `start` event arrives.
|
|
7584
|
+
*
|
|
7585
|
+
* TwiML fields are merged per-field — see
|
|
7586
|
+
* {@link TwiMLBuilderMediaStreams.build}. The WebSocket URL is derived from
|
|
7587
|
+
* `TACConfig.voicePublicDomain` + `TACConfig.voiceWebsocketPath` unless
|
|
7588
|
+
* overridden per-call via `options.websocketUrl`.
|
|
7589
|
+
*
|
|
7590
|
+
* Pass `InitiateVoiceConversationOptionsOpenAIRealtime` with `sessionConfig`
|
|
7591
|
+
* set to override `OpenAIRealtimeProviderConfig.defaultSessionConfig` for
|
|
7592
|
+
* this call.
|
|
7593
|
+
*
|
|
7594
|
+
* @param options - Outbound call options, validated in full against
|
|
7595
|
+
* `InitiateVoiceConversationOptionsOpenAIRealtimeSchema`.
|
|
7596
|
+
* @throws {TypeError} if `options` is not a valid
|
|
7597
|
+
* `InitiateVoiceConversationOptionsOpenAIRealtime` — including an unknown
|
|
7598
|
+
* key, a missing `to`, or a `twimlOptions` that is not a
|
|
7599
|
+
* `VoiceTwiMLOptionsMediaStreams`.
|
|
7600
|
+
* @throws {Error} if no WebSocket URL can be resolved — neither
|
|
7601
|
+
* `options.websocketUrl` nor any TwiML layer sets one and
|
|
7602
|
+
* `TACConfig.voicePublicDomain` is unset.
|
|
7603
|
+
*/
|
|
7604
|
+
async initiateOutboundConversation(options) {
|
|
7605
|
+
const parsedOptions = InitiateVoiceConversationOptionsOpenAIRealtimeSchema.safeParse(options);
|
|
7606
|
+
if (!parsedOptions.success) {
|
|
7607
|
+
throw new TypeError(
|
|
7608
|
+
`OpenAIRealtimeProvider.initiateOutboundConversation requires options to be an InitiateVoiceConversationOptionsOpenAIRealtime: ${describeIssues(
|
|
7609
|
+
parsedOptions.error.issues
|
|
7610
|
+
)}`
|
|
7611
|
+
);
|
|
7612
|
+
}
|
|
7613
|
+
const validated = parsedOptions.data;
|
|
7614
|
+
let twimlOptions = validated.twimlOptions;
|
|
7615
|
+
const sessionConfig = validated.sessionConfig ?? null;
|
|
7616
|
+
let sessionConfigToken = null;
|
|
7617
|
+
if (sessionConfig !== null) {
|
|
7618
|
+
sessionConfigToken = crypto.randomUUID().replace(/-/g, "");
|
|
7619
|
+
twimlOptions = {
|
|
7620
|
+
...twimlOptions,
|
|
7621
|
+
customParameters: {
|
|
7622
|
+
...twimlOptions?.customParameters,
|
|
7623
|
+
[SESSION_CONFIG_TOKEN_PARAM]: sessionConfigToken
|
|
7624
|
+
}
|
|
7625
|
+
};
|
|
7626
|
+
}
|
|
7627
|
+
const fromNumber = this.tacConfig.phoneNumber;
|
|
7628
|
+
this.logger.info(
|
|
7629
|
+
{ to: maskPhone(validated.to), from: maskPhone(fromNumber) },
|
|
7630
|
+
"Initiating outbound voice conversation"
|
|
7631
|
+
);
|
|
7632
|
+
const twiml = this.twimlBuilder.build("initiateOutboundConversation", {
|
|
7633
|
+
perCall: twimlOptions,
|
|
7634
|
+
websocketUrl: validated.websocketUrl
|
|
7635
|
+
});
|
|
7636
|
+
const callParams = this.applyCallEventCallbacks(
|
|
7637
|
+
validated.callOptions ? callOptionsToCreateParams(validated.callOptions) : {}
|
|
7638
|
+
);
|
|
7639
|
+
if (sessionConfigToken !== null && sessionConfig !== null) {
|
|
7640
|
+
this.pendingSessionConfigs.set(sessionConfigToken, sessionConfig);
|
|
7641
|
+
}
|
|
7642
|
+
try {
|
|
7643
|
+
this.logger.debug(
|
|
7644
|
+
{ twiml: redactTwimlParameters(twiml), to: maskPhone(validated.to) },
|
|
7645
|
+
"Outbound call TwiML"
|
|
7646
|
+
);
|
|
7647
|
+
const client2 = this.channel.getTwilioClientInternal();
|
|
7648
|
+
const call = await client2.calls.create({
|
|
7649
|
+
to: validated.to,
|
|
7650
|
+
from: fromNumber,
|
|
7651
|
+
twiml,
|
|
7652
|
+
...callParams
|
|
7653
|
+
});
|
|
7654
|
+
this.logger.info(
|
|
7655
|
+
{ call_sid: call.sid, to: maskPhone(validated.to) },
|
|
7656
|
+
"Outbound voice call placed"
|
|
7657
|
+
);
|
|
7658
|
+
return { callSid: call.sid };
|
|
7659
|
+
} catch (error) {
|
|
7660
|
+
if (sessionConfigToken !== null) {
|
|
7661
|
+
this.pendingSessionConfigs.delete(sessionConfigToken);
|
|
7662
|
+
}
|
|
7663
|
+
this.logger.error(
|
|
7664
|
+
{ err: error, to: maskPhone(validated.to) },
|
|
7665
|
+
"Failed to initiate outbound call"
|
|
7666
|
+
);
|
|
7667
|
+
throw error;
|
|
7668
|
+
}
|
|
7669
|
+
}
|
|
7670
|
+
// =========================================================================
|
|
7671
|
+
// Audio Bridge
|
|
7672
|
+
// =========================================================================
|
|
7673
|
+
/**
|
|
7674
|
+
* Drive one Twilio Media Stream connection from `start` to disconnect.
|
|
7675
|
+
*
|
|
7676
|
+
* Twilio's `start` event names the call, which opens the matching OpenAI
|
|
7677
|
+
* Realtime socket; from then on caller audio is relayed to the model and the
|
|
7678
|
+
* model's audio back to Twilio, until either side goes away. Whichever leg
|
|
7679
|
+
* closes first takes the other down with it, so a caller is never left
|
|
7680
|
+
* connected to silence.
|
|
7681
|
+
*
|
|
7682
|
+
* Twilio streams audio without waiting for the OpenAI socket to finish
|
|
7683
|
+
* connecting, so audio that arrives during that handshake is held and
|
|
7684
|
+
* forwarded, in order, once the model is ready — a caller who speaks the
|
|
7685
|
+
* instant the call connects is heard in full.
|
|
7686
|
+
*
|
|
7687
|
+
* Called by `VoiceChannel.handleWebSocketConnection`; hosts serve the socket
|
|
7688
|
+
* rather than calling this directly.
|
|
7689
|
+
*
|
|
7690
|
+
* @param ws - The accepted Twilio-facing WebSocket.
|
|
7691
|
+
*/
|
|
7692
|
+
handleWebSocket(ws) {
|
|
7693
|
+
let conversationId = null;
|
|
7694
|
+
ws.on("message", (data) => {
|
|
7695
|
+
void (async () => {
|
|
7696
|
+
const message = JSON.parse(data.toString());
|
|
7697
|
+
const event = typeof message.event === "string" ? message.event : "";
|
|
7698
|
+
if (event === "start") {
|
|
7699
|
+
try {
|
|
7700
|
+
const registered = this.registerCall(message.start, ws);
|
|
7701
|
+
conversationId = registered.conversationId;
|
|
7702
|
+
const connecting = this.connectModel(registered.conversationId);
|
|
7703
|
+
registered.call.modelReady = connecting.then(
|
|
7704
|
+
() => true,
|
|
7705
|
+
() => false
|
|
7706
|
+
);
|
|
7707
|
+
await connecting;
|
|
7708
|
+
} catch (err) {
|
|
7709
|
+
this.logger.error(
|
|
7710
|
+
{ err, conversation_id: conversationId },
|
|
7711
|
+
"Failed to bridge the call to OpenAI Realtime, ending the call"
|
|
7712
|
+
);
|
|
7713
|
+
ws.close();
|
|
7714
|
+
}
|
|
7715
|
+
} else if (event === "media") {
|
|
7716
|
+
const media = message.media ?? {};
|
|
7717
|
+
const payload = media.payload;
|
|
7718
|
+
if (conversationId !== null && typeof payload === "string" && payload) {
|
|
7719
|
+
const call = this.calls.get(conversationId);
|
|
7720
|
+
if (call === void 0) {
|
|
7721
|
+
return;
|
|
7722
|
+
}
|
|
7723
|
+
if (call.modelReady !== null && !await call.modelReady) {
|
|
7724
|
+
return;
|
|
7725
|
+
}
|
|
7726
|
+
this.modelSend(conversationId, {
|
|
7727
|
+
type: "input_audio_buffer.append",
|
|
7728
|
+
audio: payload
|
|
7729
|
+
});
|
|
7730
|
+
}
|
|
7731
|
+
} else if (event === "stop") {
|
|
7732
|
+
this.logger.info({ conversation_id: conversationId }, "Media stream stopped");
|
|
7733
|
+
if (conversationId !== null) {
|
|
7734
|
+
await this.cleanupCall(conversationId);
|
|
7735
|
+
}
|
|
7736
|
+
ws.close();
|
|
7737
|
+
}
|
|
7738
|
+
})().catch((err) => {
|
|
7739
|
+
this.logger.error(
|
|
7740
|
+
{ err, conversation_id: conversationId },
|
|
7741
|
+
"Unhandled error in Media Stream message handler"
|
|
7742
|
+
);
|
|
7743
|
+
});
|
|
7744
|
+
});
|
|
7745
|
+
ws.on("close", () => {
|
|
7746
|
+
this.logger.info({ conversation_id: conversationId }, "Media stream WebSocket closed");
|
|
7747
|
+
if (conversationId !== null) {
|
|
7748
|
+
trackEvent("Websocket Disconnected", {
|
|
7749
|
+
account_sid: this.tacConfig.accountSid,
|
|
7750
|
+
channel: "voice",
|
|
7751
|
+
conversation_id: conversationId,
|
|
7752
|
+
provider: this.providerId,
|
|
7753
|
+
orchestrator_enabled: this.tacConfig.isOrchestratorEnabled()
|
|
7754
|
+
});
|
|
7755
|
+
void this.cleanupCall(conversationId).catch((err) => {
|
|
7756
|
+
this.logger.error({ err, conversation_id: conversationId }, "Call cleanup error");
|
|
7757
|
+
});
|
|
7758
|
+
}
|
|
7759
|
+
});
|
|
7760
|
+
ws.on("error", (error) => {
|
|
7761
|
+
this.channel.handleErrorInternal(error, { conversationId });
|
|
7762
|
+
});
|
|
7763
|
+
}
|
|
7764
|
+
/**
|
|
7765
|
+
* Handle Twilio's `start` event: track the call and open its session.
|
|
7766
|
+
*
|
|
7767
|
+
* @param start - The event's `start` body, parsed against
|
|
7768
|
+
* `StreamStartMessageSchema`.
|
|
7769
|
+
* @param ws - The Twilio-facing socket this call arrived on.
|
|
7770
|
+
* @returns The conversation id — which is the call SID — and the call's
|
|
7771
|
+
* freshly tracked transport state.
|
|
7772
|
+
*/
|
|
7773
|
+
registerCall(start, ws) {
|
|
7774
|
+
const message = StreamStartMessageSchema.parse(start ?? {});
|
|
7775
|
+
const conversationId = message.callSid;
|
|
7776
|
+
this.cancelInboundConfigExpiry(conversationId);
|
|
7777
|
+
const token = message.customParameters[SESSION_CONFIG_TOKEN_PARAM];
|
|
7778
|
+
if (token !== void 0) {
|
|
7779
|
+
const pending = this.pendingSessionConfigs.get(token);
|
|
7780
|
+
if (pending !== void 0) {
|
|
7781
|
+
this.pendingSessionConfigs.delete(token);
|
|
7782
|
+
this.pendingSessionConfigs.set(conversationId, pending);
|
|
7783
|
+
}
|
|
7784
|
+
}
|
|
7785
|
+
try {
|
|
7786
|
+
const session = this.channel.startConversationInternal(conversationId);
|
|
7787
|
+
session.callSid = message.callSid;
|
|
7788
|
+
session.metadata.streamSid = message.streamSid;
|
|
7789
|
+
session.metadata.transcript = [];
|
|
7790
|
+
trackEvent("Conversation Initialized", {
|
|
7791
|
+
account_sid: this.tacConfig.accountSid,
|
|
7792
|
+
channel: "voice",
|
|
7793
|
+
conversation_id: conversationId,
|
|
7794
|
+
provider: this.providerId,
|
|
7795
|
+
orchestrator_enabled: this.tacConfig.isOrchestratorEnabled()
|
|
7796
|
+
});
|
|
7797
|
+
} catch (err) {
|
|
7798
|
+
this.pendingSessionConfigs.delete(conversationId);
|
|
7799
|
+
void this.channel.endConversationInternal(conversationId).catch(() => void 0);
|
|
7800
|
+
throw err;
|
|
7801
|
+
}
|
|
7802
|
+
const call = new CallState();
|
|
7803
|
+
call.twilioWs = ws;
|
|
7804
|
+
this.calls.set(conversationId, call);
|
|
7805
|
+
this.logger.debug(
|
|
7806
|
+
{ conversation_id: conversationId, media_format: message.mediaFormat },
|
|
7807
|
+
"Media stream started"
|
|
7808
|
+
);
|
|
7809
|
+
return { conversationId, call };
|
|
7810
|
+
}
|
|
7811
|
+
/**
|
|
7812
|
+
* Open this call's OpenAI Realtime socket and send its session config.
|
|
7813
|
+
*
|
|
7814
|
+
* @internal
|
|
7815
|
+
*/
|
|
7816
|
+
async connectModel(conversationId) {
|
|
7817
|
+
const sessionConfig = this.resolveSessionConfig(conversationId);
|
|
7818
|
+
const modelWs = await this.openModelSocket(
|
|
7819
|
+
`wss://api.openai.com/v1/realtime?model=${encodeURIComponent(String(sessionConfig.model))}`,
|
|
7820
|
+
{
|
|
7821
|
+
Authorization: `Bearer ${this.config.openaiApiKey}`,
|
|
7822
|
+
"User-Agent": OPENAI_USER_AGENT
|
|
7823
|
+
}
|
|
7824
|
+
);
|
|
7825
|
+
const call = this.calls.get(conversationId);
|
|
7826
|
+
if (call === void 0) {
|
|
7827
|
+
modelWs.close();
|
|
7828
|
+
return;
|
|
7829
|
+
}
|
|
7830
|
+
call.modelWs = modelWs;
|
|
7831
|
+
this.attachModelHandlers(conversationId, modelWs);
|
|
7832
|
+
this.logger.info({ conversation_id: conversationId }, "Connected to OpenAI Realtime");
|
|
7833
|
+
this.modelSend(conversationId, { type: "session.update", session: sessionConfig });
|
|
7834
|
+
if (this.config.welcomeGreetingResponse !== void 0) {
|
|
7835
|
+
this.modelSend(conversationId, {
|
|
7836
|
+
type: "response.create",
|
|
7837
|
+
response: this.config.welcomeGreetingResponse
|
|
7838
|
+
});
|
|
7839
|
+
}
|
|
7840
|
+
}
|
|
7841
|
+
/**
|
|
7842
|
+
* The validated session config for this call: its own stashed override if it
|
|
7843
|
+
* has one, else the channel-wide default.
|
|
7844
|
+
*
|
|
7845
|
+
* @throws {Error} if neither exists, if it has no `model`, or if either audio
|
|
7846
|
+
* direction is set to a format Twilio can't carry.
|
|
7847
|
+
*/
|
|
7848
|
+
resolveSessionConfig(conversationId) {
|
|
7849
|
+
const sessionConfig = this.pendingSessionConfigs.get(conversationId) ?? this.config.defaultSessionConfig;
|
|
7850
|
+
this.pendingSessionConfigs.delete(conversationId);
|
|
7851
|
+
if (sessionConfig === void 0) {
|
|
7852
|
+
throw new Error(
|
|
7853
|
+
`No sessionConfig available for call ${conversationId} \u2014 this call supplied none and defaultSessionConfig isn't set either.`
|
|
7854
|
+
);
|
|
7855
|
+
}
|
|
7856
|
+
if (!sessionConfig.model) {
|
|
7857
|
+
throw new Error(
|
|
7858
|
+
`sessionConfig for call ${conversationId} must include a 'model' field \u2014 it's used as the ?model= query param when opening the OpenAI Realtime WebSocket.`
|
|
7859
|
+
);
|
|
7860
|
+
}
|
|
7861
|
+
const audio = sessionConfig.audio ?? {};
|
|
7862
|
+
for (const direction of ["input", "output"]) {
|
|
7863
|
+
const format = (audio[direction] ?? {}).format;
|
|
7864
|
+
if (!isTwilioMediaStreamAudioFormat(format)) {
|
|
7865
|
+
throw new Error(
|
|
7866
|
+
`sessionConfig for call ${conversationId} has audio.${direction}.format=${JSON.stringify(format)}, expected ${JSON.stringify(TWILIO_AUDIO_FORMAT_FOR_REALTIME)}. Twilio Media Streams is always 8kHz G.711 u-law; set audio.${direction}.format to TWILIO_AUDIO_FORMAT_FOR_REALTIME.`
|
|
7867
|
+
);
|
|
7868
|
+
}
|
|
7869
|
+
}
|
|
7870
|
+
return sessionConfig;
|
|
7871
|
+
}
|
|
7872
|
+
/**
|
|
7873
|
+
* Wire up the model socket: dispatch its events, and tear the call down when
|
|
7874
|
+
* it goes away.
|
|
5691
7875
|
*
|
|
5692
|
-
*
|
|
7876
|
+
* The Python SDK races its two read loops so the Twilio leg dies with the
|
|
7877
|
+
* model leg; `ws` is event-driven, so the same guarantee is a close/error
|
|
7878
|
+
* handler instead. Python's sequential read loop also handles each model
|
|
7879
|
+
* event to completion before reading the next, which `ws` does not — see
|
|
7880
|
+
* {@link CallState.modelEvents} for the chain that restores it.
|
|
5693
7881
|
*/
|
|
5694
|
-
|
|
5695
|
-
|
|
5696
|
-
|
|
5697
|
-
|
|
5698
|
-
|
|
7882
|
+
attachModelHandlers(conversationId, modelWs) {
|
|
7883
|
+
modelWs.on("message", (raw) => {
|
|
7884
|
+
const call = this.calls.get(conversationId);
|
|
7885
|
+
if (call === void 0) {
|
|
7886
|
+
return;
|
|
7887
|
+
}
|
|
7888
|
+
call.modelEvents = call.modelEvents.then(() => this.handleModelMessage(conversationId, raw)).catch((err) => {
|
|
7889
|
+
this.logger.error(
|
|
7890
|
+
{ err, conversation_id: conversationId },
|
|
7891
|
+
"Unhandled error in model message handler"
|
|
7892
|
+
);
|
|
7893
|
+
});
|
|
7894
|
+
});
|
|
7895
|
+
modelWs.on("close", () => {
|
|
7896
|
+
if (this.calls.has(conversationId)) {
|
|
7897
|
+
this.logger.info({ conversation_id: conversationId }, "Model connection ended");
|
|
7898
|
+
}
|
|
7899
|
+
this.endCallFromModel(conversationId);
|
|
7900
|
+
});
|
|
7901
|
+
modelWs.on("error", (error) => {
|
|
7902
|
+
this.logger.error({ err: error, conversation_id: conversationId }, "Model socket error");
|
|
7903
|
+
this.endCallFromModel(conversationId);
|
|
7904
|
+
});
|
|
5699
7905
|
}
|
|
5700
7906
|
/**
|
|
5701
|
-
*
|
|
7907
|
+
* Hang up the Twilio leg because the model leg is gone, then clean up. A
|
|
7908
|
+
* no-op once the call has already been cleaned up, so both the model socket's
|
|
7909
|
+
* `close` and its `error` can call it.
|
|
7910
|
+
*/
|
|
7911
|
+
endCallFromModel(conversationId) {
|
|
7912
|
+
this.calls.get(conversationId)?.twilioWs?.close();
|
|
7913
|
+
void this.cleanupCall(conversationId).catch((err) => {
|
|
7914
|
+
this.logger.error({ err, conversation_id: conversationId }, "Call cleanup error");
|
|
7915
|
+
});
|
|
7916
|
+
}
|
|
7917
|
+
/** Apply one parsed OpenAI Realtime event to the call. */
|
|
7918
|
+
async dispatchModelEvent(conversationId, session, event) {
|
|
7919
|
+
const call = this.calls.get(conversationId);
|
|
7920
|
+
if (call === void 0) {
|
|
7921
|
+
return;
|
|
7922
|
+
}
|
|
7923
|
+
const bargeIn = call.bargeIn;
|
|
7924
|
+
switch (event.type) {
|
|
7925
|
+
case "error": {
|
|
7926
|
+
const error = event.error ?? {};
|
|
7927
|
+
if (error.code === "response_cancel_not_active") {
|
|
7928
|
+
this.logger.debug(
|
|
7929
|
+
{ conversation_id: conversationId, error },
|
|
7930
|
+
"response.cancel raced response.done"
|
|
7931
|
+
);
|
|
7932
|
+
} else {
|
|
7933
|
+
this.logger.error(
|
|
7934
|
+
{ conversation_id: conversationId, error },
|
|
7935
|
+
"OpenAI Realtime error event"
|
|
7936
|
+
);
|
|
7937
|
+
}
|
|
7938
|
+
break;
|
|
7939
|
+
}
|
|
7940
|
+
case "input_audio_buffer.speech_started": {
|
|
7941
|
+
this.logger.debug({ conversation_id: conversationId }, "Caller speech detected (VAD)");
|
|
7942
|
+
this.handleBargeIn(conversationId, session, call);
|
|
7943
|
+
break;
|
|
7944
|
+
}
|
|
7945
|
+
case "response.created": {
|
|
7946
|
+
bargeIn.responseActive = true;
|
|
7947
|
+
break;
|
|
7948
|
+
}
|
|
7949
|
+
case "conversation.item.input_audio_transcription.completed": {
|
|
7950
|
+
if (typeof event.transcript === "string" && event.transcript) {
|
|
7951
|
+
this.appendTranscript(session, "user", event.transcript);
|
|
7952
|
+
}
|
|
7953
|
+
break;
|
|
7954
|
+
}
|
|
7955
|
+
case "response.output_item.done": {
|
|
7956
|
+
const item = event.item ?? {};
|
|
7957
|
+
if (item.type === "function_call" && item.status === "completed") {
|
|
7958
|
+
await this.handleFunctionCall(conversationId, item);
|
|
7959
|
+
}
|
|
7960
|
+
break;
|
|
7961
|
+
}
|
|
7962
|
+
case "response.done": {
|
|
7963
|
+
bargeIn.responseActive = false;
|
|
7964
|
+
const response = event.response ?? {};
|
|
7965
|
+
const output = Array.isArray(response.output) ? response.output : [];
|
|
7966
|
+
for (const entry of output) {
|
|
7967
|
+
if (entry.role !== "assistant") {
|
|
7968
|
+
continue;
|
|
7969
|
+
}
|
|
7970
|
+
const contents = Array.isArray(entry.content) ? entry.content : [];
|
|
7971
|
+
for (const content of contents) {
|
|
7972
|
+
if (typeof content.transcript === "string" && content.transcript) {
|
|
7973
|
+
this.appendTranscript(session, "assistant", content.transcript);
|
|
7974
|
+
}
|
|
7975
|
+
}
|
|
7976
|
+
}
|
|
7977
|
+
break;
|
|
7978
|
+
}
|
|
7979
|
+
case "response.output_audio.delta": {
|
|
7980
|
+
const delta = event.delta;
|
|
7981
|
+
if (typeof delta !== "string" || !delta) {
|
|
7982
|
+
break;
|
|
7983
|
+
}
|
|
7984
|
+
const itemId = typeof event.item_id === "string" ? event.item_id : "";
|
|
7985
|
+
if (itemId && itemId === bargeIn.mutedItemId) {
|
|
7986
|
+
break;
|
|
7987
|
+
}
|
|
7988
|
+
if (itemId && itemId !== bargeIn.lastAssistantItem) {
|
|
7989
|
+
bargeIn.lastAssistantItem = itemId;
|
|
7990
|
+
bargeIn.currentItemAudioMs = 0;
|
|
7991
|
+
}
|
|
7992
|
+
bargeIn.currentItemAudioMs += Math.floor(
|
|
7993
|
+
Buffer.from(delta, "base64").length / PCMU_BYTES_PER_MS
|
|
7994
|
+
);
|
|
7995
|
+
this.twilioSend(conversationId, {
|
|
7996
|
+
event: "media",
|
|
7997
|
+
streamSid: session.metadata.streamSid,
|
|
7998
|
+
media: { payload: delta }
|
|
7999
|
+
});
|
|
8000
|
+
break;
|
|
8001
|
+
}
|
|
8002
|
+
}
|
|
8003
|
+
}
|
|
8004
|
+
/** Record one turn on the session's running transcript. */
|
|
8005
|
+
appendTranscript(session, role, text) {
|
|
8006
|
+
const existing = session.metadata.transcript;
|
|
8007
|
+
const transcript = Array.isArray(existing) ? existing : [];
|
|
8008
|
+
if (transcript !== existing) {
|
|
8009
|
+
session.metadata.transcript = transcript;
|
|
8010
|
+
}
|
|
8011
|
+
transcript.push({ role, text });
|
|
8012
|
+
}
|
|
8013
|
+
/**
|
|
8014
|
+
* The caller started talking. Cancel any response still generating, truncate
|
|
8015
|
+
* the model's memory of the last reply at the point actually heard, then
|
|
8016
|
+
* clear Twilio's buffered audio so playback stops immediately.
|
|
5702
8017
|
*
|
|
5703
|
-
*
|
|
5704
|
-
*
|
|
5705
|
-
*
|
|
5706
|
-
|
|
8018
|
+
* If no assistant audio has been sent since the last barge-in this is a
|
|
8019
|
+
* no-op: there is nothing queued at Twilio to clear, no item id to name in a
|
|
8020
|
+
* truncate, and any response still generating is left to run.
|
|
8021
|
+
*/
|
|
8022
|
+
handleBargeIn(conversationId, session, call) {
|
|
8023
|
+
const bargeIn = call.bargeIn;
|
|
8024
|
+
const lastAssistantItem = bargeIn.lastAssistantItem;
|
|
8025
|
+
if (lastAssistantItem === null) {
|
|
8026
|
+
this.logger.debug(
|
|
8027
|
+
{ conversation_id: conversationId },
|
|
8028
|
+
"Barge-in: no assistant item to interrupt"
|
|
8029
|
+
);
|
|
8030
|
+
return;
|
|
8031
|
+
}
|
|
8032
|
+
this.logger.debug({ conversation_id: conversationId }, "Barge-in: truncating assistant reply");
|
|
8033
|
+
if (bargeIn.responseActive) {
|
|
8034
|
+
this.modelSend(conversationId, { type: "response.cancel" });
|
|
8035
|
+
bargeIn.responseActive = false;
|
|
8036
|
+
}
|
|
8037
|
+
this.modelSend(conversationId, {
|
|
8038
|
+
type: "conversation.item.truncate",
|
|
8039
|
+
item_id: lastAssistantItem,
|
|
8040
|
+
content_index: 0,
|
|
8041
|
+
// Derived from bytes actually sent for this item, so for every delta
|
|
8042
|
+
// that carried an `item_id` it can never overstate the duration —
|
|
8043
|
+
// `conversation.item.truncate` rejects an `audio_end_ms` past the item's
|
|
8044
|
+
// real content.
|
|
8045
|
+
audio_end_ms: bargeIn.currentItemAudioMs
|
|
8046
|
+
});
|
|
8047
|
+
this.twilioSend(conversationId, { event: "clear", streamSid: session.metadata.streamSid });
|
|
8048
|
+
trackEvent("Voice Interrupt", {
|
|
8049
|
+
account_sid: this.tacConfig.accountSid,
|
|
8050
|
+
channel: "voice",
|
|
8051
|
+
conversation_id: conversationId,
|
|
8052
|
+
duration_until_interrupt_ms: bargeIn.currentItemAudioMs,
|
|
8053
|
+
provider: this.providerId,
|
|
8054
|
+
orchestrator_enabled: this.tacConfig.isOrchestratorEnabled()
|
|
8055
|
+
});
|
|
8056
|
+
bargeIn.mutedItemId = lastAssistantItem;
|
|
8057
|
+
bargeIn.lastAssistantItem = null;
|
|
8058
|
+
bargeIn.currentItemAudioMs = 0;
|
|
8059
|
+
}
|
|
8060
|
+
/**
|
|
8061
|
+
* Run a model-requested tool call and hand the result back.
|
|
5707
8062
|
*
|
|
5708
|
-
*
|
|
8063
|
+
* Always sends a `function_call_output` once a `call_id` is present — even a
|
|
8064
|
+
* tool that ran successfully can return something `JSON.stringify` throws on
|
|
8065
|
+
* (a circular object, a `BigInt`) or has no JSON form at all, which
|
|
8066
|
+
* `JSON.stringify` reports by returning `undefined` rather than throwing (a
|
|
8067
|
+
* void tool, a bare function, a `Symbol`). Either way the model would
|
|
8068
|
+
* otherwise be left waiting on a `call_id` it never gets a result for.
|
|
8069
|
+
* Without a `call_id` there is nothing to reply to, so the item is dropped
|
|
8070
|
+
* instead.
|
|
5709
8071
|
*/
|
|
5710
|
-
async
|
|
5711
|
-
|
|
5712
|
-
|
|
5713
|
-
|
|
5714
|
-
|
|
5715
|
-
|
|
5716
|
-
|
|
5717
|
-
|
|
8072
|
+
async handleFunctionCall(conversationId, item) {
|
|
8073
|
+
const callId = item.call_id;
|
|
8074
|
+
if (typeof callId !== "string" || !callId) {
|
|
8075
|
+
this.logger.error(
|
|
8076
|
+
{ conversation_id: conversationId, item_keys: Object.keys(item) },
|
|
8077
|
+
"Received malformed function_call item without call_id"
|
|
8078
|
+
);
|
|
8079
|
+
return;
|
|
8080
|
+
}
|
|
8081
|
+
const name = item.name;
|
|
8082
|
+
let output;
|
|
8083
|
+
if (typeof name !== "string" || !name) {
|
|
8084
|
+
this.logger.error(
|
|
8085
|
+
{ conversation_id: conversationId, call_id: callId, item_keys: Object.keys(item) },
|
|
8086
|
+
"Received malformed function_call item without tool name"
|
|
8087
|
+
);
|
|
8088
|
+
output = JSON.stringify({ error: "Malformed function call: missing tool name." });
|
|
8089
|
+
} else {
|
|
8090
|
+
const result = await this.runToolCall(conversationId, name, item.arguments);
|
|
8091
|
+
try {
|
|
8092
|
+
const serialized = JSON.stringify(result);
|
|
8093
|
+
output = serialized ?? "null";
|
|
8094
|
+
} catch (err) {
|
|
8095
|
+
this.logger.error(
|
|
8096
|
+
{ err, conversation_id: conversationId, tool_name: name },
|
|
8097
|
+
"Tool returned a non-JSON-serializable result"
|
|
8098
|
+
);
|
|
8099
|
+
output = JSON.stringify({ error: `Tool '${name}' returned a non-serializable result.` });
|
|
8100
|
+
}
|
|
8101
|
+
}
|
|
8102
|
+
this.modelSend(conversationId, {
|
|
8103
|
+
type: "conversation.item.create",
|
|
8104
|
+
item: { type: "function_call_output", call_id: callId, output }
|
|
8105
|
+
});
|
|
8106
|
+
this.modelSend(conversationId, { type: "response.create" });
|
|
5718
8107
|
}
|
|
5719
8108
|
/**
|
|
5720
|
-
*
|
|
8109
|
+
* Drop this call's transport state, close the model socket, and end the
|
|
8110
|
+
* session.
|
|
5721
8111
|
*
|
|
5722
|
-
*
|
|
5723
|
-
*
|
|
8112
|
+
* Both legs can report the call ending, and the first one to arrive tears
|
|
8113
|
+
* down the other, so this runs at most once per call: a second invocation
|
|
8114
|
+
* finds nothing tracked and returns.
|
|
8115
|
+
*/
|
|
8116
|
+
async cleanupCall(conversationId) {
|
|
8117
|
+
const call = this.calls.get(conversationId);
|
|
8118
|
+
if (call === void 0) {
|
|
8119
|
+
return;
|
|
8120
|
+
}
|
|
8121
|
+
this.calls.delete(conversationId);
|
|
8122
|
+
if (call.modelWs !== null) {
|
|
8123
|
+
try {
|
|
8124
|
+
call.modelWs.close();
|
|
8125
|
+
} catch (err) {
|
|
8126
|
+
this.logger.debug({ err, conversation_id: conversationId }, "Error closing model socket");
|
|
8127
|
+
}
|
|
8128
|
+
}
|
|
8129
|
+
await this.channel.endConversationInternal(conversationId);
|
|
8130
|
+
}
|
|
8131
|
+
};
|
|
8132
|
+
|
|
8133
|
+
// packages/core/src/channels/voice/media-streams/openai-realtime/config.ts
|
|
8134
|
+
var OpenAIRealtimeProviderConfig = class extends MediaStreamsOpenAIProviderConfig {
|
|
8135
|
+
/**
|
|
8136
|
+
* If set, sent verbatim as `response.create`'s `response` payload when the
|
|
8137
|
+
* call connects — e.g. `{ instructions: 'Hi there!' }`. No SDK-added wrapping
|
|
8138
|
+
* text or language assumption.
|
|
8139
|
+
*/
|
|
8140
|
+
welcomeGreetingResponse;
|
|
8141
|
+
constructor(options) {
|
|
8142
|
+
super(options);
|
|
8143
|
+
if (options?.welcomeGreetingResponse !== void 0) {
|
|
8144
|
+
this.welcomeGreetingResponse = options.welcomeGreetingResponse;
|
|
8145
|
+
}
|
|
8146
|
+
}
|
|
8147
|
+
createProvider(channel, tacConfig) {
|
|
8148
|
+
return new OpenAIRealtimeProvider(channel, tacConfig, this);
|
|
8149
|
+
}
|
|
8150
|
+
};
|
|
8151
|
+
|
|
8152
|
+
// packages/core/src/channels/voice/media-streams/gpt-live/state.ts
|
|
8153
|
+
var CallState2 = class extends MediaStreamsOpenAICallState {
|
|
8154
|
+
/**
|
|
8155
|
+
* Settles once `session.closed` arrives, so teardown can wait for graceful
|
|
8156
|
+
* finalization before tearing the socket down.
|
|
8157
|
+
*/
|
|
8158
|
+
closed;
|
|
8159
|
+
resolveClosed;
|
|
8160
|
+
constructor() {
|
|
8161
|
+
super();
|
|
8162
|
+
let resolve;
|
|
8163
|
+
this.closed = new Promise((r) => {
|
|
8164
|
+
resolve = r;
|
|
8165
|
+
});
|
|
8166
|
+
this.resolveClosed = resolve;
|
|
8167
|
+
}
|
|
8168
|
+
markClosed() {
|
|
8169
|
+
this.resolveClosed();
|
|
8170
|
+
}
|
|
8171
|
+
};
|
|
8172
|
+
|
|
8173
|
+
// packages/core/src/channels/voice/media-streams/gpt-live/provider.ts
|
|
8174
|
+
var TWILIO_AUDIO_FORMAT_FOR_GPT_LIVE = { type: "audio/pcmu", rate: 8e3 };
|
|
8175
|
+
var GPT_LIVE_SESSION_ID_METADATA_KEY = "gpt_live_session_id";
|
|
8176
|
+
var SESSION_CONFIG_TOKEN_PARAM2 = "_tac_session_config_token";
|
|
8177
|
+
var SESSION_CONFIG_TOKEN_TTL_MS = 12e4;
|
|
8178
|
+
var GPT_LIVE_URL = "wss://api.openai.com/v1/live/sessions";
|
|
8179
|
+
var CLOSE_TIMEOUT_MS = 5e3;
|
|
8180
|
+
function isTwilioMediaStreamAudioFormat2(value) {
|
|
8181
|
+
if (typeof value !== "object" || value === null) {
|
|
8182
|
+
return false;
|
|
8183
|
+
}
|
|
8184
|
+
const expected = TWILIO_AUDIO_FORMAT_FOR_GPT_LIVE;
|
|
8185
|
+
const actual = value;
|
|
8186
|
+
const keys = Object.keys(actual);
|
|
8187
|
+
return keys.length === Object.keys(expected).length && keys.every((key) => actual[key] === expected[key]);
|
|
8188
|
+
}
|
|
8189
|
+
async function waitWithTimeout(promise, ms) {
|
|
8190
|
+
let timer;
|
|
8191
|
+
const timeout = new Promise((resolve) => {
|
|
8192
|
+
timer = setTimeout(() => resolve(false), ms);
|
|
8193
|
+
});
|
|
8194
|
+
try {
|
|
8195
|
+
return await Promise.race([promise.then(() => true), timeout]);
|
|
8196
|
+
} finally {
|
|
8197
|
+
clearTimeout(timer);
|
|
8198
|
+
}
|
|
8199
|
+
}
|
|
8200
|
+
var GPTLiveProvider = class _GPTLiveProvider extends MediaStreamsOpenAIProvider {
|
|
8201
|
+
/** @internal */
|
|
8202
|
+
get providerId() {
|
|
8203
|
+
return "gpt_live";
|
|
8204
|
+
}
|
|
8205
|
+
pendingTokenExpiries = /* @__PURE__ */ new Map();
|
|
8206
|
+
/** Calls whose teardown has begun but is still awaiting `session.closed`. */
|
|
8207
|
+
closingCalls = /* @__PURE__ */ new Set();
|
|
8208
|
+
get channelName() {
|
|
8209
|
+
return "VOICE_MEDIA_STREAM_OPENAI_GPT_LIVE";
|
|
8210
|
+
}
|
|
8211
|
+
// =========================================================================
|
|
8212
|
+
// Outbound Call Handling
|
|
8213
|
+
// =========================================================================
|
|
8214
|
+
/**
|
|
8215
|
+
* Initiate an outbound voice conversation.
|
|
5724
8216
|
*
|
|
5725
|
-
*
|
|
5726
|
-
*
|
|
5727
|
-
*
|
|
8217
|
+
* Places an outbound call with inline TwiML that connects to a Media Stream.
|
|
8218
|
+
* Unlike inbound, there is no local session yet at this point — one is
|
|
8219
|
+
* created when Twilio's WebSocket `start` event arrives.
|
|
5728
8220
|
*
|
|
5729
|
-
*
|
|
5730
|
-
*
|
|
5731
|
-
*
|
|
5732
|
-
*
|
|
8221
|
+
* TwiML fields are merged per-field — see
|
|
8222
|
+
* {@link TwiMLBuilderMediaStreams.build}. The WebSocket URL is derived from
|
|
8223
|
+
* `TACConfig.voicePublicDomain` + `TACConfig.voiceWebsocketPath` unless
|
|
8224
|
+
* overridden per-call via `options.websocketUrl`.
|
|
8225
|
+
*
|
|
8226
|
+
* Pass `InitiateVoiceConversationOptionsGPTLive` with `sessionConfig` set to
|
|
8227
|
+
* override `GPTLiveProviderConfig.defaultSessionConfig` for this call.
|
|
8228
|
+
*
|
|
8229
|
+
* @param options - Outbound call options.
|
|
8230
|
+
* @throws {TypeError} if `options` does not satisfy
|
|
8231
|
+
* `InitiateVoiceConversationOptionsGPTLiveSchema`.
|
|
8232
|
+
* @throws {Error} if no WebSocket URL can be resolved — neither
|
|
8233
|
+
* `options.websocketUrl` nor any TwiML layer sets one and
|
|
8234
|
+
* `TACConfig.voicePublicDomain` is unset.
|
|
5733
8235
|
*/
|
|
5734
|
-
async
|
|
5735
|
-
const
|
|
5736
|
-
|
|
8236
|
+
async initiateOutboundConversation(options) {
|
|
8237
|
+
const parsedOptions = InitiateVoiceConversationOptionsGPTLiveSchema.safeParse(options);
|
|
8238
|
+
if (!parsedOptions.success) {
|
|
8239
|
+
throw new TypeError(
|
|
8240
|
+
`GPTLiveProvider.initiateOutboundConversation requires options to be an InitiateVoiceConversationOptionsGPTLive: ${describeIssues(parsedOptions.error.issues)}`
|
|
8241
|
+
);
|
|
8242
|
+
}
|
|
8243
|
+
const validated = parsedOptions.data;
|
|
8244
|
+
let twimlOptions = validated.twimlOptions;
|
|
8245
|
+
const sessionConfig = validated.sessionConfig ?? null;
|
|
8246
|
+
let sessionConfigToken = null;
|
|
8247
|
+
if (sessionConfig !== null) {
|
|
8248
|
+
sessionConfigToken = crypto.randomUUID().replace(/-/g, "");
|
|
8249
|
+
twimlOptions = {
|
|
8250
|
+
...twimlOptions,
|
|
8251
|
+
customParameters: {
|
|
8252
|
+
...twimlOptions?.customParameters,
|
|
8253
|
+
[SESSION_CONFIG_TOKEN_PARAM2]: sessionConfigToken
|
|
8254
|
+
}
|
|
8255
|
+
};
|
|
8256
|
+
}
|
|
8257
|
+
const fromNumber = this.tacConfig.phoneNumber;
|
|
8258
|
+
this.logger.info(
|
|
8259
|
+
{ to: maskPhone(validated.to), from: maskPhone(fromNumber) },
|
|
8260
|
+
"Initiating outbound voice conversation"
|
|
8261
|
+
);
|
|
8262
|
+
const twiml = this.twimlBuilder.build("initiateOutboundConversation", {
|
|
8263
|
+
perCall: twimlOptions,
|
|
8264
|
+
websocketUrl: validated.websocketUrl
|
|
8265
|
+
});
|
|
8266
|
+
const callParams = this.applyCallEventCallbacks(
|
|
8267
|
+
validated.callOptions ? callOptionsToCreateParams(validated.callOptions) : {}
|
|
8268
|
+
);
|
|
8269
|
+
if (sessionConfigToken !== null && sessionConfig !== null) {
|
|
8270
|
+
this.pendingSessionConfigs.set(sessionConfigToken, sessionConfig);
|
|
8271
|
+
}
|
|
5737
8272
|
try {
|
|
5738
|
-
|
|
8273
|
+
this.logger.debug(
|
|
8274
|
+
{ twiml: redactTwimlParameters(twiml), to: maskPhone(validated.to) },
|
|
8275
|
+
"Outbound call TwiML"
|
|
8276
|
+
);
|
|
8277
|
+
const client2 = this.channel.getTwilioClientInternal();
|
|
8278
|
+
const call = await client2.calls.create({
|
|
8279
|
+
to: validated.to,
|
|
8280
|
+
from: fromNumber,
|
|
8281
|
+
twiml,
|
|
8282
|
+
...callParams
|
|
8283
|
+
});
|
|
8284
|
+
this.logger.info(
|
|
8285
|
+
{ call_sid: call.sid, to: maskPhone(validated.to) },
|
|
8286
|
+
"Outbound voice call placed"
|
|
8287
|
+
);
|
|
8288
|
+
if (sessionConfigToken !== null) {
|
|
8289
|
+
this.armTokenExpiry(sessionConfigToken);
|
|
8290
|
+
}
|
|
8291
|
+
return { callSid: call.sid };
|
|
5739
8292
|
} catch (error) {
|
|
5740
|
-
|
|
5741
|
-
|
|
5742
|
-
|
|
5743
|
-
|
|
5744
|
-
|
|
5745
|
-
|
|
8293
|
+
if (sessionConfigToken !== null) {
|
|
8294
|
+
this.pendingSessionConfigs.delete(sessionConfigToken);
|
|
8295
|
+
}
|
|
8296
|
+
this.logger.error(
|
|
8297
|
+
{ err: error, to: maskPhone(validated.to) },
|
|
8298
|
+
"Failed to initiate outbound call"
|
|
8299
|
+
);
|
|
8300
|
+
throw error;
|
|
5746
8301
|
}
|
|
5747
|
-
return hungUp;
|
|
5748
8302
|
}
|
|
5749
8303
|
/**
|
|
5750
|
-
*
|
|
8304
|
+
* Start the clock on a stashed token, so a call that never connects cannot
|
|
8305
|
+
* strand its override in {@link pendingSessionConfigs} forever.
|
|
5751
8306
|
*
|
|
5752
|
-
*
|
|
5753
|
-
*
|
|
5754
|
-
|
|
5755
|
-
|
|
8307
|
+
* Unref'd: a two-minute timer must not be what keeps the process alive after
|
|
8308
|
+
* the call it belongs to is long over.
|
|
8309
|
+
*/
|
|
8310
|
+
armTokenExpiry(token) {
|
|
8311
|
+
const timer = setTimeout(() => {
|
|
8312
|
+
this.pendingTokenExpiries.delete(token);
|
|
8313
|
+
this.pendingSessionConfigs.delete(token);
|
|
8314
|
+
}, SESSION_CONFIG_TOKEN_TTL_MS);
|
|
8315
|
+
timer.unref();
|
|
8316
|
+
this.pendingTokenExpiries.set(token, timer);
|
|
8317
|
+
}
|
|
8318
|
+
/**
|
|
8319
|
+
* Stop the clock on a token, once the call it belongs to has claimed it.
|
|
5756
8320
|
*
|
|
5757
|
-
*
|
|
5758
|
-
*
|
|
5759
|
-
|
|
5760
|
-
|
|
8321
|
+
* Without this a two-minute timer outlives every call that connected
|
|
8322
|
+
* normally, waiting to purge an entry that is already gone.
|
|
8323
|
+
*/
|
|
8324
|
+
cancelTokenExpiry(token) {
|
|
8325
|
+
const timer = this.pendingTokenExpiries.get(token);
|
|
8326
|
+
if (timer !== void 0) {
|
|
8327
|
+
clearTimeout(timer);
|
|
8328
|
+
this.pendingTokenExpiries.delete(token);
|
|
8329
|
+
}
|
|
8330
|
+
}
|
|
8331
|
+
// =========================================================================
|
|
8332
|
+
// Audio Bridge
|
|
8333
|
+
// =========================================================================
|
|
8334
|
+
/**
|
|
8335
|
+
* Drive one Twilio Media Stream connection from `start` to disconnect.
|
|
5761
8336
|
*
|
|
5762
|
-
*
|
|
5763
|
-
*
|
|
5764
|
-
*
|
|
5765
|
-
*
|
|
8337
|
+
* Twilio's `start` event names the call, which opens the matching GPT-Live
|
|
8338
|
+
* socket; from then on caller audio is relayed to the model and the model's
|
|
8339
|
+
* audio back to Twilio, until either side goes away. Whichever leg closes
|
|
8340
|
+
* first takes the other down with it, so a caller is never left connected to
|
|
8341
|
+
* silence.
|
|
5766
8342
|
*
|
|
5767
|
-
*
|
|
5768
|
-
*
|
|
5769
|
-
*
|
|
5770
|
-
*
|
|
5771
|
-
* if (session) {
|
|
5772
|
-
* await voiceChannel.sendResponse(session.conversationId, 'Still there?');
|
|
5773
|
-
* }
|
|
5774
|
-
* }
|
|
5775
|
-
* ```
|
|
8343
|
+
* Twilio streams audio without waiting for the GPT-Live socket to finish
|
|
8344
|
+
* connecting, so audio that arrives during that handshake is held and
|
|
8345
|
+
* forwarded, in order, once the model is ready — a caller who speaks the
|
|
8346
|
+
* instant the call connects is heard in full.
|
|
5776
8347
|
*
|
|
5777
|
-
*
|
|
5778
|
-
*
|
|
5779
|
-
*
|
|
5780
|
-
*
|
|
5781
|
-
* CLAUDE.md).
|
|
8348
|
+
* Called by `VoiceChannel.handleWebSocketConnection`; hosts serve the socket
|
|
8349
|
+
* rather than calling this directly.
|
|
8350
|
+
*
|
|
8351
|
+
* @param ws - The accepted Twilio-facing WebSocket.
|
|
5782
8352
|
*/
|
|
5783
|
-
|
|
5784
|
-
|
|
5785
|
-
|
|
5786
|
-
|
|
8353
|
+
handleWebSocket(ws) {
|
|
8354
|
+
let conversationId = null;
|
|
8355
|
+
ws.on("message", (data) => {
|
|
8356
|
+
void (async () => {
|
|
8357
|
+
const message = JSON.parse(data.toString());
|
|
8358
|
+
const event = typeof message.event === "string" ? message.event : "";
|
|
8359
|
+
if (event === "start") {
|
|
8360
|
+
const registered = this.registerCall(message.start, ws);
|
|
8361
|
+
conversationId = registered.conversationId;
|
|
8362
|
+
const connecting = this.connectModel(registered.conversationId);
|
|
8363
|
+
registered.call.modelReady = connecting.then(
|
|
8364
|
+
() => true,
|
|
8365
|
+
() => false
|
|
8366
|
+
);
|
|
8367
|
+
await connecting;
|
|
8368
|
+
} else if (event === "media") {
|
|
8369
|
+
const media = message.media ?? {};
|
|
8370
|
+
const payload = media.payload;
|
|
8371
|
+
if (conversationId !== null && typeof payload === "string" && payload) {
|
|
8372
|
+
const call = this.calls.get(conversationId);
|
|
8373
|
+
if (call === void 0) {
|
|
8374
|
+
return;
|
|
8375
|
+
}
|
|
8376
|
+
if (call.modelReady !== null && !await call.modelReady) {
|
|
8377
|
+
return;
|
|
8378
|
+
}
|
|
8379
|
+
this.modelSend(conversationId, {
|
|
8380
|
+
type: "session.input_audio.append",
|
|
8381
|
+
audio: payload
|
|
8382
|
+
});
|
|
8383
|
+
}
|
|
8384
|
+
} else if (event === "stop") {
|
|
8385
|
+
this.logger.info({ conversation_id: conversationId }, "Media stream stopped");
|
|
8386
|
+
if (conversationId !== null) {
|
|
8387
|
+
await this.cleanupCall(conversationId);
|
|
8388
|
+
}
|
|
8389
|
+
ws.close();
|
|
8390
|
+
}
|
|
8391
|
+
})().catch((err) => {
|
|
8392
|
+
this.logger.error({ err, conversation_id: conversationId }, "Media stream WebSocket error");
|
|
8393
|
+
ws.close();
|
|
8394
|
+
});
|
|
8395
|
+
});
|
|
8396
|
+
ws.on("close", () => {
|
|
8397
|
+
this.logger.info({ conversation_id: conversationId }, "Media stream WebSocket closed");
|
|
8398
|
+
if (conversationId !== null) {
|
|
8399
|
+
trackEvent("Websocket Disconnected", {
|
|
8400
|
+
account_sid: this.tacConfig.accountSid,
|
|
8401
|
+
channel: "voice",
|
|
8402
|
+
conversation_id: conversationId,
|
|
8403
|
+
provider: this.providerId,
|
|
8404
|
+
orchestrator_enabled: this.tacConfig.isOrchestratorEnabled()
|
|
8405
|
+
});
|
|
8406
|
+
void this.cleanupCall(conversationId).catch((err) => {
|
|
8407
|
+
this.logger.error({ err, conversation_id: conversationId }, "Call cleanup error");
|
|
8408
|
+
});
|
|
5787
8409
|
}
|
|
5788
|
-
}
|
|
5789
|
-
|
|
8410
|
+
});
|
|
8411
|
+
ws.on("error", (error) => {
|
|
8412
|
+
this.channel.handleErrorInternal(error, { conversationId });
|
|
8413
|
+
});
|
|
5790
8414
|
}
|
|
5791
|
-
// =========================================================================
|
|
5792
|
-
// Stream Task Management
|
|
5793
|
-
// =========================================================================
|
|
5794
8415
|
/**
|
|
5795
|
-
*
|
|
5796
|
-
*
|
|
5797
|
-
* @param
|
|
5798
|
-
*
|
|
5799
|
-
|
|
5800
|
-
|
|
5801
|
-
|
|
5802
|
-
|
|
5803
|
-
|
|
5804
|
-
|
|
5805
|
-
|
|
8416
|
+
* Handle Twilio's `start` event: track the call and open its session.
|
|
8417
|
+
*
|
|
8418
|
+
* @param start - The event's `start` body, parsed against
|
|
8419
|
+
* `StreamStartMessageSchema`.
|
|
8420
|
+
* @param ws - The Twilio-facing socket this call arrived on.
|
|
8421
|
+
* @returns The conversation id — which is the call SID — and the call's
|
|
8422
|
+
* freshly tracked transport state.
|
|
8423
|
+
*/
|
|
8424
|
+
registerCall(start, ws) {
|
|
8425
|
+
const message = StreamStartMessageSchema.parse(start ?? {});
|
|
8426
|
+
const conversationId = message.callSid;
|
|
8427
|
+
this.cancelInboundConfigExpiry(conversationId);
|
|
8428
|
+
const token = message.customParameters[SESSION_CONFIG_TOKEN_PARAM2];
|
|
8429
|
+
if (token !== void 0) {
|
|
8430
|
+
const pending = this.pendingSessionConfigs.get(token);
|
|
8431
|
+
if (pending !== void 0) {
|
|
8432
|
+
this.pendingSessionConfigs.delete(token);
|
|
8433
|
+
this.pendingSessionConfigs.set(conversationId, pending);
|
|
8434
|
+
}
|
|
8435
|
+
this.cancelTokenExpiry(token);
|
|
8436
|
+
}
|
|
8437
|
+
const call = new CallState2();
|
|
8438
|
+
call.twilioWs = ws;
|
|
8439
|
+
this.calls.set(conversationId, call);
|
|
8440
|
+
try {
|
|
8441
|
+
const session = this.channel.startConversationInternal(conversationId);
|
|
8442
|
+
session.callSid = message.callSid;
|
|
8443
|
+
session.metadata.streamSid = message.streamSid;
|
|
8444
|
+
session.metadata.transcript = [];
|
|
8445
|
+
trackEvent("Conversation Initialized", {
|
|
8446
|
+
account_sid: this.tacConfig.accountSid,
|
|
8447
|
+
channel: "voice",
|
|
8448
|
+
conversation_id: conversationId,
|
|
8449
|
+
provider: this.providerId,
|
|
8450
|
+
orchestrator_enabled: this.tacConfig.isOrchestratorEnabled()
|
|
8451
|
+
});
|
|
8452
|
+
} catch (err) {
|
|
8453
|
+
this.calls.delete(conversationId);
|
|
8454
|
+
this.pendingSessionConfigs.delete(conversationId);
|
|
8455
|
+
void this.channel.endConversationInternal(conversationId).catch(() => void 0);
|
|
8456
|
+
throw err;
|
|
8457
|
+
}
|
|
8458
|
+
this.logger.debug(
|
|
8459
|
+
{ conversation_id: conversationId, media_format: message.mediaFormat },
|
|
8460
|
+
"Media stream started"
|
|
8461
|
+
);
|
|
8462
|
+
return { conversationId, call };
|
|
5806
8463
|
}
|
|
5807
8464
|
/**
|
|
5808
|
-
*
|
|
8465
|
+
* Open this call's GPT-Live socket and send its session config.
|
|
5809
8466
|
*
|
|
5810
|
-
* @
|
|
5811
|
-
* @returns true if a task was cancelled, false otherwise
|
|
8467
|
+
* @internal
|
|
5812
8468
|
*/
|
|
5813
|
-
|
|
5814
|
-
const
|
|
5815
|
-
|
|
5816
|
-
|
|
5817
|
-
|
|
5818
|
-
|
|
5819
|
-
|
|
8469
|
+
async connectModel(conversationId) {
|
|
8470
|
+
const sessionConfig = this.resolveSessionConfig(conversationId);
|
|
8471
|
+
const modelWs = await this.openModelSocket(GPT_LIVE_URL, {
|
|
8472
|
+
Authorization: `Bearer ${this.config.openaiApiKey}`,
|
|
8473
|
+
"User-Agent": OPENAI_USER_AGENT
|
|
8474
|
+
});
|
|
8475
|
+
const call = this.calls.get(conversationId);
|
|
8476
|
+
if (call === void 0) {
|
|
8477
|
+
modelWs.close();
|
|
8478
|
+
return;
|
|
5820
8479
|
}
|
|
5821
|
-
|
|
8480
|
+
call.modelWs = modelWs;
|
|
8481
|
+
this.attachModelHandlers(conversationId, modelWs);
|
|
8482
|
+
this.logger.info({ conversation_id: conversationId }, "Connected to GPT-Live");
|
|
8483
|
+
this.modelSend(conversationId, { type: "session.start", session: sessionConfig });
|
|
5822
8484
|
}
|
|
5823
8485
|
/**
|
|
5824
|
-
*
|
|
8486
|
+
* The validated session config for this call: its own stashed override if it
|
|
8487
|
+
* has one, else the channel-wide default.
|
|
5825
8488
|
*
|
|
5826
|
-
* @
|
|
8489
|
+
* @throws {Error} if neither exists, if the audio format is one Twilio can't
|
|
8490
|
+
* carry, or if it names no model.
|
|
5827
8491
|
*/
|
|
5828
|
-
|
|
5829
|
-
this.
|
|
5830
|
-
this.
|
|
8492
|
+
resolveSessionConfig(conversationId) {
|
|
8493
|
+
const sessionConfig = this.pendingSessionConfigs.get(conversationId) ?? this.config.defaultSessionConfig;
|
|
8494
|
+
this.pendingSessionConfigs.delete(conversationId);
|
|
8495
|
+
if (sessionConfig === void 0) {
|
|
8496
|
+
throw new Error(
|
|
8497
|
+
`No sessionConfig available for call ${conversationId} \u2014 this call supplied none and defaultSessionConfig is not set either.`
|
|
8498
|
+
);
|
|
8499
|
+
}
|
|
8500
|
+
const audio = sessionConfig.audio ?? {};
|
|
8501
|
+
if (!isTwilioMediaStreamAudioFormat2(audio.format)) {
|
|
8502
|
+
throw new Error(
|
|
8503
|
+
`sessionConfig for call ${conversationId} has audio.format=${JSON.stringify(audio.format)}, expected ${JSON.stringify(TWILIO_AUDIO_FORMAT_FOR_GPT_LIVE)}. Twilio Media Streams is always 8kHz G.711 u-law; set audio.format to TWILIO_AUDIO_FORMAT_FOR_GPT_LIVE.`
|
|
8504
|
+
);
|
|
8505
|
+
}
|
|
8506
|
+
if (!sessionConfig.model) {
|
|
8507
|
+
throw new Error(`sessionConfig for call ${conversationId} must include 'model'.`);
|
|
8508
|
+
}
|
|
8509
|
+
return sessionConfig;
|
|
5831
8510
|
}
|
|
5832
8511
|
/**
|
|
5833
|
-
*
|
|
8512
|
+
* Wire up the model socket: dispatch its events, and tear the call down when
|
|
8513
|
+
* it goes away.
|
|
5834
8514
|
*
|
|
5835
|
-
*
|
|
5836
|
-
*
|
|
8515
|
+
* The Python SDK races its Twilio read against its model-event reader so the
|
|
8516
|
+
* Twilio leg dies with the model leg; `ws` is event-driven, so the same
|
|
8517
|
+
* guarantee is a close/error handler instead.
|
|
5837
8518
|
*/
|
|
5838
|
-
|
|
5839
|
-
|
|
5840
|
-
|
|
8519
|
+
attachModelHandlers(conversationId, modelWs) {
|
|
8520
|
+
modelWs.on("message", (raw) => {
|
|
8521
|
+
void this.handleModelMessage(conversationId, raw);
|
|
8522
|
+
});
|
|
8523
|
+
modelWs.on("close", () => {
|
|
8524
|
+
if (this.calls.has(conversationId)) {
|
|
8525
|
+
this.logger.info({ conversation_id: conversationId }, "Model connection ended");
|
|
8526
|
+
}
|
|
8527
|
+
this.endCallFromModel(conversationId);
|
|
8528
|
+
});
|
|
8529
|
+
modelWs.on("error", (error) => {
|
|
8530
|
+
this.logger.error({ err: error, conversation_id: conversationId }, "Model socket error");
|
|
8531
|
+
this.endCallFromModel(conversationId);
|
|
8532
|
+
});
|
|
5841
8533
|
}
|
|
5842
|
-
// =========================================================================
|
|
5843
|
-
// ConversationRelay TwiML Generation
|
|
5844
|
-
// =========================================================================
|
|
5845
8534
|
/**
|
|
5846
|
-
*
|
|
5847
|
-
*
|
|
5848
|
-
*
|
|
5849
|
-
* emitted as the `url` attribute), actionUrl, languages, customParameters, extra.
|
|
8535
|
+
* Hang up the Twilio leg because the model leg is gone, then clean up. A
|
|
8536
|
+
* no-op once the call has already been cleaned up, so both the model socket's
|
|
8537
|
+
* `close` and its `error` can call it.
|
|
5850
8538
|
*/
|
|
5851
|
-
|
|
5852
|
-
|
|
5853
|
-
|
|
5854
|
-
|
|
5855
|
-
|
|
5856
|
-
|
|
5857
|
-
"transcriptionLanguage",
|
|
5858
|
-
"voice",
|
|
5859
|
-
"ttsProvider",
|
|
5860
|
-
"transcriptionProvider",
|
|
5861
|
-
"speechModel",
|
|
5862
|
-
"elevenlabsTextNormalization",
|
|
5863
|
-
"eotThreshold",
|
|
5864
|
-
"partialPrompts",
|
|
5865
|
-
"deepgramSmartFormat",
|
|
5866
|
-
"speechTimeout",
|
|
5867
|
-
"interruptible",
|
|
5868
|
-
"interruptSensitivity",
|
|
5869
|
-
"reportInputDuringAgentSpeech",
|
|
5870
|
-
"ignoreBackchannel",
|
|
5871
|
-
"preemptible",
|
|
5872
|
-
"dtmfDetection",
|
|
5873
|
-
"hints",
|
|
5874
|
-
"events",
|
|
5875
|
-
"debug",
|
|
5876
|
-
"intelligenceService"
|
|
5877
|
-
];
|
|
8539
|
+
endCallFromModel(conversationId) {
|
|
8540
|
+
this.calls.get(conversationId)?.twilioWs?.close();
|
|
8541
|
+
void this.cleanupCall(conversationId).catch((err) => {
|
|
8542
|
+
this.logger.error({ err, conversation_id: conversationId }, "Call cleanup error");
|
|
8543
|
+
});
|
|
8544
|
+
}
|
|
5878
8545
|
/**
|
|
5879
|
-
*
|
|
8546
|
+
* Close this call's GPT-Live session gracefully, drop its transport state,
|
|
8547
|
+
* and end the session.
|
|
5880
8548
|
*
|
|
5881
|
-
*
|
|
5882
|
-
*
|
|
5883
|
-
*
|
|
5884
|
-
*
|
|
5885
|
-
* channel-less caller can pass everything in one object.
|
|
8549
|
+
* `session.close` asks the server to finalize the session, and teardown waits
|
|
8550
|
+
* up to {@link CLOSE_TIMEOUT_MS} for the `session.closed` answering it before
|
|
8551
|
+
* the socket goes away — otherwise the socket would be gone before the server
|
|
8552
|
+
* could finish.
|
|
5886
8553
|
*
|
|
5887
|
-
*
|
|
5888
|
-
*
|
|
5889
|
-
*
|
|
5890
|
-
* @returns TwiML XML string ready to return to Twilio.
|
|
5891
|
-
* @throws {Error} if no WebSocket URL is provided via either source.
|
|
8554
|
+
* Both legs can report the call ending, and the first one to arrive tears
|
|
8555
|
+
* down the other, so this runs at most once per call: a second invocation
|
|
8556
|
+
* finds the call either untracked or already closing, and returns.
|
|
5892
8557
|
*/
|
|
5893
|
-
|
|
5894
|
-
const
|
|
5895
|
-
if (
|
|
5896
|
-
|
|
5897
|
-
"generateTwiml requires a WebSocket URL \u2014 pass it explicitly or set options.websocketUrl."
|
|
5898
|
-
);
|
|
8558
|
+
async cleanupCall(conversationId) {
|
|
8559
|
+
const call = this.calls.get(conversationId);
|
|
8560
|
+
if (call === void 0 || this.closingCalls.has(conversationId)) {
|
|
8561
|
+
return;
|
|
5899
8562
|
}
|
|
5900
|
-
|
|
5901
|
-
const
|
|
5902
|
-
|
|
5903
|
-
|
|
5904
|
-
|
|
5905
|
-
|
|
5906
|
-
|
|
5907
|
-
|
|
5908
|
-
|
|
5909
|
-
|
|
8563
|
+
this.closingCalls.add(conversationId);
|
|
8564
|
+
const modelWs = call.modelWs;
|
|
8565
|
+
if (modelWs !== null) {
|
|
8566
|
+
let requested = true;
|
|
8567
|
+
try {
|
|
8568
|
+
modelWs.send(JSON.stringify({ type: "session.close" }));
|
|
8569
|
+
} catch (err) {
|
|
8570
|
+
requested = false;
|
|
8571
|
+
this.logger.debug(
|
|
8572
|
+
{ err, conversation_id: conversationId },
|
|
8573
|
+
"Error sending session.close to model socket"
|
|
8574
|
+
);
|
|
5910
8575
|
}
|
|
5911
|
-
|
|
5912
|
-
|
|
5913
|
-
|
|
5914
|
-
|
|
5915
|
-
|
|
5916
|
-
|
|
5917
|
-
|
|
8576
|
+
if (requested) {
|
|
8577
|
+
try {
|
|
8578
|
+
const acknowledged = await waitWithTimeout(call.closed, CLOSE_TIMEOUT_MS);
|
|
8579
|
+
if (!acknowledged) {
|
|
8580
|
+
this.logger.debug(
|
|
8581
|
+
{ conversation_id: conversationId },
|
|
8582
|
+
"Timed out waiting for session.closed"
|
|
8583
|
+
);
|
|
8584
|
+
}
|
|
8585
|
+
} catch (err) {
|
|
8586
|
+
this.logger.debug(
|
|
8587
|
+
{ err, conversation_id: conversationId },
|
|
8588
|
+
"Error waiting for session.closed"
|
|
5918
8589
|
);
|
|
5919
|
-
continue;
|
|
5920
8590
|
}
|
|
5921
|
-
relayAttrs[key] = value;
|
|
5922
|
-
}
|
|
5923
|
-
}
|
|
5924
|
-
const relay = connect.conversationRelay(
|
|
5925
|
-
relayAttrs
|
|
5926
|
-
);
|
|
5927
|
-
if (options.languages && options.languages.length > 0) {
|
|
5928
|
-
for (const lang of options.languages) {
|
|
5929
|
-
const langAttrs = this.filterUnsetValues(lang);
|
|
5930
|
-
relay.language(langAttrs);
|
|
5931
8591
|
}
|
|
5932
8592
|
}
|
|
5933
|
-
|
|
5934
|
-
|
|
5935
|
-
|
|
5936
|
-
|
|
5937
|
-
|
|
8593
|
+
this.calls.delete(conversationId);
|
|
8594
|
+
this.closingCalls.delete(conversationId);
|
|
8595
|
+
if (modelWs !== null) {
|
|
8596
|
+
try {
|
|
8597
|
+
modelWs.close();
|
|
8598
|
+
} catch (err) {
|
|
8599
|
+
this.logger.debug({ err, conversation_id: conversationId }, "Error closing model socket");
|
|
5938
8600
|
}
|
|
5939
8601
|
}
|
|
5940
|
-
|
|
8602
|
+
await this.channel.endConversationInternal(conversationId);
|
|
5941
8603
|
}
|
|
5942
8604
|
/**
|
|
5943
|
-
*
|
|
5944
|
-
*
|
|
8605
|
+
* Drop this provider's transport state on channel shutdown, including the
|
|
8606
|
+
* bookkeeping it keeps beyond the base class's.
|
|
5945
8607
|
*
|
|
5946
|
-
*
|
|
5947
|
-
*
|
|
5948
|
-
*
|
|
5949
|
-
* @throws {Error} if config validation fails
|
|
8608
|
+
* The token expiry timers are unref'd and delete themselves, so nothing hangs
|
|
8609
|
+
* without this — but a shut-down provider must not still be holding entries
|
|
8610
|
+
* for calls that can no longer arrive.
|
|
5950
8611
|
*/
|
|
5951
|
-
|
|
5952
|
-
const
|
|
5953
|
-
|
|
5954
|
-
const errorMessage = validationResult.error.issues.map((issue) => `${issue.path.join(".")}: ${issue.message}`).join(", ");
|
|
5955
|
-
throw new Error(`Invalid ConversationRelay configuration: ${errorMessage}`);
|
|
8612
|
+
shutdown() {
|
|
8613
|
+
for (const timer of this.pendingTokenExpiries.values()) {
|
|
8614
|
+
clearTimeout(timer);
|
|
5956
8615
|
}
|
|
5957
|
-
|
|
5958
|
-
|
|
5959
|
-
|
|
5960
|
-
|
|
5961
|
-
|
|
5962
|
-
|
|
5963
|
-
|
|
5964
|
-
|
|
5965
|
-
|
|
5966
|
-
|
|
8616
|
+
this.pendingTokenExpiries.clear();
|
|
8617
|
+
this.closingCalls.clear();
|
|
8618
|
+
super.shutdown();
|
|
8619
|
+
}
|
|
8620
|
+
/** Apply one parsed GPT-Live event to the call. */
|
|
8621
|
+
async dispatchModelEvent(conversationId, session, event) {
|
|
8622
|
+
switch (event.type) {
|
|
8623
|
+
case "error": {
|
|
8624
|
+
this.logger.error(
|
|
8625
|
+
{ conversation_id: conversationId, error: event.error },
|
|
8626
|
+
"GPT-Live error event"
|
|
8627
|
+
);
|
|
8628
|
+
break;
|
|
5967
8629
|
}
|
|
5968
|
-
|
|
5969
|
-
|
|
5970
|
-
|
|
5971
|
-
|
|
8630
|
+
case "session.closed": {
|
|
8631
|
+
this.recordGptLiveSessionId(conversationId, session, event);
|
|
8632
|
+
this.calls.get(conversationId)?.markClosed();
|
|
8633
|
+
break;
|
|
8634
|
+
}
|
|
8635
|
+
case "session.started": {
|
|
8636
|
+
this.recordGptLiveSessionId(conversationId, session, event);
|
|
8637
|
+
const instruction = this.config.welcomeInstruction;
|
|
8638
|
+
if (instruction !== null) {
|
|
8639
|
+
this.modelSend(conversationId, {
|
|
8640
|
+
type: "session.commentary.append",
|
|
8641
|
+
delegation_id: null,
|
|
8642
|
+
content: instruction
|
|
8643
|
+
});
|
|
8644
|
+
}
|
|
8645
|
+
break;
|
|
8646
|
+
}
|
|
8647
|
+
case "session.input_transcript.delta": {
|
|
8648
|
+
_GPTLiveProvider.appendTranscriptDelta(session, "user", event);
|
|
8649
|
+
break;
|
|
8650
|
+
}
|
|
8651
|
+
case "session.output_transcript.delta": {
|
|
8652
|
+
_GPTLiveProvider.appendTranscriptDelta(session, "assistant", event);
|
|
8653
|
+
break;
|
|
8654
|
+
}
|
|
8655
|
+
case "session.output_audio.delta": {
|
|
8656
|
+
const delta = event.delta;
|
|
8657
|
+
if (typeof delta !== "string" || !delta) {
|
|
8658
|
+
break;
|
|
8659
|
+
}
|
|
8660
|
+
this.twilioSend(conversationId, {
|
|
8661
|
+
event: "media",
|
|
8662
|
+
streamSid: session.metadata.streamSid,
|
|
8663
|
+
media: { payload: delta }
|
|
8664
|
+
});
|
|
8665
|
+
break;
|
|
8666
|
+
}
|
|
8667
|
+
case "response.event": {
|
|
8668
|
+
const inner = event.event ?? {};
|
|
8669
|
+
if (inner.type !== "response.output_item.done") {
|
|
8670
|
+
break;
|
|
8671
|
+
}
|
|
8672
|
+
const item = inner.item ?? {};
|
|
8673
|
+
if (item.type === "function_call" && item.status === "completed") {
|
|
8674
|
+
await this.handleFunctionCall(conversationId, item);
|
|
8675
|
+
}
|
|
8676
|
+
break;
|
|
5972
8677
|
}
|
|
5973
8678
|
}
|
|
5974
|
-
return response.toString();
|
|
5975
8679
|
}
|
|
5976
8680
|
/**
|
|
5977
|
-
*
|
|
5978
|
-
*
|
|
8681
|
+
* Run a Responses-delegated tool call and hand the result back.
|
|
8682
|
+
*
|
|
8683
|
+
* Always sends a `function_call_output` once a `call_id` is present — even a
|
|
8684
|
+
* tool that ran successfully can return something `JSON.stringify` throws on
|
|
8685
|
+
* (a circular object, a `BigInt`) or has no JSON form at all, which
|
|
8686
|
+
* `JSON.stringify` reports by returning `undefined` rather than throwing (a
|
|
8687
|
+
* void tool, a bare function, a `Symbol`). Either way the model would
|
|
8688
|
+
* otherwise be left waiting on a `call_id` it never gets a result for.
|
|
8689
|
+
* Without a `call_id` there is nothing to reply to, so the item is dropped
|
|
8690
|
+
* instead.
|
|
5979
8691
|
*/
|
|
5980
|
-
|
|
5981
|
-
const
|
|
5982
|
-
|
|
5983
|
-
|
|
5984
|
-
|
|
8692
|
+
async handleFunctionCall(conversationId, item) {
|
|
8693
|
+
const callId = item.call_id;
|
|
8694
|
+
if (typeof callId !== "string" || !callId) {
|
|
8695
|
+
this.logger.error(
|
|
8696
|
+
{ conversation_id: conversationId, item_keys: Object.keys(item) },
|
|
8697
|
+
"Received malformed function_call item without call_id"
|
|
8698
|
+
);
|
|
8699
|
+
return;
|
|
8700
|
+
}
|
|
8701
|
+
const name = item.name;
|
|
8702
|
+
let output;
|
|
8703
|
+
if (typeof name !== "string" || !name) {
|
|
8704
|
+
this.logger.error(
|
|
8705
|
+
{ conversation_id: conversationId, call_id: callId, item_keys: Object.keys(item) },
|
|
8706
|
+
"Received malformed function_call item without tool name"
|
|
8707
|
+
);
|
|
8708
|
+
output = JSON.stringify({ error: "Malformed function call: missing tool name." });
|
|
8709
|
+
} else {
|
|
8710
|
+
const result = await this.runToolCall(conversationId, name, item.arguments);
|
|
8711
|
+
try {
|
|
8712
|
+
const serialized = JSON.stringify(result);
|
|
8713
|
+
output = serialized ?? "null";
|
|
8714
|
+
} catch (err) {
|
|
8715
|
+
this.logger.error(
|
|
8716
|
+
{ err, conversation_id: conversationId, tool_name: name },
|
|
8717
|
+
"Tool returned a non-JSON-serializable result"
|
|
8718
|
+
);
|
|
8719
|
+
output = JSON.stringify({ error: `Tool '${name}' returned a non-serializable result.` });
|
|
5985
8720
|
}
|
|
5986
8721
|
}
|
|
5987
|
-
|
|
8722
|
+
this.modelSend(conversationId, {
|
|
8723
|
+
type: "response.item.create",
|
|
8724
|
+
item: { type: "function_call_output", call_id: callId, output }
|
|
8725
|
+
});
|
|
8726
|
+
this.modelSend(conversationId, { type: "response.create" });
|
|
5988
8727
|
}
|
|
5989
8728
|
/**
|
|
5990
|
-
*
|
|
8729
|
+
* Surface the GPT-Live session id from a session-snapshot event.
|
|
5991
8730
|
*
|
|
5992
|
-
*
|
|
5993
|
-
*
|
|
8731
|
+
* OpenAI support asks for this id when investigating a session, so it goes
|
|
8732
|
+
* where a caller can reach it — `session.metadata`, which outlives the call
|
|
8733
|
+
* into `onConversationEnded` — and is logged once per call.
|
|
5994
8734
|
*/
|
|
5995
|
-
|
|
5996
|
-
|
|
5997
|
-
|
|
5998
|
-
|
|
5999
|
-
|
|
6000
|
-
|
|
6001
|
-
|
|
8735
|
+
recordGptLiveSessionId(conversationId, session, event) {
|
|
8736
|
+
const snapshot = event.session ?? {};
|
|
8737
|
+
const sessionId = snapshot.id;
|
|
8738
|
+
if (typeof sessionId !== "string" || sessionId === "") {
|
|
8739
|
+
return;
|
|
8740
|
+
}
|
|
8741
|
+
if (session.metadata[GPT_LIVE_SESSION_ID_METADATA_KEY] === sessionId) {
|
|
8742
|
+
return;
|
|
8743
|
+
}
|
|
8744
|
+
session.metadata[GPT_LIVE_SESSION_ID_METADATA_KEY] = sessionId;
|
|
8745
|
+
this.logger.info(
|
|
8746
|
+
{ conversation_id: conversationId, gpt_live_session_id: sessionId },
|
|
8747
|
+
"GPT-Live session id"
|
|
8748
|
+
);
|
|
8749
|
+
}
|
|
8750
|
+
/** Accumulate one transcript delta into the in-progress turn. */
|
|
8751
|
+
static appendTranscriptDelta(session, role, event) {
|
|
8752
|
+
const text = event.delta;
|
|
8753
|
+
if (typeof text !== "string" || text === "") {
|
|
8754
|
+
return;
|
|
8755
|
+
}
|
|
8756
|
+
const existing = session.metadata.transcript;
|
|
8757
|
+
const transcript = Array.isArray(existing) ? existing : [];
|
|
8758
|
+
if (transcript !== existing) {
|
|
8759
|
+
session.metadata.transcript = transcript;
|
|
8760
|
+
}
|
|
8761
|
+
const last = transcript[transcript.length - 1];
|
|
8762
|
+
if (last !== void 0 && last.role === role) {
|
|
8763
|
+
last.text += text;
|
|
8764
|
+
} else {
|
|
8765
|
+
transcript.push({ role, text });
|
|
8766
|
+
}
|
|
8767
|
+
}
|
|
8768
|
+
};
|
|
8769
|
+
|
|
8770
|
+
// packages/core/src/channels/voice/media-streams/gpt-live/config.ts
|
|
8771
|
+
var GPTLiveProviderConfig = class extends MediaStreamsOpenAIProviderConfig {
|
|
8772
|
+
welcomeInstruction;
|
|
8773
|
+
constructor(opts = {}) {
|
|
8774
|
+
super(opts);
|
|
8775
|
+
this.welcomeInstruction = opts.welcomeInstruction ?? null;
|
|
8776
|
+
}
|
|
8777
|
+
createProvider(channel, tacConfig) {
|
|
8778
|
+
return new GPTLiveProvider(channel, tacConfig, this);
|
|
6002
8779
|
}
|
|
6003
8780
|
};
|
|
6004
8781
|
|
|
@@ -6197,6 +8974,21 @@ var TACTool = class {
|
|
|
6197
8974
|
}
|
|
6198
8975
|
};
|
|
6199
8976
|
}
|
|
8977
|
+
/**
|
|
8978
|
+
* Convert to OpenAI Realtime function calling format.
|
|
8979
|
+
*
|
|
8980
|
+
* Unlike {@link TACTool.toOpenAIFormat} (Chat Completions, which nests the
|
|
8981
|
+
* schema under a `function` key), Realtime's `session.tools` expects the
|
|
8982
|
+
* fields flat on the tool object.
|
|
8983
|
+
*/
|
|
8984
|
+
toRealtimeFormat() {
|
|
8985
|
+
return {
|
|
8986
|
+
type: "function",
|
|
8987
|
+
name: this.name,
|
|
8988
|
+
description: this.description,
|
|
8989
|
+
parameters: this.parameters
|
|
8990
|
+
};
|
|
8991
|
+
}
|
|
6200
8992
|
/**
|
|
6201
8993
|
* Convert to Anthropic tool calling format
|
|
6202
8994
|
*/
|
|
@@ -6778,16 +9570,7 @@ var TACServer = class {
|
|
|
6778
9570
|
}
|
|
6779
9571
|
const voiceChannel = this.voiceChannel;
|
|
6780
9572
|
const formData = request.body;
|
|
6781
|
-
const
|
|
6782
|
-
if (!parseResult.success) {
|
|
6783
|
-
this.fastify.log.error(
|
|
6784
|
-
{ errors: parseResult.error.issues },
|
|
6785
|
-
"Invalid ConversationRelay callback payload"
|
|
6786
|
-
);
|
|
6787
|
-
await reply.code(400).send({ error: "Invalid payload" });
|
|
6788
|
-
return;
|
|
6789
|
-
}
|
|
6790
|
-
const result = await voiceChannel.handleConversationRelayCallback(parseResult.data);
|
|
9573
|
+
const result = await voiceChannel.handleTwilioProviderCallback(formData);
|
|
6791
9574
|
await reply.code(result.status).type(result.contentType).send(result.content);
|
|
6792
9575
|
} catch (error) {
|
|
6793
9576
|
this.fastify.log.error(
|
|
@@ -7016,6 +9799,6 @@ var TACServer = class {
|
|
|
7016
9799
|
}
|
|
7017
9800
|
};
|
|
7018
9801
|
|
|
7019
|
-
export { ActionChannelSettingsSchema, ActionParticipantRefSchema, ActionResponseSchema, ActionTextContentSchema, AmdEventSchema, AnthropicToolSchema, AuthorInfoSchema, BaseChannel, BaseClient, BuiltInTools, CALL_EVENT_KINDS, CallEventKindSchema, CallOptionsSchema, CallStatusEventSchema, CaptureRuleSchema, ChannelSettingsSchema, ChannelTypeSchema, ChatChannel, CintelParticipantSchema, CommunicationContentSchema, CommunicationParticipantSchema, CommunicationSchema, ConversationAddressSchema, ConversationClient, ConversationConfigurationSchema, ConversationGroupingTypeSchema, ConversationIntelligenceConfigSchema, ConversationParticipantSchema, ConversationRelayAttributesSchema, ConversationRelayCallbackPayloadSchema, ConversationRelayConfigSchema, ConversationResponseSchema, ConversationSessionSchema, ConversationSummaryItemSchema, CreateConversationSummariesResponseSchema, CreateObservationResponseSchema, CreateObservationsRequestSchema, CustomParametersSchema, EMPTY_MEMORY_RESPONSE, EnvironmentVariables, ExecutionDetailsSchema, HandoffPayloadSchema, InitiateMessagingConversationOptionsSchema, InitiateVoiceConversationOptionsSchema, IntelligenceConfigurationSchema, InterruptMessageSchema, InterruptModeSchema, JSONSchemaSchema, KnowledgeBaseSchema, KnowledgeBaseStatusSchema, KnowledgeChunkResultSchema, KnowledgeClient, KnowledgeSearchResponseSchema, LanguageAttributesSchema, LanguageConfigSchema, ListCommunicationsResponseSchema, ListConversationsResponseSchema, ListParticipantsResponseSchema, MemoryChannelTypeSchema, MemoryClient, MemoryCommunicationContentSchema, MemoryCommunicationSchema, MemoryDeliveryStatusSchema, MemoryModeSchema, MemoryParticipantSchema, MemoryParticipantTypeSchema, MemoryPromptBuilder, MemoryRetrievalRequestSchema, MemoryRetrievalResponseSchema, MessageDirectionSchema, MessagingChannel, ObservationCreateRequestSchema, ObservationInfoSchema, OpenAIToolSchema, OperatorProcessingResultSchema, OperatorResultEventSchema, OperatorResultProcessor, OperatorResultSchema, OperatorSchema, ParticipantAddressSchema, ParticipantAddressTypeSchema, PendingHandoffDataSchema, ProfileLookupResponseSchema, ProfileResponseSchema, PromptMessageSchema, RCSChannel, RecordingEventSchema, SMSChannel, SendMessageActionPayloadSchema, SendMessageActionRequestSchema, SessionInfoSchema, SessionMessageSchema, SetupMessageSchema, StatusCallbackSchema, StatusTimeoutsSchema, SummaryInfoSchema, TAC, TACChannelTypeSchema, TACCommunicationAuthorSchema, TACCommunicationContentSchema, TACCommunicationSchema, TACConfig, TACConfigSchema, TACDeliveryStatusSchema, TACMemoryResponse, TACParticipantTypeSchema, TACServer, TACTool, TextTokenMessageSchema, ToolExecutionResultSchema, TranscriptionSchema, TranscriptionWordSchema, TwiMLOptionsSchema, TwiMLRequestSchema, TwilioMemoryConfigSchema, VoiceChannel, WebSocketMessageSchema, WhatsAppChannel, amdEventFromForm, buildHandoffPayload, callOptionsToCreateParams, callStatusEventFromForm, createKnowledgeSearchTool, createKnowledgeSearchToolAsync, createKnowledgeTools, createLogger, createMemoryRetrievalTool, createMemoryTools, createMessagingTools, createSendMessageTool, createStudioHandoffTool, defineTool, isConversationId, isParticipantId, isProfileId, maskAddress, maskEmail, maskPhone, postStudioHandoff, recordingEventFromForm, redactTwimlParameters, scrubObject, scrubPii, studioExecutionsUrl, studioVoiceHandoffUrl, twiMLRequestFromForm };
|
|
9802
|
+
export { ActionChannelSettingsSchema, ActionParticipantRefSchema, ActionResponseSchema, ActionTextContentSchema, AmdEventSchema, AnthropicToolSchema, AuthorInfoSchema, BaseChannel, BaseClient, BuiltInTools, CALL_EVENT_KINDS, CallEventKindSchema, CallOptionsSchema, CallStatusEventSchema, CaptureRuleSchema, ChannelSettingsSchema, ChannelTypeSchema, ChatChannel, CintelParticipantSchema, CommunicationContentSchema, CommunicationParticipantSchema, CommunicationSchema, ConversationAddressSchema, ConversationClient, ConversationConfigurationSchema, ConversationGroupingTypeSchema, ConversationIntelligenceConfigSchema, ConversationParticipantSchema, ConversationRelayAttributesSchema, ConversationRelayCallbackPayloadSchema, ConversationRelayConfigSchema, ConversationRelayProvider, ConversationRelayProviderConfig, ConversationResponseSchema, ConversationSessionSchema, ConversationSummaryItemSchema, CreateConversationSummariesResponseSchema, CreateObservationResponseSchema, CreateObservationsRequestSchema, CustomParametersSchema, DtmfMessageSchema, EMPTY_MEMORY_RESPONSE, EnvironmentVariables, ExecutionDetailsSchema, GPTLiveProvider, GPTLiveProviderConfig, GPT_LIVE_SESSION_ID_METADATA_KEY, HandoffPayloadSchema, InitiateMessagingConversationOptionsSchema, InitiateVoiceConversationOptionsGPTLiveSchema, InitiateVoiceConversationOptionsOpenAIRealtimeSchema, InitiateVoiceConversationOptionsSchema, IntelligenceConfigurationSchema, InterruptMessageSchema, InterruptModeSchema, JSONSchemaSchema, KnowledgeBaseSchema, KnowledgeBaseStatusSchema, KnowledgeChunkResultSchema, KnowledgeClient, KnowledgeSearchResponseSchema, LanguageAttributesSchema, LanguageConfigSchema, ListCommunicationsResponseSchema, ListConversationsResponseSchema, ListParticipantsResponseSchema, MediaStreamsOpenAICallState, MediaStreamsOpenAIProvider, MediaStreamsOpenAIProviderConfig, MediaStreamsProviderConfig, MemoryChannelTypeSchema, MemoryClient, MemoryCommunicationContentSchema, MemoryCommunicationSchema, MemoryDeliveryStatusSchema, MemoryModeSchema, MemoryParticipantSchema, MemoryParticipantTypeSchema, MemoryPromptBuilder, MemoryRetrievalRequestSchema, MemoryRetrievalResponseSchema, MessageDirectionSchema, MessagingChannel, OPENAI_USER_AGENT, ObservationCreateRequestSchema, ObservationInfoSchema, OpenAIRealtimeProvider, OpenAIRealtimeProviderConfig, OpenAIRealtimeToolSchema, OpenAIToolSchema, OperatorProcessingResultSchema, OperatorResultEventSchema, OperatorResultProcessor, OperatorResultSchema, OperatorSchema, ParticipantAddressSchema, ParticipantAddressTypeSchema, PendingHandoffDataSchema, ProfileLookupResponseSchema, ProfileResponseSchema, PromptMessageSchema, RCSChannel, RecordingEventSchema, SMSChannel, SendMessageActionPayloadSchema, SendMessageActionRequestSchema, SessionInfoSchema, SessionMessageSchema, SetupMessageSchema, StatusCallbackSchema, StatusTimeoutsSchema, StreamStartMessageSchema, SummaryInfoSchema, TAC, TACChannelTypeSchema, TACCommunicationAuthorSchema, TACCommunicationContentSchema, TACCommunicationSchema, TACConfig, TACConfigSchema, TACDeliveryStatusSchema, TACMemoryResponse, TACParticipantTypeSchema, TACServer, TACTool, TWILIO_AUDIO_FORMAT_FOR_GPT_LIVE, TWILIO_AUDIO_FORMAT_FOR_REALTIME, TextTokenMessageSchema, ToolExecutionResultSchema, TranscriptionSchema, TranscriptionWordSchema, TwiMLBuilderMediaStreams, TwiMLOptionsSchema, TwiMLRequestSchema, TwilioMemoryConfigSchema, VoiceChannel, VoiceProvider, VoiceProviderConfig, VoiceTwiMLOptionsConversationRelaySchema, VoiceTwiMLOptionsMediaStreamsSchema, VoiceTwiMLOptionsSchema, WebSocketMessageSchema, WhatsAppChannel, amdEventFromForm, buildHandoffPayload, callOptionsToCreateParams, callStatusEventFromForm, createKnowledgeSearchTool, createKnowledgeSearchToolAsync, createKnowledgeTools, createLogger, createMemoryRetrievalTool, createMemoryTools, createMessagingTools, createSendMessageTool, createStudioHandoffTool, defineTool, generateStreamTwiml, isConversationId, isParticipantId, isProfileId, maskAddress, maskEmail, maskPhone, postStudioHandoff, recordingEventFromForm, redactTwimlParameters, scrubObject, scrubPii, shutdownAnalytics, studioExecutionsUrl, studioVoiceHandoffUrl, trackEvent, twiMLRequestFromForm };
|
|
7020
9803
|
//# sourceMappingURL=index.js.map
|
|
7021
9804
|
//# sourceMappingURL=index.js.map
|