realtime-voice-agents 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +298 -0
- package/assets/elevator-jazz.ulaw +1 -0
- package/assets/keyboard-typing.ulaw +1 -0
- package/assets/lofi.ulaw +1 -0
- package/assets/ringing.ulaw +1 -0
- package/assets/thinking-hum.ulaw +1 -0
- package/dist/BackgroundAudioPlayer-iMcivjis.mjs +282 -0
- package/dist/BackgroundAudioPlayer-jfRULWKC.cjs +313 -0
- package/dist/BaseRealtimeProvider-BQigr5mB.mjs +62 -0
- package/dist/BaseRealtimeProvider-BehPNT1r.d.cts +239 -0
- package/dist/BaseRealtimeProvider-ClP8Wx1X.d.mts +239 -0
- package/dist/BaseRealtimeProvider-DI4pKtOb.cjs +91 -0
- package/dist/GeminiLiveProvider-DvkTgzjG.d.cts +101 -0
- package/dist/GeminiLiveProvider-x2nyx5aO.d.mts +101 -0
- package/dist/InMemorySessionStore-B0_i-DOU.cjs +34 -0
- package/dist/InMemorySessionStore-B5_rq61L.d.cts +83 -0
- package/dist/InMemorySessionStore-B5_rq61L.d.mts +83 -0
- package/dist/InMemorySessionStore-DYParOJO.mjs +29 -0
- package/dist/OpenAICompatibleProvider-Bdtl-UXH.mjs +402 -0
- package/dist/OpenAICompatibleProvider-NS4cKVQj.cjs +409 -0
- package/dist/audio.cjs +21 -0
- package/dist/audio.d.cts +165 -0
- package/dist/audio.d.mts +165 -0
- package/dist/audio.mjs +4 -0
- package/dist/env-DSnGaERV.cjs +19 -0
- package/dist/env-DUwUWTsg.mjs +14 -0
- package/dist/events-BUMYdETO.d.cts +24 -0
- package/dist/events-BUMYdETO.d.mts +24 -0
- package/dist/events-BxDTIKKq.cjs +48 -0
- package/dist/events-BylBSBW-.mjs +43 -0
- package/dist/gemini.cjs +401 -0
- package/dist/gemini.d.cts +41 -0
- package/dist/gemini.d.mts +41 -0
- package/dist/gemini.mjs +395 -0
- package/dist/index.cjs +2142 -0
- package/dist/index.d.cts +920 -0
- package/dist/index.d.mts +920 -0
- package/dist/index.mjs +2113 -0
- package/dist/mulaw--cwU2c9L.mjs +64 -0
- package/dist/mulaw-DLUObjdP.cjs +117 -0
- package/dist/openai.cjs +61 -0
- package/dist/openai.d.cts +121 -0
- package/dist/openai.d.mts +121 -0
- package/dist/openai.mjs +53 -0
- package/dist/presets-Bf75YXs5.d.cts +24 -0
- package/dist/presets-Bf75YXs5.d.mts +24 -0
- package/dist/rest-BYqiVOhe.mjs +265 -0
- package/dist/rest-BvUKut_k.cjs +300 -0
- package/dist/rolldown-runtime-VH7oDXx4.cjs +28 -0
- package/dist/session-config-BVLl7-ha.mjs +74 -0
- package/dist/session-config-c8sOw1XL.cjs +85 -0
- package/dist/store.cjs +3 -0
- package/dist/store.d.cts +2 -0
- package/dist/store.d.mts +2 -0
- package/dist/store.mjs +2 -0
- package/dist/testing.cjs +544 -0
- package/dist/testing.d.cts +191 -0
- package/dist/testing.d.mts +191 -0
- package/dist/testing.mjs +538 -0
- package/dist/transcode-C9aJG7_W.cjs +215 -0
- package/dist/transcode-CsAp97G9.mjs +198 -0
- package/dist/transport-B_PJFIVd.d.cts +155 -0
- package/dist/transport-CEaLFV4E.d.mts +155 -0
- package/dist/twilio.cjs +8 -0
- package/dist/twilio.d.cts +3 -0
- package/dist/twilio.d.mts +3 -0
- package/dist/twilio.mjs +2 -0
- package/dist/twiml-z9LjoF4_.d.cts +67 -0
- package/dist/twiml-z9LjoF4_.d.mts +67 -0
- package/dist/xai.cjs +92 -0
- package/dist/xai.d.cts +35 -0
- package/dist/xai.d.mts +35 -0
- package/dist/xai.mjs +86 -0
- package/package.json +183 -0
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
//#region src/audio/mulaw.ts
|
|
2
|
+
/**
|
|
3
|
+
* G.711 μ-law codec (the format Twilio Media Streams speak: 8 kHz, 1 byte/sample).
|
|
4
|
+
*
|
|
5
|
+
* Decode uses a 256-entry lookup table; encode uses the canonical Sun g711.c
|
|
6
|
+
* algorithm (bias 0x84, clip 32635). μ-law silence is 0xFF.
|
|
7
|
+
*/
|
|
8
|
+
/** One byte of μ-law silence (encodes linear 0). */
|
|
9
|
+
const MULAW_SILENCE_BYTE = 255;
|
|
10
|
+
/** Samples per second on the Twilio media stream. μ-law is 1 byte per sample. */
|
|
11
|
+
const MULAW_SAMPLE_RATE = 8e3;
|
|
12
|
+
/** Bytes per 20 ms Twilio media frame at 8 kHz μ-law. */
|
|
13
|
+
const MULAW_FRAME_BYTES_20MS = 160;
|
|
14
|
+
function decodeSampleUncached(byte) {
|
|
15
|
+
const u = ~byte & 255;
|
|
16
|
+
let t = ((u & 15) << 3) + 132;
|
|
17
|
+
t <<= (u & 112) >> 4;
|
|
18
|
+
return u & 128 ? 132 - t : t - 132;
|
|
19
|
+
}
|
|
20
|
+
const DECODE_TABLE = /* @__PURE__ */ new Int16Array(256);
|
|
21
|
+
for (let i = 0; i < 256; i++) DECODE_TABLE[i] = decodeSampleUncached(i);
|
|
22
|
+
const BIAS = 132;
|
|
23
|
+
const CLIP = 32635;
|
|
24
|
+
/** Decode a single μ-law byte to a linear PCM16 sample. */
|
|
25
|
+
function mulawDecodeSample(byte) {
|
|
26
|
+
return DECODE_TABLE[byte & 255];
|
|
27
|
+
}
|
|
28
|
+
/** Encode a single linear PCM16 sample to a μ-law byte. */
|
|
29
|
+
function mulawEncodeSample(sample) {
|
|
30
|
+
let pcm = sample;
|
|
31
|
+
const sign = pcm < 0 ? 128 : 0;
|
|
32
|
+
if (sign) pcm = -pcm;
|
|
33
|
+
if (pcm > CLIP) pcm = CLIP;
|
|
34
|
+
pcm += BIAS;
|
|
35
|
+
let exponent = 7;
|
|
36
|
+
for (let mask = 16384; (pcm & mask) === 0 && exponent > 0; mask >>= 1) exponent--;
|
|
37
|
+
const mantissa = pcm >> exponent + 3 & 15;
|
|
38
|
+
return ~(sign | exponent << 4 | mantissa) & 255;
|
|
39
|
+
}
|
|
40
|
+
/** Decode a μ-law buffer to linear PCM16 samples. */
|
|
41
|
+
function mulawToPcm16(mulaw) {
|
|
42
|
+
const out = new Int16Array(mulaw.length);
|
|
43
|
+
for (let i = 0; i < mulaw.length; i++) out[i] = DECODE_TABLE[mulaw[i]];
|
|
44
|
+
return out;
|
|
45
|
+
}
|
|
46
|
+
/** Encode linear PCM16 samples to a μ-law buffer. */
|
|
47
|
+
function pcm16ToMulaw(pcm) {
|
|
48
|
+
const out = new Uint8Array(pcm.length);
|
|
49
|
+
for (let i = 0; i < pcm.length; i++) out[i] = mulawEncodeSample(pcm[i]);
|
|
50
|
+
return out;
|
|
51
|
+
}
|
|
52
|
+
/** Duration in milliseconds of a μ-law byte count at 8 kHz (1 byte = 125 µs). */
|
|
53
|
+
function mulawBytesToMs(bytes) {
|
|
54
|
+
return bytes / MULAW_SAMPLE_RATE * 1e3;
|
|
55
|
+
}
|
|
56
|
+
/** Byte length of a base64 string without decoding it. */
|
|
57
|
+
function base64ByteLength(base64) {
|
|
58
|
+
let padding = 0;
|
|
59
|
+
if (base64.endsWith("==")) padding = 2;
|
|
60
|
+
else if (base64.endsWith("=")) padding = 1;
|
|
61
|
+
return base64.length * 3 / 4 - padding;
|
|
62
|
+
}
|
|
63
|
+
//#endregion
|
|
64
|
+
export { mulawBytesToMs as a, mulawToPcm16 as c, base64ByteLength as i, pcm16ToMulaw as l, MULAW_SAMPLE_RATE as n, mulawDecodeSample as o, MULAW_SILENCE_BYTE as r, mulawEncodeSample as s, MULAW_FRAME_BYTES_20MS as t };
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
//#region src/audio/mulaw.ts
|
|
2
|
+
/**
|
|
3
|
+
* G.711 μ-law codec (the format Twilio Media Streams speak: 8 kHz, 1 byte/sample).
|
|
4
|
+
*
|
|
5
|
+
* Decode uses a 256-entry lookup table; encode uses the canonical Sun g711.c
|
|
6
|
+
* algorithm (bias 0x84, clip 32635). μ-law silence is 0xFF.
|
|
7
|
+
*/
|
|
8
|
+
/** One byte of μ-law silence (encodes linear 0). */
|
|
9
|
+
const MULAW_SILENCE_BYTE = 255;
|
|
10
|
+
/** Samples per second on the Twilio media stream. μ-law is 1 byte per sample. */
|
|
11
|
+
const MULAW_SAMPLE_RATE = 8e3;
|
|
12
|
+
/** Bytes per 20 ms Twilio media frame at 8 kHz μ-law. */
|
|
13
|
+
const MULAW_FRAME_BYTES_20MS = 160;
|
|
14
|
+
function decodeSampleUncached(byte) {
|
|
15
|
+
const u = ~byte & 255;
|
|
16
|
+
let t = ((u & 15) << 3) + 132;
|
|
17
|
+
t <<= (u & 112) >> 4;
|
|
18
|
+
return u & 128 ? 132 - t : t - 132;
|
|
19
|
+
}
|
|
20
|
+
const DECODE_TABLE = /* @__PURE__ */ new Int16Array(256);
|
|
21
|
+
for (let i = 0; i < 256; i++) DECODE_TABLE[i] = decodeSampleUncached(i);
|
|
22
|
+
const BIAS = 132;
|
|
23
|
+
const CLIP = 32635;
|
|
24
|
+
/** Decode a single μ-law byte to a linear PCM16 sample. */
|
|
25
|
+
function mulawDecodeSample(byte) {
|
|
26
|
+
return DECODE_TABLE[byte & 255];
|
|
27
|
+
}
|
|
28
|
+
/** Encode a single linear PCM16 sample to a μ-law byte. */
|
|
29
|
+
function mulawEncodeSample(sample) {
|
|
30
|
+
let pcm = sample;
|
|
31
|
+
const sign = pcm < 0 ? 128 : 0;
|
|
32
|
+
if (sign) pcm = -pcm;
|
|
33
|
+
if (pcm > CLIP) pcm = CLIP;
|
|
34
|
+
pcm += BIAS;
|
|
35
|
+
let exponent = 7;
|
|
36
|
+
for (let mask = 16384; (pcm & mask) === 0 && exponent > 0; mask >>= 1) exponent--;
|
|
37
|
+
const mantissa = pcm >> exponent + 3 & 15;
|
|
38
|
+
return ~(sign | exponent << 4 | mantissa) & 255;
|
|
39
|
+
}
|
|
40
|
+
/** Decode a μ-law buffer to linear PCM16 samples. */
|
|
41
|
+
function mulawToPcm16(mulaw) {
|
|
42
|
+
const out = new Int16Array(mulaw.length);
|
|
43
|
+
for (let i = 0; i < mulaw.length; i++) out[i] = DECODE_TABLE[mulaw[i]];
|
|
44
|
+
return out;
|
|
45
|
+
}
|
|
46
|
+
/** Encode linear PCM16 samples to a μ-law buffer. */
|
|
47
|
+
function pcm16ToMulaw(pcm) {
|
|
48
|
+
const out = new Uint8Array(pcm.length);
|
|
49
|
+
for (let i = 0; i < pcm.length; i++) out[i] = mulawEncodeSample(pcm[i]);
|
|
50
|
+
return out;
|
|
51
|
+
}
|
|
52
|
+
/** Duration in milliseconds of a μ-law byte count at 8 kHz (1 byte = 125 µs). */
|
|
53
|
+
function mulawBytesToMs(bytes) {
|
|
54
|
+
return bytes / MULAW_SAMPLE_RATE * 1e3;
|
|
55
|
+
}
|
|
56
|
+
/** Byte length of a base64 string without decoding it. */
|
|
57
|
+
function base64ByteLength(base64) {
|
|
58
|
+
let padding = 0;
|
|
59
|
+
if (base64.endsWith("==")) padding = 2;
|
|
60
|
+
else if (base64.endsWith("=")) padding = 1;
|
|
61
|
+
return base64.length * 3 / 4 - padding;
|
|
62
|
+
}
|
|
63
|
+
//#endregion
|
|
64
|
+
Object.defineProperty(exports, "MULAW_FRAME_BYTES_20MS", {
|
|
65
|
+
enumerable: true,
|
|
66
|
+
get: function() {
|
|
67
|
+
return MULAW_FRAME_BYTES_20MS;
|
|
68
|
+
}
|
|
69
|
+
});
|
|
70
|
+
Object.defineProperty(exports, "MULAW_SAMPLE_RATE", {
|
|
71
|
+
enumerable: true,
|
|
72
|
+
get: function() {
|
|
73
|
+
return MULAW_SAMPLE_RATE;
|
|
74
|
+
}
|
|
75
|
+
});
|
|
76
|
+
Object.defineProperty(exports, "MULAW_SILENCE_BYTE", {
|
|
77
|
+
enumerable: true,
|
|
78
|
+
get: function() {
|
|
79
|
+
return MULAW_SILENCE_BYTE;
|
|
80
|
+
}
|
|
81
|
+
});
|
|
82
|
+
Object.defineProperty(exports, "base64ByteLength", {
|
|
83
|
+
enumerable: true,
|
|
84
|
+
get: function() {
|
|
85
|
+
return base64ByteLength;
|
|
86
|
+
}
|
|
87
|
+
});
|
|
88
|
+
Object.defineProperty(exports, "mulawBytesToMs", {
|
|
89
|
+
enumerable: true,
|
|
90
|
+
get: function() {
|
|
91
|
+
return mulawBytesToMs;
|
|
92
|
+
}
|
|
93
|
+
});
|
|
94
|
+
Object.defineProperty(exports, "mulawDecodeSample", {
|
|
95
|
+
enumerable: true,
|
|
96
|
+
get: function() {
|
|
97
|
+
return mulawDecodeSample;
|
|
98
|
+
}
|
|
99
|
+
});
|
|
100
|
+
Object.defineProperty(exports, "mulawEncodeSample", {
|
|
101
|
+
enumerable: true,
|
|
102
|
+
get: function() {
|
|
103
|
+
return mulawEncodeSample;
|
|
104
|
+
}
|
|
105
|
+
});
|
|
106
|
+
Object.defineProperty(exports, "mulawToPcm16", {
|
|
107
|
+
enumerable: true,
|
|
108
|
+
get: function() {
|
|
109
|
+
return mulawToPcm16;
|
|
110
|
+
}
|
|
111
|
+
});
|
|
112
|
+
Object.defineProperty(exports, "pcm16ToMulaw", {
|
|
113
|
+
enumerable: true,
|
|
114
|
+
get: function() {
|
|
115
|
+
return pcm16ToMulaw;
|
|
116
|
+
}
|
|
117
|
+
});
|
package/dist/openai.cjs
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
|
+
const require_session_config = require("./session-config-c8sOw1XL.cjs");
|
|
3
|
+
const require_env = require("./env-DSnGaERV.cjs");
|
|
4
|
+
const require_OpenAICompatibleProvider = require("./OpenAICompatibleProvider-NS4cKVQj.cjs");
|
|
5
|
+
//#region src/openai.ts
|
|
6
|
+
/** OpenAI Realtime provider (GA API) — `realtime-voice-agents/openai`. */
|
|
7
|
+
/** Env vars checked (in order) when no explicit apiKey is passed. */
|
|
8
|
+
const OPENAI_KEY_ENV_VARS = [
|
|
9
|
+
"OPENAI_API_KEY",
|
|
10
|
+
"OPENAI_KEY",
|
|
11
|
+
"OPEN_AI_API_KEY"
|
|
12
|
+
];
|
|
13
|
+
const OPENAI_DEFAULT_MODEL = "gpt-realtime";
|
|
14
|
+
const OPENAI_DEFAULT_VOICE = "marin";
|
|
15
|
+
/**
|
|
16
|
+
* Voices supported by realtime models. OpenAI recommends marin/cedar (the
|
|
17
|
+
* realtime-native ones — audibly better once 8 kHz telephony takes its cut).
|
|
18
|
+
* The legacy TTS voices (fable, onyx, nova) are NOT supported and fail in
|
|
19
|
+
* unpleasant ways instead of erroring cleanly.
|
|
20
|
+
*/
|
|
21
|
+
const OPENAI_REALTIME_VOICES = [
|
|
22
|
+
"marin",
|
|
23
|
+
"cedar",
|
|
24
|
+
"alloy",
|
|
25
|
+
"ash",
|
|
26
|
+
"ballad",
|
|
27
|
+
"coral",
|
|
28
|
+
"echo",
|
|
29
|
+
"sage",
|
|
30
|
+
"shimmer",
|
|
31
|
+
"verse"
|
|
32
|
+
];
|
|
33
|
+
/** Create a provider factory for the bridge (`provider: openaiRealtime({...})`). */
|
|
34
|
+
function openaiRealtime(options = {}) {
|
|
35
|
+
const apiKey = require_env.resolveApiKey(options.apiKey, OPENAI_KEY_ENV_VARS);
|
|
36
|
+
if (!apiKey) throw new Error(`openaiRealtime: apiKey missing (pass apiKey or set one of ${OPENAI_KEY_ENV_VARS.join("/")})`);
|
|
37
|
+
const config = {
|
|
38
|
+
apiKey,
|
|
39
|
+
model: options.model ?? "gpt-realtime",
|
|
40
|
+
voice: options.voice ?? "marin",
|
|
41
|
+
baseUrl: options.baseUrl,
|
|
42
|
+
defaultVad: options.vad,
|
|
43
|
+
defaultTranscription: options.transcription,
|
|
44
|
+
headers: options.headers,
|
|
45
|
+
extraSessionOptions: options.sessionOptions,
|
|
46
|
+
connectTimeoutMs: options.connectTimeoutMs,
|
|
47
|
+
providerName: "openai"
|
|
48
|
+
};
|
|
49
|
+
return ({ logger }) => {
|
|
50
|
+
return new require_OpenAICompatibleProvider.OpenAICompatibleProvider(config, logger);
|
|
51
|
+
};
|
|
52
|
+
}
|
|
53
|
+
//#endregion
|
|
54
|
+
exports.OPENAI_DEFAULT_MODEL = OPENAI_DEFAULT_MODEL;
|
|
55
|
+
exports.OPENAI_DEFAULT_VOICE = OPENAI_DEFAULT_VOICE;
|
|
56
|
+
exports.OPENAI_KEY_ENV_VARS = OPENAI_KEY_ENV_VARS;
|
|
57
|
+
exports.OPENAI_REALTIME_VOICES = OPENAI_REALTIME_VOICES;
|
|
58
|
+
exports.OpenAICompatibleProvider = require_OpenAICompatibleProvider.OpenAICompatibleProvider;
|
|
59
|
+
exports.buildSessionUpdate = require_session_config.buildSessionUpdate;
|
|
60
|
+
exports.buildTurnDetection = require_session_config.buildTurnDetection;
|
|
61
|
+
exports.openaiRealtime = openaiRealtime;
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
import { c as VadConfig, h as Logger, i as ProviderSessionInit, m as ProviderCapabilities, n as ProviderFactory, o as SendTextOptions, s as SendToolResultOptions, t as BaseRealtimeProvider } from "./BaseRealtimeProvider-BehPNT1r.cjs";
|
|
2
|
+
//#region src/providers/openai-compatible/OpenAICompatibleProvider.d.ts
|
|
3
|
+
interface OpenAICompatibleProviderConfig {
|
|
4
|
+
apiKey: string;
|
|
5
|
+
model: string;
|
|
6
|
+
/** WS endpoint; `?model=` is appended. Default OpenAI's realtime endpoint. */
|
|
7
|
+
baseUrl?: string;
|
|
8
|
+
voice?: string;
|
|
9
|
+
/** VAD used when the session init doesn't specify one. `null` disables. */
|
|
10
|
+
defaultVad?: VadConfig | null;
|
|
11
|
+
/** Transcription used when the session init doesn't specify one. */
|
|
12
|
+
defaultTranscription?: {
|
|
13
|
+
model?: string;
|
|
14
|
+
language?: string;
|
|
15
|
+
} | false;
|
|
16
|
+
headers?: Record<string, string>;
|
|
17
|
+
/** Provider-native session fields, deep-merged into session.update last. */
|
|
18
|
+
extraSessionOptions?: Record<string, unknown>;
|
|
19
|
+
connectTimeoutMs?: number;
|
|
20
|
+
capabilityOverrides?: Partial<ProviderCapabilities>;
|
|
21
|
+
/** Provider display name for logs/events (e.g. 'openai', 'xai'). */
|
|
22
|
+
providerName?: string;
|
|
23
|
+
/**
|
|
24
|
+
* Session payload builder override for OpenAI-compatible services whose
|
|
25
|
+
* session shape diverges (xAI puts voice/turn_detection at the session
|
|
26
|
+
* root). Defaults to the OpenAI GA builder.
|
|
27
|
+
*/
|
|
28
|
+
buildSession?: (init: ProviderSessionInit, config: {
|
|
29
|
+
defaultVoice?: string;
|
|
30
|
+
extraSessionOptions?: Record<string, unknown>;
|
|
31
|
+
}) => Record<string, unknown>;
|
|
32
|
+
}
|
|
33
|
+
declare class OpenAICompatibleProvider extends BaseRealtimeProvider {
|
|
34
|
+
readonly name: string;
|
|
35
|
+
readonly capabilities: ProviderCapabilities;
|
|
36
|
+
private readonly config;
|
|
37
|
+
private readonly logger;
|
|
38
|
+
private ws;
|
|
39
|
+
private sessionInit;
|
|
40
|
+
private ready;
|
|
41
|
+
private intentionalClose;
|
|
42
|
+
private currentResponseId;
|
|
43
|
+
/**
|
|
44
|
+
* response.create serialization. The GA API rejects a `response.create`
|
|
45
|
+
* issued while another response is in flight
|
|
46
|
+
* (`conversation_already_has_active_response`), so creates requested
|
|
47
|
+
* mid-response wait in a single pending slot and fire on `response.done`.
|
|
48
|
+
* Later requests coalesce into that slot — one response reads the whole
|
|
49
|
+
* conversation state, so only the instructions payload is worth keeping.
|
|
50
|
+
*/
|
|
51
|
+
private responseActive;
|
|
52
|
+
private pendingCreate;
|
|
53
|
+
private lastCreateSent;
|
|
54
|
+
constructor(config: OpenAICompatibleProviderConfig, logger?: Logger);
|
|
55
|
+
get isConnected(): boolean;
|
|
56
|
+
connect(init: ProviderSessionInit): Promise<void>;
|
|
57
|
+
close(): Promise<void>;
|
|
58
|
+
sendAudio(base64Mulaw: string): void;
|
|
59
|
+
sendText(text: string, options?: SendTextOptions): void;
|
|
60
|
+
sendToolResult(callId: string, output: unknown, options?: SendToolResultOptions): void;
|
|
61
|
+
createResponse(options?: {
|
|
62
|
+
instructions?: string;
|
|
63
|
+
}): void;
|
|
64
|
+
private sendCreate;
|
|
65
|
+
updateSession(patch: Partial<ProviderSessionInit>): Promise<void>;
|
|
66
|
+
private buildSessionPayload;
|
|
67
|
+
cancelResponse(): void;
|
|
68
|
+
truncatePlayback(itemId: string, audioEndMs: number): void;
|
|
69
|
+
private handleEvent;
|
|
70
|
+
private parseEvent;
|
|
71
|
+
private send;
|
|
72
|
+
}
|
|
73
|
+
//#endregion
|
|
74
|
+
//#region src/providers/openai-compatible/session-config.d.ts
|
|
75
|
+
declare function buildTurnDetection(vad: VadConfig | null | undefined): Record<string, unknown> | null;
|
|
76
|
+
interface SessionPayloadConfig {
|
|
77
|
+
defaultVoice?: string;
|
|
78
|
+
/** Provider-native session fields, deep-merged last (escape hatch). */
|
|
79
|
+
extraSessionOptions?: Record<string, unknown>;
|
|
80
|
+
}
|
|
81
|
+
declare function buildSessionUpdate(init: ProviderSessionInit, config?: SessionPayloadConfig): Record<string, unknown>;
|
|
82
|
+
//#endregion
|
|
83
|
+
//#region src/openai.d.ts
|
|
84
|
+
/** Env vars checked (in order) when no explicit apiKey is passed. */
|
|
85
|
+
declare const OPENAI_KEY_ENV_VARS: readonly ["OPENAI_API_KEY", "OPENAI_KEY", "OPEN_AI_API_KEY"];
|
|
86
|
+
declare const OPENAI_DEFAULT_MODEL = "gpt-realtime";
|
|
87
|
+
declare const OPENAI_DEFAULT_VOICE = "marin";
|
|
88
|
+
/**
|
|
89
|
+
* Voices supported by realtime models. OpenAI recommends marin/cedar (the
|
|
90
|
+
* realtime-native ones — audibly better once 8 kHz telephony takes its cut).
|
|
91
|
+
* The legacy TTS voices (fable, onyx, nova) are NOT supported and fail in
|
|
92
|
+
* unpleasant ways instead of erroring cleanly.
|
|
93
|
+
*/
|
|
94
|
+
declare const OPENAI_REALTIME_VOICES: readonly ["marin", "cedar", "alloy", "ash", "ballad", "coral", "echo", "sage", "shimmer", "verse"];
|
|
95
|
+
type OpenAIRealtimeVoice = (typeof OPENAI_REALTIME_VOICES)[number];
|
|
96
|
+
interface OpenAIRealtimeOptions {
|
|
97
|
+
/** Defaults to the first of OPENAI_API_KEY / OPENAI_KEY / OPEN_AI_API_KEY set in the env. */
|
|
98
|
+
apiKey?: string;
|
|
99
|
+
/** Realtime model id. Default `gpt-realtime`. */
|
|
100
|
+
model?: string;
|
|
101
|
+
/** Default voice when the Agent doesn't set one. Default `marin`. */
|
|
102
|
+
voice?: string;
|
|
103
|
+
/** Override the WS endpoint (proxies, gateways). */
|
|
104
|
+
baseUrl?: string;
|
|
105
|
+
/** Normalized VAD config; `null` disables server turn detection. */
|
|
106
|
+
vad?: VadConfig | null;
|
|
107
|
+
/** Caller-speech transcription; `false` disables it. */
|
|
108
|
+
transcription?: {
|
|
109
|
+
model?: string;
|
|
110
|
+
language?: string;
|
|
111
|
+
} | false;
|
|
112
|
+
/** Extra WS headers. */
|
|
113
|
+
headers?: Record<string, string>;
|
|
114
|
+
/** Provider-native session fields, deep-merged into session.update last. */
|
|
115
|
+
sessionOptions?: Record<string, unknown>;
|
|
116
|
+
connectTimeoutMs?: number;
|
|
117
|
+
}
|
|
118
|
+
/** Create a provider factory for the bridge (`provider: openaiRealtime({...})`). */
|
|
119
|
+
declare function openaiRealtime(options?: OpenAIRealtimeOptions): ProviderFactory;
|
|
120
|
+
//#endregion
|
|
121
|
+
export { OPENAI_DEFAULT_MODEL, OPENAI_DEFAULT_VOICE, OPENAI_KEY_ENV_VARS, OPENAI_REALTIME_VOICES, OpenAICompatibleProvider, type OpenAICompatibleProviderConfig, OpenAIRealtimeOptions, OpenAIRealtimeVoice, buildSessionUpdate, buildTurnDetection, openaiRealtime };
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
import { c as VadConfig, h as Logger, i as ProviderSessionInit, m as ProviderCapabilities, n as ProviderFactory, o as SendTextOptions, s as SendToolResultOptions, t as BaseRealtimeProvider } from "./BaseRealtimeProvider-ClP8Wx1X.mjs";
|
|
2
|
+
//#region src/providers/openai-compatible/OpenAICompatibleProvider.d.ts
|
|
3
|
+
interface OpenAICompatibleProviderConfig {
|
|
4
|
+
apiKey: string;
|
|
5
|
+
model: string;
|
|
6
|
+
/** WS endpoint; `?model=` is appended. Default OpenAI's realtime endpoint. */
|
|
7
|
+
baseUrl?: string;
|
|
8
|
+
voice?: string;
|
|
9
|
+
/** VAD used when the session init doesn't specify one. `null` disables. */
|
|
10
|
+
defaultVad?: VadConfig | null;
|
|
11
|
+
/** Transcription used when the session init doesn't specify one. */
|
|
12
|
+
defaultTranscription?: {
|
|
13
|
+
model?: string;
|
|
14
|
+
language?: string;
|
|
15
|
+
} | false;
|
|
16
|
+
headers?: Record<string, string>;
|
|
17
|
+
/** Provider-native session fields, deep-merged into session.update last. */
|
|
18
|
+
extraSessionOptions?: Record<string, unknown>;
|
|
19
|
+
connectTimeoutMs?: number;
|
|
20
|
+
capabilityOverrides?: Partial<ProviderCapabilities>;
|
|
21
|
+
/** Provider display name for logs/events (e.g. 'openai', 'xai'). */
|
|
22
|
+
providerName?: string;
|
|
23
|
+
/**
|
|
24
|
+
* Session payload builder override for OpenAI-compatible services whose
|
|
25
|
+
* session shape diverges (xAI puts voice/turn_detection at the session
|
|
26
|
+
* root). Defaults to the OpenAI GA builder.
|
|
27
|
+
*/
|
|
28
|
+
buildSession?: (init: ProviderSessionInit, config: {
|
|
29
|
+
defaultVoice?: string;
|
|
30
|
+
extraSessionOptions?: Record<string, unknown>;
|
|
31
|
+
}) => Record<string, unknown>;
|
|
32
|
+
}
|
|
33
|
+
declare class OpenAICompatibleProvider extends BaseRealtimeProvider {
|
|
34
|
+
readonly name: string;
|
|
35
|
+
readonly capabilities: ProviderCapabilities;
|
|
36
|
+
private readonly config;
|
|
37
|
+
private readonly logger;
|
|
38
|
+
private ws;
|
|
39
|
+
private sessionInit;
|
|
40
|
+
private ready;
|
|
41
|
+
private intentionalClose;
|
|
42
|
+
private currentResponseId;
|
|
43
|
+
/**
|
|
44
|
+
* response.create serialization. The GA API rejects a `response.create`
|
|
45
|
+
* issued while another response is in flight
|
|
46
|
+
* (`conversation_already_has_active_response`), so creates requested
|
|
47
|
+
* mid-response wait in a single pending slot and fire on `response.done`.
|
|
48
|
+
* Later requests coalesce into that slot — one response reads the whole
|
|
49
|
+
* conversation state, so only the instructions payload is worth keeping.
|
|
50
|
+
*/
|
|
51
|
+
private responseActive;
|
|
52
|
+
private pendingCreate;
|
|
53
|
+
private lastCreateSent;
|
|
54
|
+
constructor(config: OpenAICompatibleProviderConfig, logger?: Logger);
|
|
55
|
+
get isConnected(): boolean;
|
|
56
|
+
connect(init: ProviderSessionInit): Promise<void>;
|
|
57
|
+
close(): Promise<void>;
|
|
58
|
+
sendAudio(base64Mulaw: string): void;
|
|
59
|
+
sendText(text: string, options?: SendTextOptions): void;
|
|
60
|
+
sendToolResult(callId: string, output: unknown, options?: SendToolResultOptions): void;
|
|
61
|
+
createResponse(options?: {
|
|
62
|
+
instructions?: string;
|
|
63
|
+
}): void;
|
|
64
|
+
private sendCreate;
|
|
65
|
+
updateSession(patch: Partial<ProviderSessionInit>): Promise<void>;
|
|
66
|
+
private buildSessionPayload;
|
|
67
|
+
cancelResponse(): void;
|
|
68
|
+
truncatePlayback(itemId: string, audioEndMs: number): void;
|
|
69
|
+
private handleEvent;
|
|
70
|
+
private parseEvent;
|
|
71
|
+
private send;
|
|
72
|
+
}
|
|
73
|
+
//#endregion
|
|
74
|
+
//#region src/providers/openai-compatible/session-config.d.ts
|
|
75
|
+
declare function buildTurnDetection(vad: VadConfig | null | undefined): Record<string, unknown> | null;
|
|
76
|
+
interface SessionPayloadConfig {
|
|
77
|
+
defaultVoice?: string;
|
|
78
|
+
/** Provider-native session fields, deep-merged last (escape hatch). */
|
|
79
|
+
extraSessionOptions?: Record<string, unknown>;
|
|
80
|
+
}
|
|
81
|
+
declare function buildSessionUpdate(init: ProviderSessionInit, config?: SessionPayloadConfig): Record<string, unknown>;
|
|
82
|
+
//#endregion
|
|
83
|
+
//#region src/openai.d.ts
|
|
84
|
+
/** Env vars checked (in order) when no explicit apiKey is passed. */
|
|
85
|
+
declare const OPENAI_KEY_ENV_VARS: readonly ["OPENAI_API_KEY", "OPENAI_KEY", "OPEN_AI_API_KEY"];
|
|
86
|
+
declare const OPENAI_DEFAULT_MODEL = "gpt-realtime";
|
|
87
|
+
declare const OPENAI_DEFAULT_VOICE = "marin";
|
|
88
|
+
/**
|
|
89
|
+
* Voices supported by realtime models. OpenAI recommends marin/cedar (the
|
|
90
|
+
* realtime-native ones — audibly better once 8 kHz telephony takes its cut).
|
|
91
|
+
* The legacy TTS voices (fable, onyx, nova) are NOT supported and fail in
|
|
92
|
+
* unpleasant ways instead of erroring cleanly.
|
|
93
|
+
*/
|
|
94
|
+
declare const OPENAI_REALTIME_VOICES: readonly ["marin", "cedar", "alloy", "ash", "ballad", "coral", "echo", "sage", "shimmer", "verse"];
|
|
95
|
+
type OpenAIRealtimeVoice = (typeof OPENAI_REALTIME_VOICES)[number];
|
|
96
|
+
interface OpenAIRealtimeOptions {
|
|
97
|
+
/** Defaults to the first of OPENAI_API_KEY / OPENAI_KEY / OPEN_AI_API_KEY set in the env. */
|
|
98
|
+
apiKey?: string;
|
|
99
|
+
/** Realtime model id. Default `gpt-realtime`. */
|
|
100
|
+
model?: string;
|
|
101
|
+
/** Default voice when the Agent doesn't set one. Default `marin`. */
|
|
102
|
+
voice?: string;
|
|
103
|
+
/** Override the WS endpoint (proxies, gateways). */
|
|
104
|
+
baseUrl?: string;
|
|
105
|
+
/** Normalized VAD config; `null` disables server turn detection. */
|
|
106
|
+
vad?: VadConfig | null;
|
|
107
|
+
/** Caller-speech transcription; `false` disables it. */
|
|
108
|
+
transcription?: {
|
|
109
|
+
model?: string;
|
|
110
|
+
language?: string;
|
|
111
|
+
} | false;
|
|
112
|
+
/** Extra WS headers. */
|
|
113
|
+
headers?: Record<string, string>;
|
|
114
|
+
/** Provider-native session fields, deep-merged into session.update last. */
|
|
115
|
+
sessionOptions?: Record<string, unknown>;
|
|
116
|
+
connectTimeoutMs?: number;
|
|
117
|
+
}
|
|
118
|
+
/** Create a provider factory for the bridge (`provider: openaiRealtime({...})`). */
|
|
119
|
+
declare function openaiRealtime(options?: OpenAIRealtimeOptions): ProviderFactory;
|
|
120
|
+
//#endregion
|
|
121
|
+
export { OPENAI_DEFAULT_MODEL, OPENAI_DEFAULT_VOICE, OPENAI_KEY_ENV_VARS, OPENAI_REALTIME_VOICES, OpenAICompatibleProvider, type OpenAICompatibleProviderConfig, OpenAIRealtimeOptions, OpenAIRealtimeVoice, buildSessionUpdate, buildTurnDetection, openaiRealtime };
|
package/dist/openai.mjs
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import { n as buildTurnDetection, t as buildSessionUpdate } from "./session-config-BVLl7-ha.mjs";
|
|
2
|
+
import { t as resolveApiKey } from "./env-DUwUWTsg.mjs";
|
|
3
|
+
import { t as OpenAICompatibleProvider } from "./OpenAICompatibleProvider-Bdtl-UXH.mjs";
|
|
4
|
+
//#region src/openai.ts
|
|
5
|
+
/** OpenAI Realtime provider (GA API) — `realtime-voice-agents/openai`. */
|
|
6
|
+
/** Env vars checked (in order) when no explicit apiKey is passed. */
|
|
7
|
+
const OPENAI_KEY_ENV_VARS = [
|
|
8
|
+
"OPENAI_API_KEY",
|
|
9
|
+
"OPENAI_KEY",
|
|
10
|
+
"OPEN_AI_API_KEY"
|
|
11
|
+
];
|
|
12
|
+
const OPENAI_DEFAULT_MODEL = "gpt-realtime";
|
|
13
|
+
const OPENAI_DEFAULT_VOICE = "marin";
|
|
14
|
+
/**
|
|
15
|
+
* Voices supported by realtime models. OpenAI recommends marin/cedar (the
|
|
16
|
+
* realtime-native ones — audibly better once 8 kHz telephony takes its cut).
|
|
17
|
+
* The legacy TTS voices (fable, onyx, nova) are NOT supported and fail in
|
|
18
|
+
* unpleasant ways instead of erroring cleanly.
|
|
19
|
+
*/
|
|
20
|
+
const OPENAI_REALTIME_VOICES = [
|
|
21
|
+
"marin",
|
|
22
|
+
"cedar",
|
|
23
|
+
"alloy",
|
|
24
|
+
"ash",
|
|
25
|
+
"ballad",
|
|
26
|
+
"coral",
|
|
27
|
+
"echo",
|
|
28
|
+
"sage",
|
|
29
|
+
"shimmer",
|
|
30
|
+
"verse"
|
|
31
|
+
];
|
|
32
|
+
/** Create a provider factory for the bridge (`provider: openaiRealtime({...})`). */
|
|
33
|
+
function openaiRealtime(options = {}) {
|
|
34
|
+
const apiKey = resolveApiKey(options.apiKey, OPENAI_KEY_ENV_VARS);
|
|
35
|
+
if (!apiKey) throw new Error(`openaiRealtime: apiKey missing (pass apiKey or set one of ${OPENAI_KEY_ENV_VARS.join("/")})`);
|
|
36
|
+
const config = {
|
|
37
|
+
apiKey,
|
|
38
|
+
model: options.model ?? "gpt-realtime",
|
|
39
|
+
voice: options.voice ?? "marin",
|
|
40
|
+
baseUrl: options.baseUrl,
|
|
41
|
+
defaultVad: options.vad,
|
|
42
|
+
defaultTranscription: options.transcription,
|
|
43
|
+
headers: options.headers,
|
|
44
|
+
extraSessionOptions: options.sessionOptions,
|
|
45
|
+
connectTimeoutMs: options.connectTimeoutMs,
|
|
46
|
+
providerName: "openai"
|
|
47
|
+
};
|
|
48
|
+
return ({ logger }) => {
|
|
49
|
+
return new OpenAICompatibleProvider(config, logger);
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
//#endregion
|
|
53
|
+
export { OPENAI_DEFAULT_MODEL, OPENAI_DEFAULT_VOICE, OPENAI_KEY_ENV_VARS, OPENAI_REALTIME_VOICES, OpenAICompatibleProvider, buildSessionUpdate, buildTurnDetection, openaiRealtime };
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
//#region src/audio/background/presets.d.ts
|
|
2
|
+
/**
|
|
3
|
+
* Background-audio sources. Presets resolve to bundled μ-law loop assets;
|
|
4
|
+
* custom sources accept a μ-law buffer or a file path to one.
|
|
5
|
+
* (The playback engine lands with the background-audio milestone; the types
|
|
6
|
+
* live here so tool definitions can reference them.)
|
|
7
|
+
*/
|
|
8
|
+
type BackgroundAudioPreset = 'elevator-jazz' | 'lofi' | 'keyboard-typing' | 'thinking-hum' | 'ringing';
|
|
9
|
+
type BackgroundAudioSpec = BackgroundAudioPreset | {
|
|
10
|
+
/** Raw 8 kHz μ-law audio, or an absolute path to a `.ulaw` file. */
|
|
11
|
+
custom: Buffer | string;
|
|
12
|
+
};
|
|
13
|
+
interface BackgroundAudioOptions {
|
|
14
|
+
/** Linear gain 0..1 applied to the loop. Default 0.4. */
|
|
15
|
+
volume?: number;
|
|
16
|
+
fadeInMs?: number;
|
|
17
|
+
fadeOutMs?: number;
|
|
18
|
+
/** Delay before audio actually starts — fast tools never trigger it. */
|
|
19
|
+
startDelayMs?: number;
|
|
20
|
+
/** Hard stop even if never released. Default 60000. */
|
|
21
|
+
maxDurationMs?: number;
|
|
22
|
+
}
|
|
23
|
+
//#endregion
|
|
24
|
+
export { BackgroundAudioPreset as n, BackgroundAudioSpec as r, BackgroundAudioOptions as t };
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
//#region src/audio/background/presets.d.ts
|
|
2
|
+
/**
|
|
3
|
+
* Background-audio sources. Presets resolve to bundled μ-law loop assets;
|
|
4
|
+
* custom sources accept a μ-law buffer or a file path to one.
|
|
5
|
+
* (The playback engine lands with the background-audio milestone; the types
|
|
6
|
+
* live here so tool definitions can reference them.)
|
|
7
|
+
*/
|
|
8
|
+
type BackgroundAudioPreset = 'elevator-jazz' | 'lofi' | 'keyboard-typing' | 'thinking-hum' | 'ringing';
|
|
9
|
+
type BackgroundAudioSpec = BackgroundAudioPreset | {
|
|
10
|
+
/** Raw 8 kHz μ-law audio, or an absolute path to a `.ulaw` file. */
|
|
11
|
+
custom: Buffer | string;
|
|
12
|
+
};
|
|
13
|
+
interface BackgroundAudioOptions {
|
|
14
|
+
/** Linear gain 0..1 applied to the loop. Default 0.4. */
|
|
15
|
+
volume?: number;
|
|
16
|
+
fadeInMs?: number;
|
|
17
|
+
fadeOutMs?: number;
|
|
18
|
+
/** Delay before audio actually starts — fast tools never trigger it. */
|
|
19
|
+
startDelayMs?: number;
|
|
20
|
+
/** Hard stop even if never released. Default 60000. */
|
|
21
|
+
maxDurationMs?: number;
|
|
22
|
+
}
|
|
23
|
+
//#endregion
|
|
24
|
+
export { BackgroundAudioPreset as n, BackgroundAudioSpec as r, BackgroundAudioOptions as t };
|