realtime-voice-agents 2.4.0 → 2.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +30 -11
- package/dist/{BaseRealtimeProvider-CWJ81HIt.d.mts → BaseRealtimeProvider-BJQO-hpu.d.mts} +44 -1
- package/dist/{BaseRealtimeProvider-BL75_HHh.d.cts → BaseRealtimeProvider-CO9N8jt-.d.cts} +44 -1
- package/dist/{BaseRealtimeProvider-C9C3s8jx.mjs → BaseRealtimeProvider-Cj6JWR2k.mjs} +1 -1
- package/dist/{BaseRealtimeProvider-C2mRn1V_.cjs → BaseRealtimeProvider-ZfFzijdr.cjs} +1 -1
- package/dist/{GeminiLiveProvider-Cc1dkxW6.d.cts → GeminiLiveProvider-C_LQCTX5.d.mts} +1 -1
- package/dist/{GeminiLiveProvider-TvY_cQQZ.d.mts → GeminiLiveProvider-DM-GwD0w.d.cts} +1 -1
- package/dist/{InMemorySessionStore-CRBmmCA6.d.cts → InMemorySessionStore-BlsyPnIp.d.cts} +2 -0
- package/dist/{InMemorySessionStore-CRBmmCA6.d.mts → InMemorySessionStore-BlsyPnIp.d.mts} +2 -0
- package/dist/{OpenAICompatibleProvider-D-2OOVBU.mjs → OpenAICompatibleProvider-Dqje-boq.mjs} +3 -3
- package/dist/{OpenAICompatibleProvider-Mp0Mefbh.cjs → OpenAICompatibleProvider-j2Nza2q2.cjs} +8 -2
- package/dist/{events-BylBSBW-.mjs → events-D-pvCH_F.mjs} +2 -1
- package/dist/{events-BxDTIKKq.cjs → events-DWX9535M.cjs} +2 -1
- package/dist/gemini.cjs +1 -1
- package/dist/gemini.d.cts +2 -2
- package/dist/gemini.d.mts +2 -2
- package/dist/gemini.mjs +1 -1
- package/dist/gpt-live.cjs +954 -0
- package/dist/gpt-live.d.cts +261 -0
- package/dist/gpt-live.d.mts +261 -0
- package/dist/gpt-live.mjs +936 -0
- package/dist/index.cjs +116 -19
- package/dist/index.d.cts +7 -3
- package/dist/index.d.mts +7 -3
- package/dist/index.mjs +116 -19
- package/dist/openai.cjs +2 -2
- package/dist/openai.d.cts +1 -1
- package/dist/openai.d.mts +1 -1
- package/dist/openai.mjs +2 -2
- package/dist/{rest-BYqiVOhe.mjs → rest-D-DWWVIj.mjs} +1 -1
- package/dist/{rest-BvUKut_k.cjs → rest-OR4VhEif.cjs} +1 -1
- package/dist/{session-config-CbifLlkV.mjs → session-config-C31WZESg.mjs} +1 -1
- package/dist/{session-config-CqJm2Kxz.cjs → session-config-CYjxBqpF.cjs} +1 -1
- package/dist/store.d.cts +1 -1
- package/dist/store.d.mts +1 -1
- package/dist/testing.cjs +376 -0
- package/dist/testing.d.cts +119 -2
- package/dist/testing.d.mts +119 -2
- package/dist/testing.mjs +374 -2
- package/dist/twilio.cjs +1 -1
- package/dist/twilio.mjs +1 -1
- package/dist/xai.cjs +3 -3
- package/dist/xai.d.cts +1 -1
- package/dist/xai.d.mts +1 -1
- package/dist/xai.mjs +3 -3
- package/package.json +14 -2
|
@@ -0,0 +1,261 @@
|
|
|
1
|
+
import { a as ProviderSessionInit, c as SendToolResultOptions, g as ProviderCapabilities, i as ProviderHistoryEntry, l as SessionUpdateOptions, n as ProviderFactory, o as ProviderToolSchema, s as SendTextOptions, t as BaseRealtimeProvider, v as Logger } from "./BaseRealtimeProvider-CO9N8jt-.cjs";
|
|
2
|
+
//#region src/providers/gpt-live/session-config.d.ts
|
|
3
|
+
interface GptLiveDelegationOptions {
|
|
4
|
+
/** Backend Responses model that reasons and calls tools. Default `gpt-5.6-terra`. */
|
|
5
|
+
model?: string;
|
|
6
|
+
/**
|
|
7
|
+
* Backend prompt — procedures, tool rules, verification. The Agent's
|
|
8
|
+
* `instructions` stay the VOICE prompt (style, interruption policy, when to
|
|
9
|
+
* delegate); GPT-Live is prompted in two halves.
|
|
10
|
+
*/
|
|
11
|
+
instructions?: string;
|
|
12
|
+
toolChoice?: 'auto' | 'required' | 'none' | {
|
|
13
|
+
type: 'function';
|
|
14
|
+
name: string;
|
|
15
|
+
};
|
|
16
|
+
parallelToolCalls?: boolean;
|
|
17
|
+
maxOutputTokens?: number;
|
|
18
|
+
reasoning?: {
|
|
19
|
+
effort?: 'none' | 'minimal' | 'low' | 'medium' | 'high' | 'xhigh';
|
|
20
|
+
summary?: string;
|
|
21
|
+
};
|
|
22
|
+
serviceTier?: 'auto' | 'default' | 'flex' | 'priority';
|
|
23
|
+
text?: {
|
|
24
|
+
verbosity?: 'low' | 'medium' | 'high';
|
|
25
|
+
};
|
|
26
|
+
/** Extra provider-native backend tools, e.g. `{ type: 'web_search' }`. */
|
|
27
|
+
extraTools?: Array<Record<string, unknown>>;
|
|
28
|
+
}
|
|
29
|
+
interface GptLiveSessionConfig {
|
|
30
|
+
defaultVoice?: string;
|
|
31
|
+
delegation?: GptLiveDelegationOptions;
|
|
32
|
+
/** Store the session server-side (30 days) for recording download / forking. */
|
|
33
|
+
store?: boolean;
|
|
34
|
+
/** Provider-native session fields, deep-merged last. Strict schema — see above. */
|
|
35
|
+
extraSessionOptions?: Record<string, unknown>;
|
|
36
|
+
/** History trimming: newest turns kept within these bounds (API: 128 messages / 8,192 tokens). */
|
|
37
|
+
historyMaxMessages?: number;
|
|
38
|
+
historyMaxChars?: number;
|
|
39
|
+
}
|
|
40
|
+
declare const GPT_LIVE_DEFAULT_BACKEND_MODEL = "gpt-5.6-terra";
|
|
41
|
+
declare const GPT_LIVE_AUDIO_FORMAT: {
|
|
42
|
+
readonly type: "audio/pcmu";
|
|
43
|
+
readonly rate: 8000;
|
|
44
|
+
};
|
|
45
|
+
declare function buildDelegation(tools: readonly ProviderToolSchema[] | undefined, options?: GptLiveDelegationOptions): Record<string, unknown>;
|
|
46
|
+
/**
|
|
47
|
+
* History → `input` items, newest-first trimming to the API bounds. A leading
|
|
48
|
+
* `developer` entry (the engine's continuation note) is pinned so trimming
|
|
49
|
+
* never drops the context that explains the rest.
|
|
50
|
+
*/
|
|
51
|
+
declare function buildHistoryItems(history: readonly ProviderHistoryEntry[] | undefined, limits?: {
|
|
52
|
+
maxMessages?: number;
|
|
53
|
+
maxChars?: number;
|
|
54
|
+
}): {
|
|
55
|
+
items: Array<Record<string, unknown>>;
|
|
56
|
+
dropped: number;
|
|
57
|
+
};
|
|
58
|
+
declare function buildSessionStart(init: ProviderSessionInit, config: GptLiveSessionConfig, eventId: string, model: string): {
|
|
59
|
+
frame: Record<string, unknown>;
|
|
60
|
+
droppedHistory: number;
|
|
61
|
+
};
|
|
62
|
+
//#endregion
|
|
63
|
+
//#region src/providers/gpt-live/speech-gate.d.ts
|
|
64
|
+
/**
|
|
65
|
+
* Speech gate for full-duplex output streams.
|
|
66
|
+
*
|
|
67
|
+
* GPT-Live streams assistant audio continuously and in real time — silence
|
|
68
|
+
* included (≈100 ms μ-law deltas, digital 0xFF while the model is quiet) —
|
|
69
|
+
* and the wire has no response boundaries: no response.created, no
|
|
70
|
+
* output_audio.done. The engine's playback tracking, tool timing and hangup
|
|
71
|
+
* watchdog all key off `responseStarted` / `responseDone`, so this gate
|
|
72
|
+
* synthesizes utterances from the audio itself: one opens on the first loud
|
|
73
|
+
* 20 ms sub-frame and closes after `quietMs` of quiet AUDIO. Quiet is
|
|
74
|
+
* measured on the stream's own clock (bytes received), never wall-clock, so
|
|
75
|
+
* a stalled socket cannot fake a finished utterance and tests can script
|
|
76
|
+
* silence instantly. Field numbers (Sept 2026, 8 kHz μ-law): quiet frames
|
|
77
|
+
* p99 ≈ −50 dBFS, speech p05 ≈ −44 dBFS; LiveKit and Pipecat both close at
|
|
78
|
+
* 0.8 s.
|
|
79
|
+
*/
|
|
80
|
+
interface SpeechGateOptions {
|
|
81
|
+
/** RMS (0..1, full scale = 1) at or above which a 20 ms sub-frame counts as speech. Default 0.004 (≈ −48 dBFS). */
|
|
82
|
+
thresholdRms?: number;
|
|
83
|
+
/** Quiet audio that closes an utterance. Default 800. */
|
|
84
|
+
quietMs?: number;
|
|
85
|
+
}
|
|
86
|
+
type SpeechGateEvent = {
|
|
87
|
+
type: 'open';
|
|
88
|
+
} | {
|
|
89
|
+
type: 'close';
|
|
90
|
+
utteranceMs: number;
|
|
91
|
+
};
|
|
92
|
+
declare const DEFAULT_GATE_THRESHOLD_RMS = 0.004;
|
|
93
|
+
declare const DEFAULT_GATE_QUIET_MS = 800;
|
|
94
|
+
declare class SpeechGate {
|
|
95
|
+
private readonly threshold;
|
|
96
|
+
private readonly quietLimitMs;
|
|
97
|
+
private open;
|
|
98
|
+
private quietMs;
|
|
99
|
+
private utteranceMs;
|
|
100
|
+
constructor(options?: SpeechGateOptions);
|
|
101
|
+
get isOpen(): boolean;
|
|
102
|
+
/** Feed one output delta; returns the transitions it caused, in order. */
|
|
103
|
+
feed(bytes: Uint8Array): SpeechGateEvent[];
|
|
104
|
+
/** Force-close an open utterance (stream stalled, session closing). */
|
|
105
|
+
close(): SpeechGateEvent | null;
|
|
106
|
+
}
|
|
107
|
+
//#endregion
|
|
108
|
+
//#region src/providers/gpt-live/GptLiveProvider.d.ts
|
|
109
|
+
declare const GPT_LIVE_DEFAULT_BASE_URL = "wss://api.openai.com/v1/live/sessions";
|
|
110
|
+
interface GptLiveProviderConfig {
|
|
111
|
+
apiKey: string;
|
|
112
|
+
/** Live model id. */
|
|
113
|
+
model: string;
|
|
114
|
+
/** Default output voice when the Agent doesn't set one. */
|
|
115
|
+
voice?: string;
|
|
116
|
+
/** Override the WS endpoint (proxies, Azure `…/openai/v1/live`). */
|
|
117
|
+
baseUrl?: string;
|
|
118
|
+
/** Extra WS headers. */
|
|
119
|
+
headers?: Record<string, string>;
|
|
120
|
+
/** Backend (Responses) delegation — the half of the prompt that reasons and calls tools. */
|
|
121
|
+
delegation?: GptLiveDelegationOptions;
|
|
122
|
+
/** Store the session server-side (30 days) for recording download / forking. Default false. */
|
|
123
|
+
store?: boolean;
|
|
124
|
+
/** Provider-native session fields, deep-merged into `session.start` last (strict schema!). */
|
|
125
|
+
extraSessionOptions?: Record<string, unknown>;
|
|
126
|
+
connectTimeoutMs?: number;
|
|
127
|
+
/** How long `close()` waits for `session.closed` (final usage) before dropping the socket. Default 3000. */
|
|
128
|
+
closeTimeoutMs?: number;
|
|
129
|
+
/** Speech gate tuning (utterance boundaries synthesized from the audio). */
|
|
130
|
+
speechGate?: SpeechGateOptions;
|
|
131
|
+
/** Session-timeline gap that splits transcript fragments into turns. Default 800. */
|
|
132
|
+
transcriptGapMs?: number;
|
|
133
|
+
/** History trimming bounds for `session.input`. */
|
|
134
|
+
historyMaxMessages?: number;
|
|
135
|
+
historyMaxChars?: number;
|
|
136
|
+
/** Ack wait for appends / updates. Default 5000. */
|
|
137
|
+
ackTimeoutMs?: number;
|
|
138
|
+
providerName?: string;
|
|
139
|
+
}
|
|
140
|
+
declare class GptLiveProvider extends BaseRealtimeProvider {
|
|
141
|
+
readonly name: string;
|
|
142
|
+
readonly capabilities: ProviderCapabilities;
|
|
143
|
+
private readonly config;
|
|
144
|
+
private readonly logger;
|
|
145
|
+
private ws;
|
|
146
|
+
private ready;
|
|
147
|
+
private intentionalClose;
|
|
148
|
+
private sessionInit;
|
|
149
|
+
/** Server-assigned session id (sideband attach, forking, recording download). */
|
|
150
|
+
sessionId: string | null;
|
|
151
|
+
/** Unix seconds at which the server expires the session (120 min at launch). */
|
|
152
|
+
expiresAt: number | null;
|
|
153
|
+
private readonly gate;
|
|
154
|
+
private gateStallTimer;
|
|
155
|
+
private utteranceCounter;
|
|
156
|
+
private currentUtteranceId;
|
|
157
|
+
private lastUtteranceId;
|
|
158
|
+
private readonly inputTranscript;
|
|
159
|
+
private readonly outputTranscript;
|
|
160
|
+
private readonly acks;
|
|
161
|
+
private readonly seenCalls;
|
|
162
|
+
private sessionClosed;
|
|
163
|
+
private closeListeners;
|
|
164
|
+
constructor(config: GptLiveProviderConfig, logger?: Logger);
|
|
165
|
+
get isConnected(): boolean;
|
|
166
|
+
connect(init: ProviderSessionInit): Promise<void>;
|
|
167
|
+
close(): Promise<void>;
|
|
168
|
+
sendAudio(base64Mulaw: string): void;
|
|
169
|
+
/**
|
|
170
|
+
* Text has three doors into a GPT-Live session, by intent rather than by
|
|
171
|
+
* conversation role: `system` → instructions (behaviour), `user` → thinking
|
|
172
|
+
* (quiet context: keypad entries, deferred results — the model decides how
|
|
173
|
+
* to react), `assistant` → thinking as well, worded as something already
|
|
174
|
+
* said (commentary would make the model say it again).
|
|
175
|
+
*/
|
|
176
|
+
sendText(text: string, options?: SendTextOptions): void;
|
|
177
|
+
sendToolResult(callId: string, output: unknown, options?: SendToolResultOptions): void;
|
|
178
|
+
/**
|
|
179
|
+
* "Speak now" has no direct verb on this API. Commentary — information the
|
|
180
|
+
* model should say aloud, paraphrasing allowed — is the append that
|
|
181
|
+
* produced speech on demand in every field test, greeting and goodbye
|
|
182
|
+
* alike; an instruction-shaped text is followed rather than read out.
|
|
183
|
+
*/
|
|
184
|
+
createResponse(options?: {
|
|
185
|
+
instructions?: string;
|
|
186
|
+
}): void;
|
|
187
|
+
/**
|
|
188
|
+
* Only the backend delegation can change mid-session (`session.update` is
|
|
189
|
+
* sparse and limited to `delegation.responses`): tool lists land there;
|
|
190
|
+
* instructions / voice / vad are immutable and are reported, not applied.
|
|
191
|
+
*/
|
|
192
|
+
updateSession(patch: Partial<ProviderSessionInit>, options?: SessionUpdateOptions): Promise<boolean>;
|
|
193
|
+
private handleEvent;
|
|
194
|
+
private handleOutputAudio;
|
|
195
|
+
private handleBackendEvent;
|
|
196
|
+
private beginUtterance;
|
|
197
|
+
private endUtterance;
|
|
198
|
+
/** The stream is real-time: no delta for a quiet window means it stalled — close the utterance. */
|
|
199
|
+
private armGateStall;
|
|
200
|
+
private clearGateStall;
|
|
201
|
+
private finishUtteranceOnClose;
|
|
202
|
+
private resetUtteranceState;
|
|
203
|
+
/** Send `content` as one or more appends (500-token cap); resolves with the last chunk's ack. */
|
|
204
|
+
private append;
|
|
205
|
+
private awaitAck;
|
|
206
|
+
private resolveAck;
|
|
207
|
+
private failPendingAcks;
|
|
208
|
+
private sessionConfig;
|
|
209
|
+
private isRetriableClose;
|
|
210
|
+
private parseEvent;
|
|
211
|
+
private send;
|
|
212
|
+
}
|
|
213
|
+
/** Split on whitespace so no chunk exceeds the per-append cap. */
|
|
214
|
+
declare function splitForAppend(content: string, maxChars?: number): string[];
|
|
215
|
+
//#endregion
|
|
216
|
+
//#region src/gpt-live.d.ts
|
|
217
|
+
/** Env vars checked (in order) when no explicit apiKey is passed — the same project key as Realtime. */
|
|
218
|
+
declare const GPT_LIVE_KEY_ENV_VARS: readonly ["OPENAI_API_KEY", "OPENAI_KEY", "OPEN_AI_API_KEY"];
|
|
219
|
+
declare const GPT_LIVE_DEFAULT_MODEL = "gpt-live-1";
|
|
220
|
+
declare const GPT_LIVE_DEFAULT_VOICE = "marin";
|
|
221
|
+
/** Built-in voices accepted by the Live API (the SDK's enum, Sept 2026). */
|
|
222
|
+
declare const GPT_LIVE_VOICES: readonly ["marin", "cedar", "alloy", "ash", "ballad", "beacon", "bossa", "cinder", "coral", "delta", "echo", "gleam", "meridian", "quartz", "ripple", "sage", "shimmer", "stone", "tempo", "verse", "vesper", "willow"];
|
|
223
|
+
type GptLiveVoice = (typeof GPT_LIVE_VOICES)[number];
|
|
224
|
+
interface GptLiveOptions {
|
|
225
|
+
/** Defaults to the first of OPENAI_API_KEY / OPENAI_KEY / OPEN_AI_API_KEY set in the env. */
|
|
226
|
+
apiKey?: string;
|
|
227
|
+
/** Live model id. Default `gpt-live-1`. */
|
|
228
|
+
model?: string;
|
|
229
|
+
/** Default voice when the Agent doesn't set one. Default `marin`. Immutable per session. */
|
|
230
|
+
voice?: string;
|
|
231
|
+
/**
|
|
232
|
+
* The backend half of the prompt: the Responses model that reasons and calls
|
|
233
|
+
* the agent's tools while the voice model keeps the conversation going.
|
|
234
|
+
* `instructions` here are the procedures; the Agent's `instructions` are
|
|
235
|
+
* the voice (style, interruption policy, when to delegate).
|
|
236
|
+
*/
|
|
237
|
+
delegation?: GptLiveDelegationOptions;
|
|
238
|
+
/** Override the WS endpoint (proxies, Azure Foundry `…/openai/v1/live`). */
|
|
239
|
+
baseUrl?: string;
|
|
240
|
+
/** Extra WS headers. */
|
|
241
|
+
headers?: Record<string, string>;
|
|
242
|
+
/** Store the session server-side (30 days) for recording download / forking. Default false. */
|
|
243
|
+
store?: boolean;
|
|
244
|
+
/** Provider-native `session.start` fields, deep-merged last. The schema is strict: an unknown field rejects the session. */
|
|
245
|
+
sessionOptions?: Record<string, unknown>;
|
|
246
|
+
connectTimeoutMs?: number;
|
|
247
|
+
/** Speech gate tuning — utterance boundaries synthesized from the continuous output stream. */
|
|
248
|
+
speechGate?: SpeechGateOptions;
|
|
249
|
+
/** Session-timeline gap that splits transcript fragments into turns. Default 800. */
|
|
250
|
+
transcriptGapMs?: number;
|
|
251
|
+
}
|
|
252
|
+
/**
|
|
253
|
+
* Create a GPT-Live provider factory for the bridge (`provider: gptLive({...})`).
|
|
254
|
+
*
|
|
255
|
+
* Credentials are resolved per call, when the factory runs (see
|
|
256
|
+
* `openaiRealtime` — a missing key fails that call's connect so a fallback
|
|
257
|
+
* chain can absorb it instead of crashing config construction).
|
|
258
|
+
*/
|
|
259
|
+
declare function gptLive(options?: GptLiveOptions): ProviderFactory;
|
|
260
|
+
//#endregion
|
|
261
|
+
export { DEFAULT_GATE_QUIET_MS, DEFAULT_GATE_THRESHOLD_RMS, GPT_LIVE_AUDIO_FORMAT, GPT_LIVE_DEFAULT_BACKEND_MODEL, GPT_LIVE_DEFAULT_BASE_URL, GPT_LIVE_DEFAULT_MODEL, GPT_LIVE_DEFAULT_VOICE, GPT_LIVE_KEY_ENV_VARS, GPT_LIVE_VOICES, type GptLiveDelegationOptions, GptLiveOptions, GptLiveProvider, type GptLiveProviderConfig, type GptLiveSessionConfig, GptLiveVoice, SpeechGate, type SpeechGateOptions, buildDelegation, buildHistoryItems, buildSessionStart, gptLive, splitForAppend };
|
|
@@ -0,0 +1,261 @@
|
|
|
1
|
+
import { a as ProviderSessionInit, c as SendToolResultOptions, g as ProviderCapabilities, i as ProviderHistoryEntry, l as SessionUpdateOptions, n as ProviderFactory, o as ProviderToolSchema, s as SendTextOptions, t as BaseRealtimeProvider, v as Logger } from "./BaseRealtimeProvider-BJQO-hpu.mjs";
|
|
2
|
+
//#region src/providers/gpt-live/session-config.d.ts
|
|
3
|
+
interface GptLiveDelegationOptions {
|
|
4
|
+
/** Backend Responses model that reasons and calls tools. Default `gpt-5.6-terra`. */
|
|
5
|
+
model?: string;
|
|
6
|
+
/**
|
|
7
|
+
* Backend prompt — procedures, tool rules, verification. The Agent's
|
|
8
|
+
* `instructions` stay the VOICE prompt (style, interruption policy, when to
|
|
9
|
+
* delegate); GPT-Live is prompted in two halves.
|
|
10
|
+
*/
|
|
11
|
+
instructions?: string;
|
|
12
|
+
toolChoice?: 'auto' | 'required' | 'none' | {
|
|
13
|
+
type: 'function';
|
|
14
|
+
name: string;
|
|
15
|
+
};
|
|
16
|
+
parallelToolCalls?: boolean;
|
|
17
|
+
maxOutputTokens?: number;
|
|
18
|
+
reasoning?: {
|
|
19
|
+
effort?: 'none' | 'minimal' | 'low' | 'medium' | 'high' | 'xhigh';
|
|
20
|
+
summary?: string;
|
|
21
|
+
};
|
|
22
|
+
serviceTier?: 'auto' | 'default' | 'flex' | 'priority';
|
|
23
|
+
text?: {
|
|
24
|
+
verbosity?: 'low' | 'medium' | 'high';
|
|
25
|
+
};
|
|
26
|
+
/** Extra provider-native backend tools, e.g. `{ type: 'web_search' }`. */
|
|
27
|
+
extraTools?: Array<Record<string, unknown>>;
|
|
28
|
+
}
|
|
29
|
+
interface GptLiveSessionConfig {
|
|
30
|
+
defaultVoice?: string;
|
|
31
|
+
delegation?: GptLiveDelegationOptions;
|
|
32
|
+
/** Store the session server-side (30 days) for recording download / forking. */
|
|
33
|
+
store?: boolean;
|
|
34
|
+
/** Provider-native session fields, deep-merged last. Strict schema — see above. */
|
|
35
|
+
extraSessionOptions?: Record<string, unknown>;
|
|
36
|
+
/** History trimming: newest turns kept within these bounds (API: 128 messages / 8,192 tokens). */
|
|
37
|
+
historyMaxMessages?: number;
|
|
38
|
+
historyMaxChars?: number;
|
|
39
|
+
}
|
|
40
|
+
declare const GPT_LIVE_DEFAULT_BACKEND_MODEL = "gpt-5.6-terra";
|
|
41
|
+
declare const GPT_LIVE_AUDIO_FORMAT: {
|
|
42
|
+
readonly type: "audio/pcmu";
|
|
43
|
+
readonly rate: 8000;
|
|
44
|
+
};
|
|
45
|
+
declare function buildDelegation(tools: readonly ProviderToolSchema[] | undefined, options?: GptLiveDelegationOptions): Record<string, unknown>;
|
|
46
|
+
/**
|
|
47
|
+
* History → `input` items, newest-first trimming to the API bounds. A leading
|
|
48
|
+
* `developer` entry (the engine's continuation note) is pinned so trimming
|
|
49
|
+
* never drops the context that explains the rest.
|
|
50
|
+
*/
|
|
51
|
+
declare function buildHistoryItems(history: readonly ProviderHistoryEntry[] | undefined, limits?: {
|
|
52
|
+
maxMessages?: number;
|
|
53
|
+
maxChars?: number;
|
|
54
|
+
}): {
|
|
55
|
+
items: Array<Record<string, unknown>>;
|
|
56
|
+
dropped: number;
|
|
57
|
+
};
|
|
58
|
+
declare function buildSessionStart(init: ProviderSessionInit, config: GptLiveSessionConfig, eventId: string, model: string): {
|
|
59
|
+
frame: Record<string, unknown>;
|
|
60
|
+
droppedHistory: number;
|
|
61
|
+
};
|
|
62
|
+
//#endregion
|
|
63
|
+
//#region src/providers/gpt-live/speech-gate.d.ts
|
|
64
|
+
/**
|
|
65
|
+
* Speech gate for full-duplex output streams.
|
|
66
|
+
*
|
|
67
|
+
* GPT-Live streams assistant audio continuously and in real time — silence
|
|
68
|
+
* included (≈100 ms μ-law deltas, digital 0xFF while the model is quiet) —
|
|
69
|
+
* and the wire has no response boundaries: no response.created, no
|
|
70
|
+
* output_audio.done. The engine's playback tracking, tool timing and hangup
|
|
71
|
+
* watchdog all key off `responseStarted` / `responseDone`, so this gate
|
|
72
|
+
* synthesizes utterances from the audio itself: one opens on the first loud
|
|
73
|
+
* 20 ms sub-frame and closes after `quietMs` of quiet AUDIO. Quiet is
|
|
74
|
+
* measured on the stream's own clock (bytes received), never wall-clock, so
|
|
75
|
+
* a stalled socket cannot fake a finished utterance and tests can script
|
|
76
|
+
* silence instantly. Field numbers (Sept 2026, 8 kHz μ-law): quiet frames
|
|
77
|
+
* p99 ≈ −50 dBFS, speech p05 ≈ −44 dBFS; LiveKit and Pipecat both close at
|
|
78
|
+
* 0.8 s.
|
|
79
|
+
*/
|
|
80
|
+
interface SpeechGateOptions {
|
|
81
|
+
/** RMS (0..1, full scale = 1) at or above which a 20 ms sub-frame counts as speech. Default 0.004 (≈ −48 dBFS). */
|
|
82
|
+
thresholdRms?: number;
|
|
83
|
+
/** Quiet audio that closes an utterance. Default 800. */
|
|
84
|
+
quietMs?: number;
|
|
85
|
+
}
|
|
86
|
+
type SpeechGateEvent = {
|
|
87
|
+
type: 'open';
|
|
88
|
+
} | {
|
|
89
|
+
type: 'close';
|
|
90
|
+
utteranceMs: number;
|
|
91
|
+
};
|
|
92
|
+
declare const DEFAULT_GATE_THRESHOLD_RMS = 0.004;
|
|
93
|
+
declare const DEFAULT_GATE_QUIET_MS = 800;
|
|
94
|
+
declare class SpeechGate {
|
|
95
|
+
private readonly threshold;
|
|
96
|
+
private readonly quietLimitMs;
|
|
97
|
+
private open;
|
|
98
|
+
private quietMs;
|
|
99
|
+
private utteranceMs;
|
|
100
|
+
constructor(options?: SpeechGateOptions);
|
|
101
|
+
get isOpen(): boolean;
|
|
102
|
+
/** Feed one output delta; returns the transitions it caused, in order. */
|
|
103
|
+
feed(bytes: Uint8Array): SpeechGateEvent[];
|
|
104
|
+
/** Force-close an open utterance (stream stalled, session closing). */
|
|
105
|
+
close(): SpeechGateEvent | null;
|
|
106
|
+
}
|
|
107
|
+
//#endregion
|
|
108
|
+
//#region src/providers/gpt-live/GptLiveProvider.d.ts
|
|
109
|
+
declare const GPT_LIVE_DEFAULT_BASE_URL = "wss://api.openai.com/v1/live/sessions";
|
|
110
|
+
interface GptLiveProviderConfig {
|
|
111
|
+
apiKey: string;
|
|
112
|
+
/** Live model id. */
|
|
113
|
+
model: string;
|
|
114
|
+
/** Default output voice when the Agent doesn't set one. */
|
|
115
|
+
voice?: string;
|
|
116
|
+
/** Override the WS endpoint (proxies, Azure `…/openai/v1/live`). */
|
|
117
|
+
baseUrl?: string;
|
|
118
|
+
/** Extra WS headers. */
|
|
119
|
+
headers?: Record<string, string>;
|
|
120
|
+
/** Backend (Responses) delegation — the half of the prompt that reasons and calls tools. */
|
|
121
|
+
delegation?: GptLiveDelegationOptions;
|
|
122
|
+
/** Store the session server-side (30 days) for recording download / forking. Default false. */
|
|
123
|
+
store?: boolean;
|
|
124
|
+
/** Provider-native session fields, deep-merged into `session.start` last (strict schema!). */
|
|
125
|
+
extraSessionOptions?: Record<string, unknown>;
|
|
126
|
+
connectTimeoutMs?: number;
|
|
127
|
+
/** How long `close()` waits for `session.closed` (final usage) before dropping the socket. Default 3000. */
|
|
128
|
+
closeTimeoutMs?: number;
|
|
129
|
+
/** Speech gate tuning (utterance boundaries synthesized from the audio). */
|
|
130
|
+
speechGate?: SpeechGateOptions;
|
|
131
|
+
/** Session-timeline gap that splits transcript fragments into turns. Default 800. */
|
|
132
|
+
transcriptGapMs?: number;
|
|
133
|
+
/** History trimming bounds for `session.input`. */
|
|
134
|
+
historyMaxMessages?: number;
|
|
135
|
+
historyMaxChars?: number;
|
|
136
|
+
/** Ack wait for appends / updates. Default 5000. */
|
|
137
|
+
ackTimeoutMs?: number;
|
|
138
|
+
providerName?: string;
|
|
139
|
+
}
|
|
140
|
+
declare class GptLiveProvider extends BaseRealtimeProvider {
|
|
141
|
+
readonly name: string;
|
|
142
|
+
readonly capabilities: ProviderCapabilities;
|
|
143
|
+
private readonly config;
|
|
144
|
+
private readonly logger;
|
|
145
|
+
private ws;
|
|
146
|
+
private ready;
|
|
147
|
+
private intentionalClose;
|
|
148
|
+
private sessionInit;
|
|
149
|
+
/** Server-assigned session id (sideband attach, forking, recording download). */
|
|
150
|
+
sessionId: string | null;
|
|
151
|
+
/** Unix seconds at which the server expires the session (120 min at launch). */
|
|
152
|
+
expiresAt: number | null;
|
|
153
|
+
private readonly gate;
|
|
154
|
+
private gateStallTimer;
|
|
155
|
+
private utteranceCounter;
|
|
156
|
+
private currentUtteranceId;
|
|
157
|
+
private lastUtteranceId;
|
|
158
|
+
private readonly inputTranscript;
|
|
159
|
+
private readonly outputTranscript;
|
|
160
|
+
private readonly acks;
|
|
161
|
+
private readonly seenCalls;
|
|
162
|
+
private sessionClosed;
|
|
163
|
+
private closeListeners;
|
|
164
|
+
constructor(config: GptLiveProviderConfig, logger?: Logger);
|
|
165
|
+
get isConnected(): boolean;
|
|
166
|
+
connect(init: ProviderSessionInit): Promise<void>;
|
|
167
|
+
close(): Promise<void>;
|
|
168
|
+
sendAudio(base64Mulaw: string): void;
|
|
169
|
+
/**
|
|
170
|
+
* Text has three doors into a GPT-Live session, by intent rather than by
|
|
171
|
+
* conversation role: `system` → instructions (behaviour), `user` → thinking
|
|
172
|
+
* (quiet context: keypad entries, deferred results — the model decides how
|
|
173
|
+
* to react), `assistant` → thinking as well, worded as something already
|
|
174
|
+
* said (commentary would make the model say it again).
|
|
175
|
+
*/
|
|
176
|
+
sendText(text: string, options?: SendTextOptions): void;
|
|
177
|
+
sendToolResult(callId: string, output: unknown, options?: SendToolResultOptions): void;
|
|
178
|
+
/**
|
|
179
|
+
* "Speak now" has no direct verb on this API. Commentary — information the
|
|
180
|
+
* model should say aloud, paraphrasing allowed — is the append that
|
|
181
|
+
* produced speech on demand in every field test, greeting and goodbye
|
|
182
|
+
* alike; an instruction-shaped text is followed rather than read out.
|
|
183
|
+
*/
|
|
184
|
+
createResponse(options?: {
|
|
185
|
+
instructions?: string;
|
|
186
|
+
}): void;
|
|
187
|
+
/**
|
|
188
|
+
* Only the backend delegation can change mid-session (`session.update` is
|
|
189
|
+
* sparse and limited to `delegation.responses`): tool lists land there;
|
|
190
|
+
* instructions / voice / vad are immutable and are reported, not applied.
|
|
191
|
+
*/
|
|
192
|
+
updateSession(patch: Partial<ProviderSessionInit>, options?: SessionUpdateOptions): Promise<boolean>;
|
|
193
|
+
private handleEvent;
|
|
194
|
+
private handleOutputAudio;
|
|
195
|
+
private handleBackendEvent;
|
|
196
|
+
private beginUtterance;
|
|
197
|
+
private endUtterance;
|
|
198
|
+
/** The stream is real-time: no delta for a quiet window means it stalled — close the utterance. */
|
|
199
|
+
private armGateStall;
|
|
200
|
+
private clearGateStall;
|
|
201
|
+
private finishUtteranceOnClose;
|
|
202
|
+
private resetUtteranceState;
|
|
203
|
+
/** Send `content` as one or more appends (500-token cap); resolves with the last chunk's ack. */
|
|
204
|
+
private append;
|
|
205
|
+
private awaitAck;
|
|
206
|
+
private resolveAck;
|
|
207
|
+
private failPendingAcks;
|
|
208
|
+
private sessionConfig;
|
|
209
|
+
private isRetriableClose;
|
|
210
|
+
private parseEvent;
|
|
211
|
+
private send;
|
|
212
|
+
}
|
|
213
|
+
/** Split on whitespace so no chunk exceeds the per-append cap. */
|
|
214
|
+
declare function splitForAppend(content: string, maxChars?: number): string[];
|
|
215
|
+
//#endregion
|
|
216
|
+
//#region src/gpt-live.d.ts
|
|
217
|
+
/** Env vars checked (in order) when no explicit apiKey is passed — the same project key as Realtime. */
|
|
218
|
+
declare const GPT_LIVE_KEY_ENV_VARS: readonly ["OPENAI_API_KEY", "OPENAI_KEY", "OPEN_AI_API_KEY"];
|
|
219
|
+
declare const GPT_LIVE_DEFAULT_MODEL = "gpt-live-1";
|
|
220
|
+
declare const GPT_LIVE_DEFAULT_VOICE = "marin";
|
|
221
|
+
/** Built-in voices accepted by the Live API (the SDK's enum, Sept 2026). */
|
|
222
|
+
declare const GPT_LIVE_VOICES: readonly ["marin", "cedar", "alloy", "ash", "ballad", "beacon", "bossa", "cinder", "coral", "delta", "echo", "gleam", "meridian", "quartz", "ripple", "sage", "shimmer", "stone", "tempo", "verse", "vesper", "willow"];
|
|
223
|
+
type GptLiveVoice = (typeof GPT_LIVE_VOICES)[number];
|
|
224
|
+
interface GptLiveOptions {
|
|
225
|
+
/** Defaults to the first of OPENAI_API_KEY / OPENAI_KEY / OPEN_AI_API_KEY set in the env. */
|
|
226
|
+
apiKey?: string;
|
|
227
|
+
/** Live model id. Default `gpt-live-1`. */
|
|
228
|
+
model?: string;
|
|
229
|
+
/** Default voice when the Agent doesn't set one. Default `marin`. Immutable per session. */
|
|
230
|
+
voice?: string;
|
|
231
|
+
/**
|
|
232
|
+
* The backend half of the prompt: the Responses model that reasons and calls
|
|
233
|
+
* the agent's tools while the voice model keeps the conversation going.
|
|
234
|
+
* `instructions` here are the procedures; the Agent's `instructions` are
|
|
235
|
+
* the voice (style, interruption policy, when to delegate).
|
|
236
|
+
*/
|
|
237
|
+
delegation?: GptLiveDelegationOptions;
|
|
238
|
+
/** Override the WS endpoint (proxies, Azure Foundry `…/openai/v1/live`). */
|
|
239
|
+
baseUrl?: string;
|
|
240
|
+
/** Extra WS headers. */
|
|
241
|
+
headers?: Record<string, string>;
|
|
242
|
+
/** Store the session server-side (30 days) for recording download / forking. Default false. */
|
|
243
|
+
store?: boolean;
|
|
244
|
+
/** Provider-native `session.start` fields, deep-merged last. The schema is strict: an unknown field rejects the session. */
|
|
245
|
+
sessionOptions?: Record<string, unknown>;
|
|
246
|
+
connectTimeoutMs?: number;
|
|
247
|
+
/** Speech gate tuning — utterance boundaries synthesized from the continuous output stream. */
|
|
248
|
+
speechGate?: SpeechGateOptions;
|
|
249
|
+
/** Session-timeline gap that splits transcript fragments into turns. Default 800. */
|
|
250
|
+
transcriptGapMs?: number;
|
|
251
|
+
}
|
|
252
|
+
/**
|
|
253
|
+
* Create a GPT-Live provider factory for the bridge (`provider: gptLive({...})`).
|
|
254
|
+
*
|
|
255
|
+
* Credentials are resolved per call, when the factory runs (see
|
|
256
|
+
* `openaiRealtime` — a missing key fails that call's connect so a fallback
|
|
257
|
+
* chain can absorb it instead of crashing config construction).
|
|
258
|
+
*/
|
|
259
|
+
declare function gptLive(options?: GptLiveOptions): ProviderFactory;
|
|
260
|
+
//#endregion
|
|
261
|
+
export { DEFAULT_GATE_QUIET_MS, DEFAULT_GATE_THRESHOLD_RMS, GPT_LIVE_AUDIO_FORMAT, GPT_LIVE_DEFAULT_BACKEND_MODEL, GPT_LIVE_DEFAULT_BASE_URL, GPT_LIVE_DEFAULT_MODEL, GPT_LIVE_DEFAULT_VOICE, GPT_LIVE_KEY_ENV_VARS, GPT_LIVE_VOICES, type GptLiveDelegationOptions, GptLiveOptions, GptLiveProvider, type GptLiveProviderConfig, type GptLiveSessionConfig, GptLiveVoice, SpeechGate, type SpeechGateOptions, buildDelegation, buildHistoryItems, buildSessionStart, gptLive, splitForAppend };
|