xo-harness 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -0
- package/dist/harness.d.ts +1 -0
- package/dist/harness.js +1 -0
- package/dist/index.d.ts +4 -0
- package/dist/index.js +4 -0
- package/dist/internal/harness/event-stream.d.ts +7 -0
- package/dist/internal/harness/event-stream.js +42 -0
- package/dist/internal/harness/index.d.ts +7 -0
- package/dist/internal/harness/index.js +7 -0
- package/dist/internal/harness/message.d.ts +26 -0
- package/dist/internal/harness/message.js +192 -0
- package/dist/internal/harness/report.d.ts +51 -0
- package/dist/internal/harness/report.js +115 -0
- package/dist/internal/harness/shadow.d.ts +28 -0
- package/dist/internal/harness/shadow.js +118 -0
- package/dist/internal/harness/socket-bridge.d.ts +65 -0
- package/dist/internal/harness/socket-bridge.js +227 -0
- package/dist/internal/harness/task-supervisor.d.ts +20 -0
- package/dist/internal/harness/task-supervisor.js +180 -0
- package/dist/internal/harness/tool-runtime.d.ts +20 -0
- package/dist/internal/harness/tool-runtime.js +93 -0
- package/dist/internal/harness/tools.d.ts +32 -0
- package/dist/internal/harness/tools.js +30 -0
- package/dist/internal/harness/voice-session.d.ts +33 -0
- package/dist/internal/harness/voice-session.js +268 -0
- package/dist/internal/harness/xo.d.ts +25 -0
- package/dist/internal/harness/xo.js +24 -0
- package/dist/internal/protocol/async-queue.d.ts +8 -0
- package/dist/internal/protocol/async-queue.js +34 -0
- package/dist/internal/protocol/audio.d.ts +12 -0
- package/dist/internal/protocol/audio.js +24 -0
- package/dist/internal/protocol/events.d.ts +350 -0
- package/dist/internal/protocol/events.js +167 -0
- package/dist/internal/protocol/index.d.ts +6 -0
- package/dist/internal/protocol/index.js +6 -0
- package/dist/internal/protocol/parts.d.ts +219 -0
- package/dist/internal/protocol/parts.js +105 -0
- package/dist/internal/protocol/provider.d.ts +118 -0
- package/dist/internal/protocol/provider.js +77 -0
- package/dist/internal/protocol/tools.d.ts +62 -0
- package/dist/internal/protocol/tools.js +47 -0
- package/dist/internal/provider/contract.d.ts +24 -0
- package/dist/internal/provider/contract.js +1 -0
- package/dist/internal/provider/grok-voice.d.ts +45 -0
- package/dist/internal/provider/grok-voice.js +77 -0
- package/dist/internal/provider/index.d.ts +6 -0
- package/dist/internal/provider/index.js +9 -0
- package/dist/internal/provider/node-socket.d.ts +3 -0
- package/dist/internal/provider/node-socket.js +74 -0
- package/dist/internal/provider/openai-realtime.d.ts +43 -0
- package/dist/internal/provider/openai-realtime.js +76 -0
- package/dist/internal/provider/realtime-session.d.ts +30 -0
- package/dist/internal/provider/realtime-session.js +461 -0
- package/dist/internal/provider/realtime-socket.d.ts +42 -0
- package/dist/internal/provider/realtime-socket.js +30 -0
- package/dist/internal/provider/workers-socket.d.ts +11 -0
- package/dist/internal/provider/workers-socket.js +108 -0
- package/dist/internal/provider/workers.d.ts +6 -0
- package/dist/internal/provider/workers.js +11 -0
- package/dist/internal/provider-fake/index.d.ts +46 -0
- package/dist/internal/provider-fake/index.js +112 -0
- package/dist/internal/provider-fake/replay-voice-provider.d.ts +47 -0
- package/dist/internal/provider-fake/replay-voice-provider.js +133 -0
- package/dist/internal/provider-fake/scripted-voice-provider.d.ts +61 -0
- package/dist/internal/provider-fake/scripted-voice-provider.js +185 -0
- package/dist/internal/storage/append-tail.d.ts +12 -0
- package/dist/internal/storage/append-tail.js +19 -0
- package/dist/internal/storage/event-store.d.ts +19 -0
- package/dist/internal/storage/event-store.js +1 -0
- package/dist/internal/storage/index.d.ts +4 -0
- package/dist/internal/storage/index.js +4 -0
- package/dist/internal/storage/jsonl-event-store.d.ts +13 -0
- package/dist/internal/storage/jsonl-event-store.js +105 -0
- package/dist/internal/storage/memory-event-store.d.ts +8 -0
- package/dist/internal/storage/memory-event-store.js +22 -0
- package/dist/protocol.d.ts +1 -0
- package/dist/protocol.js +1 -0
- package/dist/provider-workers.d.ts +1 -0
- package/dist/provider-workers.js +1 -0
- package/dist/provider.d.ts +1 -0
- package/dist/provider.js +1 -0
- package/dist/storage.d.ts +1 -0
- package/dist/storage.js +1 -0
- package/dist/testing.d.ts +1 -0
- package/dist/testing.js +1 -0
- package/package.json +76 -0
|
@@ -0,0 +1,461 @@
|
|
|
1
|
+
import { Buffer } from "node:buffer";
|
|
2
|
+
import { AsyncQueue, AudioChunkSchema, audioSampleCount, PlayoutProgressSchema, ProviderContextEventSchema, ProviderToolResultSchema, ToolCallSchema, } from "../protocol/index.js";
|
|
3
|
+
import { z } from "zod";
|
|
4
|
+
const FALLBACK_STREAM_ID = "assistant-output";
|
|
5
|
+
const CLOSE_REASON_BYTE_LIMIT = 120;
|
|
6
|
+
const ServerEnvelopeSchema = z.looseObject({ type: z.string().min(1) });
|
|
7
|
+
const OutputAudioDeltaSchema = z.looseObject({
|
|
8
|
+
item_id: z.string().min(1).optional(),
|
|
9
|
+
response_id: z.string().min(1).optional(),
|
|
10
|
+
delta: z.string().min(1).optional(),
|
|
11
|
+
data: z.string().min(1).optional(),
|
|
12
|
+
});
|
|
13
|
+
const FunctionCallItemSchema = z.looseObject({
|
|
14
|
+
type: z.literal("function_call"),
|
|
15
|
+
call_id: z.string().min(1),
|
|
16
|
+
name: z.string().min(1),
|
|
17
|
+
arguments: z.string(),
|
|
18
|
+
});
|
|
19
|
+
const FunctionCallArgumentsDoneSchema = z.looseObject({
|
|
20
|
+
call_id: z.string().min(1),
|
|
21
|
+
name: z.string().min(1),
|
|
22
|
+
arguments: z.string(),
|
|
23
|
+
});
|
|
24
|
+
const UsageDetailsSchema = z.looseObject({
|
|
25
|
+
text_tokens: z.number().int().nonnegative().optional(),
|
|
26
|
+
audio_tokens: z.number().int().nonnegative().optional(),
|
|
27
|
+
cached_tokens: z.number().int().nonnegative().optional(),
|
|
28
|
+
cached_tokens_details: z
|
|
29
|
+
.looseObject({
|
|
30
|
+
text_tokens: z.number().int().nonnegative().optional(),
|
|
31
|
+
audio_tokens: z.number().int().nonnegative().optional(),
|
|
32
|
+
})
|
|
33
|
+
.optional(),
|
|
34
|
+
});
|
|
35
|
+
const ResponseDoneSchema = z.looseObject({
|
|
36
|
+
response: z.looseObject({
|
|
37
|
+
output: z.array(z.looseObject({ type: z.string().optional() })).optional(),
|
|
38
|
+
usage: z
|
|
39
|
+
.looseObject({
|
|
40
|
+
total_tokens: z.number().int().nonnegative(),
|
|
41
|
+
input_tokens: z.number().int().nonnegative(),
|
|
42
|
+
output_tokens: z.number().int().nonnegative(),
|
|
43
|
+
input_token_details: UsageDetailsSchema.optional(),
|
|
44
|
+
output_token_details: UsageDetailsSchema.optional(),
|
|
45
|
+
})
|
|
46
|
+
.optional(),
|
|
47
|
+
}),
|
|
48
|
+
});
|
|
49
|
+
const TranscriptEventSchema = z.looseObject({
|
|
50
|
+
transcript: z.string(),
|
|
51
|
+
item_id: z.string().min(1).optional(),
|
|
52
|
+
});
|
|
53
|
+
const OutputTextDoneSchema = z.looseObject({
|
|
54
|
+
text: z.string(),
|
|
55
|
+
item_id: z.string().min(1).optional(),
|
|
56
|
+
});
|
|
57
|
+
const TranscriptDeltaWireSchema = z.looseObject({
|
|
58
|
+
delta: z.string(),
|
|
59
|
+
item_id: z.string().min(1).optional(),
|
|
60
|
+
});
|
|
61
|
+
const ServerErrorSchema = z.looseObject({
|
|
62
|
+
error: z.looseObject({ message: z.string().optional() }).optional(),
|
|
63
|
+
});
|
|
64
|
+
const SessionUpdatedSchema = z.looseObject({
|
|
65
|
+
session: z
|
|
66
|
+
.looseObject({
|
|
67
|
+
audio: z
|
|
68
|
+
.looseObject({
|
|
69
|
+
output: z
|
|
70
|
+
.looseObject({
|
|
71
|
+
format: z.looseObject({ rate: z.int().positive().optional() }).optional(),
|
|
72
|
+
})
|
|
73
|
+
.optional(),
|
|
74
|
+
})
|
|
75
|
+
.optional(),
|
|
76
|
+
})
|
|
77
|
+
.optional(),
|
|
78
|
+
});
|
|
79
|
+
export class RealtimeVoiceSession {
|
|
80
|
+
#socket;
|
|
81
|
+
#wire;
|
|
82
|
+
#queue = new AsyncQueue();
|
|
83
|
+
#outputStreams = new Map();
|
|
84
|
+
#playedThrough = new Map();
|
|
85
|
+
#deliveredToolCalls = new Set();
|
|
86
|
+
#settled;
|
|
87
|
+
#resolveSettled;
|
|
88
|
+
#outputSampleRate;
|
|
89
|
+
#activeOutputItemId;
|
|
90
|
+
#responseActive = false;
|
|
91
|
+
#responseWanted = false;
|
|
92
|
+
#ready = false;
|
|
93
|
+
#closed = false;
|
|
94
|
+
#closeReason;
|
|
95
|
+
constructor(socket, wire, options) {
|
|
96
|
+
this.#socket = socket;
|
|
97
|
+
this.#wire = wire;
|
|
98
|
+
this.#outputSampleRate = wire.outputSampleRate;
|
|
99
|
+
this.#settled = new Promise((resolve) => {
|
|
100
|
+
this.#resolveSettled = resolve;
|
|
101
|
+
});
|
|
102
|
+
socket.onMessage = (text) => this.#handleMessage(text);
|
|
103
|
+
socket.onError = (message) => this.#emitError(message, false);
|
|
104
|
+
socket.onClose = (code) => this.#settle(this.#closeReason ?? `socket_closed_${code}`);
|
|
105
|
+
options.signal.addEventListener("abort", () => {
|
|
106
|
+
void this.close(String(options.signal.reason ?? "aborted"));
|
|
107
|
+
}, { once: true });
|
|
108
|
+
this.#send({ type: "session.update", session: wire.sessionUpdate });
|
|
109
|
+
}
|
|
110
|
+
get events() {
|
|
111
|
+
return this.#queue;
|
|
112
|
+
}
|
|
113
|
+
async sendAudio(chunk) {
|
|
114
|
+
const validated = AudioChunkSchema.parse(chunk);
|
|
115
|
+
if (validated.channels !== 1) {
|
|
116
|
+
throw new Error(`${this.#wire.label} expects mono PCM input`);
|
|
117
|
+
}
|
|
118
|
+
if (validated.sampleRate !== this.#wire.inputSampleRate) {
|
|
119
|
+
throw new Error(`${this.#wire.label} session is configured for ${this.#wire.inputSampleRate} Hz input, received ${validated.sampleRate} Hz`);
|
|
120
|
+
}
|
|
121
|
+
this.#send({ type: "input_audio_buffer.append", audio: toBase64(validated.data) });
|
|
122
|
+
}
|
|
123
|
+
async sendText(text) {
|
|
124
|
+
this.#send({
|
|
125
|
+
type: "conversation.item.create",
|
|
126
|
+
item: { type: "message", role: "user", content: [{ type: "input_text", text }] },
|
|
127
|
+
});
|
|
128
|
+
this.#requestResponse();
|
|
129
|
+
}
|
|
130
|
+
async reportPlayout(progress) {
|
|
131
|
+
const validated = PlayoutProgressSchema.parse(progress);
|
|
132
|
+
this.#playedThrough.set(validated.streamId, validated.playedThroughSample);
|
|
133
|
+
}
|
|
134
|
+
async sendToolResult(result) {
|
|
135
|
+
const validated = ProviderToolResultSchema.parse(result);
|
|
136
|
+
this.#send({
|
|
137
|
+
type: "conversation.item.create",
|
|
138
|
+
item: {
|
|
139
|
+
type: "function_call_output",
|
|
140
|
+
call_id: validated.callId,
|
|
141
|
+
output: toolOutcomePayload(validated.outcome),
|
|
142
|
+
},
|
|
143
|
+
});
|
|
144
|
+
this.#requestResponse();
|
|
145
|
+
}
|
|
146
|
+
async sendContext(event) {
|
|
147
|
+
const validated = ProviderContextEventSchema.parse(event);
|
|
148
|
+
this.#send({
|
|
149
|
+
type: "conversation.item.create",
|
|
150
|
+
item: {
|
|
151
|
+
type: "message",
|
|
152
|
+
role: "system",
|
|
153
|
+
content: [{ type: "input_text", text: contextText(validated) }],
|
|
154
|
+
},
|
|
155
|
+
});
|
|
156
|
+
if (validated.type !== "task.progress" && this.#wire.announceTaskSettlement) {
|
|
157
|
+
this.#requestResponse();
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
async close(reason = "client_closed") {
|
|
161
|
+
this.#closeReason ??= reason;
|
|
162
|
+
if (!this.#closed) {
|
|
163
|
+
if (this.#socket.state === "closed") {
|
|
164
|
+
this.#settle(this.#closeReason);
|
|
165
|
+
}
|
|
166
|
+
else {
|
|
167
|
+
this.#socket.close(1000, truncateCloseReason(this.#closeReason));
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
await this.#settled;
|
|
171
|
+
}
|
|
172
|
+
#handleMessage(text) {
|
|
173
|
+
let parsed;
|
|
174
|
+
try {
|
|
175
|
+
parsed = JSON.parse(text);
|
|
176
|
+
}
|
|
177
|
+
catch {
|
|
178
|
+
this.#emitError(`Received malformed JSON from ${this.#wire.label}`, true);
|
|
179
|
+
return;
|
|
180
|
+
}
|
|
181
|
+
const envelope = ServerEnvelopeSchema.safeParse(parsed);
|
|
182
|
+
if (!envelope.success) {
|
|
183
|
+
this.#emitError(`Received a ${this.#wire.label} event without a type`, true);
|
|
184
|
+
return;
|
|
185
|
+
}
|
|
186
|
+
switch (envelope.data.type) {
|
|
187
|
+
case "session.updated":
|
|
188
|
+
this.#handleSessionUpdated(parsed);
|
|
189
|
+
break;
|
|
190
|
+
case "response.created":
|
|
191
|
+
this.#responseActive = true;
|
|
192
|
+
break;
|
|
193
|
+
case "response.output_audio.delta":
|
|
194
|
+
this.#handleAudioDelta(parsed);
|
|
195
|
+
break;
|
|
196
|
+
case "response.output_audio_transcript.delta":
|
|
197
|
+
case "response.output_text.delta":
|
|
198
|
+
this.#handleTranscriptDelta(parsed);
|
|
199
|
+
break;
|
|
200
|
+
case "response.output_audio_transcript.done":
|
|
201
|
+
this.#handleTranscript(parsed, "assistant");
|
|
202
|
+
break;
|
|
203
|
+
case "response.output_text.done":
|
|
204
|
+
this.#handleOutputText(parsed);
|
|
205
|
+
break;
|
|
206
|
+
case "conversation.item.input_audio_transcription.completed":
|
|
207
|
+
this.#handleTranscript(parsed, "user");
|
|
208
|
+
break;
|
|
209
|
+
case "response.function_call_arguments.done":
|
|
210
|
+
this.#handleFunctionCallArgumentsDone(parsed);
|
|
211
|
+
break;
|
|
212
|
+
case "response.done":
|
|
213
|
+
this.#handleResponseDone(parsed);
|
|
214
|
+
break;
|
|
215
|
+
case "input_audio_buffer.speech_started":
|
|
216
|
+
this.#handleSpeechStarted();
|
|
217
|
+
break;
|
|
218
|
+
case "error":
|
|
219
|
+
this.#handleServerError(parsed);
|
|
220
|
+
break;
|
|
221
|
+
default:
|
|
222
|
+
break;
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
#handleSessionUpdated(parsed) {
|
|
226
|
+
const updated = SessionUpdatedSchema.safeParse(parsed);
|
|
227
|
+
const rate = updated.success ? updated.data.session?.audio?.output?.format?.rate : undefined;
|
|
228
|
+
if (rate !== undefined)
|
|
229
|
+
this.#outputSampleRate = rate;
|
|
230
|
+
if (!this.#ready) {
|
|
231
|
+
this.#ready = true;
|
|
232
|
+
this.#emit({ type: "ready" });
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
#handleAudioDelta(parsed) {
|
|
236
|
+
const delta = OutputAudioDeltaSchema.safeParse(parsed);
|
|
237
|
+
if (!delta.success) {
|
|
238
|
+
this.#emitError("Received a malformed output audio delta", true);
|
|
239
|
+
return;
|
|
240
|
+
}
|
|
241
|
+
const payload = delta.data.delta ?? delta.data.data;
|
|
242
|
+
if (payload === undefined) {
|
|
243
|
+
this.#emitError("Received an output audio delta without an audio payload", true);
|
|
244
|
+
return;
|
|
245
|
+
}
|
|
246
|
+
const bytes = Buffer.from(payload, "base64");
|
|
247
|
+
if (bytes.byteLength === 0 || bytes.byteLength % 2 !== 0) {
|
|
248
|
+
this.#emitError(`Received a PCM16 delta with invalid byte length ${bytes.byteLength}`, true);
|
|
249
|
+
return;
|
|
250
|
+
}
|
|
251
|
+
const streamId = delta.data.item_id ?? delta.data.response_id ?? FALLBACK_STREAM_ID;
|
|
252
|
+
const stream = this.#outputStreams.get(streamId) ?? { sequence: 0, startSample: 0 };
|
|
253
|
+
const chunk = AudioChunkSchema.parse({
|
|
254
|
+
streamId,
|
|
255
|
+
sequence: stream.sequence,
|
|
256
|
+
encoding: "pcm_s16le",
|
|
257
|
+
sampleRate: this.#outputSampleRate,
|
|
258
|
+
channels: 1,
|
|
259
|
+
startSample: stream.startSample,
|
|
260
|
+
data: new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength),
|
|
261
|
+
});
|
|
262
|
+
stream.sequence += 1;
|
|
263
|
+
stream.startSample += audioSampleCount(chunk);
|
|
264
|
+
this.#outputStreams.set(streamId, stream);
|
|
265
|
+
this.#activeOutputItemId = streamId;
|
|
266
|
+
this.#emit({ type: "audio.output", chunk });
|
|
267
|
+
}
|
|
268
|
+
#handleTranscript(parsed, role) {
|
|
269
|
+
const transcript = TranscriptEventSchema.safeParse(parsed);
|
|
270
|
+
if (!transcript.success)
|
|
271
|
+
return;
|
|
272
|
+
this.#emitTranscript(role, transcript.data.transcript, transcript.data.item_id);
|
|
273
|
+
}
|
|
274
|
+
#handleTranscriptDelta(parsed) {
|
|
275
|
+
const delta = TranscriptDeltaWireSchema.safeParse(parsed);
|
|
276
|
+
if (!delta.success || delta.data.delta === "")
|
|
277
|
+
return;
|
|
278
|
+
this.#emit({
|
|
279
|
+
type: "transcript.delta",
|
|
280
|
+
role: "assistant",
|
|
281
|
+
delta: delta.data.delta,
|
|
282
|
+
...(delta.data.item_id === undefined ? {} : { streamId: delta.data.item_id }),
|
|
283
|
+
});
|
|
284
|
+
}
|
|
285
|
+
#handleOutputText(parsed) {
|
|
286
|
+
const output = OutputTextDoneSchema.safeParse(parsed);
|
|
287
|
+
if (!output.success)
|
|
288
|
+
return;
|
|
289
|
+
this.#emitTranscript("assistant", output.data.text, output.data.item_id);
|
|
290
|
+
}
|
|
291
|
+
#emitTranscript(role, rawText, streamId) {
|
|
292
|
+
const text = rawText.trim();
|
|
293
|
+
if (text === "")
|
|
294
|
+
return;
|
|
295
|
+
this.#emit({ type: "transcript", role, text, ...(streamId === undefined ? {} : { streamId }) });
|
|
296
|
+
}
|
|
297
|
+
#handleFunctionCallArgumentsDone(parsed) {
|
|
298
|
+
const done = FunctionCallArgumentsDoneSchema.safeParse(parsed);
|
|
299
|
+
if (!done.success)
|
|
300
|
+
return;
|
|
301
|
+
this.#emitFunctionCall(done.data.call_id, done.data.name, done.data.arguments);
|
|
302
|
+
}
|
|
303
|
+
#handleResponseDone(parsed) {
|
|
304
|
+
this.#responseActive = false;
|
|
305
|
+
this.#activeOutputItemId = undefined;
|
|
306
|
+
const done = ResponseDoneSchema.safeParse(parsed);
|
|
307
|
+
if (done.success) {
|
|
308
|
+
const usage = done.data.response.usage;
|
|
309
|
+
if (usage)
|
|
310
|
+
this.#emitUsage(usage);
|
|
311
|
+
for (const item of done.data.response.output ?? []) {
|
|
312
|
+
if (item.type !== "function_call")
|
|
313
|
+
continue;
|
|
314
|
+
const call = FunctionCallItemSchema.safeParse(item);
|
|
315
|
+
if (!call.success) {
|
|
316
|
+
this.#emitError("Received a function call item without call_id, name, or arguments", true);
|
|
317
|
+
continue;
|
|
318
|
+
}
|
|
319
|
+
this.#emitFunctionCall(call.data.call_id, call.data.name, call.data.arguments);
|
|
320
|
+
}
|
|
321
|
+
}
|
|
322
|
+
if (this.#responseWanted)
|
|
323
|
+
this.#requestResponse();
|
|
324
|
+
}
|
|
325
|
+
#emitUsage(usage) {
|
|
326
|
+
const input = usage.input_token_details;
|
|
327
|
+
const output = usage.output_token_details;
|
|
328
|
+
this.#emit({
|
|
329
|
+
type: "usage",
|
|
330
|
+
inputTokens: usage.input_tokens,
|
|
331
|
+
outputTokens: usage.output_tokens,
|
|
332
|
+
totalTokens: usage.total_tokens,
|
|
333
|
+
...(input?.cached_tokens === undefined ? {} : { cachedInputTokens: input.cached_tokens }),
|
|
334
|
+
...(input?.audio_tokens === undefined ? {} : { audioInputTokens: input.audio_tokens }),
|
|
335
|
+
...(output?.audio_tokens === undefined ? {} : { audioOutputTokens: output.audio_tokens }),
|
|
336
|
+
...(input?.cached_tokens_details?.audio_tokens === undefined
|
|
337
|
+
? {}
|
|
338
|
+
: { cachedAudioInputTokens: input.cached_tokens_details.audio_tokens }),
|
|
339
|
+
});
|
|
340
|
+
}
|
|
341
|
+
#emitFunctionCall(callId, name, rawArguments) {
|
|
342
|
+
if (this.#deliveredToolCalls.has(callId))
|
|
343
|
+
return;
|
|
344
|
+
let parsedArguments;
|
|
345
|
+
try {
|
|
346
|
+
parsedArguments = rawArguments.trim() === "" ? {} : JSON.parse(rawArguments);
|
|
347
|
+
}
|
|
348
|
+
catch {
|
|
349
|
+
this.#emitError(`Could not parse arguments for tool call ${name}`, true);
|
|
350
|
+
return;
|
|
351
|
+
}
|
|
352
|
+
if (parsedArguments === null || typeof parsedArguments !== "object" || Array.isArray(parsedArguments)) {
|
|
353
|
+
this.#emitError(`Tool call ${name} arguments are not a JSON object`, true);
|
|
354
|
+
return;
|
|
355
|
+
}
|
|
356
|
+
this.#deliveredToolCalls.add(callId);
|
|
357
|
+
this.#emit({
|
|
358
|
+
type: "tool.call",
|
|
359
|
+
call: ToolCallSchema.parse({ callId, name, arguments: parsedArguments }),
|
|
360
|
+
});
|
|
361
|
+
}
|
|
362
|
+
#handleSpeechStarted() {
|
|
363
|
+
const itemId = this.#activeOutputItemId;
|
|
364
|
+
if (itemId === undefined)
|
|
365
|
+
return;
|
|
366
|
+
this.#activeOutputItemId = undefined;
|
|
367
|
+
const playedThroughSample = this.#playedThrough.get(itemId);
|
|
368
|
+
if (playedThroughSample === undefined)
|
|
369
|
+
return;
|
|
370
|
+
this.#trySend({
|
|
371
|
+
type: "conversation.item.truncate",
|
|
372
|
+
item_id: itemId,
|
|
373
|
+
content_index: 0,
|
|
374
|
+
audio_end_ms: Math.floor((playedThroughSample / this.#outputSampleRate) * 1000),
|
|
375
|
+
});
|
|
376
|
+
}
|
|
377
|
+
#handleServerError(parsed) {
|
|
378
|
+
const error = ServerErrorSchema.safeParse(parsed);
|
|
379
|
+
const message = error.success ? error.data.error?.message : undefined;
|
|
380
|
+
this.#emitError(message ?? `${this.#wire.label} reported an error`, true);
|
|
381
|
+
}
|
|
382
|
+
#requestResponse() {
|
|
383
|
+
if (this.#responseActive) {
|
|
384
|
+
this.#responseWanted = true;
|
|
385
|
+
return;
|
|
386
|
+
}
|
|
387
|
+
this.#responseWanted = false;
|
|
388
|
+
this.#responseActive = true;
|
|
389
|
+
this.#trySend({ type: "response.create" });
|
|
390
|
+
}
|
|
391
|
+
#send(payload) {
|
|
392
|
+
if (this.#socket.state !== "open") {
|
|
393
|
+
throw new Error(`${this.#wire.label} session socket is not open`);
|
|
394
|
+
}
|
|
395
|
+
this.#socket.send(JSON.stringify(payload));
|
|
396
|
+
}
|
|
397
|
+
/** For sends triggered by incoming events: a closing socket is settlement, not an error. */
|
|
398
|
+
#trySend(payload) {
|
|
399
|
+
if (this.#socket.state !== "open")
|
|
400
|
+
return;
|
|
401
|
+
this.#send(payload);
|
|
402
|
+
}
|
|
403
|
+
#emit(event) {
|
|
404
|
+
if (this.#closed)
|
|
405
|
+
return;
|
|
406
|
+
this.#queue.push(event);
|
|
407
|
+
}
|
|
408
|
+
#emitError(message, recoverable) {
|
|
409
|
+
this.#emit({ type: "error", message, recoverable });
|
|
410
|
+
}
|
|
411
|
+
#settle(reason) {
|
|
412
|
+
if (this.#closed)
|
|
413
|
+
return;
|
|
414
|
+
this.#closed = true;
|
|
415
|
+
this.#queue.push({ type: "closed", reason });
|
|
416
|
+
this.#queue.close();
|
|
417
|
+
this.#resolveSettled();
|
|
418
|
+
}
|
|
419
|
+
}
|
|
420
|
+
export function toRealtimeFunctionTool(definition) {
|
|
421
|
+
return {
|
|
422
|
+
type: "function",
|
|
423
|
+
name: definition.name,
|
|
424
|
+
description: definition.description,
|
|
425
|
+
parameters: definition.inputSchema,
|
|
426
|
+
};
|
|
427
|
+
}
|
|
428
|
+
function toolOutcomePayload(outcome) {
|
|
429
|
+
switch (outcome.type) {
|
|
430
|
+
case "completed":
|
|
431
|
+
return JSON.stringify(outcome.value);
|
|
432
|
+
case "accepted_task":
|
|
433
|
+
return JSON.stringify({
|
|
434
|
+
status: "accepted_task",
|
|
435
|
+
taskId: outcome.taskId,
|
|
436
|
+
note: "The task is running in the background; updates will arrive as system context.",
|
|
437
|
+
});
|
|
438
|
+
case "failed":
|
|
439
|
+
return JSON.stringify({ status: "failed", error: outcome.error });
|
|
440
|
+
}
|
|
441
|
+
}
|
|
442
|
+
function contextText(event) {
|
|
443
|
+
switch (event.type) {
|
|
444
|
+
case "task.progress":
|
|
445
|
+
return `Background task ${event.taskId} progress: ${JSON.stringify(event.update)}`;
|
|
446
|
+
case "task.completed":
|
|
447
|
+
return `Background task ${event.taskId} completed with result: ${JSON.stringify(event.result)}`;
|
|
448
|
+
case "task.failed":
|
|
449
|
+
return `Background task ${event.taskId} failed: ${event.error}`;
|
|
450
|
+
case "task.cancelled":
|
|
451
|
+
return `Background task ${event.taskId} was cancelled: ${event.reason}`;
|
|
452
|
+
}
|
|
453
|
+
}
|
|
454
|
+
function toBase64(data) {
|
|
455
|
+
return Buffer.from(data.buffer, data.byteOffset, data.byteLength).toString("base64");
|
|
456
|
+
}
|
|
457
|
+
function truncateCloseReason(reason) {
|
|
458
|
+
return Buffer.byteLength(reason, "utf8") <= CLOSE_REASON_BYTE_LIMIT
|
|
459
|
+
? reason
|
|
460
|
+
: Buffer.from(reason, "utf8").subarray(0, CLOSE_REASON_BYTE_LIMIT).toString("utf8");
|
|
461
|
+
}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
export type RealtimeSocketState = "connecting" | "open" | "closing" | "closed";
|
|
2
|
+
/**
|
|
3
|
+
* The minimal transport surface the realtime session needs. Adapters normalize incoming
|
|
4
|
+
* frames to text and surface lifecycle through assignable callbacks, so the same session
|
|
5
|
+
* runs on Node (ws) and on Cloudflare Workers (fetch upgrade) unchanged.
|
|
6
|
+
*/
|
|
7
|
+
export interface RealtimeSocket {
|
|
8
|
+
readonly state: RealtimeSocketState;
|
|
9
|
+
send(data: string): void;
|
|
10
|
+
close(code?: number, reason?: string): void;
|
|
11
|
+
onMessage: ((text: string) => void) | undefined;
|
|
12
|
+
onError: ((message: string) => void) | undefined;
|
|
13
|
+
onClose: ((code: number) => void) | undefined;
|
|
14
|
+
}
|
|
15
|
+
export interface RealtimeSocketTarget {
|
|
16
|
+
/** Full endpoint including the model query parameter. */
|
|
17
|
+
url: string;
|
|
18
|
+
apiKey: string;
|
|
19
|
+
/** Subprotocol-based auth (e.g. OpenAI's), for runtimes whose standard WebSocket
|
|
20
|
+
* constructor cannot send an Authorization header. */
|
|
21
|
+
authProtocols?: readonly string[];
|
|
22
|
+
handshakeTimeoutMs: number;
|
|
23
|
+
signal: AbortSignal;
|
|
24
|
+
}
|
|
25
|
+
export type RealtimeSocketFactory = (target: RealtimeSocketTarget) => Promise<RealtimeSocket>;
|
|
26
|
+
/**
|
|
27
|
+
* Entry modules register their runtime's factory; providers resolve it lazily. Registration
|
|
28
|
+
* is write-once: re-registering the same factory is a no-op, and registering a different
|
|
29
|
+
* one throws so that mixing entries never silently depends on import order.
|
|
30
|
+
*/
|
|
31
|
+
export declare function setDefaultRealtimeSocketFactory(factory: RealtimeSocketFactory): void;
|
|
32
|
+
export declare function getDefaultRealtimeSocketFactory(): RealtimeSocketFactory;
|
|
33
|
+
export interface ConnectRealtimeSocketOptions {
|
|
34
|
+
url: string;
|
|
35
|
+
model: string;
|
|
36
|
+
apiKey: string;
|
|
37
|
+
authProtocols?: readonly string[];
|
|
38
|
+
handshakeTimeoutMs: number;
|
|
39
|
+
signal: AbortSignal;
|
|
40
|
+
factory?: RealtimeSocketFactory;
|
|
41
|
+
}
|
|
42
|
+
export declare function connectRealtimeSocket(options: ConnectRealtimeSocketOptions): Promise<RealtimeSocket>;
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
let defaultFactory;
|
|
2
|
+
/**
|
|
3
|
+
* Entry modules register their runtime's factory; providers resolve it lazily. Registration
|
|
4
|
+
* is write-once: re-registering the same factory is a no-op, and registering a different
|
|
5
|
+
* one throws so that mixing entries never silently depends on import order.
|
|
6
|
+
*/
|
|
7
|
+
export function setDefaultRealtimeSocketFactory(factory) {
|
|
8
|
+
if (defaultFactory === factory)
|
|
9
|
+
return;
|
|
10
|
+
if (defaultFactory !== undefined) {
|
|
11
|
+
throw new Error("A default realtime socket factory is already registered. Import exactly one of xo-harness/provider (Node) and xo-harness/provider/workers per process, or pass socketFactory explicitly.");
|
|
12
|
+
}
|
|
13
|
+
defaultFactory = factory;
|
|
14
|
+
}
|
|
15
|
+
export function getDefaultRealtimeSocketFactory() {
|
|
16
|
+
if (!defaultFactory) {
|
|
17
|
+
throw new Error("No realtime socket factory is registered. Import xo-harness/provider (Node) or xo-harness/provider/workers first, or pass socketFactory explicitly.");
|
|
18
|
+
}
|
|
19
|
+
return defaultFactory;
|
|
20
|
+
}
|
|
21
|
+
export function connectRealtimeSocket(options) {
|
|
22
|
+
const factory = options.factory ?? getDefaultRealtimeSocketFactory();
|
|
23
|
+
return factory({
|
|
24
|
+
url: `${options.url}?model=${encodeURIComponent(options.model)}`,
|
|
25
|
+
apiKey: options.apiKey,
|
|
26
|
+
handshakeTimeoutMs: options.handshakeTimeoutMs,
|
|
27
|
+
signal: options.signal,
|
|
28
|
+
...(options.authProtocols === undefined ? {} : { authProtocols: options.authProtocols }),
|
|
29
|
+
});
|
|
30
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { RealtimeSocketFactory } from "./realtime-socket.js";
|
|
2
|
+
/**
|
|
3
|
+
* Cloudflare Workers transport. Primary path: fetch upgrade with an Authorization
|
|
4
|
+
* header — the server-proper auth mechanism, and the one provider edges accept when
|
|
5
|
+
* they reject the browser-style subprotocol handshake from Workers egress (observed
|
|
6
|
+
* live 2026-08-11, production outage). Fallback: the standard WebSocket constructor
|
|
7
|
+
* with subprotocol auth, whose sockets get the runtime's full protocol handling
|
|
8
|
+
* (ping/pong) — a keepalive edge fetch-upgraded sockets have not always received;
|
|
9
|
+
* duplex media traffic keeps provider sessions busy enough that it has not bitten.
|
|
10
|
+
*/
|
|
11
|
+
export declare const workersSocketFactory: RealtimeSocketFactory;
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Cloudflare Workers transport. Primary path: fetch upgrade with an Authorization
|
|
3
|
+
* header — the server-proper auth mechanism, and the one provider edges accept when
|
|
4
|
+
* they reject the browser-style subprotocol handshake from Workers egress (observed
|
|
5
|
+
* live 2026-08-11, production outage). Fallback: the standard WebSocket constructor
|
|
6
|
+
* with subprotocol auth, whose sockets get the runtime's full protocol handling
|
|
7
|
+
* (ping/pong) — a keepalive edge fetch-upgraded sockets have not always received;
|
|
8
|
+
* duplex media traffic keeps provider sessions busy enough that it has not bitten.
|
|
9
|
+
*/
|
|
10
|
+
export const workersSocketFactory = async (target) => {
|
|
11
|
+
try {
|
|
12
|
+
return await fetchUpgradeSocket(target);
|
|
13
|
+
}
|
|
14
|
+
catch (error) {
|
|
15
|
+
const fallback = await subprotocolSocket(target);
|
|
16
|
+
if (!fallback)
|
|
17
|
+
throw error;
|
|
18
|
+
console.warn("fetch-upgrade handshake failed, falling back to subprotocol auth", error);
|
|
19
|
+
return fallback;
|
|
20
|
+
}
|
|
21
|
+
};
|
|
22
|
+
/** The native WebSocket + auth-subprotocol handshake; undefined when unavailable. */
|
|
23
|
+
const subprotocolSocket = async (target) => {
|
|
24
|
+
if (!target.authProtocols || target.authProtocols.length === 0)
|
|
25
|
+
return undefined;
|
|
26
|
+
const WorkersWebSocket = globalThis.WebSocket;
|
|
27
|
+
if (!WorkersWebSocket)
|
|
28
|
+
return undefined;
|
|
29
|
+
const ws = new WorkersWebSocket(target.url, [...target.authProtocols]);
|
|
30
|
+
await new Promise((resolve, reject) => {
|
|
31
|
+
const timer = setTimeout(() => reject(new Error("Realtime WebSocket handshake timed out")), target.handshakeTimeoutMs);
|
|
32
|
+
ws.addEventListener("open", () => {
|
|
33
|
+
clearTimeout(timer);
|
|
34
|
+
resolve();
|
|
35
|
+
});
|
|
36
|
+
ws.addEventListener("error", () => {
|
|
37
|
+
clearTimeout(timer);
|
|
38
|
+
reject(new Error("Realtime WebSocket handshake failed"));
|
|
39
|
+
});
|
|
40
|
+
ws.addEventListener("close", () => {
|
|
41
|
+
clearTimeout(timer);
|
|
42
|
+
reject(new Error("Realtime WebSocket closed during handshake"));
|
|
43
|
+
});
|
|
44
|
+
});
|
|
45
|
+
return wrapWorkersWebSocket(ws, target.signal);
|
|
46
|
+
};
|
|
47
|
+
const fetchUpgradeSocket = async (target) => {
|
|
48
|
+
const url = target.url.replace(/^ws(s?):/, "http$1:");
|
|
49
|
+
// The timeout must guard only the handshake. An AbortSignal passed to fetch stays
|
|
50
|
+
// attached to the upgraded connection on this runtime and would kill the live
|
|
51
|
+
// socket when it fires — so the deadline is raced, never attached.
|
|
52
|
+
let handshakeTimer;
|
|
53
|
+
const response = await Promise.race([
|
|
54
|
+
fetch(url, {
|
|
55
|
+
headers: { Upgrade: "websocket", Authorization: `Bearer ${target.apiKey}` },
|
|
56
|
+
}),
|
|
57
|
+
new Promise((_resolve, reject) => {
|
|
58
|
+
handshakeTimer = setTimeout(() => reject(new Error("Realtime WebSocket handshake timed out")), target.handshakeTimeoutMs);
|
|
59
|
+
}),
|
|
60
|
+
]).finally(() => clearTimeout(handshakeTimer));
|
|
61
|
+
const ws = response.webSocket;
|
|
62
|
+
if (!ws) {
|
|
63
|
+
throw new Error(`Realtime WebSocket upgrade failed with status ${response.status}`);
|
|
64
|
+
}
|
|
65
|
+
ws.accept();
|
|
66
|
+
return wrapWorkersWebSocket(ws, target.signal);
|
|
67
|
+
};
|
|
68
|
+
function wrapWorkersWebSocket(ws, signal) {
|
|
69
|
+
let state = "open";
|
|
70
|
+
const socket = {
|
|
71
|
+
get state() {
|
|
72
|
+
return state;
|
|
73
|
+
},
|
|
74
|
+
send: (data) => ws.send(data),
|
|
75
|
+
close: (code, reason) => {
|
|
76
|
+
if (state === "open")
|
|
77
|
+
state = "closing";
|
|
78
|
+
ws.close(code, reason);
|
|
79
|
+
},
|
|
80
|
+
onMessage: undefined,
|
|
81
|
+
onError: undefined,
|
|
82
|
+
onClose: undefined,
|
|
83
|
+
};
|
|
84
|
+
ws.addEventListener("message", (event) => {
|
|
85
|
+
if (typeof event.data === "string") {
|
|
86
|
+
socket.onMessage?.(event.data);
|
|
87
|
+
}
|
|
88
|
+
else if (event.data instanceof ArrayBuffer) {
|
|
89
|
+
socket.onMessage?.(new TextDecoder().decode(event.data));
|
|
90
|
+
}
|
|
91
|
+
});
|
|
92
|
+
ws.addEventListener("close", (event) => {
|
|
93
|
+
state = "closed";
|
|
94
|
+
socket.onClose?.(event.code ?? 1005);
|
|
95
|
+
});
|
|
96
|
+
ws.addEventListener("error", () => {
|
|
97
|
+
// Close races surface as error events on this runtime; settlement is not an error.
|
|
98
|
+
if (state !== "open")
|
|
99
|
+
return;
|
|
100
|
+
// An errored socket on workerd may never deliver a close event, so an error while
|
|
101
|
+
// open IS the close: report it, then settle, or the session waits forever.
|
|
102
|
+
state = "closed";
|
|
103
|
+
socket.onError?.("Realtime WebSocket transport error");
|
|
104
|
+
socket.onClose?.(1006);
|
|
105
|
+
});
|
|
106
|
+
signal.addEventListener("abort", () => socket.close(1000, "aborted"), { once: true });
|
|
107
|
+
return socket;
|
|
108
|
+
}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
export * from "./contract.js";
|
|
2
|
+
export * from "./grok-voice.js";
|
|
3
|
+
export * from "./openai-realtime.js";
|
|
4
|
+
export { RealtimeVoiceSession, type RealtimeWireOptions } from "./realtime-session.js";
|
|
5
|
+
export * from "./realtime-socket.js";
|
|
6
|
+
export { workersSocketFactory } from "./workers-socket.js";
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
// Cloudflare Workers entry: identical provider surface, fetch-upgrade transport,
|
|
2
|
+
// and no ws dependency anywhere in the module graph.
|
|
3
|
+
import { setDefaultRealtimeSocketFactory } from "./realtime-socket.js";
|
|
4
|
+
import { workersSocketFactory } from "./workers-socket.js";
|
|
5
|
+
setDefaultRealtimeSocketFactory(workersSocketFactory);
|
|
6
|
+
export * from "./contract.js";
|
|
7
|
+
export * from "./grok-voice.js";
|
|
8
|
+
export * from "./openai-realtime.js";
|
|
9
|
+
export { RealtimeVoiceSession } from "./realtime-session.js";
|
|
10
|
+
export * from "./realtime-socket.js";
|
|
11
|
+
export { workersSocketFactory } from "./workers-socket.js";
|