xo-harness 0.1.2 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -0
- package/dist/internal/harness/index.d.ts +3 -0
- package/dist/internal/harness/index.js +3 -0
- package/dist/internal/harness/message.d.ts +1 -2
- package/dist/internal/harness/message.js +45 -19
- package/dist/internal/harness/report-diff.d.ts +46 -0
- package/dist/internal/harness/report-diff.js +62 -0
- package/dist/internal/harness/report.d.ts +2 -0
- package/dist/internal/harness/report.js +10 -0
- package/dist/internal/harness/shadow.d.ts +9 -2
- package/dist/internal/harness/shadow.js +5 -1
- package/dist/internal/harness/task-supervisor.js +21 -6
- package/dist/internal/harness/tool-delivery.d.ts +17 -0
- package/dist/internal/harness/tool-delivery.js +53 -0
- package/dist/internal/harness/tool-policy.d.ts +52 -0
- package/dist/internal/harness/tool-policy.js +22 -0
- package/dist/internal/harness/tool-runtime.d.ts +4 -0
- package/dist/internal/harness/tool-runtime.js +82 -1
- package/dist/internal/harness/tools.d.ts +22 -1
- package/dist/internal/harness/voice-session.d.ts +9 -1
- package/dist/internal/harness/voice-session.js +22 -2
- package/dist/internal/harness/xo.d.ts +13 -0
- package/dist/internal/harness/xo.js +8 -0
- package/dist/internal/protocol/events.d.ts +72 -0
- package/dist/internal/protocol/events.js +14 -1
- package/dist/internal/protocol/provider.d.ts +84 -2
- package/dist/internal/protocol/provider.js +51 -3
- package/dist/internal/provider/grok-voice.d.ts +1 -0
- package/dist/internal/provider/grok-voice.js +4 -0
- package/dist/internal/provider/openai-realtime.d.ts +10 -1
- package/dist/internal/provider/openai-realtime.js +26 -1
- package/dist/internal/provider/realtime-session.d.ts +6 -0
- package/dist/internal/provider/realtime-session.js +267 -32
- package/dist/internal/provider-fake/replay-voice-provider.d.ts +6 -2
- package/dist/internal/provider-fake/replay-voice-provider.js +13 -3
- package/dist/internal/skills/index.d.ts +2 -0
- package/dist/internal/skills/index.js +2 -0
- package/dist/internal/skills/node.d.ts +7 -0
- package/dist/internal/skills/node.js +99 -0
- package/dist/internal/skills/skill.d.ts +17 -0
- package/dist/internal/skills/skill.js +42 -0
- package/dist/internal/skills/tools.d.ts +7 -0
- package/dist/internal/skills/tools.js +82 -0
- package/dist/internal/storage/memory.d.ts +2 -0
- package/dist/internal/storage/memory.js +1 -0
- package/dist/internal/tools-openai/delegate-conversation.d.ts +12 -0
- package/dist/internal/tools-openai/delegate-conversation.js +66 -0
- package/dist/internal/tools-openai/index.d.ts +24 -0
- package/dist/internal/tools-openai/index.js +119 -0
- package/dist/internal/tools-openai/responses.d.ts +31 -0
- package/dist/internal/tools-openai/responses.js +146 -0
- package/dist/skills-node.d.ts +1 -0
- package/dist/skills-node.js +1 -0
- package/dist/skills.d.ts +1 -0
- package/dist/skills.js +1 -0
- package/dist/storage-memory.d.ts +1 -0
- package/dist/storage-memory.js +1 -0
- package/dist/tools-openai.d.ts +1 -0
- package/dist/tools-openai.js +1 -0
- package/package.json +18 -1
|
@@ -1,10 +1,9 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { AssistantTurnRequestSchema, AsyncQueue, AudioChunkSchema, audioSampleCount, PlayoutProgressSchema, ProviderContextEventSchema, ProviderToolResultSchema, ToolCallSchema, } from "../protocol/index.js";
|
|
1
|
+
import { AssistantTurnRequestSchema, AsyncQueue, AudioChunkSchema, audioSampleCount, PlayoutProgressSchema, PROVIDER_DIAGNOSTIC_MAX_CONTENT_TYPES, PROVIDER_DIAGNOSTIC_MAX_OUTPUTS, PROVIDER_DIAGNOSTIC_STRING_MAX_LENGTH, ProviderContextEventSchema, ProviderToolResultSchema, ToolCallSchema, } from "../protocol/index.js";
|
|
3
2
|
import { z } from "zod";
|
|
4
3
|
const FALLBACK_STREAM_ID = "assistant-output";
|
|
5
4
|
const CLOSE_REASON_BYTE_LIMIT = 120;
|
|
6
|
-
const INPUT_TRANSCRIPT_SETTLE_MS = 300;
|
|
7
5
|
const EMITTED_USER_TRANSCRIPT_CACHE_LIMIT = 128;
|
|
6
|
+
const VAD_RESPONSE_RECOVERY_MS = 1_500;
|
|
8
7
|
const ServerEnvelopeSchema = z.looseObject({ type: z.string().min(1) });
|
|
9
8
|
const OutputAudioDeltaSchema = z.looseObject({
|
|
10
9
|
item_id: z.string().min(1).optional(),
|
|
@@ -34,6 +33,29 @@ const UsageDetailsSchema = z.looseObject({
|
|
|
34
33
|
})
|
|
35
34
|
.optional(),
|
|
36
35
|
});
|
|
36
|
+
const ResponseOutputContentSchema = z.looseObject({
|
|
37
|
+
type: z.string().min(1).max(PROVIDER_DIAGNOSTIC_STRING_MAX_LENGTH).optional(),
|
|
38
|
+
});
|
|
39
|
+
const ResponseOutputDiagnosticSchema = z.looseObject({
|
|
40
|
+
id: z.string().min(1).max(PROVIDER_DIAGNOSTIC_STRING_MAX_LENGTH).optional(),
|
|
41
|
+
type: z.string().min(1).max(PROVIDER_DIAGNOSTIC_STRING_MAX_LENGTH).optional(),
|
|
42
|
+
content: z.array(ResponseOutputContentSchema).max(PROVIDER_DIAGNOSTIC_MAX_CONTENT_TYPES).optional(),
|
|
43
|
+
});
|
|
44
|
+
const ResponseStatusDetailsSchema = z.looseObject({
|
|
45
|
+
reason: z.string().min(1).max(PROVIDER_DIAGNOSTIC_STRING_MAX_LENGTH).nullish(),
|
|
46
|
+
error: z
|
|
47
|
+
.looseObject({
|
|
48
|
+
code: z.string().min(1).max(PROVIDER_DIAGNOSTIC_STRING_MAX_LENGTH).optional(),
|
|
49
|
+
})
|
|
50
|
+
.nullish(),
|
|
51
|
+
});
|
|
52
|
+
const ResponseDiagnosticSchema = z.looseObject({
|
|
53
|
+
id: z.string().min(1).max(PROVIDER_DIAGNOSTIC_STRING_MAX_LENGTH).optional(),
|
|
54
|
+
status: z.string().min(1).max(PROVIDER_DIAGNOSTIC_STRING_MAX_LENGTH).optional(),
|
|
55
|
+
status_details: ResponseStatusDetailsSchema.nullish(),
|
|
56
|
+
output: z.array(ResponseOutputDiagnosticSchema).max(PROVIDER_DIAGNOSTIC_MAX_OUTPUTS).optional(),
|
|
57
|
+
});
|
|
58
|
+
/** Core response.done fields stay independent from optional diagnostic metadata. */
|
|
37
59
|
const ResponseDoneSchema = z.looseObject({
|
|
38
60
|
response: z.looseObject({
|
|
39
61
|
output: z.array(z.looseObject({ type: z.string().optional() })).optional(),
|
|
@@ -48,6 +70,13 @@ const ResponseDoneSchema = z.looseObject({
|
|
|
48
70
|
.optional(),
|
|
49
71
|
}),
|
|
50
72
|
});
|
|
73
|
+
const ResponseEnvelopeSchema = z.looseObject({
|
|
74
|
+
response: z.looseObject({}),
|
|
75
|
+
});
|
|
76
|
+
/** Response identity is core bookkeeping, separate from optional diagnostic metadata. */
|
|
77
|
+
const ResponseIdEnvelopeSchema = z.looseObject({
|
|
78
|
+
response: z.looseObject({ id: z.string().min(1).optional() }),
|
|
79
|
+
});
|
|
51
80
|
const TranscriptEventSchema = z.looseObject({
|
|
52
81
|
transcript: z.string(),
|
|
53
82
|
item_id: z.string().min(1).optional(),
|
|
@@ -66,6 +95,7 @@ const ServerErrorSchema = z.looseObject({
|
|
|
66
95
|
const SessionUpdatedSchema = z.looseObject({
|
|
67
96
|
session: z
|
|
68
97
|
.looseObject({
|
|
98
|
+
model: z.string().min(1).optional(),
|
|
69
99
|
audio: z
|
|
70
100
|
.looseObject({
|
|
71
101
|
output: z
|
|
@@ -78,21 +108,34 @@ const SessionUpdatedSchema = z.looseObject({
|
|
|
78
108
|
})
|
|
79
109
|
.optional(),
|
|
80
110
|
});
|
|
111
|
+
const SpeechBoundarySchema = z.looseObject({
|
|
112
|
+
item_id: z.string().min(1).max(PROVIDER_DIAGNOSTIC_STRING_MAX_LENGTH).optional(),
|
|
113
|
+
audio_start_ms: z.number().nonnegative().optional(),
|
|
114
|
+
audio_end_ms: z.number().nonnegative().optional(),
|
|
115
|
+
});
|
|
81
116
|
export class RealtimeVoiceSession {
|
|
82
117
|
#socket;
|
|
83
118
|
#wire;
|
|
84
119
|
#queue = new AsyncQueue();
|
|
85
120
|
#outputStreams = new Map();
|
|
86
121
|
#playedThrough = new Map();
|
|
122
|
+
#interruptedStreams = new Set();
|
|
87
123
|
#deliveredToolCalls = new Set();
|
|
124
|
+
#completedResponseIds = new Set();
|
|
125
|
+
#usageResponseIds = new Set();
|
|
88
126
|
#pendingUserTranscripts = new Map();
|
|
89
127
|
#emittedUserTranscripts = new Map();
|
|
90
128
|
#settled;
|
|
91
129
|
#resolveSettled;
|
|
92
130
|
#outputSampleRate;
|
|
93
|
-
#
|
|
131
|
+
#responseNumber = 0;
|
|
94
132
|
#responseActive = false;
|
|
133
|
+
#activeResponseId;
|
|
95
134
|
#responseWanted = false;
|
|
135
|
+
#inputSpeechActive = false;
|
|
136
|
+
#awaitingVadResponse = false;
|
|
137
|
+
#vadResponseRecoveryTimer;
|
|
138
|
+
#reportedResponseState;
|
|
96
139
|
#wantedResponseInstructions;
|
|
97
140
|
#ready = false;
|
|
98
141
|
#closed = false;
|
|
@@ -197,14 +240,22 @@ export class RealtimeVoiceSession {
|
|
|
197
240
|
this.#handleSessionUpdated(parsed);
|
|
198
241
|
break;
|
|
199
242
|
case "response.created":
|
|
243
|
+
this.#handleResponseCreated(parsed);
|
|
200
244
|
this.#responseActive = true;
|
|
245
|
+
this.#activeResponseId = responseIdFromEnvelope(parsed);
|
|
246
|
+
if (!this.#inputSpeechActive)
|
|
247
|
+
this.#clearAwaitingVadResponse();
|
|
248
|
+
this.#reportResponseState();
|
|
201
249
|
break;
|
|
202
250
|
case "response.output_audio.delta":
|
|
203
251
|
this.#handleAudioDelta(parsed);
|
|
204
252
|
break;
|
|
205
253
|
case "response.output_audio_transcript.delta":
|
|
206
254
|
case "response.output_text.delta":
|
|
207
|
-
this.#handleTranscriptDelta(parsed);
|
|
255
|
+
this.#handleTranscriptDelta(parsed, "assistant");
|
|
256
|
+
break;
|
|
257
|
+
case "conversation.item.input_audio_transcription.delta":
|
|
258
|
+
this.#handleTranscriptDelta(parsed, "user");
|
|
208
259
|
break;
|
|
209
260
|
case "response.output_audio_transcript.done":
|
|
210
261
|
this.#handleTranscript(parsed, "assistant");
|
|
@@ -222,7 +273,20 @@ export class RealtimeVoiceSession {
|
|
|
222
273
|
this.#handleResponseDone(parsed);
|
|
223
274
|
break;
|
|
224
275
|
case "input_audio_buffer.speech_started":
|
|
225
|
-
this.#handleSpeechStarted();
|
|
276
|
+
this.#handleSpeechStarted(parsed);
|
|
277
|
+
break;
|
|
278
|
+
case "input_audio_buffer.speech_stopped":
|
|
279
|
+
this.#emitSpeechDiagnostic(parsed, "stopped");
|
|
280
|
+
this.#inputSpeechActive = false;
|
|
281
|
+
if (this.#awaitingVadResponse && this.#responseActive && this.#wire.interruptOnSpeech === false) {
|
|
282
|
+
// Non-interrupting VAD can skip automatic creation while an older response
|
|
283
|
+
// is active. Request its continuation once that response finishes.
|
|
284
|
+
this.#clearAwaitingVadResponse();
|
|
285
|
+
this.#responseWanted = true;
|
|
286
|
+
}
|
|
287
|
+
else if (this.#awaitingVadResponse) {
|
|
288
|
+
this.#armVadResponseRecovery();
|
|
289
|
+
}
|
|
226
290
|
break;
|
|
227
291
|
case "error":
|
|
228
292
|
this.#handleServerError(parsed);
|
|
@@ -236,11 +300,19 @@ export class RealtimeVoiceSession {
|
|
|
236
300
|
const rate = updated.success ? updated.data.session?.audio?.output?.format?.rate : undefined;
|
|
237
301
|
if (rate !== undefined)
|
|
238
302
|
this.#outputSampleRate = rate;
|
|
303
|
+
const model = updated.success ? updated.data.session?.model : undefined;
|
|
304
|
+
if (model !== undefined && model.length <= PROVIDER_DIAGNOSTIC_STRING_MAX_LENGTH) {
|
|
305
|
+
this.#emit({ type: "diagnostic", diagnostic: { kind: "session", model } });
|
|
306
|
+
}
|
|
239
307
|
if (!this.#ready) {
|
|
240
308
|
this.#ready = true;
|
|
241
309
|
this.#emit({ type: "ready" });
|
|
310
|
+
this.#reportResponseState();
|
|
242
311
|
}
|
|
243
312
|
}
|
|
313
|
+
#handleResponseCreated(parsed) {
|
|
314
|
+
this.#emitResponseEnvelopeDiagnostic(parsed, "started");
|
|
315
|
+
}
|
|
244
316
|
#handleAudioDelta(parsed) {
|
|
245
317
|
const delta = OutputAudioDeltaSchema.safeParse(parsed);
|
|
246
318
|
if (!delta.success) {
|
|
@@ -252,13 +324,29 @@ export class RealtimeVoiceSession {
|
|
|
252
324
|
this.#emitError("Received an output audio delta without an audio payload", true);
|
|
253
325
|
return;
|
|
254
326
|
}
|
|
255
|
-
|
|
327
|
+
let bytes;
|
|
328
|
+
try {
|
|
329
|
+
bytes = Uint8Array.from(atob(payload), (character) => character.charCodeAt(0));
|
|
330
|
+
}
|
|
331
|
+
catch {
|
|
332
|
+
this.#emitError("Received an output audio delta with invalid base64", true);
|
|
333
|
+
return;
|
|
334
|
+
}
|
|
256
335
|
if (bytes.byteLength === 0 || bytes.byteLength % 2 !== 0) {
|
|
257
336
|
this.#emitError(`Received a PCM16 delta with invalid byte length ${bytes.byteLength}`, true);
|
|
258
337
|
return;
|
|
259
338
|
}
|
|
260
|
-
const streamId = delta.data.item_id ??
|
|
261
|
-
|
|
339
|
+
const streamId = delta.data.item_id ??
|
|
340
|
+
delta.data.response_id ??
|
|
341
|
+
(this.#responseNumber === 0 ? FALLBACK_STREAM_ID : `${FALLBACK_STREAM_ID}-${this.#responseNumber}`);
|
|
342
|
+
if (this.#interruptedStreams.has(streamId))
|
|
343
|
+
return;
|
|
344
|
+
const stream = this.#outputStreams.get(streamId) ?? {
|
|
345
|
+
sequence: 0,
|
|
346
|
+
startSample: 0,
|
|
347
|
+
complete: false,
|
|
348
|
+
itemId: delta.data.item_id,
|
|
349
|
+
};
|
|
262
350
|
const chunk = AudioChunkSchema.parse({
|
|
263
351
|
streamId,
|
|
264
352
|
sequence: stream.sequence,
|
|
@@ -266,12 +354,11 @@ export class RealtimeVoiceSession {
|
|
|
266
354
|
sampleRate: this.#outputSampleRate,
|
|
267
355
|
channels: 1,
|
|
268
356
|
startSample: stream.startSample,
|
|
269
|
-
data:
|
|
357
|
+
data: bytes,
|
|
270
358
|
});
|
|
271
359
|
stream.sequence += 1;
|
|
272
360
|
stream.startSample += audioSampleCount(chunk);
|
|
273
361
|
this.#outputStreams.set(streamId, stream);
|
|
274
|
-
this.#activeOutputItemId = streamId;
|
|
275
362
|
this.#emit({ type: "audio.output", chunk });
|
|
276
363
|
}
|
|
277
364
|
#handleTranscript(parsed, role) {
|
|
@@ -292,8 +379,10 @@ export class RealtimeVoiceSession {
|
|
|
292
379
|
const pending = this.#pendingUserTranscripts.get(streamId);
|
|
293
380
|
if (pending)
|
|
294
381
|
clearTimeout(pending.timer);
|
|
295
|
-
const timer = setTimeout(() => this.#flushUserTranscript(streamId),
|
|
382
|
+
const timer = setTimeout(() => this.#flushUserTranscript(streamId), this.#wire.inputTranscriptSettleMs ?? 0);
|
|
296
383
|
this.#pendingUserTranscripts.set(streamId, { text, timer });
|
|
384
|
+
if (!this.#wire.inputTranscriptSettleMs)
|
|
385
|
+
this.#flushUserTranscript(streamId);
|
|
297
386
|
}
|
|
298
387
|
#flushUserTranscript(streamId) {
|
|
299
388
|
const pending = this.#pendingUserTranscripts.get(streamId);
|
|
@@ -316,13 +405,13 @@ export class RealtimeVoiceSession {
|
|
|
316
405
|
for (const streamId of [...this.#pendingUserTranscripts.keys()])
|
|
317
406
|
this.#flushUserTranscript(streamId);
|
|
318
407
|
}
|
|
319
|
-
#handleTranscriptDelta(parsed) {
|
|
408
|
+
#handleTranscriptDelta(parsed, role) {
|
|
320
409
|
const delta = TranscriptDeltaWireSchema.safeParse(parsed);
|
|
321
410
|
if (!delta.success || delta.data.delta === "")
|
|
322
411
|
return;
|
|
323
412
|
this.#emit({
|
|
324
413
|
type: "transcript.delta",
|
|
325
|
-
role
|
|
414
|
+
role,
|
|
326
415
|
delta: delta.data.delta,
|
|
327
416
|
...(delta.data.item_id === undefined ? {} : { streamId: delta.data.item_id }),
|
|
328
417
|
});
|
|
@@ -346,13 +435,19 @@ export class RealtimeVoiceSession {
|
|
|
346
435
|
this.#emitFunctionCall(done.data.call_id, done.data.name, done.data.arguments);
|
|
347
436
|
}
|
|
348
437
|
#handleResponseDone(parsed) {
|
|
349
|
-
this.#
|
|
350
|
-
|
|
438
|
+
this.#emitResponseEnvelopeDiagnostic(parsed, "completed");
|
|
439
|
+
const responseId = responseIdFromEnvelope(parsed);
|
|
440
|
+
const duplicate = responseId !== undefined && this.#completedResponseIds.has(responseId);
|
|
441
|
+
if (responseId !== undefined)
|
|
442
|
+
this.#completedResponseIds.add(responseId);
|
|
351
443
|
const done = ResponseDoneSchema.safeParse(parsed);
|
|
352
444
|
if (done.success) {
|
|
353
445
|
const usage = done.data.response.usage;
|
|
354
|
-
if (usage)
|
|
446
|
+
if (usage && (responseId === undefined || !this.#usageResponseIds.has(responseId))) {
|
|
355
447
|
this.#emitUsage(usage);
|
|
448
|
+
if (responseId !== undefined)
|
|
449
|
+
this.#usageResponseIds.add(responseId);
|
|
450
|
+
}
|
|
356
451
|
for (const item of done.data.response.output ?? []) {
|
|
357
452
|
if (item.type !== "function_call")
|
|
358
453
|
continue;
|
|
@@ -364,11 +459,26 @@ export class RealtimeVoiceSession {
|
|
|
364
459
|
this.#emitFunctionCall(call.data.call_id, call.data.name, call.data.arguments);
|
|
365
460
|
}
|
|
366
461
|
}
|
|
462
|
+
// A duplicate or stale terminal update can refine diagnostics and carry omitted
|
|
463
|
+
// usage, but must never settle the response currently generating.
|
|
464
|
+
const settlesActiveResponse = !duplicate &&
|
|
465
|
+
(responseId === undefined ||
|
|
466
|
+
this.#activeResponseId === undefined ||
|
|
467
|
+
responseId === this.#activeResponseId);
|
|
468
|
+
if (!settlesActiveResponse)
|
|
469
|
+
return;
|
|
470
|
+
this.#responseActive = false;
|
|
471
|
+
this.#activeResponseId = undefined;
|
|
472
|
+
// Generation often ends well before the client finishes playing its queued PCM.
|
|
473
|
+
for (const stream of this.#outputStreams.values())
|
|
474
|
+
stream.complete = true;
|
|
475
|
+
this.#responseNumber += 1;
|
|
367
476
|
if (this.#responseWanted) {
|
|
368
477
|
const instructions = this.#wantedResponseInstructions;
|
|
369
478
|
this.#wantedResponseInstructions = undefined;
|
|
370
479
|
this.#requestResponse(instructions);
|
|
371
480
|
}
|
|
481
|
+
this.#reportResponseState();
|
|
372
482
|
}
|
|
373
483
|
#emitUsage(usage) {
|
|
374
484
|
const input = usage.input_token_details;
|
|
@@ -407,40 +517,151 @@ export class RealtimeVoiceSession {
|
|
|
407
517
|
call: ToolCallSchema.parse({ callId, name, arguments: parsedArguments }),
|
|
408
518
|
});
|
|
409
519
|
}
|
|
410
|
-
#handleSpeechStarted() {
|
|
411
|
-
|
|
412
|
-
|
|
520
|
+
#handleSpeechStarted(parsed) {
|
|
521
|
+
this.#emitSpeechDiagnostic(parsed, "started");
|
|
522
|
+
this.#clearVadResponseRecoveryTimer();
|
|
523
|
+
this.#inputSpeechActive = true;
|
|
524
|
+
this.#awaitingVadResponse = this.#wire.autoRespondToAudio;
|
|
525
|
+
this.#reportResponseState();
|
|
526
|
+
if (this.#wire.interruptOnSpeech === false)
|
|
413
527
|
return;
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
528
|
+
// A response can contain a preamble and an answer, both ahead of playback.
|
|
529
|
+
// Discard queued items first so stopping the audible item cannot start the next.
|
|
530
|
+
for (const [streamId, stream] of [...this.#outputStreams].reverse()) {
|
|
531
|
+
if (this.#interruptedStreams.has(streamId))
|
|
532
|
+
continue;
|
|
533
|
+
const playedThroughSample = this.#playedThrough.get(streamId) ?? 0;
|
|
534
|
+
if (stream.complete && playedThroughSample >= stream.startSample)
|
|
535
|
+
continue;
|
|
536
|
+
this.#interruptedStreams.add(streamId);
|
|
537
|
+
this.#emit({ type: "audio.interrupted", streamId });
|
|
538
|
+
// Only a real conversation item ID is valid in the vendor's truncate command.
|
|
539
|
+
if (stream.itemId === undefined)
|
|
540
|
+
continue;
|
|
541
|
+
this.#trySend({
|
|
542
|
+
type: "conversation.item.truncate",
|
|
543
|
+
item_id: stream.itemId,
|
|
544
|
+
content_index: 0,
|
|
545
|
+
audio_end_ms: Math.floor((Math.min(playedThroughSample, stream.startSample) / this.#outputSampleRate) * 1000),
|
|
546
|
+
});
|
|
547
|
+
}
|
|
548
|
+
}
|
|
549
|
+
#emitSpeechDiagnostic(parsed, phase) {
|
|
550
|
+
const boundary = SpeechBoundarySchema.safeParse(parsed);
|
|
551
|
+
if (!boundary.success)
|
|
417
552
|
return;
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
553
|
+
const audioOffsetMs = phase === "started" ? boundary.data.audio_start_ms : boundary.data.audio_end_ms;
|
|
554
|
+
this.#emit({
|
|
555
|
+
type: "diagnostic",
|
|
556
|
+
diagnostic: {
|
|
557
|
+
kind: "speech",
|
|
558
|
+
phase,
|
|
559
|
+
...(boundary.data.item_id === undefined ? {} : { streamId: boundary.data.item_id }),
|
|
560
|
+
...(audioOffsetMs === undefined ? {} : { audioOffsetMs }),
|
|
561
|
+
},
|
|
423
562
|
});
|
|
424
563
|
}
|
|
564
|
+
#emitResponseDiagnostic(parsed, phase) {
|
|
565
|
+
const response = ResponseDiagnosticSchema.safeParse(parsed);
|
|
566
|
+
if (!response.success)
|
|
567
|
+
return;
|
|
568
|
+
const details = response.data.status_details;
|
|
569
|
+
const outputs = response.data.output
|
|
570
|
+
?.filter((item) => item.type !== undefined)
|
|
571
|
+
.map((item) => ({
|
|
572
|
+
...(item.id === undefined ? {} : { streamId: item.id }),
|
|
573
|
+
type: item.type,
|
|
574
|
+
contentTypes: item.content?.flatMap((content) => (content.type === undefined ? [] : [content.type])) ?? [],
|
|
575
|
+
}));
|
|
576
|
+
this.#emit({
|
|
577
|
+
type: "diagnostic",
|
|
578
|
+
diagnostic: {
|
|
579
|
+
kind: "response",
|
|
580
|
+
phase,
|
|
581
|
+
...(response.data.id === undefined ? {} : { responseId: response.data.id }),
|
|
582
|
+
...(response.data.status === undefined ? {} : { status: response.data.status }),
|
|
583
|
+
...(details?.reason == null ? {} : { reason: details.reason }),
|
|
584
|
+
...(details?.error?.code === undefined ? {} : { code: details.error.code }),
|
|
585
|
+
...(outputs === undefined ? {} : { outputs }),
|
|
586
|
+
},
|
|
587
|
+
});
|
|
588
|
+
}
|
|
589
|
+
#emitResponseEnvelopeDiagnostic(parsed, phase) {
|
|
590
|
+
const envelope = ResponseEnvelopeSchema.safeParse(parsed);
|
|
591
|
+
if (envelope.success)
|
|
592
|
+
this.#emitResponseDiagnostic(envelope.data.response, phase);
|
|
593
|
+
}
|
|
425
594
|
#handleServerError(parsed) {
|
|
426
595
|
const error = ServerErrorSchema.safeParse(parsed);
|
|
427
596
|
const message = error.success ? error.data.error?.message : undefined;
|
|
428
597
|
this.#emitError(message ?? `${this.#wire.label} reported an error`, true);
|
|
598
|
+
if (this.#awaitingVadResponse && !this.#inputSpeechActive) {
|
|
599
|
+
this.#clearAwaitingVadResponse();
|
|
600
|
+
this.#resumeQueuedResponse();
|
|
601
|
+
}
|
|
429
602
|
}
|
|
430
603
|
#requestResponse(instructions) {
|
|
604
|
+
if (this.#awaitingVadResponse) {
|
|
605
|
+
// Tool/task context is already in the conversation. Let VAD answer after the
|
|
606
|
+
// user finishes, including the gap between speech_stopped and response.created.
|
|
607
|
+
if (instructions !== undefined)
|
|
608
|
+
this.#wantedResponseInstructions = instructions;
|
|
609
|
+
this.#responseWanted = this.#wantedResponseInstructions !== undefined;
|
|
610
|
+
this.#reportResponseState();
|
|
611
|
+
return;
|
|
612
|
+
}
|
|
431
613
|
if (this.#responseActive) {
|
|
432
614
|
this.#responseWanted = true;
|
|
433
615
|
if (instructions !== undefined)
|
|
434
616
|
this.#wantedResponseInstructions = instructions;
|
|
617
|
+
this.#reportResponseState();
|
|
435
618
|
return;
|
|
436
619
|
}
|
|
437
620
|
this.#responseWanted = false;
|
|
438
621
|
this.#responseActive = true;
|
|
622
|
+
this.#reportResponseState();
|
|
439
623
|
this.#trySend({
|
|
440
624
|
type: "response.create",
|
|
441
625
|
...(instructions === undefined ? {} : { response: { instructions } }),
|
|
442
626
|
});
|
|
443
627
|
}
|
|
628
|
+
#armVadResponseRecovery() {
|
|
629
|
+
this.#clearVadResponseRecoveryTimer();
|
|
630
|
+
this.#vadResponseRecoveryTimer = setTimeout(() => {
|
|
631
|
+
this.#vadResponseRecoveryTimer = undefined;
|
|
632
|
+
if (this.#closed || !this.#awaitingVadResponse)
|
|
633
|
+
return;
|
|
634
|
+
this.#clearAwaitingVadResponse();
|
|
635
|
+
this.#emitError("Provider did not create the expected VAD response", true);
|
|
636
|
+
this.#resumeQueuedResponse();
|
|
637
|
+
}, VAD_RESPONSE_RECOVERY_MS);
|
|
638
|
+
}
|
|
639
|
+
#clearAwaitingVadResponse() {
|
|
640
|
+
this.#awaitingVadResponse = false;
|
|
641
|
+
this.#clearVadResponseRecoveryTimer();
|
|
642
|
+
}
|
|
643
|
+
#clearVadResponseRecoveryTimer() {
|
|
644
|
+
if (this.#vadResponseRecoveryTimer !== undefined)
|
|
645
|
+
clearTimeout(this.#vadResponseRecoveryTimer);
|
|
646
|
+
this.#vadResponseRecoveryTimer = undefined;
|
|
647
|
+
}
|
|
648
|
+
#resumeQueuedResponse() {
|
|
649
|
+
if (!this.#responseWanted) {
|
|
650
|
+
this.#reportResponseState();
|
|
651
|
+
return;
|
|
652
|
+
}
|
|
653
|
+
const instructions = this.#wantedResponseInstructions;
|
|
654
|
+
this.#responseWanted = false;
|
|
655
|
+
this.#wantedResponseInstructions = undefined;
|
|
656
|
+
this.#requestResponse(instructions);
|
|
657
|
+
}
|
|
658
|
+
#reportResponseState() {
|
|
659
|
+
const state = this.#responseActive || this.#responseWanted || this.#awaitingVadResponse ? "pending" : "idle";
|
|
660
|
+
if (state === this.#reportedResponseState)
|
|
661
|
+
return;
|
|
662
|
+
this.#reportedResponseState = state;
|
|
663
|
+
this.#emit({ type: "response.state", state });
|
|
664
|
+
}
|
|
444
665
|
#send(payload) {
|
|
445
666
|
if (this.#socket.state !== "open") {
|
|
446
667
|
throw new Error(`${this.#wire.label} session socket is not open`);
|
|
@@ -464,6 +685,7 @@ export class RealtimeVoiceSession {
|
|
|
464
685
|
#settle(reason) {
|
|
465
686
|
if (this.#closed)
|
|
466
687
|
return;
|
|
688
|
+
this.#clearVadResponseRecoveryTimer();
|
|
467
689
|
this.#flushUserTranscripts();
|
|
468
690
|
this.#closed = true;
|
|
469
691
|
this.#queue.push({ type: "closed", reason });
|
|
@@ -506,10 +728,23 @@ function contextText(event) {
|
|
|
506
728
|
}
|
|
507
729
|
}
|
|
508
730
|
function toBase64(data) {
|
|
509
|
-
|
|
731
|
+
let binary = "";
|
|
732
|
+
// Bound each argument list; microphone buffers need not fit the engine's call stack.
|
|
733
|
+
for (let offset = 0; offset < data.length; offset += 0x8000) {
|
|
734
|
+
binary += String.fromCharCode(...data.subarray(offset, offset + 0x8000));
|
|
735
|
+
}
|
|
736
|
+
return btoa(binary);
|
|
510
737
|
}
|
|
511
738
|
function truncateCloseReason(reason) {
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
739
|
+
const bytes = new TextEncoder().encode(reason);
|
|
740
|
+
if (bytes.byteLength <= CLOSE_REASON_BYTE_LIMIT)
|
|
741
|
+
return reason;
|
|
742
|
+
// Streaming decode leaves a partial final code point buffered instead of replacing it.
|
|
743
|
+
return new TextDecoder("utf-8", { ignoreBOM: true }).decode(bytes.subarray(0, CLOSE_REASON_BYTE_LIMIT), {
|
|
744
|
+
stream: true,
|
|
745
|
+
});
|
|
746
|
+
}
|
|
747
|
+
function responseIdFromEnvelope(parsed) {
|
|
748
|
+
const envelope = ResponseIdEnvelopeSchema.safeParse(parsed);
|
|
749
|
+
return envelope.success ? envelope.data.response.id : undefined;
|
|
515
750
|
}
|
|
@@ -12,8 +12,12 @@ export interface ReplayVoiceProviderOptions {
|
|
|
12
12
|
* Replays the provider-visible half of a recorded session: model audio, tool requests,
|
|
13
13
|
* and provider errors, in recorded order and (optionally) recorded pacing. Input audio,
|
|
14
14
|
* playout acknowledgements, tool results, and context are accepted and captured but do
|
|
15
|
-
* not influence playback. This
|
|
16
|
-
*
|
|
15
|
+
* not influence playback. This re-drives UIs and stores from a recorded log without a
|
|
16
|
+
* live model or an API key.
|
|
17
|
+
*
|
|
18
|
+
* Playback closes after the last recorded output, even if a replayed tool request is
|
|
19
|
+
* still awaiting admission or execution. Recorded tool outcomes are not replayed.
|
|
20
|
+
* Use ScriptedVoiceProvider when subsequent output must wait for and verify results.
|
|
17
21
|
*/
|
|
18
22
|
export declare class ReplayVoiceProvider implements VoiceProvider {
|
|
19
23
|
#private;
|
|
@@ -3,8 +3,12 @@ import { AssistantTurnRequestSchema, AsyncQueue, AudioChunkSchema, PlayoutProgre
|
|
|
3
3
|
* Replays the provider-visible half of a recorded session: model audio, tool requests,
|
|
4
4
|
* and provider errors, in recorded order and (optionally) recorded pacing. Input audio,
|
|
5
5
|
* playout acknowledgements, tool results, and context are accepted and captured but do
|
|
6
|
-
* not influence playback. This
|
|
7
|
-
*
|
|
6
|
+
* not influence playback. This re-drives UIs and stores from a recorded log without a
|
|
7
|
+
* live model or an API key.
|
|
8
|
+
*
|
|
9
|
+
* Playback closes after the last recorded output, even if a replayed tool request is
|
|
10
|
+
* still awaiting admission or execution. Recorded tool outcomes are not replayed.
|
|
11
|
+
* Use ScriptedVoiceProvider when subsequent output must wait for and verify results.
|
|
8
12
|
*/
|
|
9
13
|
export class ReplayVoiceProvider {
|
|
10
14
|
id = "replay";
|
|
@@ -114,14 +118,18 @@ export class ReplayVoiceProviderSession {
|
|
|
114
118
|
}
|
|
115
119
|
function isReplayable(event) {
|
|
116
120
|
return (event.type === "audio.output" ||
|
|
121
|
+
event.type === "audio.interrupted" ||
|
|
117
122
|
event.type === "transcript" ||
|
|
118
123
|
event.type === "tool.requested" ||
|
|
119
|
-
event.type === "provider.error"
|
|
124
|
+
event.type === "provider.error" ||
|
|
125
|
+
event.type === "provider.diagnostic");
|
|
120
126
|
}
|
|
121
127
|
function toProviderEvent(event) {
|
|
122
128
|
switch (event.type) {
|
|
123
129
|
case "audio.output":
|
|
124
130
|
return { type: "audio.output", chunk: event.chunk };
|
|
131
|
+
case "audio.interrupted":
|
|
132
|
+
return { type: "audio.interrupted", streamId: event.streamId };
|
|
125
133
|
case "transcript":
|
|
126
134
|
return {
|
|
127
135
|
type: "transcript",
|
|
@@ -133,5 +141,7 @@ function toProviderEvent(event) {
|
|
|
133
141
|
return { type: "tool.call", call: event.call };
|
|
134
142
|
case "provider.error":
|
|
135
143
|
return { type: "error", message: event.message, recoverable: event.recoverable };
|
|
144
|
+
case "provider.diagnostic":
|
|
145
|
+
return { type: "diagnostic", diagnostic: event.diagnostic };
|
|
136
146
|
}
|
|
137
147
|
}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import { type Skill } from "./skill.js";
|
|
2
|
+
/**
|
|
3
|
+
* Discovers immediate skill directories in explicitly trusted roots. Missing roots
|
|
4
|
+
* are ignored; malformed skills and duplicates fail with a path for the host to fix.
|
|
5
|
+
* Nothing scans the user's home, executes scripts, or grants allowed-tools.
|
|
6
|
+
*/
|
|
7
|
+
export declare function loadSkillsFromDirectories(directories: readonly string[]): Promise<readonly Skill[]>;
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
import { constants } from "node:fs";
|
|
2
|
+
import { open, readdir, realpath } from "node:fs/promises";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import { parseSkillMarkdown, validateSkillPath } from "./skill.js";
|
|
5
|
+
// Skills are text guidance, not an unbounded file export API. Resource paging bounds
|
|
6
|
+
// model context; this limit also bounds host memory during discovery and reads.
|
|
7
|
+
const MAX_FILE_BYTES = 1024 * 1024;
|
|
8
|
+
/**
|
|
9
|
+
* Discovers immediate skill directories in explicitly trusted roots. Missing roots
|
|
10
|
+
* are ignored; malformed skills and duplicates fail with a path for the host to fix.
|
|
11
|
+
* Nothing scans the user's home, executes scripts, or grants allowed-tools.
|
|
12
|
+
*/
|
|
13
|
+
export async function loadSkillsFromDirectories(directories) {
|
|
14
|
+
const skills = new Map();
|
|
15
|
+
for (const directory of directories) {
|
|
16
|
+
let entries;
|
|
17
|
+
try {
|
|
18
|
+
entries = await readdir(directory, { withFileTypes: true });
|
|
19
|
+
}
|
|
20
|
+
catch (error) {
|
|
21
|
+
if (isMissing(error))
|
|
22
|
+
continue;
|
|
23
|
+
throw error;
|
|
24
|
+
}
|
|
25
|
+
for (const entry of entries.sort((a, b) => a.name.localeCompare(b.name))) {
|
|
26
|
+
if ((!entry.isDirectory() && !entry.isSymbolicLink()) || entry.name.startsWith("."))
|
|
27
|
+
continue;
|
|
28
|
+
const location = path.resolve(directory, entry.name);
|
|
29
|
+
let root;
|
|
30
|
+
let markdown;
|
|
31
|
+
try {
|
|
32
|
+
root = await realpath(location);
|
|
33
|
+
markdown = await readSkillFile(root, "SKILL.md");
|
|
34
|
+
}
|
|
35
|
+
catch (error) {
|
|
36
|
+
if (isMissing(error))
|
|
37
|
+
continue;
|
|
38
|
+
throw new Error(`Cannot load skill at ${location}`, { cause: error });
|
|
39
|
+
}
|
|
40
|
+
let metadata;
|
|
41
|
+
try {
|
|
42
|
+
metadata = parseSkillMarkdown(markdown);
|
|
43
|
+
if (metadata.name !== entry.name)
|
|
44
|
+
throw new Error("Skill name must match its directory name");
|
|
45
|
+
if (skills.has(metadata.name))
|
|
46
|
+
throw new Error(`Duplicate skill name: ${metadata.name}`);
|
|
47
|
+
}
|
|
48
|
+
catch (error) {
|
|
49
|
+
throw new Error(`Invalid skill at ${location}: ${error instanceof Error ? error.message : String(error)}`, { cause: error });
|
|
50
|
+
}
|
|
51
|
+
skills.set(metadata.name, {
|
|
52
|
+
...metadata,
|
|
53
|
+
// Freeze the activated instructions to the catalog that was discovered.
|
|
54
|
+
read: (resource, signal) => {
|
|
55
|
+
signal.throwIfAborted();
|
|
56
|
+
return resource === "SKILL.md" ? markdown : readSkillFile(root, resource, signal);
|
|
57
|
+
},
|
|
58
|
+
});
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
return [...skills.values()];
|
|
62
|
+
}
|
|
63
|
+
async function readSkillFile(root, resource, signal) {
|
|
64
|
+
validateSkillPath(resource);
|
|
65
|
+
signal?.throwIfAborted();
|
|
66
|
+
const target = await realpath(path.join(root, resource));
|
|
67
|
+
const relative = path.relative(root, target);
|
|
68
|
+
if (relative === ".." || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) {
|
|
69
|
+
throw new Error("Skill resource resolves outside its skill directory");
|
|
70
|
+
}
|
|
71
|
+
// Nonblocking open lets fstat reject FIFOs instead of waiting forever for a writer.
|
|
72
|
+
const file = await open(target, constants.O_RDONLY | constants.O_NONBLOCK);
|
|
73
|
+
try {
|
|
74
|
+
const stat = await file.stat();
|
|
75
|
+
if (!stat.isFile())
|
|
76
|
+
throw new Error("Skill resource must be a regular text file");
|
|
77
|
+
const bytes = new Uint8Array(MAX_FILE_BYTES + 1);
|
|
78
|
+
let size = 0;
|
|
79
|
+
while (size < bytes.length) {
|
|
80
|
+
signal?.throwIfAborted();
|
|
81
|
+
const { bytesRead } = await file.read(bytes, size, bytes.length - size, null);
|
|
82
|
+
if (bytesRead === 0)
|
|
83
|
+
break;
|
|
84
|
+
size += bytesRead;
|
|
85
|
+
}
|
|
86
|
+
if (size > MAX_FILE_BYTES)
|
|
87
|
+
throw new Error("Skill resource exceeds the 1 MiB text limit");
|
|
88
|
+
const text = new TextDecoder("utf-8", { fatal: true }).decode(bytes.subarray(0, size));
|
|
89
|
+
if (text.includes("\u0000"))
|
|
90
|
+
throw new Error("Skill resource must be UTF-8 text without NUL bytes");
|
|
91
|
+
return text;
|
|
92
|
+
}
|
|
93
|
+
finally {
|
|
94
|
+
await file.close();
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
function isMissing(error) {
|
|
98
|
+
return error instanceof Error && "code" in error && (error.code === "ENOENT" || error.code === "ENOTDIR");
|
|
99
|
+
}
|