xo-harness 0.1.2 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. package/README.md +17 -0
  2. package/dist/internal/harness/index.d.ts +3 -0
  3. package/dist/internal/harness/index.js +3 -0
  4. package/dist/internal/harness/message.d.ts +1 -2
  5. package/dist/internal/harness/message.js +45 -19
  6. package/dist/internal/harness/report-diff.d.ts +46 -0
  7. package/dist/internal/harness/report-diff.js +62 -0
  8. package/dist/internal/harness/report.d.ts +2 -0
  9. package/dist/internal/harness/report.js +10 -0
  10. package/dist/internal/harness/shadow.d.ts +9 -2
  11. package/dist/internal/harness/shadow.js +5 -1
  12. package/dist/internal/harness/task-supervisor.js +21 -6
  13. package/dist/internal/harness/tool-delivery.d.ts +17 -0
  14. package/dist/internal/harness/tool-delivery.js +53 -0
  15. package/dist/internal/harness/tool-policy.d.ts +52 -0
  16. package/dist/internal/harness/tool-policy.js +22 -0
  17. package/dist/internal/harness/tool-runtime.d.ts +4 -0
  18. package/dist/internal/harness/tool-runtime.js +82 -1
  19. package/dist/internal/harness/tools.d.ts +22 -1
  20. package/dist/internal/harness/voice-session.d.ts +9 -1
  21. package/dist/internal/harness/voice-session.js +22 -2
  22. package/dist/internal/harness/xo.d.ts +13 -0
  23. package/dist/internal/harness/xo.js +8 -0
  24. package/dist/internal/protocol/events.d.ts +72 -0
  25. package/dist/internal/protocol/events.js +14 -1
  26. package/dist/internal/protocol/provider.d.ts +84 -2
  27. package/dist/internal/protocol/provider.js +51 -3
  28. package/dist/internal/provider/grok-voice.d.ts +1 -0
  29. package/dist/internal/provider/grok-voice.js +4 -0
  30. package/dist/internal/provider/openai-realtime.d.ts +10 -1
  31. package/dist/internal/provider/openai-realtime.js +26 -1
  32. package/dist/internal/provider/realtime-session.d.ts +6 -0
  33. package/dist/internal/provider/realtime-session.js +267 -32
  34. package/dist/internal/provider-fake/replay-voice-provider.d.ts +6 -2
  35. package/dist/internal/provider-fake/replay-voice-provider.js +13 -3
  36. package/dist/internal/skills/index.d.ts +2 -0
  37. package/dist/internal/skills/index.js +2 -0
  38. package/dist/internal/skills/node.d.ts +7 -0
  39. package/dist/internal/skills/node.js +99 -0
  40. package/dist/internal/skills/skill.d.ts +17 -0
  41. package/dist/internal/skills/skill.js +42 -0
  42. package/dist/internal/skills/tools.d.ts +7 -0
  43. package/dist/internal/skills/tools.js +82 -0
  44. package/dist/internal/storage/memory.d.ts +2 -0
  45. package/dist/internal/storage/memory.js +1 -0
  46. package/dist/internal/tools-openai/delegate-conversation.d.ts +12 -0
  47. package/dist/internal/tools-openai/delegate-conversation.js +66 -0
  48. package/dist/internal/tools-openai/index.d.ts +24 -0
  49. package/dist/internal/tools-openai/index.js +119 -0
  50. package/dist/internal/tools-openai/responses.d.ts +31 -0
  51. package/dist/internal/tools-openai/responses.js +146 -0
  52. package/dist/skills-node.d.ts +1 -0
  53. package/dist/skills-node.js +1 -0
  54. package/dist/skills.d.ts +1 -0
  55. package/dist/skills.js +1 -0
  56. package/dist/storage-memory.d.ts +1 -0
  57. package/dist/storage-memory.js +1 -0
  58. package/dist/tools-openai.d.ts +1 -0
  59. package/dist/tools-openai.js +1 -0
  60. package/package.json +18 -1
@@ -1,10 +1,9 @@
1
- import { Buffer } from "node:buffer";
2
- import { AssistantTurnRequestSchema, AsyncQueue, AudioChunkSchema, audioSampleCount, PlayoutProgressSchema, ProviderContextEventSchema, ProviderToolResultSchema, ToolCallSchema, } from "../protocol/index.js";
1
+ import { AssistantTurnRequestSchema, AsyncQueue, AudioChunkSchema, audioSampleCount, PlayoutProgressSchema, PROVIDER_DIAGNOSTIC_MAX_CONTENT_TYPES, PROVIDER_DIAGNOSTIC_MAX_OUTPUTS, PROVIDER_DIAGNOSTIC_STRING_MAX_LENGTH, ProviderContextEventSchema, ProviderToolResultSchema, ToolCallSchema, } from "../protocol/index.js";
3
2
  import { z } from "zod";
4
3
  const FALLBACK_STREAM_ID = "assistant-output";
5
4
  const CLOSE_REASON_BYTE_LIMIT = 120;
6
- const INPUT_TRANSCRIPT_SETTLE_MS = 300;
7
5
  const EMITTED_USER_TRANSCRIPT_CACHE_LIMIT = 128;
6
+ const VAD_RESPONSE_RECOVERY_MS = 1_500;
8
7
  const ServerEnvelopeSchema = z.looseObject({ type: z.string().min(1) });
9
8
  const OutputAudioDeltaSchema = z.looseObject({
10
9
  item_id: z.string().min(1).optional(),
@@ -34,6 +33,29 @@ const UsageDetailsSchema = z.looseObject({
34
33
  })
35
34
  .optional(),
36
35
  });
36
+ const ResponseOutputContentSchema = z.looseObject({
37
+ type: z.string().min(1).max(PROVIDER_DIAGNOSTIC_STRING_MAX_LENGTH).optional(),
38
+ });
39
+ const ResponseOutputDiagnosticSchema = z.looseObject({
40
+ id: z.string().min(1).max(PROVIDER_DIAGNOSTIC_STRING_MAX_LENGTH).optional(),
41
+ type: z.string().min(1).max(PROVIDER_DIAGNOSTIC_STRING_MAX_LENGTH).optional(),
42
+ content: z.array(ResponseOutputContentSchema).max(PROVIDER_DIAGNOSTIC_MAX_CONTENT_TYPES).optional(),
43
+ });
44
+ const ResponseStatusDetailsSchema = z.looseObject({
45
+ reason: z.string().min(1).max(PROVIDER_DIAGNOSTIC_STRING_MAX_LENGTH).nullish(),
46
+ error: z
47
+ .looseObject({
48
+ code: z.string().min(1).max(PROVIDER_DIAGNOSTIC_STRING_MAX_LENGTH).optional(),
49
+ })
50
+ .nullish(),
51
+ });
52
+ const ResponseDiagnosticSchema = z.looseObject({
53
+ id: z.string().min(1).max(PROVIDER_DIAGNOSTIC_STRING_MAX_LENGTH).optional(),
54
+ status: z.string().min(1).max(PROVIDER_DIAGNOSTIC_STRING_MAX_LENGTH).optional(),
55
+ status_details: ResponseStatusDetailsSchema.nullish(),
56
+ output: z.array(ResponseOutputDiagnosticSchema).max(PROVIDER_DIAGNOSTIC_MAX_OUTPUTS).optional(),
57
+ });
58
+ /** Core response.done fields stay independent from optional diagnostic metadata. */
37
59
  const ResponseDoneSchema = z.looseObject({
38
60
  response: z.looseObject({
39
61
  output: z.array(z.looseObject({ type: z.string().optional() })).optional(),
@@ -48,6 +70,13 @@ const ResponseDoneSchema = z.looseObject({
48
70
  .optional(),
49
71
  }),
50
72
  });
73
+ const ResponseEnvelopeSchema = z.looseObject({
74
+ response: z.looseObject({}),
75
+ });
76
+ /** Response identity is core bookkeeping, separate from optional diagnostic metadata. */
77
+ const ResponseIdEnvelopeSchema = z.looseObject({
78
+ response: z.looseObject({ id: z.string().min(1).optional() }),
79
+ });
51
80
  const TranscriptEventSchema = z.looseObject({
52
81
  transcript: z.string(),
53
82
  item_id: z.string().min(1).optional(),
@@ -66,6 +95,7 @@ const ServerErrorSchema = z.looseObject({
66
95
  const SessionUpdatedSchema = z.looseObject({
67
96
  session: z
68
97
  .looseObject({
98
+ model: z.string().min(1).optional(),
69
99
  audio: z
70
100
  .looseObject({
71
101
  output: z
@@ -78,21 +108,34 @@ const SessionUpdatedSchema = z.looseObject({
78
108
  })
79
109
  .optional(),
80
110
  });
111
+ const SpeechBoundarySchema = z.looseObject({
112
+ item_id: z.string().min(1).max(PROVIDER_DIAGNOSTIC_STRING_MAX_LENGTH).optional(),
113
+ audio_start_ms: z.number().nonnegative().optional(),
114
+ audio_end_ms: z.number().nonnegative().optional(),
115
+ });
81
116
  export class RealtimeVoiceSession {
82
117
  #socket;
83
118
  #wire;
84
119
  #queue = new AsyncQueue();
85
120
  #outputStreams = new Map();
86
121
  #playedThrough = new Map();
122
+ #interruptedStreams = new Set();
87
123
  #deliveredToolCalls = new Set();
124
+ #completedResponseIds = new Set();
125
+ #usageResponseIds = new Set();
88
126
  #pendingUserTranscripts = new Map();
89
127
  #emittedUserTranscripts = new Map();
90
128
  #settled;
91
129
  #resolveSettled;
92
130
  #outputSampleRate;
93
- #activeOutputItemId;
131
+ #responseNumber = 0;
94
132
  #responseActive = false;
133
+ #activeResponseId;
95
134
  #responseWanted = false;
135
+ #inputSpeechActive = false;
136
+ #awaitingVadResponse = false;
137
+ #vadResponseRecoveryTimer;
138
+ #reportedResponseState;
96
139
  #wantedResponseInstructions;
97
140
  #ready = false;
98
141
  #closed = false;
@@ -197,14 +240,22 @@ export class RealtimeVoiceSession {
197
240
  this.#handleSessionUpdated(parsed);
198
241
  break;
199
242
  case "response.created":
243
+ this.#handleResponseCreated(parsed);
200
244
  this.#responseActive = true;
245
+ this.#activeResponseId = responseIdFromEnvelope(parsed);
246
+ if (!this.#inputSpeechActive)
247
+ this.#clearAwaitingVadResponse();
248
+ this.#reportResponseState();
201
249
  break;
202
250
  case "response.output_audio.delta":
203
251
  this.#handleAudioDelta(parsed);
204
252
  break;
205
253
  case "response.output_audio_transcript.delta":
206
254
  case "response.output_text.delta":
207
- this.#handleTranscriptDelta(parsed);
255
+ this.#handleTranscriptDelta(parsed, "assistant");
256
+ break;
257
+ case "conversation.item.input_audio_transcription.delta":
258
+ this.#handleTranscriptDelta(parsed, "user");
208
259
  break;
209
260
  case "response.output_audio_transcript.done":
210
261
  this.#handleTranscript(parsed, "assistant");
@@ -222,7 +273,20 @@ export class RealtimeVoiceSession {
222
273
  this.#handleResponseDone(parsed);
223
274
  break;
224
275
  case "input_audio_buffer.speech_started":
225
- this.#handleSpeechStarted();
276
+ this.#handleSpeechStarted(parsed);
277
+ break;
278
+ case "input_audio_buffer.speech_stopped":
279
+ this.#emitSpeechDiagnostic(parsed, "stopped");
280
+ this.#inputSpeechActive = false;
281
+ if (this.#awaitingVadResponse && this.#responseActive && this.#wire.interruptOnSpeech === false) {
282
+ // Non-interrupting VAD can skip automatic creation while an older response
283
+ // is active. Request its continuation once that response finishes.
284
+ this.#clearAwaitingVadResponse();
285
+ this.#responseWanted = true;
286
+ }
287
+ else if (this.#awaitingVadResponse) {
288
+ this.#armVadResponseRecovery();
289
+ }
226
290
  break;
227
291
  case "error":
228
292
  this.#handleServerError(parsed);
@@ -236,11 +300,19 @@ export class RealtimeVoiceSession {
236
300
  const rate = updated.success ? updated.data.session?.audio?.output?.format?.rate : undefined;
237
301
  if (rate !== undefined)
238
302
  this.#outputSampleRate = rate;
303
+ const model = updated.success ? updated.data.session?.model : undefined;
304
+ if (model !== undefined && model.length <= PROVIDER_DIAGNOSTIC_STRING_MAX_LENGTH) {
305
+ this.#emit({ type: "diagnostic", diagnostic: { kind: "session", model } });
306
+ }
239
307
  if (!this.#ready) {
240
308
  this.#ready = true;
241
309
  this.#emit({ type: "ready" });
310
+ this.#reportResponseState();
242
311
  }
243
312
  }
313
+ #handleResponseCreated(parsed) {
314
+ this.#emitResponseEnvelopeDiagnostic(parsed, "started");
315
+ }
244
316
  #handleAudioDelta(parsed) {
245
317
  const delta = OutputAudioDeltaSchema.safeParse(parsed);
246
318
  if (!delta.success) {
@@ -252,13 +324,29 @@ export class RealtimeVoiceSession {
252
324
  this.#emitError("Received an output audio delta without an audio payload", true);
253
325
  return;
254
326
  }
255
- const bytes = Buffer.from(payload, "base64");
327
+ let bytes;
328
+ try {
329
+ bytes = Uint8Array.from(atob(payload), (character) => character.charCodeAt(0));
330
+ }
331
+ catch {
332
+ this.#emitError("Received an output audio delta with invalid base64", true);
333
+ return;
334
+ }
256
335
  if (bytes.byteLength === 0 || bytes.byteLength % 2 !== 0) {
257
336
  this.#emitError(`Received a PCM16 delta with invalid byte length ${bytes.byteLength}`, true);
258
337
  return;
259
338
  }
260
- const streamId = delta.data.item_id ?? delta.data.response_id ?? FALLBACK_STREAM_ID;
261
- const stream = this.#outputStreams.get(streamId) ?? { sequence: 0, startSample: 0 };
339
+ const streamId = delta.data.item_id ??
340
+ delta.data.response_id ??
341
+ (this.#responseNumber === 0 ? FALLBACK_STREAM_ID : `${FALLBACK_STREAM_ID}-${this.#responseNumber}`);
342
+ if (this.#interruptedStreams.has(streamId))
343
+ return;
344
+ const stream = this.#outputStreams.get(streamId) ?? {
345
+ sequence: 0,
346
+ startSample: 0,
347
+ complete: false,
348
+ itemId: delta.data.item_id,
349
+ };
262
350
  const chunk = AudioChunkSchema.parse({
263
351
  streamId,
264
352
  sequence: stream.sequence,
@@ -266,12 +354,11 @@ export class RealtimeVoiceSession {
266
354
  sampleRate: this.#outputSampleRate,
267
355
  channels: 1,
268
356
  startSample: stream.startSample,
269
- data: new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength),
357
+ data: bytes,
270
358
  });
271
359
  stream.sequence += 1;
272
360
  stream.startSample += audioSampleCount(chunk);
273
361
  this.#outputStreams.set(streamId, stream);
274
- this.#activeOutputItemId = streamId;
275
362
  this.#emit({ type: "audio.output", chunk });
276
363
  }
277
364
  #handleTranscript(parsed, role) {
@@ -292,8 +379,10 @@ export class RealtimeVoiceSession {
292
379
  const pending = this.#pendingUserTranscripts.get(streamId);
293
380
  if (pending)
294
381
  clearTimeout(pending.timer);
295
- const timer = setTimeout(() => this.#flushUserTranscript(streamId), INPUT_TRANSCRIPT_SETTLE_MS);
382
+ const timer = setTimeout(() => this.#flushUserTranscript(streamId), this.#wire.inputTranscriptSettleMs ?? 0);
296
383
  this.#pendingUserTranscripts.set(streamId, { text, timer });
384
+ if (!this.#wire.inputTranscriptSettleMs)
385
+ this.#flushUserTranscript(streamId);
297
386
  }
298
387
  #flushUserTranscript(streamId) {
299
388
  const pending = this.#pendingUserTranscripts.get(streamId);
@@ -316,13 +405,13 @@ export class RealtimeVoiceSession {
316
405
  for (const streamId of [...this.#pendingUserTranscripts.keys()])
317
406
  this.#flushUserTranscript(streamId);
318
407
  }
319
- #handleTranscriptDelta(parsed) {
408
+ #handleTranscriptDelta(parsed, role) {
320
409
  const delta = TranscriptDeltaWireSchema.safeParse(parsed);
321
410
  if (!delta.success || delta.data.delta === "")
322
411
  return;
323
412
  this.#emit({
324
413
  type: "transcript.delta",
325
- role: "assistant",
414
+ role,
326
415
  delta: delta.data.delta,
327
416
  ...(delta.data.item_id === undefined ? {} : { streamId: delta.data.item_id }),
328
417
  });
@@ -346,13 +435,19 @@ export class RealtimeVoiceSession {
346
435
  this.#emitFunctionCall(done.data.call_id, done.data.name, done.data.arguments);
347
436
  }
348
437
  #handleResponseDone(parsed) {
349
- this.#responseActive = false;
350
- this.#activeOutputItemId = undefined;
438
+ this.#emitResponseEnvelopeDiagnostic(parsed, "completed");
439
+ const responseId = responseIdFromEnvelope(parsed);
440
+ const duplicate = responseId !== undefined && this.#completedResponseIds.has(responseId);
441
+ if (responseId !== undefined)
442
+ this.#completedResponseIds.add(responseId);
351
443
  const done = ResponseDoneSchema.safeParse(parsed);
352
444
  if (done.success) {
353
445
  const usage = done.data.response.usage;
354
- if (usage)
446
+ if (usage && (responseId === undefined || !this.#usageResponseIds.has(responseId))) {
355
447
  this.#emitUsage(usage);
448
+ if (responseId !== undefined)
449
+ this.#usageResponseIds.add(responseId);
450
+ }
356
451
  for (const item of done.data.response.output ?? []) {
357
452
  if (item.type !== "function_call")
358
453
  continue;
@@ -364,11 +459,26 @@ export class RealtimeVoiceSession {
364
459
  this.#emitFunctionCall(call.data.call_id, call.data.name, call.data.arguments);
365
460
  }
366
461
  }
462
+ // A duplicate or stale terminal update can refine diagnostics and carry omitted
463
+ // usage, but must never settle the response currently generating.
464
+ const settlesActiveResponse = !duplicate &&
465
+ (responseId === undefined ||
466
+ this.#activeResponseId === undefined ||
467
+ responseId === this.#activeResponseId);
468
+ if (!settlesActiveResponse)
469
+ return;
470
+ this.#responseActive = false;
471
+ this.#activeResponseId = undefined;
472
+ // Generation often ends well before the client finishes playing its queued PCM.
473
+ for (const stream of this.#outputStreams.values())
474
+ stream.complete = true;
475
+ this.#responseNumber += 1;
367
476
  if (this.#responseWanted) {
368
477
  const instructions = this.#wantedResponseInstructions;
369
478
  this.#wantedResponseInstructions = undefined;
370
479
  this.#requestResponse(instructions);
371
480
  }
481
+ this.#reportResponseState();
372
482
  }
373
483
  #emitUsage(usage) {
374
484
  const input = usage.input_token_details;
@@ -407,40 +517,151 @@ export class RealtimeVoiceSession {
407
517
  call: ToolCallSchema.parse({ callId, name, arguments: parsedArguments }),
408
518
  });
409
519
  }
410
- #handleSpeechStarted() {
411
- const itemId = this.#activeOutputItemId;
412
- if (itemId === undefined)
520
+ #handleSpeechStarted(parsed) {
521
+ this.#emitSpeechDiagnostic(parsed, "started");
522
+ this.#clearVadResponseRecoveryTimer();
523
+ this.#inputSpeechActive = true;
524
+ this.#awaitingVadResponse = this.#wire.autoRespondToAudio;
525
+ this.#reportResponseState();
526
+ if (this.#wire.interruptOnSpeech === false)
413
527
  return;
414
- this.#activeOutputItemId = undefined;
415
- const playedThroughSample = this.#playedThrough.get(itemId);
416
- if (playedThroughSample === undefined)
528
+ // A response can contain a preamble and an answer, both ahead of playback.
529
+ // Discard queued items first so stopping the audible item cannot start the next.
530
+ for (const [streamId, stream] of [...this.#outputStreams].reverse()) {
531
+ if (this.#interruptedStreams.has(streamId))
532
+ continue;
533
+ const playedThroughSample = this.#playedThrough.get(streamId) ?? 0;
534
+ if (stream.complete && playedThroughSample >= stream.startSample)
535
+ continue;
536
+ this.#interruptedStreams.add(streamId);
537
+ this.#emit({ type: "audio.interrupted", streamId });
538
+ // Only a real conversation item ID is valid in the vendor's truncate command.
539
+ if (stream.itemId === undefined)
540
+ continue;
541
+ this.#trySend({
542
+ type: "conversation.item.truncate",
543
+ item_id: stream.itemId,
544
+ content_index: 0,
545
+ audio_end_ms: Math.floor((Math.min(playedThroughSample, stream.startSample) / this.#outputSampleRate) * 1000),
546
+ });
547
+ }
548
+ }
549
+ #emitSpeechDiagnostic(parsed, phase) {
550
+ const boundary = SpeechBoundarySchema.safeParse(parsed);
551
+ if (!boundary.success)
417
552
  return;
418
- this.#trySend({
419
- type: "conversation.item.truncate",
420
- item_id: itemId,
421
- content_index: 0,
422
- audio_end_ms: Math.floor((playedThroughSample / this.#outputSampleRate) * 1000),
553
+ const audioOffsetMs = phase === "started" ? boundary.data.audio_start_ms : boundary.data.audio_end_ms;
554
+ this.#emit({
555
+ type: "diagnostic",
556
+ diagnostic: {
557
+ kind: "speech",
558
+ phase,
559
+ ...(boundary.data.item_id === undefined ? {} : { streamId: boundary.data.item_id }),
560
+ ...(audioOffsetMs === undefined ? {} : { audioOffsetMs }),
561
+ },
423
562
  });
424
563
  }
564
+ #emitResponseDiagnostic(parsed, phase) {
565
+ const response = ResponseDiagnosticSchema.safeParse(parsed);
566
+ if (!response.success)
567
+ return;
568
+ const details = response.data.status_details;
569
+ const outputs = response.data.output
570
+ ?.filter((item) => item.type !== undefined)
571
+ .map((item) => ({
572
+ ...(item.id === undefined ? {} : { streamId: item.id }),
573
+ type: item.type,
574
+ contentTypes: item.content?.flatMap((content) => (content.type === undefined ? [] : [content.type])) ?? [],
575
+ }));
576
+ this.#emit({
577
+ type: "diagnostic",
578
+ diagnostic: {
579
+ kind: "response",
580
+ phase,
581
+ ...(response.data.id === undefined ? {} : { responseId: response.data.id }),
582
+ ...(response.data.status === undefined ? {} : { status: response.data.status }),
583
+ ...(details?.reason == null ? {} : { reason: details.reason }),
584
+ ...(details?.error?.code === undefined ? {} : { code: details.error.code }),
585
+ ...(outputs === undefined ? {} : { outputs }),
586
+ },
587
+ });
588
+ }
589
+ #emitResponseEnvelopeDiagnostic(parsed, phase) {
590
+ const envelope = ResponseEnvelopeSchema.safeParse(parsed);
591
+ if (envelope.success)
592
+ this.#emitResponseDiagnostic(envelope.data.response, phase);
593
+ }
425
594
  #handleServerError(parsed) {
426
595
  const error = ServerErrorSchema.safeParse(parsed);
427
596
  const message = error.success ? error.data.error?.message : undefined;
428
597
  this.#emitError(message ?? `${this.#wire.label} reported an error`, true);
598
+ if (this.#awaitingVadResponse && !this.#inputSpeechActive) {
599
+ this.#clearAwaitingVadResponse();
600
+ this.#resumeQueuedResponse();
601
+ }
429
602
  }
430
603
  #requestResponse(instructions) {
604
+ if (this.#awaitingVadResponse) {
605
+ // Tool/task context is already in the conversation. Let VAD answer after the
606
+ // user finishes, including the gap between speech_stopped and response.created.
607
+ if (instructions !== undefined)
608
+ this.#wantedResponseInstructions = instructions;
609
+ this.#responseWanted = this.#wantedResponseInstructions !== undefined;
610
+ this.#reportResponseState();
611
+ return;
612
+ }
431
613
  if (this.#responseActive) {
432
614
  this.#responseWanted = true;
433
615
  if (instructions !== undefined)
434
616
  this.#wantedResponseInstructions = instructions;
617
+ this.#reportResponseState();
435
618
  return;
436
619
  }
437
620
  this.#responseWanted = false;
438
621
  this.#responseActive = true;
622
+ this.#reportResponseState();
439
623
  this.#trySend({
440
624
  type: "response.create",
441
625
  ...(instructions === undefined ? {} : { response: { instructions } }),
442
626
  });
443
627
  }
628
+ #armVadResponseRecovery() {
629
+ this.#clearVadResponseRecoveryTimer();
630
+ this.#vadResponseRecoveryTimer = setTimeout(() => {
631
+ this.#vadResponseRecoveryTimer = undefined;
632
+ if (this.#closed || !this.#awaitingVadResponse)
633
+ return;
634
+ this.#clearAwaitingVadResponse();
635
+ this.#emitError("Provider did not create the expected VAD response", true);
636
+ this.#resumeQueuedResponse();
637
+ }, VAD_RESPONSE_RECOVERY_MS);
638
+ }
639
+ #clearAwaitingVadResponse() {
640
+ this.#awaitingVadResponse = false;
641
+ this.#clearVadResponseRecoveryTimer();
642
+ }
643
+ #clearVadResponseRecoveryTimer() {
644
+ if (this.#vadResponseRecoveryTimer !== undefined)
645
+ clearTimeout(this.#vadResponseRecoveryTimer);
646
+ this.#vadResponseRecoveryTimer = undefined;
647
+ }
648
+ #resumeQueuedResponse() {
649
+ if (!this.#responseWanted) {
650
+ this.#reportResponseState();
651
+ return;
652
+ }
653
+ const instructions = this.#wantedResponseInstructions;
654
+ this.#responseWanted = false;
655
+ this.#wantedResponseInstructions = undefined;
656
+ this.#requestResponse(instructions);
657
+ }
658
+ #reportResponseState() {
659
+ const state = this.#responseActive || this.#responseWanted || this.#awaitingVadResponse ? "pending" : "idle";
660
+ if (state === this.#reportedResponseState)
661
+ return;
662
+ this.#reportedResponseState = state;
663
+ this.#emit({ type: "response.state", state });
664
+ }
444
665
  #send(payload) {
445
666
  if (this.#socket.state !== "open") {
446
667
  throw new Error(`${this.#wire.label} session socket is not open`);
@@ -464,6 +685,7 @@ export class RealtimeVoiceSession {
464
685
  #settle(reason) {
465
686
  if (this.#closed)
466
687
  return;
688
+ this.#clearVadResponseRecoveryTimer();
467
689
  this.#flushUserTranscripts();
468
690
  this.#closed = true;
469
691
  this.#queue.push({ type: "closed", reason });
@@ -506,10 +728,23 @@ function contextText(event) {
506
728
  }
507
729
  }
508
730
  function toBase64(data) {
509
- return Buffer.from(data.buffer, data.byteOffset, data.byteLength).toString("base64");
731
+ let binary = "";
732
+ // Bound each argument list; microphone buffers need not fit the engine's call stack.
733
+ for (let offset = 0; offset < data.length; offset += 0x8000) {
734
+ binary += String.fromCharCode(...data.subarray(offset, offset + 0x8000));
735
+ }
736
+ return btoa(binary);
510
737
  }
511
738
  function truncateCloseReason(reason) {
512
- return Buffer.byteLength(reason, "utf8") <= CLOSE_REASON_BYTE_LIMIT
513
- ? reason
514
- : Buffer.from(reason, "utf8").subarray(0, CLOSE_REASON_BYTE_LIMIT).toString("utf8");
739
+ const bytes = new TextEncoder().encode(reason);
740
+ if (bytes.byteLength <= CLOSE_REASON_BYTE_LIMIT)
741
+ return reason;
742
+ // Streaming decode leaves a partial final code point buffered instead of replacing it.
743
+ return new TextDecoder("utf-8", { ignoreBOM: true }).decode(bytes.subarray(0, CLOSE_REASON_BYTE_LIMIT), {
744
+ stream: true,
745
+ });
746
+ }
747
+ function responseIdFromEnvelope(parsed) {
748
+ const envelope = ResponseIdEnvelopeSchema.safeParse(parsed);
749
+ return envelope.success ? envelope.data.response.id : undefined;
515
750
  }
@@ -12,8 +12,12 @@ export interface ReplayVoiceProviderOptions {
12
12
  * Replays the provider-visible half of a recorded session: model audio, tool requests,
13
13
  * and provider errors, in recorded order and (optionally) recorded pacing. Input audio,
14
14
  * playout acknowledgements, tool results, and context are accepted and captured but do
15
- * not influence playback. This is the shadow-test primitive: re-drive UIs, tools, and
16
- * stores from an authoritative log without a live model or an API key.
15
+ * not influence playback. This re-drives UIs and stores from a recorded log without a
16
+ * live model or an API key.
17
+ *
18
+ * Playback closes after the last recorded output, even if a replayed tool request is
19
+ * still awaiting admission or execution. Recorded tool outcomes are not replayed.
20
+ * Use ScriptedVoiceProvider when subsequent output must wait for and verify results.
17
21
  */
18
22
  export declare class ReplayVoiceProvider implements VoiceProvider {
19
23
  #private;
@@ -3,8 +3,12 @@ import { AssistantTurnRequestSchema, AsyncQueue, AudioChunkSchema, PlayoutProgre
3
3
  * Replays the provider-visible half of a recorded session: model audio, tool requests,
4
4
  * and provider errors, in recorded order and (optionally) recorded pacing. Input audio,
5
5
  * playout acknowledgements, tool results, and context are accepted and captured but do
6
- * not influence playback. This is the shadow-test primitive: re-drive UIs, tools, and
7
- * stores from an authoritative log without a live model or an API key.
6
+ * not influence playback. This re-drives UIs and stores from a recorded log without a
7
+ * live model or an API key.
8
+ *
9
+ * Playback closes after the last recorded output, even if a replayed tool request is
10
+ * still awaiting admission or execution. Recorded tool outcomes are not replayed.
11
+ * Use ScriptedVoiceProvider when subsequent output must wait for and verify results.
8
12
  */
9
13
  export class ReplayVoiceProvider {
10
14
  id = "replay";
@@ -114,14 +118,18 @@ export class ReplayVoiceProviderSession {
114
118
  }
115
119
  function isReplayable(event) {
116
120
  return (event.type === "audio.output" ||
121
+ event.type === "audio.interrupted" ||
117
122
  event.type === "transcript" ||
118
123
  event.type === "tool.requested" ||
119
- event.type === "provider.error");
124
+ event.type === "provider.error" ||
125
+ event.type === "provider.diagnostic");
120
126
  }
121
127
  function toProviderEvent(event) {
122
128
  switch (event.type) {
123
129
  case "audio.output":
124
130
  return { type: "audio.output", chunk: event.chunk };
131
+ case "audio.interrupted":
132
+ return { type: "audio.interrupted", streamId: event.streamId };
125
133
  case "transcript":
126
134
  return {
127
135
  type: "transcript",
@@ -133,5 +141,7 @@ function toProviderEvent(event) {
133
141
  return { type: "tool.call", call: event.call };
134
142
  case "provider.error":
135
143
  return { type: "error", message: event.message, recoverable: event.recoverable };
144
+ case "provider.diagnostic":
145
+ return { type: "diagnostic", diagnostic: event.diagnostic };
136
146
  }
137
147
  }
@@ -0,0 +1,2 @@
1
+ export { parseSkillMarkdown, type Skill, type SkillMetadata, skillFromMarkdown } from "./skill.js";
2
+ export { createSkillTools } from "./tools.js";
@@ -0,0 +1,2 @@
1
+ export { parseSkillMarkdown, skillFromMarkdown } from "./skill.js";
2
+ export { createSkillTools } from "./tools.js";
@@ -0,0 +1,7 @@
1
+ import { type Skill } from "./skill.js";
2
+ /**
3
+ * Discovers immediate skill directories in explicitly trusted roots. Missing roots
4
+ * are ignored; malformed skills and duplicates fail with a path for the host to fix.
5
+ * Nothing scans the user's home, executes scripts, or grants allowed-tools.
6
+ */
7
+ export declare function loadSkillsFromDirectories(directories: readonly string[]): Promise<readonly Skill[]>;
@@ -0,0 +1,99 @@
1
+ import { constants } from "node:fs";
2
+ import { open, readdir, realpath } from "node:fs/promises";
3
+ import path from "node:path";
4
+ import { parseSkillMarkdown, validateSkillPath } from "./skill.js";
5
+ // Skills are text guidance, not an unbounded file export API. Resource paging bounds
6
+ // model context; this limit also bounds host memory during discovery and reads.
7
+ const MAX_FILE_BYTES = 1024 * 1024;
8
+ /**
9
+ * Discovers immediate skill directories in explicitly trusted roots. Missing roots
10
+ * are ignored; malformed skills and duplicates fail with a path for the host to fix.
11
+ * Nothing scans the user's home, executes scripts, or grants allowed-tools.
12
+ */
13
+ export async function loadSkillsFromDirectories(directories) {
14
+ const skills = new Map();
15
+ for (const directory of directories) {
16
+ let entries;
17
+ try {
18
+ entries = await readdir(directory, { withFileTypes: true });
19
+ }
20
+ catch (error) {
21
+ if (isMissing(error))
22
+ continue;
23
+ throw error;
24
+ }
25
+ for (const entry of entries.sort((a, b) => a.name.localeCompare(b.name))) {
26
+ if ((!entry.isDirectory() && !entry.isSymbolicLink()) || entry.name.startsWith("."))
27
+ continue;
28
+ const location = path.resolve(directory, entry.name);
29
+ let root;
30
+ let markdown;
31
+ try {
32
+ root = await realpath(location);
33
+ markdown = await readSkillFile(root, "SKILL.md");
34
+ }
35
+ catch (error) {
36
+ if (isMissing(error))
37
+ continue;
38
+ throw new Error(`Cannot load skill at ${location}`, { cause: error });
39
+ }
40
+ let metadata;
41
+ try {
42
+ metadata = parseSkillMarkdown(markdown);
43
+ if (metadata.name !== entry.name)
44
+ throw new Error("Skill name must match its directory name");
45
+ if (skills.has(metadata.name))
46
+ throw new Error(`Duplicate skill name: ${metadata.name}`);
47
+ }
48
+ catch (error) {
49
+ throw new Error(`Invalid skill at ${location}: ${error instanceof Error ? error.message : String(error)}`, { cause: error });
50
+ }
51
+ skills.set(metadata.name, {
52
+ ...metadata,
53
+ // Freeze the activated instructions to the catalog that was discovered.
54
+ read: (resource, signal) => {
55
+ signal.throwIfAborted();
56
+ return resource === "SKILL.md" ? markdown : readSkillFile(root, resource, signal);
57
+ },
58
+ });
59
+ }
60
+ }
61
+ return [...skills.values()];
62
+ }
63
+ async function readSkillFile(root, resource, signal) {
64
+ validateSkillPath(resource);
65
+ signal?.throwIfAborted();
66
+ const target = await realpath(path.join(root, resource));
67
+ const relative = path.relative(root, target);
68
+ if (relative === ".." || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) {
69
+ throw new Error("Skill resource resolves outside its skill directory");
70
+ }
71
+ // Nonblocking open lets fstat reject FIFOs instead of waiting forever for a writer.
72
+ const file = await open(target, constants.O_RDONLY | constants.O_NONBLOCK);
73
+ try {
74
+ const stat = await file.stat();
75
+ if (!stat.isFile())
76
+ throw new Error("Skill resource must be a regular text file");
77
+ const bytes = new Uint8Array(MAX_FILE_BYTES + 1);
78
+ let size = 0;
79
+ while (size < bytes.length) {
80
+ signal?.throwIfAborted();
81
+ const { bytesRead } = await file.read(bytes, size, bytes.length - size, null);
82
+ if (bytesRead === 0)
83
+ break;
84
+ size += bytesRead;
85
+ }
86
+ if (size > MAX_FILE_BYTES)
87
+ throw new Error("Skill resource exceeds the 1 MiB text limit");
88
+ const text = new TextDecoder("utf-8", { fatal: true }).decode(bytes.subarray(0, size));
89
+ if (text.includes("\u0000"))
90
+ throw new Error("Skill resource must be UTF-8 text without NUL bytes");
91
+ return text;
92
+ }
93
+ finally {
94
+ await file.close();
95
+ }
96
+ }
97
+ function isMissing(error) {
98
+ return error instanceof Error && "code" in error && (error.code === "ENOENT" || error.code === "ENOTDIR");
99
+ }