xo-harness 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (115) hide show
  1. package/README.md +92 -5
  2. package/dist/browser/worklets/capture-processor.js +34 -0
  3. package/dist/browser/worklets/playback-processor.js +188 -0
  4. package/dist/browser.d.ts +1 -0
  5. package/dist/browser.js +1 -0
  6. package/dist/internal/browser/browser-voice-client.d.ts +33 -0
  7. package/dist/internal/browser/browser-voice-client.js +316 -0
  8. package/dist/internal/browser/index.d.ts +2 -0
  9. package/dist/internal/browser/index.js +1 -0
  10. package/dist/internal/harness/conversation-context.d.ts +45 -0
  11. package/dist/internal/harness/conversation-context.js +79 -0
  12. package/dist/internal/harness/conversation-projection.d.ts +55 -0
  13. package/dist/internal/harness/conversation-projection.js +145 -0
  14. package/dist/internal/harness/event-stream.d.ts +23 -2
  15. package/dist/internal/harness/event-stream.js +148 -18
  16. package/dist/internal/harness/index.d.ts +7 -1
  17. package/dist/internal/harness/index.js +7 -1
  18. package/dist/internal/harness/message.d.ts +24 -11
  19. package/dist/internal/harness/message.js +271 -77
  20. package/dist/internal/harness/report-diff.d.ts +3 -0
  21. package/dist/internal/harness/report-diff.js +10 -0
  22. package/dist/internal/harness/report.d.ts +17 -14
  23. package/dist/internal/harness/report.js +41 -24
  24. package/dist/internal/harness/runtime-limits.d.ts +18 -0
  25. package/dist/internal/harness/runtime-limits.js +19 -0
  26. package/dist/internal/harness/session-persistence.d.ts +16 -0
  27. package/dist/internal/harness/session-persistence.js +106 -0
  28. package/dist/internal/harness/shadow.d.ts +5 -5
  29. package/dist/internal/harness/shadow.js +214 -52
  30. package/dist/internal/harness/socket-bridge.d.ts +45 -4
  31. package/dist/internal/harness/socket-bridge.js +215 -46
  32. package/dist/internal/harness/task-supervisor.d.ts +6 -2
  33. package/dist/internal/harness/task-supervisor.js +97 -15
  34. package/dist/internal/harness/tool-calls.d.ts +40 -0
  35. package/dist/internal/harness/tool-calls.js +132 -0
  36. package/dist/internal/harness/tool-policy.d.ts +2 -2
  37. package/dist/internal/harness/tool-runtime.d.ts +2 -2
  38. package/dist/internal/harness/tool-runtime.js +37 -40
  39. package/dist/internal/harness/tools.d.ts +21 -0
  40. package/dist/internal/harness/tools.js +7 -1
  41. package/dist/internal/harness/usage-tracker.d.ts +39 -0
  42. package/dist/internal/harness/usage-tracker.js +104 -0
  43. package/dist/internal/harness/voice-session.d.ts +39 -8
  44. package/dist/internal/harness/voice-session.js +353 -58
  45. package/dist/internal/harness/xo.d.ts +7 -12
  46. package/dist/internal/harness/xo.js +25 -13
  47. package/dist/internal/protocol/async-queue.d.ts +22 -1
  48. package/dist/internal/protocol/async-queue.js +86 -12
  49. package/dist/internal/protocol/audio.d.ts +16 -2
  50. package/dist/internal/protocol/audio.js +23 -5
  51. package/dist/internal/protocol/backend-output.d.ts +70 -0
  52. package/dist/internal/protocol/backend-output.js +39 -0
  53. package/dist/internal/protocol/event-json.d.ts +3 -0
  54. package/dist/internal/protocol/event-json.js +15 -0
  55. package/dist/internal/protocol/events.d.ts +342 -4
  56. package/dist/internal/protocol/events.js +53 -25
  57. package/dist/internal/protocol/index.d.ts +5 -0
  58. package/dist/internal/protocol/index.js +5 -0
  59. package/dist/internal/protocol/output-source.d.ts +18 -0
  60. package/dist/internal/protocol/output-source.js +15 -0
  61. package/dist/internal/protocol/paced-audio.d.ts +27 -0
  62. package/dist/internal/protocol/paced-audio.js +117 -0
  63. package/dist/internal/protocol/parts.d.ts +154 -0
  64. package/dist/internal/protocol/parts.js +32 -6
  65. package/dist/internal/protocol/provider.d.ts +308 -1
  66. package/dist/internal/protocol/provider.js +102 -3
  67. package/dist/internal/protocol/records.d.ts +773 -0
  68. package/dist/internal/protocol/records.js +46 -0
  69. package/dist/internal/protocol/tools.d.ts +40 -0
  70. package/dist/internal/protocol/tools.js +12 -0
  71. package/dist/internal/protocol/transcript.d.ts +19 -0
  72. package/dist/internal/protocol/transcript.js +53 -0
  73. package/dist/internal/provider/contract.d.ts +25 -2
  74. package/dist/internal/provider/event-queue.d.ts +17 -0
  75. package/dist/internal/provider/event-queue.js +78 -0
  76. package/dist/internal/provider/grok-voice.d.ts +12 -9
  77. package/dist/internal/provider/grok-voice.js +26 -15
  78. package/dist/internal/provider/index.d.ts +1 -0
  79. package/dist/internal/provider/index.js +1 -0
  80. package/dist/internal/provider/live-session.d.ts +29 -0
  81. package/dist/internal/provider/live-session.js +840 -0
  82. package/dist/internal/provider/node-socket.js +6 -0
  83. package/dist/internal/provider/openai-live.d.ts +62 -0
  84. package/dist/internal/provider/openai-live.js +160 -0
  85. package/dist/internal/provider/openai-realtime.d.ts +9 -5
  86. package/dist/internal/provider/openai-realtime.js +24 -10
  87. package/dist/internal/provider/realtime-session.d.ts +12 -1
  88. package/dist/internal/provider/realtime-session.js +373 -70
  89. package/dist/internal/provider/realtime-socket.d.ts +3 -1
  90. package/dist/internal/provider/tool-status-context.d.ts +10 -0
  91. package/dist/internal/provider/tool-status-context.js +30 -0
  92. package/dist/internal/provider/workers-socket.js +63 -15
  93. package/dist/internal/provider/workers.d.ts +1 -0
  94. package/dist/internal/provider/workers.js +1 -0
  95. package/dist/internal/provider-fake/replay-voice-provider.d.ts +9 -11
  96. package/dist/internal/provider-fake/replay-voice-provider.js +47 -23
  97. package/dist/internal/storage/event-store.d.ts +22 -4
  98. package/dist/internal/storage/jsonl-event-store.d.ts +4 -4
  99. package/dist/internal/storage/jsonl-event-store.js +21 -16
  100. package/dist/internal/storage/memory-event-store.d.ts +4 -3
  101. package/dist/internal/storage/memory-event-store.js +8 -2
  102. package/dist/internal/storage/memory.d.ts +1 -1
  103. package/dist/internal/testkit/events.d.ts +14 -0
  104. package/dist/internal/testkit/events.js +33 -0
  105. package/dist/internal/testkit/runtime.d.ts +3 -0
  106. package/dist/internal/testkit/runtime.js +3 -0
  107. package/dist/internal/testkit/trajectory.d.ts +23 -0
  108. package/dist/internal/testkit/trajectory.js +58 -0
  109. package/dist/internal/tools-openai/index.d.ts +2 -0
  110. package/dist/internal/tools-openai/index.js +79 -72
  111. package/dist/internal/tools-openai/responses.d.ts +5 -1
  112. package/dist/internal/tools-openai/responses.js +53 -5
  113. package/dist/testing.d.ts +1 -0
  114. package/dist/testing.js +1 -0
  115. package/package.json +7 -1
@@ -0,0 +1,18 @@
1
+ /** Opt-in per-session limits. Omitted fields preserve full replay and unlimited storage waits. */
2
+ export interface SessionRuntimeLimits {
3
+ /** Oldest events are evicted together; lagging readers receive EventReplayUnavailableError. */
4
+ maxReplayEvents?: number;
5
+ /** PCM bytes retained in the shared replay window, independent of the EventStore's retention. */
6
+ maxReplayAudioBytes?: number;
7
+ /** Maximum underlying appends in flight. Overflow fails the session instead of dropping records. */
8
+ maxPendingAppends?: number;
9
+ /** PCM bytes held by underlying appends, independent of chunk size. */
10
+ maxPendingAudioBytes?: number;
11
+ /** Deadline for each store operation. A timeout does not cancel an underlying external write. */
12
+ persistenceTimeoutMs?: number;
13
+ }
14
+ export declare class SessionPersistenceError extends Error {
15
+ readonly code: "persistence_failed" | "persistence_timeout" | "pending_appends_exceeded" | "pending_audio_bytes_exceeded";
16
+ constructor(code: "persistence_failed" | "persistence_timeout" | "pending_appends_exceeded" | "pending_audio_bytes_exceeded", message: string, options?: ErrorOptions);
17
+ }
18
+ export declare function validateRuntimeLimits(limits?: SessionRuntimeLimits): Readonly<SessionRuntimeLimits>;
@@ -0,0 +1,19 @@
1
+ export class SessionPersistenceError extends Error {
2
+ code;
3
+ constructor(code, message, options) {
4
+ super(message, options);
5
+ this.code = code;
6
+ this.name = "SessionPersistenceError";
7
+ }
8
+ }
9
+ export function validateRuntimeLimits(limits = {}) {
10
+ for (const [key, value] of Object.entries(limits)) {
11
+ if (value === undefined)
12
+ continue;
13
+ const minimum = key.endsWith("Bytes") || key === "maxReplayEvents" ? 0 : 1;
14
+ const maximum = key === "persistenceTimeoutMs" ? 2_147_483_647 : Number.MAX_SAFE_INTEGER;
15
+ if (!Number.isSafeInteger(value) || value < minimum || value > maximum)
16
+ throw new Error(`${key} must be an integer between ${minimum} and ${maximum}`);
17
+ }
18
+ return Object.freeze({ ...limits });
19
+ }
@@ -0,0 +1,16 @@
1
+ import type { HarnessEvent, HarnessRecord, UnsequencedHarnessEvent } from "../protocol/index.js";
2
+ import type { EventStore } from "../storage/index.js";
3
+ import { SessionPersistenceError, type SessionRuntimeLimits } from "./runtime-limits.js";
4
+ /** Session-local admission/deadlines around the host store; never bypasses durable publication. */
5
+ export declare class SessionPersistence implements EventStore {
6
+ #private;
7
+ onFailure: ((error: SessionPersistenceError) => void) | undefined;
8
+ constructor(store: EventStore, limits: Readonly<SessionRuntimeLimits>);
9
+ stats(): {
10
+ pendingAppends: number;
11
+ pendingAppendAudioBytes: number;
12
+ };
13
+ append(event: UnsequencedHarnessEvent): Promise<HarnessEvent>;
14
+ flush(sessionId: string): Promise<void>;
15
+ list(sessionId: string): Promise<readonly HarnessRecord[]>;
16
+ }
@@ -0,0 +1,106 @@
1
+ import { SessionPersistenceError } from "./runtime-limits.js";
2
+ /** Session-local admission/deadlines around the host store; never bypasses durable publication. */
3
+ export class SessionPersistence {
4
+ #store;
5
+ #limits;
6
+ #pendingAppends = 0;
7
+ #pendingAudioBytes = 0;
8
+ #failure;
9
+ #failureWaiters = new Set();
10
+ onFailure;
11
+ constructor(store, limits) {
12
+ this.#store = store;
13
+ this.#limits = limits;
14
+ }
15
+ stats() {
16
+ return { pendingAppends: this.#pendingAppends, pendingAppendAudioBytes: this.#pendingAudioBytes };
17
+ }
18
+ append(event) {
19
+ if (this.#failure)
20
+ return Promise.reject(this.#failure);
21
+ if (this.#pendingAppends >= (this.#limits.maxPendingAppends ?? Infinity))
22
+ return Promise.reject(this.#fail(new SessionPersistenceError("pending_appends_exceeded", `Pending event appends exceeded ${this.#limits.maxPendingAppends}; session recording is incomplete`)));
23
+ const bytes = event.type === "audio.input" || event.type === "audio.output" ? event.chunk.data.byteLength : 0;
24
+ if (this.#pendingAudioBytes + bytes > (this.#limits.maxPendingAudioBytes ?? Infinity))
25
+ return Promise.reject(this.#fail(new SessionPersistenceError("pending_audio_bytes_exceeded", `Pending audio appends exceeded ${this.#limits.maxPendingAudioBytes} bytes; session recording is incomplete`)));
26
+ this.#pendingAppends++;
27
+ this.#pendingAudioBytes += bytes;
28
+ // Invoke synchronously: the store reserves ordering before any provider forwarding.
29
+ let append;
30
+ try {
31
+ append = this.#store.append(event);
32
+ }
33
+ catch (error) {
34
+ append = Promise.reject(this.#operationFailed(error, "append"));
35
+ }
36
+ const settled = append.finally(() => {
37
+ // Counts underlying writes, including writes still outstanding after a deadline.
38
+ this.#pendingAppends--;
39
+ this.#pendingAudioBytes -= bytes;
40
+ });
41
+ return this.#observe(settled, "append");
42
+ }
43
+ flush(sessionId) {
44
+ return this.#call(() => this.#store.flush(sessionId), "flush");
45
+ }
46
+ list(sessionId) {
47
+ return this.#call(() => this.#store.list(sessionId), "list");
48
+ }
49
+ #call(operation, label) {
50
+ if (this.#failure)
51
+ return Promise.reject(this.#failure);
52
+ try {
53
+ return this.#observe(operation(), label);
54
+ }
55
+ catch (error) {
56
+ return Promise.reject(this.#operationFailed(error, label));
57
+ }
58
+ }
59
+ async #observe(operation, label) {
60
+ let timer;
61
+ let rejectFailure;
62
+ try {
63
+ const timeout = this.#limits.persistenceTimeoutMs;
64
+ const pending = [operation];
65
+ pending.push(new Promise((_resolve, reject) => {
66
+ if (this.#failure)
67
+ reject(this.#failure);
68
+ else {
69
+ rejectFailure = reject;
70
+ this.#failureWaiters.add(reject);
71
+ }
72
+ }));
73
+ if (timeout !== undefined)
74
+ pending.push(new Promise((_resolve, reject) => {
75
+ timer = setTimeout(() => reject(this.#fail(new SessionPersistenceError("persistence_timeout", `EventStore ${label} exceeded ${timeout} ms; persistence remains unconfirmed`))), timeout);
76
+ }));
77
+ const value = await Promise.race(pending);
78
+ if (this.#failure)
79
+ throw this.#failure;
80
+ return value;
81
+ }
82
+ catch (error) {
83
+ throw this.#operationFailed(error, label);
84
+ }
85
+ finally {
86
+ clearTimeout(timer);
87
+ if (rejectFailure)
88
+ this.#failureWaiters.delete(rejectFailure);
89
+ }
90
+ }
91
+ #operationFailed(error, label) {
92
+ return this.#fail(error instanceof SessionPersistenceError
93
+ ? error
94
+ : new SessionPersistenceError("persistence_failed", `EventStore ${label} failed: ${error instanceof Error ? error.message : String(error)}`, { cause: error }));
95
+ }
96
+ #fail(error) {
97
+ if (!this.#failure) {
98
+ this.#failure = error;
99
+ for (const reject of this.#failureWaiters)
100
+ reject(error);
101
+ this.#failureWaiters.clear();
102
+ this.onFailure?.(error);
103
+ }
104
+ return this.#failure;
105
+ }
106
+ }
@@ -1,4 +1,4 @@
1
- import type { HarnessEvent } from "../protocol/index.js";
1
+ import { type HarnessRecord } from "../protocol/index.js";
2
2
  import type { VoiceProvider } from "../provider/index.js";
3
3
  import { type EventStore } from "../storage/memory.js";
4
4
  import { type SessionReport } from "./report.js";
@@ -6,7 +6,7 @@ import type { ToolPolicy } from "./tool-policy.js";
6
6
  import type { VoiceTool } from "./tools.js";
7
7
  export interface ShadowRunOptions {
8
8
  /** The recorded session whose microphone audio is re-driven. */
9
- events: readonly HarnessEvent[];
9
+ events: readonly HarnessRecord[];
10
10
  /** The provider to shadow-test with the recorded input. */
11
11
  provider: VoiceProvider;
12
12
  /** Tools to execute during the shadow run; use substitutes when side effects are unwanted. */
@@ -20,7 +20,7 @@ export interface ShadowRunOptions {
20
20
  sessionId?: string;
21
21
  /** Defaults to the recorded session's instructions. */
22
22
  instructions?: string;
23
- /** The run settles once the provider has produced audio and then stayed quiet this long. */
23
+ /** Waits this long after meaningful output, with no pending backend, tool, or task work. */
24
24
  quietMs?: number;
25
25
  timeoutMs?: number;
26
26
  }
@@ -29,7 +29,7 @@ export interface ShadowRunOptions {
29
29
  * provider and returns the resulting session report for comparison with the original.
30
30
  * The shadow run executes supplied tools and applies only the policy provided here;
31
31
  * a recording's denials do not establish permissions for a new session.
32
- * Input is sent as fast as the provider accepts it; server-side VAD sees the recorded
33
- * speech and trailing silence exactly as the original session did.
32
+ * Continuous-input providers receive paced PCM and silence through settlement.
33
+ * Other providers retain the existing push-input behavior.
34
34
  */
35
35
  export declare function runShadowSession(options: ShadowRunOptions): Promise<SessionReport>;
@@ -1,22 +1,35 @@
1
+ import { PacedAudioInput, requireRecordedAudioHistory, } from "../protocol/index.js";
1
2
  import { MemoryEventStore } from "../storage/memory.js";
2
3
  import { summarizeSession } from "./report.js";
4
+ import { ToolCallTracker } from "./tool-calls.js";
3
5
  import { XO } from "./xo.js";
4
6
  /**
5
7
  * The silent-test primitive: re-drives a recorded session's input audio against another
6
8
  * provider and returns the resulting session report for comparison with the original.
7
9
  * The shadow run executes supplied tools and applies only the policy provided here;
8
10
  * a recording's denials do not establish permissions for a new session.
9
- * Input is sent as fast as the provider accepts it; server-side VAD sees the recorded
10
- * speech and trailing silence exactly as the original session did.
11
+ * Continuous-input providers receive paced PCM and silence through settlement.
12
+ * Other providers retain the existing push-input behavior.
11
13
  */
12
14
  export async function runShadowSession(options) {
13
15
  const recordedStart = options.events.find((event) => event.type === "session.started");
14
- const inputs = options.events.filter((event) => event.type === "audio.input");
15
- if (inputs.length === 0)
16
+ // Validate every required input before opening a provider or executing tools.
17
+ const inputs = requireRecordedAudioHistory(options.events.filter((event) => event.type === "audio.input")).filter((event) => event.type === "audio.input");
18
+ const firstInput = inputs[0];
19
+ if (!firstInput)
16
20
  throw new Error("Recorded session has no input audio to re-drive");
17
21
  const instructions = options.instructions ?? recordedStart?.instructions;
18
22
  const quietMs = options.quietMs ?? 3_000;
19
23
  const timeoutMs = options.timeoutMs ?? 60_000;
24
+ if (!Number.isFinite(quietMs) || quietMs < 0 || !Number.isFinite(timeoutMs) || timeoutMs < 1) {
25
+ throw new Error("Shadow quietMs must be nonnegative and timeoutMs must be positive and finite");
26
+ }
27
+ const continuousInput = options.provider.capabilities.inputCadence === "continuous";
28
+ const continuousOutput = options.provider.capabilities.outputLifecycle === "continuous";
29
+ if (continuousInput &&
30
+ inputs.some((input) => input.chunk.channels !== 1 || input.chunk.sampleRate !== firstInput.chunk.sampleRate)) {
31
+ throw new Error("Continuous shadow input requires one consistent mono PCM sample rate");
32
+ }
20
33
  const harness = new XO({
21
34
  store: options.store ?? new MemoryEventStore(),
22
35
  ...(options.tools === undefined ? {} : { tools: options.tools }),
@@ -29,18 +42,130 @@ export async function runShadowSession(options) {
29
42
  ...(options.sessionId === undefined ? {} : { sessionId: options.sessionId }),
30
43
  ...(instructions === undefined ? {} : { instructions }),
31
44
  });
45
+ const deadline = performance.now() + timeoutMs;
46
+ const inputController = new AbortController();
47
+ let pacedInput;
48
+ let closeReason = "shadow_complete";
49
+ const inputDeadline = setTimeout(() => {
50
+ closeReason = "shadow_timeout";
51
+ inputController.abort(new Error("Shadow input exceeded the run deadline"));
52
+ }, timeoutMs);
32
53
  try {
33
- await waitForReady(session, timeoutMs);
34
- for (const input of inputs) {
35
- await session.sendAudio(input.chunk);
54
+ await waitOrAbort(session.waitUntilReady(), inputController.signal);
55
+ if (continuousInput) {
56
+ const sampleRate = firstInput.chunk.sampleRate;
57
+ let sequence = 0;
58
+ let startSample = 0;
59
+ let frameSubmitted = Promise.withResolvers();
60
+ pacedInput = new PacedAudioInput({
61
+ sampleRate,
62
+ signal: inputController.signal,
63
+ send: async (data, syntheticSilence) => {
64
+ const chunk = {
65
+ streamId: "shadow-input",
66
+ sequence: sequence++,
67
+ encoding: "pcm_s16le",
68
+ sampleRate,
69
+ channels: 1,
70
+ startSample,
71
+ data: Uint8Array.from(data),
72
+ };
73
+ await session.sendAudio(chunk, { syntheticSilence });
74
+ startSample += data.byteLength / 2;
75
+ const submitted = frameSubmitted;
76
+ frameSubmitted = Promise.withResolvers();
77
+ submitted.resolve();
78
+ },
79
+ });
80
+ // A rejected input clock must not become an unhandled rejection while the
81
+ // recording is waiting for its next source timestamp.
82
+ void pacedInput.done.catch((error) => inputController.abort(error));
83
+ await driveContinuousInput(inputs, pacedInput, deadline, recordedStart?.capabilities.inputCadence !== "continuous", async (targetSample) => {
84
+ while (startSample < targetSample)
85
+ await waitOrAbort(frameSubmitted.promise, inputController.signal);
86
+ });
36
87
  }
37
- await settleAfterOutput(session, quietMs, timeoutMs);
88
+ else {
89
+ for (const input of inputs)
90
+ await session.sendAudio(input.chunk);
91
+ }
92
+ const settlement = settleAfterOutput(session, quietMs, deadline, continuousOutput);
93
+ const result = await (pacedInput
94
+ ? Promise.race([settlement, pacedInput.done.then(() => "closed")])
95
+ : settlement);
96
+ if (result === "timeout")
97
+ closeReason = "shadow_timeout";
38
98
  }
39
99
  finally {
40
- await session.close("shadow_complete");
100
+ clearTimeout(inputDeadline);
101
+ inputController.abort("shadow_complete");
102
+ await pacedInput?.done.catch(() => undefined);
103
+ await session.close(closeReason);
41
104
  }
42
105
  return summarizeSession(await session.history());
43
106
  }
107
+ async function driveContinuousInput(inputs, pacedInput, deadline, preserveCapturePauses, waitUntilSample) {
108
+ const first = inputs[0];
109
+ if (!first)
110
+ return;
111
+ const sourceStart = first.sessionTimeMs - (first.chunk.data.byteLength / 2 / first.chunk.sampleRate) * 1000;
112
+ const frameSamples = Math.round(first.chunk.sampleRate / 10);
113
+ const streams = new Map();
114
+ let frame = new Uint8Array(frameSamples * 2);
115
+ let filled = 0;
116
+ let position = 0;
117
+ let hasSourceAudio = false;
118
+ const flushFrame = async () => {
119
+ if (performance.now() >= deadline)
120
+ throw new Error("Shadow input exceeds the run deadline");
121
+ if (hasSourceAudio)
122
+ await pacedInput.write(frame);
123
+ else
124
+ await waitUntilSample(position);
125
+ frame = new Uint8Array(frameSamples * 2);
126
+ filled = 0;
127
+ hasSourceAudio = false;
128
+ };
129
+ const appendSamples = async (count, data) => {
130
+ let consumed = 0;
131
+ while (consumed < count) {
132
+ const take = Math.min(frameSamples - filled, count - consumed);
133
+ if (data) {
134
+ frame.set(data.subarray(consumed * 2, (consumed + take) * 2), filled * 2);
135
+ hasSourceAudio = true;
136
+ }
137
+ filled += take;
138
+ consumed += take;
139
+ position += take;
140
+ if (filled === frameSamples)
141
+ await flushFrame();
142
+ }
143
+ };
144
+ for (const input of inputs) {
145
+ const { chunk } = input;
146
+ const observedOffsetMs = input.sessionTimeMs - (chunk.data.byteLength / 2 / chunk.sampleRate) * 1000 - sourceStart;
147
+ let origin = streams.get(chunk.streamId);
148
+ if (!origin) {
149
+ origin = { sample: chunk.startSample, offsetMs: observedOffsetMs };
150
+ streams.set(chunk.streamId, origin);
151
+ }
152
+ let offsetMs = Math.max(0, origin.offsetMs + ((chunk.startSample - origin.sample) / chunk.sampleRate) * 1_000);
153
+ if (preserveCapturePauses && observedOffsetMs - offsetMs > 250) {
154
+ // On-demand capture can stop while its sample counter stays contiguous. Preserve
155
+ // that pause; continuous recordings keep their sample clock despite receipt jitter.
156
+ offsetMs = observedOffsetMs;
157
+ streams.set(chunk.streamId, { sample: chunk.startSample, offsetMs });
158
+ }
159
+ // Coalesce source packets before awaiting the clock: a 20 ms packet must not
160
+ // acquire 80 ms of padding merely because the transport uses 100 ms frames.
161
+ // Sample positions preserve gaps, including silence inside a transport frame.
162
+ const targetSample = Math.round((offsetMs * chunk.sampleRate) / 1_000);
163
+ await appendSamples(Math.max(0, targetSample - position));
164
+ await appendSamples(chunk.data.byteLength / 2, input.syntheticSilence ? undefined : chunk.data);
165
+ }
166
+ if (filled)
167
+ await flushFrame();
168
+ }
44
169
  function createEventCursor(session) {
45
170
  const iterator = session.events()[Symbol.asyncIterator]();
46
171
  // A raced-and-lost next() must be reused, not abandoned: an abandoned call would
@@ -49,7 +174,13 @@ function createEventCursor(session) {
49
174
  return {
50
175
  async next(timeoutMs) {
51
176
  pendingNext ??= iterator.next();
52
- const result = await Promise.race([pendingNext, delay(timeoutMs).then(() => "timeout")]);
177
+ let timer;
178
+ const result = await Promise.race([
179
+ pendingNext,
180
+ new Promise((resolve) => {
181
+ timer = setTimeout(() => resolve("timeout"), timeoutMs);
182
+ }),
183
+ ]).finally(() => clearTimeout(timer));
53
184
  if (result === "timeout")
54
185
  return "timeout";
55
186
  pendingNext = undefined;
@@ -62,54 +193,72 @@ function createEventCursor(session) {
62
193
  },
63
194
  };
64
195
  }
65
- async function waitForReady(session, timeoutMs) {
196
+ async function settleAfterOutput(session, quietMs, deadline, continuousOutput) {
66
197
  const cursor = createEventCursor(session);
67
- const deadline = Date.now() + timeoutMs;
198
+ let lastActivity = performance.now();
199
+ let sawOutputSinceWork = false;
200
+ let responseIdle = true;
201
+ const toolCalls = new ToolCallTracker();
202
+ const backends = new Set();
203
+ const hostedTools = new Set();
68
204
  try {
69
205
  while (true) {
70
- const remaining = deadline - Date.now();
71
- if (remaining <= 0)
72
- throw new Error("Shadow provider did not become ready in time");
73
- const event = await cursor.next(remaining);
74
- if (event === "timeout")
75
- throw new Error("Shadow provider did not become ready in time");
76
- if (event === "done")
77
- throw new Error("Shadow session ended before the provider was ready");
78
- if (event.type === "provider.ready")
79
- return;
80
- }
81
- }
82
- finally {
83
- cursor.dispose();
84
- }
85
- }
86
- async function settleAfterOutput(session, quietMs, timeoutMs) {
87
- const cursor = createEventCursor(session);
88
- const deadline = Date.now() + timeoutMs;
89
- let lastActivity = Date.now();
90
- let sawOutput = false;
91
- try {
92
- while (true) {
93
- const untilDeadline = deadline - Date.now();
206
+ const untilDeadline = deadline - performance.now();
94
207
  if (untilDeadline <= 0)
95
- return;
96
- const untilQuiet = quietMs - (Date.now() - lastActivity);
97
- if (sawOutput && untilQuiet <= 0)
98
- return;
99
- const wait = Math.max(10, sawOutput ? Math.min(untilQuiet, untilDeadline) : untilDeadline);
208
+ return "timeout";
209
+ const untilQuiet = quietMs - (performance.now() - lastActivity);
210
+ const canSettle = sawOutputSinceWork &&
211
+ responseIdle &&
212
+ !toolCalls.hasPending() &&
213
+ backends.size + hostedTools.size === 0;
214
+ const wait = Math.max(0, canSettle ? Math.min(untilQuiet, untilDeadline) : untilDeadline);
100
215
  const event = await cursor.next(wait);
101
- if (event === "timeout")
216
+ if (event === "timeout") {
217
+ if (canSettle && performance.now() - lastActivity >= quietMs)
218
+ return "quiet";
102
219
  continue;
220
+ }
103
221
  if (event === "done")
104
- return;
222
+ return "closed";
105
223
  const type = event.type;
106
- if (type === "audio.output" ||
107
- type === "transcript" ||
108
- type === "tool.requested" ||
109
- type === "tool.completed") {
110
- lastActivity = Date.now();
111
- if (type === "audio.output")
112
- sawOutput = true;
224
+ const previousToolCall = "callId" in event && event.callId !== undefined ? toolCalls.get(event.callId) : undefined;
225
+ const toolCall = toolCalls.apply(event);
226
+ // Only a new execution result requires another answer. Retransmissions and
227
+ // delivery diagnostics must not erase an answer already observed.
228
+ let workSettled = toolCall?.status === "result_ready" && toolCall.state !== previousToolCall?.state;
229
+ if (type === "backend.response") {
230
+ // Voice may announce a result before the backend emits its trailing terminal.
231
+ // That terminal releases the pending gate; it does not require another answer.
232
+ if (event.phase === "completed")
233
+ backends.delete(event.responseId);
234
+ else
235
+ backends.add(event.responseId);
236
+ }
237
+ if (type === "backend.tool") {
238
+ const key = JSON.stringify([event.responseId, event.toolId]);
239
+ if (event.phase === "started")
240
+ hostedTools.add(key);
241
+ else
242
+ workSettled = hostedTools.delete(key);
243
+ }
244
+ if (type === "delegation.created" && event.target === "responses" && event.responseId)
245
+ backends.add(event.responseId);
246
+ if (type === "response.state")
247
+ responseIdle = event.state === "idle";
248
+ if (workSettled)
249
+ sawOutputSinceWork = false;
250
+ const spokenAudio = type === "audio.output" && (!continuousOutput || hasAudibleSamples(event.chunk.data));
251
+ const assistantText = (type === "transcript" || type === "transcript.delta" || type === "transcript.fragment") &&
252
+ event.role === "assistant";
253
+ if (spokenAudio || assistantText)
254
+ sawOutputSinceWork = true;
255
+ if (spokenAudio ||
256
+ assistantText ||
257
+ type === "response.state" ||
258
+ type.startsWith("tool.") ||
259
+ type.startsWith("task.") ||
260
+ type.startsWith("backend.")) {
261
+ lastActivity = performance.now();
113
262
  }
114
263
  }
115
264
  }
@@ -117,6 +266,19 @@ async function settleAfterOutput(session, quietMs, timeoutMs) {
117
266
  cursor.dispose();
118
267
  }
119
268
  }
120
- function delay(ms) {
121
- return new Promise((resolve) => setTimeout(resolve, ms));
269
+ function hasAudibleSamples(data) {
270
+ const samples = new DataView(data.buffer, data.byteOffset, data.byteLength);
271
+ let squares = 0;
272
+ for (let offset = 0; offset < data.byteLength; offset += 2)
273
+ squares += (samples.getInt16(offset, true) / 32_768) ** 2;
274
+ return Math.sqrt(squares / Math.max(1, data.byteLength / 2)) >= 0.003;
275
+ }
276
+ function waitOrAbort(promise, signal) {
277
+ return new Promise((resolve, reject) => {
278
+ const abort = () => reject(signal.reason);
279
+ signal.addEventListener("abort", abort, { once: true });
280
+ if (signal.aborted)
281
+ abort();
282
+ void promise.then(resolve, reject).finally(() => signal.removeEventListener("abort", abort));
283
+ });
122
284
  }
@@ -4,7 +4,7 @@ import type { VoiceSession } from "./voice-session.js";
4
4
  /**
5
5
  * The client wire protocol every XO transport speaks:
6
6
  * - text frames are JSON control messages (start / playout / stop up; started / event /
7
- * error / closed down),
7
+ * interrupt / error / closed down),
8
8
  * - binary frames are PCM16 mono audio (microphone up; length-prefixed model audio down:
9
9
  * u32le header length, JSON header, PCM bytes).
10
10
  *
@@ -30,6 +30,15 @@ export declare const VoiceClientMessageSchema: z.ZodDiscriminatedUnion<[z.ZodObj
30
30
  type: z.ZodLiteral<"playout">;
31
31
  streamId: z.ZodString;
32
32
  playedThroughSample: z.ZodNumber;
33
+ }, z.core.$strip>, z.ZodObject<{
34
+ streamId: z.ZodString;
35
+ startSample: z.ZodInt;
36
+ endSample: z.ZodInt;
37
+ status: z.ZodEnum<{
38
+ discarded: "discarded";
39
+ played: "played";
40
+ }>;
41
+ type: z.ZodLiteral<"playout.range">;
33
42
  }, z.core.$strip>, z.ZodObject<{
34
43
  type: z.ZodLiteral<"stop">;
35
44
  reason: z.ZodOptional<z.ZodString>;
@@ -40,13 +49,41 @@ export type VoiceStartRequest = Omit<Extract<VoiceClientMessage, {
40
49
  type: "start";
41
50
  }>, "type">;
42
51
  export interface VoiceBridgeSocket {
52
+ /** Throw if the frame cannot be accepted locally; returning does not confirm client receipt. */
43
53
  sendText(text: string): void;
44
54
  sendBinary(frame: ArrayBuffer): void;
45
55
  }
56
+ export interface VoiceBridgeStartupContext {
57
+ /** Aborted on disconnect only until the factory returns; safe to pass to XO.startSession. */
58
+ startupSignal: AbortSignal;
59
+ }
60
+ export interface VoiceBridgeStartedContext {
61
+ /** Cancels opening work on close, independently of the provider's owner signal. */
62
+ signal: AbortSignal;
63
+ }
64
+ export interface VoiceBridgeCloseResult {
65
+ /** Whether pending creation/late cleanup settled before the bridge's deadline, not provider finalization. */
66
+ readonly startup: "settled" | "unconfirmed";
67
+ }
46
68
  export interface VoiceSocketBridgeOptions {
47
69
  socket: VoiceBridgeSocket;
48
70
  /** Owns provider resolution and XO construction; the bridge owns only the wire. */
49
- startSession(start: VoiceStartRequest): Promise<VoiceSession>;
71
+ startSession(start: VoiceStartRequest, context: VoiceBridgeStartupContext): Promise<VoiceSession>;
72
+ /**
73
+ * Runs after media is enabled and started is submitted locally; its promise never gates media.
74
+ * The started control frame precedes the provider.ready event frame. Earlier startup events may precede it.
75
+ */
76
+ onStarted?(session: VoiceSession, context: VoiceBridgeStartedContext): void | Promise<void>;
77
+ /**
78
+ * Synchronous visibility policy for JSON event frames. Default: send every event.
79
+ * Does not filter playback configuration/interruption controls, binary audio, stored history, or server observers.
80
+ * Inspect without mutating the event; throwing closes the bridge with a generic client error.
81
+ */
82
+ shouldSendEvent?(event: HarnessEvent): boolean;
83
+ /** Observes command/hook errors and socket send failures, including when wire reporting is impossible. */
84
+ onError?(error: Error): void;
85
+ /** Bounds waiting for pending creation on close. Integer 1–2147483647 ms; default: 5 seconds. */
86
+ startupCloseTimeoutMs?: number;
50
87
  inputSampleRate?: number;
51
88
  }
52
89
  export declare const VOICE_BRIDGE_INPUT_SAMPLE_RATE = 24000;
@@ -57,8 +94,12 @@ export declare class VoiceSocketBridge {
57
94
  handleText(text: string): Promise<void>;
58
95
  /** Feed every binary frame from the client socket here. */
59
96
  handleBinary(data: Uint8Array): Promise<void>;
60
- /** Call when the client socket closes; settles the session. */
61
- handleClose(reason?: string): Promise<void>;
97
+ /**
98
+ * Closes an attached session gracefully, or cancels pending creation and waits up to
99
+ * startupCloseTimeoutMs. Repeated calls share one result. Unconfirmed startup is still
100
+ * cleaned up if it returns later; this result never claims provider finalization.
101
+ */
102
+ handleClose(reason?: string): Promise<VoiceBridgeCloseResult>;
62
103
  }
63
104
  export declare function encodeVoiceAudioFrame(chunk: AudioChunk): ArrayBuffer;
64
105
  /** Event payload for clients: audio chunks shrink to metadata, bytes stay server-side. */