xo-harness 0.2.1 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +109 -5
- package/dist/browser/worklets/capture-processor.js +34 -0
- package/dist/browser/worklets/playback-processor.js +188 -0
- package/dist/browser.d.ts +1 -0
- package/dist/browser.js +1 -0
- package/dist/internal/browser/browser-voice-client.d.ts +33 -0
- package/dist/internal/browser/browser-voice-client.js +316 -0
- package/dist/internal/browser/index.d.ts +2 -0
- package/dist/internal/browser/index.js +1 -0
- package/dist/internal/harness/conversation-context.d.ts +45 -0
- package/dist/internal/harness/conversation-context.js +79 -0
- package/dist/internal/harness/conversation-projection.d.ts +55 -0
- package/dist/internal/harness/conversation-projection.js +145 -0
- package/dist/internal/harness/event-stream.d.ts +23 -2
- package/dist/internal/harness/event-stream.js +148 -18
- package/dist/internal/harness/index.d.ts +7 -1
- package/dist/internal/harness/index.js +7 -1
- package/dist/internal/harness/message.d.ts +24 -11
- package/dist/internal/harness/message.js +271 -77
- package/dist/internal/harness/report-diff.d.ts +3 -0
- package/dist/internal/harness/report-diff.js +10 -0
- package/dist/internal/harness/report.d.ts +17 -14
- package/dist/internal/harness/report.js +41 -24
- package/dist/internal/harness/runtime-limits.d.ts +18 -0
- package/dist/internal/harness/runtime-limits.js +19 -0
- package/dist/internal/harness/session-persistence.d.ts +16 -0
- package/dist/internal/harness/session-persistence.js +106 -0
- package/dist/internal/harness/shadow.d.ts +5 -5
- package/dist/internal/harness/shadow.js +214 -52
- package/dist/internal/harness/socket-bridge.d.ts +45 -4
- package/dist/internal/harness/socket-bridge.js +216 -49
- package/dist/internal/harness/task-supervisor.d.ts +6 -2
- package/dist/internal/harness/task-supervisor.js +97 -17
- package/dist/internal/harness/tool-calls.d.ts +40 -0
- package/dist/internal/harness/tool-calls.js +132 -0
- package/dist/internal/harness/tool-policy.d.ts +2 -2
- package/dist/internal/harness/tool-runtime.d.ts +2 -2
- package/dist/internal/harness/tool-runtime.js +38 -43
- package/dist/internal/harness/tools.d.ts +21 -0
- package/dist/internal/harness/tools.js +7 -1
- package/dist/internal/harness/usage-tracker.d.ts +39 -0
- package/dist/internal/harness/usage-tracker.js +104 -0
- package/dist/internal/harness/voice-session.d.ts +39 -8
- package/dist/internal/harness/voice-session.js +354 -61
- package/dist/internal/harness/xo.d.ts +7 -12
- package/dist/internal/harness/xo.js +25 -13
- package/dist/internal/protocol/async-queue.d.ts +22 -1
- package/dist/internal/protocol/async-queue.js +86 -12
- package/dist/internal/protocol/audio.d.ts +21 -2
- package/dist/internal/protocol/audio.js +56 -5
- package/dist/internal/protocol/backend-output.d.ts +70 -0
- package/dist/internal/protocol/backend-output.js +39 -0
- package/dist/internal/protocol/error-message.d.ts +6 -0
- package/dist/internal/protocol/error-message.js +8 -0
- package/dist/internal/protocol/event-json.d.ts +3 -0
- package/dist/internal/protocol/event-json.js +15 -0
- package/dist/internal/protocol/events.d.ts +342 -4
- package/dist/internal/protocol/events.js +53 -25
- package/dist/internal/protocol/index.d.ts +5 -0
- package/dist/internal/protocol/index.js +5 -0
- package/dist/internal/protocol/internal.d.ts +5 -0
- package/dist/internal/protocol/internal.js +5 -0
- package/dist/internal/protocol/output-source.d.ts +18 -0
- package/dist/internal/protocol/output-source.js +15 -0
- package/dist/internal/protocol/paced-audio.d.ts +27 -0
- package/dist/internal/protocol/paced-audio.js +117 -0
- package/dist/internal/protocol/parts.d.ts +154 -0
- package/dist/internal/protocol/parts.js +32 -6
- package/dist/internal/protocol/provider.d.ts +308 -1
- package/dist/internal/protocol/provider.js +102 -3
- package/dist/internal/protocol/records.d.ts +773 -0
- package/dist/internal/protocol/records.js +46 -0
- package/dist/internal/protocol/tools.d.ts +40 -0
- package/dist/internal/protocol/tools.js +12 -0
- package/dist/internal/protocol/transcript.d.ts +19 -0
- package/dist/internal/protocol/transcript.js +53 -0
- package/dist/internal/provider/contract.d.ts +25 -2
- package/dist/internal/provider/event-queue.d.ts +17 -0
- package/dist/internal/provider/event-queue.js +78 -0
- package/dist/internal/provider/grok-voice.d.ts +12 -9
- package/dist/internal/provider/grok-voice.js +26 -15
- package/dist/internal/provider/index.d.ts +1 -0
- package/dist/internal/provider/index.js +1 -0
- package/dist/internal/provider/live-session.d.ts +29 -0
- package/dist/internal/provider/live-session.js +838 -0
- package/dist/internal/provider/node-socket.js +7 -3
- package/dist/internal/provider/openai-live.d.ts +62 -0
- package/dist/internal/provider/openai-live.js +161 -0
- package/dist/internal/provider/openai-realtime.d.ts +9 -5
- package/dist/internal/provider/openai-realtime.js +24 -10
- package/dist/internal/provider/realtime-session.d.ts +12 -1
- package/dist/internal/provider/realtime-session.js +369 -68
- package/dist/internal/provider/realtime-socket.d.ts +3 -1
- package/dist/internal/provider/tool-status-context.d.ts +10 -0
- package/dist/internal/provider/tool-status-context.js +30 -0
- package/dist/internal/provider/workers-socket.js +63 -15
- package/dist/internal/provider/workers.d.ts +1 -0
- package/dist/internal/provider/workers.js +1 -0
- package/dist/internal/provider-fake/replay-voice-provider.d.ts +9 -11
- package/dist/internal/provider-fake/replay-voice-provider.js +47 -23
- package/dist/internal/storage/event-store.d.ts +22 -4
- package/dist/internal/storage/jsonl-event-store.d.ts +4 -4
- package/dist/internal/storage/jsonl-event-store.js +21 -16
- package/dist/internal/storage/memory-event-store.d.ts +4 -3
- package/dist/internal/storage/memory-event-store.js +8 -2
- package/dist/internal/storage/memory.d.ts +1 -1
- package/dist/internal/testkit/events.d.ts +14 -0
- package/dist/internal/testkit/events.js +33 -0
- package/dist/internal/testkit/runtime.d.ts +3 -0
- package/dist/internal/testkit/runtime.js +3 -0
- package/dist/internal/testkit/trajectory.d.ts +23 -0
- package/dist/internal/testkit/trajectory.js +58 -0
- package/dist/internal/tools-openai/index.d.ts +2 -0
- package/dist/internal/tools-openai/index.js +79 -72
- package/dist/internal/tools-openai/responses.d.ts +5 -1
- package/dist/internal/tools-openai/responses.js +53 -5
- package/dist/testing.d.ts +1 -0
- package/dist/testing.js +1 -0
- package/package.json +7 -1
|
@@ -1,6 +1,10 @@
|
|
|
1
|
-
import { AgentInputSchema, AssistantTurnRequestSchema, AudioChunkSchema, agentInputToText, MessageInputTextSchema, PlayoutProgressSchema, ProviderEventSchema, } from "../protocol/index.js";
|
|
1
|
+
import { AgentInputSchema, AssistantTurnRequestSchema, AudioChunkSchema, agentInputToText, ConversationContextUpdateSchema, MessageInputTextSchema, PlayoutProgressSchema, PlayoutRangeSchema, ProviderCapabilitiesSchema, ProviderEventSchema, } from "../protocol/index.js";
|
|
2
|
+
import { errorMessage } from "../protocol/internal.js";
|
|
2
3
|
import { ReplayEventStream } from "./event-stream.js";
|
|
4
|
+
import { validateRuntimeLimits, } from "./runtime-limits.js";
|
|
5
|
+
import { SessionPersistence } from "./session-persistence.js";
|
|
3
6
|
import { TaskSupervisor } from "./task-supervisor.js";
|
|
7
|
+
import { ToolCallTracker } from "./tool-calls.js";
|
|
4
8
|
import { DEFAULT_MAX_TOOL_RESULT_BYTES, validateMaxToolResultBytes } from "./tool-delivery.js";
|
|
5
9
|
import { allowAllToolPolicy } from "./tool-policy.js";
|
|
6
10
|
import { ToolRuntime } from "./tool-runtime.js";
|
|
@@ -9,32 +13,63 @@ export class VoiceSession {
|
|
|
9
13
|
id;
|
|
10
14
|
#providerSession;
|
|
11
15
|
#store;
|
|
12
|
-
#events
|
|
16
|
+
#events;
|
|
13
17
|
#controller = new AbortController();
|
|
18
|
+
#providerController;
|
|
19
|
+
#capabilities;
|
|
20
|
+
#providerId;
|
|
21
|
+
#closeTimeoutMs;
|
|
22
|
+
#ownAudioBuffers;
|
|
23
|
+
#recordingEnded = false;
|
|
24
|
+
#providerClosed = false;
|
|
25
|
+
#persistenceFailure;
|
|
26
|
+
#providerFailure;
|
|
14
27
|
#startedAt;
|
|
15
28
|
#toolExecutions = new Set();
|
|
16
29
|
#pendingRecords = new Set();
|
|
17
30
|
#tasks;
|
|
18
31
|
#toolRuntime;
|
|
32
|
+
#toolCalls = new ToolCallTracker();
|
|
33
|
+
#toolStatusDelivery = Promise.resolve();
|
|
19
34
|
#providerReadiness;
|
|
20
35
|
#settleProviderReadiness = () => undefined;
|
|
21
36
|
#providerReadinessSettled = false;
|
|
22
37
|
#detachOwnerAbort;
|
|
38
|
+
#detachProviderFailure;
|
|
23
39
|
#providerPump;
|
|
24
40
|
#closePromise;
|
|
25
41
|
#maxDurationTimer;
|
|
26
|
-
constructor(options, providerSession, startedAt, startedEvent) {
|
|
42
|
+
constructor(options, providerSession, startedAt, startedEvent, providerController, capabilities, persistence) {
|
|
27
43
|
this.id = options.sessionId;
|
|
28
44
|
this.#providerSession = providerSession;
|
|
29
|
-
this.#
|
|
45
|
+
this.#providerController = providerController;
|
|
46
|
+
this.#capabilities = capabilities;
|
|
47
|
+
this.#providerId = options.provider.id;
|
|
48
|
+
this.#closeTimeoutMs = options.closeTimeoutMs ?? 15_000;
|
|
49
|
+
this.#ownAudioBuffers =
|
|
50
|
+
options.runtimeLimits?.maxReplayAudioBytes !== undefined ||
|
|
51
|
+
options.runtimeLimits?.maxPendingAudioBytes !== undefined;
|
|
52
|
+
this.#store = persistence;
|
|
53
|
+
this.#events = new ReplayEventStream({
|
|
54
|
+
maxEvents: options.runtimeLimits?.maxReplayEvents,
|
|
55
|
+
maxAudioBytes: options.runtimeLimits?.maxReplayAudioBytes,
|
|
56
|
+
});
|
|
30
57
|
this.#startedAt = startedAt;
|
|
31
58
|
this.#events.publish(startedEvent);
|
|
59
|
+
persistence.onFailure = (error) => {
|
|
60
|
+
this.#persistenceFailure = error;
|
|
61
|
+
this.#recordingEnded = true;
|
|
62
|
+
this.#events.fail(error);
|
|
63
|
+
this.#providerController.abort(error.code);
|
|
64
|
+
this.#closeFromInternal(error.code);
|
|
65
|
+
};
|
|
32
66
|
this.#providerReadiness = new Promise((resolve) => {
|
|
33
67
|
this.#settleProviderReadiness = resolve;
|
|
34
68
|
});
|
|
35
69
|
this.#tasks = new TaskSupervisor({
|
|
36
70
|
record: (event) => this.#record(event),
|
|
37
|
-
|
|
71
|
+
// Shutdown still records task outcomes; it must not prompt another spoken response.
|
|
72
|
+
sendContext: (event) => this.#controller.signal.aborted ? Promise.resolve() : this.#providerSession.sendContext(event),
|
|
38
73
|
flush: () => this.#store.flush(this.id),
|
|
39
74
|
sessionSignal: this.#controller.signal,
|
|
40
75
|
onError: () => {
|
|
@@ -56,29 +91,44 @@ export class VoiceSession {
|
|
|
56
91
|
});
|
|
57
92
|
}
|
|
58
93
|
static async create(options) {
|
|
94
|
+
// Capture declared behavior before any application/provider startup awaits.
|
|
95
|
+
const capabilities = Object.freeze(ProviderCapabilitiesSchema.parse(options.provider.capabilities));
|
|
96
|
+
const runtimeLimits = validateRuntimeLimits(options.runtimeLimits);
|
|
97
|
+
const persistence = new SessionPersistence(options.store, runtimeLimits);
|
|
59
98
|
const maxToolResultBytes = options.maxToolResultBytes ?? DEFAULT_MAX_TOOL_RESULT_BYTES;
|
|
60
99
|
validateMaxToolResultBytes(maxToolResultBytes);
|
|
100
|
+
for (const [name, value] of [
|
|
101
|
+
["closeTimeoutMs", options.closeTimeoutMs],
|
|
102
|
+
["maxDurationMs", options.maxDurationMs],
|
|
103
|
+
])
|
|
104
|
+
if (value !== undefined && (!Number.isInteger(value) || value < 1 || value > 2_147_483_647))
|
|
105
|
+
throw new Error(`${name} must be a positive integer no greater than 2147483647 ms`);
|
|
106
|
+
if (options.outputModalities?.includes("text") &&
|
|
107
|
+
!options.outputModalities.includes("audio") &&
|
|
108
|
+
capabilities.supportsTextOutput === false) {
|
|
109
|
+
throw new Error("This provider does not support text-only output");
|
|
110
|
+
}
|
|
61
111
|
// Registrations made after startup begins belong to future sessions. Execution
|
|
62
112
|
// must use the same catalog that this session advertises to its provider.
|
|
63
113
|
const tools = new ToolRegistry(options.tools.list());
|
|
64
|
-
const existingEvents = await
|
|
114
|
+
const existingEvents = await persistence.list(options.sessionId);
|
|
65
115
|
if (existingEvents.length > 0) {
|
|
66
116
|
throw new Error(`Session id already exists: ${options.sessionId}`);
|
|
67
117
|
}
|
|
68
118
|
const startedAt = performance.now();
|
|
69
|
-
const startedEvent = await
|
|
119
|
+
const startedEvent = await persistence.append(createUnsequencedEvent(options.sessionId, startedAt, {
|
|
70
120
|
type: "session.started",
|
|
71
|
-
formatVersion: 1,
|
|
121
|
+
formatVersion: capabilities.outputLifecycle === "continuous" ? 2 : 1,
|
|
72
122
|
providerId: options.provider.id,
|
|
73
|
-
capabilities
|
|
123
|
+
capabilities,
|
|
74
124
|
...(options.instructions === undefined ? {} : { instructions: options.instructions }),
|
|
75
125
|
...(options.outputModalities === undefined
|
|
76
126
|
? {}
|
|
77
127
|
: { outputModalities: [...options.outputModalities] }),
|
|
78
128
|
}));
|
|
79
|
-
await
|
|
129
|
+
await persistence.flush(options.sessionId);
|
|
80
130
|
if (options.signal?.aborted) {
|
|
81
|
-
await recordFailedCreation(
|
|
131
|
+
await recordFailedCreation(persistence, options.sessionId, startedAt, "owner_revoked");
|
|
82
132
|
throw options.signal.reason ?? new Error("Session owner revoked before startup");
|
|
83
133
|
}
|
|
84
134
|
const providerController = new AbortController();
|
|
@@ -99,29 +149,37 @@ export class VoiceSession {
|
|
|
99
149
|
catch (error) {
|
|
100
150
|
options.signal?.removeEventListener("abort", abortProviderFromOwner);
|
|
101
151
|
providerController.abort(error);
|
|
102
|
-
await recordFailedCreation(
|
|
152
|
+
await recordFailedCreation(persistence, options.sessionId, startedAt, options.signal?.aborted ? "owner_revoked" : "provider_create_failed");
|
|
103
153
|
throw error;
|
|
104
154
|
}
|
|
155
|
+
const session = new VoiceSession({ ...options, tools, maxToolResultBytes, runtimeLimits }, providerSession, startedAt, startedEvent, providerController, capabilities, persistence);
|
|
156
|
+
options.signal?.removeEventListener("abort", abortProviderFromOwner);
|
|
157
|
+
session.#providerPump = session.#pumpProviderEvents();
|
|
105
158
|
if (options.signal?.aborted) {
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
await
|
|
159
|
+
// A returned provider now belongs to a session, including its normal close deadline
|
|
160
|
+
// and terminal recording. Do not leave startup waiting on an unbounded close handshake.
|
|
161
|
+
await session.close("owner_revoked");
|
|
109
162
|
throw options.signal.reason ?? new Error("Session owner revoked during startup");
|
|
110
163
|
}
|
|
111
|
-
const session = new VoiceSession({ ...options, tools, maxToolResultBytes }, providerSession, startedAt, startedEvent);
|
|
112
|
-
session.#controller.signal.addEventListener("abort", () => providerController.abort(session.#controller.signal.reason), {
|
|
113
|
-
once: true,
|
|
114
|
-
});
|
|
115
|
-
options.signal?.removeEventListener("abort", abortProviderFromOwner);
|
|
116
164
|
if (options.signal) {
|
|
117
165
|
const abortSessionFromOwner = () => {
|
|
166
|
+
providerController.abort(options.signal?.reason ?? "owner_revoked");
|
|
118
167
|
session.#closeFromInternal("owner_revoked");
|
|
119
168
|
};
|
|
120
169
|
options.signal.addEventListener("abort", abortSessionFromOwner, { once: true });
|
|
121
170
|
session.#detachOwnerAbort = () => options.signal?.removeEventListener("abort", abortSessionFromOwner);
|
|
122
171
|
}
|
|
123
|
-
|
|
124
|
-
|
|
172
|
+
// An ingress failure must fence tools/input even while the event pump awaits storage.
|
|
173
|
+
// Install after assigning the pump so an already-failed provider can drain safely.
|
|
174
|
+
const failureSignal = providerSession.failureSignal;
|
|
175
|
+
if (failureSignal) {
|
|
176
|
+
const onFailure = () => session.#failProvider(failureSignal.reason);
|
|
177
|
+
failureSignal.addEventListener("abort", onFailure, { once: true });
|
|
178
|
+
session.#detachProviderFailure = () => failureSignal.removeEventListener("abort", onFailure);
|
|
179
|
+
if (failureSignal.aborted)
|
|
180
|
+
onFailure();
|
|
181
|
+
}
|
|
182
|
+
if (options.maxDurationMs !== undefined && !session.#closePromise) {
|
|
125
183
|
session.#maxDurationTimer = setTimeout(() => {
|
|
126
184
|
session.#closeFromInternal("max_duration_reached");
|
|
127
185
|
}, options.maxDurationMs);
|
|
@@ -134,37 +192,88 @@ export class VoiceSession {
|
|
|
134
192
|
history() {
|
|
135
193
|
return this.#store.list(this.id);
|
|
136
194
|
}
|
|
137
|
-
|
|
195
|
+
/** Separate buffer owners; these counters are not a whole-process memory ceiling. */
|
|
196
|
+
runtimeStats() {
|
|
197
|
+
const providerQueue = this.#providerSession.eventQueueStats?.();
|
|
198
|
+
return {
|
|
199
|
+
replay: this.#events.stats(),
|
|
200
|
+
...this.#store.stats(),
|
|
201
|
+
...(providerQueue === undefined ? {} : { providerQueue }),
|
|
202
|
+
};
|
|
203
|
+
}
|
|
204
|
+
/** Immutable startup snapshot; missing optional fields retain their documented legacy meaning. */
|
|
205
|
+
get capabilities() {
|
|
206
|
+
return this.#capabilities;
|
|
207
|
+
}
|
|
208
|
+
/** Recorded tool/task execution. Completion does not imply a spoken announcement. */
|
|
209
|
+
toolCalls() {
|
|
210
|
+
return this.#toolCalls.list();
|
|
211
|
+
}
|
|
212
|
+
/** Requests one task's cancellation. The terminal task event, not this result, records settlement. */
|
|
213
|
+
cancelTask(taskId, reason) {
|
|
214
|
+
this.#assertOpen();
|
|
215
|
+
return this.#tasks.requestCancel(taskId, reason);
|
|
216
|
+
}
|
|
217
|
+
async sendAudio(chunk, metadata = {}) {
|
|
138
218
|
this.#assertOpen();
|
|
139
219
|
const validated = AudioChunkSchema.parse(chunk);
|
|
140
220
|
// The store reserves this event's log position synchronously, so the durable append
|
|
141
221
|
// and the provider forward can run in parallel without reordering the log. Media
|
|
142
222
|
// latency must not wait on persistence; the call still settles only once both are done.
|
|
143
|
-
const recorded = this.#record({ type: "audio.input", chunk: validated });
|
|
223
|
+
const recorded = this.#record({ type: "audio.input", chunk: validated, ...metadata });
|
|
144
224
|
recorded.catch(() => undefined);
|
|
225
|
+
this.#assertOpen();
|
|
145
226
|
await this.#providerSession.sendAudio(validated);
|
|
146
227
|
await recorded;
|
|
147
228
|
}
|
|
148
229
|
/**
|
|
149
|
-
*
|
|
150
|
-
*
|
|
151
|
-
*
|
|
152
|
-
*
|
|
230
|
+
* Submits a string or text-only AgentInput to the declared provider destination.
|
|
231
|
+
* Image/file parts are rejected before recording or partial delivery. Applications
|
|
232
|
+
* route those parts to their own backend. input.delivery records local submission,
|
|
233
|
+
* not backend consumption; interrupted shutdown can leave delivery unconfirmed.
|
|
153
234
|
*/
|
|
154
235
|
async sendText(input) {
|
|
155
236
|
this.#assertOpen();
|
|
156
237
|
const parsed = AgentInputSchema.parse(input);
|
|
238
|
+
const target = this.#capabilities.textInput ?? "conversation";
|
|
239
|
+
if (target === "unsupported")
|
|
240
|
+
throw new Error("This session does not accept typed input; send it to the application backend");
|
|
241
|
+
if (typeof parsed !== "string" && parsed.some((part) => part.type !== "text"))
|
|
242
|
+
throw new Error("This session accepts text parts only; send image/file input to the application backend");
|
|
157
243
|
const collapsed = agentInputToText(parsed);
|
|
158
244
|
if (collapsed === "") {
|
|
159
|
-
throw new Error("
|
|
245
|
+
throw new Error("Typed input requires at least one nonempty text part");
|
|
160
246
|
}
|
|
161
247
|
const text = MessageInputTextSchema.parse(collapsed);
|
|
162
|
-
await this.#record({
|
|
248
|
+
const recorded = await this.#record({
|
|
163
249
|
type: "message.input",
|
|
164
250
|
text,
|
|
165
251
|
...(typeof parsed === "string" ? {} : { parts: parsed }),
|
|
166
252
|
});
|
|
167
|
-
|
|
253
|
+
try {
|
|
254
|
+
this.#assertOpen();
|
|
255
|
+
await this.#providerSession.sendText(text);
|
|
256
|
+
}
|
|
257
|
+
catch (error) {
|
|
258
|
+
if (!this.#recordingEnded)
|
|
259
|
+
await this.#record({
|
|
260
|
+
type: "input.delivery",
|
|
261
|
+
inputId: recorded.id,
|
|
262
|
+
providerId: this.#providerId,
|
|
263
|
+
target,
|
|
264
|
+
phase: "failed",
|
|
265
|
+
error: errorMessage(error).slice(0, 2000) || "Input submission failed",
|
|
266
|
+
});
|
|
267
|
+
throw error;
|
|
268
|
+
}
|
|
269
|
+
if (!this.#recordingEnded)
|
|
270
|
+
await this.#record({
|
|
271
|
+
type: "input.delivery",
|
|
272
|
+
inputId: recorded.id,
|
|
273
|
+
providerId: this.#providerId,
|
|
274
|
+
target,
|
|
275
|
+
phase: "submitted",
|
|
276
|
+
});
|
|
168
277
|
}
|
|
169
278
|
/**
|
|
170
279
|
* Resolves only after the provider has acknowledged its session configuration.
|
|
@@ -183,50 +292,136 @@ export class VoiceSession {
|
|
|
183
292
|
* creates a synthetic user message in the session log.
|
|
184
293
|
*/
|
|
185
294
|
async requestAssistantTurn(request = {}) {
|
|
295
|
+
if (this.#capabilities.supportsAssistantTurn === false) {
|
|
296
|
+
throw new Error("This provider does not support assistant turns; use appendContext for conversation steering");
|
|
297
|
+
}
|
|
186
298
|
const validated = AssistantTurnRequestSchema.parse(request);
|
|
187
299
|
await this.waitUntilReady();
|
|
188
300
|
await this.#record({ type: "assistant.turn.requested" });
|
|
301
|
+
this.#assertOpen();
|
|
189
302
|
await this.#providerSession.requestAssistantTurn(validated);
|
|
190
303
|
}
|
|
191
304
|
async reportPlayout(progress) {
|
|
192
305
|
this.#assertOpen();
|
|
193
306
|
const validated = PlayoutProgressSchema.parse(progress);
|
|
194
307
|
await this.#record({ type: "playout.progress", ...validated });
|
|
308
|
+
// An admitted receipt remains valid local evidence when shutdown wins the append.
|
|
309
|
+
// Avoid further transport work without turning normal playback cleanup into an error.
|
|
310
|
+
if (this.#closePromise || this.#controller.signal.aborted)
|
|
311
|
+
return;
|
|
195
312
|
await this.#providerSession.reportPlayout(validated);
|
|
196
313
|
}
|
|
314
|
+
/** Records local evidence even when a provider cannot consume playout acknowledgements. */
|
|
315
|
+
async reportPlayoutRange(range) {
|
|
316
|
+
this.#assertOpen();
|
|
317
|
+
await this.#record({ type: "playout.range", ...PlayoutRangeSchema.parse(range) });
|
|
318
|
+
}
|
|
319
|
+
async appendContext(update) {
|
|
320
|
+
const validated = ConversationContextUpdateSchema.parse(update);
|
|
321
|
+
const limit = this.#capabilities.maxContextUpdateBytes;
|
|
322
|
+
if (limit !== undefined && new TextEncoder().encode(validated.content).byteLength > limit)
|
|
323
|
+
throw new Error(`Context update exceeds this session's ${limit}-byte UTF-8 limit`);
|
|
324
|
+
await this.waitUntilReady();
|
|
325
|
+
if (!this.#providerSession.appendContext)
|
|
326
|
+
throw new Error("This provider does not support direct context updates");
|
|
327
|
+
await this.#providerSession.appendContext(validated);
|
|
328
|
+
}
|
|
197
329
|
close(reason = "client_closed") {
|
|
198
|
-
this.#closePromise
|
|
330
|
+
if (this.#closePromise)
|
|
331
|
+
return this.#closePromise;
|
|
332
|
+
const completion = Promise.withResolvers();
|
|
333
|
+
// Abort callbacks can call close synchronously, so reserve the shared result first.
|
|
334
|
+
this.#closePromise = completion.promise;
|
|
335
|
+
void this.#performClose(reason).then(completion.resolve, completion.reject);
|
|
199
336
|
return this.#closePromise;
|
|
200
337
|
}
|
|
201
338
|
#closeFromInternal(reason) {
|
|
202
339
|
void this.close(reason).catch(() => undefined);
|
|
203
340
|
}
|
|
341
|
+
#failProvider(reason) {
|
|
342
|
+
if (this.#providerFailure || this.#recordingEnded)
|
|
343
|
+
return;
|
|
344
|
+
const error = reason instanceof Error ? reason : new Error(errorMessage(reason));
|
|
345
|
+
this.#providerFailure = error;
|
|
346
|
+
this.#controller.abort("provider_error");
|
|
347
|
+
this.#settleReadiness(false);
|
|
348
|
+
// Stop subscribers now; accepted durable writes may still settle afterward.
|
|
349
|
+
this.#events.fail(error);
|
|
350
|
+
this.#trackRecord(this.#record({ type: "provider.error", message: error.message, recoverable: false }));
|
|
351
|
+
this.#closeFromInternal("provider_error");
|
|
352
|
+
}
|
|
204
353
|
async #performClose(reason) {
|
|
205
|
-
this.#detachOwnerAbort?.();
|
|
206
|
-
this.#detachOwnerAbort = undefined;
|
|
207
354
|
if (this.#maxDurationTimer)
|
|
208
355
|
clearTimeout(this.#maxDurationTimer);
|
|
209
356
|
this.#controller.abort(reason);
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
await this.#
|
|
357
|
+
let deadline;
|
|
358
|
+
const drain = async () => {
|
|
359
|
+
await Promise.allSettled([...this.#toolExecutions]);
|
|
360
|
+
await this.#tasks.cancelAll(reason);
|
|
361
|
+
try {
|
|
362
|
+
await this.#providerSession.close(reason);
|
|
363
|
+
}
|
|
364
|
+
catch (error) {
|
|
365
|
+
// Keep the original shutdown failure even if this best-effort terminal
|
|
366
|
+
// observation cannot be persisted. A failed close never confirms usage.
|
|
367
|
+
await this.#recordProviderClosed("provider_close_failed", false).catch(() => undefined);
|
|
368
|
+
throw error;
|
|
369
|
+
}
|
|
214
370
|
await this.#providerPump;
|
|
371
|
+
// A resolved close method is not a provider finalization receipt. The pump
|
|
372
|
+
// may have stopped on invalid input or ended without a terminal event.
|
|
373
|
+
await this.#recordProviderClosed("provider_stream_ended", false);
|
|
374
|
+
};
|
|
375
|
+
try {
|
|
376
|
+
const completed = await Promise.race([
|
|
377
|
+
drain().then(() => true),
|
|
378
|
+
new Promise((resolve) => {
|
|
379
|
+
deadline = setTimeout(() => resolve(false), this.#closeTimeoutMs);
|
|
380
|
+
}),
|
|
381
|
+
]);
|
|
382
|
+
if (!completed) {
|
|
383
|
+
this.#providerController.abort("close_timeout");
|
|
384
|
+
await this.#recordProviderClosed("close_timeout", false);
|
|
385
|
+
}
|
|
215
386
|
}
|
|
216
387
|
finally {
|
|
388
|
+
clearTimeout(deadline);
|
|
389
|
+
this.#providerController.abort(reason);
|
|
390
|
+
this.#detachOwnerAbort?.();
|
|
391
|
+
this.#detachOwnerAbort = undefined;
|
|
392
|
+
this.#detachProviderFailure?.();
|
|
393
|
+
this.#detachProviderFailure = undefined;
|
|
394
|
+
this.#settleReadiness(false);
|
|
395
|
+
// Fence late/uncooperative producers before the final append; accepted writes retain order.
|
|
396
|
+
this.#recordingEnded = true;
|
|
217
397
|
await Promise.allSettled([...this.#pendingRecords]);
|
|
218
|
-
|
|
219
|
-
|
|
398
|
+
await this.#finishRecording(reason);
|
|
399
|
+
}
|
|
400
|
+
}
|
|
401
|
+
async #finishRecording(reason) {
|
|
402
|
+
try {
|
|
403
|
+
if (this.#providerFailure) {
|
|
404
|
+
// Preserve any accepted records, but a lost ingress event cannot become a complete log.
|
|
220
405
|
await this.#store.flush(this.id);
|
|
406
|
+
throw this.#providerFailure;
|
|
221
407
|
}
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
408
|
+
const ended = await this.#store.append(createUnsequencedEvent(this.id, this.#startedAt, { type: "session.ended", reason }));
|
|
409
|
+
await this.#store.flush(this.id);
|
|
410
|
+
this.#events.publish(ended);
|
|
411
|
+
}
|
|
412
|
+
catch (error) {
|
|
413
|
+
this.#events.fail(error instanceof Error ? error : new Error(String(error)));
|
|
414
|
+
throw error;
|
|
415
|
+
}
|
|
416
|
+
finally {
|
|
417
|
+
this.#events.close();
|
|
225
418
|
}
|
|
226
419
|
}
|
|
227
420
|
async #pumpProviderEvents() {
|
|
228
421
|
try {
|
|
229
422
|
for await (const candidate of this.#providerSession.events) {
|
|
423
|
+
if (this.#recordingEnded)
|
|
424
|
+
break;
|
|
230
425
|
const event = ProviderEventSchema.parse(candidate);
|
|
231
426
|
switch (event.type) {
|
|
232
427
|
case "ready":
|
|
@@ -237,32 +432,33 @@ export class VoiceSession {
|
|
|
237
432
|
// Pipelined: the append is enqueued (log position reserved) without stalling
|
|
238
433
|
// the pump, so a burst of audio deltas cannot delay a trailing tool call or
|
|
239
434
|
// error behind per-chunk persistence.
|
|
240
|
-
this.#trackRecord(this.#record(
|
|
435
|
+
this.#trackRecord(this.#record(event));
|
|
241
436
|
break;
|
|
242
437
|
case "audio.interrupted":
|
|
243
438
|
this.#trackRecord(this.#record({ type: "audio.interrupted", streamId: event.streamId }));
|
|
244
439
|
break;
|
|
245
440
|
case "transcript":
|
|
246
|
-
await this.#record(
|
|
247
|
-
type: "transcript",
|
|
248
|
-
role: event.role,
|
|
249
|
-
text: event.text,
|
|
250
|
-
...(event.streamId === undefined ? {} : { streamId: event.streamId }),
|
|
251
|
-
});
|
|
441
|
+
await this.#record(event);
|
|
252
442
|
break;
|
|
253
443
|
case "transcript.delta":
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
role: event.role,
|
|
257
|
-
delta: event.delta,
|
|
258
|
-
...(event.streamId === undefined ? {} : { streamId: event.streamId }),
|
|
259
|
-
}));
|
|
444
|
+
case "backend.output.delta":
|
|
445
|
+
this.#trackRecord(this.#record(event));
|
|
260
446
|
break;
|
|
261
447
|
case "tool.call":
|
|
448
|
+
if (this.#controller.signal.aborted)
|
|
449
|
+
break;
|
|
262
450
|
await this.#record({ type: "tool.requested", call: event.call });
|
|
263
451
|
this.#trackToolExecution(this.#toolRuntime.execute(event.call));
|
|
264
452
|
break;
|
|
265
453
|
case "response.state":
|
|
454
|
+
case "transcript.fragment":
|
|
455
|
+
case "backend.response":
|
|
456
|
+
case "backend.output":
|
|
457
|
+
case "backend.tool":
|
|
458
|
+
case "delegation.created":
|
|
459
|
+
case "context.delivery":
|
|
460
|
+
case "usage.duration":
|
|
461
|
+
case "tool.delivery_failed":
|
|
266
462
|
await this.#record(event);
|
|
267
463
|
break;
|
|
268
464
|
case "diagnostic":
|
|
@@ -286,7 +482,7 @@ export class VoiceSession {
|
|
|
286
482
|
break;
|
|
287
483
|
case "closed":
|
|
288
484
|
this.#settleReadiness(false);
|
|
289
|
-
await this.#
|
|
485
|
+
await this.#recordProviderClosed(event.reason, event.finalized);
|
|
290
486
|
this.#closeFromInternal("provider_closed");
|
|
291
487
|
break;
|
|
292
488
|
}
|
|
@@ -299,10 +495,15 @@ export class VoiceSession {
|
|
|
299
495
|
message: errorMessage(error),
|
|
300
496
|
recoverable: false,
|
|
301
497
|
});
|
|
498
|
+
this.#closeFromInternal("provider_error");
|
|
302
499
|
}
|
|
303
500
|
}
|
|
304
501
|
finally {
|
|
305
502
|
this.#settleReadiness(false);
|
|
503
|
+
// A custom adapter may end its iterator without a terminal receipt. Reuse
|
|
504
|
+
// shutdown to fence input and seal the log with unconfirmed finalization.
|
|
505
|
+
if (!this.#controller.signal.aborted)
|
|
506
|
+
this.#closeFromInternal("provider_stream_ended");
|
|
306
507
|
}
|
|
307
508
|
}
|
|
308
509
|
#settleReadiness(ready) {
|
|
@@ -311,6 +512,20 @@ export class VoiceSession {
|
|
|
311
512
|
this.#providerReadinessSettled = true;
|
|
312
513
|
this.#settleProviderReadiness(ready);
|
|
313
514
|
}
|
|
515
|
+
async #recordProviderClosed(reason, finalized) {
|
|
516
|
+
if (this.#providerClosed)
|
|
517
|
+
return;
|
|
518
|
+
// Mark before persistence: a transport abort can enqueue the same terminal
|
|
519
|
+
// observation while the close deadline is recording its fallback.
|
|
520
|
+
this.#providerClosed = true;
|
|
521
|
+
if (this.#providerFailure)
|
|
522
|
+
finalized = false;
|
|
523
|
+
await this.#record({
|
|
524
|
+
type: "provider.closed",
|
|
525
|
+
reason,
|
|
526
|
+
...(finalized === undefined ? {} : { finalized }),
|
|
527
|
+
});
|
|
528
|
+
}
|
|
314
529
|
#trackToolExecution(execution) {
|
|
315
530
|
this.#toolExecutions.add(execution);
|
|
316
531
|
void execution.then(() => this.#toolExecutions.delete(execution), () => {
|
|
@@ -326,16 +541,97 @@ export class VoiceSession {
|
|
|
326
541
|
});
|
|
327
542
|
}
|
|
328
543
|
async #record(event) {
|
|
544
|
+
if (this.#recordingEnded)
|
|
545
|
+
throw new Error("Session recording has ended");
|
|
546
|
+
if (this.#ownAudioBuffers && (event.type === "audio.input" || event.type === "audio.output")) {
|
|
547
|
+
const data = event.chunk.data;
|
|
548
|
+
// A small view can otherwise retain a whole recording or pooled buffer beyond the byte limit.
|
|
549
|
+
// Storage and replay share this one exact-sized payload; caller/provider references are separate.
|
|
550
|
+
if (data.byteLength !== data.buffer.byteLength)
|
|
551
|
+
event = { ...event, chunk: { ...event.chunk, data: Uint8Array.from(data) } };
|
|
552
|
+
}
|
|
329
553
|
const stored = await this.#store.append(createUnsequencedEvent(this.id, this.#startedAt, event));
|
|
330
|
-
this.#
|
|
554
|
+
const changed = this.#toolCalls.apply(stored);
|
|
555
|
+
this.#syncToolStatus(stored, changed);
|
|
556
|
+
if (!this.#providerFailure)
|
|
557
|
+
this.#events.publish(stored);
|
|
331
558
|
return stored;
|
|
332
559
|
}
|
|
560
|
+
/** Status is optional observation: it must not hold up results, tasks, or media. */
|
|
561
|
+
#sendToolStatus(event) {
|
|
562
|
+
const delivery = this.#toolStatusDelivery.then(async () => {
|
|
563
|
+
if (this.#controller.signal.aborted)
|
|
564
|
+
return;
|
|
565
|
+
await this.#store.flush(this.id);
|
|
566
|
+
if (this.#controller.signal.aborted)
|
|
567
|
+
return;
|
|
568
|
+
if (this.#recordingEnded || this.#providerClosed || this.#providerController.signal.aborted)
|
|
569
|
+
throw new Error("Provider closed before context could be submitted");
|
|
570
|
+
const current = this.#toolCalls.get(event.call.callId);
|
|
571
|
+
// A slow transport may leave progress queued after its execution already settled.
|
|
572
|
+
if (!current || toolStatus(current) !== event.status)
|
|
573
|
+
return;
|
|
574
|
+
await this.#providerSession.sendContext(event);
|
|
575
|
+
});
|
|
576
|
+
// Each caller records its own delivery failure; one failure must not poison later updates.
|
|
577
|
+
this.#toolStatusDelivery = delivery.catch(() => undefined);
|
|
578
|
+
return delivery;
|
|
579
|
+
}
|
|
580
|
+
#syncToolStatus(event, changed) {
|
|
581
|
+
if (!this.#capabilities.supportsToolStatusContext || this.#controller.signal.aborted)
|
|
582
|
+
return;
|
|
583
|
+
// A request may still be denied by admission. Report execution only after tool.started.
|
|
584
|
+
const call = event.type === "tool.started" ? this.#toolCalls.get(event.callId) : changed;
|
|
585
|
+
if (!call)
|
|
586
|
+
return;
|
|
587
|
+
if (event.type !== "tool.started" &&
|
|
588
|
+
event.type !== "tool.completed" &&
|
|
589
|
+
event.type !== "tool.failed" &&
|
|
590
|
+
event.type !== "tool.denied" &&
|
|
591
|
+
event.type !== "task.completed" &&
|
|
592
|
+
event.type !== "task.failed" &&
|
|
593
|
+
event.type !== "task.cancelled")
|
|
594
|
+
return;
|
|
595
|
+
const contextId = `tool-status:${event.id}`;
|
|
596
|
+
this.#toolStatusDelivery = this.#sendToolStatus({
|
|
597
|
+
type: "tool.status",
|
|
598
|
+
contextId,
|
|
599
|
+
sourceSequence: event.sequence,
|
|
600
|
+
call: { callId: call.callId, name: call.name, arguments: call.state.input },
|
|
601
|
+
status: toolStatus(call),
|
|
602
|
+
...(call.taskId === undefined ? {} : { taskId: call.taskId }),
|
|
603
|
+
}).catch(async (error) => {
|
|
604
|
+
if (this.#recordingEnded)
|
|
605
|
+
return;
|
|
606
|
+
await this.#record({
|
|
607
|
+
type: "context.delivery",
|
|
608
|
+
contextId,
|
|
609
|
+
callId: call.callId,
|
|
610
|
+
...(call.taskId === undefined ? {} : { taskId: call.taskId }),
|
|
611
|
+
kind: "thinking",
|
|
612
|
+
phase: "failed",
|
|
613
|
+
message: error instanceof Error ? error.message.slice(0, 2000) : "Tool status context failed",
|
|
614
|
+
});
|
|
615
|
+
});
|
|
616
|
+
void this.#toolStatusDelivery.catch(() => this.#closeFromInternal("storage_error"));
|
|
617
|
+
}
|
|
333
618
|
#assertOpen() {
|
|
619
|
+
if (this.#persistenceFailure)
|
|
620
|
+
throw this.#persistenceFailure;
|
|
621
|
+
if (this.#providerFailure)
|
|
622
|
+
throw this.#providerFailure;
|
|
334
623
|
if (this.#closePromise || this.#controller.signal.aborted) {
|
|
335
624
|
throw new Error("Voice session is closed");
|
|
336
625
|
}
|
|
337
626
|
}
|
|
338
627
|
}
|
|
628
|
+
function toolStatus(call) {
|
|
629
|
+
if (call.state.status === "running")
|
|
630
|
+
return "running";
|
|
631
|
+
if (call.state.outcome.type === "accepted_task")
|
|
632
|
+
return "background";
|
|
633
|
+
return call.state.outcome.type === "failed" ? "failed" : "completed";
|
|
634
|
+
}
|
|
339
635
|
function createUnsequencedEvent(sessionId, startedAt, event) {
|
|
340
636
|
return {
|
|
341
637
|
...event,
|
|
@@ -357,6 +653,3 @@ async function recordFailedCreation(store, sessionId, startedAt, reason) {
|
|
|
357
653
|
// Preserve the provider creation error; the original started event remains inspectable.
|
|
358
654
|
}
|
|
359
655
|
}
|
|
360
|
-
function errorMessage(error) {
|
|
361
|
-
return error instanceof Error ? error.message : String(error);
|
|
362
|
-
}
|
|
@@ -1,9 +1,8 @@
|
|
|
1
|
-
import type { OutputModality } from "../protocol/index.js";
|
|
2
|
-
import type { VoiceProvider } from "../provider/index.js";
|
|
3
1
|
import type { EventStore } from "../storage/index.js";
|
|
2
|
+
import type { SessionRuntimeLimits } from "./runtime-limits.js";
|
|
4
3
|
import type { ToolPolicy } from "./tool-policy.js";
|
|
5
4
|
import { ToolRegistry, type VoiceTool } from "./tools.js";
|
|
6
|
-
import { VoiceSession } from "./voice-session.js";
|
|
5
|
+
import { type CreateVoiceSessionOptions, VoiceSession } from "./voice-session.js";
|
|
7
6
|
export interface XOOptions {
|
|
8
7
|
store: EventStore;
|
|
9
8
|
tools?: readonly VoiceTool[];
|
|
@@ -14,21 +13,17 @@ export interface XOOptions {
|
|
|
14
13
|
* An integer of at least 128, or Infinity (the default). The log keeps the full outcome.
|
|
15
14
|
*/
|
|
16
15
|
maxToolResultBytes?: number;
|
|
16
|
+
runtimeLimits?: SessionRuntimeLimits;
|
|
17
17
|
}
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
/** Cancels provider startup and closes the live session when its owner is revoked. */
|
|
21
|
-
signal?: AbortSignal;
|
|
18
|
+
/** Session options share the low-level contract; XO supplies storage and the tool registry. */
|
|
19
|
+
export interface StartSessionOptions extends Omit<CreateVoiceSessionOptions, "sessionId" | "store" | "tools"> {
|
|
22
20
|
sessionId?: string;
|
|
23
21
|
/** Overrides the harness-level tool policy for this session. */
|
|
24
22
|
toolPolicy?: ToolPolicy;
|
|
25
23
|
/** Overrides the harness-level delivery budget for this session. */
|
|
26
24
|
maxToolResultBytes?: number;
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
outputModalities?: readonly OutputModality[];
|
|
30
|
-
/** Closes the session with max_duration_reached once elapsed. Guards runaway metered sessions. */
|
|
31
|
-
maxDurationMs?: number;
|
|
25
|
+
/** Overrides individual harness runtime limits for this session. */
|
|
26
|
+
runtimeLimits?: SessionRuntimeLimits;
|
|
32
27
|
}
|
|
33
28
|
export declare class XO {
|
|
34
29
|
#private;
|