xo-harness 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +92 -5
- package/dist/browser/worklets/capture-processor.js +34 -0
- package/dist/browser/worklets/playback-processor.js +188 -0
- package/dist/browser.d.ts +1 -0
- package/dist/browser.js +1 -0
- package/dist/internal/browser/browser-voice-client.d.ts +33 -0
- package/dist/internal/browser/browser-voice-client.js +316 -0
- package/dist/internal/browser/index.d.ts +2 -0
- package/dist/internal/browser/index.js +1 -0
- package/dist/internal/harness/conversation-context.d.ts +45 -0
- package/dist/internal/harness/conversation-context.js +79 -0
- package/dist/internal/harness/conversation-projection.d.ts +55 -0
- package/dist/internal/harness/conversation-projection.js +145 -0
- package/dist/internal/harness/event-stream.d.ts +23 -2
- package/dist/internal/harness/event-stream.js +148 -18
- package/dist/internal/harness/index.d.ts +7 -1
- package/dist/internal/harness/index.js +7 -1
- package/dist/internal/harness/message.d.ts +24 -11
- package/dist/internal/harness/message.js +271 -77
- package/dist/internal/harness/report-diff.d.ts +3 -0
- package/dist/internal/harness/report-diff.js +10 -0
- package/dist/internal/harness/report.d.ts +17 -14
- package/dist/internal/harness/report.js +41 -24
- package/dist/internal/harness/runtime-limits.d.ts +18 -0
- package/dist/internal/harness/runtime-limits.js +19 -0
- package/dist/internal/harness/session-persistence.d.ts +16 -0
- package/dist/internal/harness/session-persistence.js +106 -0
- package/dist/internal/harness/shadow.d.ts +5 -5
- package/dist/internal/harness/shadow.js +214 -52
- package/dist/internal/harness/socket-bridge.d.ts +45 -4
- package/dist/internal/harness/socket-bridge.js +215 -46
- package/dist/internal/harness/task-supervisor.d.ts +6 -2
- package/dist/internal/harness/task-supervisor.js +97 -15
- package/dist/internal/harness/tool-calls.d.ts +40 -0
- package/dist/internal/harness/tool-calls.js +132 -0
- package/dist/internal/harness/tool-policy.d.ts +2 -2
- package/dist/internal/harness/tool-runtime.d.ts +2 -2
- package/dist/internal/harness/tool-runtime.js +37 -40
- package/dist/internal/harness/tools.d.ts +21 -0
- package/dist/internal/harness/tools.js +7 -1
- package/dist/internal/harness/usage-tracker.d.ts +39 -0
- package/dist/internal/harness/usage-tracker.js +104 -0
- package/dist/internal/harness/voice-session.d.ts +39 -8
- package/dist/internal/harness/voice-session.js +353 -58
- package/dist/internal/harness/xo.d.ts +7 -12
- package/dist/internal/harness/xo.js +25 -13
- package/dist/internal/protocol/async-queue.d.ts +22 -1
- package/dist/internal/protocol/async-queue.js +86 -12
- package/dist/internal/protocol/audio.d.ts +16 -2
- package/dist/internal/protocol/audio.js +23 -5
- package/dist/internal/protocol/backend-output.d.ts +70 -0
- package/dist/internal/protocol/backend-output.js +39 -0
- package/dist/internal/protocol/event-json.d.ts +3 -0
- package/dist/internal/protocol/event-json.js +15 -0
- package/dist/internal/protocol/events.d.ts +342 -4
- package/dist/internal/protocol/events.js +53 -25
- package/dist/internal/protocol/index.d.ts +5 -0
- package/dist/internal/protocol/index.js +5 -0
- package/dist/internal/protocol/output-source.d.ts +18 -0
- package/dist/internal/protocol/output-source.js +15 -0
- package/dist/internal/protocol/paced-audio.d.ts +27 -0
- package/dist/internal/protocol/paced-audio.js +117 -0
- package/dist/internal/protocol/parts.d.ts +154 -0
- package/dist/internal/protocol/parts.js +32 -6
- package/dist/internal/protocol/provider.d.ts +308 -1
- package/dist/internal/protocol/provider.js +102 -3
- package/dist/internal/protocol/records.d.ts +773 -0
- package/dist/internal/protocol/records.js +46 -0
- package/dist/internal/protocol/tools.d.ts +40 -0
- package/dist/internal/protocol/tools.js +12 -0
- package/dist/internal/protocol/transcript.d.ts +19 -0
- package/dist/internal/protocol/transcript.js +53 -0
- package/dist/internal/provider/contract.d.ts +25 -2
- package/dist/internal/provider/event-queue.d.ts +17 -0
- package/dist/internal/provider/event-queue.js +78 -0
- package/dist/internal/provider/grok-voice.d.ts +12 -9
- package/dist/internal/provider/grok-voice.js +26 -15
- package/dist/internal/provider/index.d.ts +1 -0
- package/dist/internal/provider/index.js +1 -0
- package/dist/internal/provider/live-session.d.ts +29 -0
- package/dist/internal/provider/live-session.js +840 -0
- package/dist/internal/provider/node-socket.js +6 -0
- package/dist/internal/provider/openai-live.d.ts +62 -0
- package/dist/internal/provider/openai-live.js +160 -0
- package/dist/internal/provider/openai-realtime.d.ts +9 -5
- package/dist/internal/provider/openai-realtime.js +24 -10
- package/dist/internal/provider/realtime-session.d.ts +12 -1
- package/dist/internal/provider/realtime-session.js +373 -70
- package/dist/internal/provider/realtime-socket.d.ts +3 -1
- package/dist/internal/provider/tool-status-context.d.ts +10 -0
- package/dist/internal/provider/tool-status-context.js +30 -0
- package/dist/internal/provider/workers-socket.js +63 -15
- package/dist/internal/provider/workers.d.ts +1 -0
- package/dist/internal/provider/workers.js +1 -0
- package/dist/internal/provider-fake/replay-voice-provider.d.ts +9 -11
- package/dist/internal/provider-fake/replay-voice-provider.js +47 -23
- package/dist/internal/storage/event-store.d.ts +22 -4
- package/dist/internal/storage/jsonl-event-store.d.ts +4 -4
- package/dist/internal/storage/jsonl-event-store.js +21 -16
- package/dist/internal/storage/memory-event-store.d.ts +4 -3
- package/dist/internal/storage/memory-event-store.js +8 -2
- package/dist/internal/storage/memory.d.ts +1 -1
- package/dist/internal/testkit/events.d.ts +14 -0
- package/dist/internal/testkit/events.js +33 -0
- package/dist/internal/testkit/runtime.d.ts +3 -0
- package/dist/internal/testkit/runtime.js +3 -0
- package/dist/internal/testkit/trajectory.d.ts +23 -0
- package/dist/internal/testkit/trajectory.js +58 -0
- package/dist/internal/tools-openai/index.d.ts +2 -0
- package/dist/internal/tools-openai/index.js +79 -72
- package/dist/internal/tools-openai/responses.d.ts +5 -1
- package/dist/internal/tools-openai/responses.js +53 -5
- package/dist/testing.d.ts +1 -0
- package/dist/testing.js +1 -0
- package/package.json +7 -1
|
@@ -1,6 +1,9 @@
|
|
|
1
|
-
import { AgentInputSchema, AssistantTurnRequestSchema, AudioChunkSchema, agentInputToText, MessageInputTextSchema, PlayoutProgressSchema, ProviderEventSchema, } from "../protocol/index.js";
|
|
1
|
+
import { AgentInputSchema, AssistantTurnRequestSchema, AudioChunkSchema, agentInputToText, ConversationContextUpdateSchema, MessageInputTextSchema, PlayoutProgressSchema, PlayoutRangeSchema, ProviderCapabilitiesSchema, ProviderEventSchema, } from "../protocol/index.js";
|
|
2
2
|
import { ReplayEventStream } from "./event-stream.js";
|
|
3
|
+
import { validateRuntimeLimits, } from "./runtime-limits.js";
|
|
4
|
+
import { SessionPersistence } from "./session-persistence.js";
|
|
3
5
|
import { TaskSupervisor } from "./task-supervisor.js";
|
|
6
|
+
import { ToolCallTracker } from "./tool-calls.js";
|
|
4
7
|
import { DEFAULT_MAX_TOOL_RESULT_BYTES, validateMaxToolResultBytes } from "./tool-delivery.js";
|
|
5
8
|
import { allowAllToolPolicy } from "./tool-policy.js";
|
|
6
9
|
import { ToolRuntime } from "./tool-runtime.js";
|
|
@@ -9,32 +12,63 @@ export class VoiceSession {
|
|
|
9
12
|
id;
|
|
10
13
|
#providerSession;
|
|
11
14
|
#store;
|
|
12
|
-
#events
|
|
15
|
+
#events;
|
|
13
16
|
#controller = new AbortController();
|
|
17
|
+
#providerController;
|
|
18
|
+
#capabilities;
|
|
19
|
+
#providerId;
|
|
20
|
+
#closeTimeoutMs;
|
|
21
|
+
#ownAudioBuffers;
|
|
22
|
+
#recordingEnded = false;
|
|
23
|
+
#providerClosed = false;
|
|
24
|
+
#persistenceFailure;
|
|
25
|
+
#providerFailure;
|
|
14
26
|
#startedAt;
|
|
15
27
|
#toolExecutions = new Set();
|
|
16
28
|
#pendingRecords = new Set();
|
|
17
29
|
#tasks;
|
|
18
30
|
#toolRuntime;
|
|
31
|
+
#toolCalls = new ToolCallTracker();
|
|
32
|
+
#toolStatusDelivery = Promise.resolve();
|
|
19
33
|
#providerReadiness;
|
|
20
34
|
#settleProviderReadiness = () => undefined;
|
|
21
35
|
#providerReadinessSettled = false;
|
|
22
36
|
#detachOwnerAbort;
|
|
37
|
+
#detachProviderFailure;
|
|
23
38
|
#providerPump;
|
|
24
39
|
#closePromise;
|
|
25
40
|
#maxDurationTimer;
|
|
26
|
-
constructor(options, providerSession, startedAt, startedEvent) {
|
|
41
|
+
constructor(options, providerSession, startedAt, startedEvent, providerController, capabilities, persistence) {
|
|
27
42
|
this.id = options.sessionId;
|
|
28
43
|
this.#providerSession = providerSession;
|
|
29
|
-
this.#
|
|
44
|
+
this.#providerController = providerController;
|
|
45
|
+
this.#capabilities = capabilities;
|
|
46
|
+
this.#providerId = options.provider.id;
|
|
47
|
+
this.#closeTimeoutMs = options.closeTimeoutMs ?? 15_000;
|
|
48
|
+
this.#ownAudioBuffers =
|
|
49
|
+
options.runtimeLimits?.maxReplayAudioBytes !== undefined ||
|
|
50
|
+
options.runtimeLimits?.maxPendingAudioBytes !== undefined;
|
|
51
|
+
this.#store = persistence;
|
|
52
|
+
this.#events = new ReplayEventStream({
|
|
53
|
+
maxEvents: options.runtimeLimits?.maxReplayEvents,
|
|
54
|
+
maxAudioBytes: options.runtimeLimits?.maxReplayAudioBytes,
|
|
55
|
+
});
|
|
30
56
|
this.#startedAt = startedAt;
|
|
31
57
|
this.#events.publish(startedEvent);
|
|
58
|
+
persistence.onFailure = (error) => {
|
|
59
|
+
this.#persistenceFailure = error;
|
|
60
|
+
this.#recordingEnded = true;
|
|
61
|
+
this.#events.fail(error);
|
|
62
|
+
this.#providerController.abort(error.code);
|
|
63
|
+
this.#closeFromInternal(error.code);
|
|
64
|
+
};
|
|
32
65
|
this.#providerReadiness = new Promise((resolve) => {
|
|
33
66
|
this.#settleProviderReadiness = resolve;
|
|
34
67
|
});
|
|
35
68
|
this.#tasks = new TaskSupervisor({
|
|
36
69
|
record: (event) => this.#record(event),
|
|
37
|
-
|
|
70
|
+
// Shutdown still records task outcomes; it must not prompt another spoken response.
|
|
71
|
+
sendContext: (event) => this.#controller.signal.aborted ? Promise.resolve() : this.#providerSession.sendContext(event),
|
|
38
72
|
flush: () => this.#store.flush(this.id),
|
|
39
73
|
sessionSignal: this.#controller.signal,
|
|
40
74
|
onError: () => {
|
|
@@ -56,29 +90,44 @@ export class VoiceSession {
|
|
|
56
90
|
});
|
|
57
91
|
}
|
|
58
92
|
static async create(options) {
|
|
93
|
+
// Capture declared behavior before any application/provider startup awaits.
|
|
94
|
+
const capabilities = Object.freeze(ProviderCapabilitiesSchema.parse(options.provider.capabilities));
|
|
95
|
+
const runtimeLimits = validateRuntimeLimits(options.runtimeLimits);
|
|
96
|
+
const persistence = new SessionPersistence(options.store, runtimeLimits);
|
|
59
97
|
const maxToolResultBytes = options.maxToolResultBytes ?? DEFAULT_MAX_TOOL_RESULT_BYTES;
|
|
60
98
|
validateMaxToolResultBytes(maxToolResultBytes);
|
|
99
|
+
for (const [name, value] of [
|
|
100
|
+
["closeTimeoutMs", options.closeTimeoutMs],
|
|
101
|
+
["maxDurationMs", options.maxDurationMs],
|
|
102
|
+
])
|
|
103
|
+
if (value !== undefined && (!Number.isInteger(value) || value < 1 || value > 2_147_483_647))
|
|
104
|
+
throw new Error(`${name} must be a positive integer no greater than 2147483647 ms`);
|
|
105
|
+
if (options.outputModalities?.includes("text") &&
|
|
106
|
+
!options.outputModalities.includes("audio") &&
|
|
107
|
+
capabilities.supportsTextOutput === false) {
|
|
108
|
+
throw new Error("This provider does not support text-only output");
|
|
109
|
+
}
|
|
61
110
|
// Registrations made after startup begins belong to future sessions. Execution
|
|
62
111
|
// must use the same catalog that this session advertises to its provider.
|
|
63
112
|
const tools = new ToolRegistry(options.tools.list());
|
|
64
|
-
const existingEvents = await
|
|
113
|
+
const existingEvents = await persistence.list(options.sessionId);
|
|
65
114
|
if (existingEvents.length > 0) {
|
|
66
115
|
throw new Error(`Session id already exists: ${options.sessionId}`);
|
|
67
116
|
}
|
|
68
117
|
const startedAt = performance.now();
|
|
69
|
-
const startedEvent = await
|
|
118
|
+
const startedEvent = await persistence.append(createUnsequencedEvent(options.sessionId, startedAt, {
|
|
70
119
|
type: "session.started",
|
|
71
|
-
formatVersion: 1,
|
|
120
|
+
formatVersion: capabilities.outputLifecycle === "continuous" ? 2 : 1,
|
|
72
121
|
providerId: options.provider.id,
|
|
73
|
-
capabilities
|
|
122
|
+
capabilities,
|
|
74
123
|
...(options.instructions === undefined ? {} : { instructions: options.instructions }),
|
|
75
124
|
...(options.outputModalities === undefined
|
|
76
125
|
? {}
|
|
77
126
|
: { outputModalities: [...options.outputModalities] }),
|
|
78
127
|
}));
|
|
79
|
-
await
|
|
128
|
+
await persistence.flush(options.sessionId);
|
|
80
129
|
if (options.signal?.aborted) {
|
|
81
|
-
await recordFailedCreation(
|
|
130
|
+
await recordFailedCreation(persistence, options.sessionId, startedAt, "owner_revoked");
|
|
82
131
|
throw options.signal.reason ?? new Error("Session owner revoked before startup");
|
|
83
132
|
}
|
|
84
133
|
const providerController = new AbortController();
|
|
@@ -99,29 +148,37 @@ export class VoiceSession {
|
|
|
99
148
|
catch (error) {
|
|
100
149
|
options.signal?.removeEventListener("abort", abortProviderFromOwner);
|
|
101
150
|
providerController.abort(error);
|
|
102
|
-
await recordFailedCreation(
|
|
151
|
+
await recordFailedCreation(persistence, options.sessionId, startedAt, options.signal?.aborted ? "owner_revoked" : "provider_create_failed");
|
|
103
152
|
throw error;
|
|
104
153
|
}
|
|
154
|
+
const session = new VoiceSession({ ...options, tools, maxToolResultBytes, runtimeLimits }, providerSession, startedAt, startedEvent, providerController, capabilities, persistence);
|
|
155
|
+
options.signal?.removeEventListener("abort", abortProviderFromOwner);
|
|
156
|
+
session.#providerPump = session.#pumpProviderEvents();
|
|
105
157
|
if (options.signal?.aborted) {
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
await
|
|
158
|
+
// A returned provider now belongs to a session, including its normal close deadline
|
|
159
|
+
// and terminal recording. Do not leave startup waiting on an unbounded close handshake.
|
|
160
|
+
await session.close("owner_revoked");
|
|
109
161
|
throw options.signal.reason ?? new Error("Session owner revoked during startup");
|
|
110
162
|
}
|
|
111
|
-
const session = new VoiceSession({ ...options, tools, maxToolResultBytes }, providerSession, startedAt, startedEvent);
|
|
112
|
-
session.#controller.signal.addEventListener("abort", () => providerController.abort(session.#controller.signal.reason), {
|
|
113
|
-
once: true,
|
|
114
|
-
});
|
|
115
|
-
options.signal?.removeEventListener("abort", abortProviderFromOwner);
|
|
116
163
|
if (options.signal) {
|
|
117
164
|
const abortSessionFromOwner = () => {
|
|
165
|
+
providerController.abort(options.signal?.reason ?? "owner_revoked");
|
|
118
166
|
session.#closeFromInternal("owner_revoked");
|
|
119
167
|
};
|
|
120
168
|
options.signal.addEventListener("abort", abortSessionFromOwner, { once: true });
|
|
121
169
|
session.#detachOwnerAbort = () => options.signal?.removeEventListener("abort", abortSessionFromOwner);
|
|
122
170
|
}
|
|
123
|
-
|
|
124
|
-
|
|
171
|
+
// An ingress failure must fence tools/input even while the event pump awaits storage.
|
|
172
|
+
// Install after assigning the pump so an already-failed provider can drain safely.
|
|
173
|
+
const failureSignal = providerSession.failureSignal;
|
|
174
|
+
if (failureSignal) {
|
|
175
|
+
const onFailure = () => session.#failProvider(failureSignal.reason);
|
|
176
|
+
failureSignal.addEventListener("abort", onFailure, { once: true });
|
|
177
|
+
session.#detachProviderFailure = () => failureSignal.removeEventListener("abort", onFailure);
|
|
178
|
+
if (failureSignal.aborted)
|
|
179
|
+
onFailure();
|
|
180
|
+
}
|
|
181
|
+
if (options.maxDurationMs !== undefined && !session.#closePromise) {
|
|
125
182
|
session.#maxDurationTimer = setTimeout(() => {
|
|
126
183
|
session.#closeFromInternal("max_duration_reached");
|
|
127
184
|
}, options.maxDurationMs);
|
|
@@ -134,37 +191,88 @@ export class VoiceSession {
|
|
|
134
191
|
history() {
|
|
135
192
|
return this.#store.list(this.id);
|
|
136
193
|
}
|
|
137
|
-
|
|
194
|
+
/** Separate buffer owners; these counters are not a whole-process memory ceiling. */
|
|
195
|
+
runtimeStats() {
|
|
196
|
+
const providerQueue = this.#providerSession.eventQueueStats?.();
|
|
197
|
+
return {
|
|
198
|
+
replay: this.#events.stats(),
|
|
199
|
+
...this.#store.stats(),
|
|
200
|
+
...(providerQueue === undefined ? {} : { providerQueue }),
|
|
201
|
+
};
|
|
202
|
+
}
|
|
203
|
+
/** Immutable startup snapshot; missing optional fields retain their documented legacy meaning. */
|
|
204
|
+
get capabilities() {
|
|
205
|
+
return this.#capabilities;
|
|
206
|
+
}
|
|
207
|
+
/** Recorded tool/task execution. Completion does not imply a spoken announcement. */
|
|
208
|
+
toolCalls() {
|
|
209
|
+
return this.#toolCalls.list();
|
|
210
|
+
}
|
|
211
|
+
/** Requests one task's cancellation. The terminal task event, not this result, records settlement. */
|
|
212
|
+
cancelTask(taskId, reason) {
|
|
213
|
+
this.#assertOpen();
|
|
214
|
+
return this.#tasks.requestCancel(taskId, reason);
|
|
215
|
+
}
|
|
216
|
+
async sendAudio(chunk, metadata = {}) {
|
|
138
217
|
this.#assertOpen();
|
|
139
218
|
const validated = AudioChunkSchema.parse(chunk);
|
|
140
219
|
// The store reserves this event's log position synchronously, so the durable append
|
|
141
220
|
// and the provider forward can run in parallel without reordering the log. Media
|
|
142
221
|
// latency must not wait on persistence; the call still settles only once both are done.
|
|
143
|
-
const recorded = this.#record({ type: "audio.input", chunk: validated });
|
|
222
|
+
const recorded = this.#record({ type: "audio.input", chunk: validated, ...metadata });
|
|
144
223
|
recorded.catch(() => undefined);
|
|
224
|
+
this.#assertOpen();
|
|
145
225
|
await this.#providerSession.sendAudio(validated);
|
|
146
226
|
await recorded;
|
|
147
227
|
}
|
|
148
228
|
/**
|
|
149
|
-
*
|
|
150
|
-
*
|
|
151
|
-
*
|
|
152
|
-
*
|
|
229
|
+
* Submits a string or text-only AgentInput to the declared provider destination.
|
|
230
|
+
* Image/file parts are rejected before recording or partial delivery. Applications
|
|
231
|
+
* route those parts to their own backend. input.delivery records local submission,
|
|
232
|
+
* not backend consumption; interrupted shutdown can leave delivery unconfirmed.
|
|
153
233
|
*/
|
|
154
234
|
async sendText(input) {
|
|
155
235
|
this.#assertOpen();
|
|
156
236
|
const parsed = AgentInputSchema.parse(input);
|
|
237
|
+
const target = this.#capabilities.textInput ?? "conversation";
|
|
238
|
+
if (target === "unsupported")
|
|
239
|
+
throw new Error("This session does not accept typed input; send it to the application backend");
|
|
240
|
+
if (typeof parsed !== "string" && parsed.some((part) => part.type !== "text"))
|
|
241
|
+
throw new Error("This session accepts text parts only; send image/file input to the application backend");
|
|
157
242
|
const collapsed = agentInputToText(parsed);
|
|
158
243
|
if (collapsed === "") {
|
|
159
|
-
throw new Error("
|
|
244
|
+
throw new Error("Typed input requires at least one nonempty text part");
|
|
160
245
|
}
|
|
161
246
|
const text = MessageInputTextSchema.parse(collapsed);
|
|
162
|
-
await this.#record({
|
|
247
|
+
const recorded = await this.#record({
|
|
163
248
|
type: "message.input",
|
|
164
249
|
text,
|
|
165
250
|
...(typeof parsed === "string" ? {} : { parts: parsed }),
|
|
166
251
|
});
|
|
167
|
-
|
|
252
|
+
try {
|
|
253
|
+
this.#assertOpen();
|
|
254
|
+
await this.#providerSession.sendText(text);
|
|
255
|
+
}
|
|
256
|
+
catch (error) {
|
|
257
|
+
if (!this.#recordingEnded)
|
|
258
|
+
await this.#record({
|
|
259
|
+
type: "input.delivery",
|
|
260
|
+
inputId: recorded.id,
|
|
261
|
+
providerId: this.#providerId,
|
|
262
|
+
target,
|
|
263
|
+
phase: "failed",
|
|
264
|
+
error: errorMessage(error).slice(0, 2000) || "Input submission failed",
|
|
265
|
+
});
|
|
266
|
+
throw error;
|
|
267
|
+
}
|
|
268
|
+
if (!this.#recordingEnded)
|
|
269
|
+
await this.#record({
|
|
270
|
+
type: "input.delivery",
|
|
271
|
+
inputId: recorded.id,
|
|
272
|
+
providerId: this.#providerId,
|
|
273
|
+
target,
|
|
274
|
+
phase: "submitted",
|
|
275
|
+
});
|
|
168
276
|
}
|
|
169
277
|
/**
|
|
170
278
|
* Resolves only after the provider has acknowledged its session configuration.
|
|
@@ -183,50 +291,136 @@ export class VoiceSession {
|
|
|
183
291
|
* creates a synthetic user message in the session log.
|
|
184
292
|
*/
|
|
185
293
|
async requestAssistantTurn(request = {}) {
|
|
294
|
+
if (this.#capabilities.supportsAssistantTurn === false) {
|
|
295
|
+
throw new Error("This provider does not support assistant turns; use appendContext for conversation steering");
|
|
296
|
+
}
|
|
186
297
|
const validated = AssistantTurnRequestSchema.parse(request);
|
|
187
298
|
await this.waitUntilReady();
|
|
188
299
|
await this.#record({ type: "assistant.turn.requested" });
|
|
300
|
+
this.#assertOpen();
|
|
189
301
|
await this.#providerSession.requestAssistantTurn(validated);
|
|
190
302
|
}
|
|
191
303
|
async reportPlayout(progress) {
|
|
192
304
|
this.#assertOpen();
|
|
193
305
|
const validated = PlayoutProgressSchema.parse(progress);
|
|
194
306
|
await this.#record({ type: "playout.progress", ...validated });
|
|
307
|
+
// An admitted receipt remains valid local evidence when shutdown wins the append.
|
|
308
|
+
// Avoid further transport work without turning normal playback cleanup into an error.
|
|
309
|
+
if (this.#closePromise || this.#controller.signal.aborted)
|
|
310
|
+
return;
|
|
195
311
|
await this.#providerSession.reportPlayout(validated);
|
|
196
312
|
}
|
|
313
|
+
/** Records local evidence even when a provider cannot consume playout acknowledgements. */
|
|
314
|
+
async reportPlayoutRange(range) {
|
|
315
|
+
this.#assertOpen();
|
|
316
|
+
await this.#record({ type: "playout.range", ...PlayoutRangeSchema.parse(range) });
|
|
317
|
+
}
|
|
318
|
+
async appendContext(update) {
|
|
319
|
+
const validated = ConversationContextUpdateSchema.parse(update);
|
|
320
|
+
const limit = this.#capabilities.maxContextUpdateBytes;
|
|
321
|
+
if (limit !== undefined && new TextEncoder().encode(validated.content).byteLength > limit)
|
|
322
|
+
throw new Error(`Context update exceeds this session's ${limit}-byte UTF-8 limit`);
|
|
323
|
+
await this.waitUntilReady();
|
|
324
|
+
if (!this.#providerSession.appendContext)
|
|
325
|
+
throw new Error("This provider does not support direct context updates");
|
|
326
|
+
await this.#providerSession.appendContext(validated);
|
|
327
|
+
}
|
|
197
328
|
close(reason = "client_closed") {
|
|
198
|
-
this.#closePromise
|
|
329
|
+
if (this.#closePromise)
|
|
330
|
+
return this.#closePromise;
|
|
331
|
+
const completion = Promise.withResolvers();
|
|
332
|
+
// Abort callbacks can call close synchronously, so reserve the shared result first.
|
|
333
|
+
this.#closePromise = completion.promise;
|
|
334
|
+
void this.#performClose(reason).then(completion.resolve, completion.reject);
|
|
199
335
|
return this.#closePromise;
|
|
200
336
|
}
|
|
201
337
|
#closeFromInternal(reason) {
|
|
202
338
|
void this.close(reason).catch(() => undefined);
|
|
203
339
|
}
|
|
340
|
+
#failProvider(reason) {
|
|
341
|
+
if (this.#providerFailure || this.#recordingEnded)
|
|
342
|
+
return;
|
|
343
|
+
const error = reason instanceof Error ? reason : new Error(errorMessage(reason));
|
|
344
|
+
this.#providerFailure = error;
|
|
345
|
+
this.#controller.abort("provider_error");
|
|
346
|
+
this.#settleReadiness(false);
|
|
347
|
+
// Stop subscribers now; accepted durable writes may still settle afterward.
|
|
348
|
+
this.#events.fail(error);
|
|
349
|
+
this.#trackRecord(this.#record({ type: "provider.error", message: error.message, recoverable: false }));
|
|
350
|
+
this.#closeFromInternal("provider_error");
|
|
351
|
+
}
|
|
204
352
|
async #performClose(reason) {
|
|
205
|
-
this.#detachOwnerAbort?.();
|
|
206
|
-
this.#detachOwnerAbort = undefined;
|
|
207
353
|
if (this.#maxDurationTimer)
|
|
208
354
|
clearTimeout(this.#maxDurationTimer);
|
|
209
355
|
this.#controller.abort(reason);
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
await this.#
|
|
356
|
+
let deadline;
|
|
357
|
+
const drain = async () => {
|
|
358
|
+
await Promise.allSettled([...this.#toolExecutions]);
|
|
359
|
+
await this.#tasks.cancelAll(reason);
|
|
360
|
+
try {
|
|
361
|
+
await this.#providerSession.close(reason);
|
|
362
|
+
}
|
|
363
|
+
catch (error) {
|
|
364
|
+
// Keep the original shutdown failure even if this best-effort terminal
|
|
365
|
+
// observation cannot be persisted. A failed close never confirms usage.
|
|
366
|
+
await this.#recordProviderClosed("provider_close_failed", false).catch(() => undefined);
|
|
367
|
+
throw error;
|
|
368
|
+
}
|
|
214
369
|
await this.#providerPump;
|
|
370
|
+
// A resolved close method is not a provider finalization receipt. The pump
|
|
371
|
+
// may have stopped on invalid input or ended without a terminal event.
|
|
372
|
+
await this.#recordProviderClosed("provider_stream_ended", false);
|
|
373
|
+
};
|
|
374
|
+
try {
|
|
375
|
+
const completed = await Promise.race([
|
|
376
|
+
drain().then(() => true),
|
|
377
|
+
new Promise((resolve) => {
|
|
378
|
+
deadline = setTimeout(() => resolve(false), this.#closeTimeoutMs);
|
|
379
|
+
}),
|
|
380
|
+
]);
|
|
381
|
+
if (!completed) {
|
|
382
|
+
this.#providerController.abort("close_timeout");
|
|
383
|
+
await this.#recordProviderClosed("close_timeout", false);
|
|
384
|
+
}
|
|
215
385
|
}
|
|
216
386
|
finally {
|
|
387
|
+
clearTimeout(deadline);
|
|
388
|
+
this.#providerController.abort(reason);
|
|
389
|
+
this.#detachOwnerAbort?.();
|
|
390
|
+
this.#detachOwnerAbort = undefined;
|
|
391
|
+
this.#detachProviderFailure?.();
|
|
392
|
+
this.#detachProviderFailure = undefined;
|
|
393
|
+
this.#settleReadiness(false);
|
|
394
|
+
// Fence late/uncooperative producers before the final append; accepted writes retain order.
|
|
395
|
+
this.#recordingEnded = true;
|
|
217
396
|
await Promise.allSettled([...this.#pendingRecords]);
|
|
218
|
-
|
|
219
|
-
|
|
397
|
+
await this.#finishRecording(reason);
|
|
398
|
+
}
|
|
399
|
+
}
|
|
400
|
+
async #finishRecording(reason) {
|
|
401
|
+
try {
|
|
402
|
+
if (this.#providerFailure) {
|
|
403
|
+
// Preserve any accepted records, but a lost ingress event cannot become a complete log.
|
|
220
404
|
await this.#store.flush(this.id);
|
|
405
|
+
throw this.#providerFailure;
|
|
221
406
|
}
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
407
|
+
const ended = await this.#store.append(createUnsequencedEvent(this.id, this.#startedAt, { type: "session.ended", reason }));
|
|
408
|
+
await this.#store.flush(this.id);
|
|
409
|
+
this.#events.publish(ended);
|
|
410
|
+
}
|
|
411
|
+
catch (error) {
|
|
412
|
+
this.#events.fail(error instanceof Error ? error : new Error(String(error)));
|
|
413
|
+
throw error;
|
|
414
|
+
}
|
|
415
|
+
finally {
|
|
416
|
+
this.#events.close();
|
|
225
417
|
}
|
|
226
418
|
}
|
|
227
419
|
async #pumpProviderEvents() {
|
|
228
420
|
try {
|
|
229
421
|
for await (const candidate of this.#providerSession.events) {
|
|
422
|
+
if (this.#recordingEnded)
|
|
423
|
+
break;
|
|
230
424
|
const event = ProviderEventSchema.parse(candidate);
|
|
231
425
|
switch (event.type) {
|
|
232
426
|
case "ready":
|
|
@@ -237,32 +431,33 @@ export class VoiceSession {
|
|
|
237
431
|
// Pipelined: the append is enqueued (log position reserved) without stalling
|
|
238
432
|
// the pump, so a burst of audio deltas cannot delay a trailing tool call or
|
|
239
433
|
// error behind per-chunk persistence.
|
|
240
|
-
this.#trackRecord(this.#record(
|
|
434
|
+
this.#trackRecord(this.#record(event));
|
|
241
435
|
break;
|
|
242
436
|
case "audio.interrupted":
|
|
243
437
|
this.#trackRecord(this.#record({ type: "audio.interrupted", streamId: event.streamId }));
|
|
244
438
|
break;
|
|
245
439
|
case "transcript":
|
|
246
|
-
await this.#record(
|
|
247
|
-
type: "transcript",
|
|
248
|
-
role: event.role,
|
|
249
|
-
text: event.text,
|
|
250
|
-
...(event.streamId === undefined ? {} : { streamId: event.streamId }),
|
|
251
|
-
});
|
|
440
|
+
await this.#record(event);
|
|
252
441
|
break;
|
|
253
442
|
case "transcript.delta":
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
role: event.role,
|
|
257
|
-
delta: event.delta,
|
|
258
|
-
...(event.streamId === undefined ? {} : { streamId: event.streamId }),
|
|
259
|
-
}));
|
|
443
|
+
case "backend.output.delta":
|
|
444
|
+
this.#trackRecord(this.#record(event));
|
|
260
445
|
break;
|
|
261
446
|
case "tool.call":
|
|
447
|
+
if (this.#controller.signal.aborted)
|
|
448
|
+
break;
|
|
262
449
|
await this.#record({ type: "tool.requested", call: event.call });
|
|
263
450
|
this.#trackToolExecution(this.#toolRuntime.execute(event.call));
|
|
264
451
|
break;
|
|
265
452
|
case "response.state":
|
|
453
|
+
case "transcript.fragment":
|
|
454
|
+
case "backend.response":
|
|
455
|
+
case "backend.output":
|
|
456
|
+
case "backend.tool":
|
|
457
|
+
case "delegation.created":
|
|
458
|
+
case "context.delivery":
|
|
459
|
+
case "usage.duration":
|
|
460
|
+
case "tool.delivery_failed":
|
|
266
461
|
await this.#record(event);
|
|
267
462
|
break;
|
|
268
463
|
case "diagnostic":
|
|
@@ -286,7 +481,7 @@ export class VoiceSession {
|
|
|
286
481
|
break;
|
|
287
482
|
case "closed":
|
|
288
483
|
this.#settleReadiness(false);
|
|
289
|
-
await this.#
|
|
484
|
+
await this.#recordProviderClosed(event.reason, event.finalized);
|
|
290
485
|
this.#closeFromInternal("provider_closed");
|
|
291
486
|
break;
|
|
292
487
|
}
|
|
@@ -299,10 +494,15 @@ export class VoiceSession {
|
|
|
299
494
|
message: errorMessage(error),
|
|
300
495
|
recoverable: false,
|
|
301
496
|
});
|
|
497
|
+
this.#closeFromInternal("provider_error");
|
|
302
498
|
}
|
|
303
499
|
}
|
|
304
500
|
finally {
|
|
305
501
|
this.#settleReadiness(false);
|
|
502
|
+
// A custom adapter may end its iterator without a terminal receipt. Reuse
|
|
503
|
+
// shutdown to fence input and seal the log with unconfirmed finalization.
|
|
504
|
+
if (!this.#controller.signal.aborted)
|
|
505
|
+
this.#closeFromInternal("provider_stream_ended");
|
|
306
506
|
}
|
|
307
507
|
}
|
|
308
508
|
#settleReadiness(ready) {
|
|
@@ -311,6 +511,20 @@ export class VoiceSession {
|
|
|
311
511
|
this.#providerReadinessSettled = true;
|
|
312
512
|
this.#settleProviderReadiness(ready);
|
|
313
513
|
}
|
|
514
|
+
async #recordProviderClosed(reason, finalized) {
|
|
515
|
+
if (this.#providerClosed)
|
|
516
|
+
return;
|
|
517
|
+
// Mark before persistence: a transport abort can enqueue the same terminal
|
|
518
|
+
// observation while the close deadline is recording its fallback.
|
|
519
|
+
this.#providerClosed = true;
|
|
520
|
+
if (this.#providerFailure)
|
|
521
|
+
finalized = false;
|
|
522
|
+
await this.#record({
|
|
523
|
+
type: "provider.closed",
|
|
524
|
+
reason,
|
|
525
|
+
...(finalized === undefined ? {} : { finalized }),
|
|
526
|
+
});
|
|
527
|
+
}
|
|
314
528
|
#trackToolExecution(execution) {
|
|
315
529
|
this.#toolExecutions.add(execution);
|
|
316
530
|
void execution.then(() => this.#toolExecutions.delete(execution), () => {
|
|
@@ -326,16 +540,97 @@ export class VoiceSession {
|
|
|
326
540
|
});
|
|
327
541
|
}
|
|
328
542
|
async #record(event) {
|
|
543
|
+
if (this.#recordingEnded)
|
|
544
|
+
throw new Error("Session recording has ended");
|
|
545
|
+
if (this.#ownAudioBuffers && (event.type === "audio.input" || event.type === "audio.output")) {
|
|
546
|
+
const data = event.chunk.data;
|
|
547
|
+
// A small view can otherwise retain a whole recording or pooled buffer beyond the byte limit.
|
|
548
|
+
// Storage and replay share this one exact-sized payload; caller/provider references are separate.
|
|
549
|
+
if (data.byteLength !== data.buffer.byteLength)
|
|
550
|
+
event = { ...event, chunk: { ...event.chunk, data: Uint8Array.from(data) } };
|
|
551
|
+
}
|
|
329
552
|
const stored = await this.#store.append(createUnsequencedEvent(this.id, this.#startedAt, event));
|
|
330
|
-
this.#
|
|
553
|
+
const changed = this.#toolCalls.apply(stored);
|
|
554
|
+
this.#syncToolStatus(stored, changed);
|
|
555
|
+
if (!this.#providerFailure)
|
|
556
|
+
this.#events.publish(stored);
|
|
331
557
|
return stored;
|
|
332
558
|
}
|
|
559
|
+
/** Status is optional observation: it must not hold up results, tasks, or media. */
|
|
560
|
+
#sendToolStatus(event) {
|
|
561
|
+
const delivery = this.#toolStatusDelivery.then(async () => {
|
|
562
|
+
if (this.#controller.signal.aborted)
|
|
563
|
+
return;
|
|
564
|
+
await this.#store.flush(this.id);
|
|
565
|
+
if (this.#controller.signal.aborted)
|
|
566
|
+
return;
|
|
567
|
+
if (this.#recordingEnded || this.#providerClosed || this.#providerController.signal.aborted)
|
|
568
|
+
throw new Error("Provider closed before context could be submitted");
|
|
569
|
+
const current = this.#toolCalls.get(event.call.callId);
|
|
570
|
+
// A slow transport may leave progress queued after its execution already settled.
|
|
571
|
+
if (!current || toolStatus(current) !== event.status)
|
|
572
|
+
return;
|
|
573
|
+
await this.#providerSession.sendContext(event);
|
|
574
|
+
});
|
|
575
|
+
// Each caller records its own delivery failure; one failure must not poison later updates.
|
|
576
|
+
this.#toolStatusDelivery = delivery.catch(() => undefined);
|
|
577
|
+
return delivery;
|
|
578
|
+
}
|
|
579
|
+
#syncToolStatus(event, changed) {
|
|
580
|
+
if (!this.#capabilities.supportsToolStatusContext || this.#controller.signal.aborted)
|
|
581
|
+
return;
|
|
582
|
+
// A request may still be denied by admission. Report execution only after tool.started.
|
|
583
|
+
const call = event.type === "tool.started" ? this.#toolCalls.get(event.callId) : changed;
|
|
584
|
+
if (!call)
|
|
585
|
+
return;
|
|
586
|
+
if (event.type !== "tool.started" &&
|
|
587
|
+
event.type !== "tool.completed" &&
|
|
588
|
+
event.type !== "tool.failed" &&
|
|
589
|
+
event.type !== "tool.denied" &&
|
|
590
|
+
event.type !== "task.completed" &&
|
|
591
|
+
event.type !== "task.failed" &&
|
|
592
|
+
event.type !== "task.cancelled")
|
|
593
|
+
return;
|
|
594
|
+
const contextId = `tool-status:${event.id}`;
|
|
595
|
+
this.#toolStatusDelivery = this.#sendToolStatus({
|
|
596
|
+
type: "tool.status",
|
|
597
|
+
contextId,
|
|
598
|
+
sourceSequence: event.sequence,
|
|
599
|
+
call: { callId: call.callId, name: call.name, arguments: call.state.input },
|
|
600
|
+
status: toolStatus(call),
|
|
601
|
+
...(call.taskId === undefined ? {} : { taskId: call.taskId }),
|
|
602
|
+
}).catch(async (error) => {
|
|
603
|
+
if (this.#recordingEnded)
|
|
604
|
+
return;
|
|
605
|
+
await this.#record({
|
|
606
|
+
type: "context.delivery",
|
|
607
|
+
contextId,
|
|
608
|
+
callId: call.callId,
|
|
609
|
+
...(call.taskId === undefined ? {} : { taskId: call.taskId }),
|
|
610
|
+
kind: "thinking",
|
|
611
|
+
phase: "failed",
|
|
612
|
+
message: error instanceof Error ? error.message.slice(0, 2000) : "Tool status context failed",
|
|
613
|
+
});
|
|
614
|
+
});
|
|
615
|
+
void this.#toolStatusDelivery.catch(() => this.#closeFromInternal("storage_error"));
|
|
616
|
+
}
|
|
333
617
|
#assertOpen() {
|
|
618
|
+
if (this.#persistenceFailure)
|
|
619
|
+
throw this.#persistenceFailure;
|
|
620
|
+
if (this.#providerFailure)
|
|
621
|
+
throw this.#providerFailure;
|
|
334
622
|
if (this.#closePromise || this.#controller.signal.aborted) {
|
|
335
623
|
throw new Error("Voice session is closed");
|
|
336
624
|
}
|
|
337
625
|
}
|
|
338
626
|
}
|
|
627
|
+
function toolStatus(call) {
|
|
628
|
+
if (call.state.status === "running")
|
|
629
|
+
return "running";
|
|
630
|
+
if (call.state.outcome.type === "accepted_task")
|
|
631
|
+
return "background";
|
|
632
|
+
return call.state.outcome.type === "failed" ? "failed" : "completed";
|
|
633
|
+
}
|
|
339
634
|
function createUnsequencedEvent(sessionId, startedAt, event) {
|
|
340
635
|
return {
|
|
341
636
|
...event,
|
|
@@ -1,9 +1,8 @@
|
|
|
1
|
-
import type { OutputModality } from "../protocol/index.js";
|
|
2
|
-
import type { VoiceProvider } from "../provider/index.js";
|
|
3
1
|
import type { EventStore } from "../storage/index.js";
|
|
2
|
+
import type { SessionRuntimeLimits } from "./runtime-limits.js";
|
|
4
3
|
import type { ToolPolicy } from "./tool-policy.js";
|
|
5
4
|
import { ToolRegistry, type VoiceTool } from "./tools.js";
|
|
6
|
-
import { VoiceSession } from "./voice-session.js";
|
|
5
|
+
import { type CreateVoiceSessionOptions, VoiceSession } from "./voice-session.js";
|
|
7
6
|
export interface XOOptions {
|
|
8
7
|
store: EventStore;
|
|
9
8
|
tools?: readonly VoiceTool[];
|
|
@@ -14,21 +13,17 @@ export interface XOOptions {
|
|
|
14
13
|
* An integer of at least 128, or Infinity (the default). The log keeps the full outcome.
|
|
15
14
|
*/
|
|
16
15
|
maxToolResultBytes?: number;
|
|
16
|
+
runtimeLimits?: SessionRuntimeLimits;
|
|
17
17
|
}
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
/** Cancels provider startup and closes the live session when its owner is revoked. */
|
|
21
|
-
signal?: AbortSignal;
|
|
18
|
+
/** Session options share the low-level contract; XO supplies storage and the tool registry. */
|
|
19
|
+
export interface StartSessionOptions extends Omit<CreateVoiceSessionOptions, "sessionId" | "store" | "tools"> {
|
|
22
20
|
sessionId?: string;
|
|
23
21
|
/** Overrides the harness-level tool policy for this session. */
|
|
24
22
|
toolPolicy?: ToolPolicy;
|
|
25
23
|
/** Overrides the harness-level delivery budget for this session. */
|
|
26
24
|
maxToolResultBytes?: number;
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
outputModalities?: readonly OutputModality[];
|
|
30
|
-
/** Closes the session with max_duration_reached once elapsed. Guards runaway metered sessions. */
|
|
31
|
-
maxDurationMs?: number;
|
|
25
|
+
/** Overrides individual harness runtime limits for this session. */
|
|
26
|
+
runtimeLimits?: SessionRuntimeLimits;
|
|
32
27
|
}
|
|
33
28
|
export declare class XO {
|
|
34
29
|
#private;
|