xo-harness 0.1.0 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -0
- package/dist/internal/harness/task-supervisor.js +7 -1
- package/dist/internal/harness/tool-runtime.js +14 -2
- package/dist/internal/harness/voice-session.d.ts +14 -1
- package/dist/internal/harness/voice-session.js +85 -11
- package/dist/internal/harness/xo.d.ts +2 -0
- package/dist/internal/harness/xo.js +1 -0
- package/dist/internal/protocol/events.d.ts +9 -0
- package/dist/internal/protocol/events.js +2 -0
- package/dist/internal/protocol/provider.d.ts +8 -0
- package/dist/internal/protocol/provider.js +8 -0
- package/dist/internal/provider/contract.d.ts +3 -1
- package/dist/internal/provider/realtime-session.d.ts +2 -1
- package/dist/internal/provider/realtime-session.js +59 -5
- package/dist/internal/provider-fake/index.d.ts +3 -1
- package/dist/internal/provider-fake/index.js +5 -1
- package/dist/internal/provider-fake/replay-voice-provider.d.ts +3 -1
- package/dist/internal/provider-fake/replay-voice-provider.js +5 -1
- package/dist/internal/provider-fake/scripted-voice-provider.d.ts +5 -1
- package/dist/internal/provider-fake/scripted-voice-provider.js +4 -1
- package/dist/internal/storage/jsonl-event-store.js +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -17,6 +17,9 @@ const harness = new XO({ store: new JsonlEventStore(".xo/sessions") });
|
|
|
17
17
|
const session = await harness.startSession({
|
|
18
18
|
provider: new OpenAIRealtimeVoiceProvider({ apiKey: process.env.OPENAI_API_KEY! }),
|
|
19
19
|
});
|
|
20
|
+
|
|
21
|
+
// The model can open from its existing instructions/context; no fake user turn is added.
|
|
22
|
+
await session.requestAssistantTurn({ instructions: "Welcome the caller in one sentence." });
|
|
20
23
|
```
|
|
21
24
|
|
|
22
25
|
Additional entry points are available for focused imports:
|
|
@@ -149,8 +149,14 @@ export class TaskSupervisor {
|
|
|
149
149
|
const key = taskKey(event.taskId, event.callId);
|
|
150
150
|
if (this.#terminalTasks.has(key))
|
|
151
151
|
return false;
|
|
152
|
-
await this.#record(event);
|
|
153
152
|
this.#terminalTasks.add(key);
|
|
153
|
+
try {
|
|
154
|
+
await this.#record(event);
|
|
155
|
+
}
|
|
156
|
+
catch (error) {
|
|
157
|
+
this.#terminalTasks.delete(key);
|
|
158
|
+
throw error;
|
|
159
|
+
}
|
|
154
160
|
await this.#flush();
|
|
155
161
|
if (deliverContext)
|
|
156
162
|
await this.#deliverContext(event, "terminal");
|
|
@@ -54,15 +54,27 @@ export class ToolRuntime {
|
|
|
54
54
|
}
|
|
55
55
|
async #settleCompleted(callId, outcome) {
|
|
56
56
|
this.#assertNotTerminal(callId);
|
|
57
|
-
await this.#record({ type: "tool.completed", callId, outcome });
|
|
58
57
|
this.#terminal.add(callId);
|
|
58
|
+
try {
|
|
59
|
+
await this.#record({ type: "tool.completed", callId, outcome });
|
|
60
|
+
}
|
|
61
|
+
catch (error) {
|
|
62
|
+
this.#terminal.delete(callId);
|
|
63
|
+
throw error;
|
|
64
|
+
}
|
|
59
65
|
await this.#flush();
|
|
60
66
|
await this.#deliver(callId, outcome);
|
|
61
67
|
}
|
|
62
68
|
async #settleFailure(callId, message) {
|
|
63
69
|
this.#assertNotTerminal(callId);
|
|
64
|
-
await this.#record({ type: "tool.failed", callId, error: message });
|
|
65
70
|
this.#terminal.add(callId);
|
|
71
|
+
try {
|
|
72
|
+
await this.#record({ type: "tool.failed", callId, error: message });
|
|
73
|
+
}
|
|
74
|
+
catch (error) {
|
|
75
|
+
this.#terminal.delete(callId);
|
|
76
|
+
throw error;
|
|
77
|
+
}
|
|
66
78
|
await this.#flush();
|
|
67
79
|
await this.#deliver(callId, { type: "failed", error: message });
|
|
68
80
|
}
|
|
@@ -1,10 +1,12 @@
|
|
|
1
|
-
import { type AgentInput, type AudioChunk, type HarnessEvent, type OutputModality, type PlayoutProgress } from "../protocol/index.js";
|
|
1
|
+
import { type AgentInput, type AssistantTurnRequest, type AudioChunk, type HarnessEvent, type OutputModality, type PlayoutProgress } from "../protocol/index.js";
|
|
2
2
|
import type { VoiceProvider } from "../provider/index.js";
|
|
3
3
|
import type { EventStore } from "../storage/index.js";
|
|
4
4
|
import type { ToolRegistry } from "./tools.js";
|
|
5
5
|
export interface CreateVoiceSessionOptions {
|
|
6
6
|
sessionId: string;
|
|
7
7
|
provider: VoiceProvider;
|
|
8
|
+
/** Cancels provider startup and closes the live session when its owner is revoked. */
|
|
9
|
+
signal?: AbortSignal;
|
|
8
10
|
store: EventStore;
|
|
9
11
|
tools: ToolRegistry;
|
|
10
12
|
instructions?: string;
|
|
@@ -28,6 +30,17 @@ export declare class VoiceSession {
|
|
|
28
30
|
* the collapsed text is what goes over the wire (media-native providers wire later).
|
|
29
31
|
*/
|
|
30
32
|
sendText(input: AgentInput): Promise<void>;
|
|
33
|
+
/**
|
|
34
|
+
* Resolves only after the provider has acknowledged its session configuration.
|
|
35
|
+
* Hosts can apply their own timeout policy around this promise.
|
|
36
|
+
*/
|
|
37
|
+
waitUntilReady(): Promise<void>;
|
|
38
|
+
/**
|
|
39
|
+
* Begins an assistant turn from existing provider context. The request waits for
|
|
40
|
+
* provider.ready, is recorded without its potentially private guidance, and never
|
|
41
|
+
* creates a synthetic user message in the session log.
|
|
42
|
+
*/
|
|
43
|
+
requestAssistantTurn(request?: AssistantTurnRequest): Promise<void>;
|
|
31
44
|
reportPlayout(progress: PlayoutProgress): Promise<void>;
|
|
32
45
|
close(reason?: string): Promise<void>;
|
|
33
46
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { AgentInputSchema, AudioChunkSchema, agentInputToText, MessageInputTextSchema, PlayoutProgressSchema, ProviderEventSchema, } from "../protocol/index.js";
|
|
1
|
+
import { AgentInputSchema, AssistantTurnRequestSchema, AudioChunkSchema, agentInputToText, MessageInputTextSchema, PlayoutProgressSchema, ProviderEventSchema, } from "../protocol/index.js";
|
|
2
2
|
import { ReplayEventStream } from "./event-stream.js";
|
|
3
3
|
import { TaskSupervisor } from "./task-supervisor.js";
|
|
4
4
|
import { ToolRuntime } from "./tool-runtime.js";
|
|
@@ -13,6 +13,10 @@ export class VoiceSession {
|
|
|
13
13
|
#pendingRecords = new Set();
|
|
14
14
|
#tasks;
|
|
15
15
|
#toolRuntime;
|
|
16
|
+
#providerReadiness;
|
|
17
|
+
#settleProviderReadiness = () => undefined;
|
|
18
|
+
#providerReadinessSettled = false;
|
|
19
|
+
#detachOwnerAbort;
|
|
16
20
|
#providerPump;
|
|
17
21
|
#closePromise;
|
|
18
22
|
#maxDurationTimer;
|
|
@@ -22,13 +26,16 @@ export class VoiceSession {
|
|
|
22
26
|
this.#store = options.store;
|
|
23
27
|
this.#startedAt = startedAt;
|
|
24
28
|
this.#events.publish(startedEvent);
|
|
29
|
+
this.#providerReadiness = new Promise((resolve) => {
|
|
30
|
+
this.#settleProviderReadiness = resolve;
|
|
31
|
+
});
|
|
25
32
|
this.#tasks = new TaskSupervisor({
|
|
26
33
|
record: (event) => this.#record(event),
|
|
27
34
|
sendContext: (event) => this.#providerSession.sendContext(event),
|
|
28
35
|
flush: () => this.#store.flush(this.id),
|
|
29
36
|
sessionSignal: this.#controller.signal,
|
|
30
37
|
onError: () => {
|
|
31
|
-
|
|
38
|
+
this.#closeFromInternal("storage_error");
|
|
32
39
|
},
|
|
33
40
|
});
|
|
34
41
|
this.#toolRuntime = new ToolRuntime({
|
|
@@ -43,6 +50,10 @@ export class VoiceSession {
|
|
|
43
50
|
});
|
|
44
51
|
}
|
|
45
52
|
static async create(options) {
|
|
53
|
+
const existingEvents = await options.store.list(options.sessionId);
|
|
54
|
+
if (existingEvents.length > 0) {
|
|
55
|
+
throw new Error(`Session id already exists: ${options.sessionId}`);
|
|
56
|
+
}
|
|
46
57
|
const startedAt = performance.now();
|
|
47
58
|
const startedEvent = await options.store.append(createUnsequencedEvent(options.sessionId, startedAt, {
|
|
48
59
|
type: "session.started",
|
|
@@ -55,7 +66,15 @@ export class VoiceSession {
|
|
|
55
66
|
: { outputModalities: [...options.outputModalities] }),
|
|
56
67
|
}));
|
|
57
68
|
await options.store.flush(options.sessionId);
|
|
69
|
+
if (options.signal?.aborted) {
|
|
70
|
+
await recordFailedCreation(options.store, options.sessionId, startedAt, "owner_revoked");
|
|
71
|
+
throw options.signal.reason ?? new Error("Session owner revoked before startup");
|
|
72
|
+
}
|
|
58
73
|
const providerController = new AbortController();
|
|
74
|
+
const abortProviderFromOwner = () => providerController.abort(options.signal?.reason ?? "owner_revoked");
|
|
75
|
+
if (options.signal?.aborted)
|
|
76
|
+
abortProviderFromOwner();
|
|
77
|
+
options.signal?.addEventListener("abort", abortProviderFromOwner, { once: true });
|
|
59
78
|
let providerSession;
|
|
60
79
|
try {
|
|
61
80
|
providerSession = await options.provider.createSession({
|
|
@@ -67,18 +86,33 @@ export class VoiceSession {
|
|
|
67
86
|
});
|
|
68
87
|
}
|
|
69
88
|
catch (error) {
|
|
89
|
+
options.signal?.removeEventListener("abort", abortProviderFromOwner);
|
|
70
90
|
providerController.abort(error);
|
|
71
|
-
await recordFailedCreation(options.store, options.sessionId, startedAt);
|
|
91
|
+
await recordFailedCreation(options.store, options.sessionId, startedAt, options.signal?.aborted ? "owner_revoked" : "provider_create_failed");
|
|
72
92
|
throw error;
|
|
73
93
|
}
|
|
94
|
+
if (options.signal?.aborted) {
|
|
95
|
+
options.signal.removeEventListener("abort", abortProviderFromOwner);
|
|
96
|
+
await providerSession.close("owner_revoked").catch(() => undefined);
|
|
97
|
+
await recordFailedCreation(options.store, options.sessionId, startedAt, "owner_revoked");
|
|
98
|
+
throw options.signal.reason ?? new Error("Session owner revoked during startup");
|
|
99
|
+
}
|
|
74
100
|
const session = new VoiceSession(options, providerSession, startedAt, startedEvent);
|
|
75
101
|
session.#controller.signal.addEventListener("abort", () => providerController.abort(session.#controller.signal.reason), {
|
|
76
102
|
once: true,
|
|
77
103
|
});
|
|
104
|
+
options.signal?.removeEventListener("abort", abortProviderFromOwner);
|
|
105
|
+
if (options.signal) {
|
|
106
|
+
const abortSessionFromOwner = () => {
|
|
107
|
+
session.#closeFromInternal("owner_revoked");
|
|
108
|
+
};
|
|
109
|
+
options.signal.addEventListener("abort", abortSessionFromOwner, { once: true });
|
|
110
|
+
session.#detachOwnerAbort = () => options.signal?.removeEventListener("abort", abortSessionFromOwner);
|
|
111
|
+
}
|
|
78
112
|
session.#providerPump = session.#pumpProviderEvents();
|
|
79
113
|
if (options.maxDurationMs !== undefined) {
|
|
80
114
|
session.#maxDurationTimer = setTimeout(() => {
|
|
81
|
-
|
|
115
|
+
session.#closeFromInternal("max_duration_reached");
|
|
82
116
|
}, options.maxDurationMs);
|
|
83
117
|
}
|
|
84
118
|
return session;
|
|
@@ -121,6 +155,28 @@ export class VoiceSession {
|
|
|
121
155
|
});
|
|
122
156
|
await this.#providerSession.sendText(text);
|
|
123
157
|
}
|
|
158
|
+
/**
|
|
159
|
+
* Resolves only after the provider has acknowledged its session configuration.
|
|
160
|
+
* Hosts can apply their own timeout policy around this promise.
|
|
161
|
+
*/
|
|
162
|
+
async waitUntilReady() {
|
|
163
|
+
this.#assertOpen();
|
|
164
|
+
if (!(await this.#providerReadiness)) {
|
|
165
|
+
throw new Error("Voice provider closed before it became ready");
|
|
166
|
+
}
|
|
167
|
+
this.#assertOpen();
|
|
168
|
+
}
|
|
169
|
+
/**
|
|
170
|
+
* Begins an assistant turn from existing provider context. The request waits for
|
|
171
|
+
* provider.ready, is recorded without its potentially private guidance, and never
|
|
172
|
+
* creates a synthetic user message in the session log.
|
|
173
|
+
*/
|
|
174
|
+
async requestAssistantTurn(request = {}) {
|
|
175
|
+
const validated = AssistantTurnRequestSchema.parse(request);
|
|
176
|
+
await this.waitUntilReady();
|
|
177
|
+
await this.#record({ type: "assistant.turn.requested" });
|
|
178
|
+
await this.#providerSession.requestAssistantTurn(validated);
|
|
179
|
+
}
|
|
124
180
|
async reportPlayout(progress) {
|
|
125
181
|
this.#assertOpen();
|
|
126
182
|
const validated = PlayoutProgressSchema.parse(progress);
|
|
@@ -131,7 +187,12 @@ export class VoiceSession {
|
|
|
131
187
|
this.#closePromise ??= this.#performClose(reason);
|
|
132
188
|
return this.#closePromise;
|
|
133
189
|
}
|
|
190
|
+
#closeFromInternal(reason) {
|
|
191
|
+
void this.close(reason).catch(() => undefined);
|
|
192
|
+
}
|
|
134
193
|
async #performClose(reason) {
|
|
194
|
+
this.#detachOwnerAbort?.();
|
|
195
|
+
this.#detachOwnerAbort = undefined;
|
|
135
196
|
if (this.#maxDurationTimer)
|
|
136
197
|
clearTimeout(this.#maxDurationTimer);
|
|
137
198
|
this.#controller.abort(reason);
|
|
@@ -159,6 +220,7 @@ export class VoiceSession {
|
|
|
159
220
|
switch (event.type) {
|
|
160
221
|
case "ready":
|
|
161
222
|
await this.#record({ type: "provider.ready" });
|
|
223
|
+
this.#settleReadiness(true);
|
|
162
224
|
break;
|
|
163
225
|
case "audio.output":
|
|
164
226
|
// Pipelined: the append is enqueued (log position reserved) without stalling
|
|
@@ -197,12 +259,15 @@ export class VoiceSession {
|
|
|
197
259
|
message: event.message,
|
|
198
260
|
recoverable: event.recoverable,
|
|
199
261
|
});
|
|
200
|
-
if (!event.recoverable)
|
|
201
|
-
|
|
262
|
+
if (!event.recoverable) {
|
|
263
|
+
this.#settleReadiness(false);
|
|
264
|
+
this.#closeFromInternal("provider_error");
|
|
265
|
+
}
|
|
202
266
|
break;
|
|
203
267
|
case "closed":
|
|
268
|
+
this.#settleReadiness(false);
|
|
204
269
|
await this.#record({ type: "provider.closed", reason: event.reason });
|
|
205
|
-
|
|
270
|
+
this.#closeFromInternal("provider_closed");
|
|
206
271
|
break;
|
|
207
272
|
}
|
|
208
273
|
}
|
|
@@ -216,19 +281,28 @@ export class VoiceSession {
|
|
|
216
281
|
});
|
|
217
282
|
}
|
|
218
283
|
}
|
|
284
|
+
finally {
|
|
285
|
+
this.#settleReadiness(false);
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
#settleReadiness(ready) {
|
|
289
|
+
if (this.#providerReadinessSettled)
|
|
290
|
+
return;
|
|
291
|
+
this.#providerReadinessSettled = true;
|
|
292
|
+
this.#settleProviderReadiness(ready);
|
|
219
293
|
}
|
|
220
294
|
#trackToolExecution(execution) {
|
|
221
295
|
this.#toolExecutions.add(execution);
|
|
222
296
|
void execution.then(() => this.#toolExecutions.delete(execution), () => {
|
|
223
297
|
this.#toolExecutions.delete(execution);
|
|
224
|
-
|
|
298
|
+
this.#closeFromInternal("storage_error");
|
|
225
299
|
});
|
|
226
300
|
}
|
|
227
301
|
#trackRecord(record) {
|
|
228
302
|
this.#pendingRecords.add(record);
|
|
229
303
|
void record.then(() => this.#pendingRecords.delete(record), () => {
|
|
230
304
|
this.#pendingRecords.delete(record);
|
|
231
|
-
|
|
305
|
+
this.#closeFromInternal("storage_error");
|
|
232
306
|
});
|
|
233
307
|
}
|
|
234
308
|
async #record(event) {
|
|
@@ -251,11 +325,11 @@ function createUnsequencedEvent(sessionId, startedAt, event) {
|
|
|
251
325
|
sessionTimeMs: performance.now() - startedAt,
|
|
252
326
|
};
|
|
253
327
|
}
|
|
254
|
-
async function recordFailedCreation(store, sessionId, startedAt) {
|
|
328
|
+
async function recordFailedCreation(store, sessionId, startedAt, reason) {
|
|
255
329
|
try {
|
|
256
330
|
await store.append(createUnsequencedEvent(sessionId, startedAt, {
|
|
257
331
|
type: "session.ended",
|
|
258
|
-
reason
|
|
332
|
+
reason,
|
|
259
333
|
}));
|
|
260
334
|
await store.flush(sessionId);
|
|
261
335
|
}
|
|
@@ -9,6 +9,8 @@ export interface XOOptions {
|
|
|
9
9
|
}
|
|
10
10
|
export interface StartSessionOptions {
|
|
11
11
|
provider: VoiceProvider;
|
|
12
|
+
/** Cancels provider startup and closes the live session when its owner is revoked. */
|
|
13
|
+
signal?: AbortSignal;
|
|
12
14
|
sessionId?: string;
|
|
13
15
|
instructions?: string;
|
|
14
16
|
/** What the model may produce this session; ["text"] disables audio output. Default: audio. */
|
|
@@ -14,6 +14,7 @@ export class XO {
|
|
|
14
14
|
return VoiceSession.create({
|
|
15
15
|
sessionId: options.sessionId ?? crypto.randomUUID(),
|
|
16
16
|
provider: options.provider,
|
|
17
|
+
...(options.signal === undefined ? {} : { signal: options.signal }),
|
|
17
18
|
store: this.store,
|
|
18
19
|
tools: this.tools,
|
|
19
20
|
...(options.instructions === undefined ? {} : { instructions: options.instructions }),
|
|
@@ -49,6 +49,15 @@ export declare const HarnessEventSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
49
49
|
correlationId: z.ZodOptional<z.ZodString>;
|
|
50
50
|
causationId: z.ZodOptional<z.ZodString>;
|
|
51
51
|
type: z.ZodLiteral<"provider.ready">;
|
|
52
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
53
|
+
id: z.ZodString;
|
|
54
|
+
sessionId: z.ZodString;
|
|
55
|
+
sequence: z.ZodInt;
|
|
56
|
+
recordedAtMs: z.ZodNumber;
|
|
57
|
+
sessionTimeMs: z.ZodNumber;
|
|
58
|
+
correlationId: z.ZodOptional<z.ZodString>;
|
|
59
|
+
causationId: z.ZodOptional<z.ZodString>;
|
|
60
|
+
type: z.ZodLiteral<"assistant.turn.requested">;
|
|
52
61
|
}, z.core.$strip>, z.ZodObject<{
|
|
53
62
|
id: z.ZodString;
|
|
54
63
|
sessionId: z.ZodString;
|
|
@@ -34,6 +34,8 @@ export const HarnessEventSchema = z.discriminatedUnion("type", [
|
|
|
34
34
|
EventBaseSchema.extend({ type: z.literal("session.ended"), reason: z.string().min(1) }),
|
|
35
35
|
/** The provider accepted the session configuration; media may now flow. */
|
|
36
36
|
EventBaseSchema.extend({ type: z.literal("provider.ready") }),
|
|
37
|
+
/** A host asked the ready provider to begin an assistant turn without fabricating user input. */
|
|
38
|
+
EventBaseSchema.extend({ type: z.literal("assistant.turn.requested") }),
|
|
37
39
|
/** The provider connection settled; the harness closes the session in response. */
|
|
38
40
|
EventBaseSchema.extend({ type: z.literal("provider.closed"), reason: z.string().min(1) }),
|
|
39
41
|
/** Provider-reported fault; recoverable errors leave the session running. */
|
|
@@ -22,6 +22,14 @@ export declare const OutputModalitySchema: z.ZodEnum<{
|
|
|
22
22
|
text: "text";
|
|
23
23
|
}>;
|
|
24
24
|
export type OutputModality = z.infer<typeof OutputModalitySchema>;
|
|
25
|
+
/**
|
|
26
|
+
* A host-initiated assistant turn. This is deliberately not a synthetic user
|
|
27
|
+
* message: the model responds from the existing conversation and session context.
|
|
28
|
+
*/
|
|
29
|
+
export declare const AssistantTurnRequestSchema: z.ZodObject<{
|
|
30
|
+
instructions: z.ZodOptional<z.ZodString>;
|
|
31
|
+
}, z.core.$strip>;
|
|
32
|
+
export type AssistantTurnRequest = z.infer<typeof AssistantTurnRequestSchema>;
|
|
25
33
|
/**
|
|
26
34
|
* Token accounting for one completed model response, as reported by the provider.
|
|
27
35
|
* cachedInputTokens is the slice of input served from the provider's prompt cache —
|
|
@@ -22,6 +22,14 @@ export const PlayoutProgressSchema = z.object({
|
|
|
22
22
|
});
|
|
23
23
|
/** What the model may produce; ["text"] runs a session without audio output. */
|
|
24
24
|
export const OutputModalitySchema = z.enum(["audio", "text"]);
|
|
25
|
+
/**
|
|
26
|
+
* A host-initiated assistant turn. This is deliberately not a synthetic user
|
|
27
|
+
* message: the model responds from the existing conversation and session context.
|
|
28
|
+
*/
|
|
29
|
+
export const AssistantTurnRequestSchema = z.object({
|
|
30
|
+
/** Per-response guidance; it overrides session instructions for this turn only. */
|
|
31
|
+
instructions: z.string().min(1).max(4_000).optional(),
|
|
32
|
+
});
|
|
25
33
|
/**
|
|
26
34
|
* Token accounting for one completed model response, as reported by the provider.
|
|
27
35
|
* cachedInputTokens is the slice of input served from the provider's prompt cache —
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { AudioChunk, OutputModality, PlayoutProgress, ProviderCapabilities, ProviderContextEvent, ProviderEvent, ProviderToolDefinition, ProviderToolResult } from "../protocol/index.js";
|
|
1
|
+
import type { AssistantTurnRequest, AudioChunk, OutputModality, PlayoutProgress, ProviderCapabilities, ProviderContextEvent, ProviderEvent, ProviderToolDefinition, ProviderToolResult } from "../protocol/index.js";
|
|
2
2
|
export interface CreateProviderSessionOptions {
|
|
3
3
|
sessionId: string;
|
|
4
4
|
signal: AbortSignal;
|
|
@@ -12,6 +12,8 @@ export interface VoiceProviderSession {
|
|
|
12
12
|
sendAudio(chunk: AudioChunk): Promise<void>;
|
|
13
13
|
/** Injects a typed user message into the conversation; the model responds as usual. */
|
|
14
14
|
sendText(text: string): Promise<void>;
|
|
15
|
+
/** Starts a model turn from existing context without adding a fake user message. */
|
|
16
|
+
requestAssistantTurn(request?: AssistantTurnRequest): Promise<void>;
|
|
15
17
|
reportPlayout(progress: PlayoutProgress): Promise<void>;
|
|
16
18
|
sendToolResult(result: ProviderToolResult): Promise<void>;
|
|
17
19
|
sendContext(event: ProviderContextEvent): Promise<void>;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { type AudioChunk, type PlayoutProgress, type ProviderContextEvent, type ProviderEvent, type ProviderToolDefinition, type ProviderToolResult } from "../protocol/index.js";
|
|
1
|
+
import { type AssistantTurnRequest, type AudioChunk, type PlayoutProgress, type ProviderContextEvent, type ProviderEvent, type ProviderToolDefinition, type ProviderToolResult } from "../protocol/index.js";
|
|
2
2
|
import type { CreateProviderSessionOptions, VoiceProviderSession } from "./contract.js";
|
|
3
3
|
import type { RealtimeSocket } from "./realtime-socket.js";
|
|
4
4
|
/**
|
|
@@ -22,6 +22,7 @@ export declare class RealtimeVoiceSession implements VoiceProviderSession {
|
|
|
22
22
|
get events(): AsyncIterable<ProviderEvent>;
|
|
23
23
|
sendAudio(chunk: AudioChunk): Promise<void>;
|
|
24
24
|
sendText(text: string): Promise<void>;
|
|
25
|
+
requestAssistantTurn(request?: AssistantTurnRequest): Promise<void>;
|
|
25
26
|
reportPlayout(progress: PlayoutProgress): Promise<void>;
|
|
26
27
|
sendToolResult(result: ProviderToolResult): Promise<void>;
|
|
27
28
|
sendContext(event: ProviderContextEvent): Promise<void>;
|
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
import { Buffer } from "node:buffer";
|
|
2
|
-
import { AsyncQueue, AudioChunkSchema, audioSampleCount, PlayoutProgressSchema, ProviderContextEventSchema, ProviderToolResultSchema, ToolCallSchema, } from "../protocol/index.js";
|
|
2
|
+
import { AssistantTurnRequestSchema, AsyncQueue, AudioChunkSchema, audioSampleCount, PlayoutProgressSchema, ProviderContextEventSchema, ProviderToolResultSchema, ToolCallSchema, } from "../protocol/index.js";
|
|
3
3
|
import { z } from "zod";
|
|
4
4
|
const FALLBACK_STREAM_ID = "assistant-output";
|
|
5
5
|
const CLOSE_REASON_BYTE_LIMIT = 120;
|
|
6
|
+
const INPUT_TRANSCRIPT_SETTLE_MS = 300;
|
|
7
|
+
const EMITTED_USER_TRANSCRIPT_CACHE_LIMIT = 128;
|
|
6
8
|
const ServerEnvelopeSchema = z.looseObject({ type: z.string().min(1) });
|
|
7
9
|
const OutputAudioDeltaSchema = z.looseObject({
|
|
8
10
|
item_id: z.string().min(1).optional(),
|
|
@@ -83,12 +85,15 @@ export class RealtimeVoiceSession {
|
|
|
83
85
|
#outputStreams = new Map();
|
|
84
86
|
#playedThrough = new Map();
|
|
85
87
|
#deliveredToolCalls = new Set();
|
|
88
|
+
#pendingUserTranscripts = new Map();
|
|
89
|
+
#emittedUserTranscripts = new Map();
|
|
86
90
|
#settled;
|
|
87
91
|
#resolveSettled;
|
|
88
92
|
#outputSampleRate;
|
|
89
93
|
#activeOutputItemId;
|
|
90
94
|
#responseActive = false;
|
|
91
95
|
#responseWanted = false;
|
|
96
|
+
#wantedResponseInstructions;
|
|
92
97
|
#ready = false;
|
|
93
98
|
#closed = false;
|
|
94
99
|
#closeReason;
|
|
@@ -127,6 +132,10 @@ export class RealtimeVoiceSession {
|
|
|
127
132
|
});
|
|
128
133
|
this.#requestResponse();
|
|
129
134
|
}
|
|
135
|
+
async requestAssistantTurn(request = {}) {
|
|
136
|
+
const validated = AssistantTurnRequestSchema.parse(request);
|
|
137
|
+
this.#requestResponse(validated.instructions);
|
|
138
|
+
}
|
|
130
139
|
async reportPlayout(progress) {
|
|
131
140
|
const validated = PlayoutProgressSchema.parse(progress);
|
|
132
141
|
this.#playedThrough.set(validated.streamId, validated.playedThroughSample);
|
|
@@ -269,8 +278,44 @@ export class RealtimeVoiceSession {
|
|
|
269
278
|
const transcript = TranscriptEventSchema.safeParse(parsed);
|
|
270
279
|
if (!transcript.success)
|
|
271
280
|
return;
|
|
281
|
+
if (role === "user" && transcript.data.item_id !== undefined) {
|
|
282
|
+
this.#queueUserTranscript(transcript.data.item_id, transcript.data.transcript);
|
|
283
|
+
return;
|
|
284
|
+
}
|
|
272
285
|
this.#emitTranscript(role, transcript.data.transcript, transcript.data.item_id);
|
|
273
286
|
}
|
|
287
|
+
/** xAI can send several "completed" revisions for one input item; emit only the settled text. */
|
|
288
|
+
#queueUserTranscript(streamId, rawText) {
|
|
289
|
+
const text = rawText.trim();
|
|
290
|
+
if (text === "")
|
|
291
|
+
return;
|
|
292
|
+
const pending = this.#pendingUserTranscripts.get(streamId);
|
|
293
|
+
if (pending)
|
|
294
|
+
clearTimeout(pending.timer);
|
|
295
|
+
const timer = setTimeout(() => this.#flushUserTranscript(streamId), INPUT_TRANSCRIPT_SETTLE_MS);
|
|
296
|
+
this.#pendingUserTranscripts.set(streamId, { text, timer });
|
|
297
|
+
}
|
|
298
|
+
#flushUserTranscript(streamId) {
|
|
299
|
+
const pending = this.#pendingUserTranscripts.get(streamId);
|
|
300
|
+
if (!pending)
|
|
301
|
+
return;
|
|
302
|
+
clearTimeout(pending.timer);
|
|
303
|
+
this.#pendingUserTranscripts.delete(streamId);
|
|
304
|
+
if (this.#emittedUserTranscripts.get(streamId) === pending.text)
|
|
305
|
+
return;
|
|
306
|
+
this.#emittedUserTranscripts.delete(streamId);
|
|
307
|
+
this.#emittedUserTranscripts.set(streamId, pending.text);
|
|
308
|
+
if (this.#emittedUserTranscripts.size > EMITTED_USER_TRANSCRIPT_CACHE_LIMIT) {
|
|
309
|
+
const oldestStreamId = this.#emittedUserTranscripts.keys().next().value;
|
|
310
|
+
if (oldestStreamId !== undefined)
|
|
311
|
+
this.#emittedUserTranscripts.delete(oldestStreamId);
|
|
312
|
+
}
|
|
313
|
+
this.#emitTranscript("user", pending.text, streamId);
|
|
314
|
+
}
|
|
315
|
+
#flushUserTranscripts() {
|
|
316
|
+
for (const streamId of [...this.#pendingUserTranscripts.keys()])
|
|
317
|
+
this.#flushUserTranscript(streamId);
|
|
318
|
+
}
|
|
274
319
|
#handleTranscriptDelta(parsed) {
|
|
275
320
|
const delta = TranscriptDeltaWireSchema.safeParse(parsed);
|
|
276
321
|
if (!delta.success || delta.data.delta === "")
|
|
@@ -319,8 +364,11 @@ export class RealtimeVoiceSession {
|
|
|
319
364
|
this.#emitFunctionCall(call.data.call_id, call.data.name, call.data.arguments);
|
|
320
365
|
}
|
|
321
366
|
}
|
|
322
|
-
if (this.#responseWanted)
|
|
323
|
-
this.#
|
|
367
|
+
if (this.#responseWanted) {
|
|
368
|
+
const instructions = this.#wantedResponseInstructions;
|
|
369
|
+
this.#wantedResponseInstructions = undefined;
|
|
370
|
+
this.#requestResponse(instructions);
|
|
371
|
+
}
|
|
324
372
|
}
|
|
325
373
|
#emitUsage(usage) {
|
|
326
374
|
const input = usage.input_token_details;
|
|
@@ -379,14 +427,19 @@ export class RealtimeVoiceSession {
|
|
|
379
427
|
const message = error.success ? error.data.error?.message : undefined;
|
|
380
428
|
this.#emitError(message ?? `${this.#wire.label} reported an error`, true);
|
|
381
429
|
}
|
|
382
|
-
#requestResponse() {
|
|
430
|
+
#requestResponse(instructions) {
|
|
383
431
|
if (this.#responseActive) {
|
|
384
432
|
this.#responseWanted = true;
|
|
433
|
+
if (instructions !== undefined)
|
|
434
|
+
this.#wantedResponseInstructions = instructions;
|
|
385
435
|
return;
|
|
386
436
|
}
|
|
387
437
|
this.#responseWanted = false;
|
|
388
438
|
this.#responseActive = true;
|
|
389
|
-
this.#trySend({
|
|
439
|
+
this.#trySend({
|
|
440
|
+
type: "response.create",
|
|
441
|
+
...(instructions === undefined ? {} : { response: { instructions } }),
|
|
442
|
+
});
|
|
390
443
|
}
|
|
391
444
|
#send(payload) {
|
|
392
445
|
if (this.#socket.state !== "open") {
|
|
@@ -411,6 +464,7 @@ export class RealtimeVoiceSession {
|
|
|
411
464
|
#settle(reason) {
|
|
412
465
|
if (this.#closed)
|
|
413
466
|
return;
|
|
467
|
+
this.#flushUserTranscripts();
|
|
414
468
|
this.#closed = true;
|
|
415
469
|
this.#queue.push({ type: "closed", reason });
|
|
416
470
|
this.#queue.close();
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { type AudioChunk, type OutputModality, type PlayoutProgress, type ProviderContextEvent, type ProviderEvent, type ProviderToolDefinition, type ProviderToolResult, type ToolCall } from "../protocol/index.js";
|
|
1
|
+
import { type AssistantTurnRequest, type AudioChunk, type OutputModality, type PlayoutProgress, type ProviderContextEvent, type ProviderEvent, type ProviderToolDefinition, type ProviderToolResult, type ToolCall } from "../protocol/index.js";
|
|
2
2
|
import type { CreateProviderSessionOptions, VoiceProvider, VoiceProviderSession } from "../provider/index.js";
|
|
3
3
|
export * from "./replay-voice-provider.js";
|
|
4
4
|
export * from "./scripted-voice-provider.js";
|
|
@@ -24,6 +24,7 @@ export declare class FakeVoiceProviderSession implements VoiceProviderSession {
|
|
|
24
24
|
#private;
|
|
25
25
|
readonly receivedAudio: AudioChunk[];
|
|
26
26
|
readonly receivedTexts: string[];
|
|
27
|
+
readonly assistantTurnRequests: AssistantTurnRequest[];
|
|
27
28
|
readonly playoutProgress: PlayoutProgress[];
|
|
28
29
|
readonly toolResults: ProviderToolResult[];
|
|
29
30
|
readonly contextEvents: ProviderContextEvent[];
|
|
@@ -35,6 +36,7 @@ export declare class FakeVoiceProviderSession implements VoiceProviderSession {
|
|
|
35
36
|
get closed(): boolean;
|
|
36
37
|
sendAudio(chunk: AudioChunk): Promise<void>;
|
|
37
38
|
sendText(text: string): Promise<void>;
|
|
39
|
+
requestAssistantTurn(request?: AssistantTurnRequest): Promise<void>;
|
|
38
40
|
reportPlayout(progress: PlayoutProgress): Promise<void>;
|
|
39
41
|
sendToolResult(result: ProviderToolResult): Promise<void>;
|
|
40
42
|
sendContext(event: ProviderContextEvent): Promise<void>;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { AsyncQueue, AudioChunkSchema, PlayoutProgressSchema, ProviderContextEventSchema, ProviderEventSchema, ProviderToolResultSchema, } from "../protocol/index.js";
|
|
1
|
+
import { AssistantTurnRequestSchema, AsyncQueue, AudioChunkSchema, PlayoutProgressSchema, ProviderContextEventSchema, ProviderEventSchema, ProviderToolResultSchema, } from "../protocol/index.js";
|
|
2
2
|
export * from "./replay-voice-provider.js";
|
|
3
3
|
export * from "./scripted-voice-provider.js";
|
|
4
4
|
export class FakeVoiceProvider {
|
|
@@ -34,6 +34,7 @@ export class FakeVoiceProvider {
|
|
|
34
34
|
export class FakeVoiceProviderSession {
|
|
35
35
|
receivedAudio = [];
|
|
36
36
|
receivedTexts = [];
|
|
37
|
+
assistantTurnRequests = [];
|
|
37
38
|
playoutProgress = [];
|
|
38
39
|
toolResults = [];
|
|
39
40
|
contextEvents = [];
|
|
@@ -81,6 +82,9 @@ export class FakeVoiceProviderSession {
|
|
|
81
82
|
this.emitTranscript("assistant", text);
|
|
82
83
|
}
|
|
83
84
|
}
|
|
85
|
+
async requestAssistantTurn(request = {}) {
|
|
86
|
+
this.assistantTurnRequests.push(AssistantTurnRequestSchema.parse(request));
|
|
87
|
+
}
|
|
84
88
|
async reportPlayout(progress) {
|
|
85
89
|
this.playoutProgress.push(PlayoutProgressSchema.parse(progress));
|
|
86
90
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { type AudioChunk, type HarnessEvent, type PlayoutProgress, type ProviderContextEvent, type ProviderEvent, type ProviderToolResult } from "../protocol/index.js";
|
|
1
|
+
import { type AssistantTurnRequest, type AudioChunk, type HarnessEvent, type PlayoutProgress, type ProviderContextEvent, type ProviderEvent, type ProviderToolResult } from "../protocol/index.js";
|
|
2
2
|
import type { CreateProviderSessionOptions, VoiceProvider, VoiceProviderSession } from "../provider/index.js";
|
|
3
3
|
export interface ReplayVoiceProviderOptions {
|
|
4
4
|
/** Recorded harness events to replay. Takes precedence over load. */
|
|
@@ -32,6 +32,7 @@ export declare class ReplayVoiceProviderSession implements VoiceProviderSession
|
|
|
32
32
|
#private;
|
|
33
33
|
readonly receivedAudio: AudioChunk[];
|
|
34
34
|
readonly receivedTexts: string[];
|
|
35
|
+
readonly assistantTurnRequests: AssistantTurnRequest[];
|
|
35
36
|
readonly playoutProgress: PlayoutProgress[];
|
|
36
37
|
readonly toolResults: ProviderToolResult[];
|
|
37
38
|
readonly contextEvents: ProviderContextEvent[];
|
|
@@ -40,6 +41,7 @@ export declare class ReplayVoiceProviderSession implements VoiceProviderSession
|
|
|
40
41
|
get closed(): boolean;
|
|
41
42
|
sendAudio(chunk: AudioChunk): Promise<void>;
|
|
42
43
|
sendText(text: string): Promise<void>;
|
|
44
|
+
requestAssistantTurn(request?: AssistantTurnRequest): Promise<void>;
|
|
43
45
|
reportPlayout(progress: PlayoutProgress): Promise<void>;
|
|
44
46
|
sendToolResult(result: ProviderToolResult): Promise<void>;
|
|
45
47
|
sendContext(event: ProviderContextEvent): Promise<void>;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { AsyncQueue, AudioChunkSchema, PlayoutProgressSchema, ProviderContextEventSchema, ProviderToolResultSchema, } from "../protocol/index.js";
|
|
1
|
+
import { AssistantTurnRequestSchema, AsyncQueue, AudioChunkSchema, PlayoutProgressSchema, ProviderContextEventSchema, ProviderToolResultSchema, } from "../protocol/index.js";
|
|
2
2
|
/**
|
|
3
3
|
* Replays the provider-visible half of a recorded session: model audio, tool requests,
|
|
4
4
|
* and provider errors, in recorded order and (optionally) recorded pacing. Input audio,
|
|
@@ -30,6 +30,7 @@ export class ReplayVoiceProvider {
|
|
|
30
30
|
export class ReplayVoiceProviderSession {
|
|
31
31
|
receivedAudio = [];
|
|
32
32
|
receivedTexts = [];
|
|
33
|
+
assistantTurnRequests = [];
|
|
33
34
|
playoutProgress = [];
|
|
34
35
|
toolResults = [];
|
|
35
36
|
contextEvents = [];
|
|
@@ -54,6 +55,9 @@ export class ReplayVoiceProviderSession {
|
|
|
54
55
|
async sendText(text) {
|
|
55
56
|
this.receivedTexts.push(text);
|
|
56
57
|
}
|
|
58
|
+
async requestAssistantTurn(request = {}) {
|
|
59
|
+
this.assistantTurnRequests.push(AssistantTurnRequestSchema.parse(request));
|
|
60
|
+
}
|
|
57
61
|
async reportPlayout(progress) {
|
|
58
62
|
this.playoutProgress.push(PlayoutProgressSchema.parse(progress));
|
|
59
63
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { type AudioChunk, type OutputModality, type PlayoutProgress, type ProviderContextEvent, type ProviderEvent, type ProviderToolDefinition, type ProviderToolResult } from "../protocol/index.js";
|
|
1
|
+
import { type AssistantTurnRequest, type AudioChunk, type OutputModality, type PlayoutProgress, type ProviderContextEvent, type ProviderEvent, type ProviderToolDefinition, type ProviderToolResult } from "../protocol/index.js";
|
|
2
2
|
import type { CreateProviderSessionOptions, VoiceProvider, VoiceProviderSession } from "../provider/index.js";
|
|
3
3
|
export type ScriptedVoiceStep = {
|
|
4
4
|
type: "emit";
|
|
@@ -9,6 +9,9 @@ export type ScriptedVoiceStep = {
|
|
|
9
9
|
} | {
|
|
10
10
|
type: "expect.text";
|
|
11
11
|
value: string;
|
|
12
|
+
} | {
|
|
13
|
+
type: "expect.assistant_turn";
|
|
14
|
+
value: AssistantTurnRequest;
|
|
12
15
|
} | {
|
|
13
16
|
type: "expect.playout";
|
|
14
17
|
value: PlayoutProgress;
|
|
@@ -54,6 +57,7 @@ export declare class ScriptedVoiceProviderSession implements VoiceProviderSessio
|
|
|
54
57
|
completed(): Promise<void>;
|
|
55
58
|
sendAudio(chunk: AudioChunk): Promise<void>;
|
|
56
59
|
sendText(text: string): Promise<void>;
|
|
60
|
+
requestAssistantTurn(request?: AssistantTurnRequest): Promise<void>;
|
|
57
61
|
reportPlayout(progress: PlayoutProgress): Promise<void>;
|
|
58
62
|
sendToolResult(result: ProviderToolResult): Promise<void>;
|
|
59
63
|
sendContext(event: ProviderContextEvent): Promise<void>;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { AsyncQueue, AudioChunkSchema, PlayoutProgressSchema, ProviderContextEventSchema, ProviderEventSchema, ProviderToolResultSchema, } from "../protocol/index.js";
|
|
1
|
+
import { AssistantTurnRequestSchema, AsyncQueue, AudioChunkSchema, PlayoutProgressSchema, ProviderContextEventSchema, ProviderEventSchema, ProviderToolResultSchema, } from "../protocol/index.js";
|
|
2
2
|
/**
|
|
3
3
|
* A closed-loop deterministic provider. Unlike recorded replay, scripts can stop after
|
|
4
4
|
* emitting a tool call and verify the exact result before the next model event appears.
|
|
@@ -84,6 +84,9 @@ export class ScriptedVoiceProviderSession {
|
|
|
84
84
|
async sendText(text) {
|
|
85
85
|
this.#accept("expect.text", text);
|
|
86
86
|
}
|
|
87
|
+
async requestAssistantTurn(request = {}) {
|
|
88
|
+
this.#accept("expect.assistant_turn", AssistantTurnRequestSchema.parse(request));
|
|
89
|
+
}
|
|
87
90
|
async reportPlayout(progress) {
|
|
88
91
|
this.#accept("expect.playout", PlayoutProgressSchema.parse(progress));
|
|
89
92
|
}
|
|
@@ -28,7 +28,7 @@ export class JsonlEventStore {
|
|
|
28
28
|
this.#assertSessionId(sessionId);
|
|
29
29
|
return this.#tails.run(sessionId, async () => {
|
|
30
30
|
try {
|
|
31
|
-
const file = await open(this.#sessionPath(sessionId), "r");
|
|
31
|
+
const file = await open(this.#sessionPath(sessionId), "r+");
|
|
32
32
|
try {
|
|
33
33
|
await file.sync();
|
|
34
34
|
}
|