xo-harness 0.1.1 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -0
- package/dist/internal/harness/voice-session.d.ts +12 -1
- package/dist/internal/harness/voice-session.js +43 -2
- package/dist/internal/protocol/events.d.ts +9 -0
- package/dist/internal/protocol/events.js +2 -0
- package/dist/internal/protocol/provider.d.ts +8 -0
- package/dist/internal/protocol/provider.js +8 -0
- package/dist/internal/provider/contract.d.ts +3 -1
- package/dist/internal/provider/realtime-session.d.ts +2 -1
- package/dist/internal/provider/realtime-session.js +59 -5
- package/dist/internal/provider-fake/index.d.ts +3 -1
- package/dist/internal/provider-fake/index.js +5 -1
- package/dist/internal/provider-fake/replay-voice-provider.d.ts +3 -1
- package/dist/internal/provider-fake/replay-voice-provider.js +5 -1
- package/dist/internal/provider-fake/scripted-voice-provider.d.ts +5 -1
- package/dist/internal/provider-fake/scripted-voice-provider.js +4 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -17,6 +17,9 @@ const harness = new XO({ store: new JsonlEventStore(".xo/sessions") });
|
|
|
17
17
|
const session = await harness.startSession({
|
|
18
18
|
provider: new OpenAIRealtimeVoiceProvider({ apiKey: process.env.OPENAI_API_KEY! }),
|
|
19
19
|
});
|
|
20
|
+
|
|
21
|
+
// The model can open from its existing instructions/context; no fake user turn is added.
|
|
22
|
+
await session.requestAssistantTurn({ instructions: "Welcome the caller in one sentence." });
|
|
20
23
|
```
|
|
21
24
|
|
|
22
25
|
Additional entry points are available for focused imports:
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { type AgentInput, type AudioChunk, type HarnessEvent, type OutputModality, type PlayoutProgress } from "../protocol/index.js";
|
|
1
|
+
import { type AgentInput, type AssistantTurnRequest, type AudioChunk, type HarnessEvent, type OutputModality, type PlayoutProgress } from "../protocol/index.js";
|
|
2
2
|
import type { VoiceProvider } from "../provider/index.js";
|
|
3
3
|
import type { EventStore } from "../storage/index.js";
|
|
4
4
|
import type { ToolRegistry } from "./tools.js";
|
|
@@ -30,6 +30,17 @@ export declare class VoiceSession {
|
|
|
30
30
|
* the collapsed text is what goes over the wire (media-native providers wire later).
|
|
31
31
|
*/
|
|
32
32
|
sendText(input: AgentInput): Promise<void>;
|
|
33
|
+
/**
|
|
34
|
+
* Resolves only after the provider has acknowledged its session configuration.
|
|
35
|
+
* Hosts can apply their own timeout policy around this promise.
|
|
36
|
+
*/
|
|
37
|
+
waitUntilReady(): Promise<void>;
|
|
38
|
+
/**
|
|
39
|
+
* Begins an assistant turn from existing provider context. The request waits for
|
|
40
|
+
* provider.ready, is recorded without its potentially private guidance, and never
|
|
41
|
+
* creates a synthetic user message in the session log.
|
|
42
|
+
*/
|
|
43
|
+
requestAssistantTurn(request?: AssistantTurnRequest): Promise<void>;
|
|
33
44
|
reportPlayout(progress: PlayoutProgress): Promise<void>;
|
|
34
45
|
close(reason?: string): Promise<void>;
|
|
35
46
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { AgentInputSchema, AudioChunkSchema, agentInputToText, MessageInputTextSchema, PlayoutProgressSchema, ProviderEventSchema, } from "../protocol/index.js";
|
|
1
|
+
import { AgentInputSchema, AssistantTurnRequestSchema, AudioChunkSchema, agentInputToText, MessageInputTextSchema, PlayoutProgressSchema, ProviderEventSchema, } from "../protocol/index.js";
|
|
2
2
|
import { ReplayEventStream } from "./event-stream.js";
|
|
3
3
|
import { TaskSupervisor } from "./task-supervisor.js";
|
|
4
4
|
import { ToolRuntime } from "./tool-runtime.js";
|
|
@@ -13,6 +13,9 @@ export class VoiceSession {
|
|
|
13
13
|
#pendingRecords = new Set();
|
|
14
14
|
#tasks;
|
|
15
15
|
#toolRuntime;
|
|
16
|
+
#providerReadiness;
|
|
17
|
+
#settleProviderReadiness = () => undefined;
|
|
18
|
+
#providerReadinessSettled = false;
|
|
16
19
|
#detachOwnerAbort;
|
|
17
20
|
#providerPump;
|
|
18
21
|
#closePromise;
|
|
@@ -23,6 +26,9 @@ export class VoiceSession {
|
|
|
23
26
|
this.#store = options.store;
|
|
24
27
|
this.#startedAt = startedAt;
|
|
25
28
|
this.#events.publish(startedEvent);
|
|
29
|
+
this.#providerReadiness = new Promise((resolve) => {
|
|
30
|
+
this.#settleProviderReadiness = resolve;
|
|
31
|
+
});
|
|
26
32
|
this.#tasks = new TaskSupervisor({
|
|
27
33
|
record: (event) => this.#record(event),
|
|
28
34
|
sendContext: (event) => this.#providerSession.sendContext(event),
|
|
@@ -149,6 +155,28 @@ export class VoiceSession {
|
|
|
149
155
|
});
|
|
150
156
|
await this.#providerSession.sendText(text);
|
|
151
157
|
}
|
|
158
|
+
/**
|
|
159
|
+
* Resolves only after the provider has acknowledged its session configuration.
|
|
160
|
+
* Hosts can apply their own timeout policy around this promise.
|
|
161
|
+
*/
|
|
162
|
+
async waitUntilReady() {
|
|
163
|
+
this.#assertOpen();
|
|
164
|
+
if (!(await this.#providerReadiness)) {
|
|
165
|
+
throw new Error("Voice provider closed before it became ready");
|
|
166
|
+
}
|
|
167
|
+
this.#assertOpen();
|
|
168
|
+
}
|
|
169
|
+
/**
|
|
170
|
+
* Begins an assistant turn from existing provider context. The request waits for
|
|
171
|
+
* provider.ready, is recorded without its potentially private guidance, and never
|
|
172
|
+
* creates a synthetic user message in the session log.
|
|
173
|
+
*/
|
|
174
|
+
async requestAssistantTurn(request = {}) {
|
|
175
|
+
const validated = AssistantTurnRequestSchema.parse(request);
|
|
176
|
+
await this.waitUntilReady();
|
|
177
|
+
await this.#record({ type: "assistant.turn.requested" });
|
|
178
|
+
await this.#providerSession.requestAssistantTurn(validated);
|
|
179
|
+
}
|
|
152
180
|
async reportPlayout(progress) {
|
|
153
181
|
this.#assertOpen();
|
|
154
182
|
const validated = PlayoutProgressSchema.parse(progress);
|
|
@@ -192,6 +220,7 @@ export class VoiceSession {
|
|
|
192
220
|
switch (event.type) {
|
|
193
221
|
case "ready":
|
|
194
222
|
await this.#record({ type: "provider.ready" });
|
|
223
|
+
this.#settleReadiness(true);
|
|
195
224
|
break;
|
|
196
225
|
case "audio.output":
|
|
197
226
|
// Pipelined: the append is enqueued (log position reserved) without stalling
|
|
@@ -230,10 +259,13 @@ export class VoiceSession {
|
|
|
230
259
|
message: event.message,
|
|
231
260
|
recoverable: event.recoverable,
|
|
232
261
|
});
|
|
233
|
-
if (!event.recoverable)
|
|
262
|
+
if (!event.recoverable) {
|
|
263
|
+
this.#settleReadiness(false);
|
|
234
264
|
this.#closeFromInternal("provider_error");
|
|
265
|
+
}
|
|
235
266
|
break;
|
|
236
267
|
case "closed":
|
|
268
|
+
this.#settleReadiness(false);
|
|
237
269
|
await this.#record({ type: "provider.closed", reason: event.reason });
|
|
238
270
|
this.#closeFromInternal("provider_closed");
|
|
239
271
|
break;
|
|
@@ -249,6 +281,15 @@ export class VoiceSession {
|
|
|
249
281
|
});
|
|
250
282
|
}
|
|
251
283
|
}
|
|
284
|
+
finally {
|
|
285
|
+
this.#settleReadiness(false);
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
#settleReadiness(ready) {
|
|
289
|
+
if (this.#providerReadinessSettled)
|
|
290
|
+
return;
|
|
291
|
+
this.#providerReadinessSettled = true;
|
|
292
|
+
this.#settleProviderReadiness(ready);
|
|
252
293
|
}
|
|
253
294
|
#trackToolExecution(execution) {
|
|
254
295
|
this.#toolExecutions.add(execution);
|
|
@@ -49,6 +49,15 @@ export declare const HarnessEventSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
49
49
|
correlationId: z.ZodOptional<z.ZodString>;
|
|
50
50
|
causationId: z.ZodOptional<z.ZodString>;
|
|
51
51
|
type: z.ZodLiteral<"provider.ready">;
|
|
52
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
53
|
+
id: z.ZodString;
|
|
54
|
+
sessionId: z.ZodString;
|
|
55
|
+
sequence: z.ZodInt;
|
|
56
|
+
recordedAtMs: z.ZodNumber;
|
|
57
|
+
sessionTimeMs: z.ZodNumber;
|
|
58
|
+
correlationId: z.ZodOptional<z.ZodString>;
|
|
59
|
+
causationId: z.ZodOptional<z.ZodString>;
|
|
60
|
+
type: z.ZodLiteral<"assistant.turn.requested">;
|
|
52
61
|
}, z.core.$strip>, z.ZodObject<{
|
|
53
62
|
id: z.ZodString;
|
|
54
63
|
sessionId: z.ZodString;
|
|
@@ -34,6 +34,8 @@ export const HarnessEventSchema = z.discriminatedUnion("type", [
|
|
|
34
34
|
EventBaseSchema.extend({ type: z.literal("session.ended"), reason: z.string().min(1) }),
|
|
35
35
|
/** The provider accepted the session configuration; media may now flow. */
|
|
36
36
|
EventBaseSchema.extend({ type: z.literal("provider.ready") }),
|
|
37
|
+
/** A host asked the ready provider to begin an assistant turn without fabricating user input. */
|
|
38
|
+
EventBaseSchema.extend({ type: z.literal("assistant.turn.requested") }),
|
|
37
39
|
/** The provider connection settled; the harness closes the session in response. */
|
|
38
40
|
EventBaseSchema.extend({ type: z.literal("provider.closed"), reason: z.string().min(1) }),
|
|
39
41
|
/** Provider-reported fault; recoverable errors leave the session running. */
|
|
@@ -22,6 +22,14 @@ export declare const OutputModalitySchema: z.ZodEnum<{
|
|
|
22
22
|
text: "text";
|
|
23
23
|
}>;
|
|
24
24
|
export type OutputModality = z.infer<typeof OutputModalitySchema>;
|
|
25
|
+
/**
|
|
26
|
+
* A host-initiated assistant turn. This is deliberately not a synthetic user
|
|
27
|
+
* message: the model responds from the existing conversation and session context.
|
|
28
|
+
*/
|
|
29
|
+
export declare const AssistantTurnRequestSchema: z.ZodObject<{
|
|
30
|
+
instructions: z.ZodOptional<z.ZodString>;
|
|
31
|
+
}, z.core.$strip>;
|
|
32
|
+
export type AssistantTurnRequest = z.infer<typeof AssistantTurnRequestSchema>;
|
|
25
33
|
/**
|
|
26
34
|
* Token accounting for one completed model response, as reported by the provider.
|
|
27
35
|
* cachedInputTokens is the slice of input served from the provider's prompt cache —
|
|
@@ -22,6 +22,14 @@ export const PlayoutProgressSchema = z.object({
|
|
|
22
22
|
});
|
|
23
23
|
/** What the model may produce; ["text"] runs a session without audio output. */
|
|
24
24
|
export const OutputModalitySchema = z.enum(["audio", "text"]);
|
|
25
|
+
/**
|
|
26
|
+
* A host-initiated assistant turn. This is deliberately not a synthetic user
|
|
27
|
+
* message: the model responds from the existing conversation and session context.
|
|
28
|
+
*/
|
|
29
|
+
export const AssistantTurnRequestSchema = z.object({
|
|
30
|
+
/** Per-response guidance; it overrides session instructions for this turn only. */
|
|
31
|
+
instructions: z.string().min(1).max(4_000).optional(),
|
|
32
|
+
});
|
|
25
33
|
/**
|
|
26
34
|
* Token accounting for one completed model response, as reported by the provider.
|
|
27
35
|
* cachedInputTokens is the slice of input served from the provider's prompt cache —
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { AudioChunk, OutputModality, PlayoutProgress, ProviderCapabilities, ProviderContextEvent, ProviderEvent, ProviderToolDefinition, ProviderToolResult } from "../protocol/index.js";
|
|
1
|
+
import type { AssistantTurnRequest, AudioChunk, OutputModality, PlayoutProgress, ProviderCapabilities, ProviderContextEvent, ProviderEvent, ProviderToolDefinition, ProviderToolResult } from "../protocol/index.js";
|
|
2
2
|
export interface CreateProviderSessionOptions {
|
|
3
3
|
sessionId: string;
|
|
4
4
|
signal: AbortSignal;
|
|
@@ -12,6 +12,8 @@ export interface VoiceProviderSession {
|
|
|
12
12
|
sendAudio(chunk: AudioChunk): Promise<void>;
|
|
13
13
|
/** Injects a typed user message into the conversation; the model responds as usual. */
|
|
14
14
|
sendText(text: string): Promise<void>;
|
|
15
|
+
/** Starts a model turn from existing context without adding a fake user message. */
|
|
16
|
+
requestAssistantTurn(request?: AssistantTurnRequest): Promise<void>;
|
|
15
17
|
reportPlayout(progress: PlayoutProgress): Promise<void>;
|
|
16
18
|
sendToolResult(result: ProviderToolResult): Promise<void>;
|
|
17
19
|
sendContext(event: ProviderContextEvent): Promise<void>;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { type AudioChunk, type PlayoutProgress, type ProviderContextEvent, type ProviderEvent, type ProviderToolDefinition, type ProviderToolResult } from "../protocol/index.js";
|
|
1
|
+
import { type AssistantTurnRequest, type AudioChunk, type PlayoutProgress, type ProviderContextEvent, type ProviderEvent, type ProviderToolDefinition, type ProviderToolResult } from "../protocol/index.js";
|
|
2
2
|
import type { CreateProviderSessionOptions, VoiceProviderSession } from "./contract.js";
|
|
3
3
|
import type { RealtimeSocket } from "./realtime-socket.js";
|
|
4
4
|
/**
|
|
@@ -22,6 +22,7 @@ export declare class RealtimeVoiceSession implements VoiceProviderSession {
|
|
|
22
22
|
get events(): AsyncIterable<ProviderEvent>;
|
|
23
23
|
sendAudio(chunk: AudioChunk): Promise<void>;
|
|
24
24
|
sendText(text: string): Promise<void>;
|
|
25
|
+
requestAssistantTurn(request?: AssistantTurnRequest): Promise<void>;
|
|
25
26
|
reportPlayout(progress: PlayoutProgress): Promise<void>;
|
|
26
27
|
sendToolResult(result: ProviderToolResult): Promise<void>;
|
|
27
28
|
sendContext(event: ProviderContextEvent): Promise<void>;
|
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
import { Buffer } from "node:buffer";
|
|
2
|
-
import { AsyncQueue, AudioChunkSchema, audioSampleCount, PlayoutProgressSchema, ProviderContextEventSchema, ProviderToolResultSchema, ToolCallSchema, } from "../protocol/index.js";
|
|
2
|
+
import { AssistantTurnRequestSchema, AsyncQueue, AudioChunkSchema, audioSampleCount, PlayoutProgressSchema, ProviderContextEventSchema, ProviderToolResultSchema, ToolCallSchema, } from "../protocol/index.js";
|
|
3
3
|
import { z } from "zod";
|
|
4
4
|
const FALLBACK_STREAM_ID = "assistant-output";
|
|
5
5
|
const CLOSE_REASON_BYTE_LIMIT = 120;
|
|
6
|
+
const INPUT_TRANSCRIPT_SETTLE_MS = 300;
|
|
7
|
+
const EMITTED_USER_TRANSCRIPT_CACHE_LIMIT = 128;
|
|
6
8
|
const ServerEnvelopeSchema = z.looseObject({ type: z.string().min(1) });
|
|
7
9
|
const OutputAudioDeltaSchema = z.looseObject({
|
|
8
10
|
item_id: z.string().min(1).optional(),
|
|
@@ -83,12 +85,15 @@ export class RealtimeVoiceSession {
|
|
|
83
85
|
#outputStreams = new Map();
|
|
84
86
|
#playedThrough = new Map();
|
|
85
87
|
#deliveredToolCalls = new Set();
|
|
88
|
+
#pendingUserTranscripts = new Map();
|
|
89
|
+
#emittedUserTranscripts = new Map();
|
|
86
90
|
#settled;
|
|
87
91
|
#resolveSettled;
|
|
88
92
|
#outputSampleRate;
|
|
89
93
|
#activeOutputItemId;
|
|
90
94
|
#responseActive = false;
|
|
91
95
|
#responseWanted = false;
|
|
96
|
+
#wantedResponseInstructions;
|
|
92
97
|
#ready = false;
|
|
93
98
|
#closed = false;
|
|
94
99
|
#closeReason;
|
|
@@ -127,6 +132,10 @@ export class RealtimeVoiceSession {
|
|
|
127
132
|
});
|
|
128
133
|
this.#requestResponse();
|
|
129
134
|
}
|
|
135
|
+
async requestAssistantTurn(request = {}) {
|
|
136
|
+
const validated = AssistantTurnRequestSchema.parse(request);
|
|
137
|
+
this.#requestResponse(validated.instructions);
|
|
138
|
+
}
|
|
130
139
|
async reportPlayout(progress) {
|
|
131
140
|
const validated = PlayoutProgressSchema.parse(progress);
|
|
132
141
|
this.#playedThrough.set(validated.streamId, validated.playedThroughSample);
|
|
@@ -269,8 +278,44 @@ export class RealtimeVoiceSession {
|
|
|
269
278
|
const transcript = TranscriptEventSchema.safeParse(parsed);
|
|
270
279
|
if (!transcript.success)
|
|
271
280
|
return;
|
|
281
|
+
if (role === "user" && transcript.data.item_id !== undefined) {
|
|
282
|
+
this.#queueUserTranscript(transcript.data.item_id, transcript.data.transcript);
|
|
283
|
+
return;
|
|
284
|
+
}
|
|
272
285
|
this.#emitTranscript(role, transcript.data.transcript, transcript.data.item_id);
|
|
273
286
|
}
|
|
287
|
+
/** xAI can send several "completed" revisions for one input item; emit only the settled text. */
|
|
288
|
+
#queueUserTranscript(streamId, rawText) {
|
|
289
|
+
const text = rawText.trim();
|
|
290
|
+
if (text === "")
|
|
291
|
+
return;
|
|
292
|
+
const pending = this.#pendingUserTranscripts.get(streamId);
|
|
293
|
+
if (pending)
|
|
294
|
+
clearTimeout(pending.timer);
|
|
295
|
+
const timer = setTimeout(() => this.#flushUserTranscript(streamId), INPUT_TRANSCRIPT_SETTLE_MS);
|
|
296
|
+
this.#pendingUserTranscripts.set(streamId, { text, timer });
|
|
297
|
+
}
|
|
298
|
+
#flushUserTranscript(streamId) {
|
|
299
|
+
const pending = this.#pendingUserTranscripts.get(streamId);
|
|
300
|
+
if (!pending)
|
|
301
|
+
return;
|
|
302
|
+
clearTimeout(pending.timer);
|
|
303
|
+
this.#pendingUserTranscripts.delete(streamId);
|
|
304
|
+
if (this.#emittedUserTranscripts.get(streamId) === pending.text)
|
|
305
|
+
return;
|
|
306
|
+
this.#emittedUserTranscripts.delete(streamId);
|
|
307
|
+
this.#emittedUserTranscripts.set(streamId, pending.text);
|
|
308
|
+
if (this.#emittedUserTranscripts.size > EMITTED_USER_TRANSCRIPT_CACHE_LIMIT) {
|
|
309
|
+
const oldestStreamId = this.#emittedUserTranscripts.keys().next().value;
|
|
310
|
+
if (oldestStreamId !== undefined)
|
|
311
|
+
this.#emittedUserTranscripts.delete(oldestStreamId);
|
|
312
|
+
}
|
|
313
|
+
this.#emitTranscript("user", pending.text, streamId);
|
|
314
|
+
}
|
|
315
|
+
#flushUserTranscripts() {
|
|
316
|
+
for (const streamId of [...this.#pendingUserTranscripts.keys()])
|
|
317
|
+
this.#flushUserTranscript(streamId);
|
|
318
|
+
}
|
|
274
319
|
#handleTranscriptDelta(parsed) {
|
|
275
320
|
const delta = TranscriptDeltaWireSchema.safeParse(parsed);
|
|
276
321
|
if (!delta.success || delta.data.delta === "")
|
|
@@ -319,8 +364,11 @@ export class RealtimeVoiceSession {
|
|
|
319
364
|
this.#emitFunctionCall(call.data.call_id, call.data.name, call.data.arguments);
|
|
320
365
|
}
|
|
321
366
|
}
|
|
322
|
-
if (this.#responseWanted)
|
|
323
|
-
this.#
|
|
367
|
+
if (this.#responseWanted) {
|
|
368
|
+
const instructions = this.#wantedResponseInstructions;
|
|
369
|
+
this.#wantedResponseInstructions = undefined;
|
|
370
|
+
this.#requestResponse(instructions);
|
|
371
|
+
}
|
|
324
372
|
}
|
|
325
373
|
#emitUsage(usage) {
|
|
326
374
|
const input = usage.input_token_details;
|
|
@@ -379,14 +427,19 @@ export class RealtimeVoiceSession {
|
|
|
379
427
|
const message = error.success ? error.data.error?.message : undefined;
|
|
380
428
|
this.#emitError(message ?? `${this.#wire.label} reported an error`, true);
|
|
381
429
|
}
|
|
382
|
-
#requestResponse() {
|
|
430
|
+
#requestResponse(instructions) {
|
|
383
431
|
if (this.#responseActive) {
|
|
384
432
|
this.#responseWanted = true;
|
|
433
|
+
if (instructions !== undefined)
|
|
434
|
+
this.#wantedResponseInstructions = instructions;
|
|
385
435
|
return;
|
|
386
436
|
}
|
|
387
437
|
this.#responseWanted = false;
|
|
388
438
|
this.#responseActive = true;
|
|
389
|
-
this.#trySend({
|
|
439
|
+
this.#trySend({
|
|
440
|
+
type: "response.create",
|
|
441
|
+
...(instructions === undefined ? {} : { response: { instructions } }),
|
|
442
|
+
});
|
|
390
443
|
}
|
|
391
444
|
#send(payload) {
|
|
392
445
|
if (this.#socket.state !== "open") {
|
|
@@ -411,6 +464,7 @@ export class RealtimeVoiceSession {
|
|
|
411
464
|
#settle(reason) {
|
|
412
465
|
if (this.#closed)
|
|
413
466
|
return;
|
|
467
|
+
this.#flushUserTranscripts();
|
|
414
468
|
this.#closed = true;
|
|
415
469
|
this.#queue.push({ type: "closed", reason });
|
|
416
470
|
this.#queue.close();
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { type AudioChunk, type OutputModality, type PlayoutProgress, type ProviderContextEvent, type ProviderEvent, type ProviderToolDefinition, type ProviderToolResult, type ToolCall } from "../protocol/index.js";
|
|
1
|
+
import { type AssistantTurnRequest, type AudioChunk, type OutputModality, type PlayoutProgress, type ProviderContextEvent, type ProviderEvent, type ProviderToolDefinition, type ProviderToolResult, type ToolCall } from "../protocol/index.js";
|
|
2
2
|
import type { CreateProviderSessionOptions, VoiceProvider, VoiceProviderSession } from "../provider/index.js";
|
|
3
3
|
export * from "./replay-voice-provider.js";
|
|
4
4
|
export * from "./scripted-voice-provider.js";
|
|
@@ -24,6 +24,7 @@ export declare class FakeVoiceProviderSession implements VoiceProviderSession {
|
|
|
24
24
|
#private;
|
|
25
25
|
readonly receivedAudio: AudioChunk[];
|
|
26
26
|
readonly receivedTexts: string[];
|
|
27
|
+
readonly assistantTurnRequests: AssistantTurnRequest[];
|
|
27
28
|
readonly playoutProgress: PlayoutProgress[];
|
|
28
29
|
readonly toolResults: ProviderToolResult[];
|
|
29
30
|
readonly contextEvents: ProviderContextEvent[];
|
|
@@ -35,6 +36,7 @@ export declare class FakeVoiceProviderSession implements VoiceProviderSession {
|
|
|
35
36
|
get closed(): boolean;
|
|
36
37
|
sendAudio(chunk: AudioChunk): Promise<void>;
|
|
37
38
|
sendText(text: string): Promise<void>;
|
|
39
|
+
requestAssistantTurn(request?: AssistantTurnRequest): Promise<void>;
|
|
38
40
|
reportPlayout(progress: PlayoutProgress): Promise<void>;
|
|
39
41
|
sendToolResult(result: ProviderToolResult): Promise<void>;
|
|
40
42
|
sendContext(event: ProviderContextEvent): Promise<void>;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { AsyncQueue, AudioChunkSchema, PlayoutProgressSchema, ProviderContextEventSchema, ProviderEventSchema, ProviderToolResultSchema, } from "../protocol/index.js";
|
|
1
|
+
import { AssistantTurnRequestSchema, AsyncQueue, AudioChunkSchema, PlayoutProgressSchema, ProviderContextEventSchema, ProviderEventSchema, ProviderToolResultSchema, } from "../protocol/index.js";
|
|
2
2
|
export * from "./replay-voice-provider.js";
|
|
3
3
|
export * from "./scripted-voice-provider.js";
|
|
4
4
|
export class FakeVoiceProvider {
|
|
@@ -34,6 +34,7 @@ export class FakeVoiceProvider {
|
|
|
34
34
|
export class FakeVoiceProviderSession {
|
|
35
35
|
receivedAudio = [];
|
|
36
36
|
receivedTexts = [];
|
|
37
|
+
assistantTurnRequests = [];
|
|
37
38
|
playoutProgress = [];
|
|
38
39
|
toolResults = [];
|
|
39
40
|
contextEvents = [];
|
|
@@ -81,6 +82,9 @@ export class FakeVoiceProviderSession {
|
|
|
81
82
|
this.emitTranscript("assistant", text);
|
|
82
83
|
}
|
|
83
84
|
}
|
|
85
|
+
async requestAssistantTurn(request = {}) {
|
|
86
|
+
this.assistantTurnRequests.push(AssistantTurnRequestSchema.parse(request));
|
|
87
|
+
}
|
|
84
88
|
async reportPlayout(progress) {
|
|
85
89
|
this.playoutProgress.push(PlayoutProgressSchema.parse(progress));
|
|
86
90
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { type AudioChunk, type HarnessEvent, type PlayoutProgress, type ProviderContextEvent, type ProviderEvent, type ProviderToolResult } from "../protocol/index.js";
|
|
1
|
+
import { type AssistantTurnRequest, type AudioChunk, type HarnessEvent, type PlayoutProgress, type ProviderContextEvent, type ProviderEvent, type ProviderToolResult } from "../protocol/index.js";
|
|
2
2
|
import type { CreateProviderSessionOptions, VoiceProvider, VoiceProviderSession } from "../provider/index.js";
|
|
3
3
|
export interface ReplayVoiceProviderOptions {
|
|
4
4
|
/** Recorded harness events to replay. Takes precedence over load. */
|
|
@@ -32,6 +32,7 @@ export declare class ReplayVoiceProviderSession implements VoiceProviderSession
|
|
|
32
32
|
#private;
|
|
33
33
|
readonly receivedAudio: AudioChunk[];
|
|
34
34
|
readonly receivedTexts: string[];
|
|
35
|
+
readonly assistantTurnRequests: AssistantTurnRequest[];
|
|
35
36
|
readonly playoutProgress: PlayoutProgress[];
|
|
36
37
|
readonly toolResults: ProviderToolResult[];
|
|
37
38
|
readonly contextEvents: ProviderContextEvent[];
|
|
@@ -40,6 +41,7 @@ export declare class ReplayVoiceProviderSession implements VoiceProviderSession
|
|
|
40
41
|
get closed(): boolean;
|
|
41
42
|
sendAudio(chunk: AudioChunk): Promise<void>;
|
|
42
43
|
sendText(text: string): Promise<void>;
|
|
44
|
+
requestAssistantTurn(request?: AssistantTurnRequest): Promise<void>;
|
|
43
45
|
reportPlayout(progress: PlayoutProgress): Promise<void>;
|
|
44
46
|
sendToolResult(result: ProviderToolResult): Promise<void>;
|
|
45
47
|
sendContext(event: ProviderContextEvent): Promise<void>;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { AsyncQueue, AudioChunkSchema, PlayoutProgressSchema, ProviderContextEventSchema, ProviderToolResultSchema, } from "../protocol/index.js";
|
|
1
|
+
import { AssistantTurnRequestSchema, AsyncQueue, AudioChunkSchema, PlayoutProgressSchema, ProviderContextEventSchema, ProviderToolResultSchema, } from "../protocol/index.js";
|
|
2
2
|
/**
|
|
3
3
|
* Replays the provider-visible half of a recorded session: model audio, tool requests,
|
|
4
4
|
* and provider errors, in recorded order and (optionally) recorded pacing. Input audio,
|
|
@@ -30,6 +30,7 @@ export class ReplayVoiceProvider {
|
|
|
30
30
|
export class ReplayVoiceProviderSession {
|
|
31
31
|
receivedAudio = [];
|
|
32
32
|
receivedTexts = [];
|
|
33
|
+
assistantTurnRequests = [];
|
|
33
34
|
playoutProgress = [];
|
|
34
35
|
toolResults = [];
|
|
35
36
|
contextEvents = [];
|
|
@@ -54,6 +55,9 @@ export class ReplayVoiceProviderSession {
|
|
|
54
55
|
async sendText(text) {
|
|
55
56
|
this.receivedTexts.push(text);
|
|
56
57
|
}
|
|
58
|
+
async requestAssistantTurn(request = {}) {
|
|
59
|
+
this.assistantTurnRequests.push(AssistantTurnRequestSchema.parse(request));
|
|
60
|
+
}
|
|
57
61
|
async reportPlayout(progress) {
|
|
58
62
|
this.playoutProgress.push(PlayoutProgressSchema.parse(progress));
|
|
59
63
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { type AudioChunk, type OutputModality, type PlayoutProgress, type ProviderContextEvent, type ProviderEvent, type ProviderToolDefinition, type ProviderToolResult } from "../protocol/index.js";
|
|
1
|
+
import { type AssistantTurnRequest, type AudioChunk, type OutputModality, type PlayoutProgress, type ProviderContextEvent, type ProviderEvent, type ProviderToolDefinition, type ProviderToolResult } from "../protocol/index.js";
|
|
2
2
|
import type { CreateProviderSessionOptions, VoiceProvider, VoiceProviderSession } from "../provider/index.js";
|
|
3
3
|
export type ScriptedVoiceStep = {
|
|
4
4
|
type: "emit";
|
|
@@ -9,6 +9,9 @@ export type ScriptedVoiceStep = {
|
|
|
9
9
|
} | {
|
|
10
10
|
type: "expect.text";
|
|
11
11
|
value: string;
|
|
12
|
+
} | {
|
|
13
|
+
type: "expect.assistant_turn";
|
|
14
|
+
value: AssistantTurnRequest;
|
|
12
15
|
} | {
|
|
13
16
|
type: "expect.playout";
|
|
14
17
|
value: PlayoutProgress;
|
|
@@ -54,6 +57,7 @@ export declare class ScriptedVoiceProviderSession implements VoiceProviderSessio
|
|
|
54
57
|
completed(): Promise<void>;
|
|
55
58
|
sendAudio(chunk: AudioChunk): Promise<void>;
|
|
56
59
|
sendText(text: string): Promise<void>;
|
|
60
|
+
requestAssistantTurn(request?: AssistantTurnRequest): Promise<void>;
|
|
57
61
|
reportPlayout(progress: PlayoutProgress): Promise<void>;
|
|
58
62
|
sendToolResult(result: ProviderToolResult): Promise<void>;
|
|
59
63
|
sendContext(event: ProviderContextEvent): Promise<void>;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { AsyncQueue, AudioChunkSchema, PlayoutProgressSchema, ProviderContextEventSchema, ProviderEventSchema, ProviderToolResultSchema, } from "../protocol/index.js";
|
|
1
|
+
import { AssistantTurnRequestSchema, AsyncQueue, AudioChunkSchema, PlayoutProgressSchema, ProviderContextEventSchema, ProviderEventSchema, ProviderToolResultSchema, } from "../protocol/index.js";
|
|
2
2
|
/**
|
|
3
3
|
* A closed-loop deterministic provider. Unlike recorded replay, scripts can stop after
|
|
4
4
|
* emitting a tool call and verify the exact result before the next model event appears.
|
|
@@ -84,6 +84,9 @@ export class ScriptedVoiceProviderSession {
|
|
|
84
84
|
async sendText(text) {
|
|
85
85
|
this.#accept("expect.text", text);
|
|
86
86
|
}
|
|
87
|
+
async requestAssistantTurn(request = {}) {
|
|
88
|
+
this.#accept("expect.assistant_turn", AssistantTurnRequestSchema.parse(request));
|
|
89
|
+
}
|
|
87
90
|
async reportPlayout(progress) {
|
|
88
91
|
this.#accept("expect.playout", PlayoutProgressSchema.parse(progress));
|
|
89
92
|
}
|