xo-harness 0.2.0 → 0.3.0

package/dist/internal/provider/realtime-session.js

@@ -1,27 +1,41 @@
-import { AssistantTurnRequestSchema, AsyncQueue, AudioChunkSchema, audioSampleCount, PlayoutProgressSchema, PROVIDER_DIAGNOSTIC_MAX_CONTENT_TYPES, PROVIDER_DIAGNOSTIC_MAX_OUTPUTS, PROVIDER_DIAGNOSTIC_STRING_MAX_LENGTH, ProviderContextEventSchema, ProviderToolResultSchema, ToolCallSchema, } from "../protocol/index.js";
+import { AssistantTurnRequestSchema, AudioChunkSchema, audioSampleCount, PlayoutProgressSchema, PROVIDER_DIAGNOSTIC_MAX_CONTENT_TYPES, PROVIDER_DIAGNOSTIC_MAX_OUTPUTS, PROVIDER_DIAGNOSTIC_STRING_MAX_LENGTH, ProviderContextEventSchema, ProviderToolResultSchema, ToolCallSchema, } from "../protocol/index.js";
 import { z } from "zod";
+import { ProviderEventQueue } from "./event-queue.js";
+import { toolStatusContext } from "./tool-status-context.js";
 const FALLBACK_STREAM_ID = "assistant-output";
 const CLOSE_REASON_BYTE_LIMIT = 120;
 const EMITTED_USER_TRANSCRIPT_CACHE_LIMIT = 128;
+const TOOL_RESULT_CORRELATION_LIMIT = 4096;
 const VAD_RESPONSE_RECOVERY_MS = 1_500;
 const ServerEnvelopeSchema = z.looseObject({ type: z.string().min(1) });
-const OutputAudioDeltaSchema = z.looseObject({
+const OutputCoordinatesSchema = z.object({
     item_id: z.string().min(1).optional(),
     response_id: z.string().min(1).optional(),
+    output_index: z.int().nonnegative().optional(),
+    content_index: z.int().nonnegative().optional(),
+});
+const OutputAudioDeltaSchema = z.looseObject({
+    ...OutputCoordinatesSchema.shape,
     delta: z.string().min(1).optional(),
     data: z.string().min(1).optional(),
 });
 const FunctionCallItemSchema = z.looseObject({
     type: z.literal("function_call"),
+    id: z.string().min(1).optional(),
     call_id: z.string().min(1),
     name: z.string().min(1),
     arguments: z.string(),
 });
 const FunctionCallArgumentsDoneSchema = z.looseObject({
+    ...OutputCoordinatesSchema.shape,
     call_id: z.string().min(1),
     name: z.string().min(1),
     arguments: z.string(),
 });
+const OutputItemDoneSchema = z.looseObject({
+    ...OutputCoordinatesSchema.shape,
+    item: FunctionCallItemSchema,
+});
 const UsageDetailsSchema = z.looseObject({
     text_tokens: z.number().int().nonnegative().optional(),
     audio_tokens: z.number().int().nonnegative().optional(),
@@ -55,19 +69,18 @@ const ResponseDiagnosticSchema = z.looseObject({
     status_details: ResponseStatusDetailsSchema.nullish(),
     output: z.array(ResponseOutputDiagnosticSchema).max(PROVIDER_DIAGNOSTIC_MAX_OUTPUTS).optional(),
 });
-/** Core response.done fields stay independent from optional diagnostic metadata. */
+const ResponseUsageSchema = z.looseObject({
+    total_tokens: z.number().int().nonnegative(),
+    input_tokens: z.number().int().nonnegative(),
+    output_tokens: z.number().int().nonnegative(),
+    input_token_details: UsageDetailsSchema.optional(),
+    output_token_details: UsageDetailsSchema.optional(),
+});
+/** Optional usage and diagnostics must never decide whether a function completed. */
 const ResponseDoneSchema = z.looseObject({
     response: z.looseObject({
         output: z.array(z.looseObject({ type: z.string().optional() })).optional(),
-        usage: z
-            .looseObject({
-            total_tokens: z.number().int().nonnegative(),
-            input_tokens: z.number().int().nonnegative(),
-            output_tokens: z.number().int().nonnegative(),
-            input_token_details: UsageDetailsSchema.optional(),
-            output_token_details: UsageDetailsSchema.optional(),
-        })
-            .optional(),
+        usage: z.unknown().optional(),
     }),
 });
 const ResponseEnvelopeSchema = z.looseObject({
@@ -79,18 +92,23 @@ const ResponseIdEnvelopeSchema = z.looseObject({
 });
 const TranscriptEventSchema = z.looseObject({
     transcript: z.string(),
-    item_id: z.string().min(1).optional(),
+    ...OutputCoordinatesSchema.shape,
+});
+const InputTranscriptionFailedSchema = z.looseObject({
+    item_id: z.string().min(1),
+    content_index: z.int().nonnegative(),
+    error: z.looseObject({ message: z.string().nullish(), code: z.string().nullish() }),
 });
 const OutputTextDoneSchema = z.looseObject({
     text: z.string(),
-    item_id: z.string().min(1).optional(),
+    ...OutputCoordinatesSchema.shape,
 });
 const TranscriptDeltaWireSchema = z.looseObject({
     delta: z.string(),
-    item_id: z.string().min(1).optional(),
+    ...OutputCoordinatesSchema.shape,
 });
 const ServerErrorSchema = z.looseObject({
-    error: z.looseObject({ message: z.string().optional() }).optional(),
+    error: z.looseObject({ message: z.string().optional(), event_id: z.string().min(1).nullish() }).optional(),
 });
 const SessionUpdatedSchema = z.looseObject({
     session: z
@@ -116,12 +134,20 @@ const SpeechBoundarySchema = z.looseObject({
 export class RealtimeVoiceSession {
     #socket;
     #wire;
-    #queue = new AsyncQueue();
+    #signal;
+    #onAbort = () => {
+        void this.close(String(this.#signal.reason ?? "aborted"));
+    };
+    #queue;
     #outputStreams = new Map();
     #playedThrough = new Map();
     #interruptedStreams = new Set();
+    #interruptedResponseIds = new Set();
     #deliveredToolCalls = new Set();
-    #completedResponseIds = new Set();
+    #rejectedToolCalls = new Set();
+    #backgroundCalls = new Map();
+    #toolResultEvents = new Map();
+    #completedResponses = new Map();
     #usageResponseIds = new Set();
     #pendingUserTranscripts = new Map();
     #emittedUserTranscripts = new Map();
@@ -131,6 +157,8 @@ export class RealtimeVoiceSession {
     #responseNumber = 0;
     #responseActive = false;
     #activeResponseId;
+    #pendingResponseEventId;
+    #clientEventSequence = 0;
     #responseWanted = false;
     #inputSpeechActive = false;
     #awaitingVadResponse = false;
@@ -139,26 +167,49 @@ export class RealtimeVoiceSession {
     #wantedResponseInstructions;
     #ready = false;
     #closed = false;
+    #settling = false;
     #closeReason;
     constructor(socket, wire, options) {
         this.#socket = socket;
         this.#wire = wire;
+        this.#signal = options.signal;
         this.#outputSampleRate = wire.outputSampleRate;
         this.#settled = new Promise((resolve) => {
             this.#resolveSettled = resolve;
         });
+        this.#queue = new ProviderEventQueue(wire.eventQueueLimits, () => {
+            this.#settle("provider_event_queue_failed");
+            if (socket.state !== "closed") {
+                if (socket.terminate)
+                    socket.terminate();
+                else
+                    socket.close(1011, "provider_event_queue_failed");
+            }
+        });
         socket.onMessage = (text) => this.#handleMessage(text);
         socket.onError = (message) => this.#emitError(message, false);
         socket.onClose = (code) => this.#settle(this.#closeReason ?? `socket_closed_${code}`);
-        options.signal.addEventListener("abort", () => {
-            void this.close(String(options.signal.reason ?? "aborted"));
-        }, { once: true });
+        this.#signal.addEventListener("abort", this.#onAbort, { once: true });
+        if (this.#signal.aborted) {
+            this.#onAbort();
+            return;
+        }
         this.#send({ type: "session.update", session: wire.sessionUpdate });
     }
+    get failureSignal() {
+        return this.#queue.failureSignal;
+    }
+    eventQueueStats() {
+        return this.#queue.stats();
+    }
     get events() {
         return this.#queue;
     }
     async sendAudio(chunk) {
+        this.failureSignal.throwIfAborted();
+        if (this.#wire.autoCommitAudio === false) {
+            throw new Error(`${this.#wire.label} cannot accept audio with turnDetection: null; XO does not expose manual input-audio commit. Enable turn detection, optionally with create_response: false for host-triggered replies.`);
+        }
         const validated = AudioChunkSchema.parse(chunk);
         if (validated.channels !== 1) {
             throw new Error(`${this.#wire.label} expects mono PCM input`);
@@ -169,6 +220,7 @@ export class RealtimeVoiceSession {
         this.#send({ type: "input_audio_buffer.append", audio: toBase64(validated.data) });
     }
     async sendText(text) {
+        this.failureSignal.throwIfAborted();
         this.#send({
             type: "conversation.item.create",
             item: { type: "message", role: "user", content: [{ type: "input_text", text }] },
@@ -176,38 +228,110 @@ export class RealtimeVoiceSession {
         this.#requestResponse();
     }
     async requestAssistantTurn(request = {}) {
+        this.failureSignal.throwIfAborted();
         const validated = AssistantTurnRequestSchema.parse(request);
         this.#requestResponse(validated.instructions);
     }
     async reportPlayout(progress) {
+        this.failureSignal.throwIfAborted();
         const validated = PlayoutProgressSchema.parse(progress);
-        this.#playedThrough.set(validated.streamId, validated.playedThroughSample);
+        this.#playedThrough.set(validated.streamId, Math.max(this.#playedThrough.get(validated.streamId) ?? 0, validated.playedThroughSample));
     }
     async sendToolResult(result) {
+        this.failureSignal.throwIfAborted();
         const validated = ProviderToolResultSchema.parse(result);
-        this.#send({
-            type: "conversation.item.create",
-            item: {
-                type: "function_call_output",
-                call_id: validated.callId,
-                output: toolOutcomePayload(validated.outcome),
-            },
-        });
+        if (this.#wire.nativeAsyncTools && validated.outcome.type === "accepted_task") {
+            const { taskId } = validated.outcome;
+            const existing = this.#backgroundCalls.get(validated.callId);
+            if (existing) {
+                if (existing.taskId !== taskId)
+                    throw new Error("A background tool call cannot be assigned to a different task");
+                return;
+            }
+            if ([...this.#backgroundCalls.values()].some((call) => call.taskId === taskId))
+                throw new Error("A background task cannot belong to a different tool call");
+            // A task handle is acceptance, not the result of the original function call.
+            // Keeping that call open lets native async-tool reasoning recognize pending work.
+            this.#sendContextItem(JSON.stringify({
+                type: "accepted_task",
+                callId: validated.callId,
+                taskId,
+                note: "The original function call remains pending. Its final output will arrive automatically; continue independent work while waiting.",
+            }));
+            this.#backgroundCalls.set(validated.callId, { taskId, settled: false });
+            this.#requestResponse();
+            return;
+        }
+        this.#sendFunctionResult(validated.callId, toolOutcomePayload(validated.outcome));
         this.#requestResponse();
     }
+    #sendFunctionResult(callId, output) {
+        const eventId = this.#wire.correlateClientErrors
+            ? `xo-realtime-tool-result-${++this.#clientEventSequence}`
+            : undefined;
+        if (eventId !== undefined)
+            this.#toolResultEvents.set(eventId, callId);
+        try {
+            this.#send({
+                type: "conversation.item.create",
+                ...(eventId === undefined ? {} : { event_id: eventId }),
+                item: {
+                    type: "function_call_output",
+                    call_id: callId,
+                    output,
+                },
+            });
+        }
+        catch (error) {
+            if (eventId !== undefined)
+                this.#toolResultEvents.delete(eventId);
+            throw error;
+        }
+        // Retain recent submissions for asynchronous rejections without an unbounded
+        // per-session command log. An error never retries the action or its output.
+        if (this.#toolResultEvents.size > TOOL_RESULT_CORRELATION_LIMIT) {
+            const oldest = this.#toolResultEvents.keys().next().value;
+            if (oldest !== undefined)
+                this.#toolResultEvents.delete(oldest);
+        }
+    }
     async sendContext(event) {
+        this.failureSignal.throwIfAborted();
         const validated = ProviderContextEventSchema.parse(event);
+        if (validated.type !== "tool.status") {
+            const background = this.#backgroundCalls.get(validated.callId);
+            if (!background && [...this.#backgroundCalls.values()].some((call) => call.taskId === validated.taskId))
+                throw new Error("Task context does not match the accepted background tool call");
+            if (background) {
+                if (background.taskId !== validated.taskId)
+                    throw new Error("Task context does not match the accepted background tool call");
+                if (background.settled)
+                    return;
+                if (validated.type !== "task.progress") {
+                    this.#sendFunctionResult(validated.callId, taskResultPayload(validated));
+                    background.settled = true;
+                    if (this.#wire.announceTaskSettlement)
+                        this.#requestResponse();
+                    return;
+                }
+            }
+        }
+        this.#sendContextItem(contextText(validated));
+        if (validated.type !== "tool.status" &&
+            validated.type !== "task.progress" &&
+            this.#wire.announceTaskSettlement) {
+            this.#requestResponse();
+        }
+    }
+    #sendContextItem(text) {
         this.#send({
             type: "conversation.item.create",
             item: {
                 type: "message",
                 role: "system",
-                content: [{ type: "input_text", text: contextText(validated) }],
+                content: [{ type: "input_text", text }],
             },
         });
-        if (validated.type !== "task.progress" && this.#wire.announceTaskSettlement) {
-            this.#requestResponse();
-        }
     }
     async close(reason = "client_closed") {
         this.#closeReason ??= reason;
@@ -222,6 +346,8 @@ export class RealtimeVoiceSession {
         await this.#settled;
     }
     #handleMessage(text) {
+        if (this.#closed)
+            return;
         let parsed;
         try {
             parsed = JSON.parse(text);
@@ -241,8 +367,19 @@ export class RealtimeVoiceSession {
                 break;
             case "response.created":
                 this.#handleResponseCreated(parsed);
+                this.#pendingResponseEventId = undefined;
                 this.#responseActive = true;
                 this.#activeResponseId = responseIdFromEnvelope(parsed);
+                // A response.create already in flight can arrive after the server's VAD start.
+                // Automatic interruption only cancelled responses that existed at that boundary.
+                if (this.#inputSpeechActive && this.#awaitingVadResponse && this.#wire.interruptOnSpeech !== false) {
+                    if (this.#activeResponseId !== undefined)
+                        this.#interruptedResponseIds.add(this.#activeResponseId);
+                    this.#trySend({
+                        type: "response.cancel",
+                        ...(this.#activeResponseId === undefined ? {} : { response_id: this.#activeResponseId }),
+                    });
+                }
                 if (!this.#inputSpeechActive)
                     this.#clearAwaitingVadResponse();
                 this.#reportResponseState();
@@ -266,9 +403,19 @@ export class RealtimeVoiceSession {
             case "conversation.item.input_audio_transcription.completed":
                 this.#handleTranscript(parsed, "user");
                 break;
+            case "conversation.item.input_audio_transcription.failed": {
+                const failed = InputTranscriptionFailedSchema.safeParse(parsed);
+                this.#emitError(failed.success
+                    ? `Input audio transcription failed for ${failed.data.item_id} content ${failed.data.content_index}: ${failed.data.error.message ?? failed.data.error.code ?? "Unknown transcription error"}`
+                    : "Received a malformed input audio transcription failure", true);
+                break;
+            }
             case "response.function_call_arguments.done":
                 this.#handleFunctionCallArgumentsDone(parsed);
                 break;
+            case "response.output_item.done":
+                this.#handleOutputItemDone(parsed);
+                break;
             case "response.done":
                 this.#handleResponseDone(parsed);
                 break;
@@ -319,6 +466,8 @@ export class RealtimeVoiceSession {
             this.#emitError("Received a malformed output audio delta", true);
             return;
         }
+        if (delta.data.response_id !== undefined && this.#interruptedResponseIds.has(delta.data.response_id))
+            return;
         const payload = delta.data.delta ?? delta.data.data;
         if (payload === undefined) {
             this.#emitError("Received an output audio delta without an audio payload", true);
@@ -336,9 +485,11 @@ export class RealtimeVoiceSession {
             this.#emitError(`Received a PCM16 delta with invalid byte length ${bytes.byteLength}`, true);
             return;
         }
-        const streamId = delta.data.item_id ??
+        const baseStreamId = delta.data.item_id ??
             delta.data.response_id ??
             (this.#responseNumber === 0 ? FALLBACK_STREAM_ID : `${FALLBACK_STREAM_ID}-${this.#responseNumber}`);
+        const streamId = contentStreamId(baseStreamId, delta.data.content_index);
+        const source = outputSource(delta.data);
         if (this.#interruptedStreams.has(streamId))
             return;
         const stream = this.#outputStreams.get(streamId) ?? {
@@ -346,6 +497,8 @@ export class RealtimeVoiceSession {
             startSample: 0,
             complete: false,
             itemId: delta.data.item_id,
+            responseId: delta.data.response_id ?? this.#activeResponseId,
+            contentIndex: delta.data.content_index ?? 0,
         };
         const chunk = AudioChunkSchema.parse({
             streamId,
@@ -359,20 +512,22 @@ export class RealtimeVoiceSession {
         stream.sequence += 1;
         stream.startSample += audioSampleCount(chunk);
         this.#outputStreams.set(streamId, stream);
-        this.#emit({ type: "audio.output", chunk });
+        this.#emit({ type: "audio.output", chunk, ...(source === undefined ? {} : { source }) });
     }
     #handleTranscript(parsed, role) {
         const transcript = TranscriptEventSchema.safeParse(parsed);
         if (!transcript.success)
             return;
+        const source = outputSource(transcript.data);
+        const streamId = transcriptStreamId(transcript.data);
         if (role === "user" && transcript.data.item_id !== undefined) {
-            this.#queueUserTranscript(transcript.data.item_id, transcript.data.transcript);
+            this.#queueUserTranscript(streamId ?? transcript.data.item_id, transcript.data.transcript, source);
             return;
         }
-        this.#emitTranscript(role, transcript.data.transcript, transcript.data.item_id);
+        this.#emitTranscript(role, transcript.data.transcript, streamId, source);
     }
     /** xAI can send several "completed" revisions for one input item; emit only the settled text. */
-    #queueUserTranscript(streamId, rawText) {
+    #queueUserTranscript(streamId, rawText, source) {
         const text = rawText.trim();
         if (text === "")
             return;
@@ -380,7 +535,11 @@ export class RealtimeVoiceSession {
         if (pending)
             clearTimeout(pending.timer);
         const timer = setTimeout(() => this.#flushUserTranscript(streamId), this.#wire.inputTranscriptSettleMs ?? 0);
-        this.#pendingUserTranscripts.set(streamId, { text, timer });
+        this.#pendingUserTranscripts.set(streamId, {
+            text,
+            source: source === undefined ? pending?.source : { ...pending?.source, ...source },
+            timer,
+        });
         if (!this.#wire.inputTranscriptSettleMs)
             this.#flushUserTranscript(streamId);
     }
@@ -399,7 +558,7 @@ export class RealtimeVoiceSession {
             if (oldestStreamId !== undefined)
                 this.#emittedUserTranscripts.delete(oldestStreamId);
         }
-        this.#emitTranscript("user", pending.text, streamId);
+        this.#emitTranscript("user", pending.text, streamId, pending.source);
     }
     #flushUserTranscripts() {
         for (const streamId of [...this.#pendingUserTranscripts.keys()])
@@ -409,46 +568,86 @@ export class RealtimeVoiceSession {
         const delta = TranscriptDeltaWireSchema.safeParse(parsed);
         if (!delta.success || delta.data.delta === "")
             return;
+        const source = outputSource(delta.data);
+        const streamId = transcriptStreamId(delta.data);
         this.#emit({
             type: "transcript.delta",
             role,
             delta: delta.data.delta,
-            ...(delta.data.item_id === undefined ? {} : { streamId: delta.data.item_id }),
+            ...(streamId === undefined ? {} : { streamId }),
+            ...(source === undefined ? {} : { source }),
         });
     }
     #handleOutputText(parsed) {
         const output = OutputTextDoneSchema.safeParse(parsed);
         if (!output.success)
             return;
-        this.#emitTranscript("assistant", output.data.text, output.data.item_id);
+        this.#emitTranscript("assistant", output.data.text, transcriptStreamId(output.data), outputSource(output.data));
     }
-    #emitTranscript(role, rawText, streamId) {
+    #emitTranscript(role, rawText, streamId, source) {
         const text = rawText.trim();
         if (text === "")
             return;
-        this.#emit({ type: "transcript", role, text, ...(streamId === undefined ? {} : { streamId }) });
+        this.#emit({
+            type: "transcript",
+            role,
+            text,
+            ...(streamId === undefined ? {} : { streamId }),
+            ...(source === undefined ? {} : { source }),
+        });
     }
     #handleFunctionCallArgumentsDone(parsed) {
+        // OpenAI also emits this for cancelled/incomplete calls. Its authoritative
+        // item/response completion carries full arguments, so no speculative buffer is needed.
+        if (this.#wire.toolCallCompletion !== "arguments")
+            return;
         const done = FunctionCallArgumentsDoneSchema.safeParse(parsed);
         if (!done.success)
             return;
-        this.#emitFunctionCall(done.data.call_id, done.data.name, done.data.arguments);
+        this.#emitFunctionCall(done.data.call_id, done.data.name, done.data.arguments, outputSource(done.data));
+    }
+    #handleOutputItemDone(parsed) {
+        const done = OutputItemDoneSchema.safeParse(parsed);
+        if (!done.success)
+            return;
+        const item = done.data.item;
+        if (item.status !== undefined && item.status !== "completed") {
+            this.#rejectToolCall(item.call_id);
+            return;
+        }
+        if (item.status !== "completed" && this.#wire.toolCallCompletion === "confirmed")
+            return;
+        this.#emitFunctionCall(item.call_id, item.name, item.arguments, outputSource({
+            ...done.data,
+            ...(item.id === undefined ? {} : { item_id: item.id }),
+        }));
     }
     #handleResponseDone(parsed) {
         this.#emitResponseEnvelopeDiagnostic(parsed, "completed");
         const responseId = responseIdFromEnvelope(parsed);
-        const duplicate = responseId !== undefined && this.#completedResponseIds.has(responseId);
-        if (responseId !== undefined)
-            this.#completedResponseIds.add(responseId);
+        const duplicate = responseId !== undefined && this.#completedResponses.has(responseId);
         const done = ResponseDoneSchema.safeParse(parsed);
+        const status = done.success ? done.data.response.status : undefined;
+        const completed = done.success &&
+            (status === "completed" || (status === undefined && this.#wire.toolCallCompletion === "arguments"));
+        if (responseId !== undefined && !duplicate)
+            this.#completedResponses.set(responseId, completed);
         if (done.success) {
             const usage = done.data.response.usage;
-            if (usage && (responseId === undefined || !this.#usageResponseIds.has(responseId))) {
-                this.#emitUsage(usage);
-                if (responseId !== undefined)
-                    this.#usageResponseIds.add(responseId);
+            if (usage !== undefined && (responseId === undefined || !this.#usageResponseIds.has(responseId))) {
+                const parsedUsage = ResponseUsageSchema.safeParse(usage);
+                if (parsedUsage.success) {
+                    this.#emitUsage(parsedUsage.data);
+                    if (responseId !== undefined)
+                        this.#usageResponseIds.add(responseId);
+                }
+                else {
+                    this.#emitError("Received incomplete or invalid response usage; token accounting remains unconfirmed", true);
+                }
             }
-            for (const item of done.data.response.output ?? []) {
+            for (const [outputIndex, item] of (done.data.response.output ?? []).entries()) {
+                if (duplicate)
+                    break;
                 if (item.type !== "function_call")
                     continue;
                 const call = FunctionCallItemSchema.safeParse(item);
@@ -456,7 +655,15 @@ export class RealtimeVoiceSession {
                     this.#emitError("Received a function call item without call_id, name, or arguments", true);
                     continue;
                 }
-                this.#emitFunctionCall(call.data.call_id, call.data.name, call.data.arguments);
+                if (!completed || (call.data.status !== undefined && call.data.status !== "completed")) {
+                    this.#rejectToolCall(call.data.call_id);
+                    continue;
+                }
+                this.#emitFunctionCall(call.data.call_id, call.data.name, call.data.arguments, outputSource({
+                    output_index: outputIndex,
+                    ...(responseId === undefined ? {} : { response_id: responseId }),
+                    ...(call.data.id === undefined ? {} : { item_id: call.data.id }),
+                }));
             }
         }
         // A duplicate or stale terminal update can refine diagnostics and carry omitted
@@ -469,6 +676,7 @@ export class RealtimeVoiceSession {
             return;
         this.#responseActive = false;
         this.#activeResponseId = undefined;
+        this.#pendingResponseEventId = undefined;
         // Generation often ends well before the client finishes playing its queued PCM.
         for (const stream of this.#outputStreams.values())
             stream.complete = true;
@@ -496,25 +704,44 @@ export class RealtimeVoiceSession {
                 : { cachedAudioInputTokens: input.cached_tokens_details.audio_tokens }),
         });
     }
-    #emitFunctionCall(callId, name, rawArguments) {
-        if (this.#deliveredToolCalls.has(callId))
+    #rejectToolCall(callId) {
+        if (!this.#deliveredToolCalls.has(callId))
+            this.#rejectedToolCalls.add(callId);
+    }
+    #emitFunctionCall(callId, name, rawArguments, source) {
+        if (this.#closed || this.#deliveredToolCalls.has(callId) || this.#rejectedToolCalls.has(callId))
+            return;
+        if (source?.responseId !== undefined &&
+            (this.#completedResponses.get(source.responseId) === false ||
+                this.#interruptedResponseIds.has(source.responseId)))
             return;
         let parsedArguments;
         try {
             parsedArguments = rawArguments.trim() === "" ? {} : JSON.parse(rawArguments);
+            if (parsedArguments === null || typeof parsedArguments !== "object" || Array.isArray(parsedArguments))
+                throw new Error("Function arguments must be a valid JSON object");
         }
         catch {
-            this.#emitError(`Could not parse arguments for tool call ${name}`, true);
-            return;
-        }
-        if (parsedArguments === null || typeof parsedArguments !== "object" || Array.isArray(parsedArguments)) {
-            this.#emitError(`Tool call ${name} arguments are not a JSON object`, true);
+            // This is confirmed native intent, even though it cannot be admitted to the
+            // harness. Settle it once so the model can recover instead of waiting forever.
+            this.#rejectToolCall(callId);
+            const error = "Function arguments must be a valid JSON object";
+            this.#emitError(`Tool call ${name} (${callId}) was not executed: ${error}`, true);
+            // Finish the incoming response's bookkeeping before scheduling its recovery.
+            void Promise.resolve()
+                .then(() => this.sendToolResult({ callId, outcome: { type: "failed", error } }))
+                .catch((error) => this.#emitError(`Could not return invalid tool arguments: ${String(error)}`, true));
             return;
         }
         this.#deliveredToolCalls.add(callId);
         this.#emit({
             type: "tool.call",
-            call: ToolCallSchema.parse({ callId, name, arguments: parsedArguments }),
+            call: ToolCallSchema.parse({
+                callId,
+                name,
+                arguments: parsedArguments,
+                ...(source === undefined ? {} : { source }),
+            }),
         });
     }
     #handleSpeechStarted(parsed) {
@@ -525,6 +752,8 @@ export class RealtimeVoiceSession {
         this.#reportResponseState();
         if (this.#wire.interruptOnSpeech === false)
             return;
+        if (this.#activeResponseId !== undefined)
+            this.#interruptedResponseIds.add(this.#activeResponseId);
         // A response can contain a preamble and an answer, both ahead of playback.
         // Discard queued items first so stopping the audible item cannot start the next.
         for (const [streamId, stream] of [...this.#outputStreams].reverse()) {
@@ -534,14 +763,24 @@ export class RealtimeVoiceSession {
             if (stream.complete && playedThroughSample >= stream.startSample)
                 continue;
             this.#interruptedStreams.add(streamId);
+            if (stream.responseId !== undefined)
+                this.#interruptedResponseIds.add(stream.responseId);
             this.#emit({ type: "audio.interrupted", streamId });
-            // Only a real conversation item ID is valid in the vendor's truncate command.
-            if (stream.itemId === undefined)
+            // Legacy clients never acknowledge playback. Preserve their conversation
+            // context instead of mistaking unknown playback for zero heard samples.
+            // Once a client reports playback, other queued streams can start at zero.
+            if (stream.itemId === undefined || this.#playedThrough.size === 0)
                 continue;
+            if (stream.contentIndex !== 0) {
+                // The public truncate contract currently permits only content_index 0.
+                // Discard locally without truncating a different content part on the server.
+                this.#emitError(`Interrupted audio content ${stream.contentIndex} locally; native truncation only supports content index 0`, true);
+                continue;
+            }
             this.#trySend({
                 type: "conversation.item.truncate",
                 item_id: stream.itemId,
-                content_index: 0,
+                content_index: stream.contentIndex,
                 audio_end_ms: Math.floor((Math.min(playedThroughSample, stream.startSample) / this.#outputSampleRate) * 1000),
             });
         }
@@ -595,8 +834,22 @@ export class RealtimeVoiceSession {
         const error = ServerErrorSchema.safeParse(parsed);
         const message = error.success ? error.data.error?.message : undefined;
         this.#emitError(message ?? `${this.#wire.label} reported an error`, true);
-        if (this.#awaitingVadResponse && !this.#inputSpeechActive) {
-            this.#clearAwaitingVadResponse();
+        const eventId = error.success ? error.data.error?.event_id : undefined;
+        const callId = eventId == null ? undefined : this.#toolResultEvents.get(eventId);
+        if (callId !== undefined && eventId != null) {
+            this.#toolResultEvents.delete(eventId);
+            this.#emit({
+                type: "tool.delivery_failed",
+                callId,
+                error: message ?? `${this.#wire.label} rejected the function result`,
+            });
+        }
+        if (eventId != null && eventId === this.#pendingResponseEventId) {
+            // A rejected create has no response.done to release the scheduler. Only its
+            // correlated error may clear this latch; accepted responses still own it.
+            this.#pendingResponseEventId = undefined;
+            this.#responseActive = false;
+            // Do not retry the rejected request; only resume a separately queued turn.
             this.#resumeQueuedResponse();
         }
     }
@@ -619,14 +872,21 @@ export class RealtimeVoiceSession {
         }
         this.#responseWanted = false;
         this.#responseActive = true;
+        this.#pendingResponseEventId = this.#wire.correlateClientErrors
+            ? `xo-realtime-response-${++this.#clientEventSequence}`
+            : undefined;
         this.#reportResponseState();
         this.#trySend({
             type: "response.create",
+            ...(this.#pendingResponseEventId === undefined ? {} : { event_id: this.#pendingResponseEventId }),
             ...(instructions === undefined ? {} : { response: { instructions } }),
         });
     }
     #armVadResponseRecovery() {
         this.#clearVadResponseRecoveryTimer();
+        // The preceding speech diagnostic can synchronously fail the ingress queue.
+        if (this.#closed)
+            return;
         this.#vadResponseRecoveryTimer = setTimeout(() => {
             this.#vadResponseRecoveryTimer = undefined;
             if (this.#closed || !this.#awaitingVadResponse)
@@ -663,6 +923,7 @@ export class RealtimeVoiceSession {
         this.#emit({ type: "response.state", state });
     }
     #send(payload) {
+        this.failureSignal.throwIfAborted();
         if (this.#socket.state !== "open") {
             throw new Error(`${this.#wire.label} session socket is not open`);
         }
@@ -670,7 +931,7 @@ export class RealtimeVoiceSession {
     }
     /** For sends triggered by incoming events: a closing socket is settlement, not an error. */
     #trySend(payload) {
-        if (this.#socket.state !== "open")
+        if (this.#closed || this.failureSignal.aborted || this.#socket.state !== "open")
             return;
         this.#send(payload);
     }
@@ -683,9 +944,12 @@ export class RealtimeVoiceSession {
         this.#emit({ type: "error", message, recoverable });
     }
     #settle(reason) {
-        if (this.#closed)
+        if (this.#closed || this.#settling)
             return;
+        this.#settling = true;
+        this.#signal.removeEventListener("abort", this.#onAbort);
         this.#clearVadResponseRecoveryTimer();
+        this.#toolResultEvents.clear();
         this.#flushUserTranscripts();
         this.#closed = true;
         this.#queue.push({ type: "closed", reason });
@@ -693,6 +957,33 @@ export class RealtimeVoiceSession {
         this.#resolveSettled();
     }
 }
+/** Native coordinates are evidence only; never fill them from the active response or a counter. */
+function outputSource(coordinates) {
+    const { response_id, item_id, output_index, content_index } = coordinates;
+    if (response_id === undefined &&
+        item_id === undefined &&
+        output_index === undefined &&
+        content_index === undefined)
+        return undefined;
+    return {
+        scope: "voice",
+        ...(response_id === undefined ? {} : { responseId: response_id }),
+        ...(item_id === undefined ? {} : { itemId: item_id }),
+        ...(output_index === undefined ? {} : { outputIndex: output_index }),
+        ...(content_index === undefined ? {} : { contentIndex: content_index }),
+    };
+}
+function transcriptStreamId(coordinates) {
+    const base = coordinates.item_id ?? coordinates.response_id;
+    return base === undefined ? undefined : contentStreamId(base, coordinates.content_index);
+}
+/** Preserve legacy zero-index IDs; escape the namespace to keep native IDs collision-free. */
+function contentStreamId(base, contentIndex = 0) {
+    const prefix = "xo-content:";
+    return contentIndex === 0 && !base.startsWith(prefix)
+        ? base
+        : `${prefix}${JSON.stringify([base, contentIndex])}`;
+}
 export function toRealtimeFunctionTool(definition) {
     return {
         type: "function",
@@ -717,6 +1008,8 @@ function toolOutcomePayload(outcome) {
 }
 function contextText(event) {
     switch (event.type) {
+        case "tool.status":
+            return toolStatusContext(event);
         case "task.progress":
             return `Background task ${event.taskId} progress: ${JSON.stringify(event.update)}`;
         case "task.completed":
@@ -727,6 +1020,16 @@ function contextText(event) {
             return `Background task ${event.taskId} was cancelled: ${event.reason}`;
     }
 }
+function taskResultPayload(event) {
+    switch (event.type) {
+        case "task.completed":
+            return JSON.stringify(event.result);
+        case "task.failed":
+            return JSON.stringify({ status: "failed", error: event.error });
+        case "task.cancelled":
+            return JSON.stringify({ status: "cancelled", reason: event.reason });
+    }
+}
 function toBase64(data) {
     let binary = "";
     // Bound each argument list; microphone buffers need not fit the engine's call stack.