videobook 0.0.0-stage → 0.2.0

package/dist/voice-session.js

@@ -0,0 +1,992 @@
+import { createOpenAi } from "./usage-providers.js";
+import { randomBytes, randomUUID, timingSafeEqual } from "node:crypto";
+import { OpenAiError } from "@apicity/openai";
+import { connectOpenAiRealtime } from "@apicity/openai/realtime";
+import { XaiError } from "@apicity/xai";
+import { connectXaiRealtime } from "@apicity/xai/realtime";
+import { z } from "zod";
+import { getApiKey } from "./api-key.js";
+import { withBookContext } from "./book-context.js";
+import { discoverVoiceActions, voiceWorkspaceCells, voiceWorkspaceContext, voiceWorkspaceSignature } from "./voice-actions.js";
+import { DEFAULT_VOICE_MODEL, DEFAULT_XAI_VOICE_MODEL, VOICE_TRANSCRIPTION_MODEL, VOICE_ACTIVATION_PHRASE, VOICE_PCM_RATE, openAiVoiceActivationEvent, voiceInstructions, voiceSessionConfig, xaiVoiceActivationEvent, xaiVoiceSessionConfig } from "./voice-config.js";
+import { recordVoiceUsage, recordTranscriptionUsage } from "./provider-usage.js";
+import { chatActionPlanSchema, prepareVoiceChatAction } from "./tools/chat.js";
+import { appendVoiceHistory } from "./tools/chat-history.js";
+import { appendDiagnostic } from "./diagnostics.js";
+import { VoiceReplyProgress } from "./voice-reply-progress.js";
+import { isVoiceTurnContinuation, joinVoiceTranscripts } from "./voice-continuation.js";
+class VoiceError extends Error {
+    status;
+    constructor(message, status) {
+        super(message);
+        this.status = status;
+    }
+}
+const providerErrorSchema = z.object({ code: z.string().nullish(), type: z.string().nullish() });
+const quotaMessage = "OpenAI API quota is exhausted. Check credits and usage limits in OpenAI settings before restarting voice.";
+const rateLimitMessage = "OpenAI is temporarily rate limiting voice. Wait before starting voice again.";
+const providerMessages = new Map([
+    ["credit_balance_exhausted", "OpenAI API credits are exhausted. Add credits in OpenAI billing, then start voice again."],
+    ["organization_spend_limit_exceeded", "The OpenAI organization spend limit has been reached. Review the limit in OpenAI settings before restarting voice."],
+    ["project_spend_limit_exceeded", "The OpenAI project spend limit has been reached. Review the project limit in OpenAI settings before restarting voice."],
+    ["organization_usage_limit_exceeded", "The OpenAI organization usage limit has been reached. Review usage limits in OpenAI settings before restarting voice."],
+    ["insufficient_quota", quotaMessage],
+    ["rate_limit_exceeded", rateLimitMessage],
+    ["slow_down", rateLimitMessage],
+]);
+function providerErrorMessage(details, fallbackCode, provider = "openai") {
+    const parsed = providerErrorSchema.safeParse(details);
+    const code = (parsed.success ? parsed.data.code : undefined) ?? fallbackCode;
+    const message = (code ? providerMessages.get(code) : undefined)
+        ?? (parsed.success && parsed.data.type === "insufficient_quota" ? quotaMessage : undefined)
+        ?? (parsed.success && parsed.data.type === "rate_limit_error" ? rateLimitMessage : undefined);
+    return message?.replaceAll("OpenAI", provider === "xai" ? "xAI" : "OpenAI");
+}
+export function voiceError(error) {
+    if (error instanceof VoiceError)
+        return error;
+    if (error instanceof XaiError) {
+        const body = z.object({ error: z.unknown() }).safeParse(error.body);
+        const message = providerErrorMessage(body.success ? body.data.error : undefined, error.code, "xai");
+        if (message)
+            return new VoiceError(message, error.status);
+        if (error.status === 401 || error.status === 403)
+            return new VoiceError("xAI voice access was denied. Check XAI_API_KEY and voice model access on the server.", 503);
+        if (error.status === 429)
+            return new VoiceError("xAI voice reached an account limit. Check xAI credits and usage limits before retrying.", 429);
+        return new VoiceError("xAI voice could not connect. Check the voice model and network, then try again.", 502);
+    }
+    if (error instanceof OpenAiError) {
+        const body = z.object({ error: z.unknown() }).safeParse(error.body);
+        const message = providerErrorMessage(body.success ? body.data.error : undefined, error.code);
+        if (message)
+            return new VoiceError(message, error.status);
+        if (error.status === 401 || error.status === 403)
+            return new VoiceError("OpenAI voice access was denied. Check OPENAI_API_KEY and model access on the server.", 503);
+        if (error.status === 429)
+            return new VoiceError("OpenAI rejected voice because of an account limit. Check OpenAI billing and usage limits before retrying.", 429);
+        return new VoiceError("OpenAI voice could not connect. Check the voice model and network, then try again.", 502);
+    }
+    return new VoiceError("Voice disconnected. Start voice again to reconnect.", 502);
+}
+function deferred() {
+    let resolve;
+    const promise = new Promise((done) => { resolve = done; });
+    return { promise, resolve };
+}
+const actionSchema = z.object({ tool: z.string().min(1).max(120), params: z.record(z.string(), z.unknown()) }).strict();
+const discoverySchema = z.object({ query: z.string().min(1).max(160) }).strict();
+const reviewSchema = z.object({ decision: z.enum(["apply", "cancel", "revise"]), prompt: z.string().max(8000).optional(), model: z.string().max(200).optional(), settings: z.record(z.string().max(120), z.union([z.string().max(400), z.number(), z.boolean()])).optional() }).strict();
+export class VoiceSession {
+    books;
+    bookId;
+    notebookId;
+    engine;
+    idleTimeoutMs;
+    onClose;
+    providerName;
+    inputRate;
+    id = randomUUID();
+    token = randomBytes(32).toString("hex");
+    model;
+    provider;
+    control = null;
+    // Provider-specific, because the two realtime APIs take different session
+    // shapes. Set once the connection is open.
+    pushSession = null;
+    sendActivation = null;
+    sendContextNote = null;
+    workspaceSignature = null;
+    callId = null;
+    negotiation = null;
+    abort = new AbortController();
+    stopped = false;
+    closing = null;
+    listener = null;
+    backlog = [];
+    seen = new Set();
+    captions = new Map();
+    turns = new Map();
+    currentTurn = "";
+    responseTurns = new Map();
+    responseItems = new Map();
+    interruptedResponses = new Set();
+    pendingAssistantHistory = new Map();
+    activeResponse = false;
+    greeted = false;
+    activationResponseId = null;
+    activationItems = new Set();
+    activationPending = false;
+    activationCaptionFinal = false;
+    replyProgress = new VoiceReplyProgress();
+    abandonedResponses = new Set();
+    recoveringTurn = null;
+    lastAction = null;
+    needsResponse = false;
+    speaking = false;
+    waitingAction = null;
+    reviewCall = null;
+    toolChain = Promise.resolve();
+    pendingTools = 0;
+    audioAttached = false;
+    audioClose;
+    writes = Promise.resolve();
+    lastActivity = Date.now();
+    startedAt = Date.now();
+    usage = { responses: 0, inputTokens: 0, outputTokens: 0 };
+    pendingUsage = new Set();
+    timer;
+    constructor(books, bookId, notebookId, engine, apiKey, idleTimeoutMs, onClose, providerName = "openai", inputRate = VOICE_PCM_RATE) {
+        this.books = books;
+        this.bookId = bookId;
+        this.notebookId = notebookId;
+        this.engine = engine;
+        this.idleTimeoutMs = idleTimeoutMs;
+        this.onClose = onClose;
+        this.providerName = providerName;
+        this.inputRate = inputRate;
+        this.model = providerName === "xai" ? process.env.VIDEOBOOK_XAI_VOICE_MODEL?.trim() || DEFAULT_XAI_VOICE_MODEL : process.env.VIDEOBOOK_VOICE_MODEL?.trim() || DEFAULT_VOICE_MODEL;
+        this.provider = createOpenAi({ apiKey, timeout: 20_000 });
+        this.apiKey = apiKey;
+        this.timer = setInterval(() => {
+            if (!this.scopeAlive())
+                void this.close("Workspace closed. Start voice in the current project.");
+            else if (Date.now() - this.lastActivity > this.idleTimeoutMs)
+                void this.close("Comms idle. Start voice to resume.");
+            else if (Date.now() - this.startedAt > 30 * 60_000)
+                void this.close("Comms session ended. Start voice to resume.");
+            else {
+                try {
+                    this.recoverReply();
+                    this.syncWorkspace();
+                }
+                catch (error) {
+                    this.fail(voiceError(error).message);
+                }
+            }
+        }, Math.min(1000, idleTimeoutMs));
+        this.timer.unref();
+    }
+    apiKey;
+    owns(token) {
+        const candidate = Buffer.from(token);
+        const expected = Buffer.from(this.token);
+        return candidate.length === expected.length && timingSafeEqual(candidate, expected);
+    }
+    scopeAlive() {
+        const book = this.books.get(this.bookId);
+        if (!book.ok || book.value.engine !== this.engine)
+            return false;
+        return !this.notebookId || this.engine.notebooks.read(this.notebookId).ok;
+    }
+    async start(sdp, signal) {
+        const onAbort = () => { void this.close("Voice start cancelled."); };
+        signal.addEventListener("abort", onAbort, { once: true });
+        try {
+            if (signal.aborted || this.stopped)
+                throw new VoiceError("Voice start cancelled.", 499);
+            if (this.providerName === "xai")
+                return await this.startXai();
+            this.negotiation = this.provider.post.v1.realtime.calls({
+                sdp, session: voiceSessionConfig(voiceWorkspaceContext(this.engine, this.notebookId), this.model),
+            });
+            const call = await this.negotiation;
+            this.callId = call.callId;
+            if (this.stopped) {
+                await this.provider.post.v1.realtime.calls.hangup(call.callId).catch(() => undefined);
+                throw new VoiceError("Voice start cancelled.", 499);
+            }
+            const connection = await connectOpenAiRealtime({
+                apiKey: this.apiKey, callId: call.callId, signal: this.abort.signal, timeout: 10_000,
+                onEvent: (event) => { try {
+                    this.receive(event);
+                }
+                catch {
+                    this.fail("Voice could not process an event. Start voice to reconnect.");
+                } },
+                onError: (error) => { this.fail(voiceError(error).message); },
+                onClose: () => { if (!this.stopped)
+                    this.fail("Voice connection lost. Start voice to reconnect."); },
+            });
+            this.control = connection;
+            if (this.stopped)
+                throw new VoiceError("Voice start cancelled.", 499);
+            // Only instructions change mid-call; resending model or audio settings
+            // would renegotiate a live call for nothing.
+            this.pushSession = (context) => connection.send({
+                type: "session.update",
+                session: { type: "realtime", instructions: voiceInstructions(context) },
+            });
+            this.sendActivation = () => connection.send(openAiVoiceActivationEvent());
+            this.sendContextNote = (text) => connection.send({ type: "conversation.item.create", item: { type: "message", role: "system", content: [{ type: "input_text", text }] } });
+            this.workspaceSignature = this.signature();
+            return { id: this.id, token: this.token, provider: "openai", transport: "webrtc", sdp: call.sdp, model: this.model, idleTimeoutMs: this.idleTimeoutMs };
+        }
+        finally {
+            signal.removeEventListener("abort", onAbort);
+        }
+    }
+    async startXai() {
+        const configured = deferred();
+        let setupError;
+        // Unset in production: voice talks to xAI. Set, it points the same client at
+        // another realtime endpoint -- a local gateway, or the integration test's
+        // server, which is the only way to watch what we send the model.
+        const endpoint = process.env.VIDEOBOOK_XAI_REALTIME_URL?.trim();
+        const connection = await connectXaiRealtime({
+            apiKey: this.apiKey, model: this.model, signal: this.abort.signal, timeout: 10_000,
+            ...(endpoint ? { baseURL: endpoint } : {}),
+            onEvent: event => {
+                if (event.type === "session.updated")
+                    configured.resolve();
+                if (event.type === "error") {
+                    setupError = new XaiError("xAI voice configuration failed", 502, { error: event.error });
+                    configured.resolve();
+                }
+                try {
+                    this.receive(event);
+                }
+                catch {
+                    this.fail("Voice could not process an event. Start voice to reconnect.");
+                }
+            },
+            onError: error => { setupError = error; configured.resolve(); if (this.control)
+                this.fail(voiceError(error).message); },
+            onClose: () => { configured.resolve(); if (!this.stopped)
+                this.fail("xAI voice connection lost. Start voice to reconnect."); },
+        });
+        this.control = connection;
+        if (this.stopped) {
+            connection.close();
+            throw new VoiceError("Voice start cancelled.", 499);
+        }
+        this.pushSession = (context) => connection.send({
+            type: "session.update",
+            session: xaiVoiceSessionConfig(context, this.inputRate),
+        });
+        this.sendActivation = () => connection.send(xaiVoiceActivationEvent());
+        this.sendContextNote = (text) => connection.send({ type: "conversation.item.create", item: { type: "message", role: "user", content: [{ type: "input_text", text }] } });
+        this.pushSession(voiceWorkspaceContext(this.engine, this.notebookId));
+        this.workspaceSignature = this.signature();
+        const deadline = setTimeout(() => { setupError = new XaiError("xAI voice setup timed out", 504); configured.resolve(); }, 10_000);
+        const aborted = () => configured.resolve();
+        this.abort.signal.addEventListener("abort", aborted, { once: true });
+        try {
+            await configured.promise;
+            if (setupError)
+                throw setupError;
+            if (this.stopped)
+                throw new VoiceError("Voice start cancelled.", 499);
+            return { id: this.id, token: this.token, provider: "xai", transport: "pcm", sampleRate: VOICE_PCM_RATE, model: this.model, idleTimeoutMs: this.idleTimeoutMs };
+        }
+        finally {
+            clearTimeout(deadline);
+            this.abort.signal.removeEventListener("abort", aborted);
+        }
+    }
+    attachAudio(close) {
+        if (this.stopped || this.providerName !== "xai" || this.audioAttached)
+            throw new VoiceError("Voice audio is unavailable or already connected.", 409);
+        this.audioAttached = true;
+        this.audioClose = close;
+    }
+    appendAudio(audio) {
+        if (this.stopped || !this.audioAttached || this.providerName !== "xai")
+            throw new VoiceError("Voice audio is closed.", 409);
+        if (audio.length === 0 || audio.length > 16_000 || audio.length % 2 !== 0)
+            throw new VoiceError("Invalid voice audio frame.", 400);
+        this.control?.send({ type: "input_audio_buffer.append", audio: audio.toString("base64") });
+    }
+    interruptPlayback(responseId, itemId, audioEndMs) {
+        this.interruptTranscript(responseId);
+        if (this.stopped || this.providerName !== "xai" || this.abandonedResponses.has(responseId) || !itemId || audioEndMs === undefined || !this.responseItems.get(responseId)?.has(itemId))
+            return;
+        this.control?.send({ type: "conversation.item.truncate", item_id: itemId, content_index: 0, audio_end_ms: audioEndMs });
+    }
+    subscribe(listener) {
+        if (this.listener || this.stopped)
+            throw new VoiceError("Voice session is already connected or closed.", 409);
+        this.listener = listener;
+        this.backlog.splice(0).forEach(listener);
+    }
+    emit(event) {
+        if (this.listener)
+            this.listener(event);
+        else {
+            this.backlog.push(event);
+            if (this.backlog.length > 256)
+                this.fail("Voice event connection was not established. Start voice again.");
+        }
+    }
+    greet() {
+        if (this.greeted || this.stopped || !this.control || !this.sendActivation)
+            return;
+        this.greeted = true;
+        if (this.currentTurn || this.speaking || this.activeResponse || this.needsResponse || this.pendingTools > 0 || this.waitingAction || this.reviewCall)
+            return;
+        this.activationPending = true;
+        this.activeResponse = true;
+        this.sendActivation();
+    }
+    isLineResponse(responseId) {
+        return !!responseId && responseId === this.activationResponseId;
+    }
+    dropLine() {
+        this.activationPending = false;
+        this.activeResponse = false;
+    }
+    finalizeActivationCaption(responseId) {
+        if (!responseId || responseId !== this.activationResponseId || this.activationCaptionFinal)
+            return;
+        const id = [...(this.responseItems.get(responseId) ?? [])][0] ?? `activation:${responseId}`;
+        this.activationItems.add(id);
+        this.caption(id, "assistant", VOICE_ACTIVATION_PHRASE, true);
+        this.activationCaptionFinal = true;
+    }
+    turn(id) {
+        let turn = this.turns.get(id);
+        if (!turn) {
+            turn = { text: "", saved: deferred(), final: false };
+            this.turns.set(id, turn);
+        }
+        return turn;
+    }
+    save(operation) {
+        this.writes = this.writes.then(async () => {
+            if (!this.scopeAlive())
+                return;
+            await operation();
+        }).catch(() => { this.fail("Could not save voice history. Voice stopped; your typed draft is safe."); });
+    }
+    caption(id, role, text, final, userTurnId) {
+        if (this.captions.get(id)?.final)
+            return;
+        const value = { id, role, text: text.slice(0, 16_000), final };
+        this.captions.set(id, value);
+        this.emit({ type: "caption", caption: value });
+        if (!final)
+            return;
+        if (role === "assistant" && userTurnId && !this.turn(userTurnId).final) {
+            const waiting = this.pendingAssistantHistory.get(userTurnId) ?? [];
+            waiting.push(value);
+            this.pendingAssistantHistory.set(userTurnId, waiting);
+            return;
+        }
+        const turn = role === "user" ? this.turn(id) : null;
+        if (turn) {
+            turn.final = true;
+            turn.text = value.text;
+        }
+        this.save(async () => {
+            if (turn && value.text.trim()) {
+                const result = await this.engine.prompts.record({ surface: "chat", prompt: value.text, context: { source: "voice", sessionId: this.id, turnId: id, model: this.model, notebookId: this.notebookId } });
+                if (!result.ok) {
+                    turn.saved.resolve(false);
+                    throw new Error(result.error.message);
+                }
+            }
+            if (!this.activationItems.has(value.id)) {
+                await appendVoiceHistory(this.engine, { ...value, sessionId: this.id, notebookId: this.notebookId, model: this.model });
+            }
+            if (turn) {
+                for (const pending of this.pendingAssistantHistory.get(id) ?? []) {
+                    const latest = this.captions.get(pending.id) ?? pending;
+                    await appendVoiceHistory(this.engine, { ...latest, text: latest.interrupted ? "" : latest.text, sessionId: this.id, notebookId: this.notebookId, model: this.model });
+                }
+                this.pendingAssistantHistory.delete(id);
+            }
+            turn?.saved.resolve(true);
+        });
+    }
+    receive(event) {
+        if (this.stopped)
+            return;
+        if (event.event_id && this.seen.has(event.event_id))
+            return;
+        if (event.event_id)
+            this.seen.add(event.event_id);
+        if (this.seen.size > 20_000) {
+            void this.close("Comms session limit reached. Start voice to resume.");
+            return;
+        }
+        if (!this.scopeAlive()) {
+            void this.close("Workspace closed.");
+            return;
+        }
+        const responseId = event.response_id ?? (typeof event.response?.id === "string" ? event.response.id : undefined);
+        if (responseId && this.abandonedResponses.has(responseId) && event.type !== "response.done")
+            return;
+        if (responseId)
+            this.replyProgress.progress(responseId, Date.now());
+        const id = event.item_id ?? "";
+        if (id && event.response_id && (event.type.startsWith("response.output_audio") || event.type.startsWith("response.audio."))) {
+            const items = this.responseItems.get(event.response_id) ?? new Set();
+            items.add(id);
+            this.responseItems.set(event.response_id, items);
+            if (this.interruptedResponses.has(event.response_id))
+                return;
+        }
+        if (event.type === "input_audio_buffer.speech_started") {
+            this.lastActivity = Date.now();
+            this.currentTurn = id;
+            this.replyProgress.beginTurn();
+            this.recoveringTurn = null;
+            this.turn(id);
+            this.caption(id, "user", "", false);
+            this.speaking = true;
+            this.needsResponse = false;
+            this.activeResponse = false;
+            this.emit({ type: "activity", phase: "listening", interrupt: this.providerName === "xai" });
+        }
+        else if (event.type === "input_audio_buffer.speech_stopped") {
+            this.speaking = false;
+            this.lastActivity = Date.now();
+            this.replyProgress.expect(this.lastActivity);
+            this.emit({ type: "activity", phase: "thinking" });
+        }
+        else if (event.type === "conversation.item.input_audio_transcription.delta") {
+            this.caption(id, "user", (this.captions.get(id)?.text ?? "") + (event.delta ?? ""), false);
+        }
+        else if (event.type === "conversation.item.input_audio_transcription.updated") {
+            this.caption(id, "user", event.transcript ?? "", false);
+        }
+        else if (event.type === "conversation.item.input_audio_transcription.completed") {
+            if (id && this.providerName === "openai")
+                recordTranscriptionUsage(this.engine, VOICE_TRANSCRIPTION_MODEL, this.id, id, event);
+            this.caption(id, "user", event.transcript ?? "", true);
+        }
+        else if (event.type === "conversation.item.input_audio_transcription.failed") {
+            if (id && this.providerName === "openai")
+                recordTranscriptionUsage(this.engine, VOICE_TRANSCRIPTION_MODEL, this.id, id, event);
+            this.fail(providerErrorMessage(event.error, undefined, this.providerName) ?? "Could not transcribe this turn. Start voice and try again.");
+        }
+        else if (this.providerName === "xai" && (event.type === "response.output_audio.delta" || event.type === "response.audio.delta")) {
+            if (event.response_id && event.item_id && event.delta && !this.interruptedResponses.has(event.response_id)) {
+                this.emit({ type: "audio", audio: event.delta, responseId: event.response_id, itemId: event.item_id });
+            }
+        }
+        else if (this.providerName === "xai" && (event.type === "response.output_audio.done" || event.type === "response.audio.done")) {
+            if (event.response_id)
+                this.emit({ type: "audio-done", responseId: event.response_id });
+        }
+        else if (event.type === "response.output_audio_transcript.delta") {
+            const line = this.isLineResponse(responseId);
+            if (line && id)
+                this.activationItems.add(id);
+            this.caption(id, "assistant", line ? VOICE_ACTIVATION_PHRASE : (this.captions.get(id)?.text ?? "") + (event.delta ?? ""), false);
+        }
+        else if (event.type === "response.output_audio_transcript.done") {
+            const line = this.isLineResponse(responseId);
+            if (line && id)
+                this.activationItems.add(id);
+            this.caption(id, "assistant", line ? VOICE_ACTIVATION_PHRASE : event.transcript ?? this.captions.get(id)?.text ?? "", true, event.response_id ? this.responseTurns.get(event.response_id) : undefined);
+            if (line)
+                this.activationCaptionFinal = true;
+        }
+        else if (event.type === "conversation.item.truncated") {
+            const caption = this.captions.get(id);
+            if (caption && !caption.interrupted) {
+                const interrupted = { ...caption, final: true, interrupted: true };
+                this.captions.set(id, interrupted);
+                this.emit({ type: "caption", caption: interrupted });
+                if (!this.activationItems.has(id)) {
+                    this.save(() => appendVoiceHistory(this.engine, { ...interrupted, text: "", sessionId: this.id, notebookId: this.notebookId, model: this.model }));
+                }
+            }
+        }
+        else if (event.type === "response.created") {
+            this.activeResponse = true;
+            if (responseId) {
+                this.pendingUsage.add(responseId);
+                recordVoiceUsage(this.engine, this.providerName, this.model, this.id, responseId, { ...event.response, status: "in_progress" });
+                this.responseTurns.set(responseId, this.currentTurn);
+                this.replyProgress.started(responseId, Date.now());
+                if (this.activationPending && (this.providerName === "xai" || event.response?.metadata?.purpose === "activation")) {
+                    this.activationPending = false;
+                    this.activationResponseId = responseId;
+                }
+            }
+        }
+        else if (event.type === "response.done") {
+            if (responseId) {
+                this.pendingUsage.delete(responseId);
+                recordVoiceUsage(this.engine, this.providerName, this.model, this.id, responseId, event.response);
+            }
+            const completion = this.replyProgress.finished(responseId, Date.now());
+            const usage = event.response?.usage;
+            this.usage.responses += 1;
+            this.usage.inputTokens += typeof usage?.input_tokens === "number" ? usage.input_tokens : 0;
+            this.usage.outputTokens += typeof usage?.output_tokens === "number" ? usage.output_tokens : 0;
+            if (completion === "stale")
+                return;
+            this.activeResponse = false;
+            if (completion === "retry")
+                this.needsResponse = true;
+            if (event.response?.status === "failed") {
+                const details = z.object({ error: z.unknown().optional() }).safeParse(event.response.status_details);
+                const guidance = providerErrorMessage(details.success ? details.data.error : undefined, undefined, this.providerName);
+                if (guidance) {
+                    this.fail(guidance);
+                    return;
+                }
+                if (this.activationPending || this.isLineResponse(responseId)) {
+                    this.dropLine();
+                    return;
+                }
+                this.fail(guidance ?? "The voice reply failed. Start voice to retry.");
+                return;
+            }
+            this.finalizeActivationCaption(responseId);
+            this.continueResponse();
+        }
+        else if (event.type === "response.function_call_arguments.done" && event.call_id) {
+            const key = `call:${event.call_id}`;
+            if (this.seen.has(key))
+                return;
+            this.seen.add(key);
+            const turnId = (event.response_id && this.responseTurns.get(event.response_id)) || this.currentTurn;
+            this.pendingTools += 1;
+            this.toolChain = this.toolChain.then(() => this.runTool(event, turnId)).catch(() => { this.fail("Voice action failed. Check the command and try again."); }).finally(() => { this.pendingTools -= 1; this.continueResponse(); });
+        }
+        else if (event.type === "error") {
+            if (event.error?.code === "response_cancel_not_active") {
+                if (this.replyProgress.cancelled(Date.now())) {
+                    this.activeResponse = false;
+                    this.needsResponse = true;
+                    this.continueResponse();
+                }
+                return;
+            }
+            const guidance = providerErrorMessage(event.error, undefined, this.providerName);
+            if (guidance) {
+                this.fail(guidance);
+                return;
+            }
+            if (this.activationPending || this.isLineResponse(responseId) || event.error?.code === "conversation_already_has_active_response") {
+                this.dropLine();
+                return;
+            }
+            this.fail("Voice service reported an error. Start voice again; check model access if it repeats.");
+        }
+    }
+    continueResponse() {
+        if (!this.needsResponse || this.activeResponse || this.pendingTools > 0 || this.speaking || this.stopped)
+            return;
+        this.needsResponse = false;
+        this.activeResponse = true;
+        this.replyProgress.expect(Date.now());
+        this.control?.send({ type: "response.create" });
+    }
+    recoverReply() {
+        if (this.stopped || !this.control || !this.currentTurn || this.speaking || this.pendingTools > 0)
+            return;
+        const recovery = this.replyProgress.check(Date.now());
+        if (!recovery)
+            return;
+        if (recovery.type === "failed") {
+            this.fail("The voice service stopped responding. Start voice to try again.");
+            return;
+        }
+        this.emit({ type: "activity", phase: "thinking", interrupt: this.providerName === "xai", retry: true });
+        this.recoveringTurn = this.currentTurn;
+        if (recovery.responseId) {
+            this.abandonedResponses.add(recovery.responseId);
+            this.interruptTranscript(recovery.responseId);
+            this.control.send({ type: "response.cancel", response_id: recovery.responseId });
+            return;
+        }
+        this.activeResponse = false;
+        this.needsResponse = true;
+        this.continueResponse();
+    }
+    signature() {
+        try {
+            return voiceWorkspaceSignature(this.engine, this.notebookId);
+        }
+        catch {
+            return this.workspaceSignature ?? "";
+        }
+    }
+    /**
+     * Push the board back into the live session when it has actually changed.
+     * The instructions carry a snapshot taken at connect, and a session lasts half
+     * an hour, so without this the model argues about cells it made itself. Only
+     * between turns: a session.update mid-response would land in the middle of the
+     * model's own sentence.
+     */
+    syncWorkspace() {
+        if (!this.pushSession || !this.control || this.stopped)
+            return;
+        if (this.activeResponse || this.speaking || this.pendingTools > 0 || this.needsResponse)
+            return;
+        const signature = this.signature();
+        if (signature === this.workspaceSignature)
+            return;
+        this.workspaceSignature = signature;
+        this.pushSession(voiceWorkspaceContext(this.engine, this.notebookId));
+    }
+    output(callId, result, turnId) {
+        if (this.stopped)
+            return;
+        const serialized = JSON.stringify(this.withWorkspace(result));
+        const output = serialized.length <= 40_000 ? serialized : JSON.stringify({ truncated: true, resultText: serialized.slice(0, 36_000) });
+        this.control?.send({ type: "conversation.item.create", item: { type: "function_call_output", call_id: callId, output } });
+        if (turnId === this.currentTurn) {
+            this.needsResponse = true;
+            this.continueResponse();
+        }
+    }
+    /**
+     * A tool call outlives the turn that asked for it whenever the speaker keeps
+     * talking -- a second clause, or just "hello?" -- because every new speech
+     * segment starts a new turn. Continuation clauses reuse the pending
+     * prepare_workspace_action against the joined transcript. A genuinely new
+     * request still has to be answered so the model does not hang believing work
+     * is running: say it was dropped and let the model re-plan. The stale turn
+     * never provokes a reply of its own; output() only continues the response
+     * for the current turn.
+     */
+    abandon(callId, turnId) {
+        this.output(callId, {
+            status: "failed",
+            message: "Dropped: the speaker talked again, so this call belongs to an earlier turn and did not go through. Nothing changed in the workspace and nothing is running. Answer the newest request, repeating this action for it if it is still wanted.",
+        }, turnId);
+    }
+    async waitTurnText(turnId) {
+        const turn = this.turn(turnId);
+        if (turn.final)
+            return turn.text;
+        const timeout = setTimeout(() => turn.saved.resolve(false), 10_000);
+        await turn.saved.promise.finally(() => clearTimeout(timeout));
+        return turn.text;
+    }
+    /**
+     * Bind a superseded prepare_workspace_action to the conversation: join the
+     * original turn with later continuation clauses, or signal abandon when the
+     * speaker started a new request.
+     */
+    async resolvePreparePrompt(originalTurnId) {
+        const original = (await this.waitTurnText(originalTurnId)).trim();
+        if (this.stopped)
+            return null;
+        let latestId = this.currentTurn || originalTurnId;
+        let latest = latestId === originalTurnId ? original : (await this.waitTurnText(latestId)).trim();
+        if (this.stopped)
+            return null;
+        if (this.currentTurn && this.currentTurn !== latestId) {
+            latestId = this.currentTurn;
+            latest = (await this.waitTurnText(latestId)).trim();
+        }
+        if (!latestId || latestId === originalTurnId) {
+            if (!original)
+                return null;
+            return { prompt: original, outputTurnId: originalTurnId };
+        }
+        if (!isVoiceTurnContinuation(original, latest))
+            return null;
+        return { prompt: joinVoiceTranscripts(original, latest), outputTurnId: latestId };
+    }
+    /**
+     * Every tool result carries the current board. The model reaches for a tool
+     * exactly when it is about to act on an address, so this is the cheapest
+     * moment to correct a stale picture. Cells only -- the conversation is
+     * already in its context and would crowd out the result itself.
+     */
+    withWorkspace(result) {
+        if (!result || typeof result !== "object" || Array.isArray(result))
+            return result;
+        try {
+            return { ...result, workspace: voiceWorkspaceCells(this.engine, this.notebookId) };
+        }
+        catch {
+            return result;
+        }
+    }
+    async runTool(event, turnId) {
+        const callId = event.call_id;
+        if (this.stopped)
+            return;
+        if (turnId !== this.currentTurn && event.name !== "prepare_workspace_action") {
+            this.abandon(callId, turnId);
+            return;
+        }
+        try {
+            if (!this.control || !turnId)
+                throw new Error("Wait for a spoken user request.");
+            const args = JSON.parse(event.arguments ?? "{}");
+            if (event.name === "find_workspace_actions") {
+                const { query } = discoverySchema.parse(args);
+                this.output(callId, { actions: discoverVoiceActions(this.engine, query) }, turnId);
+                return;
+            }
+            if (event.name === "resolve_pending_review") {
+                this.output(callId, await this.resolveReview(callId, reviewSchema.parse(args)), turnId);
+                return;
+            }
+            if (event.name !== "prepare_workspace_action")
+                throw new Error("Unknown voice action.");
+            if (this.recoveringTurn === turnId && this.lastAction?.turnId === turnId) {
+                this.output(callId, this.lastAction.outcome, turnId);
+                return;
+            }
+            if (this.reviewCall || this.waitingAction)
+                throw new Error("A review is already on screen. Use resolve_pending_review to apply, cancel, or revise it before requesting another workspace action.");
+            const resolved = await this.resolvePreparePrompt(turnId);
+            if (this.stopped)
+                return;
+            if (!resolved?.prompt) {
+                if (turnId !== this.currentTurn) {
+                    this.abandon(callId, turnId);
+                    return;
+                }
+                throw new Error("The user transcript could not be saved. Ask the user to repeat the request.");
+            }
+            let prompt = resolved.prompt;
+            let outputTurnId = resolved.outputTurnId;
+            if (!this.scopeAlive())
+                throw new Error("The workspace closed before this action ran. Nothing was prepared.");
+            const parsed = actionSchema.parse(args);
+            if (parsed.params.book_id !== undefined && parsed.params.book_id !== this.bookId)
+                throw new Error("Voice actions must stay in the current project.");
+            if (parsed.params.notebook_id !== undefined && parsed.params.notebook_id !== this.notebookId)
+                throw new Error("Voice actions must stay in the current notebook.");
+            let result;
+            for (let attempt = 0; attempt < 3; attempt += 1) {
+                const decision = {
+                    ...parsed,
+                    message: prompt,
+                    params: typeof parsed.params.prompt === "string"
+                        ? { ...parsed.params, prompt }
+                        : parsed.params,
+                };
+                result = await withBookContext({ bookId: this.bookId, engine: this.engine }, () => prepareVoiceChatAction(this.engine, { book_id: this.bookId, notebook_id: this.notebookId, prompt }, decision));
+                if (this.stopped)
+                    return;
+                if (this.currentTurn && this.currentTurn !== outputTurnId) {
+                    const newer = (await this.waitTurnText(this.currentTurn)).trim();
+                    if (!isVoiceTurnContinuation(prompt, newer)) {
+                        this.abandon(callId, turnId);
+                        return;
+                    }
+                    prompt = joinVoiceTranscripts(prompt, newer);
+                    outputTurnId = this.currentTurn;
+                    continue;
+                }
+                break;
+            }
+            if (!result) {
+                this.abandon(callId, turnId);
+                return;
+            }
+            const raw = result.content[0]?.text ?? "null";
+            if (result.isError) {
+                this.output(callId, { status: "failed", message: raw }, outputTurnId);
+                return;
+            }
+            const value = JSON.parse(raw);
+            const plan = chatActionPlanSchema.safeParse(value);
+            if (!plan.success) {
+                this.output(callId, { status: "completed", result: value }, outputTurnId);
+                return;
+            }
+            const done = deferred();
+            this.waitingAction = { callId, kind: "action", done };
+            this.emit({ type: "action", callId, prompt, plan: plan.data });
+            const deadline = setTimeout(() => done.resolve({ status: "failed", message: "The command UI did not acknowledge this action. Check its current status before retrying." }), 60_000);
+            const outcome = await done.promise.finally(() => clearTimeout(deadline));
+            this.lastAction = { turnId: outputTurnId, callId, outcome };
+            if (this.waitingAction?.callId === callId)
+                this.waitingAction = null;
+            if (outcome.status === "review")
+                this.reviewCall = callId;
+            this.output(callId, outcome, outputTurnId);
+        }
+        catch (error) {
+            this.output(callId, { status: "failed", message: error instanceof z.ZodError ? "Invalid action parameters. Use the discovered schema." : error instanceof Error ? error.message : "The voice action failed." }, turnId);
+        }
+    }
+    /**
+     * Hand a spoken decision about the on-screen review to the command UI and
+     * wait for what actually happened. The review stays open on a revise, so the
+     * speaker can keep changing it before spending anything.
+     */
+    async resolveReview(callId, command) {
+        if (!this.reviewCall)
+            throw new Error("There is no review on screen to act on.");
+        if (this.waitingAction)
+            throw new Error("The command UI is still working on the last spoken decision.");
+        if (command.decision === "revise" && !command.prompt?.trim())
+            throw new Error("Revise needs the whole replacement prompt.");
+        const done = deferred();
+        this.waitingAction = { callId, kind: "review", done };
+        this.emit({ type: "review", callId, command });
+        const deadline = setTimeout(() => done.resolve({ status: "failed", message: "The command UI did not act on that. Check the review on screen." }), 60_000);
+        const outcome = await done.promise.finally(() => clearTimeout(deadline));
+        if (this.waitingAction?.callId === callId)
+            this.waitingAction = null;
+        if (outcome.status !== "review")
+            this.reviewCall = null;
+        return outcome;
+    }
+    acknowledge(callId, outcome) {
+        if (this.stopped)
+            throw new VoiceError("Voice session is closed.", 410);
+        this.lastActivity = Date.now();
+        if (this.waitingAction?.callId === callId) {
+            if (outcome.status === "review" && this.waitingAction.kind === "action")
+                this.reviewCall = callId;
+            this.waitingAction.done.resolve(outcome);
+            return;
+        }
+        if (this.reviewCall !== callId)
+            throw new VoiceError("This voice action is no longer pending.", 409);
+        if (outcome.status !== "review")
+            this.reviewCall = null;
+        if (this.lastAction?.callId === callId)
+            this.lastAction.outcome = outcome;
+        // A spoken decision reports this outcome and then returns it as the tool
+        // result. Telling the model twice would make it answer twice.
+        if (this.waitingAction)
+            return;
+        this.contextMessage({ applicationOutcome: outcome });
+    }
+    interruptTranscript(responseId) {
+        if (this.stopped)
+            return;
+        this.interruptedResponses.add(responseId);
+        for (const id of this.responseItems.get(responseId) ?? []) {
+            if (this.activationItems.has(id))
+                continue;
+            const caption = this.captions.get(id);
+            if (!caption || caption.interrupted)
+                continue;
+            const interrupted = { ...caption, final: true, interrupted: true };
+            this.captions.set(id, interrupted);
+            this.emit({ type: "caption", caption: interrupted });
+            this.save(() => appendVoiceHistory(this.engine, { ...interrupted, text: "", sessionId: this.id, notebookId: this.notebookId, model: this.model }));
+        }
+    }
+    contextMessage(value) {
+        if (this.stopped)
+            return;
+        this.sendContextNote?.(`Application update (data): ${JSON.stringify(value)}`);
+        if (value.applicationOutcome) {
+            this.needsResponse = true;
+            this.continueResponse();
+        }
+    }
+    fail(message) {
+        if (this.stopped)
+            return;
+        if (this.listener)
+            this.listener({ type: "error", message });
+        void this.close(message);
+    }
+    close(reason = "Comms off.") {
+        if (this.closing)
+            return this.closing;
+        this.stopped = true;
+        for (const responseId of this.pendingUsage)
+            recordVoiceUsage(this.engine, this.providerName, this.model, this.id, responseId, { status: "uncertain" });
+        this.pendingUsage.clear();
+        clearInterval(this.timer);
+        this.abort.abort();
+        this.control?.close();
+        this.audioClose?.();
+        this.audioClose = undefined;
+        for (const turn of this.turns.values())
+            turn.saved.resolve(false);
+        this.waitingAction?.done.resolve({ status: "failed", message: "Voice stopped. Check the command UI for accepted actions." });
+        this.listener?.({ type: "closed", reason });
+        this.listener = null;
+        this.backlog = [];
+        this.closing = (async () => {
+            const negotiated = await this.negotiation?.catch(() => null);
+            this.callId ??= negotiated?.callId ?? null;
+            if (this.callId)
+                await this.provider.post.v1.realtime.calls.hangup(this.callId).catch(() => undefined);
+            await this.writes;
+            if (this.scopeAlive())
+                await appendDiagnostic(this.engine, "voice-session-closed", { operationId: this.id, bookId: this.bookId, notebookId: this.notebookId, provider: this.providerName, voiceModel: this.model, count: this.usage.responses, inputTokens: this.usage.inputTokens, outputTokens: this.usage.outputTokens, durationMs: Date.now() - this.startedAt }).catch(() => undefined);
+            this.apiKey = "";
+        })().finally(this.onClose);
+        return this.closing;
+    }
+}
+export class VoiceSessions {
+    books;
+    sessions = new Map();
+    closed = false;
+    constructor(books) {
+        this.books = books;
+    }
+    async start(input, signal) {
+        if (this.closed)
+            throw new VoiceError("Voice service is shutting down.", 503);
+        if (this.sessions.size >= 4)
+            throw new VoiceError("Four voice sessions are already active. Stop another session first.", 429);
+        const book = this.books.get(input.bookId);
+        if (!book.ok)
+            throw new VoiceError("Open an available project before starting voice.", 404);
+        if (input.notebookId && !book.value.engine.notebooks.read(input.notebookId).ok)
+            throw new VoiceError("This notebook is no longer available.", 404);
+        const configured = Number(process.env.VIDEOBOOK_VOICE_IDLE_TIMEOUT_MS ?? 120_000);
+        const idle = Number.isFinite(configured) ? Math.min(600_000, Math.max(10_000, configured)) : 120_000;
+        const attempt = async (provider) => {
+            if (signal.aborted || this.closed)
+                throw new VoiceError("Voice start cancelled.", 499);
+            if (this.sessions.size >= 4)
+                throw new VoiceError("Four voice sessions are already active. Stop another session first.", 429);
+            const keyName = provider === "xai" ? "XAI_API_KEY" : "OPENAI_API_KEY";
+            const apiKey = getApiKey(keyName);
+            if (!apiKey)
+                throw new VoiceError(`Voice needs ${keyName} on the server. Configure it, then start voice again.`, 503);
+            const session = new VoiceSession(this.books, input.bookId, input.notebookId, book.value.engine, apiKey, idle, () => this.sessions.delete(session.id), provider, input.sampleRate);
+            this.sessions.set(session.id, session);
+            try {
+                return await session.start(input.sdp ?? "", signal);
+            }
+            catch (error) {
+                await session.close();
+                throw voiceError(error);
+            }
+        };
+        if (input.mode !== "auto")
+            return await attempt(input.mode ?? "openai");
+        try {
+            return await attempt("openai");
+        }
+        catch (error) {
+            const first = voiceError(error);
+            if (signal.aborted || this.closed || !(first.status >= 500 || [401, 403, 404, 408, 429].includes(first.status)))
+                throw first;
+            try {
+                return { ...await attempt("xai"), fallbackReason: first.message };
+            }
+            catch (failure) {
+                const last = voiceError(failure);
+                if (signal.aborted || this.closed)
+                    throw last;
+                throw new VoiceError(`OpenAI: ${first.message} xAI: ${last.message}`, last.status);
+            }
+        }
+    }
+    get(id, token) {
+        const session = this.sessions.get(id);
+        if (!session || !session.owns(token))
+            throw new VoiceError("Voice session was not found. Start voice again.", 404);
+        return session;
+    }
+    async stop(id, token, interruptedResponseId) {
+        if (!this.sessions.has(id))
+            return;
+        const session = this.get(id, token);
+        if (interruptedResponseId)
+            session.interruptTranscript(interruptedResponseId);
+        await session.close();
+    }
+    async close() {
+        this.closed = true;
+        await Promise.all([...this.sessions.values()].map((session) => session.close("Server is shutting down.")));
+    }
+}