stelo 1.0.1

package/dist/cortex.js

@@ -0,0 +1,1479 @@
+"use strict";
+// ============================================================================
+// Cortex — Real-Time Agent Nervous System
+// ============================================================================
+//
+//  ██████  ██████  ██████  ████████ ███████ ██   ██
+// ██      ██    ██ ██   ██    ██    ██       ██ ██
+// ██      ██    ██ ██████     ██    █████     ███
+// ██      ██    ██ ██   ██    ██    ██       ██ ██
+//  ██████  ██████  ██   ██    ██    ███████ ██   ██
+//
+// The missing layer between real-time AI models and the physical computer.
+// Cortex gives a model eyes (screen), ears (audio), hands (mouse/keyboard),
+// and a voice — all flowing bidirectionally in real time.
+//
+// Provider-agnostic: works with Gemini Live, OpenAI Realtime, xAI, or any
+// WebSocket-based model. The Cortex doesn't care what model is behind it.
+// It just bridges senses ↔ intelligence ↔ actions.
+//
+// ============================================================================
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Cortex = void 0;
+exports.geminiLiveProvider = geminiLiveProvider;
+exports.openAIRealtimeProvider = openAIRealtimeProvider;
+exports.customProvider = customProvider;
+exports.createCortex = createCortex;
+const native = require('../index.js');
+class Cortex {
+    config = null;
+    transport = null;
+    _state = 'disconnected';
+    _turn = 'none';
+    _connected = false;
+    // Event listeners
+    listeners = new Map();
+    // Tool registry
+    tools = new Map();
+    pendingToolCalls = new Map();
+    // Sensory feeds
+    screenFeedInterval = null;
+    audioFeedInterval = null;
+    _screenFeedActive = false;
+    _audioFeedActive = false;
+    // Idle tracking
+    idleTimer = null;
+    lastActivityTime = 0;
+    // Session
+    _resumeToken = null;
+    _totalTokensUsed = 0;
+    // Accumulation buffers
+    modelTextBuffer = '';
+    userTranscriptBuffer = '';
+    // ═══ CONNECTION ═══════════════════════════════════════════════════════
+    /**
+     * Connect to a real-time AI model through a provider.
+     *
+     * @example
+     * ```typescript
+     * const cortex = new Cortex();
+     * await cortex.connect({
+     *   provider: geminiLiveProvider(),
+     *   connection: {
+     *     apiKey: process.env.GEMINI_API_KEY!,
+     *     model: 'gemini-3.1-flash-live-preview',
+     *     systemInstruction: 'You are a desktop assistant. Use tools to interact with the computer.',
+     *     responseModalities: ['audio'],
+     *     voice: 'Kore',
+     *   },
+     *   screenFeed: { fps: 1, quality: 60, scale: 0.5 },
+     *   audioFeed: { source: 'system' },
+     *   autoRegisterSteloTools: true,
+     * });
+     * ```
+     */
+    async connect(config) {
+        if (this._connected) {
+            await this.disconnect();
+        }
+        this.config = config;
+        // Merge custom tools into connection config
+        const allToolDefs = [...(config.connection.tools ?? [])];
+        for (const tool of config.tools ?? []) {
+            this.registerTool(tool.name, tool.handler, tool);
+            allToolDefs.push({ name: tool.name, description: tool.description, parameters: tool.parameters, nonBlocking: tool.nonBlocking });
+        }
+        // Auto-register Stelo tools
+        if (config.autoRegisterSteloTools) {
+            this.registerSteloTools();
+            for (const [, entry] of this.tools) {
+                allToolDefs.push(entry.definition);
+            }
+        }
+        const connectionConfig = { ...config.connection, tools: allToolDefs };
+        // Build transport event handlers
+        const handlers = {
+            audio: (data, sampleRate) => this.handleModelAudio(data, sampleRate),
+            text: (text, isFinal) => this.handleModelText(text, isFinal),
+            transcript: (text, source, isFinal) => this.handleTranscript(text, source, isFinal),
+            toolCall: (id, name, args) => this.handleToolCall(id, name, args),
+            turnComplete: () => this.handleTurnComplete(),
+            generationComplete: () => this.handleGenerationComplete(),
+            interrupted: () => this.handleInterrupted(),
+            thought: (text) => this.emit('thought', text),
+            usage: (total, input, output) => {
+                this._totalTokensUsed = total;
+                this.emit('usage', total, input, output);
+            },
+            error: (err) => this.emit('error', err),
+            close: (reason) => this.handleDisconnect(reason),
+            resumeToken: (token) => {
+                this._resumeToken = token;
+                this.emit('resumeToken', token);
+            },
+        };
+        // Connect
+        this.transport = await config.provider.connect(connectionConfig, handlers);
+        this._connected = true;
+        this.setState('idle');
+        this.setTurn('none');
+        this.touchActivity();
+        this.emit('connected');
+        // Auto-start sensory feeds
+        if (config.screenFeed) {
+            const screenConfig = config.screenFeed === true ? {} : config.screenFeed;
+            this.startScreenFeed(screenConfig);
+        }
+        if (config.audioFeed) {
+            const audioConfig = config.audioFeed === true ? { source: 'system' } : config.audioFeed;
+            this.startAudioFeed(audioConfig);
+        }
+        // Idle timeout
+        if (config.idleTimeoutMs && config.idleTimeoutMs > 0) {
+            this.startIdleTimer(config.idleTimeoutMs);
+        }
+    }
+    /**
+     * Disconnect from the model and stop all feeds.
+     */
+    async disconnect() {
+        this.stopScreenFeed();
+        this.stopAudioFeed();
+        this.stopIdleTimer();
+        if (this.transport) {
+            await this.transport.close();
+            this.transport = null;
+        }
+        this._connected = false;
+        this.setState('disconnected');
+        this.setTurn('none');
+        this.pendingToolCalls.clear();
+        this.emit('disconnected');
+    }
+    // ═══ SEND TO MODEL ════════════════════════════════════════════════════
+    /**
+     * Say something to the model (text).
+     *
+     * @example
+     * ```typescript
+     * cortex.say("Click the Save button");
+     * cortex.say("What do you see on the screen?");
+     * ```
+     */
+    say(text) {
+        this.ensureConnected();
+        this.transport.sendText(text, 'user');
+        this.setState('listening');
+        this.setTurn('user');
+        this.touchActivity();
+    }
+    /**
+     * Send raw audio to the model (PCM).
+     */
+    sendAudio(pcm, sampleRate = 16000) {
+        this.ensureConnected();
+        this.transport.sendAudio(pcm, sampleRate);
+        this.touchActivity();
+    }
+    /**
+     * Send a screenshot to the model right now.
+     *
+     * @example
+     * ```typescript
+     * cortex.sendScreenshot(); // sends current screen as JPEG
+     * ```
+     */
+    sendScreenshot(region) {
+        this.ensureConnected();
+        const jpeg = region
+            ? native.screenCapturePng(region.x, region.y, region.width, region.height)
+            : native.screenCapturePng();
+        this.transport.sendVideo(jpeg);
+        this.touchActivity();
+    }
+    /**
+     * Send a system instruction mid-session.
+     */
+    instruct(text) {
+        this.ensureConnected();
+        this.transport.sendText(text, 'system');
+        this.touchActivity();
+    }
+    /**
+     * Interrupt the model's current output.
+     * Stops any audio/text generation in progress.
+     *
+     * @example
+     * ```typescript
+     * cortex.on('modelAudio', () => {
+     *   if (userPressedStop) cortex.interrupt();
+     * });
+     * ```
+     */
+    interrupt() {
+        this.ensureConnected();
+        this.transport.interrupt();
+        this.setState('interrupted');
+        this.touchActivity();
+    }
+    /**
+     * Signal that the user started speaking (manual VAD).
+     */
+    activityStart() {
+        this.ensureConnected();
+        this.transport.sendActivityStart();
+        this.setState('listening');
+        this.setTurn('user');
+        this.touchActivity();
+    }
+    /**
+     * Signal that the user stopped speaking (manual VAD).
+     */
+    activityEnd() {
+        this.ensureConnected();
+        this.transport.sendActivityEnd();
+        this.touchActivity();
+    }
+    // ═══ SENSORY FEEDS ════════════════════════════════════════════════════
+    // Auto-feed screen/audio to the model at a configurable rate.
+    /**
+     * Start streaming screen frames to the model.
+     * The model "sees" the screen in real-time.
+     *
+     * @example
+     * ```typescript
+     * cortex.startScreenFeed({ fps: 1, quality: 60 });
+     * // Model now receives a screenshot every second
+     * ```
+     */
+    startScreenFeed(config) {
+        this.ensureConnected();
+        if (this._screenFeedActive)
+            this.stopScreenFeed();
+        const fps = Math.min(config?.fps ?? 1, 2); // Models usually cap at 1fps
+        const quality = config?.quality ?? 60;
+        const scale = config?.scale ?? 0.5;
+        const region = config?.region;
+        const intervalMs = Math.floor(1000 / fps);
+        this._screenFeedActive = true;
+        this.screenFeedInterval = setInterval(() => {
+            if (!this._connected || !this.transport)
+                return;
+            try {
+                let jpeg;
+                if (region) {
+                    // Capture region, scale it
+                    const cap = native.screenCapture(region.x, region.y, region.width, region.height);
+                    const sw = Math.floor(cap.width * scale);
+                    const sh = Math.floor(cap.height * scale);
+                    jpeg = native.screenCapturePng(region.x, region.y, sw, sh);
+                }
+                else {
+                    jpeg = native.screenCapturePng();
+                }
+                this.transport.sendVideo(jpeg);
+                this.emit('screenFrameSent', jpeg.length);
+            }
+            catch { /* non-fatal */ }
+        }, intervalMs);
+    }
+    /**
+     * Stop the automatic screen feed.
+     */
+    stopScreenFeed() {
+        if (this.screenFeedInterval) {
+            clearInterval(this.screenFeedInterval);
+            this.screenFeedInterval = null;
+        }
+        this._screenFeedActive = false;
+    }
+    /**
+     * Start streaming audio to the model.
+     * Uses WASAPI loopback (system audio) or microphone.
+     *
+     * @example
+     * ```typescript
+     * cortex.startAudioFeed({ source: 'system', targetSampleRate: 16000 });
+     * // Model now hears everything playing on the desktop
+     * ```
+     */
+    startAudioFeed(config) {
+        this.ensureConnected();
+        if (this._audioFeedActive)
+            this.stopAudioFeed();
+        const source = config?.source ?? 'system';
+        const targetRate = config?.targetSampleRate ?? 16000;
+        const pollMs = 100; // Feed audio every 100ms
+        this._audioFeedActive = true;
+        if (source === 'system') {
+            // Use Stelo audioStream for system audio capture
+            try {
+                native.audioStreamStart({ bufferSize: 30 });
+            }
+            catch { /* already started */ }
+            this.audioFeedInterval = setInterval(() => {
+                if (!this._connected || !this.transport)
+                    return;
+                try {
+                    const chunks = native.audioStreamDrain ? native.audioStreamDrain() : [];
+                    for (const chunk of chunks) {
+                        // Convert float32 to int16 PCM for model
+                        const pcm = float32ToInt16(chunk.data, chunk.sampleRate ?? 48000, targetRate);
+                        this.transport.sendAudio(pcm, targetRate);
+                        this.emit('audioChunkSent', (chunk.frames / (chunk.sampleRate ?? 48000)) * 1000);
+                    }
+                }
+                catch { /* non-fatal */ }
+            }, pollMs);
+        }
+        // 'mic' source would require a separate mic capture implementation
+        // which could be added as a native module extension
+    }
+    /**
+     * Stop the automatic audio feed.
+     */
+    stopAudioFeed() {
+        if (this.audioFeedInterval) {
+            clearInterval(this.audioFeedInterval);
+            this.audioFeedInterval = null;
+        }
+        if (this._audioFeedActive) {
+            try {
+                native.audioStreamStop();
+            }
+            catch { /* ok */ }
+        }
+        this._audioFeedActive = false;
+    }
+    // ═══ TOOL REGISTRY ════════════════════════════════════════════════════
+    // Register tools that the model can call. When the model issues a tool_call,
+    // Cortex automatically routes it to the handler and sends the result back.
+    /**
+     * Register a tool the model can call.
+     *
+     * @example
+     * ```typescript
+     * cortex.registerTool('get_weather', async (city: string) => {
+     *   return { temp: 72, condition: 'sunny' };
+     * }, {
+     *   name: 'get_weather',
+     *   description: 'Get current weather for a city',
+     *   parameters: { city: { type: 'string', description: 'City name', required: true } },
+     * });
+     * ```
+     */
+    registerTool(name, handler, definition) {
+        this.tools.set(name, {
+            definition: {
+                name,
+                description: definition?.description ?? name,
+                parameters: definition?.parameters,
+                nonBlocking: definition?.nonBlocking,
+            },
+            handler,
+        });
+    }
+    /**
+     * Auto-register Stelo SDK functions as model-callable tools.
+     * Gives the model hands to control the computer.
+     *
+     * @example
+     * ```typescript
+     * cortex.registerSteloTools();
+     * // Now the model can call: mouse_click, keyboard_type, screen_capture, etc.
+     * ```
+     */
+    registerSteloTools() {
+        // Mouse tools
+        this.registerTool('mouse_click', (x, y, button) => {
+            native.mouseClickAt(x, y, button ?? 'left', false);
+            return { success: true, x, y };
+        }, {
+            description: 'Click the mouse at screen coordinates (x, y)',
+            parameters: {
+                x: { type: 'number', description: 'X coordinate', required: true },
+                y: { type: 'number', description: 'Y coordinate', required: true },
+                button: { type: 'string', description: 'Mouse button', enum: ['left', 'right', 'middle'] },
+            },
+        });
+        this.registerTool('mouse_double_click', (x, y) => {
+            native.mouseMove(x, y);
+            native.mouseDoubleClick('left');
+            return { success: true, x, y };
+        }, {
+            description: 'Double-click at screen coordinates',
+            parameters: {
+                x: { type: 'number', description: 'X coordinate', required: true },
+                y: { type: 'number', description: 'Y coordinate', required: true },
+            },
+        });
+        this.registerTool('mouse_move', (x, y, smooth) => {
+            if (smooth) {
+                native.mouseMoveSmooth(x, y, 300, 'easeInOut');
+            }
+            else {
+                native.mouseMove(x, y);
+            }
+            return { success: true, x, y };
+        }, {
+            description: 'Move the mouse cursor to screen coordinates',
+            parameters: {
+                x: { type: 'number', description: 'X coordinate', required: true },
+                y: { type: 'number', description: 'Y coordinate', required: true },
+                smooth: { type: 'boolean', description: 'Use smooth movement' },
+            },
+        });
+        this.registerTool('mouse_scroll', (amount, direction) => {
+            native.mouseScroll(amount, direction ?? 'down');
+            return { success: true };
+        }, {
+            description: 'Scroll the mouse wheel',
+            parameters: {
+                amount: { type: 'number', description: 'Scroll amount', required: true },
+                direction: { type: 'string', description: 'Scroll direction', enum: ['up', 'down', 'left', 'right'] },
+            },
+        });
+        this.registerTool('mouse_drag', (fromX, fromY, toX, toY) => {
+            native.mouseDrag(fromX, fromY, toX, toY, 300, 'left');
+            return { success: true };
+        }, {
+            description: 'Drag from one position to another',
+            parameters: {
+                fromX: { type: 'number', required: true },
+                fromY: { type: 'number', required: true },
+                toX: { type: 'number', required: true },
+                toY: { type: 'number', required: true },
+            },
+        });
+        // Keyboard tools
+        this.registerTool('keyboard_type', (text) => {
+            native.keyboardType(text);
+            return { success: true, typed: text };
+        }, {
+            description: 'Type text using the keyboard',
+            parameters: {
+                text: { type: 'string', description: 'Text to type', required: true },
+            },
+        });
+        this.registerTool('keyboard_press', (key) => {
+            native.keyboardPress(key);
+            return { success: true, key };
+        }, {
+            description: 'Press a single key (Enter, Tab, Escape, Backspace, etc.)',
+            parameters: {
+                key: { type: 'string', description: 'Key name', required: true },
+            },
+        });
+        this.registerTool('keyboard_hotkey', (...keys) => {
+            native.keyboardHotkey(keys);
+            return { success: true, keys };
+        }, {
+            description: 'Press a keyboard shortcut (e.g. ctrl+s, alt+f4, ctrl+shift+t)',
+            parameters: {
+                keys: { type: 'string', description: 'Comma-separated key names for the hotkey combo', required: true },
+            },
+        });
+        // Screen tools
+        this.registerTool('screen_capture', () => {
+            const cap = native.screenCapture();
+            return { width: cap.width, height: cap.height, captured: true };
+        }, {
+            description: 'Capture the current screen and return dimensions',
+        });
+        this.registerTool('screen_get_size', () => {
+            return native.screenGetSize();
+        }, {
+            description: 'Get the screen resolution (width, height)',
+        });
+        this.registerTool('screen_get_pixel_color', (x, y) => {
+            return native.screenGetPixelColor(x, y);
+        }, {
+            description: 'Get the color of a pixel at coordinates',
+            parameters: {
+                x: { type: 'number', required: true },
+                y: { type: 'number', required: true },
+            },
+        });
+        // OCR tools
+        this.registerTool('ocr_read_screen', (x, y, width, height) => {
+            const result = native.ocrRecognize(x, y, width, height);
+            return { text: result.text, words: result.words?.length ?? 0 };
+        }, {
+            description: 'Read text from the screen using OCR. Can target a specific region.',
+            parameters: {
+                x: { type: 'number', description: 'Region X (optional)' },
+                y: { type: 'number', description: 'Region Y (optional)' },
+                width: { type: 'number', description: 'Region width (optional)' },
+                height: { type: 'number', description: 'Region height (optional)' },
+            },
+        });
+        this.registerTool('ocr_find_text', (text) => {
+            const result = native.ocrFindText(text);
+            return result ? { found: true, x: result.x, y: result.y, width: result.width, height: result.height } : { found: false };
+        }, {
+            description: 'Find specific text on the screen using OCR and return its position',
+            parameters: {
+                text: { type: 'string', description: 'Text to find', required: true },
+            },
+        });
+        // Window tools
+        this.registerTool('window_list', () => {
+            return native.windowList().map((w) => ({
+                id: w.id, title: w.title, x: w.x, y: w.y, width: w.width, height: w.height,
+                processName: w.processName, isVisible: w.isVisible,
+            }));
+        }, {
+            description: 'List all open windows with their titles, positions, and sizes',
+        });
+        this.registerTool('window_focus', (titleSubstring) => {
+            const windows = native.windowList();
+            const match = windows.find((w) => w.title.toLowerCase().includes(titleSubstring.toLowerCase()) && w.isVisible);
+            if (match) {
+                native.windowFocus(match.id);
+                return { success: true, title: match.title };
+            }
+            return { success: false, error: 'Window not found' };
+        }, {
+            description: 'Focus a window by partial title match',
+            parameters: {
+                titleSubstring: { type: 'string', description: 'Part of the window title', required: true },
+            },
+        });
+        // Clipboard tools
+        this.registerTool('clipboard_read', () => {
+            return { text: native.clipboardRead() };
+        }, {
+            description: 'Read the current clipboard text content',
+        });
+        this.registerTool('clipboard_write', (text) => {
+            native.clipboardWrite(text);
+            return { success: true };
+        }, {
+            description: 'Write text to the clipboard',
+            parameters: {
+                text: { type: 'string', description: 'Text to copy', required: true },
+            },
+        });
+        // Wait/verify tools
+        this.registerTool('wait_ms', async (ms) => {
+            await new Promise(r => setTimeout(r, Math.min(ms, 30000)));
+            return { waited: ms };
+        }, {
+            description: 'Wait for a specified number of milliseconds',
+            parameters: {
+                ms: { type: 'number', description: 'Milliseconds to wait', required: true },
+            },
+            nonBlocking: true,
+        });
+    }
+    /**
+     * Get all registered tool names.
+     */
+    get registeredTools() {
+        return Array.from(this.tools.keys());
+    }
+    // ═══ EVENTS ═══════════════════════════════════════════════════════════
+    /**
+     * Listen for Cortex events.
+     *
+     * @example
+     * ```typescript
+     * cortex.on('stateChange', (state, prev) => {
+     *   console.log(`${prev} → ${state}`);
+     * });
+     *
+     * cortex.on('modelText', (text, isFinal) => {
+     *   if (isFinal) console.log('Model said:', text);
+     * });
+     *
+     * cortex.on('toolCall', (name, args) => {
+     *   console.log(`Model calling: ${name}(${JSON.stringify(args)})`);
+     * });
+     *
+     * cortex.on('transcript', (text, source, isFinal) => {
+     *   if (source === 'user' && isFinal) console.log('User said:', text);
+     * });
+     * ```
+     */
+    on(event, handler) {
+        const handlers = this.listeners.get(event) ?? [];
+        handlers.push(handler);
+        this.listeners.set(event, handlers);
+        return () => {
+            const list = this.listeners.get(event);
+            if (list) {
+                const idx = list.indexOf(handler);
+                if (idx >= 0)
+                    list.splice(idx, 1);
+            }
+        };
+    }
+    /**
+     * Listen for an event once.
+     */
+    once(event, handler) {
+        const unsub = this.on(event, ((...args) => {
+            unsub();
+            handler(...args);
+        }));
+        return unsub;
+    }
+    /**
+     * Wait for a specific event. Returns a promise that resolves with the event args.
+     *
+     * @example
+     * ```typescript
+     * cortex.say("What's on the screen?");
+     * const [text] = await cortex.waitFor('modelText', 10000);
+     * console.log('Model responded:', text);
+     * ```
+     */
+    waitFor(event, timeoutMs = 30000) {
+        return new Promise((resolve, reject) => {
+            const timer = setTimeout(() => {
+                unsub();
+                reject(new Error(`Cortex.waitFor('${event}') timed out after ${timeoutMs}ms`));
+            }, timeoutMs);
+            const unsub = this.on(event, ((...args) => {
+                clearTimeout(timer);
+                unsub();
+                resolve(args);
+            }));
+        });
+    }
+    // ═══ STATE ════════════════════════════════════════════════════════════
+    /** Current conversation state. */
+    get state() { return this._state; }
+    /** Who currently holds the conversational floor. */
+    get turn() { return this._turn; }
+    /** Whether connected to a model. */
+    get isConnected() { return this._connected; }
+    /** Whether screen feed is active. */
+    get isScreenFeedActive() { return this._screenFeedActive; }
+    /** Whether audio feed is active. */
+    get isAudioFeedActive() { return this._audioFeedActive; }
+    /** Last session resume token for reconnection. */
+    get resumeToken() { return this._resumeToken; }
+    /** Total tokens consumed this session. */
+    get totalTokensUsed() { return this._totalTokensUsed; }
+    /** Number of pending (in-flight) tool calls. */
+    get pendingToolCallCount() { return this.pendingToolCalls.size; }
+    // ═══ CONVENIENCE: Ask & Wait ══════════════════════════════════════════
+    /**
+     * Send text and wait for the model's complete text response.
+     * This is the simplest way to have a conversation turn.
+     *
+     * @example
+     * ```typescript
+     * const response = await cortex.ask("What application is currently focused?");
+     * console.log(response); // "I can see that Notepad is currently focused..."
+     * ```
+     */
+    async ask(text, timeoutMs = 30000) {
+        this.modelTextBuffer = '';
+        let transcriptBuffer = '';
+        this.say(text);
+        return new Promise((resolve, reject) => {
+            const timer = setTimeout(() => {
+                cleanup();
+                resolve(this.modelTextBuffer || transcriptBuffer || '(no response)');
+            }, timeoutMs);
+            const unsubText = this.on('modelText', (chunk, isFinal) => {
+                if (isFinal && this.modelTextBuffer) {
+                    clearTimeout(timer);
+                    cleanup();
+                    resolve(this.modelTextBuffer);
+                }
+            });
+            // Also listen for model transcripts (when model responds with audio)
+            const unsubTranscript = this.on('transcript', (t, source, isFinal) => {
+                if (source === 'model') {
+                    transcriptBuffer += t;
+                }
+            });
+            const unsubComplete = this.on('turnChange', (owner) => {
+                if (owner !== 'model' && (this.modelTextBuffer || transcriptBuffer)) {
+                    clearTimeout(timer);
+                    cleanup();
+                    resolve(this.modelTextBuffer || transcriptBuffer);
+                }
+            });
+            const unsubErr = this.on('error', (err) => {
+                clearTimeout(timer);
+                cleanup();
+                reject(err);
+            });
+            const cleanup = () => { unsubText(); unsubComplete(); unsubErr(); unsubTranscript(); };
+        });
+    }
+    /**
+     * Tell the model to perform an action and wait for tool execution to complete.
+     *
+     * @example
+     * ```typescript
+     * await cortex.command("Click on the Start menu");
+     * // Model calls mouse_click tool, Cortex executes it
+     * await cortex.command("Type 'notepad' and press Enter");
+     * // Model calls keyboard_type then keyboard_press
+     * ```
+     */
+    async command(instruction, timeoutMs = 30000) {
+        this.say(instruction);
+        return new Promise((resolve, reject) => {
+            const timer = setTimeout(() => {
+                cleanup();
+                resolve();
+            }, timeoutMs);
+            // Resolve when model turn completes and no pending tool calls
+            const unsubTurn = this.on('turnChange', (owner) => {
+                if (owner !== 'model' && this.pendingToolCalls.size === 0) {
+                    clearTimeout(timer);
+                    cleanup();
+                    resolve();
+                }
+            });
+            const unsubErr = this.on('error', (err) => {
+                clearTimeout(timer);
+                cleanup();
+                reject(err);
+            });
+            const cleanup = () => { unsubTurn(); unsubErr(); };
+        });
+    }
+    // ═══ INTERNAL HANDLERS ════════════════════════════════════════════════
+    handleModelAudio(data, sampleRate) {
+        if (this._state !== 'speaking') {
+            this.setState('speaking');
+            this.setTurn('model');
+        }
+        this.emit('modelAudio', data, sampleRate);
+        this.touchActivity();
+    }
+    handleModelText(text, isFinal) {
+        this.modelTextBuffer += text;
+        if (this._state !== 'speaking') {
+            this.setState('speaking');
+            this.setTurn('model');
+        }
+        this.emit('modelText', text, isFinal);
+        if (isFinal) {
+            this.modelTextBuffer = '';
+        }
+        this.touchActivity();
+    }
+    handleTranscript(text, source, isFinal) {
+        if (source === 'user') {
+            this.userTranscriptBuffer += text;
+            if (this._state !== 'listening') {
+                this.setState('listening');
+                this.setTurn('user');
+            }
+            if (isFinal)
+                this.userTranscriptBuffer = '';
+        }
+        this.emit('transcript', text, source, isFinal);
+    }
+    async handleToolCall(id, name, args) {
+        this.setState('acting');
+        this.emit('toolCall', name, args, id);
+        const tool = this.tools.get(name);
+        if (!tool) {
+            // Unknown tool — send error response
+            this.transport?.sendToolResponse(id, { error: `Unknown tool: ${name}` });
+            return;
+        }
+        const pending = { id, name, args, startTime: Date.now(), resolved: false };
+        this.pendingToolCalls.set(id, pending);
+        try {
+            // Parse args: if tool handler expects positional args, spread them
+            let result;
+            if (typeof args === 'object' && args !== null && !Array.isArray(args)) {
+                const paramNames = Object.keys(tool.definition.parameters ?? {});
+                if (paramNames.length > 0) {
+                    const positionalArgs = paramNames.map(p => args[p]);
+                    result = await tool.handler(...positionalArgs);
+                }
+                else {
+                    result = await tool.handler(args);
+                }
+            }
+            else {
+                result = await tool.handler(args);
+            }
+            const durationMs = Date.now() - pending.startTime;
+            pending.resolved = true;
+            this.pendingToolCalls.delete(id);
+            // Send result back to model
+            const scheduling = tool.definition.nonBlocking ? 'when_idle' : undefined;
+            this.transport?.sendToolResponse(id, result, scheduling);
+            this.emit('toolResult', name, result, durationMs);
+            // If no more pending calls, return to idle
+            if (this.pendingToolCalls.size === 0 && this._state === 'acting') {
+                this.setState('idle');
+            }
+        }
+        catch (err) {
+            this.pendingToolCalls.delete(id);
+            this.transport?.sendToolResponse(id, { error: err.message ?? String(err) });
+            this.emit('error', err instanceof Error ? err : new Error(String(err)));
+        }
+        this.touchActivity();
+    }
+    handleTurnComplete() {
+        this.setTurn('none');
+        if (this._state === 'speaking' || this._state === 'thinking') {
+            this.setState('idle');
+        }
+        this.modelTextBuffer = '';
+    }
+    handleGenerationComplete() {
+        if (this.pendingToolCalls.size === 0) {
+            this.setState('idle');
+            this.setTurn('none');
+        }
+    }
+    handleInterrupted() {
+        this.setState('interrupted');
+        this.emit('interrupted');
+        // After interruption, go back to listening (user is speaking)
+        setTimeout(() => {
+            if (this._state === 'interrupted') {
+                this.setState('listening');
+                this.setTurn('user');
+            }
+        }, 100);
+    }
+    async handleDisconnect(reason) {
+        const wasConnected = this._connected;
+        this._connected = false;
+        this.stopScreenFeed();
+        this.stopAudioFeed();
+        this.stopIdleTimer();
+        this.setState('disconnected');
+        if (wasConnected) {
+            this.emit('disconnected', reason);
+            // Auto-reconnect
+            if (this.config?.autoReconnect && this._resumeToken) {
+                try {
+                    await new Promise(r => setTimeout(r, 1000)); // Brief delay
+                    const reconnectConfig = { ...this.config };
+                    reconnectConfig.resumeToken = this._resumeToken;
+                    await this.connect(reconnectConfig);
+                }
+                catch (err) {
+                    this.emit('error', err instanceof Error ? err : new Error('Reconnection failed'));
+                }
+            }
+        }
+    }
+    // ═══ STATE MACHINE ════════════════════════════════════════════════════
+    setState(newState) {
+        if (this._state === newState)
+            return;
+        const oldState = this._state;
+        this._state = newState;
+        this.emit('stateChange', newState, oldState);
+    }
+    setTurn(owner) {
+        if (this._turn === owner)
+            return;
+        this._turn = owner;
+        this.emit('turnChange', owner);
+    }
+    emit(event, ...args) {
+        const handlers = this.listeners.get(event);
+        if (!handlers)
+            return;
+        for (const handler of handlers) {
+            try {
+                handler(...args);
+            }
+            catch { /* listener error, don't crash cortex */ }
+        }
+    }
+    // ═══ UTILITIES ════════════════════════════════════════════════════════
+    ensureConnected() {
+        if (!this._connected || !this.transport) {
+            throw new Error('Cortex is not connected. Call connect() first.');
+        }
+    }
+    touchActivity() {
+        this.lastActivityTime = Date.now();
+    }
+    startIdleTimer(timeoutMs) {
+        this.stopIdleTimer();
+        this.idleTimer = setInterval(() => {
+            if (Date.now() - this.lastActivityTime > timeoutMs) {
+                this.disconnect();
+            }
+        }, Math.min(timeoutMs / 2, 30000));
+    }
+    stopIdleTimer() {
+        if (this.idleTimer) {
+            clearInterval(this.idleTimer);
+            this.idleTimer = null;
+        }
+    }
+}
+exports.Cortex = Cortex;
+// ═════════════════════════════════════════════════════════════════════════════
+// AUDIO UTILITIES
+// ═════════════════════════════════════════════════════════════════════════════
+/**
+ * Convert Float32 PCM audio to Int16 PCM with optional resampling.
+ * Most real-time APIs expect 16-bit PCM at 16kHz.
+ */
+function float32ToInt16(input, inputRate, outputRate) {
+    // Interpret as float32
+    const floatCount = Math.floor(input.length / 4);
+    const floats = new Float32Array(input.buffer, input.byteOffset, floatCount);
+    // Resample if needed (simple linear interpolation)
+    const ratio = inputRate / outputRate;
+    const outputLength = Math.floor(floatCount / ratio);
+    const output = Buffer.alloc(outputLength * 2); // 2 bytes per int16
+    for (let i = 0; i < outputLength; i++) {
+        const srcIdx = i * ratio;
+        const idx0 = Math.floor(srcIdx);
+        const idx1 = Math.min(idx0 + 1, floatCount - 1);
+        const frac = srcIdx - idx0;
+        // Interpolate
+        const sample = floats[idx0] * (1 - frac) + floats[idx1] * frac;
+        // Clamp and convert to int16
+        const clamped = Math.max(-1, Math.min(1, sample));
+        output.writeInt16LE(Math.round(clamped * 32767), i * 2);
+    }
+    return output;
+}
+// ═════════════════════════════════════════════════════════════════════════════
+// BUILT-IN PROVIDER FACTORIES
+// ═════════════════════════════════════════════════════════════════════════════
+//
+// These create CortexProvider instances for popular real-time AI APIs.
+// They handle the WebSocket protocol translation so Cortex doesn't have to
+// know about Gemini's BidiGenerateContent or OpenAI's realtime protocol.
+//
+/**
+ * Create a Gemini Live API provider.
+ *
+ * @example
+ * ```typescript
+ * const cortex = new Cortex();
+ * await cortex.connect({
+ *   provider: geminiLiveProvider(),
+ *   connection: {
+ *     apiKey: 'YOUR_API_KEY',
+ *     model: 'gemini-3.1-flash-live-preview',
+ *     responseModalities: ['audio'],
+ *     voice: 'Kore',
+ *     systemInstruction: 'You are a desktop assistant.',
+ *     transcribeInput: true,
+ *     transcribeOutput: true,
+ *     providerOptions: {
+ *       // Gemini-specific
+ *       enableAffectiveDialog: true,
+ *       proactiveAudio: true,
+ *       thinkingLevel: 'low',
+ *       contextWindowCompression: true,
+ *     },
+ *   },
+ * });
+ * ```
+ */
+function geminiLiveProvider() {
+    return {
+        name: 'gemini-live',
+        async connect(config, handlers) {
+            const apiVersion = config.providerOptions?.apiVersion ?? 'v1beta';
+            const host = config.providerOptions?.host ?? 'generativelanguage.googleapis.com';
+            const wsUrl = `wss://${host}/ws/google.ai.generativelanguage.${apiVersion}.GenerativeService.BidiGenerateContent?key=${config.apiKey}`;
+            // Build setup message — uses snake_case per Gemini BidiGenerateContent protocol
+            const setupMsg = {
+                setup: {
+                    model: `models/${config.model}`,
+                    generation_config: {
+                        response_modalities: (config.responseModalities ?? ['audio']).map(m => m.toUpperCase()),
+                    },
+                },
+            };
+            if (config.systemInstruction) {
+                setupMsg.setup.system_instruction = { parts: [{ text: config.systemInstruction }] };
+            }
+            if (config.voice) {
+                setupMsg.setup.generation_config.speech_config = {
+                    voice_config: { prebuilt_voice_config: { voice_name: config.voice } },
+                };
+            }
+            if (config.transcribeInput) {
+                setupMsg.setup.input_audio_transcription = {};
+            }
+            if (config.transcribeOutput) {
+                setupMsg.setup.output_audio_transcription = {};
+            }
+            if (config.tools && config.tools.length > 0) {
+                setupMsg.setup.tools = [{
+                        function_declarations: config.tools.map(t => ({
+                            name: t.name,
+                            description: t.description,
+                            parameters: t.parameters ? {
+                                type: 'OBJECT',
+                                properties: Object.fromEntries(Object.entries(t.parameters).map(([k, v]) => [k, { type: v.type.toUpperCase(), description: v.description }])),
+                                required: Object.entries(t.parameters).filter(([, v]) => v.required).map(([k]) => k),
+                            } : undefined,
+                            ...(t.nonBlocking ? { behavior: 'NON_BLOCKING' } : {}),
+                        })),
+                    }];
+            }
+            // Provider-specific options
+            const opts = config.providerOptions ?? {};
+            if (opts.enableAffectiveDialog) {
+                setupMsg.setup.enable_affective_dialog = true;
+            }
+            if (opts.proactiveAudio) {
+                setupMsg.setup.proactivity = { proactive_audio: true };
+            }
+            if (opts.thinkingLevel) {
+                setupMsg.setup.generation_config.thinking_config = { thinking_level: opts.thinkingLevel };
+            }
+            if (opts.contextWindowCompression) {
+                setupMsg.setup.context_window_compression = { sliding_window: {} };
+            }
+            if (opts.sessionResumptionHandle) {
+                setupMsg.setup.session_resumption = { handle: opts.sessionResumptionHandle };
+            }
+            // Connect WebSocket and wait for setupComplete acknowledgment
+            const ws = new WebSocket(wsUrl);
+            await new Promise((resolve, reject) => {
+                const timeout = setTimeout(() => reject(new Error('Connection timeout')), 15000);
+                let setupSent = false;
+                ws.onopen = () => {
+                    ws.send(JSON.stringify(setupMsg));
+                    setupSent = true;
+                };
+                ws.onmessage = async (event) => {
+                    // First message back is { setupComplete: {} }
+                    if (setupSent) {
+                        clearTimeout(timeout);
+                        resolve();
+                    }
+                };
+                ws.onerror = (e) => { clearTimeout(timeout); reject(new Error(`WebSocket error: ${e.message ?? 'connection failed'}`)); };
+                ws.onclose = (ev) => { clearTimeout(timeout); reject(new Error(`WebSocket closed during setup (code=${ev?.code} reason=${ev?.reason || 'none'})`)); };
+            });
+            // Helper to extract text from MessageEvent.data (handles string and Blob)
+            async function extractText(data) {
+                if (typeof data === 'string')
+                    return data;
+                if (typeof Blob !== 'undefined' && data instanceof Blob)
+                    return data.text();
+                if (Buffer.isBuffer(data))
+                    return data.toString('utf-8');
+                return String(data);
+            }
+            // Message router
+            ws.onmessage = (event) => {
+                extractText(event.data).then(raw => {
+                    const msg = JSON.parse(raw);
+                    if (msg.serverContent) {
+                        const sc = msg.serverContent;
+                        if (sc.interrupted) {
+                            handlers.interrupted();
+                        }
+                        if (sc.modelTurn?.parts) {
+                            for (const part of sc.modelTurn.parts) {
+                                if (part.inlineData?.data) {
+                                    const audioData = Buffer.from(part.inlineData.data, 'base64');
+                                    handlers.audio(audioData, 24000);
+                                }
+                                if (part.text) {
+                                    handlers.text(part.text, false);
+                                }
+                            }
+                        }
+                        if (sc.outputTranscription?.text) {
+                            handlers.transcript(sc.outputTranscription.text, 'model', true);
+                        }
+                        if (sc.inputTranscription?.text) {
+                            handlers.transcript(sc.inputTranscription.text, 'user', true);
+                        }
+                        if (sc.turnComplete) {
+                            handlers.text('', true); // Signal final text
+                            handlers.turnComplete();
+                        }
+                        if (sc.generationComplete) {
+                            handlers.generationComplete();
+                        }
+                    }
+                    if (msg.toolCall?.functionCalls) {
+                        for (const fc of msg.toolCall.functionCalls) {
+                            handlers.toolCall(fc.id, fc.name, fc.args ?? {});
+                        }
+                    }
+                    if (msg.usageMetadata) {
+                        handlers.usage(msg.usageMetadata.totalTokenCount ?? msg.usageMetadata.promptTokenCount ?? 0, msg.usageMetadata.promptTokenCount ?? 0, msg.usageMetadata.candidatesTokenCount ?? 0);
+                    }
+                    if (msg.sessionResumptionUpdate?.newHandle) {
+                        handlers.resumeToken(msg.sessionResumptionUpdate.newHandle);
+                    }
+                    if (msg.goAway) {
+                        handlers.close('server_go_away');
+                    }
+                }).catch((err) => {
+                    handlers.error(new Error(`Message parse error: ${err.message}`));
+                });
+            };
+            ws.onerror = (e) => handlers.error(new Error(`Gemini WS error: ${e.message ?? 'unknown'}`));
+            ws.onclose = (ev) => handlers.close(`gemini_closed code=${ev?.code} reason=${ev?.reason || 'none'}`);
+            // Build transport — uses snake_case per Gemini protocol
+            const transport = {
+                sendAudio(pcm, sampleRate) {
+                    if (ws.readyState !== WebSocket.OPEN)
+                        return;
+                    ws.send(JSON.stringify({
+                        realtime_input: {
+                            audio: {
+                                data: pcm.toString('base64'),
+                                mimeType: `audio/pcm;rate=${sampleRate}`,
+                            },
+                        },
+                    }));
+                },
+                sendVideo(jpeg) {
+                    if (ws.readyState !== WebSocket.OPEN)
+                        return;
+                    ws.send(JSON.stringify({
+                        realtime_input: {
+                            video: {
+                                data: jpeg.toString('base64'),
+                                mimeType: 'image/jpeg',
+                            },
+                        },
+                    }));
+                },
+                sendText(text, role) {
+                    if (ws.readyState !== WebSocket.OPEN)
+                        return;
+                    if (role === 'system') {
+                        ws.send(JSON.stringify({
+                            realtime_input: { text: `[System]: ${text}` },
+                        }));
+                    }
+                    else {
+                        ws.send(JSON.stringify({
+                            realtime_input: { text },
+                        }));
+                    }
+                },
+                sendToolResponse(callId, result, scheduling) {
+                    if (ws.readyState !== WebSocket.OPEN)
+                        return;
+                    const funcResponse = {
+                        name: callId,
+                        id: callId,
+                        response: typeof result === 'string' ? { result } : { result: JSON.stringify(result) },
+                    };
+                    if (scheduling) {
+                        funcResponse.response.scheduling = scheduling.toUpperCase();
+                    }
+                    ws.send(JSON.stringify({
+                        tool_response: {
+                            function_responses: [funcResponse],
+                        },
+                    }));
+                },
+                sendActivityStart() {
+                    if (ws.readyState !== WebSocket.OPEN)
+                        return;
+                    ws.send(JSON.stringify({ realtime_input: { activity_start: {} } }));
+                },
+                sendActivityEnd() {
+                    if (ws.readyState !== WebSocket.OPEN)
+                        return;
+                    ws.send(JSON.stringify({ realtime_input: { activity_end: {} } }));
+                },
+                interrupt() {
+                    if (ws.readyState !== WebSocket.OPEN)
+                        return;
+                    ws.send(JSON.stringify({ realtime_input: { text: '' } }));
+                },
+                async close() {
+                    ws.close();
+                },
+            };
+            return transport;
+        },
+    };
+}
+/**
+ * Create an OpenAI Realtime API provider.
+ *
+ * @example
+ * ```typescript
+ * await cortex.connect({
+ *   provider: openAIRealtimeProvider(),
+ *   connection: {
+ *     apiKey: 'YOUR_OPENAI_KEY',
+ *     model: 'gpt-4o-realtime-preview',
+ *     responseModalities: ['audio', 'text'],
+ *     voice: 'alloy',
+ *   },
+ * });
+ * ```
+ */
+function openAIRealtimeProvider() {
+    return {
+        name: 'openai-realtime',
+        async connect(config, handlers) {
+            const host = config.providerOptions?.host ?? 'api.openai.com';
+            const wsUrl = `wss://${host}/v1/realtime?model=${config.model}`;
+            const ws = new WebSocket(wsUrl, {
+                headers: {
+                    'Authorization': `Bearer ${config.apiKey}`,
+                    'OpenAI-Beta': 'realtime=v1',
+                },
+            });
+            await new Promise((resolve, reject) => {
+                const timeout = setTimeout(() => reject(new Error('Connection timeout')), 15000);
+                ws.onopen = () => {
+                    clearTimeout(timeout);
+                    // Send session.update with configuration
+                    const sessionUpdate = {
+                        type: 'session.update',
+                        session: {
+                            modalities: config.responseModalities ?? ['audio', 'text'],
+                            instructions: config.systemInstruction ?? '',
+                            voice: config.voice ?? 'alloy',
+                            input_audio_format: 'pcm16',
+                            output_audio_format: 'pcm16',
+                            input_audio_transcription: config.transcribeInput ? { model: 'whisper-1' } : undefined,
+                            turn_detection: config.providerOptions?.manualVAD ? null : { type: 'server_vad' },
+                        },
+                    };
+                    if (config.tools && config.tools.length > 0) {
+                        sessionUpdate.session.tools = config.tools.map(t => ({
+                            type: 'function',
+                            name: t.name,
+                            description: t.description,
+                            parameters: t.parameters ? {
+                                type: 'object',
+                                properties: Object.fromEntries(Object.entries(t.parameters).map(([k, v]) => [k, { type: v.type, description: v.description }])),
+                                required: Object.entries(t.parameters).filter(([, v]) => v.required).map(([k]) => k),
+                            } : { type: 'object', properties: {} },
+                        }));
+                    }
+                    ws.send(JSON.stringify(sessionUpdate));
+                    resolve();
+                };
+                ws.onerror = (e) => { clearTimeout(timeout); reject(new Error(`WebSocket error: ${e.message ?? 'connection failed'}`)); };
+            });
+            // Message router
+            ws.onmessage = (event) => {
+                try {
+                    const msg = JSON.parse(typeof event.data === 'string' ? event.data : event.data.toString());
+                    switch (msg.type) {
+                        case 'response.audio.delta':
+                            if (msg.delta) {
+                                handlers.audio(Buffer.from(msg.delta, 'base64'), 24000);
+                            }
+                            break;
+                        case 'response.text.delta':
+                            if (msg.delta) {
+                                handlers.text(msg.delta, false);
+                            }
+                            break;
+                        case 'response.text.done':
+                            if (msg.text)
+                                handlers.text(msg.text, true);
+                            break;
+                        case 'response.audio_transcript.delta':
+                            handlers.transcript(msg.delta ?? '', 'model', false);
+                            break;
+                        case 'response.audio_transcript.done':
+                            handlers.transcript(msg.transcript ?? '', 'model', true);
+                            break;
+                        case 'conversation.item.input_audio_transcription.completed':
+                            handlers.transcript(msg.transcript ?? '', 'user', true);
+                            break;
+                        case 'response.function_call_arguments.done':
+                            try {
+                                const args = JSON.parse(msg.arguments ?? '{}');
+                                handlers.toolCall(msg.call_id, msg.name, args);
+                            }
+                            catch { /* parse error */ }
+                            break;
+                        case 'response.done':
+                            handlers.turnComplete();
+                            if (msg.response?.usage) {
+                                handlers.usage(msg.response.usage.total_tokens ?? 0, msg.response.usage.input_tokens ?? 0, msg.response.usage.output_tokens ?? 0);
+                            }
+                            break;
+                        case 'input_audio_buffer.speech_started':
+                            // User started speaking — model will be interrupted
+                            break;
+                        case 'input_audio_buffer.speech_stopped':
+                            break;
+                        case 'error':
+                            handlers.error(new Error(`OpenAI: ${msg.error?.message ?? JSON.stringify(msg.error)}`));
+                            break;
+                    }
+                }
+                catch (err) {
+                    handlers.error(new Error(`Message parse error: ${err.message}`));
+                }
+            };
+            ws.onerror = (e) => handlers.error(new Error(`WebSocket error: ${e.message ?? 'unknown'}`));
+            ws.onclose = () => handlers.close('connection_closed');
+            const transport = {
+                sendAudio(pcm, _sampleRate) {
+                    if (ws.readyState !== WebSocket.OPEN)
+                        return;
+                    ws.send(JSON.stringify({
+                        type: 'input_audio_buffer.append',
+                        audio: pcm.toString('base64'),
+                    }));
+                },
+                sendVideo(_jpeg) {
+                    // OpenAI Realtime doesn't support video frames directly
+                    // Could be sent as a conversation item with image
+                },
+                sendText(text, role) {
+                    if (ws.readyState !== WebSocket.OPEN)
+                        return;
+                    ws.send(JSON.stringify({
+                        type: 'conversation.item.create',
+                        item: {
+                            type: 'message',
+                            role: role ?? 'user',
+                            content: [{ type: 'input_text', text }],
+                        },
+                    }));
+                    ws.send(JSON.stringify({ type: 'response.create' }));
+                },
+                sendToolResponse(callId, result) {
+                    if (ws.readyState !== WebSocket.OPEN)
+                        return;
+                    ws.send(JSON.stringify({
+                        type: 'conversation.item.create',
+                        item: {
+                            type: 'function_call_output',
+                            call_id: callId,
+                            output: JSON.stringify(result),
+                        },
+                    }));
+                    ws.send(JSON.stringify({ type: 'response.create' }));
+                },
+                sendActivityStart() {
+                    // OpenAI uses server VAD by default
+                },
+                sendActivityEnd() {
+                    if (ws.readyState !== WebSocket.OPEN)
+                        return;
+                    ws.send(JSON.stringify({ type: 'input_audio_buffer.commit' }));
+                    ws.send(JSON.stringify({ type: 'response.create' }));
+                },
+                interrupt() {
+                    if (ws.readyState !== WebSocket.OPEN)
+                        return;
+                    ws.send(JSON.stringify({ type: 'response.cancel' }));
+                },
+                async close() {
+                    ws.close();
+                },
+            };
+            return transport;
+        },
+    };
+}
+function customProvider(custom) {
+    return {
+        name: custom.name,
+        async connect(config, handlers) {
+            const url = custom.buildUrl(config);
+            const headers = custom.buildHeaders?.(config);
+            const ws = new WebSocket(url, headers ? { headers } : undefined);
+            await new Promise((resolve, reject) => {
+                const timeout = setTimeout(() => reject(new Error('Connection timeout')), 15000);
+                ws.onopen = () => {
+                    clearTimeout(timeout);
+                    const setupMsg = custom.buildSetupMessage?.(config);
+                    if (setupMsg)
+                        ws.send(JSON.stringify(setupMsg));
+                    resolve();
+                };
+                ws.onerror = (e) => { clearTimeout(timeout); reject(new Error(e.message ?? 'connection failed')); };
+            });
+            ws.onmessage = (event) => {
+                try {
+                    const raw = typeof event.data === 'string' ? event.data : event.data.toString();
+                    const parsed = custom.parseMessage(raw);
+                    if (!parsed)
+                        return;
+                    switch (parsed.kind) {
+                        case 'audio':
+                            handlers.audio(parsed.data, parsed.sampleRate);
+                            break;
+                        case 'text':
+                            handlers.text(parsed.text, parsed.isFinal);
+                            break;
+                        case 'transcript':
+                            handlers.transcript(parsed.text, parsed.source, parsed.isFinal);
+                            break;
+                        case 'toolCall':
+                            handlers.toolCall(parsed.id, parsed.name, parsed.args);
+                            break;
+                        case 'turnComplete':
+                            handlers.turnComplete();
+                            break;
+                        case 'interrupted':
+                            handlers.interrupted();
+                            break;
+                        case 'thought':
+                            handlers.thought(parsed.text);
+                            break;
+                        case 'usage':
+                            handlers.usage(parsed.total, parsed.input, parsed.output);
+                            break;
+                        case 'error':
+                            handlers.error(new Error(parsed.message));
+                            break;
+                        case 'close':
+                            handlers.close(parsed.reason);
+                            break;
+                        case 'resumeToken':
+                            handlers.resumeToken(parsed.token);
+                            break;
+                    }
+                }
+                catch (err) {
+                    handlers.error(new Error(`Parse error: ${err.message}`));
+                }
+            };
+            ws.onerror = (e) => handlers.error(new Error(e.message ?? 'unknown'));
+            ws.onclose = () => handlers.close('connection_closed');
+            return {
+                sendAudio(pcm, rate) { if (ws.readyState === WebSocket.OPEN)
+                    ws.send(custom.formatAudio(pcm, rate)); },
+                sendVideo(jpeg) { if (ws.readyState === WebSocket.OPEN && custom.formatVideo)
+                    ws.send(custom.formatVideo(jpeg)); },
+                sendText(text, role) { if (ws.readyState === WebSocket.OPEN)
+                    ws.send(custom.formatText(text, role)); },
+                sendToolResponse(id, result, sched) { if (ws.readyState === WebSocket.OPEN)
+                    ws.send(custom.formatToolResponse(id, result, sched)); },
+                sendActivityStart() { if (ws.readyState === WebSocket.OPEN && custom.formatActivityStart)
+                    ws.send(custom.formatActivityStart()); },
+                sendActivityEnd() { if (ws.readyState === WebSocket.OPEN && custom.formatActivityEnd)
+                    ws.send(custom.formatActivityEnd()); },
+                interrupt() { if (ws.readyState === WebSocket.OPEN && custom.formatInterrupt)
+                    ws.send(custom.formatInterrupt()); },
+                async close() { ws.close(); },
+            };
+        },
+    };
+}
+// ═════════════════════════════════════════════════════════════════════════════
+// FACTORY
+// ═════════════════════════════════════════════════════════════════════════════
+/** Create a new Cortex instance. */
+function createCortex() {
+    return new Cortex();
+}
+//# sourceMappingURL=cortex.js.map