tywrap 0.1.2 → 0.2.1

package/dist/runtime/process-io.js

@@ -0,0 +1,695 @@
+/**
+ * ProcessIO Transport - Subprocess-based Python communication for Node.js.
+ *
+ * This transport implements the Transport interface for spawning and communicating
+ * with a Python subprocess via stdio streams. It provides:
+ * - JSONL (JSON Lines) protocol for request/response messaging
+ * - Backpressure handling for stdin writes
+ * - Line-based stdout parsing with buffering
+ * - Stderr capture for error diagnostics
+ * - Request timeout management
+ * - Optional process restart after N requests
+ *
+ * @see https://github.com/bbopen/tywrap/issues/149
+ */
+import { spawn } from 'child_process';
+import { BoundedContext } from './bounded-context.js';
+import { BridgeDisposedError, BridgeProtocolError, BridgeTimeoutError, BridgeExecutionError, } from './errors.js';
+// =============================================================================
+// CONSTANTS
+// =============================================================================
+/** Default maximum response line length: 100MB */
+const DEFAULT_MAX_LINE_LENGTH = 100 * 1024 * 1024;
+/** Maximum stderr bytes to retain for diagnostics: 8KB */
+const MAX_STDERR_BYTES = 8 * 1024;
+/** Default write queue timeout: 30 seconds */
+const DEFAULT_WRITE_QUEUE_TIMEOUT_MS = 30_000;
+/** Regex for ANSI escape sequences */
+const ANSI_ESCAPE_RE = /\u001b\[[0-9;]*[A-Za-z]/g;
+/** Regex for control characters */
+const CONTROL_CHARS_RE = /[\u0000-\u0008\u000B\u000C\u000E-\u001F\u007F\u0080-\u009F]/g;
+// =============================================================================
+// UTILITIES
+// =============================================================================
+/**
+ * Sanitize stderr output by removing ANSI codes and control characters.
+ */
+function sanitizeStderr(value) {
+    return value.replace(ANSI_ESCAPE_RE, '').replace(CONTROL_CHARS_RE, '');
+}
+/**
+ * Extract message ID from a JSON string without full parsing.
+ * Returns null if ID cannot be extracted.
+ */
+function extractMessageId(json) {
+    // Look for "id": <number> (integer IDs)
+    const match = json.match(/"id"\s*:\s*(-?\d+)/);
+    return match?.[1] ? parseInt(match[1], 10) : null;
+}
+// =============================================================================
+// PROCESS IO TRANSPORT
+// =============================================================================
+/**
+ * Transport implementation for subprocess-based Python communication.
+ *
+ * ProcessIO spawns a Python child process and communicates via stdio:
+ * - Requests are written to stdin as JSON lines
+ * - Responses are read from stdout as JSON lines
+ * - Stderr is captured for diagnostics
+ *
+ * @example
+ * ```typescript
+ * const transport = new ProcessIO({
+ *   bridgeScript: '/path/to/bridge.py',
+ *   pythonPath: 'python3',
+ * });
+ *
+ * await transport.init();
+ *
+ * const response = await transport.send(
+ *   JSON.stringify({ id: '1', type: 'call', module: 'math', args: [16] }),
+ *   5000
+ * );
+ *
+ * await transport.dispose();
+ * ```
+ */
+export class ProcessIO extends BoundedContext {
+    // Configuration
+    pythonPath;
+    bridgeScript;
+    envOverrides;
+    cwd;
+    maxLineLength;
+    restartAfterRequests;
+    writeQueueTimeoutMs;
+    // Process state
+    process = null;
+    processExited = false;
+    processError = null;
+    // Stream buffers
+    stdoutBuffer = '';
+    stderrBuffer = '';
+    // Request tracking
+    pending = new Map();
+    requestCount = 0;
+    needsRestart = false;
+    // Write queue for backpressure
+    writeQueue = [];
+    draining = false;
+    /**
+     * Create a new ProcessIO transport.
+     *
+     * @param options - Transport configuration options
+     */
+    constructor(options) {
+        super();
+        this.pythonPath = options.pythonPath ?? 'python3';
+        this.bridgeScript = options.bridgeScript;
+        this.envOverrides = options.env ?? {};
+        this.cwd = options.cwd;
+        this.maxLineLength = options.maxLineLength ?? DEFAULT_MAX_LINE_LENGTH;
+        this.restartAfterRequests = options.restartAfterRequests ?? 0;
+        this.writeQueueTimeoutMs = options.writeQueueTimeoutMs ?? DEFAULT_WRITE_QUEUE_TIMEOUT_MS;
+    }
+    // ===========================================================================
+    // TRANSPORT INTERFACE
+    // ===========================================================================
+    /**
+     * Send a message and wait for the response.
+     *
+     * @param message - The JSON-encoded protocol message
+     * @param timeoutMs - Timeout in milliseconds (0 = no timeout)
+     * @param signal - Optional AbortSignal for cancellation
+     * @returns The raw JSON response string
+     */
+    async send(message, timeoutMs, signal) {
+        // Check disposed state
+        if (this.isDisposed) {
+            throw new BridgeDisposedError('Transport has been disposed');
+        }
+        // Auto-initialize if needed
+        if (!this.isReady) {
+            await this.init();
+        }
+        // Check if process is alive
+        if (this.processExited || !this.process) {
+            const stderrTail = this.getStderrTail();
+            const baseMsg = 'Python process is not running';
+            const msg = stderrTail ? `${baseMsg}. Stderr:\n${stderrTail}` : baseMsg;
+            throw new BridgeProtocolError(msg);
+        }
+        // Check if already aborted
+        if (signal?.aborted) {
+            throw new BridgeTimeoutError('Operation aborted');
+        }
+        // Extract message ID for response correlation
+        const messageId = extractMessageId(message);
+        if (!messageId) {
+            throw new BridgeProtocolError('Message must contain an "id" field');
+        }
+        // Check for restart condition (either scheduled restart or forced by stream error)
+        if (this.needsRestart || (this.restartAfterRequests > 0 && this.requestCount >= this.restartAfterRequests)) {
+            await this.restartProcess();
+        }
+        // Create promise for response
+        return new Promise((resolve, reject) => {
+            // Set up timeout if specified
+            let timer;
+            if (timeoutMs > 0) {
+                timer = setTimeout(() => {
+                    this.pending.delete(messageId);
+                    const stderrTail = this.getStderrTail();
+                    const baseMsg = `Operation timed out after ${timeoutMs}ms`;
+                    const msg = stderrTail ? `${baseMsg}. Recent stderr:\n${stderrTail}` : baseMsg;
+                    reject(new BridgeTimeoutError(msg));
+                }, timeoutMs);
+            }
+            // Set up abort handler
+            const abortHandler = () => {
+                if (timer) {
+                    clearTimeout(timer);
+                }
+                this.pending.delete(messageId);
+                reject(new BridgeTimeoutError('Operation aborted'));
+            };
+            if (signal) {
+                signal.addEventListener('abort', abortHandler, { once: true });
+            }
+            // Wrap resolve/reject to clean up abort listener
+            const wrappedResolve = (value) => {
+                if (timer) {
+                    clearTimeout(timer);
+                }
+                signal?.removeEventListener('abort', abortHandler);
+                resolve(value);
+            };
+            const wrappedReject = (error) => {
+                if (timer) {
+                    clearTimeout(timer);
+                }
+                signal?.removeEventListener('abort', abortHandler);
+                reject(error);
+            };
+            // Register pending request
+            this.pending.set(messageId, {
+                resolve: wrappedResolve,
+                reject: wrappedReject,
+                timer,
+            });
+            // Write message to stdin
+            this.writeToStdin(`${message}\n`).catch(err => {
+                this.pending.delete(messageId);
+                if (timer) {
+                    clearTimeout(timer);
+                }
+                signal?.removeEventListener('abort', abortHandler);
+                reject(this.classifyError(err));
+            });
+            this.requestCount++;
+        });
+    }
+    // ===========================================================================
+    // BOUNDED CONTEXT LIFECYCLE
+    // ===========================================================================
+    /**
+     * Initialize the transport by spawning the Python process.
+     */
+    async doInit() {
+        await this.spawnProcess();
+    }
+    /**
+     * Dispose the transport by killing the Python process.
+     */
+    async doDispose() {
+        // Reject all pending requests
+        const stderrTail = this.getStderrTail();
+        const msg = stderrTail
+            ? `Transport disposed. Stderr:\n${stderrTail}`
+            : 'Transport disposed';
+        const error = new BridgeDisposedError(msg);
+        for (const [, pending] of this.pending) {
+            if (pending.timer) {
+                clearTimeout(pending.timer);
+            }
+            pending.reject(error);
+        }
+        this.pending.clear();
+        // Clear write queue
+        for (const queued of this.writeQueue) {
+            this.clearQueuedWriteTimeout(queued);
+            queued.reject(error);
+        }
+        this.writeQueue.length = 0;
+        // Kill process
+        await this.killProcess();
+        // Clear buffers
+        this.stdoutBuffer = '';
+        this.stderrBuffer = '';
+        this.requestCount = 0;
+    }
+    // ===========================================================================
+    // ABSTRACT METHOD STUBS (from BoundedContext)
+    // ===========================================================================
+    /**
+     * Not implemented - ProcessIO is a transport, not a full bridge.
+     */
+    call(_module, _functionName, _args, _kwargs) {
+        throw new BridgeExecutionError('ProcessIO is a transport, use BridgeProtocol for operations');
+    }
+    /**
+     * Not implemented - ProcessIO is a transport, not a full bridge.
+     */
+    instantiate(_module, _className, _args, _kwargs) {
+        throw new BridgeExecutionError('ProcessIO is a transport, use BridgeProtocol for operations');
+    }
+    /**
+     * Not implemented - ProcessIO is a transport, not a full bridge.
+     */
+    callMethod(_handle, _methodName, _args, _kwargs) {
+        throw new BridgeExecutionError('ProcessIO is a transport, use BridgeProtocol for operations');
+    }
+    /**
+     * Not implemented - ProcessIO is a transport, not a full bridge.
+     */
+    disposeInstance(_handle) {
+        throw new BridgeExecutionError('ProcessIO is a transport, use BridgeProtocol for operations');
+    }
+    // ===========================================================================
+    // PROCESS MANAGEMENT
+    // ===========================================================================
+    /**
+     * Spawn the Python subprocess.
+     */
+    async spawnProcess() {
+        // Build environment - use provided env or inherit from process.env
+        // If env is provided, it should be the complete environment (already filtered by NodeBridge)
+        // We only add Python-specific variables on top
+        const baseEnv = Object.keys(this.envOverrides).length > 0 ? this.envOverrides : process.env;
+        const env = {
+            ...baseEnv,
+            // Ensure Python uses UTF-8
+            PYTHONUTF8: '1',
+            PYTHONIOENCODING: 'UTF-8',
+            // Disable Python buffering
+            PYTHONUNBUFFERED: '1',
+        };
+        // Spawn process
+        this.process = spawn(this.pythonPath, [this.bridgeScript], {
+            stdio: ['pipe', 'pipe', 'pipe'],
+            env,
+            cwd: this.cwd,
+        });
+        this.processExited = false;
+        this.processError = null;
+        // Set up event handlers
+        this.process.on('error', this.handleProcessError.bind(this));
+        this.process.on('exit', this.handleProcessExit.bind(this));
+        if (this.process.stdout) {
+            this.process.stdout.on('data', this.handleStdoutData.bind(this));
+            this.process.stdout.on('error', this.handleStdoutError.bind(this));
+        }
+        if (this.process.stderr) {
+            this.process.stderr.on('data', this.handleStderrData.bind(this));
+            this.process.stderr.on('error', this.handleStderrError.bind(this));
+        }
+        if (this.process.stdin) {
+            this.process.stdin.on('drain', this.handleStdinDrain.bind(this));
+            this.process.stdin.on('error', this.handleStdinError.bind(this));
+        }
+        // Wait for process to be ready (first heartbeat could be here)
+        // For now, just resolve immediately - the process is spawned
+        await Promise.resolve();
+    }
+    /**
+     * Kill the Python subprocess.
+     */
+    async killProcess() {
+        if (!this.process) {
+            return;
+        }
+        const proc = this.process;
+        this.process = null;
+        // Add a catch-all error handler to prevent uncaught exceptions during shutdown
+        // This must be added BEFORE removing other listeners and ending stdin
+        const noopErrorHandler = () => {
+            // Ignore errors during shutdown (e.g., EPIPE)
+        };
+        proc.stdin?.on('error', noopErrorHandler);
+        proc.on('error', noopErrorHandler);
+        // Gracefully end stdin to prevent EPIPE on pending writes
+        try {
+            proc.stdin?.end();
+        }
+        catch {
+            // Ignore errors ending stdin
+        }
+        // Remove other listeners to prevent callbacks after disposal
+        proc.removeAllListeners('exit');
+        proc.removeAllListeners('close');
+        proc.stdout?.removeAllListeners();
+        proc.stderr?.removeAllListeners();
+        // Kill the process
+        if (!proc.killed) {
+            proc.kill('SIGTERM');
+            // Wait briefly for graceful exit
+            await new Promise(resolve => {
+                const timeout = setTimeout(() => {
+                    if (!proc.killed) {
+                        proc.kill('SIGKILL');
+                    }
+                    resolve();
+                }, 1000);
+                proc.once('exit', () => {
+                    clearTimeout(timeout);
+                    resolve();
+                });
+            });
+        }
+    }
+    /**
+     * Restart the Python process.
+     */
+    async restartProcess() {
+        // Kill existing process
+        await this.killProcess();
+        // Clear buffers and restart flags
+        this.stdoutBuffer = '';
+        this.stderrBuffer = '';
+        this.requestCount = 0;
+        this.needsRestart = false;
+        // Spawn new process
+        await this.spawnProcess();
+    }
+    /**
+     * Mark the process for restart on the next send.
+     * This is called after stream errors to ensure the next request uses a fresh process.
+     * Works independently of restartAfterRequests setting.
+     */
+    markForRestart() {
+        this.needsRestart = true;
+    }
+    // ===========================================================================
+    // STREAM HANDLERS
+    // ===========================================================================
+    /**
+     * Handle stdout data from the Python process.
+     */
+    handleStdoutData(chunk) {
+        this.stdoutBuffer += chunk.toString();
+        // Check for excessive line length without newline
+        if (this.stdoutBuffer.length > this.maxLineLength &&
+            !this.stdoutBuffer.includes('\n')) {
+            const snippet = this.stdoutBuffer.slice(0, 500);
+            this.stdoutBuffer = '';
+            this.handleProtocolError(`Response line exceeded ${this.maxLineLength} bytes`, snippet);
+            return;
+        }
+        // Process complete lines
+        let newlineIndex;
+        while ((newlineIndex = this.stdoutBuffer.indexOf('\n')) !== -1) {
+            const line = this.stdoutBuffer.slice(0, newlineIndex);
+            this.stdoutBuffer = this.stdoutBuffer.slice(newlineIndex + 1);
+            // Skip empty lines
+            if (!line.trim()) {
+                continue;
+            }
+            // Check line length
+            if (line.length > this.maxLineLength) {
+                const snippet = line.slice(0, 500);
+                this.handleProtocolError(`Response line exceeded ${this.maxLineLength} bytes`, snippet);
+                return;
+            }
+            this.handleResponseLine(line);
+        }
+    }
+    /**
+     * Handle a complete response line from stdout.
+     */
+    handleResponseLine(line) {
+        // Extract ID to find pending request
+        const messageId = extractMessageId(line);
+        if (!messageId) {
+            this.handleProtocolError('Response missing "id" field', line);
+            return;
+        }
+        const pending = this.pending.get(messageId);
+        if (!pending) {
+            // Response for unknown request - could be for a timed-out request
+            // Log but don't fail
+            return;
+        }
+        // Remove from pending
+        this.pending.delete(messageId);
+        // Clear timeout
+        if (pending.timer) {
+            clearTimeout(pending.timer);
+        }
+        // Resolve with raw response
+        pending.resolve(line);
+    }
+    /**
+     * Handle stderr data from the Python process.
+     */
+    handleStderrData(chunk) {
+        try {
+            this.stderrBuffer += sanitizeStderr(chunk.toString());
+            // Keep only the tail
+            if (this.stderrBuffer.length > MAX_STDERR_BYTES) {
+                this.stderrBuffer = this.stderrBuffer.slice(this.stderrBuffer.length - MAX_STDERR_BYTES);
+            }
+        }
+        catch {
+            // Ignore stderr buffering errors
+        }
+    }
+    /**
+     * Handle process error event.
+     */
+    handleProcessError(err) {
+        this.processError = err;
+        this.processExited = true;
+        const msg = `Python process error: ${err.message}`;
+        const error = new BridgeProtocolError(msg);
+        this.rejectAllPending(error);
+    }
+    /**
+     * Handle process exit event.
+     */
+    handleProcessExit(code, signal) {
+        this.processExited = true;
+        const stderrTail = this.getStderrTail();
+        let msg;
+        if (signal) {
+            msg = `Python process killed by signal ${signal}`;
+        }
+        else if (code !== null && code !== 0) {
+            msg = `Python process exited with code ${code}`;
+        }
+        else {
+            msg = 'Python process exited';
+        }
+        if (stderrTail) {
+            msg += `. Stderr:\n${stderrTail}`;
+        }
+        const error = new BridgeProtocolError(msg);
+        this.rejectAllPending(error);
+    }
+    /**
+     * Handle stdin drain event (backpressure relief).
+     */
+    handleStdinDrain() {
+        this.draining = false;
+        this.flushWriteQueue();
+    }
+    /**
+     * Handle stdin error event.
+     */
+    handleStdinError(err) {
+        // EPIPE means process died
+        const error = new BridgeProtocolError(`stdin error: ${err.message}`);
+        // Reject all pending writes
+        for (const queued of this.writeQueue) {
+            this.clearQueuedWriteTimeout(queued);
+            queued.reject(error);
+        }
+        this.writeQueue.length = 0;
+        // Reject all pending requests
+        this.rejectAllPending(error);
+        // Mark for restart on next send
+        this.markForRestart();
+    }
+    /**
+     * Handle stdout error event.
+     * This can occur during pipe errors or when the process crashes.
+     */
+    handleStdoutError(err) {
+        const error = new BridgeProtocolError(`stdout error: ${err.message}`);
+        this.rejectAllPending(error);
+        this.markForRestart();
+    }
+    /**
+     * Handle stderr error event.
+     * This can occur during pipe errors or when the process crashes.
+     */
+    handleStderrError(err) {
+        // Stderr errors are less critical but still indicate process health issues
+        const error = new BridgeProtocolError(`stderr error: ${err.message}`);
+        this.rejectAllPending(error);
+        this.markForRestart();
+    }
+    // ===========================================================================
+    // WRITE MANAGEMENT
+    // ===========================================================================
+    /**
+     * Create a queued write entry with a timeout timer.
+     * The timer fires if the drain event never comes.
+     */
+    createQueuedWrite(data, resolve, reject) {
+        const queuedAt = Date.now();
+        const entry = { data, resolve, reject, queuedAt };
+        // Set up timeout timer that fires if drain never happens
+        entry.timeoutHandle = setTimeout(() => {
+            // Remove this entry from the queue
+            const index = this.writeQueue.indexOf(entry);
+            if (index !== -1) {
+                this.writeQueue.splice(index, 1);
+                reject(new BridgeTimeoutError(`Write queue timeout: entry waited ${this.writeQueueTimeoutMs}ms without drain`));
+            }
+        }, this.writeQueueTimeoutMs);
+        // Unref the timer so it doesn't keep the process alive
+        entry.timeoutHandle.unref();
+        return entry;
+    }
+    /**
+     * Clear the timeout for a queued write entry.
+     */
+    clearQueuedWriteTimeout(entry) {
+        if (entry.timeoutHandle) {
+            clearTimeout(entry.timeoutHandle);
+            entry.timeoutHandle = undefined;
+        }
+    }
+    /**
+     * Write data to stdin with backpressure handling.
+     */
+    writeToStdin(data) {
+        return new Promise((resolve, reject) => {
+            if (!this.process?.stdin || this.processExited) {
+                reject(new BridgeProtocolError('Process stdin not available'));
+                return;
+            }
+            if (this.draining || this.writeQueue.length > 0) {
+                // Queue the write with timestamp and timeout timer
+                this.writeQueue.push(this.createQueuedWrite(data, resolve, reject));
+                return;
+            }
+            // Try direct write (wrap in try-catch for synchronous EPIPE errors)
+            try {
+                const canWrite = this.process.stdin.write(data);
+                if (canWrite) {
+                    resolve();
+                }
+                else {
+                    // Backpressure - queue this write and set draining flag
+                    this.draining = true;
+                    this.writeQueue.push(this.createQueuedWrite(data, resolve, reject));
+                }
+            }
+            catch (err) {
+                // Synchronous write error (e.g., EPIPE)
+                this.markForRestart();
+                reject(new BridgeProtocolError(`Write error: ${err instanceof Error ? err.message : 'unknown'}`));
+            }
+        });
+    }
+    /**
+     * Flush queued writes when backpressure clears.
+     */
+    flushWriteQueue() {
+        const now = Date.now();
+        while (this.writeQueue.length > 0 && !this.draining) {
+            if (!this.process?.stdin || this.processExited) {
+                // Process died - reject all queued writes
+                for (const q of this.writeQueue) {
+                    this.clearQueuedWriteTimeout(q);
+                    q.reject(new BridgeProtocolError('Process stdin not available'));
+                }
+                this.writeQueue.length = 0;
+                this.markForRestart();
+                return;
+            }
+            const queued = this.writeQueue.shift();
+            if (!queued) {
+                return;
+            }
+            // Clear the timeout since we're processing this entry now
+            this.clearQueuedWriteTimeout(queued);
+            // Check for write queue timeout (fallback check, timer should have handled this)
+            if (now - queued.queuedAt > this.writeQueueTimeoutMs) {
+                queued.reject(new BridgeTimeoutError(`Write queue timeout: entry waited ${now - queued.queuedAt}ms (limit: ${this.writeQueueTimeoutMs}ms)`));
+                continue; // Process next entry
+            }
+            try {
+                const canWrite = this.process.stdin.write(queued.data);
+                if (canWrite) {
+                    queued.resolve();
+                }
+                else {
+                    // Still under pressure - put it back with a new timeout
+                    this.writeQueue.unshift(this.createQueuedWrite(queued.data, queued.resolve, queued.reject));
+                    this.draining = true;
+                    return;
+                }
+            }
+            catch (err) {
+                // Synchronous write error (e.g., EPIPE) - reject this and all remaining writes
+                const error = new BridgeProtocolError(`Write error: ${err instanceof Error ? err.message : 'unknown'}`);
+                queued.reject(error);
+                for (const q of this.writeQueue) {
+                    this.clearQueuedWriteTimeout(q);
+                    q.reject(error);
+                }
+                this.writeQueue.length = 0;
+                this.markForRestart();
+                return;
+            }
+        }
+    }
+    // ===========================================================================
+    // HELPERS
+    // ===========================================================================
+    /**
+     * Get the tail of stderr for diagnostics.
+     */
+    getStderrTail() {
+        return this.stderrBuffer.trim();
+    }
+    /**
+     * Handle a protocol error by rejecting all pending requests.
+     */
+    handleProtocolError(details, line) {
+        const snippet = line
+            ? line.length > 500 ? `${line.slice(0, 500)}...` : line
+            : undefined;
+        const hint = 'Ensure Python code does not print to stdout and bridge outputs only JSON lines.';
+        const msg = snippet
+            ? `Protocol error: ${details}\n${hint}\nOffending line: ${snippet}`
+            : `Protocol error: ${details}\n${hint}`;
+        const error = new BridgeProtocolError(msg);
+        this.rejectAllPending(error);
+    }
+    /**
+     * Reject all pending requests with an error.
+     */
+    rejectAllPending(error) {
+        for (const [, pending] of this.pending) {
+            if (pending.timer) {
+                clearTimeout(pending.timer);
+            }
+            pending.reject(error);
+        }
+        this.pending.clear();
+    }
+}
+//# sourceMappingURL=process-io.js.map