tywrap 0.4.0 → 0.5.1

package/dist/utils/parallel-processor.js

@@ -1,707 +0,0 @@
-/**
- * Parallel Processing System for tywrap
- * High-performance parallel processing for large Python codebases
- */
-import { EventEmitter } from 'events';
-import { cpus } from 'os';
-import { performance } from 'perf_hooks';
-import { Worker, isMainThread, parentPort, workerData } from 'worker_threads';
-import { fileURLToPath } from 'node:url';
-import { CodeGenerator } from '../core/generator.js';
-import { PyAnalyzer } from '../core/analyzer.js';
-import { globalCache } from './cache.js';
-import { getComponentLogger } from './logger.js';
-const log = getComponentLogger('ParallelProcessor');
-export class ParallelProcessor extends EventEmitter {
-    workers = new Map();
-    workerStats = new Map();
-    taskQueue = [];
-    queueInterval;
-    activeTasks = new Map();
-    pendingResults = new Map();
-    roundRobinIndex = 0;
-    options;
-    disposed = false;
-    debug = false;
-    constructor(options = {}) {
-        super();
-        const defaultWorkerScript = fileURLToPath(import.meta.url);
-        this.options = {
-            maxWorkers: options.maxWorkers ?? Math.min(cpus().length, 8),
-            taskTimeout: options.taskTimeout ?? 30000,
-            retryAttempts: options.retryAttempts ?? 2,
-            enableMemoryMonitoring: options.enableMemoryMonitoring ?? true,
-            enableCaching: options.enableCaching ?? true,
-            workerScript: options.workerScript ?? defaultWorkerScript,
-            batchSize: options.batchSize ?? 1,
-            loadBalancing: options.loadBalancing ?? 'least-loaded',
-            debug: options.debug ?? false,
-        };
-        this.debug = this.options.debug;
-        if (this.debug) {
-            this.debugLog(`🚀 Parallel processor initialized with ${this.options.maxWorkers} workers`);
-        }
-    }
-    debugLog(message) {
-        if (!this.debug) {
-            return;
-        }
-        process.stdout.write(`${message}\n`);
-    }
-    /**
-     * Initialize worker pool
-     */
-    async init() {
-        if (this.disposed) {
-            throw new Error('Processor has been disposed');
-        }
-        // Spawn initial worker pool
-        for (let i = 0; i < this.options.maxWorkers; i++) {
-            await this.spawnWorker(`worker_${i}`);
-        }
-        // Start task processing
-        this.processQueue();
-        this.debugLog(`✅ Worker pool initialized with ${this.workers.size} workers`);
-    }
-    /**
-     * Process Python module analysis in parallel
-     */
-    async analyzeModulesParallel(sources, options = {}) {
-        const chunkSize = options.chunkSize ?? Math.ceil(sources.length / this.options.maxWorkers);
-        const chunks = this.chunkArray(sources, chunkSize);
-        this.debugLog(`📊 Analyzing ${sources.length} modules in ${chunks.length} chunks`);
-        const tasks = chunks.map((chunk, index) => ({
-            id: `analyze_chunk_${index}`,
-            type: 'analyze',
-            data: { sources: chunk },
-            priority: 1,
-        }));
-        const results = await this.executeTasks(tasks);
-        // Flatten chunked results
-        const flatResults = [];
-        for (const result of results) {
-            if (!result.success || !result.result) {
-                continue;
-            }
-            if (result.success && result.result) {
-                const chunkResults = result.result;
-                if (Array.isArray(chunkResults)) {
-                    flatResults.push(...chunkResults);
-                }
-            }
-        }
-        return flatResults;
-    }
-    /**
-     * Generate TypeScript wrappers in parallel
-     */
-    async generateWrappersParallel(modules, options = {}) {
-        const chunkSize = options.chunkSize ?? Math.ceil(modules.length / this.options.maxWorkers);
-        const chunks = this.chunkArray(modules, chunkSize);
-        this.debugLog(`🏗️  Generating ${modules.length} wrappers in ${chunks.length} chunks`);
-        const tasks = chunks.map((chunk, index) => ({
-            id: `generate_chunk_${index}`,
-            type: 'generate',
-            data: { modules: chunk },
-            priority: 1,
-        }));
-        const results = await this.executeTasks(tasks);
-        // Flatten chunked results
-        const flatResults = [];
-        for (const result of results) {
-            if (result.success && result.result) {
-                const chunkResults = result.result;
-                if (Array.isArray(chunkResults)) {
-                    flatResults.push(...chunkResults);
-                }
-            }
-        }
-        return flatResults;
-    }
-    /**
-     * Execute tasks in parallel with load balancing
-     */
-    async executeTasks(tasks) {
-        if (this.workers.size === 0) {
-            await this.init();
-        }
-        // Sort tasks by priority
-        const sortedTasks = [...tasks].sort((a, b) => (b.priority ?? 0) - (a.priority ?? 0));
-        // Execute tasks with batching
-        const results = [];
-        const inFlight = new Set();
-        const maxInFlight = this.workers.size * 2;
-        for (const task of sortedTasks) {
-            const promise = this.executeTask(task);
-            inFlight.add(promise);
-            promise
-                .finally(() => {
-                inFlight.delete(promise);
-            })
-                .catch(() => {
-                // no-op: individual task failures are encoded in results
-            });
-            // Limit concurrent tasks to prevent overwhelming workers
-            if (inFlight.size >= maxInFlight) {
-                const completed = await Promise.race(inFlight);
-                results.push(completed);
-            }
-        }
-        // Wait for remaining tasks
-        const remaining = await Promise.all(inFlight);
-        results.push(...remaining);
-        return results;
-    }
-    /**
-     * Execute single task
-     */
-    async executeTask(task) {
-        // Check cache first if enabled
-        if (this.options.enableCaching) {
-            const cacheKey = globalCache.generateKey('parallel_task', task.type, task.data);
-            const cached = await globalCache.get(cacheKey);
-            if (cached) {
-                this.debugLog(`🎯 Cache HIT for task ${task.id}`);
-                return cached;
-            }
-        }
-        let attempts = 0;
-        let lastError;
-        while (attempts < this.options.retryAttempts) {
-            attempts++;
-            try {
-                const selected = this.selectOptimalWorker();
-                if (!selected) {
-                    throw new Error('No available workers');
-                }
-                const result = await this.sendTaskToWorker(selected.workerId, selected.worker, task);
-                // Cache successful results
-                if (this.options.enableCaching && result.success) {
-                    const cacheKey = globalCache.generateKey('parallel_task', task.type, task.data);
-                    await globalCache.set(cacheKey, result, { computeTime: result.duration });
-                }
-                return result;
-            }
-            catch (error) {
-                lastError = error;
-                log.warn('Task failed', { taskId: task.id, attempt: attempts, error: String(error) });
-                if (attempts < this.options.retryAttempts) {
-                    // Exponential backoff
-                    await this.sleep(Math.pow(2, attempts) * 1000);
-                }
-            }
-        }
-        // Return failure result
-        return {
-            taskId: task.id,
-            success: false,
-            error: lastError?.message ?? 'Task failed after retries',
-            duration: 0,
-        };
-    }
-    /**
-     * Select optimal worker using load balancing strategy
-     */
-    selectOptimalWorker() {
-        const availableWorkers = Array.from(this.workers.entries())
-            .map(([workerId, worker]) => ({ workerId, worker }))
-            .filter(({ workerId, worker }) => {
-            if (worker.threadId <= 0) {
-                return false;
-            }
-            const stats = this.workerStats.get(workerId);
-            return stats?.isActive !== false;
-        });
-        if (availableWorkers.length === 0) {
-            return null;
-        }
-        switch (this.options.loadBalancing) {
-            case 'round-robin': {
-                const selected = availableWorkers[this.roundRobinIndex % availableWorkers.length];
-                this.roundRobinIndex = (this.roundRobinIndex + 1) % availableWorkers.length;
-                return selected ?? null;
-            }
-            case 'least-loaded': {
-                // Find worker with least active tasks
-                const workerLoads = availableWorkers.map(({ workerId, worker }) => {
-                    const stats = this.workerStats.get(workerId);
-                    return { workerId, worker, load: stats?.tasksCompleted ?? 0 };
-                });
-                workerLoads.sort((a, b) => a.load - b.load);
-                const first = workerLoads[0];
-                return first ? { workerId: first.workerId, worker: first.worker } : null;
-            }
-            case 'weighted': {
-                // Weight by average task completion time
-                const workerWeights = availableWorkers.map(({ workerId, worker }) => {
-                    const stats = this.workerStats.get(workerId);
-                    const avgTime = stats?.averageTime ?? 1000;
-                    return { workerId, worker, weight: 1 / avgTime };
-                });
-                // Weighted random selection
-                const totalWeight = workerWeights.reduce((sum, w) => sum + w.weight, 0);
-                let random = Math.random() * totalWeight;
-                for (const { workerId, worker: candidate, weight } of workerWeights) {
-                    random -= weight;
-                    if (random <= 0) {
-                        return { workerId, worker: candidate };
-                    }
-                }
-                const fallback = workerWeights[0];
-                return fallback ? { workerId: fallback.workerId, worker: fallback.worker } : null;
-            }
-            default:
-                return availableWorkers[0] ?? null;
-        }
-    }
-    /**
-     * Send task to specific worker
-     */
-    async sendTaskToWorker(workerId, worker, task) {
-        return new Promise((resolve, reject) => {
-            if (this.disposed) {
-                reject(new Error('Processor has been disposed'));
-                return;
-            }
-            const timeoutMs = task.timeout ?? this.options.taskTimeout;
-            const timeout = setTimeout(() => {
-                const pending = this.pendingResults.get(task.id);
-                if (!pending) {
-                    return;
-                }
-                this.pendingResults.delete(task.id);
-                this.activeTasks.delete(task.id);
-                pending.reject(new Error(`Task ${task.id} timed out after ${timeoutMs}ms`));
-            }, timeoutMs);
-            this.pendingResults.set(task.id, {
-                workerId,
-                resolve: resolve,
-                reject,
-                timeout,
-            });
-            this.activeTasks.set(task.id, task);
-            try {
-                worker.postMessage({
-                    type: 'task',
-                    task,
-                });
-            }
-            catch (error) {
-                this.pendingResults.delete(task.id);
-                this.activeTasks.delete(task.id);
-                clearTimeout(timeout);
-                reject(error instanceof Error ? error : new Error(String(error)));
-            }
-        });
-    }
-    /**
-     * Spawn new worker
-     */
-    async spawnWorker(workerId) {
-        const worker = new Worker(this.options.workerScript, {
-            workerData: {
-                workerId,
-                options: {
-                    enableMemoryMonitoring: this.options.enableMemoryMonitoring,
-                    enableCaching: this.options.enableCaching,
-                },
-            },
-        });
-        // Setup worker message handling
-        worker.on('message', message => {
-            this.handleWorkerMessage(workerId, message);
-        });
-        // Setup worker error handling
-        worker.on('error', error => {
-            const workerError = error instanceof Error ? error : new Error(String(error));
-            log.error('Worker error', { workerId, error: String(workerError) });
-            this.handleWorkerError(workerId, workerError);
-        });
-        // Setup worker exit handling
-        worker.on('exit', code => {
-            log.warn('Worker exited', { workerId, code });
-            this.handleWorkerExit(workerId, code);
-        });
-        this.workers.set(workerId, worker);
-        this.workerStats.set(workerId, {
-            workerId,
-            tasksCompleted: 0,
-            totalTime: 0,
-            averageTime: 0,
-            errorCount: 0,
-            memoryPeak: 0,
-            isActive: true,
-        });
-        this.debugLog(`👷 Spawned worker ${workerId}`);
-        return worker;
-    }
-    /**
-     * Handle message from worker
-     */
-    handleWorkerMessage(workerId, message) {
-        if (message.type === 'task_complete') {
-            const { taskId, result, error, duration, memoryUsage } = message;
-            const pending = this.pendingResults.get(taskId);
-            if (pending) {
-                this.pendingResults.delete(taskId);
-                this.activeTasks.delete(taskId);
-                if (pending.timeout) {
-                    clearTimeout(pending.timeout);
-                }
-                // Update worker stats
-                const stats = this.workerStats.get(workerId);
-                if (stats) {
-                    stats.tasksCompleted++;
-                    stats.totalTime += duration;
-                    stats.averageTime = stats.totalTime / stats.tasksCompleted;
-                    if (error) {
-                        stats.errorCount++;
-                    }
-                    if (memoryUsage && memoryUsage > stats.memoryPeak) {
-                        stats.memoryPeak = memoryUsage;
-                    }
-                }
-                const parallelResult = {
-                    taskId,
-                    success: !error,
-                    result,
-                    error,
-                    duration,
-                    memoryUsage,
-                };
-                pending.resolve(parallelResult);
-            }
-        }
-        else if (message.type === 'worker_ready') {
-            this.emit('worker_ready', workerId);
-        }
-    }
-    /**
-     * Handle worker error
-     */
-    handleWorkerError(workerId, error) {
-        const stats = this.workerStats.get(workerId);
-        if (stats) {
-            stats.errorCount++;
-            stats.isActive = false;
-        }
-        // Reject only tasks owned by this worker.
-        const ownedTasks = [];
-        for (const [taskId, pending] of this.pendingResults) {
-            if (pending.workerId === workerId) {
-                ownedTasks.push(taskId);
-            }
-        }
-        for (const taskId of ownedTasks) {
-            const pending = this.pendingResults.get(taskId);
-            if (!pending) {
-                continue;
-            }
-            if (pending.timeout) {
-                clearTimeout(pending.timeout);
-            }
-            pending.reject(new Error(`Worker ${workerId} error: ${error.message}`));
-            this.pendingResults.delete(taskId);
-            this.activeTasks.delete(taskId);
-        }
-        this.emit('worker_error', workerId, error);
-    }
-    /**
-     * Handle worker exit
-     */
-    handleWorkerExit(workerId, code) {
-        this.workers.delete(workerId);
-        const stats = this.workerStats.get(workerId);
-        if (stats) {
-            stats.isActive = false;
-        }
-        // Reject any tasks that were in-flight on this worker.
-        const ownedTasks = [];
-        for (const [taskId, pending] of this.pendingResults) {
-            if (pending.workerId === workerId) {
-                ownedTasks.push(taskId);
-            }
-        }
-        for (const taskId of ownedTasks) {
-            const pending = this.pendingResults.get(taskId);
-            if (!pending) {
-                continue;
-            }
-            if (pending.timeout) {
-                clearTimeout(pending.timeout);
-            }
-            pending.reject(new Error(`Worker ${workerId} exited with code ${code}`));
-            this.pendingResults.delete(taskId);
-            this.activeTasks.delete(taskId);
-        }
-        // Respawn worker if not disposing
-        if (!this.disposed && this.workers.size < this.options.maxWorkers) {
-            this.spawnWorker(`${workerId}_respawn_${Date.now()}`).catch(error => {
-                log.error('Failed to respawn worker', { workerId, error: String(error) });
-            });
-        }
-        this.emit('worker_exit', workerId, code);
-    }
-    /**
-     * Process task queue
-     */
-    processQueue() {
-        // Simple queue processing - could be enhanced with priority scheduling
-        if (this.queueInterval) {
-            return;
-        }
-        this.queueInterval = setInterval(() => {
-            if (this.taskQueue.length > 0 && this.workers.size > 0) {
-                const task = this.taskQueue.shift();
-                if (task) {
-                    this.executeTask(task).catch(error => {
-                        log.error('Queue task execution failed', { error: String(error) });
-                    });
-                }
-            }
-        }, 100);
-        // Don't keep the event loop alive just for an idle queue pump.
-        this.queueInterval.unref?.();
-    }
-    /**
-     * Get worker statistics
-     */
-    getWorkerStats() {
-        return Array.from(this.workerStats.values());
-    }
-    /**
-     * Get overall processing statistics
-     */
-    getStats() {
-        const workers = Array.from(this.workerStats.values());
-        const totalTasks = workers.reduce((sum, w) => sum + w.tasksCompleted, 0);
-        const totalErrors = workers.reduce((sum, w) => sum + w.errorCount, 0);
-        const totalTime = workers.reduce((sum, w) => sum + w.totalTime, 0);
-        const avgTime = totalTasks > 0 ? totalTime / totalTasks : 0;
-        return {
-            activeWorkers: workers.filter(w => w.isActive).length,
-            totalWorkers: workers.length,
-            tasksCompleted: totalTasks,
-            totalErrors,
-            averageTaskTime: avgTime,
-            queueLength: this.taskQueue.length,
-            activeTasks: this.activeTasks.size,
-            workerStats: workers,
-        };
-    }
-    /**
-     * Utility methods
-     */
-    chunkArray(array, chunkSize) {
-        const chunks = [];
-        for (let i = 0; i < array.length; i += chunkSize) {
-            chunks.push(array.slice(i, i + chunkSize));
-        }
-        return chunks;
-    }
-    sleep(ms) {
-        return new Promise(resolve => setTimeout(resolve, ms));
-    }
-    /**
-     * Gracefully dispose all workers
-     */
-    async dispose() {
-        if (this.disposed) {
-            return;
-        }
-        this.disposed = true;
-        this.debugLog('🛑 Disposing parallel processor...');
-        if (this.queueInterval) {
-            clearInterval(this.queueInterval);
-            this.queueInterval = undefined;
-        }
-        // Reject any pending results immediately so callers don't hang.
-        const disposedError = new Error('Processor has been disposed');
-        for (const pending of this.pendingResults.values()) {
-            if (pending.timeout) {
-                clearTimeout(pending.timeout);
-            }
-            pending.reject(disposedError);
-        }
-        this.pendingResults.clear();
-        this.activeTasks.clear();
-        // Terminate all workers
-        const terminationPromises = Array.from(this.workers.values()).map(worker => {
-            return new Promise(resolve => {
-                const timeout = setTimeout(() => {
-                    worker.terminate();
-                    resolve();
-                }, 5000);
-                worker.once('exit', () => {
-                    clearTimeout(timeout);
-                    resolve();
-                });
-                try {
-                    worker.postMessage({ type: 'shutdown' });
-                }
-                catch {
-                    // Ignore: worker may already be dead.
-                }
-            });
-        });
-        await Promise.all(terminationPromises);
-        // Clear data structures
-        this.workers.clear();
-        this.workerStats.clear();
-        this.taskQueue.length = 0;
-        this.removeAllListeners();
-        this.debugLog('✅ Parallel processor disposed');
-    }
-    setDebug(debug) {
-        this.debug = debug;
-    }
-}
-// Worker script implementation (this runs in worker threads)
-if (!isMainThread && parentPort) {
-    const port = parentPort;
-    const { workerId } = workerData;
-    let analyzer = null;
-    let analyzerInitialized = false;
-    let generator = null;
-    const getAnalyzer = async () => {
-        analyzer ??= new PyAnalyzer();
-        if (!analyzerInitialized) {
-            await analyzer.initialize();
-            analyzerInitialized = true;
-        }
-        return analyzer;
-    };
-    const getGenerator = () => {
-        generator ??= new CodeGenerator();
-        return generator;
-    };
-    const createResultId = (prefix, name, index) => `${prefix}:${name}:${index}`;
-    // Import required modules in worker context
-    const workerImplementation = {
-        async processTask(task) {
-            const startTime = performance.now();
-            let result;
-            let error;
-            try {
-                switch (task.type) {
-                    case 'analyze':
-                        result = await this.processAnalysisTask(task.data);
-                        break;
-                    case 'generate':
-                        result = await this.processGenerationTask(task.data);
-                        break;
-                    case 'validate':
-                        result = await this.processValidationTask(task.data);
-                        break;
-                    default:
-                        throw new Error(`Unknown task type: ${task.type}`);
-                }
-            }
-            catch (err) {
-                error = String(err);
-            }
-            const duration = performance.now() - startTime;
-            const memoryUsage = process.memoryUsage().heapUsed;
-            return { result, error, duration, memoryUsage };
-        },
-        async processAnalysisTask(data) {
-            const results = [];
-            const activeAnalyzer = await getAnalyzer();
-            for (const [index, source] of data.sources.entries()) {
-                const start = performance.now();
-                try {
-                    const modulePath = source.path ?? `${source.name}.py`;
-                    const analysis = await activeAnalyzer.analyzePythonModule(source.content, modulePath);
-                    results.push({
-                        taskId: createResultId('analyze', source.name, index),
-                        success: true,
-                        result: analysis,
-                        duration: performance.now() - start,
-                        memoryUsage: process.memoryUsage().heapUsed,
-                    });
-                }
-                catch (error) {
-                    results.push({
-                        taskId: createResultId('analyze', source.name, index),
-                        success: false,
-                        error: String(error),
-                        duration: performance.now() - start,
-                        memoryUsage: process.memoryUsage().heapUsed,
-                    });
-                }
-            }
-            return results;
-        },
-        async processGenerationTask(data) {
-            const results = [];
-            const activeGenerator = getGenerator();
-            for (const [index, moduleData] of data.modules.entries()) {
-                const start = performance.now();
-                try {
-                    const generationOptions = {
-                        moduleName: moduleData.name,
-                        ...moduleData.options,
-                    };
-                    const generated = await activeGenerator.generateModule(moduleData.module, generationOptions);
-                    results.push({
-                        taskId: createResultId('generate', moduleData.name, index),
-                        success: true,
-                        result: generated,
-                        duration: performance.now() - start,
-                        memoryUsage: process.memoryUsage().heapUsed,
-                    });
-                }
-                catch (error) {
-                    results.push({
-                        taskId: createResultId('generate', moduleData.name, index),
-                        success: false,
-                        error: String(error),
-                        duration: performance.now() - start,
-                        memoryUsage: process.memoryUsage().heapUsed,
-                    });
-                }
-            }
-            return results;
-        },
-        async processValidationTask(_data) {
-            // Validation logic would go here
-            return { validated: true };
-        },
-    };
-    port.on('message', async (raw) => {
-        const message = raw;
-        if (!message || typeof message !== 'object') {
-            return;
-        }
-        if (message.type === 'task') {
-            try {
-                const taskResult = await workerImplementation.processTask(message.task);
-                port.postMessage({
-                    type: 'task_complete',
-                    taskId: message.task.id,
-                    ...(typeof taskResult === 'object' && taskResult !== null
-                        ? taskResult
-                        : { result: taskResult }),
-                });
-            }
-            catch (error) {
-                port.postMessage({
-                    type: 'task_complete',
-                    taskId: message.task.id,
-                    error: String(error),
-                    duration: 0,
-                });
-            }
-        }
-        else if (message.type === 'shutdown') {
-            process.exit(0);
-        }
-    });
-    // Signal that worker is ready
-    port.postMessage({
-        type: 'worker_ready',
-        workerId,
-    });
-}
-export const globalParallelProcessor = new ParallelProcessor();
-//# sourceMappingURL=parallel-processor.js.map