xypriss 1.0.0 → 1.0.1

package/dist/esm/cluster/cluster-manager.js

@@ -0,0 +1,990 @@
+import * as os from 'os';
+import { EventEmitter } from 'events';
+import { WorkerManager } from './modules/WorkerManager.js';
+import { HealthMonitor } from './modules/HealthMonitor.js';
+import { LoadBalancer } from './modules/LoadBalancer.js';
+import { IPCManager } from './modules/IPCManager.js';
+import { MetricsCollector } from './modules/MetricsCollector.js';
+import { AutoScaler } from './modules/AutoScaler.js';
+import { ClusterPersistenceManager } from './modules/ClusterPersistenceManager.js';
+import { SecurityErrorLogger, createSecurityError, ErrorSeverity, ErrorType } from '../mods/toolkit/src/utils/errorHandler.js';
+import { DEFAULT_CLUSTER_CONFIGS } from '../server/const/Cluster.config.js';
+import { logger } from '../server/utils/Logger.js';
+
+/**
+ * XyPrissJS Cluster Manager
+ *  cluster management for Express applications with advanced monitoring
+ */
+/**
+ *  cluster manager with comprehensive monitoring and auto-scaling
+ */
+class ClusterManager extends EventEmitter {
+    constructor(config = {}) {
+        super();
+        this.state = "initializing";
+        this.startTime = new Date();
+        this.isShuttingDown = false;
+        // Apply intelligent defaults
+        this.config = this.applyDefaults(config);
+        // Initialize components with fortified security
+        this.errorLogger = new SecurityErrorLogger();
+        this.workerManager = new WorkerManager(this.config, this.errorLogger);
+        this.healthMonitor = new HealthMonitor(this.config, this.errorLogger);
+        this.loadBalancer = new LoadBalancer(this.config);
+        this.ipcManager = new IPCManager(this.config, this.errorLogger);
+        this.metricsCollector = new MetricsCollector(this.config);
+        this.autoScaler = new AutoScaler(this.config, this.errorLogger);
+        // Initialize persistence manager if configured
+        if (this.config.persistence?.enabled && this.config.persistence.type) {
+            this.persistenceManager = new ClusterPersistenceManager(this.config.persistence);
+        }
+        // Setup component integrations
+        this.setupComponentIntegrations();
+        // Setup event forwarding
+        this.setupEventForwarding();
+        // Setup error handling
+        this.setupErrorHandling();
+    }
+    /**
+     * Apply intelligent defaults to cluster configuration
+     */
+    applyDefaults(config) {
+        const defaults = DEFAULT_CLUSTER_CONFIGS;
+        return this.mergeDeep(defaults, config);
+    }
+    /**
+     * Deep merge configuration objects
+     */
+    mergeDeep(target, source) {
+        const result = { ...target };
+        for (const key in source) {
+            if (source[key] &&
+                typeof source[key] === "object" &&
+                !Array.isArray(source[key])) {
+                result[key] = this.mergeDeep(target[key] || {}, source[key]);
+            }
+            else {
+                result[key] = source[key];
+            }
+        }
+        return result;
+    }
+    /**
+     * Setup component integrations for cross-component communication
+     */
+    setupComponentIntegrations() {
+        // Integrate IPC Manager with Worker Manager
+        this.ipcManager.setWorkerManager(this.workerManager);
+        // Setup auto-scaler with worker manager and metrics collector integration
+        this.autoScaler.setWorkerManager(this.workerManager);
+        this.autoScaler.setMetricsCollector(this.metricsCollector);
+        // Setup metrics collector with worker manager integration
+        this.metricsCollector.setWorkerManager(this.workerManager);
+        // Setup health monitor with worker manager integration
+        this.healthMonitor.setWorkerManager(this.workerManager);
+        // Setup load balancer with IPC manager integration
+        this.loadBalancer.setIPCManager(this.ipcManager);
+    }
+    /**
+     * Setup event forwarding between components
+     */
+    setupEventForwarding() {
+        // Forward worker manager events
+        this.workerManager.on("worker:started", (workerId, pid) => {
+            this.emit("worker:started", workerId, pid);
+        });
+        this.workerManager.on("worker:died", (workerId, code, signal) => {
+            this.emit("worker:died", workerId, code, signal);
+        });
+        this.workerManager.on("worker:restarted", (workerId, reason) => {
+            this.emit("worker:restarted", workerId, reason);
+        });
+        // Forward health monitor events
+        this.healthMonitor.on("worker:health:warning", (workerId, metrics) => {
+            this.emit("worker:health:warning", workerId, metrics);
+        });
+        this.healthMonitor.on("worker:health:critical", (workerId, metrics) => {
+            this.emit("worker:health:critical", workerId, metrics);
+        });
+        // Forward auto-scaler events
+        this.autoScaler.on("cluster:scaled", (action, newWorkerCount) => {
+            this.emit("cluster:scaled", action, newWorkerCount);
+        });
+        this.autoScaler.on("scaling:triggered", (reason, currentWorkers, targetWorkers) => {
+            this.emit("scaling:triggered", reason, currentWorkers, targetWorkers);
+        });
+        // Forward load balancer events
+        this.loadBalancer.on("loadbalancer:updated", (strategy, weights) => {
+            this.emit("loadbalancer:updated", strategy, weights);
+        });
+        // Forward IPC events
+        this.ipcManager.on("ipc:message", (from, to, message) => {
+            this.emit("ipc:message", from, to, message);
+        });
+        this.ipcManager.on("ipc:broadcast", (from, message) => {
+            this.emit("ipc:broadcast", from, message);
+        });
+        // Forward metrics events
+        this.metricsCollector.on("metrics:updated", (metrics) => {
+            this.updateClusterState(metrics);
+        });
+    }
+    /**
+     * Setup comprehensive error handling
+     */
+    setupErrorHandling() {
+        // Handle uncaught exceptions
+        process.on("uncaughtException", (error) => {
+            const securityError = createSecurityError(`Uncaught exception in cluster manager: ${error.message}`, ErrorType.INTERNAL, ErrorSeverity.CRITICAL, "CLUSTER_UNCAUGHT_EXCEPTION", { operation: "cluster_manager" });
+            this.errorLogger.logError(securityError);
+            if (this.config.errorHandling?.uncaughtException === "restart") {
+                this.handleCriticalError("uncaught_exception");
+            }
+        });
+        // Handle unhandled rejections
+        process.on("unhandledRejection", (reason) => {
+            const securityError = createSecurityError(`Unhandled rejection in cluster manager: ${reason}`, ErrorType.INTERNAL, ErrorSeverity.CRITICAL, "CLUSTER_UNHANDLED_REJECTION", { operation: "cluster_manager" });
+            this.errorLogger.logError(securityError);
+            if (this.config.errorHandling?.unhandledRejection === "restart") {
+                this.handleCriticalError("unhandled_rejection");
+            }
+        });
+    }
+    /**
+     * Update cluster state based on metrics
+     */
+    updateClusterState(metrics) {
+        // Update state based on cluster health
+        if (metrics.healthOverall.status === "critical") {
+            this.state = "degraded";
+        }
+        else if (metrics.activeWorkers === 0) {
+            this.state = "stopped";
+        }
+        else if (this.state === "initializing" || this.state === "starting") {
+            this.state = "running";
+        }
+    }
+    /**
+     * Handle critical errors with recovery strategies
+     */
+    async handleCriticalError(errorType) {
+        logger.error("cluster", `Critical error detected: ${errorType}`);
+        try {
+            // Attempt graceful recovery
+            await this.restart();
+        }
+        catch (error) {
+            logger.error("cluster", "Failed to recover from critical error:", error);
+            process.exit(1);
+        }
+    }
+    // ===== CORE CLUSTER MANAGEMENT METHODS =====
+    /**
+     * Start the cluster with intelligent worker management
+     */
+    async start() {
+        if (this.state === "running") {
+            logger.debug("cluster", "Cluster is already running");
+            return;
+        }
+        this.state = "starting";
+        try {
+            // logger.debug( "cluster","Starting cluster...");
+            // Determine if we're in master or worker process
+            const clusterModule = require("cluster");
+            if (clusterModule.isMaster) {
+                logger.debug("cluster", "Starting as cluster master process");
+                // Start monitoring components
+                this.healthMonitor.startMonitoring();
+                this.metricsCollector.startCollection();
+                // Start workers
+                await this.workerManager.startWorkers();
+                // Update auto-scaler with initial worker count
+                const workerCount = this.workerManager.getActiveWorkers().length;
+                this.autoScaler.updateWorkerCount(workerCount);
+                // Setup graceful shutdown
+                this.setupGracefulShutdown();
+                logger.debug("cluster", `Cluster master started with ${workerCount} workers`);
+            }
+            else {
+                logger.debug("cluster", `Worker ${process.pid} started`);
+                // Worker-specific initialization
+                this.setupWorkerProcess();
+            }
+            this.state = "running";
+            // logger.debug( "cluster","XyPrissJS cluster started successfully");
+        }
+        catch (error) {
+            this.state = "error";
+            const securityError = createSecurityError(`Cluster start failed: ${error.message}`, ErrorType.INTERNAL, ErrorSeverity.CRITICAL, "CLUSTER_START_ERROR", { operation: "start" });
+            this.errorLogger.logError(securityError);
+            throw error;
+        }
+    }
+    /**
+     * Setup worker process initialization
+     */
+    setupWorkerProcess() {
+        // Set worker environment
+        process.env.WORKER_ID =
+            process.env.WORKER_ID || process.pid?.toString() || "unknown";
+        process.env.NODE_ENV = "worker";
+        // Setup worker-specific error handling
+        process.on("uncaughtException", (error) => {
+            logger.error("cluster", "Worker uncaught exception:", error);
+            process.exit(1);
+        });
+        process.on("unhandledRejection", (reason, promise) => {
+            logger.error("cluster", "Worker unhandled rejection at:", promise, "reason:", reason);
+            process.exit(1);
+        });
+        // Send ready signal to master
+        if (process.send) {
+            process.send({
+                type: "worker_ready",
+                workerId: process.env.WORKER_ID,
+                pid: process.pid,
+            });
+        }
+    }
+    /**
+     * Setup graceful shutdown handling
+     */
+    setupGracefulShutdown() {
+        const gracefulShutdown = async (signal) => {
+            logger.debug("cluster", `Received ${signal}, starting graceful shutdown...`);
+            try {
+                await this.stop();
+                process.exit(0);
+            }
+            catch (error) {
+                logger.error("cluster", "Error during graceful shutdown:", error);
+                process.exit(1);
+            }
+        };
+        process.on("SIGTERM", () => gracefulShutdown("SIGTERM"));
+        process.on("SIGINT", () => gracefulShutdown("SIGINT"));
+    }
+    /**
+     * Stop the cluster gracefully or forcefully
+     */
+    async stop(graceful = true) {
+        if (this.state === "stopped" || this.state === "stopping") {
+            logger.debug("cluster", "Cluster is already stopped or stopping");
+            return;
+        }
+        this.state = "stopping";
+        try {
+            logger.debug("cluster", `Stopping cluster ${graceful ? "gracefully" : "forcefully"}...`);
+            // Stop monitoring
+            this.healthMonitor.stopMonitoring();
+            this.metricsCollector.stopCollection();
+            this.autoScaler.stopScaling();
+            // Stop all workers
+            await this.workerManager.stopAllWorkers(graceful);
+            // Close persistence manager
+            await this.closePersistenceManager();
+            this.state = "stopped";
+            logger.debug("cluster", "Cluster stopped successfully");
+        }
+        catch (error) {
+            this.state = "error";
+            const securityError = createSecurityError(`Cluster stop failed: ${error.message}`, ErrorType.INTERNAL, ErrorSeverity.HIGH, "CLUSTER_STOP_ERROR", { operation: "stop" });
+            this.errorLogger.logError(securityError);
+            throw error;
+        }
+    }
+    /**
+     * Restart the cluster or specific worker
+     */
+    async restart(workerId) {
+        if (workerId) {
+            // Restart specific worker using WorkerManager
+            try {
+                logger.debug("cluster", `Restarting worker ${workerId}...`);
+                // Use WorkerManager to restart the specific worker
+                const newWorkerId = await this.replaceWorker(workerId);
+                this.emit("worker:restarted", newWorkerId, "manual_restart");
+                logger.debug("cluster", `Worker ${workerId} restarted as ${newWorkerId}`);
+            }
+            catch (error) {
+                const securityError = createSecurityError(`Failed to restart worker ${workerId}: ${error.message}`, ErrorType.INTERNAL, ErrorSeverity.HIGH, "WORKER_RESTART_ERROR", { operation: "restart_worker" });
+                this.errorLogger.logError(securityError);
+                throw error;
+            }
+        }
+        else {
+            // Restart entire cluster
+            try {
+                logger.debug("cluster", "Restarting entire cluster...");
+                await this.stop(true);
+                await new Promise((resolve) => setTimeout(resolve, 2000)); // Brief pause
+                await this.start();
+                logger.debug("cluster", "Cluster restarted successfully");
+            }
+            catch (error) {
+                const securityError = createSecurityError(`Failed to restart cluster: ${error.message}`, ErrorType.INTERNAL, ErrorSeverity.CRITICAL, "CLUSTER_RESTART_ERROR", { operation: "restart_cluster" });
+                this.errorLogger.logError(securityError);
+                throw error;
+            }
+        }
+    }
+    /**
+     * Pause cluster operations
+     */
+    async pause() {
+        if (this.state !== "running") {
+            throw new Error("Cannot pause cluster: not running");
+        }
+        this.state = "paused";
+        // Pause monitoring
+        this.healthMonitor.stopMonitoring();
+        this.autoScaler.disable();
+        logger.debug("cluster", "Cluster paused");
+    }
+    /**
+     * Resume cluster operations
+     */
+    async resume() {
+        if (this.state !== "paused") {
+            throw new Error("Cannot resume cluster: not paused");
+        }
+        this.state = "running";
+        // Resume monitoring
+        this.healthMonitor.startMonitoring();
+        this.autoScaler.enable();
+        logger.debug("cluster", "Cluster resumed");
+    }
+    // ===== WORKER MANAGEMENT METHODS =====
+    /**
+     * Add new worker to the cluster
+     */
+    async addWorker() {
+        const workers = this.workerManager.getActiveWorkers();
+        const maxWorkers = this.config.autoScaling?.maxWorkers || 8;
+        if (workers.length >= maxWorkers) {
+            throw new Error("Cannot add worker: maximum worker limit reached");
+        }
+        try {
+            // Use WorkerManager to actually start a worker
+            const workerId = await this.workerManager.startSingleWorker();
+            // Update auto-scaler with new worker count
+            this.autoScaler.updateWorkerCount(workers.length + 1);
+            this.emit("worker:started", workerId, 0);
+            return workerId;
+        }
+        catch (error) {
+            const securityError = createSecurityError(`Failed to add worker: ${error.message}`, ErrorType.INTERNAL, ErrorSeverity.HIGH, "WORKER_ADD_ERROR", { operation: "add_worker" });
+            this.errorLogger.logError(securityError);
+            throw error;
+        }
+    }
+    /**
+     * Remove worker from the cluster
+     */
+    async removeWorker(workerId, graceful = true) {
+        const workers = this.workerManager.getActiveWorkers();
+        const minWorkers = this.config.autoScaling?.minWorkers || 1;
+        if (workers.length <= minWorkers) {
+            throw new Error("Cannot remove worker: minimum worker limit reached");
+        }
+        try {
+            // Use WorkerManager to actually stop the worker
+            await this.workerManager.stopSingleWorker(workerId, graceful);
+            // Update auto-scaler with new worker count
+            this.autoScaler.updateWorkerCount(workers.length - 1);
+            this.emit("worker:died", workerId, 0, graceful ? "SIGTERM" : "SIGKILL");
+        }
+        catch (error) {
+            const securityError = createSecurityError(`Failed to remove worker ${workerId}: ${error.message}`, ErrorType.INTERNAL, ErrorSeverity.HIGH, "WORKER_REMOVE_ERROR", { operation: "remove_worker" });
+            this.errorLogger.logError(securityError);
+            throw error;
+        }
+    }
+    /**
+     * Replace worker with a new one
+     */
+    async replaceWorker(workerId) {
+        logger.debug("cluster", `Replacing worker ${workerId}...`);
+        // Start new worker first
+        const newWorkerId = await this.addWorker();
+        // Wait for new worker to be ready
+        await new Promise((resolve) => setTimeout(resolve, 2000));
+        // Remove old worker
+        await this.removeWorker(workerId, true);
+        return newWorkerId;
+    }
+    /**
+     * Get worker by ID
+     */
+    getWorker(workerId) {
+        return this.workerManager.getWorker(workerId);
+    }
+    /**
+     * Get all workers
+     */
+    getAllWorkers() {
+        return this.workerManager.getActiveWorkers();
+    }
+    /**
+     * Get active workers
+     */
+    getActiveWorkers() {
+        return this.workerManager
+            .getActiveWorkers()
+            .filter((w) => w.health.status === "healthy" ||
+            w.health.status === "warning");
+    }
+    /**
+     * Get unhealthy workers
+     */
+    getUnhealthyWorkers() {
+        return this.workerManager
+            .getActiveWorkers()
+            .filter((w) => w.health.status === "critical" || w.health.status === "dead");
+    }
+    // ===== HEALTH MONITORING METHODS =====
+    /**
+     * Check health of specific worker or all workers
+     */
+    async checkHealth(workerId) {
+        if (workerId) {
+            return await this.healthMonitor.checkWorkerHealth(workerId);
+        }
+        else {
+            await this.healthMonitor.performHealthChecks();
+            const healthStatus = this.healthMonitor.getHealthStatus();
+            return Object.values(healthStatus).every((healthy) => healthy);
+        }
+    }
+    /**
+     * Get health status of all workers
+     */
+    async getHealthStatus() {
+        return this.healthMonitor.getHealthStatus();
+    }
+    /**
+     * Run comprehensive health check
+     */
+    async runHealthCheck() {
+        await this.healthMonitor.performHealthChecks();
+    }
+    // ===== METRICS AND MONITORING METHODS =====
+    /**
+     * Get comprehensive cluster metrics
+     */
+    async getMetrics() {
+        return this.metricsCollector.getClusterMetrics();
+    }
+    /**
+     * Get metrics for specific worker
+     */
+    async getWorkerMetrics(workerId) {
+        return this.metricsCollector.getWorkerMetrics(workerId);
+    }
+    /**
+     * Get aggregated metrics summary
+     */
+    async getAggregatedMetrics() {
+        return this.metricsCollector.getAggregatedMetrics();
+    }
+    /**
+     * Start monitoring systems
+     */
+    startMonitoring() {
+        this.healthMonitor.startMonitoring();
+        this.metricsCollector.startCollection();
+        logger.debug("cluster", "Monitoring systems started");
+    }
+    /**
+     * Stop monitoring systems
+     */
+    stopMonitoring() {
+        this.healthMonitor.stopMonitoring();
+        this.metricsCollector.stopCollection();
+        logger.debug("cluster", "Monitoring systems stopped");
+    }
+    /**
+     * Export metrics in specified format
+     */
+    async exportMetrics(format = "json") {
+        return await this.metricsCollector.exportMetrics(format);
+    }
+    // ===== SCALING METHODS =====
+    /**
+     * Scale up by adding workers
+     */
+    async scaleUp(count = 1) {
+        await this.autoScaler.scaleUp(count);
+    }
+    /**
+     * Scale down by removing workers
+     */
+    async scaleDown(count = 1) {
+        await this.autoScaler.scaleDown(count);
+    }
+    /**
+     * Perform auto-scaling evaluation
+     */
+    async autoScale() {
+        await this.autoScaler.autoScale();
+    }
+    /**
+     * Get optimal worker count
+     */
+    async getOptimalWorkerCount() {
+        return await this.autoScaler.getOptimalWorkerCount();
+    }
+    // ===== LOAD BALANCING METHODS =====
+    /**
+     * Update load balancing strategy
+     */
+    async updateLoadBalancingStrategy(strategy, options) {
+        await this.loadBalancer.updateStrategy(strategy, options);
+    }
+    /**
+     * Get load balance status
+     */
+    async getLoadBalanceStatus() {
+        const status = this.loadBalancer.getLoadBalanceStatus();
+        return status.connections;
+    }
+    /**
+     * Redistribute load across workers
+     */
+    async redistributeLoad() {
+        await this.loadBalancer.redistributeLoad();
+    }
+    // ===== IPC METHODS =====
+    /**
+     * Send message to specific worker
+     */
+    async sendToWorker(workerId, message) {
+        await this.ipcManager.sendToWorker(workerId, message);
+    }
+    /**
+     * Send message to all workers
+     */
+    async sendToAllWorkers(message) {
+        await this.ipcManager.sendToAllWorkers(message);
+    }
+    /**
+     * Broadcast message to all workers
+     */
+    async broadcast(message) {
+        await this.ipcManager.broadcast(message);
+    }
+    /**
+     * Send message to random worker
+     */
+    async sendToRandomWorker(message) {
+        await this.ipcManager.sendToRandomWorker(message);
+    }
+    /**
+     * Send message to least loaded worker
+     */
+    async sendToLeastLoadedWorker(message) {
+        await this.ipcManager.sendToLeastLoadedWorker(message);
+    }
+    // ===== EVENT HANDLING METHODS =====
+    /**
+     * Register event handler
+     */
+    addListener(event, handler) {
+        return super.on(event, handler);
+    }
+    /**
+     * Remove event handler
+     */
+    removeListener(event, handler) {
+        return super.off(event, handler);
+    }
+    /**
+     * Emit cluster event
+     */
+    emitEvent(event, ...args) {
+        return super.emit(event, ...args);
+    }
+    // ===== STATE MANAGEMENT METHODS =====
+    /**
+     * Save cluster state
+     */
+    async saveState() {
+        try {
+            // Create comprehensive persistent state
+            const persistentState = {
+                state: this.state,
+                config: this.config,
+                workers: this.workerManager
+                    .getActiveWorkers()
+                    .map((worker) => ({
+                    id: worker.workerId,
+                    pid: worker.pid,
+                    status: worker.health.status === "healthy"
+                        ? "running"
+                        : "stopped",
+                    startTime: worker.lastRestart || new Date(),
+                    restarts: worker.restarts,
+                    metrics: worker,
+                })),
+                metrics: this.metricsCollector.exportMetricsForPersistence(),
+                loadBalancer: {
+                    strategy: this.config.loadBalancing?.strategy || "round-robin",
+                    weights: {},
+                    distribution: {},
+                    historicalTrends: {
+                        requestsPerMinute: [],
+                        averageResponseTimes: [],
+                        errorRates: [],
+                    },
+                },
+                timestamp: new Date(),
+                version: "1.0.0",
+            };
+            // Save to persistent storage
+            await this.saveClusterStateToPersistentStorage(persistentState);
+        }
+        catch (error) {
+            const securityError = createSecurityError(`Failed to save cluster state: ${error.message}`, ErrorType.INTERNAL, ErrorSeverity.MEDIUM, "CLUSTER_STATE_SAVE_ERROR", { operation: "save_state" });
+            this.errorLogger.logError(securityError);
+            throw error;
+        }
+    }
+    /**
+     * Restore cluster state
+     */
+    async restoreState() {
+        try {
+            // Restore from persistent storage
+            const savedState = await this.loadClusterStateFromPersistentStorage();
+            if (savedState) {
+                await this.applyRestoredState(savedState);
+                logger.debug("cluster", " Cluster state restored from persistent storage");
+            }
+            else {
+                logger.debug("cluster", "No saved cluster state found, starting fresh");
+            }
+            // if (savedState) {
+            //     await this.applyRestoredState(savedState);
+            // }
+        }
+        catch (error) {
+            const securityError = createSecurityError(`Failed to restore cluster state: ${error.message}`, ErrorType.INTERNAL, ErrorSeverity.MEDIUM, "CLUSTER_STATE_RESTORE_ERROR", { operation: "restore_state" });
+            this.errorLogger.logError(securityError);
+            throw error;
+        }
+    }
+    /**
+     * Get current cluster state
+     */
+    async getState() {
+        return this.state;
+    }
+    /**
+     * Export cluster configuration
+     */
+    async exportConfiguration() {
+        return { ...this.config };
+    }
+    // ===== UTILITY METHODS =====
+    /**
+     * Check if cluster is healthy
+     */
+    isHealthy() {
+        const activeWorkers = this.getActiveWorkers();
+        const totalWorkers = this.getAllWorkers();
+        if (totalWorkers.length === 0)
+            return false;
+        const healthyPercentage = (activeWorkers.length / totalWorkers.length) * 100;
+        return healthyPercentage >= 70; // 70% threshold
+    }
+    /**
+     * Get load balance score
+     */
+    getLoadBalance() {
+        return this.loadBalancer.getLoadDistributionEfficiency();
+    }
+    /**
+     * Get recommended worker count
+     */
+    getRecommendedWorkerCount() {
+        const cpuCount = os.cpus().length;
+        const currentLoad = this.getActiveWorkers().length;
+        // Simple recommendation based on CPU cores and current load
+        return Math.max(1, Math.min(cpuCount - 1, currentLoad));
+    }
+    /**
+     * Get cluster efficiency score
+     */
+    getClusterEfficiency() {
+        const loadBalanceEfficiency = this.getLoadBalance();
+        const healthScore = this.isHealthy() ? 100 : 50;
+        const utilizationScore = this.calculateUtilizationScore();
+        return (loadBalanceEfficiency + healthScore + utilizationScore) / 3;
+    }
+    /**
+     * Calculate utilization score
+     */
+    calculateUtilizationScore() {
+        const workers = this.getActiveWorkers();
+        if (workers.length === 0)
+            return 0;
+        const avgCpu = workers.reduce((sum, w) => sum + w.cpu.usage, 0) / workers.length;
+        const avgMemory = workers.reduce((sum, w) => sum + w.memory.percentage, 0) /
+            workers.length;
+        // Optimal utilization is around 60-70%
+        const optimalCpu = 65;
+        const optimalMemory = 65;
+        const cpuScore = 100 - Math.abs(avgCpu - optimalCpu);
+        const memoryScore = 100 - Math.abs(avgMemory - optimalMemory);
+        return (cpuScore + memoryScore) / 2;
+    }
+    // ===== ADVANCED FEATURES =====
+    /**
+     * Enable profiling for worker
+     */
+    async enableProfiling(workerId) {
+        logger.debug("cluster", `Enabling profiling for worker ${workerId}`);
+        await this.sendToWorker(workerId, { type: "enable_profiling" });
+    }
+    /**
+     * Disable profiling for worker
+     */
+    async disableProfiling(workerId) {
+        logger.debug("cluster", `Disabling profiling for worker ${workerId}`);
+        await this.sendToWorker(workerId, { type: "disable_profiling" });
+    }
+    /**
+     * Take heap snapshot of worker
+     */
+    async takeHeapSnapshot(workerId) {
+        const snapshotPath = `/tmp/heap-snapshot-${workerId}-${Date.now()}.heapsnapshot`;
+        logger.debug("cluster", `Taking heap snapshot for worker ${workerId}: ${snapshotPath}`);
+        await this.sendToWorker(workerId, {
+            type: "take_heap_snapshot",
+            path: snapshotPath,
+        });
+        return snapshotPath;
+    }
+    /**
+     * Enable debug mode for worker
+     */
+    async enableDebugMode(workerId, port) {
+        const debugPort = port || 9229;
+        logger.debug("cluster", `Enabling debug mode for worker ${workerId} on port ${debugPort}`);
+        await this.sendToWorker(workerId, {
+            type: "enable_debug",
+            port: debugPort,
+        });
+    }
+    /**
+     * Disable debug mode for worker
+     */
+    async disableDebugMode(workerId) {
+        logger.debug("cluster", `Disabling debug mode for worker ${workerId}`);
+        await this.sendToWorker(workerId, { type: "disable_debug" });
+    }
+    // ===== ROLLING UPDATES =====
+    /**
+     * Perform rolling update
+     */
+    async performRollingUpdate(updateFn) {
+        logger.debug("cluster", "Starting rolling update...");
+        const workers = this.getAllWorkers();
+        const maxUnavailable = this.config.advanced?.deployment?.maxUnavailable || 1;
+        for (let i = 0; i < workers.length; i += maxUnavailable) {
+            const batch = workers.slice(i, i + maxUnavailable);
+            // Drain workers in batch
+            for (const worker of batch) {
+                await this.drainWorker(worker.workerId);
+            }
+            // Perform update
+            await updateFn();
+            // Replace workers
+            for (const worker of batch) {
+                await this.replaceWorker(worker.workerId);
+            }
+            // Wait for new workers to be ready
+            await new Promise((resolve) => setTimeout(resolve, 5000));
+        }
+        logger.debug("cluster", "Rolling update completed");
+    }
+    /**
+     * Drain worker connections
+     */
+    async drainWorker(workerId) {
+        logger.debug("cluster", `Draining worker ${workerId}...`);
+        await this.sendToWorker(workerId, { type: "drain_connections" });
+        // Wait for connections to drain
+        await new Promise((resolve) => setTimeout(resolve, 10000));
+    }
+    // ===== CIRCUIT BREAKER =====
+    /**
+     * Check if circuit breaker is open
+     */
+    isCircuitOpen(workerId) {
+        if (workerId) {
+            const worker = this.getWorker(workerId);
+            return worker ? worker.health.consecutiveFailures >= 5 : false;
+        }
+        // Check overall cluster circuit breaker
+        const unhealthyWorkers = this.getUnhealthyWorkers();
+        const totalWorkers = this.getAllWorkers();
+        return (totalWorkers.length > 0 &&
+            unhealthyWorkers.length / totalWorkers.length > 0.5);
+    }
+    /**
+     * Reset circuit breaker
+     */
+    async resetCircuitBreaker(workerId) {
+        if (workerId) {
+            logger.debug("cluster", `Resetting circuit breaker for worker ${workerId}`);
+            await this.sendToWorker(workerId, {
+                type: "reset_circuit_breaker",
+            });
+        }
+        else {
+            logger.debug("cluster", "Resetting cluster circuit breaker");
+            // Reset all worker circuit breakers
+            const workers = this.getAllWorkers();
+            for (const worker of workers) {
+                await this.sendToWorker(worker.workerId, {
+                    type: "reset_circuit_breaker",
+                });
+            }
+        }
+    }
+    // ===== CLEANUP METHODS =====
+    /**
+     * Cleanup cluster resources
+     */
+    async cleanup() {
+        logger.debug("cluster", "Cleaning up cluster resources...");
+        await this.stop(true);
+        // Cleanup monitoring
+        this.healthMonitor.stopMonitoring();
+        this.metricsCollector.stopCollection();
+        this.autoScaler.stopScaling();
+        logger.debug("cluster", "Cluster cleanup completed");
+    }
+    /**
+     * Force cleanup cluster resources
+     */
+    async forceCleanup() {
+        logger.debug("cluster", "Force cleaning up cluster resources...");
+        await this.stop(false);
+        // Force cleanup
+        this.removeAllListeners();
+        logger.debug("cluster", "Force cleanup completed");
+    }
+    // ===== PERSISTENCE METHODS =====
+    /**
+     * Save cluster state to persistent storage
+     */
+    async saveClusterStateToPersistentStorage(clusterState) {
+        if (!this.persistenceManager) {
+            // Fallback: emit event for external handling
+            this.emit("cluster:state:saved", clusterState);
+            return;
+        }
+        try {
+            await this.persistenceManager.saveClusterState(clusterState);
+        }
+        catch (error) {
+            logger.warn("cluster", `Failed to save cluster state: ${error.message}`);
+            // Emit event as fallback
+            this.emit("cluster:state:saved", clusterState);
+        }
+    }
+    /**
+     * Load cluster state from persistent storage
+     */
+    async loadClusterStateFromPersistentStorage() {
+        if (!this.persistenceManager) {
+            // Fallback: emit event for external handling
+            this.emit("cluster:state:restore_requested");
+            return null;
+        }
+        try {
+            return await this.persistenceManager.loadClusterState();
+        }
+        catch (error) {
+            logger.warn("cluster", `Failed to load cluster state: ${error.message}`);
+            return null;
+        }
+    }
+    /**
+     * Apply restored cluster state
+     */
+    async applyRestoredState(savedState) {
+        try {
+            if (savedState.workers) {
+                // Restore worker configurations
+                for (const workerState of savedState.workers) {
+                    if (workerState.status === "running") {
+                        // Try to reconnect to existing worker or spawn new one
+                        await this.restoreOrSpawnWorker(workerState);
+                    }
+                }
+            }
+            if (savedState.config) {
+                // Apply saved configuration (merge with current)
+                this.config = { ...this.config, ...savedState.config };
+            }
+            if (savedState.metrics) {
+                // Restore metrics using the MetricsCollector
+                if (savedState.metrics.historicalData) {
+                    this.metricsCollector.restoreHistoricalData(savedState.metrics.historicalData);
+                }
+                if (savedState.metrics.workerHistory) {
+                    this.metricsCollector.restoreWorkerHistoricalData(savedState.metrics.workerHistory);
+                }
+                logger.debug("cluster", "✔ Restored metrics from persistent storage");
+            }
+        }
+        catch (error) {
+            logger.warn("cluster", `Failed to apply restored state: ${error.message}`);
+        }
+    }
+    /**
+     * Restore or spawn worker from saved state
+     */
+    async restoreOrSpawnWorker(workerState) {
+        try {
+            // Check if worker process still exists
+            const existingWorker = this.workerManager.getWorker(workerState.id);
+            if (!existingWorker) {
+                // Worker doesn't exist, spawn a new one
+                logger.debug("cluster", `Spawning new worker to replace ${workerState.id}`);
+                await this.addWorker();
+            }
+            else {
+                logger.debug("cluster", `Worker ${workerState.id} already exists and running`);
+            }
+        }
+        catch (error) {
+            logger.warn("cluster", `Failed to restore worker ${workerState.id}: ${error.message}`);
+        }
+    }
+    /**
+     * Get persistence manager statistics
+     */
+    getPersistenceStats() {
+        if (!this.persistenceManager) {
+            return { enabled: false, type: "none" };
+        }
+        return this.persistenceManager.getStats();
+    }
+    /**
+     * Close persistence manager
+     */
+    async closePersistenceManager() {
+        if (this.persistenceManager) {
+            await this.persistenceManager.close();
+        }
+    }
+}
+
+export { ClusterManager };
+//# sourceMappingURL=cluster-manager.js.map