react-on-rails-pro-node-renderer 17.0.0-rc.2 → 17.0.0-rc.3

package/lib/master.js

@@ -66,6 +66,8 @@ const errorReporter = __importStar(require("./shared/errorReporter.js"));
 const licenseValidator_js_1 = require("./shared/licenseValidator.js");
 const runRscPeerCompatibilityCheck_js_1 = require("./shared/runRscPeerCompatibilityCheck.js");
 const workerMessages_js_1 = require("./shared/workerMessages.js");
+const utils_js_1 = require("./shared/utils.js");
+const shutdownHooks_js_1 = require("./worker/shutdownHooks.js");
 const MILLISECONDS_IN_MINUTE = 60000;
 // How often to scan for orphaned upload directories.
 const ORPHAN_CLEANUP_INTERVAL_MS = 5 * MILLISECONDS_IN_MINUTE;
@@ -73,6 +75,26 @@ const ORPHAN_CLEANUP_INTERVAL_MS = 5 * MILLISECONDS_IN_MINUTE;
 // Set well above the longest realistic upload duration so that large bundle
 // uploads in progress are never deleted by the cleanup timer.
 const ORPHAN_AGE_THRESHOLD_MS = 30 * MILLISECONDS_IN_MINUTE;
+// Give workers a short window to receive SHUTDOWN_WORKER_MESSAGE and enter
+// their own graceful shutdown path before the master falls back to disconnect().
+const MASTER_WORKER_SHUTDOWN_MESSAGE_GRACE_MS = 1000;
+// Hard deadline for the master to exit after it begins draining workers. It
+// starts before workers receive SHUTDOWN_WORKER_MESSAGE, so it must include the
+// message grace window plus the worker hook budget. If a worker is stuck after
+// that point, this guarantees the master still exits.
+const MASTER_SHUTDOWN_TIMEOUT_MS = MASTER_WORKER_SHUTDOWN_MESSAGE_GRACE_MS + shutdownHooks_js_1.WORKER_SHUTDOWN_HOOKS_TIMEOUT_MS;
+// When a worker's event loop is blocked (synchronous render) it can process
+// neither SHUTDOWN_WORKER_MESSAGE nor a disconnect, so only SIGKILL can stop
+// it — and the master must send that SIGKILL while it is still alive itself.
+// Process supervisors kill the master well before MASTER_SHUTDOWN_TIMEOUT_MS
+// (Foreman's default SIGTERM-to-SIGKILL window is 5s), so survivors are
+// force-killed early: after the message grace window has given draining a
+// chance, but safely inside the supervisor's window.
+const SHUTDOWN_WORKER_FORCE_KILL_TIMEOUT_MS = 2000;
+const SIGNAL_EXIT_CODES = {
+    SIGINT: 130,
+    SIGTERM: 143,
+};
 function masterRun(runningConfig) {
     // This is memoized after the wrapper path runs, but still protects direct `./master` entrypoint users.
     (0, runRscPeerCompatibilityCheck_js_1.runRscPeerCompatibilityCheck)({ proVersion: packageJson_js_1.default.version });
@@ -124,34 +146,130 @@ function masterRun(runningConfig) {
         })();
     }, ORPHAN_CLEANUP_INTERVAL_MS);
     let isAbortingForStartupFailure = false;
+    let hasReportedStartupFailure = false;
     let fatalStartupFailure = null;
-    let hasInitiatedShutdown = false;
+    // Set as soon as any shutdown path (external signal or startup-failure abort)
+    // begins. Read by the `cluster.on('exit')` handler to suppress re-forking
+    // workers that exit because we are intentionally tearing the cluster down.
+    let isShuttingDown = false;
+    // Drain every live worker, then exit with `exitCode`. Idempotent: only the
+    // first call performs the disconnect + exit; later calls are no-ops so that
+    // an external signal and a near-simultaneous startup-failure abort can't both
+    // disconnect or schedule duplicate exits.
+    //
+    // cluster.disconnect() is async, but its callback only proves workers have
+    // disconnected from IPC. Workers may still be draining in-flight requests, so
+    // external signal shutdown also waits for their exit events before the master
+    // exits. A hard-deadline timer guarantees the master still exits if a worker
+    // is stuck.
+    const currentWorkers = () => Object.values(cluster_1.default.workers ?? {}).filter((worker) => Boolean(worker));
+    const forceKillSurvivingWorkers = (workers) => {
+        workers.forEach((worker) => {
+            const workerProcess = worker.process;
+            // isDead() only: ChildProcess.killed means a signal was sent, not that
+            // the process died — e.g. a blocked worker surviving destroy()'s SIGTERM.
+            if (worker.isDead())
+                return;
+            try {
+                workerProcess.kill('SIGKILL');
+            }
+            catch (err) {
+                log_js_1.default.warn({ msg: 'Error sending SIGKILL to worker during node renderer master shutdown', err });
+            }
+        });
+    };
+    const sendGracefulShutdownMessageToWorkers = (workers) => {
+        workers.forEach((worker) => {
+            try {
+                worker.send(utils_js_1.SHUTDOWN_WORKER_MESSAGE);
+            }
+            catch (err) {
+                log_js_1.default.warn({ msg: 'Error sending graceful shutdown message to worker', err });
+            }
+        });
+    };
+    const waitForWorkerExits = (workers, onAllWorkersExited) => {
+        let remainingWorkers = 0;
+        let hasFinished = false;
+        const finishIfComplete = () => {
+            if (hasFinished || remainingWorkers > 0)
+                return;
+            hasFinished = true;
+            onAllWorkersExited();
+        };
+        workers.forEach((worker) => {
+            if (worker.isDead())
+                return;
+            remainingWorkers += 1;
+            const markWorkerExited = () => {
+                remainingWorkers -= 1;
+                finishIfComplete();
+            };
+            worker.once('exit', markWorkerExited);
+            if (worker.isDead()) {
+                worker.off('exit', markWorkerExited);
+                markWorkerExited();
+            }
+        });
+        finishIfComplete();
+    };
+    const shutdownGracefully = (exitCode, notifyWorkersBeforeDisconnect = false) => {
+        if (isShuttingDown)
+            return;
+        isShuttingDown = true;
+        const workersAtShutdown = currentWorkers();
+        const shutdownTimer = setTimeout(() => {
+            forceKillSurvivingWorkers(workersAtShutdown);
+            process.exit(exitCode);
+        }, MASTER_SHUTDOWN_TIMEOUT_MS);
+        if (typeof shutdownTimer.unref === 'function')
+            shutdownTimer.unref();
+        // Early force-kill of workers that cannot drain (blocked event loop).
+        // Their SIGKILL-induced exits complete the disconnect, which lets the
+        // normal waitForWorkerExits path below finish the shutdown promptly.
+        const forceKillTimer = setTimeout(() => {
+            forceKillSurvivingWorkers(workersAtShutdown);
+        }, SHUTDOWN_WORKER_FORCE_KILL_TIMEOUT_MS);
+        if (typeof forceKillTimer.unref === 'function')
+            forceKillTimer.unref();
+        const finishShutdown = () => {
+            clearTimeout(shutdownTimer);
+            clearTimeout(forceKillTimer);
+            process.exit(exitCode);
+        };
+        const disconnectCluster = () => {
+            cluster_1.default.disconnect(() => {
+                if (notifyWorkersBeforeDisconnect) {
+                    waitForWorkerExits(workersAtShutdown, finishShutdown);
+                    return;
+                }
+                finishShutdown();
+            });
+        };
+        if (notifyWorkersBeforeDisconnect) {
+            sendGracefulShutdownMessageToWorkers(workersAtShutdown);
+            setTimeout(disconnectCluster, MASTER_WORKER_SHUTDOWN_MESSAGE_GRACE_MS);
+            return;
+        }
+        disconnectCluster();
+    };
     const abortForStartupFailure = () => {
         if (!(isAbortingForStartupFailure && fatalStartupFailure))
             return false;
-        if (!hasInitiatedShutdown) {
-            hasInitiatedShutdown = true;
-            // Note: the exiting worker may differ from the one that sent the
-            // failure message if multiple workers exit in rapid succession.
-            // We always report the first failure received.
-            const { failure, workerId: failedWorkerId } = fatalStartupFailure;
-            const msg = failure.code === 'EADDRINUSE'
-                ? `Node renderer startup failed: ${failure.host}:${failure.port} is already in use`
-                : `Node renderer startup failed in worker ${failedWorkerId}: ${failure.message}`;
-            errorReporter.message(msg);
-            // Disconnect all live workers so they release their ports before the
-            // master exits. cluster.disconnect() is async — the callback fires
-            // once every worker has disconnected. A hard-deadline timer guarantees
-            // the master still exits if a worker is stuck (leaked handle, blocking
-            // syscall, etc.), following the same pattern as restartWorkers.ts.
-            const MASTER_SHUTDOWN_TIMEOUT_MS = 5000;
-            const shutdownTimer = setTimeout(() => process.exit(1), MASTER_SHUTDOWN_TIMEOUT_MS);
-            if (typeof shutdownTimer.unref === 'function')
-                shutdownTimer.unref();
-            cluster_1.default.disconnect(() => {
-                clearTimeout(shutdownTimer);
-                process.exit(1);
-            });
+        if (hasReportedStartupFailure)
+            return true;
+        // Note: the exiting worker may differ from the one that sent the
+        // failure message if multiple workers exit in rapid succession.
+        // We always report the first failure received.
+        const { failure, workerId: failedWorkerId } = fatalStartupFailure;
+        const msg = failure.code === 'EADDRINUSE'
+            ? `Node renderer startup failed: ${failure.host}:${failure.port} is already in use`
+            : `Node renderer startup failed in worker ${failedWorkerId}: ${failure.message}`;
+        errorReporter.message(msg);
+        hasReportedStartupFailure = true;
+        if (!isShuttingDown) {
+            // Exit non-zero so supervisors (Foreman/systemd/k8s) see the failed start.
+            shutdownGracefully(1);
         }
         return true;
     };
@@ -163,17 +281,23 @@ function masterRun(runningConfig) {
         isAbortingForStartupFailure = true;
         fatalStartupFailure = { workerId: worker.id, failure: message };
     });
-    for (let i = 0; i < workersCount; i += 1) {
-        cluster_1.default.fork();
-    }
     // Listen for dying workers:
     cluster_1.default.on('exit', (worker) => {
         // Once a startup failure has been detected, abort regardless of whether
         // this particular exit was from the failing worker, a scheduled restart,
-        // or an unrelated crash. Don't fork any more workers.
+        // or an unrelated crash. Don't fork any more workers. If an external signal
+        // already started shutdown, still report the recorded failure once while
+        // preserving the signal-style exit code and avoiding duplicate disconnects.
         if (abortForStartupFailure()) {
             return;
         }
+        // If we are intentionally tearing the cluster down (external SIGTERM/SIGINT
+        // or a startup-failure abort), workers are expected to exit — never re-fork
+        // them, or we would resurrect the processes we are trying to drain and leave
+        // them orphaned after the master exits.
+        if (isShuttingDown) {
+            return;
+        }
         if (worker.isScheduledRestart) {
             log_js_1.default.info('Restarting worker #%d on schedule', worker.id);
             cluster_1.default.fork();
@@ -182,7 +306,9 @@ function masterRun(runningConfig) {
         // Give in-flight startup-failure IPC messages one event-loop turn to be
         // processed before classifying this as an ordinary runtime crash.
         setImmediate(() => {
-            if (abortForStartupFailure())
+            // A startup failure may have arrived during this event-loop turn; report
+            // it before any signal-shutdown bail-out suppresses crash classification.
+            if (abortForStartupFailure() || isShuttingDown)
                 return;
             // TODO: Track last rendering request per worker.id
             // TODO: Consider blocking a given rendering request if it kills a worker more than X times
@@ -191,6 +317,20 @@ function masterRun(runningConfig) {
             cluster_1.default.fork();
         });
     });
+    // Drain workers on external shutdown signals. Process managers (Foreman,
+    // systemd, Kubernetes, Docker) send SIGTERM (and SIGINT on Ctrl-C in a
+    // foreground terminal) to the master. Without these handlers the master is
+    // killed immediately and its forked workers are left orphaned, holding their
+    // ports.
+    const handleShutdownSignal = (signal) => {
+        log_js_1.default.info('Received %s, draining workers before shutting down the node renderer master', signal);
+        shutdownGracefully(SIGNAL_EXIT_CODES[signal] ?? 0, true);
+    };
+    process.on('SIGTERM', () => handleShutdownSignal('SIGTERM'));
+    process.on('SIGINT', () => handleShutdownSignal('SIGINT'));
+    for (let i = 0; i < workersCount; i += 1) {
+        cluster_1.default.fork();
+    }
     // Schedule regular restarts of workers
     if (allWorkersRestartInterval && delayBetweenIndividualWorkerRestarts) {
         log_js_1.default.info('Scheduled workers restarts every %d minutes (%d minutes btw each)', allWorkersRestartInterval, delayBetweenIndividualWorkerRestarts);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "react-on-rails-pro-node-renderer",
-  "version": "17.0.0-rc.2",
+  "version": "17.0.0-rc.3",
   "protocolVersion": "2.0.0",
   "description": "React on Rails Pro Node Renderer for server-side rendering",
   "main": "lib/ReactOnRailsProNodeRenderer.js",
@@ -74,7 +74,7 @@
     "sentry-testkit": "^5.0.6",
     "touch": "^3.1.0",
     "typescript": "^5.4.3",
-    "react-on-rails": "17.0.0-rc.2"
+    "react-on-rails": "17.0.0-rc.3"
   },
   "peerDependencies": {
     "@honeybadger-io/js": ">=4.0.0",