velocious 1.0.681 → 1.0.683

package/build/background-jobs/pooled-runner-child.js

@@ -1,6 +1,7 @@
 // @ts-check
 
 import { randomUUID } from "node:crypto"
+import v8 from "node:v8"
 import timeout from "awaitery/build/timeout.js"
 import runJobPayload, { BackgroundJobPerformedFailure } from "./job-runner.js"
 import { boundedPooledRunnerInflightJobIds, isPooledChildShutdownReason, isPooledChildShutdownSignal } from "./pooled-runner-shutdown.js"
@@ -14,6 +15,10 @@ const BASE_PROCESS_TITLE = "velocious background-jobs-runner"
 const SHUTDOWN_OBSERVATION_SEND_TIMEOUT_MS = 100
 /** Stable identity of this pooled child process for the life of the process. */
 const childInstanceId = randomUUID()
+/** Sampling cadence for the memory observation sampler. */
+const MEMORY_OBSERVATION_INTERVAL_MS = 10000
+/** Bound on in-flight job ids carried in a memory observation. */
+const MEMORY_OBSERVATION_JOB_ID_LIMIT = 32
 
 setRunnerProcessTitle()
 
@@ -148,6 +153,93 @@ function sendChildAcceptance(type, payload, observedAtMs) {
   }
 }
 
+/**
+ * Collects a bounded diagnostic snapshot of this child's memory state. The
+ * pooled child's stdio is ignored by the worker fork, so IPC is the only channel
+ * to the worker's log surface — this snapshot (sent periodically and on demand)
+ * is how a memory problem names itself in production. The V8 heap-stat
+ * breakdown (not a full heap snapshot, which would be far too expensive while a
+ * child runs 25 concurrent jobs) distinguishes a V8 heap-growth leak from
+ * external/array-buffer (native resource) growth, and the in-flight job ids
+ * tie the observation to the work that was running.
+ * @returns {{activeJobIds: string[], activeJobIdsTruncatedCount: number, childInstanceId: string, childPid: number, childUptimeMs: number, heapStatistics: ReturnType<typeof v8.getHeapStatistics>, jobCount: number, memoryUsage: ReturnType<typeof process.memoryUsage>, observedAtMs: number, rssBytes: number, type: "pooled-child-memory", uptimeMs: number}} - Bounded memory observation for this child.
+ */
+function collectMemoryObservation() {
+  const jobIds = [...runningJobIds]
+  const activeJobIds = jobIds.slice(0, MEMORY_OBSERVATION_JOB_ID_LIMIT)
+  const memoryUsage = process.memoryUsage()
+
+  return {
+    activeJobIds,
+    activeJobIdsTruncatedCount: Math.max(0, jobIds.length - activeJobIds.length),
+    childInstanceId,
+    childPid: process.pid,
+    childUptimeMs: Math.floor(process.uptime() * 1000),
+    heapStatistics: v8.getHeapStatistics(),
+    jobCount: jobIds.length,
+    memoryUsage,
+    observedAtMs: Date.now(),
+    rssBytes: memoryUsage.rss,
+    type: "pooled-child-memory",
+    uptimeMs: Math.floor(process.uptime() * 1000)
+  }
+}
+
+/**
+ * Sends a memory observation to the worker. Sampling is cheap (process + V8
+ * heap stats) and only runs while jobs are in flight; a closed IPC channel is
+ * terminal for this child (the disconnect handler owns shutdown), so a failed
+ * send is swallowed.
+ * @returns {void}
+ */
+function sendMemoryObservation() {
+  if (!process.send || runningJobIds.size === 0) return
+
+  try {
+    process.send(collectMemoryObservation())
+  } catch {
+    // The IPC channel is already gone; the disconnect handler owns shutdown.
+  }
+}
+
+/** @type {ReturnType<typeof setInterval> | undefined} */
+let memoryObservationTimer
+
+/**
+ * Checks whether an IPC value requests this child to send a memory observation.
+ * @param {ReturnType<typeof JSON.parse>} message - IPC message.
+ * @returns {message is {type: "memory-observation-request"}} - Whether the message requests one.
+ */
+function isMemoryObservationRequestMessage(message) {
+  return message !== null
+    && typeof message === "object"
+    && message.type === "memory-observation-request"
+}
+
+/**
+ * Starts the periodic memory observation sampler if it is not already running.
+ * @returns {void}
+ */
+function startMemoryObservationSampling() {
+  if (memoryObservationTimer || !process.send) return
+
+  memoryObservationTimer = setInterval(() => {
+    sendMemoryObservation()
+  }, MEMORY_OBSERVATION_INTERVAL_MS)
+  memoryObservationTimer.unref()
+}
+
+/**
+ * Stops the periodic memory observation sampler.
+ * @returns {void}
+ */
+function stopMemoryObservationSampling() {
+  if (!memoryObservationTimer) return
+
+  clearInterval(memoryObservationTimer)
+  memoryObservationTimer = undefined
+}
+
 /**
  * Checks whether an IPC value is a runnable pooled job message.
  * @param {ReturnType<typeof JSON.parse>} message - IPC message.
@@ -238,6 +330,10 @@ async function runJob(payload, sharedTransactionBroker) {
   } finally {
     runningJobIds.delete(payload.id)
     updateProcessTitle()
+
+    if (runningJobIds.size === 0) {
+      stopMemoryObservationSampling()
+    }
   }
 }
 
@@ -257,11 +353,17 @@ function handleMessage(message) {
     return
   }
 
+  if (isMemoryObservationRequestMessage(message)) {
+    sendMemoryObservation()
+    return
+  }
+
   if (!isJobMessage(message) || runningJobIds.has(message.payload.id)) return
 
   runningJobIds.add(message.payload.id)
   updateProcessTitle()
   sendChildAcceptance("job-received", message.payload, Date.now())
+  startMemoryObservationSampling()
   void runJob(message.payload, message.sharedTransactionBroker || {expected: false})
 }
 

package/build/background-jobs/types.js

@@ -73,6 +73,27 @@
  * @property {number | null} shutdownRequestedAtMs - Parent request timestamp when supplied over IPC.
  * @property {import("node:child_process").ChildProcess["signalCode"]} signal - Requested or observed signal when available.
  */
+/**
+ * Bounded pooled-child memory observation sent over the child IPC channel.
+ * The pooled child's stdio is ignored by the worker fork, so this observation
+ * (sent periodically while jobs run, plus on demand) is how a memory problem
+ * in a running child names itself. `heapStatistics` and `memoryUsage`
+ * distinguish V8-heap growth from external/array-buffer (native resource)
+ * growth; `activeJobIds` ties the sample to the work that was in flight.
+ * @typedef {object} PooledChildMemoryObservation
+ * @property {string[]} activeJobIds - In-flight job ids, bounded.
+ * @property {number} activeJobIdsTruncatedCount - In-flight job ids omitted by the bound.
+ * @property {string} childInstanceId - Stable identity reported by the child.
+ * @property {number} childPid - Child process id.
+ * @property {number} childUptimeMs - Child process uptime in ms.
+ * @property {ReturnType<typeof import("node:v8").getHeapStatistics>} heapStatistics - V8 heap-stat breakdown at the sample.
+ * @property {number} jobCount - In-flight job count.
+ * @property {ReturnType<typeof import("node:process").memoryUsage>} memoryUsage - Process memory breakdown at the sample.
+ * @property {number} observedAtMs - Epoch ms the child sampled.
+ * @property {number} rssBytes - Resident set size in bytes at the sample.
+ * @property {"pooled-child-memory"} type - Discriminator.
+ * @property {number} uptimeMs - Process uptime in ms.
+ */
 /**
  * @typedef {object} LocalBackgroundJobsClock
  * @property {() => number} now - Current epoch milliseconds.

package/build/background-jobs/worker.js

@@ -41,6 +41,7 @@ import { POOLED_RUNNER_INFLIGHT_JOB_ID_LIMIT, boundedPooledRunnerInflightJobIds,
  * @property {boolean} retiring - Whether this child is draining before retirement.
  * @property {boolean} [started] - Whether the child completed its startup handshake.
  * @property {boolean} [settling] - Whether failure handling already owns this child.
+ * @property {import("./types.js").PooledChildMemoryObservation} [lastMemoryObservation] - Latest memory observation received from this child.
  * @property {number} [ipcDisconnectedAtMs] - Parent observation of IPC disconnect.
  * @property {import("./types.js").PooledChildShutdownObservation} [shutdownObservation] - Child observation sent before teardown.
  * @property {import("./types.js").PooledChildShutdownReason} [shutdownReason] - Exact parent-requested shutdown reason.
@@ -132,6 +133,38 @@ function isChildShutdownObservationMessage(message) {
     && isPooledChildShutdownSignal(record.signal)
 }
 
+/**
+ * Checks whether an IPC value is a pooled child's bounded memory observation.
+ * The pooled child's stdio is ignored by the worker fork, so this observation
+ * (periodic while jobs run, plus on demand) is how a memory problem in a
+ * running child names itself. The worker logs it (its stderr reaches the prod
+ * log) and forwards it to the optional `onPooledRunnerMemoryObservation` hook.
+ * @param {ReturnType<typeof JSON.parse>} message - IPC message.
+ * @returns {message is import("./types.js").PooledChildMemoryObservation} - Whether this is a valid memory observation.
+ */
+function isPooledChildMemoryObservationMessage(message) {
+  if (!message || typeof message !== "object") return false
+  const record = /** @type {Record<string, ReturnType<typeof JSON.parse>>} */ (message)
+  const heapStatistics = /** @type {Record<string, ReturnType<typeof JSON.parse>> | undefined} */ (record.heapStatistics)
+  const memoryUsage = /** @type {Record<string, ReturnType<typeof JSON.parse>> | undefined} */ (record.memoryUsage)
+
+  return record.type === "pooled-child-memory"
+    && typeof record.childInstanceId === "string"
+    && Number.isInteger(record.childPid)
+    && typeof record.rssBytes === "number"
+    && Number.isFinite(record.rssBytes)
+    && Number.isInteger(record.jobCount)
+    && record.jobCount >= 0
+    && Array.isArray(record.activeJobIds)
+    && record.activeJobIds.every((jobId) => typeof jobId === "string")
+    && Number.isInteger(record.activeJobIdsTruncatedCount)
+    && record.activeJobIdsTruncatedCount >= 0
+    && typeof record.observedAtMs === "number"
+    && Number.isFinite(record.observedAtMs)
+    && heapStatistics !== undefined && typeof heapStatistics === "object"
+    && memoryUsage !== undefined && typeof memoryUsage === "object"
+}
+
 /**
  * Normalizes a candidate pooled-runner resource limit.
  * @param {number | undefined} value - Candidate positive number.
@@ -166,8 +199,9 @@ export default class BackgroundJobsWorker {
    * @param {() => void | Promise<void>} [args.onStopped] - Lifecycle hook invoked after the worker finishes stopping.
    * @param {() => void} [args.onGenerationAccepted] - Explicit generation-acceptance observation hook.
    * @param {() => void} [args.onRetireMessage] - Explicit retire-message observation hook.
+   * @param {(observation: import("./types.js").PooledChildMemoryObservation) => void | Promise<void>} [args.onPooledRunnerMemoryObservation] - Explicit pooled-child memory observation hook. Every validated observation (periodic while a child has in-flight jobs, plus on demand) is forwarded here, in addition to the worker's compact stderr log line, so an application can route memory diagnostics (e.g. to a bug reporter) without parsing logs.
    */
-  constructor({configuration, host, port, generationId, workerInstanceId, maxConcurrentForkedJobs, maxConcurrentInlineJobs, pooledRunnerCount, pooledRunnerConcurrency, pooledRunnerMaxJobs, pooledRunnerMaxRssBytes, pooledRunnerMaxLifetimeMs, forkedChildSigkillGraceMs, heartbeatIntervalMs, generationHandshakeTimeoutMs = DEFAULT_GENERATION_HANDSHAKE_TIMEOUT_MS, reconnectDelayMs = 1000, jobTimeoutMs, closeDatabaseConnectionsOnStop = true, onStopped, onGenerationAccepted, onRetireMessage} = {}) {
+  constructor({configuration, host, port, generationId, workerInstanceId, maxConcurrentForkedJobs, maxConcurrentInlineJobs, pooledRunnerCount, pooledRunnerConcurrency, pooledRunnerMaxJobs, pooledRunnerMaxRssBytes, pooledRunnerMaxLifetimeMs, forkedChildSigkillGraceMs, heartbeatIntervalMs, generationHandshakeTimeoutMs = DEFAULT_GENERATION_HANDSHAKE_TIMEOUT_MS, reconnectDelayMs = 1000, jobTimeoutMs, closeDatabaseConnectionsOnStop = true, onStopped, onGenerationAccepted, onRetireMessage, onPooledRunnerMemoryObservation} = {}) {
     /**
      * Narrows the runtime value to the documented type.
      * @type {Promise<import("../configuration.js").default>} */
@@ -186,6 +220,7 @@ export default class BackgroundJobsWorker {
     this.onStopped = onStopped
     this.onGenerationAccepted = onGenerationAccepted
     this.onRetireMessage = onRetireMessage
+    this.onPooledRunnerMemoryObservation = onPooledRunnerMemoryObservation
     /**
      * Constructor override for the inline-job concurrency cap. When unset
      * the cap is read from `configuration.getBackgroundJobsConfig()` in
@@ -322,6 +357,28 @@ export default class BackgroundJobsWorker {
     // Monotonic dispatch counter for round-robin child selection: each dispatch stamps
     // the chosen child, and selection prefers the child dispatched least recently.
     this._pooledDispatchSeq = 0
+    // Waiters blocked in _runPooledJob because the pool is at its hard cap: a job may
+    // not spawn a child while total live children (working + draining) is at the cap.
+    /** @type {Set<() => void>} */
+    this._pooledSlotWaiters = new Set()
+    /** @type {ReturnType<typeof setInterval> | undefined} - Safety poll that re-checks the slot condition. */
+    this._pooledSlotWaitTimer = undefined
+  }
+
+  /** Starts the slot-waiter safety poll if it is not already running. */
+  _startPooledSlotWaitPoll() {
+    if (this._pooledSlotWaitTimer) return
+
+    this._pooledSlotWaitTimer = setInterval(() => this._wakePooledSlotWaiters(), 50)
+    this._pooledSlotWaitTimer.unref()
+  }
+
+  /** Stops the slot-waiter safety poll once no waiter is registered. */
+  _stopPooledSlotWaitPollIfIdle() {
+    if (this._pooledSlotWaiters.size > 0 || !this._pooledSlotWaitTimer) return
+
+    clearInterval(this._pooledSlotWaitTimer)
+    this._pooledSlotWaitTimer = undefined
   }
 
   /**
@@ -1079,27 +1136,72 @@ export default class BackgroundJobsWorker {
     }
   }
 
+  /**
+   * Hard cap on total live pooled children (working + draining): children the
+   * pool may still spawn. Counting the whole live set — draining children
+   * included — is what bounds pool memory: a draining child still holds its
+   * RSS until its last in-flight job finishes, so it must occupy a cap slot.
+   * @returns {number} - Number of children the pool may still spawn.
+   */
+  _spawnablePooledChildren() {
+    return Math.max(0, this.pooledRunnerCount - this.pooledChildren.size)
+  }
+
+  /**
+   * Resolves once a non-retiring pooled child has a free concurrency slot or the
+   * pool may spawn a new one. Pooled jobs admitted while the pool is at its
+   * hard cap wait here instead of spawning an over-capacity child; the wake
+   * points are the only capacity-freeing transitions (a job outcome and a child
+   * exit), so no polling is needed.
+   * @returns {Promise<void>} - Resolves when a slot is available.
+   */
+  _waitPooledSlot() {
+    this._startPooledSlotWaitPoll()
+
+    return new Promise((resolve) => {
+      const waiter = () => {
+        this._pooledSlotWaiters.delete(waiter)
+        this._stopPooledSlotWaitPollIfIdle()
+        resolve()
+      }
+
+      this._pooledSlotWaiters.add(waiter)
+    })
+  }
+
+  /**
+   * Resolves every registered waiter; waiters re-check the slot condition
+   * themselves and only proceed when it holds.
+   * @returns {void}
+   */
+  _wakePooledSlotWaiters() {
+    if (this._pooledSlotWaiters.size === 0) return
+
+    for (const resolve of [...this._pooledSlotWaiters]) resolve()
+  }
+
   /**
    * Free pooled slots across the pool: open slots in non-retiring children plus
-   * the slots we could add by spawning more children up to `pooledRunnerCount`.
-   * Retiring children (draining before replacement) never contribute capacity.
+   * the slots we could add by spawning more children up to the hard cap on total
+   * live children. Retiring children (draining before replacement) never
+   * contribute capacity, and they count against the cap: while one is still
+   * draining, no replacement is advertised (or spawned) — the pool advertises
+   * exactly what it can serve instead of phantom capacity.
    * @returns {number} - Number of pooled jobs the worker can accept right now.
    */
   _availablePooledSlots() {
     let openInExisting = 0
-    let nonRetiringChildren = 0
     let queuedReservations = 0
 
     for (const child of this.pooledChildren) {
       const state = this.pooledChildStates.get(child)
       if (!state || state.retiring) continue
-      nonRetiringChildren += 1
       openInExisting += this.pooledRunnerConcurrency - state.inflight.size
     }
 
     for (const queue of this.pooledJobQueues.values()) queuedReservations += queue.length
 
-    const spawnableChildren = Math.max(0, this.pooledRunnerCount - nonRetiringChildren)
+    const spawnableChildren = Math.max(0, this.pooledRunnerCount - this.pooledChildren.size)
 
     return Math.max(0, openInExisting + spawnableChildren * this.pooledRunnerConcurrency - queuedReservations)
   }
@@ -1109,11 +1211,32 @@ export default class BackgroundJobsWorker {
    * new child when every non-retiring child is full and the pool is below
    * `pooledRunnerCount`. Each child runs up to `pooledRunnerConcurrency` jobs at
    * once on its own event loop.
+   *
+   * When the pool is already at its hard cap (total live children, draining
+   * included), the job waits for a slot instead of spawning: that is what keeps
+   * the live-child count — and therefore the pool's total RSS — bounded. The
+   * wait resolves on the only two capacity-freeing transitions (a job outcome,
+   * a child exit); a safety poll covers anything missed.
    * @param {import("./types.js").BackgroundJobPayload & {id: string}} payload - Job payload.
    * @returns {Promise<void>} - Resolves after the durable report.
    */
-  _runPooledJob(payload) {
-    const child = this._selectPooledChild() || this._createPooledChild()
+  async _runPooledJob(payload) {
+    // At the hard cap (no free slot, no spawnable child) the job waits for a
+    // slot instead of spawning an over-capacity child — that is what bounds
+    // the live-child count and the pool's total RSS.
+    let child = this._selectPooledChild()
+    while (!child) {
+      if (this._spawnablePooledChildren() === 0) {
+        // Shutdown: main no longer dispatches and no slot will ever free —
+        // stop waiting so the tracked job can settle and the drain completes.
+        if (this.shouldStop) return
+        await this._waitPooledSlot()
+        child = this._selectPooledChild()
+        continue
+      }
+      child = this._selectPooledChild() || this._createPooledChild()
+    }
+
     const state = this.pooledChildStates.get(child)
     if (!state) throw new Error("Pooled runner state missing")
 
@@ -1232,12 +1355,16 @@ export default class BackgroundJobsWorker {
   }
 
   /**
-   * Creates a reusable pooled child.
-   * @returns {import("node:child_process").ChildProcess} - New pooled child.
+   * Creates a reusable pooled child, enforcing the hard cap on total live
+   * children (working + draining). Returns undefined when the cap is already
+   * met — the only way a new child may exist is a slot being open, so the
+   * caller re-checks and waits again.
+   * @returns {import("node:child_process").ChildProcess | undefined} - The new child, or undefined when the pool is at its cap.
    */
   _createPooledChild() {
     const configuration = this.configuration
     if (!configuration) throw new Error("Background jobs worker configuration not initialized")
+    if (this.pooledChildren.size >= this.pooledRunnerCount) return undefined
     const child = fork(POOLED_RUNNER_ENTRY_PATH, [], {
       cwd: configuration.getDirectory(), execArgv: [], stdio: ["ignore", "ignore", "ignore", "ipc"],
       env: Object.assign({}, process.env, this._childBackgroundJobsEnvironment())
@@ -1301,6 +1428,10 @@ export default class BackgroundJobsWorker {
       this._reportChildAccepted(message)
       return
     }
+    if (isPooledChildMemoryObservationMessage(message)) {
+      this._handlePooledChildMemoryObservation({child, message})
+      return
+    }
     if (record.type !== "job-outcome" || !state || state.settling || typeof record.jobId !== "string") return
     state.started = true
     const entry = state.inflight.get(record.jobId)
@@ -1332,6 +1463,9 @@ export default class BackgroundJobsWorker {
       this._beginRetirePooledChild(child)
     }
     this._terminateIfDrained(child)
+    // A job outcome frees a concurrency slot and may drain a retiring child —
+    // the only two transitions that free capacity for waiters at the hard cap.
+    this._wakePooledSlotWaiters()
   }
 
   /**
@@ -1362,11 +1496,85 @@ export default class BackgroundJobsWorker {
   }
 
   /**
-   * Marks a pooled child for retirement and eagerly spawns a single replacement
-   * (1-for-1) so its capacity is restored immediately without waiting for it to
-   * finish draining. The retiring child stops receiving new jobs and is
-   * terminated only once its in-flight set drains, so a long-running job (e.g. a
-   * build) is never cut off.
+   * Handles a pooled child's bounded memory observation. The pooled child's
+   * stdio is ignored by the worker fork, so this IPC observation is how a
+   * memory problem in a running child names itself. The worker (a) records the
+   * latest observation on the child's state for later correlation, (b) logs one
+   * compact line to its own stderr (which reaches the prod log, unlike the
+   * child's ignored stdio), and (c) forwards the full observation to the
+   * optional `onPooledRunnerMemoryObservation` hook so an application can route
+   * it (e.g. to a bug reporter) without parsing logs. The heap-stat breakdown
+   * distinguishes V8-heap growth from external/array-buffer (native) growth. A
+   * hook failure is swallowed — diagnostics must never take down the worker or
+   * fail the jobs running on that child.
+   * @param {object} args - Message details.
+   * @param {import("node:child_process").ChildProcess} args.child - Pooled child.
+   * @param {import("./types.js").PooledChildMemoryObservation} args.message - Validated memory observation.
+   * @returns {void}
+   */
+  _handlePooledChildMemoryObservation({child, message}) {
+    const state = this.pooledChildStates.get(child)
+    if (state) state.lastMemoryObservation = message
+
+    const heap = message.heapStatistics
+    console.error(
+      JSON.stringify({
+        event: "pooled-child-memory",
+        childInstanceId: state?.childInstanceId ?? message.childInstanceId,
+        childPid: message.childPid,
+        childUptimeS: Math.round(message.childUptimeMs / 1000),
+        rssMb: Math.round(message.rssBytes / (1024 * 1024)),
+        heapUsedMb: Math.round(heap.used_heap_size / (1024 * 1024)),
+        heapTotalMb: Math.round(heap.total_heap_size / (1024 * 1024)),
+        heapLimitMb: Math.round(heap.heap_size_limit / (1024 * 1024)),
+        externalMb: Math.round(message.memoryUsage.external / (1024 * 1024)),
+        arrayBuffersMb: Math.round(message.memoryUsage.arrayBuffers / (1024 * 1024)),
+        jobs: message.jobCount,
+        activeJobIds: message.activeJobIds
+      })
+    )
+
+    if (this.onPooledRunnerMemoryObservation) {
+      try {
+        const result = this.onPooledRunnerMemoryObservation(message)
+        if (result && typeof result.catch === "function") result.catch((error) => {
+          console.error("Pooled runner memory observation hook failed:", error)
+        })
+      } catch (error) {
+        console.error("Pooled runner memory observation hook failed:", error)
+      }
+    }
+  }
+
+  /**
+   * Requests an immediate memory observation from one pooled child. The child
+   * replies over IPC with its current snapshot (the same shape as the periodic
+   * sampler), which the worker records, logs, and forwards to the
+   * `onPooledRunnerMemoryObservation` hook. Use this to pull a snapshot on
+   * suspicion (e.g. after an OOM report) without waiting for the next periodic
+   * sample. A no-op when the child is gone or its IPC channel is closed.
+   * @param {import("node:child_process").ChildProcess} child - Pooled child to sample.
+   * @returns {void}
+   */
+  requestPooledChildMemoryObservation(child) {
+    if (!child.connected) return
+
+    try {
+      child.send({type: "memory-observation-request"})
+    } catch {
+      // The IPC channel is already gone; the disconnect/exit handler owns teardown.
+    }
+  }
+
+  /**
+   * Marks a pooled child for retirement and — when the pool is below its hard
+   * cap — eagerly spawns a single replacement (1-for-1) so its capacity is
+   * restored immediately without waiting for it to finish draining. The
+   * replacement spawn is gated by the cap (the retiring child still counts as
+   * live until it exits), so a full pool simply defers the replacement to the
+   * retiring child's drain instead of spawning over capacity. The retiring
+   * child stops receiving new jobs and is terminated only once its in-flight
+   * set drains, so a long-running job (e.g. a build) is never cut off.
    * @param {import("node:child_process").ChildProcess} child - Child to retire.
    * @returns {void}
    */
@@ -1376,7 +1584,9 @@ export default class BackgroundJobsWorker {
 
     state.retiring = true
     // Best-effort pre-warm: skip when stopping (no new work) or before the
-    // worker is initialized (no configuration to fork a child from).
+    // worker is initialized (no configuration to fork a child from). The cap
+    // inside _createPooledChild refuses the spawn while the pool is full, in
+    // which case the replacement is deferred to the drain path.
     if (!this.shouldStop && this.configuration) this._createPooledChild()
   }
 
@@ -1394,6 +1604,9 @@ export default class BackgroundJobsWorker {
 
   /**
    * Retires a drained pooled child (removes it from tracking, then SIGTERMs it).
+   * Because the hard cap counts live children, the exit of this child frees a
+   * slot: any deferred replacement (the pool was full when the child retired)
+   * is spawned now, and capacity is re-advertised so main can dispatch into it.
    * @param {import("node:child_process").ChildProcess} child - Child process to retire.
    * @returns {void}
    */
@@ -1503,6 +1716,9 @@ export default class BackgroundJobsWorker {
     }
     this.pooledChildren.delete(child)
     this.inflightProcessChildren.delete(child)
+    // Child exit frees a hard-cap slot even while its in-flight set is still
+    // being reported — wake waiters now; their reports settle independently.
+    this._wakePooledSlotWaiters()
 
     const entries = state ? [...state.inflight.values()] : []
     const runnerFailure = state