velocious 1.0.684 → 1.0.686

package/src/background-jobs/pooled-runner-child.js

@@ -1,6 +1,7 @@
 // @ts-check
 
 import { randomUUID } from "node:crypto"
+import { readFileSync } from "node:fs"
 import v8 from "node:v8"
 import timeout from "awaitery/build/timeout.js"
 import runJobPayload, { BackgroundJobPerformedFailure } from "./job-runner.js"
@@ -20,6 +21,31 @@ const MEMORY_OBSERVATION_INTERVAL_MS = 10000
 /** Bound on in-flight job ids carried in a memory observation. */
 const MEMORY_OBSERVATION_JOB_ID_LIMIT = 32
 
+/**
+ * Reads this process's peak resident set size (the kernel high-water mark,
+ * `VmHWM` in `/proc/self/status`). Unlike a point-in-time RSS sample, this is
+ * monotonic for the life of the process: once a burst ratchets the working set
+ * up, `VmHWM` remembers it even after garbage collection returns the pages.
+ * That makes it the honest signal for recycling a pooled child — a child that
+ * spiked to several gigabytes during a build burst keeps its ratcheted floor
+ * and must be replaced, even though its settled RSS at job-outcome time looks
+ * modest. A read failure (no `/proc`) degrades to the current RSS sample so the
+ * observation is never blocked on it.
+ * @returns {number} Peak RSS in bytes, or the current RSS sample when unreadable.
+ */
+function readPeakRssBytes() {
+  const fallbackBytes = process.memoryUsage().rss
+  try {
+    const status = readFileSync("/proc/self/status", "utf8")
+    const match = /^VmHWM:\s+(\d+)\s+kB$/m.exec(status)
+    if (!match) return fallbackBytes
+    const kibibytes = Number(match[1])
+    return Number.isFinite(kibibytes) && kibibytes > 0 ? kibibytes * 1024 : fallbackBytes
+  } catch {
+    return fallbackBytes
+  }
+}
+
 setRunnerProcessTitle()
 
 /** @type {Promise<void> | undefined} */
@@ -162,7 +188,7 @@ function sendChildAcceptance(type, payload, observedAtMs) {
  * child runs 25 concurrent jobs) distinguishes a V8 heap-growth leak from
  * external/array-buffer (native resource) growth, and the in-flight job ids
  * tie the observation to the work that was running.
- * @returns {{activeJobIds: string[], activeJobIdsTruncatedCount: number, childInstanceId: string, childPid: number, childUptimeMs: number, heapStatistics: ReturnType<typeof v8.getHeapStatistics>, jobCount: number, memoryUsage: ReturnType<typeof process.memoryUsage>, observedAtMs: number, rssBytes: number, type: "pooled-child-memory", uptimeMs: number}} - Bounded memory observation for this child.
+ * @returns {{activeJobIds: string[], activeJobIdsTruncatedCount: number, childInstanceId: string, childPid: number, childUptimeMs: number, heapStatistics: ReturnType<typeof v8.getHeapStatistics>, jobCount: number, memoryUsage: ReturnType<typeof process.memoryUsage>, observedAtMs: number, peakRssBytes: number, rssBytes: number, type: "pooled-child-memory", uptimeMs: number}} - Bounded memory observation for this child.
  */
 function collectMemoryObservation() {
   const jobIds = [...runningJobIds]
@@ -179,6 +205,7 @@ function collectMemoryObservation() {
     jobCount: jobIds.length,
     memoryUsage,
     observedAtMs: Date.now(),
+    peakRssBytes: readPeakRssBytes(),
     rssBytes: memoryUsage.rss,
     type: "pooled-child-memory",
     uptimeMs: Math.floor(process.uptime() * 1000)
@@ -289,6 +316,7 @@ function sendOutcome({jobId, acknowledged, status, error}) {
       jobId,
       acknowledged,
       status,
+      peakRssBytes: readPeakRssBytes(),
       rssBytes: process.memoryUsage().rss,
       error: error?.message
     }, () => resolve(undefined))

package/src/background-jobs/types.js

@@ -90,6 +90,7 @@
  * @property {number} jobCount - In-flight job count.
  * @property {ReturnType<typeof import("node:process").memoryUsage>} memoryUsage - Process memory breakdown at the sample.
  * @property {number} observedAtMs - Epoch ms the child sampled.
+ * @property {number} peakRssBytes - Child-reported peak RSS (kernel `VmHWM` high-water mark) in bytes, monotonic for the child's life.
  * @property {number} rssBytes - Resident set size in bytes at the sample.
  * @property {"pooled-child-memory"} type - Discriminator.
  * @property {number} uptimeMs - Process uptime in ms.

package/src/background-jobs/worker.js

@@ -1404,7 +1404,7 @@ export default class BackgroundJobsWorker {
    */
   _handlePooledChildMessage({child, message}) {
     if (!message || typeof message !== "object") return
-    const record = /** @type {{type?: ReturnType<typeof JSON.parse>, childInstanceId?: ReturnType<typeof JSON.parse>, jobId?: ReturnType<typeof JSON.parse>, acknowledged?: ReturnType<typeof JSON.parse>, rssBytes?: ReturnType<typeof JSON.parse>, error?: ReturnType<typeof JSON.parse>}} */ (message)
+    const record = /** @type {{type?: ReturnType<typeof JSON.parse>, childInstanceId?: ReturnType<typeof JSON.parse>, jobId?: ReturnType<typeof JSON.parse>, acknowledged?: ReturnType<typeof JSON.parse>, rssBytes?: ReturnType<typeof JSON.parse>, peakRssBytes?: ReturnType<typeof JSON.parse>, error?: ReturnType<typeof JSON.parse>}} */ (message)
     const state = this.pooledChildStates.get(child)
     if (record.type === "ready") {
       if (state) {
@@ -1458,8 +1458,13 @@ export default class BackgroundJobsWorker {
     }
 
     const rssBytes = typeof record.rssBytes === "number" ? record.rssBytes : Number.POSITIVE_INFINITY
+    // The child's peak RSS (VmHWM) is monotonic and outlives the settled
+    // sample: a ratcheted burst working-set must recycle the child even when
+    // settled RSS at outcome time looks modest. A child without peak support
+    // falls back to the settled sample.
+    const peakRssBytes = typeof record.peakRssBytes === "number" ? record.peakRssBytes : rssBytes
     const runnerAgeMs = Date.now() - state.createdAtMs
-    if (!state.retiring && (state.jobsRun >= this.pooledRunnerMaxJobs || rssBytes >= this.pooledRunnerMaxRssBytes || runnerAgeMs >= this.pooledRunnerMaxLifetimeMs || this.shouldStop)) {
+    if (!state.retiring && (state.jobsRun >= this.pooledRunnerMaxJobs || peakRssBytes >= this.pooledRunnerMaxRssBytes || runnerAgeMs >= this.pooledRunnerMaxLifetimeMs || this.shouldStop)) {
       this._beginRetirePooledChild(child)
     }
     this._terminateIfDrained(child)
@@ -1524,6 +1529,7 @@ export default class BackgroundJobsWorker {
         childPid: message.childPid,
         childUptimeS: Math.round(message.childUptimeMs / 1000),
         rssMb: Math.round(message.rssBytes / (1024 * 1024)),
+        peakRssMb: Math.round((typeof message.peakRssBytes === "number" ? message.peakRssBytes : message.rssBytes) / (1024 * 1024)),
         heapUsedMb: Math.round(heap.used_heap_size / (1024 * 1024)),
         heapTotalMb: Math.round(heap.total_heap_size / (1024 * 1024)),
         heapLimitMb: Math.round(heap.heap_size_limit / (1024 * 1024)),

package/src/configuration-types.js

@@ -194,7 +194,7 @@
  * @property {number} [pooledRunnerCount] - Number of warm, reusable child runners owned by each worker. Pooled capacity is separate from inline and forked/spawned capacity. Default: `4`.
  * @property {number} [pooledRunnerConcurrency] - Number of jobs each pooled child runs concurrently on its own event loop. Total per-worker pooled capacity is `pooledRunnerCount × pooledRunnerConcurrency`. `1` (default) keeps each child serial; raise it for I/O-bound jobs so a bounded set of isolated processes handles high concurrency (like the inline lane) without one process per concurrent job. Default: `1`.
  * @property {number} [pooledRunnerMaxJobs] - Number of sequential jobs a pooled child runs before it is replaced, bounding process-level resource accumulation. Default: `100`.
- * @property {number} [pooledRunnerMaxRssBytes] - RSS bytes after an acknowledged job at which a pooled child is replaced. Default: `536870912` (512 MiB).
+ * @property {number} [pooledRunnerMaxRssBytes] - Peak RSS (the kernel `VmHWM` high-water mark reported by the child) at which a pooled child is replaced after an acknowledged job. The child's peak, not its settled RSS at job-outcome time, is the trigger, so a burst working-set that reclaims before the outcome still retires the child. Default: `536870912` (512 MiB).
  * @property {number} [pooledRunnerMaxLifetimeMs] - Age after an acknowledged job at which a pooled child is replaced. Default: `3600000` (one hour).
  * @property {Record<string, {maxConcurrent?: number, priority?: number}>} [queues] - Per-queue
  *   concurrency caps and dispatch priorities, Sidekiq-style. A job declares its queue (static