velocious 1.0.725 → 1.0.726

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. package/README.md +1 -1
  2. package/build/background-jobs/adapter.js +16 -0
  3. package/build/background-jobs/job-record.js +3 -1
  4. package/build/background-jobs/main.js +62 -2
  5. package/build/background-jobs/owner-liveness.js +27 -5
  6. package/build/background-jobs/store.js +74 -0
  7. package/build/background-jobs/types.js +3 -0
  8. package/build/configuration.js +36 -1
  9. package/build/src/background-jobs/adapter.d.ts +23 -0
  10. package/build/src/background-jobs/adapter.d.ts.map +1 -1
  11. package/build/src/background-jobs/adapter.js +15 -1
  12. package/build/src/background-jobs/job-record.d.ts +3 -1
  13. package/build/src/background-jobs/job-record.d.ts.map +1 -1
  14. package/build/src/background-jobs/job-record.js +4 -2
  15. package/build/src/background-jobs/main.d.ts +33 -0
  16. package/build/src/background-jobs/main.d.ts.map +1 -1
  17. package/build/src/background-jobs/main.js +59 -3
  18. package/build/src/background-jobs/owner-liveness.d.ts +25 -3
  19. package/build/src/background-jobs/owner-liveness.d.ts.map +1 -1
  20. package/build/src/background-jobs/owner-liveness.js +26 -7
  21. package/build/src/background-jobs/store.d.ts +39 -0
  22. package/build/src/background-jobs/store.d.ts.map +1 -1
  23. package/build/src/background-jobs/store.js +66 -1
  24. package/build/src/background-jobs/types.d.ts +6 -0
  25. package/build/src/background-jobs/types.d.ts.map +1 -1
  26. package/build/src/background-jobs/types.js +4 -1
  27. package/build/src/configuration.d.ts +22 -0
  28. package/build/src/configuration.d.ts.map +1 -1
  29. package/build/src/configuration.js +30 -2
  30. package/package.json +1 -1
  31. package/src/background-jobs/adapter.js +16 -0
  32. package/src/background-jobs/job-record.js +3 -1
  33. package/src/background-jobs/main.js +62 -2
  34. package/src/background-jobs/owner-liveness.js +27 -5
  35. package/src/background-jobs/store.js +74 -0
  36. package/src/background-jobs/types.js +3 -0
  37. package/src/configuration.js +36 -1
package/package.json CHANGED
@@ -3,7 +3,7 @@
3
3
  "velocious": "build/bin/velocious.js"
4
4
  },
5
5
  "name": "velocious",
6
- "version": "1.0.725",
6
+ "version": "1.0.726",
7
7
  "main": "build/index.js",
8
8
  "types": "build/index.d.ts",
9
9
  "files": [
@@ -190,6 +190,22 @@ export default class BackgroundJobsAdapter {
190
190
  */
191
191
  async markOrphanedHandoffs(_args) { return [] }
192
192
 
193
+ /**
194
+ * Evaluates recorded runner-process liveness for exact active handoffs.
195
+ * Adapters without durable runner identity may return none.
196
+ * @param {{hostname?: string, liveChildren?: import("./owner-liveness.js").BackgroundJobLiveChildIdentity[], probe?: (pid: number) => boolean}} [_args] - Probe controls.
197
+ * @returns {Promise<import("./types.js").BackgroundJobHandoffOwnershipVerdict[]>} - One verdict per exact active handoff.
198
+ */
199
+ async snapshotHandedOffJobOwnership(_args) { return [] }
200
+
201
+ /**
202
+ * Settles exact active handoffs whose recorded runner process is provably
203
+ * dead. Adapters without durable runner identity settle none.
204
+ * @param {{error?: ReturnType<typeof JSON.parse>, hostname?: string, liveChildren?: import("./owner-liveness.js").BackgroundJobLiveChildIdentity[], probe?: (pid: number) => boolean}} [_args] - Orphan reason and probe controls.
205
+ * @returns {Promise<import("./types.js").BackgroundJobRow[]>} - Accepted orphan transitions.
206
+ */
207
+ async markOrphanedHandoffsByDeadOwner(_args) { return [] }
208
+
193
209
  /**
194
210
  * Marks a handed-off job failed or retryable.
195
211
  * @param {{jobId: string, error: ReturnType<typeof JSON.parse>, handoffId?: string, workerId?: string, handedOffAtMs?: number}} _args - Failure report.
@@ -21,14 +21,16 @@ export default class BackgroundJobRecord extends DatabaseRecord {
21
21
  * registry, which this helper deliberately does not invent.
22
22
  * @param {object} [args] - Options.
23
23
  * @param {string} [args.hostname] - Hostname to compare against, defaults to this process's hostname.
24
+ * @param {import("./owner-liveness.js").BackgroundJobLiveChildIdentity[]} [args.liveChildren] - Runner identities observed as live, for the pid-recycling check.
24
25
  * @param {(pid: number) => boolean} [args.probe] - Process probe, injectable for tests.
25
26
  * @returns {import("./owner-liveness.js").BackgroundJobOwnerLivenessResult} - Liveness verdict.
26
27
  */
27
- ownerIsAlive({hostname, probe} = {}) {
28
+ ownerIsAlive({hostname, liveChildren, probe} = {}) {
28
29
  return evaluateBackgroundJobOwnerLiveness({
29
30
  childInstanceId: /** @type {string | null} */ (this.readAttribute("childInstanceId")),
30
31
  childPid: /** @type {number | null} */ (this.readAttribute("childPid")),
31
32
  currentHostname: hostname,
33
+ liveChildren,
32
34
  probe,
33
35
  runnerHostname: /** @type {string | null} */ (this.readAttribute("runnerHostname"))
34
36
  })
@@ -190,6 +190,13 @@ export default class BackgroundJobsMain {
190
190
  * @type {Set<string>}
191
191
  */
192
192
  this.reconnectedWorkerIds = new Set()
193
+ /**
194
+ * Pooled-runner identities this main observed accepting jobs, keyed by
195
+ * child instance id. A dead-owner sweep uses them to disprove a recorded
196
+ * pid that the OS recycled onto a newer runner before orphaning its rows.
197
+ * @type {Map<string, import("./owner-liveness.js").BackgroundJobLiveChildIdentity>}
198
+ */
199
+ this.liveChildIdentities = new Map()
193
200
  /** @type {import("./types.js").BackgroundJobHandoffSnapshot[]} */
194
201
  this.startupHandoffSnapshot = []
195
202
  /** @type {Promise<void>[]} */
@@ -637,6 +644,7 @@ export default class BackgroundJobsMain {
637
644
  this.lifecycleState = "active"
638
645
  this._creditReadyWorkers()
639
646
  await this._settleForeignGenerationHandoffs()
647
+ await this._settleDeadOwnerHandoffs()
640
648
  this.logger.info(() => ["Background jobs generation activation acknowledged", {generationId: this.generationId}])
641
649
  void this._drain().catch((error) => {
642
650
  this.logger.error(() => ["Background jobs generation post-activation drain failed", {error, generationId: this.generationId}])
@@ -912,6 +920,23 @@ export default class BackgroundJobsMain {
912
920
  }
913
921
  }
914
922
 
923
+ /**
924
+ * One dead-owner settlement pass: evaluate every exact active handoff's
925
+ * recorded runner process through the owner-liveness helper and settle the
926
+ * provably-dead ones regardless of which generation created the lease.
927
+ * Owners answering `alive` (the runner is still running the job) or `unknown`
928
+ * (remote host, or identity recorded before pids/hostnames existed) stay with
929
+ * their existing recovery paths. Live child identities disprove recorded pids
930
+ * the OS recycled onto a newer runner before anything is orphaned.
931
+ * @returns {Promise<import("./types.js").BackgroundJobRow[]>} - Accepted orphan transitions.
932
+ */
933
+ async _settleDeadOwnerHandoffs() {
934
+ return await this.store.markOrphanedHandoffsByDeadOwner({
935
+ error: "Job orphaned when its recorded runner process was found dead",
936
+ liveChildren: [...this.liveChildIdentities.values()]
937
+ })
938
+ }
939
+
915
940
  _setupStartupHandoffReclaim() {
916
941
  if (this.startupHandoffSnapshot.length === 0) return
917
942
  if (this._startupHandoffReclaimTimer || this._startupHandoffReclaimPromise || this._startupHandoffGraceElapsed) return
@@ -1445,6 +1470,9 @@ export default class BackgroundJobsMain {
1445
1470
  startedAtMs: message.startedAtMs,
1446
1471
  workerId: message.workerId
1447
1472
  })
1473
+ if (typeof message.childInstanceId === "string" && typeof message.childPid === "number") {
1474
+ this._recordLiveChildIdentity({childInstanceId: message.childInstanceId, childPid: message.childPid})
1475
+ }
1448
1476
  jsonSocket.send({type: "job-updated", jobId: message.jobId})
1449
1477
  } catch (error) {
1450
1478
  this._reportJobUpdateFailure({error, jobId: message.jobId, stage: "background-job-accepted"})
@@ -1452,6 +1480,25 @@ export default class BackgroundJobsMain {
1452
1480
  }
1453
1481
  }
1454
1482
 
1483
+ /**
1484
+ * Records one pooled-runner identity as live. A pid belongs to a single live
1485
+ * process, so an identity whose pid is now reported by a different child
1486
+ * instance proves that the older runner already exited: keeping its entry
1487
+ * would make the dead-owner pass answer `dead` for the newer, still-running
1488
+ * child that inherited the pid.
1489
+ * @param {object} args - Observed runner identity.
1490
+ * @param {string} args.childInstanceId - Stable pooled-child identity of the live runner.
1491
+ * @param {number} args.childPid - OS pid of the live runner.
1492
+ * @returns {void} - Nothing.
1493
+ */
1494
+ _recordLiveChildIdentity({childInstanceId, childPid}) {
1495
+ for (const [recordedInstanceId, identity] of this.liveChildIdentities) {
1496
+ if (recordedInstanceId !== childInstanceId && identity.childPid === childPid) this.liveChildIdentities.delete(recordedInstanceId)
1497
+ }
1498
+
1499
+ this.liveChildIdentities.set(childInstanceId, {childInstanceId, childPid})
1500
+ }
1501
+
1455
1502
  /**
1456
1503
  * Requires the complete durable lease identity before a generation-mode
1457
1504
  * reporter can mutate a job. Legacy reporters keep their permissive protocol.
@@ -2669,11 +2716,24 @@ export default class BackgroundJobsMain {
2669
2716
  const handoffs = (await this._generationOwnedHandoffSnapshot()).filter((handoff) => {
2670
2717
  return handoff.handedOffAtMs <= cutoff && !connectedWorkerIds.has(handoff.workerId)
2671
2718
  })
2672
- orphanedJobs = handoffs.length === 0
2719
+ const generationOrphanedJobs = handoffs.length === 0
2673
2720
  ? []
2674
2721
  : await this.store.markOrphanedHandoffs({handoffs, error: "Job orphaned after its generation owner disappeared"})
2722
+ // Cross-generation rows are invisible to the fenced snapshot above; the
2723
+ // dead-owner pass settles them from the recorded runner process instead
2724
+ // of an age rule, so a retired generation can never strand a lease. The
2725
+ // fenced pass already settled its own rows, so only ids it did not
2726
+ // already report are appended.
2727
+ const deadOwnerJobs = await this._settleDeadOwnerHandoffs()
2728
+ const generationOrphanedJobIds = new Set(generationOrphanedJobs.map((job) => job.id))
2729
+
2730
+ orphanedJobs = [...generationOrphanedJobs, ...deadOwnerJobs.filter((job) => !generationOrphanedJobIds.has(job.id))]
2675
2731
  } else {
2676
- orphanedJobs = await this.store.markOrphanedJobs()
2732
+ const legacyOrphanedJobs = await this.store.markOrphanedJobs()
2733
+ const deadOwnerJobs = await this._settleDeadOwnerHandoffs()
2734
+ const legacyJobIds = new Set(legacyOrphanedJobs.map((job) => job.id))
2735
+
2736
+ orphanedJobs = [...legacyOrphanedJobs, ...deadOwnerJobs.filter((job) => !legacyJobIds.has(job.id))]
2677
2737
  }
2678
2738
 
2679
2739
  await this._handleOrphanedJobs({jobs: orphanedJobs, warning: "Marked orphaned background jobs"})
@@ -18,6 +18,14 @@ import os from "node:os"
18
18
  * @property {BackgroundJobOwnerLivenessStatus} status - Liveness verdict.
19
19
  */
20
20
 
21
+ /**
22
+ * A runner process this main currently observes as live, used to disprove a
23
+ * recorded pid that the OS has recycled onto a different runner.
24
+ * @typedef {object} BackgroundJobLiveChildIdentity
25
+ * @property {string | null} childInstanceId - Stable pooled-child identity of the live runner.
26
+ * @property {number} childPid - OS pid of the live runner.
27
+ */
28
+
21
29
  /**
22
30
  * Probes a local pid without signalling the target process. `EPERM` means the
23
31
  * process exists but belongs to another user, so it counts as alive; `ESRCH`
@@ -49,12 +57,16 @@ export function isProcessAlive(pid) {
49
57
  * another machine the result is `unknown`, because liveness there needs a
50
58
  * heartbeat or instance registry rather than a local probe. A pid alone can be
51
59
  * recycled by the OS after a process exits, so treat `alive` as "a process with
52
- * that pid exists now", not proof that it is the same process; pair it with
53
- * `childStartedAtMs` when that distinction matters.
60
+ * that pid exists now", not proof that it is the same process. `child_instance_id`
61
+ * is a UUID and cannot be recycled: when `liveChildren` shows the recorded pid
62
+ * currently belongs to a runner with a different instance identity, the recorded
63
+ * runner is provably gone and the verdict is `dead` (`local_pid_recycled`)
64
+ * instead of a false `alive`.
54
65
  * @param {object} [args] - Options.
55
66
  * @param {string | null} [args.childInstanceId] - Recorded pooled-child identity.
56
67
  * @param {number | null} [args.childPid] - Recorded OS pid.
57
68
  * @param {string} [args.currentHostname] - Hostname of this process, defaults to `os.hostname()`.
69
+ * @param {BackgroundJobLiveChildIdentity[]} [args.liveChildren] - Runner identities this main observes as live, for the pid-recycling check.
58
70
  * @param {(pid: number) => boolean} [args.probe] - Process probe, injectable for tests.
59
71
  * @param {string | null} [args.runnerHostname] - Hostname recorded with the job.
60
72
  * @returns {BackgroundJobOwnerLivenessResult} - Liveness verdict with its reason.
@@ -63,6 +75,7 @@ export function evaluateBackgroundJobOwnerLiveness({
63
75
  childInstanceId = null,
64
76
  childPid = null,
65
77
  currentHostname = os.hostname(),
78
+ liveChildren = [],
66
79
  probe = isProcessAlive,
67
80
  runnerHostname = null
68
81
  } = {}) {
@@ -86,7 +99,16 @@ export function evaluateBackgroundJobOwnerLiveness({
86
99
 
87
100
  const probedPid = /** @type {number} */ (owner.childPid)
88
101
 
89
- return probe(probedPid)
90
- ? {...owner, reason: "local_pid_alive", status: "alive"}
91
- : {...owner, reason: "local_pid_missing", status: "dead"}
102
+ if (!probe(probedPid)) return {...owner, reason: "local_pid_missing", status: "dead"}
103
+
104
+ // PID-recycling guard: a live pid whose instance identity differs from the
105
+ // recorded one proves the recorded runner exited and the OS handed its pid
106
+ // to a new runner — the honest verdict is dead, not a false alive.
107
+ if (owner.childInstanceId) {
108
+ const recycledBy = liveChildren.find((child) => child.childPid === probedPid && child.childInstanceId !== null && child.childInstanceId !== owner.childInstanceId)
109
+
110
+ if (recycledBy) return {...owner, reason: "local_pid_recycled", status: "dead"}
111
+ }
112
+
113
+ return {...owner, reason: "local_pid_alive", status: "alive"}
92
114
  }
@@ -1691,6 +1691,80 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
1691
1691
  })
1692
1692
  }
1693
1693
 
1694
+ /**
1695
+ * Evaluates the recorded runner-process liveness of every exact, lease-aware
1696
+ * active handoff through {@link BackgroundJobRecord#ownerIsAlive}. Rows without
1697
+ * a complete lease identity stay out of the verdicts — they belong to the
1698
+ * age-based legacy sweep. The probe and live-child list are injectable so
1699
+ * specs can prove dead/recycled verdicts deterministically.
1700
+ * @param {object} [args] - Options.
1701
+ * @param {string} [args.hostname] - Hostname whose local probe answers ownership, defaults to this process's hostname.
1702
+ * @param {import("./owner-liveness.js").BackgroundJobLiveChildIdentity[]} [args.liveChildren] - Runner identities this main observes as live, for the pid-recycling check.
1703
+ * @param {(pid: number) => boolean} [args.probe] - Process probe, injectable for tests.
1704
+ * @returns {Promise<import("./types.js").BackgroundJobHandoffOwnershipVerdict[]>} - One verdict per exact active handoff.
1705
+ */
1706
+ async snapshotHandedOffJobOwnership({hostname, liveChildren = [], probe} = {}) {
1707
+ await this.ensureReady()
1708
+
1709
+ return await this._withDb(async (db) => {
1710
+ const rows = await db
1711
+ .newQuery()
1712
+ .from(JOBS_TABLE)
1713
+ .where({status: "handed_off"})
1714
+ .order("created_at_ms ASC")
1715
+ .order("id ASC")
1716
+ .results()
1717
+ /** @type {import("./types.js").BackgroundJobHandoffOwnershipVerdict[]} */
1718
+ const verdicts = []
1719
+
1720
+ for (const row of rows) {
1721
+ const job = this._normalizeJobRow(row)
1722
+
1723
+ if (!job.handoffId || !job.workerId || typeof job.handedOffAtMs !== "number") continue
1724
+
1725
+ const record = new BackgroundJobRecord()
1726
+
1727
+ record.loadExistingRecord(row)
1728
+ verdicts.push({
1729
+ handedOffAtMs: job.handedOffAtMs,
1730
+ handoffId: job.handoffId,
1731
+ jobId: job.id,
1732
+ liveness: record.ownerIsAlive({hostname, liveChildren, probe}),
1733
+ workerId: job.workerId
1734
+ })
1735
+ }
1736
+
1737
+ return verdicts
1738
+ })
1739
+ }
1740
+
1741
+ /**
1742
+ * Settles every exact active handoff whose recorded runner process is
1743
+ * provably dead on this host, regardless of which generation created the
1744
+ * lease. Owners that answer `alive` (still running) or `unknown` (remote
1745
+ * host, identity recorded before hostnames/pids existed) stay untouched for
1746
+ * their existing recovery paths. Each settled row goes through the same
1747
+ * fenced orphan transition as the orphan sweep, so retries, terminal status,
1748
+ * `orphaned_at_ms`/`last_error` stamping, count transitions, schedule
1749
+ * ownership, and concurrency release all follow.
1750
+ * @param {object} [args] - Options.
1751
+ * @param {ReturnType<typeof JSON.parse>} [args.error] - Orphan reason recorded on settled rows.
1752
+ * @param {string} [args.hostname] - Hostname whose local probe answers ownership, defaults to this process's hostname.
1753
+ * @param {import("./owner-liveness.js").BackgroundJobLiveChildIdentity[]} [args.liveChildren] - Runner identities this main observes as live, for the pid-recycling check.
1754
+ * @param {(pid: number) => boolean} [args.probe] - Process probe, injectable for tests.
1755
+ * @returns {Promise<import("./types.js").BackgroundJobRow[]>} - Accepted orphan transitions.
1756
+ */
1757
+ async markOrphanedHandoffsByDeadOwner({error = "Job orphaned when its recorded runner process was found dead", hostname, liveChildren = [], probe} = {}) {
1758
+ await this.ensureReady()
1759
+
1760
+ const verdicts = await this.snapshotHandedOffJobOwnership({hostname, liveChildren, probe})
1761
+ const deadHandoffs = verdicts.filter((verdict) => verdict.liveness.status === "dead")
1762
+
1763
+ if (deadHandoffs.length === 0) return []
1764
+
1765
+ return await this.markOrphanedHandoffs({error, handoffs: deadHandoffs})
1766
+ }
1767
+
1694
1768
  /**
1695
1769
  * Runs mark failed.
1696
1770
  * @param {object} args - Options.
@@ -159,6 +159,9 @@
159
159
  * @property {string} workerId - Stable worker id that received the lease.
160
160
  * @property {number} handedOffAtMs - Time handed to the worker in ms.
161
161
  */
162
+ /**
163
+ * @typedef {BackgroundJobHandoffSnapshot & {liveness: import("./owner-liveness.js").BackgroundJobOwnerLivenessResult}} BackgroundJobHandoffOwnershipVerdict
164
+ */
162
165
  /**
163
166
  * @typedef {object} BackgroundJobHandoffRequest
164
167
  * @property {string} jobId - Job to claim.
@@ -275,6 +275,14 @@ export default class VelociousConfiguration {
275
275
  _advisoryLockAcquisitions = new Map()
276
276
  _advisoryLockReentries = new Map()
277
277
 
278
+ /**
279
+ * Per-tier advisory-lock acquisition counters — the "how many locks are
280
+ * served by which tier" attribution surface (STC-218 scope 1). Keyed by
281
+ * tier so production reads tier attribution directly instead of ad-hoc
282
+ * processlist probing.
283
+ * @type {Map<import("./database/advisory-lock-backend-policy.js").AdvisoryLockTier, {count: number, lastServedAt: string}>} */
284
+ _advisoryLockTierCounts = new Map()
285
+
278
286
  /** @type {Map<string, number>} */
279
287
  _schemaCacheGenerationsByReuseKey = new Map()
280
288
 
@@ -4353,7 +4361,34 @@ export default class VelociousConfiguration {
4353
4361
  * @returns {void}
4354
4362
  */
4355
4363
  recordAdvisoryLockAcquisition({lockName, backend, fencingSupported}) {
4356
- this._advisoryLockAcquisitions.set(lockName, {backend, fencingSupported, servedAt: new Date().toISOString()})
4364
+ const servedAt = new Date().toISOString()
4365
+
4366
+ this._advisoryLockAcquisitions.set(lockName, {backend, fencingSupported, servedAt})
4367
+
4368
+ const tierCount = this._advisoryLockTierCounts.get(backend) ?? {count: 0, lastServedAt: servedAt}
4369
+
4370
+ tierCount.count += 1
4371
+ tierCount.lastServedAt = servedAt
4372
+ this._advisoryLockTierCounts.set(backend, tierCount)
4373
+ }
4374
+
4375
+ /**
4376
+ * The "how many locks are served by which tier" attribution surface
4377
+ * (STC-218 scope 1): per tier, the total acquisitions this process served
4378
+ * and when the tier last served one. Together with
4379
+ * `getAdvisoryLockBackendStatus(lockName)` this is what production reads to
4380
+ * attribute lock traffic to backends without ad-hoc probing.
4381
+ * @returns {Record<string, {count: number, lastServedAt: string}>} - Snapshot of the per-tier counters.
4382
+ */
4383
+ getAdvisoryLockBackendStats() {
4384
+ /** @type {Record<string, {count: number, lastServedAt: string}>} */
4385
+ const snapshot = {}
4386
+
4387
+ for (const [tier, counter] of this._advisoryLockTierCounts) {
4388
+ snapshot[tier] = {count: counter.count, lastServedAt: counter.lastServedAt}
4389
+ }
4390
+
4391
+ return snapshot
4357
4392
  }
4358
4393
 
4359
4394
  /**