velocious 1.0.725 → 1.0.727

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (102) hide show
  1. package/README.md +1 -1
  2. package/build/background-jobs/adapter.js +16 -0
  3. package/build/background-jobs/job-record.js +3 -1
  4. package/build/background-jobs/main.js +62 -2
  5. package/build/background-jobs/owner-liveness.js +27 -5
  6. package/build/background-jobs/store.js +74 -0
  7. package/build/background-jobs/types.js +3 -0
  8. package/build/configuration-types.js +1 -1
  9. package/build/configuration.js +53 -3
  10. package/build/database/advisory-lock-lock-server-backend.js +30 -24
  11. package/build/database/advisory-lock-lock-server-embedded-client.js +74 -0
  12. package/build/database/advisory-lock-lock-server-engine.js +544 -0
  13. package/build/database/advisory-lock-lock-server-local-client.js +260 -0
  14. package/build/database/advisory-lock-lock-server-socket-path.js +17 -0
  15. package/build/database/advisory-lock-lock-server-transport.js +30 -0
  16. package/build/database/record/auditing.js +89 -11
  17. package/build/database/record/index.js +1 -1
  18. package/build/environment-handlers/base.js +50 -9
  19. package/build/environment-handlers/node/advisory-lock-lock-server-local-server.js +206 -0
  20. package/build/environment-handlers/node.js +30 -3
  21. package/build/http-server/server-lock.js +29 -16
  22. package/build/src/background-jobs/adapter.d.ts +23 -0
  23. package/build/src/background-jobs/adapter.d.ts.map +1 -1
  24. package/build/src/background-jobs/adapter.js +15 -1
  25. package/build/src/background-jobs/job-record.d.ts +3 -1
  26. package/build/src/background-jobs/job-record.d.ts.map +1 -1
  27. package/build/src/background-jobs/job-record.js +4 -2
  28. package/build/src/background-jobs/main.d.ts +33 -0
  29. package/build/src/background-jobs/main.d.ts.map +1 -1
  30. package/build/src/background-jobs/main.js +59 -3
  31. package/build/src/background-jobs/owner-liveness.d.ts +25 -3
  32. package/build/src/background-jobs/owner-liveness.d.ts.map +1 -1
  33. package/build/src/background-jobs/owner-liveness.js +26 -7
  34. package/build/src/background-jobs/store.d.ts +39 -0
  35. package/build/src/background-jobs/store.d.ts.map +1 -1
  36. package/build/src/background-jobs/store.js +66 -1
  37. package/build/src/background-jobs/types.d.ts +6 -0
  38. package/build/src/background-jobs/types.d.ts.map +1 -1
  39. package/build/src/background-jobs/types.js +4 -1
  40. package/build/src/configuration-types.d.ts +3 -1
  41. package/build/src/configuration-types.d.ts.map +1 -1
  42. package/build/src/configuration-types.js +2 -2
  43. package/build/src/configuration.d.ts +25 -1
  44. package/build/src/configuration.d.ts.map +1 -1
  45. package/build/src/configuration.js +47 -4
  46. package/build/src/database/advisory-lock-lock-server-backend.d.ts +10 -3
  47. package/build/src/database/advisory-lock-lock-server-backend.d.ts.map +1 -1
  48. package/build/src/database/advisory-lock-lock-server-backend.js +25 -22
  49. package/build/src/database/advisory-lock-lock-server-embedded-client.d.ts +24 -0
  50. package/build/src/database/advisory-lock-lock-server-embedded-client.d.ts.map +1 -0
  51. package/build/src/database/advisory-lock-lock-server-embedded-client.js +64 -0
  52. package/build/src/database/advisory-lock-lock-server-engine.d.ts +285 -0
  53. package/build/src/database/advisory-lock-lock-server-engine.d.ts.map +1 -0
  54. package/build/src/database/advisory-lock-lock-server-engine.js +460 -0
  55. package/build/src/database/advisory-lock-lock-server-local-client.d.ts +114 -0
  56. package/build/src/database/advisory-lock-lock-server-local-client.d.ts.map +1 -0
  57. package/build/src/database/advisory-lock-lock-server-local-client.js +227 -0
  58. package/build/src/database/advisory-lock-lock-server-socket-path.d.ts +9 -0
  59. package/build/src/database/advisory-lock-lock-server-socket-path.d.ts.map +1 -0
  60. package/build/src/database/advisory-lock-lock-server-socket-path.js +16 -0
  61. package/build/src/database/advisory-lock-lock-server-transport.d.ts +18 -0
  62. package/build/src/database/advisory-lock-lock-server-transport.d.ts.map +1 -0
  63. package/build/src/database/advisory-lock-lock-server-transport.js +27 -0
  64. package/build/src/database/record/auditing.d.ts +14 -4
  65. package/build/src/database/record/auditing.d.ts.map +1 -1
  66. package/build/src/database/record/auditing.js +77 -12
  67. package/build/src/database/record/index.d.ts +4 -2
  68. package/build/src/database/record/index.d.ts.map +1 -1
  69. package/build/src/database/record/index.js +2 -2
  70. package/build/src/environment-handlers/base.d.ts +86 -7
  71. package/build/src/environment-handlers/base.d.ts.map +1 -1
  72. package/build/src/environment-handlers/base.js +48 -9
  73. package/build/src/environment-handlers/node/advisory-lock-lock-server-local-server.d.ts +2 -0
  74. package/build/src/environment-handlers/node/advisory-lock-lock-server-local-server.d.ts.map +1 -0
  75. package/build/src/environment-handlers/node/advisory-lock-lock-server-local-server.js +179 -0
  76. package/build/src/environment-handlers/node.d.ts +10 -4
  77. package/build/src/environment-handlers/node.d.ts.map +1 -1
  78. package/build/src/environment-handlers/node.js +30 -4
  79. package/build/src/http-server/server-lock.d.ts +22 -11
  80. package/build/src/http-server/server-lock.d.ts.map +1 -1
  81. package/build/src/http-server/server-lock.js +30 -17
  82. package/package.json +1 -1
  83. package/src/background-jobs/adapter.js +16 -0
  84. package/src/background-jobs/job-record.js +3 -1
  85. package/src/background-jobs/main.js +62 -2
  86. package/src/background-jobs/owner-liveness.js +27 -5
  87. package/src/background-jobs/store.js +74 -0
  88. package/src/background-jobs/types.js +3 -0
  89. package/src/configuration-types.js +1 -1
  90. package/src/configuration.js +53 -3
  91. package/src/database/advisory-lock-lock-server-backend.js +30 -24
  92. package/src/database/advisory-lock-lock-server-embedded-client.js +74 -0
  93. package/src/database/advisory-lock-lock-server-engine.js +544 -0
  94. package/src/database/advisory-lock-lock-server-local-client.js +260 -0
  95. package/src/database/advisory-lock-lock-server-socket-path.js +17 -0
  96. package/src/database/advisory-lock-lock-server-transport.js +30 -0
  97. package/src/database/record/auditing.js +89 -11
  98. package/src/database/record/index.js +1 -1
  99. package/src/environment-handlers/base.js +50 -9
  100. package/src/environment-handlers/node/advisory-lock-lock-server-local-server.js +206 -0
  101. package/src/environment-handlers/node.js +30 -3
  102. package/src/http-server/server-lock.js +29 -16
package/README.md CHANGED
@@ -4,7 +4,7 @@
4
4
  * Database framework with familiar MVC concepts
5
5
  * Database models with migrations and validations
6
6
  * Database models that work almost the same in frontend and backend, including online CRUD for [composite primary keys](docs/composite-primary-keys.md)
7
- * Connection-scoped advisory locks with automatic cleanup before pooled connections are reused or closed, a non-throwing `withAdvisoryLockOrDefer` result for contended hot paths, event-driven slow acquire/hold duration telemetry with holder attribution, and an explicitly opt-in file-lock fallback tier with per-lock-name minimum-tier enforcement and acquisition provenance, and a policy-selected centralized lock-server tier that holds leases in a separate process instead of pinning database connections (see [docs/advisory-locks.md](docs/advisory-locks.md) and [docs/advisory-lock-backends.md](docs/advisory-lock-backends.md))* Built-in record auditing for model lifecycle changes (see [docs/auditing.md](docs/auditing.md))
7
+ * Connection-scoped advisory locks with automatic cleanup before pooled connections are reused or closed, a non-throwing `withAdvisoryLockOrDefer` result for contended hot paths, event-driven slow acquire/hold duration telemetry with holder attribution, and an explicitly opt-in file-lock fallback tier with per-lock-name minimum-tier enforcement and acquisition provenance, a policy-selected centralized lock-server tier that holds leases in a separate process instead of pinning database connections, and per-tier attribution counters via `configuration.getAdvisoryLockBackendStats()` (see [docs/advisory-locks.md](docs/advisory-locks.md) and [docs/advisory-lock-backends.md](docs/advisory-lock-backends.md))* Built-in record auditing for model lifecycle changes (see [docs/auditing.md](docs/auditing.md))
8
8
  * Declarative state machines for models, with typed event methods generated into the base model (see [docs/state-machine.md](docs/state-machine.md))
9
9
  * Migrations for schema changes and UTC datetime storage, including caller-selected `pre-runtime` and `post-publication` execution sets and recorded `changeTable` batches that combine operations into one `ALTER` on bulk-capable drivers (see [docs/database-migrations.md](docs/database-migrations.md), [docs/migration-execution-phases.md](docs/migration-execution-phases.md), and [docs/change-table.md](docs/change-table.md))
10
10
  * Tenant-selected base-model and structure generation with one immutable, fail-closed physical database context; tenant-only model metadata initializes only after that context is active (see [docs/tenant-selected-database-generation.md](docs/tenant-selected-database-generation.md))
@@ -190,6 +190,22 @@ export default class BackgroundJobsAdapter {
190
190
  */
191
191
  async markOrphanedHandoffs(_args) { return [] }
192
192
 
193
+ /**
194
+ * Evaluates recorded runner-process liveness for exact active handoffs.
195
+ * Adapters without durable runner identity may return none.
196
+ * @param {{hostname?: string, liveChildren?: import("./owner-liveness.js").BackgroundJobLiveChildIdentity[], probe?: (pid: number) => boolean}} [_args] - Probe controls.
197
+ * @returns {Promise<import("./types.js").BackgroundJobHandoffOwnershipVerdict[]>} - One verdict per exact active handoff.
198
+ */
199
+ async snapshotHandedOffJobOwnership(_args) { return [] }
200
+
201
+ /**
202
+ * Settles exact active handoffs whose recorded runner process is provably
203
+ * dead. Adapters without durable runner identity settle none.
204
+ * @param {{error?: ReturnType<typeof JSON.parse>, hostname?: string, liveChildren?: import("./owner-liveness.js").BackgroundJobLiveChildIdentity[], probe?: (pid: number) => boolean}} [_args] - Orphan reason and probe controls.
205
+ * @returns {Promise<import("./types.js").BackgroundJobRow[]>} - Accepted orphan transitions.
206
+ */
207
+ async markOrphanedHandoffsByDeadOwner(_args) { return [] }
208
+
193
209
  /**
194
210
  * Marks a handed-off job failed or retryable.
195
211
  * @param {{jobId: string, error: ReturnType<typeof JSON.parse>, handoffId?: string, workerId?: string, handedOffAtMs?: number}} _args - Failure report.
@@ -21,14 +21,16 @@ export default class BackgroundJobRecord extends DatabaseRecord {
21
21
  * registry, which this helper deliberately does not invent.
22
22
  * @param {object} [args] - Options.
23
23
  * @param {string} [args.hostname] - Hostname to compare against, defaults to this process's hostname.
24
+ * @param {import("./owner-liveness.js").BackgroundJobLiveChildIdentity[]} [args.liveChildren] - Runner identities observed as live, for the pid-recycling check.
24
25
  * @param {(pid: number) => boolean} [args.probe] - Process probe, injectable for tests.
25
26
  * @returns {import("./owner-liveness.js").BackgroundJobOwnerLivenessResult} - Liveness verdict.
26
27
  */
27
- ownerIsAlive({hostname, probe} = {}) {
28
+ ownerIsAlive({hostname, liveChildren, probe} = {}) {
28
29
  return evaluateBackgroundJobOwnerLiveness({
29
30
  childInstanceId: /** @type {string | null} */ (this.readAttribute("childInstanceId")),
30
31
  childPid: /** @type {number | null} */ (this.readAttribute("childPid")),
31
32
  currentHostname: hostname,
33
+ liveChildren,
32
34
  probe,
33
35
  runnerHostname: /** @type {string | null} */ (this.readAttribute("runnerHostname"))
34
36
  })
@@ -190,6 +190,13 @@ export default class BackgroundJobsMain {
190
190
  * @type {Set<string>}
191
191
  */
192
192
  this.reconnectedWorkerIds = new Set()
193
+ /**
194
+ * Pooled-runner identities this main observed accepting jobs, keyed by
195
+ * child instance id. A dead-owner sweep uses them to disprove a recorded
196
+ * pid that the OS recycled onto a newer runner before orphaning its rows.
197
+ * @type {Map<string, import("./owner-liveness.js").BackgroundJobLiveChildIdentity>}
198
+ */
199
+ this.liveChildIdentities = new Map()
193
200
  /** @type {import("./types.js").BackgroundJobHandoffSnapshot[]} */
194
201
  this.startupHandoffSnapshot = []
195
202
  /** @type {Promise<void>[]} */
@@ -637,6 +644,7 @@ export default class BackgroundJobsMain {
637
644
  this.lifecycleState = "active"
638
645
  this._creditReadyWorkers()
639
646
  await this._settleForeignGenerationHandoffs()
647
+ await this._settleDeadOwnerHandoffs()
640
648
  this.logger.info(() => ["Background jobs generation activation acknowledged", {generationId: this.generationId}])
641
649
  void this._drain().catch((error) => {
642
650
  this.logger.error(() => ["Background jobs generation post-activation drain failed", {error, generationId: this.generationId}])
@@ -912,6 +920,23 @@ export default class BackgroundJobsMain {
912
920
  }
913
921
  }
914
922
 
923
+ /**
924
+ * One dead-owner settlement pass: evaluate every exact active handoff's
925
+ * recorded runner process through the owner-liveness helper and settle the
926
+ * provably-dead ones regardless of which generation created the lease.
927
+ * Owners answering `alive` (the runner is still running the job) or `unknown`
928
+ * (remote host, or identity recorded before pids/hostnames existed) stay with
929
+ * their existing recovery paths. Live child identities disprove recorded pids
930
+ * the OS recycled onto a newer runner before anything is orphaned.
931
+ * @returns {Promise<import("./types.js").BackgroundJobRow[]>} - Accepted orphan transitions.
932
+ */
933
+ async _settleDeadOwnerHandoffs() {
934
+ return await this.store.markOrphanedHandoffsByDeadOwner({
935
+ error: "Job orphaned when its recorded runner process was found dead",
936
+ liveChildren: [...this.liveChildIdentities.values()]
937
+ })
938
+ }
939
+
915
940
  _setupStartupHandoffReclaim() {
916
941
  if (this.startupHandoffSnapshot.length === 0) return
917
942
  if (this._startupHandoffReclaimTimer || this._startupHandoffReclaimPromise || this._startupHandoffGraceElapsed) return
@@ -1445,6 +1470,9 @@ export default class BackgroundJobsMain {
1445
1470
  startedAtMs: message.startedAtMs,
1446
1471
  workerId: message.workerId
1447
1472
  })
1473
+ if (typeof message.childInstanceId === "string" && typeof message.childPid === "number") {
1474
+ this._recordLiveChildIdentity({childInstanceId: message.childInstanceId, childPid: message.childPid})
1475
+ }
1448
1476
  jsonSocket.send({type: "job-updated", jobId: message.jobId})
1449
1477
  } catch (error) {
1450
1478
  this._reportJobUpdateFailure({error, jobId: message.jobId, stage: "background-job-accepted"})
@@ -1452,6 +1480,25 @@ export default class BackgroundJobsMain {
1452
1480
  }
1453
1481
  }
1454
1482
 
1483
+ /**
1484
+ * Records one pooled-runner identity as live. A pid belongs to a single live
1485
+ * process, so an identity whose pid is now reported by a different child
1486
+ * instance proves that the older runner already exited: keeping its entry
1487
+ * would make the dead-owner pass answer `dead` for the newer, still-running
1488
+ * child that inherited the pid.
1489
+ * @param {object} args - Observed runner identity.
1490
+ * @param {string} args.childInstanceId - Stable pooled-child identity of the live runner.
1491
+ * @param {number} args.childPid - OS pid of the live runner.
1492
+ * @returns {void} - Nothing.
1493
+ */
1494
+ _recordLiveChildIdentity({childInstanceId, childPid}) {
1495
+ for (const [recordedInstanceId, identity] of this.liveChildIdentities) {
1496
+ if (recordedInstanceId !== childInstanceId && identity.childPid === childPid) this.liveChildIdentities.delete(recordedInstanceId)
1497
+ }
1498
+
1499
+ this.liveChildIdentities.set(childInstanceId, {childInstanceId, childPid})
1500
+ }
1501
+
1455
1502
  /**
1456
1503
  * Requires the complete durable lease identity before a generation-mode
1457
1504
  * reporter can mutate a job. Legacy reporters keep their permissive protocol.
@@ -2669,11 +2716,24 @@ export default class BackgroundJobsMain {
2669
2716
  const handoffs = (await this._generationOwnedHandoffSnapshot()).filter((handoff) => {
2670
2717
  return handoff.handedOffAtMs <= cutoff && !connectedWorkerIds.has(handoff.workerId)
2671
2718
  })
2672
- orphanedJobs = handoffs.length === 0
2719
+ const generationOrphanedJobs = handoffs.length === 0
2673
2720
  ? []
2674
2721
  : await this.store.markOrphanedHandoffs({handoffs, error: "Job orphaned after its generation owner disappeared"})
2722
+ // Cross-generation rows are invisible to the fenced snapshot above; the
2723
+ // dead-owner pass settles them from the recorded runner process instead
2724
+ // of an age rule, so a retired generation can never strand a lease. The
2725
+ // fenced pass already settled its own rows, so only ids it did not
2726
+ // already report are appended.
2727
+ const deadOwnerJobs = await this._settleDeadOwnerHandoffs()
2728
+ const generationOrphanedJobIds = new Set(generationOrphanedJobs.map((job) => job.id))
2729
+
2730
+ orphanedJobs = [...generationOrphanedJobs, ...deadOwnerJobs.filter((job) => !generationOrphanedJobIds.has(job.id))]
2675
2731
  } else {
2676
- orphanedJobs = await this.store.markOrphanedJobs()
2732
+ const legacyOrphanedJobs = await this.store.markOrphanedJobs()
2733
+ const deadOwnerJobs = await this._settleDeadOwnerHandoffs()
2734
+ const legacyJobIds = new Set(legacyOrphanedJobs.map((job) => job.id))
2735
+
2736
+ orphanedJobs = [...legacyOrphanedJobs, ...deadOwnerJobs.filter((job) => !legacyJobIds.has(job.id))]
2677
2737
  }
2678
2738
 
2679
2739
  await this._handleOrphanedJobs({jobs: orphanedJobs, warning: "Marked orphaned background jobs"})
@@ -18,6 +18,14 @@ import os from "node:os"
18
18
  * @property {BackgroundJobOwnerLivenessStatus} status - Liveness verdict.
19
19
  */
20
20
 
21
+ /**
22
+ * A runner process this main currently observes as live, used to disprove a
23
+ * recorded pid that the OS has recycled onto a different runner.
24
+ * @typedef {object} BackgroundJobLiveChildIdentity
25
+ * @property {string | null} childInstanceId - Stable pooled-child identity of the live runner.
26
+ * @property {number} childPid - OS pid of the live runner.
27
+ */
28
+
21
29
  /**
22
30
  * Probes a local pid without signalling the target process. `EPERM` means the
23
31
  * process exists but belongs to another user, so it counts as alive; `ESRCH`
@@ -49,12 +57,16 @@ export function isProcessAlive(pid) {
49
57
  * another machine the result is `unknown`, because liveness there needs a
50
58
  * heartbeat or instance registry rather than a local probe. A pid alone can be
51
59
  * recycled by the OS after a process exits, so treat `alive` as "a process with
52
- * that pid exists now", not proof that it is the same process; pair it with
53
- * `childStartedAtMs` when that distinction matters.
60
+ * that pid exists now", not proof that it is the same process. `child_instance_id`
61
+ * is a UUID and cannot be recycled: when `liveChildren` shows the recorded pid
62
+ * currently belongs to a runner with a different instance identity, the recorded
63
+ * runner is provably gone and the verdict is `dead` (`local_pid_recycled`)
64
+ * instead of a false `alive`.
54
65
  * @param {object} [args] - Options.
55
66
  * @param {string | null} [args.childInstanceId] - Recorded pooled-child identity.
56
67
  * @param {number | null} [args.childPid] - Recorded OS pid.
57
68
  * @param {string} [args.currentHostname] - Hostname of this process, defaults to `os.hostname()`.
69
+ * @param {BackgroundJobLiveChildIdentity[]} [args.liveChildren] - Runner identities this main observes as live, for the pid-recycling check.
58
70
  * @param {(pid: number) => boolean} [args.probe] - Process probe, injectable for tests.
59
71
  * @param {string | null} [args.runnerHostname] - Hostname recorded with the job.
60
72
  * @returns {BackgroundJobOwnerLivenessResult} - Liveness verdict with its reason.
@@ -63,6 +75,7 @@ export function evaluateBackgroundJobOwnerLiveness({
63
75
  childInstanceId = null,
64
76
  childPid = null,
65
77
  currentHostname = os.hostname(),
78
+ liveChildren = [],
66
79
  probe = isProcessAlive,
67
80
  runnerHostname = null
68
81
  } = {}) {
@@ -86,7 +99,16 @@ export function evaluateBackgroundJobOwnerLiveness({
86
99
 
87
100
  const probedPid = /** @type {number} */ (owner.childPid)
88
101
 
89
- return probe(probedPid)
90
- ? {...owner, reason: "local_pid_alive", status: "alive"}
91
- : {...owner, reason: "local_pid_missing", status: "dead"}
102
+ if (!probe(probedPid)) return {...owner, reason: "local_pid_missing", status: "dead"}
103
+
104
+ // PID-recycling guard: a live pid whose instance identity differs from the
105
+ // recorded one proves the recorded runner exited and the OS handed its pid
106
+ // to a new runner — the honest verdict is dead, not a false alive.
107
+ if (owner.childInstanceId) {
108
+ const recycledBy = liveChildren.find((child) => child.childPid === probedPid && child.childInstanceId !== null && child.childInstanceId !== owner.childInstanceId)
109
+
110
+ if (recycledBy) return {...owner, reason: "local_pid_recycled", status: "dead"}
111
+ }
112
+
113
+ return {...owner, reason: "local_pid_alive", status: "alive"}
92
114
  }
@@ -1691,6 +1691,80 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
1691
1691
  })
1692
1692
  }
1693
1693
 
1694
+ /**
1695
+ * Evaluates the recorded runner-process liveness of every exact, lease-aware
1696
+ * active handoff through {@link BackgroundJobRecord#ownerIsAlive}. Rows without
1697
+ * a complete lease identity stay out of the verdicts — they belong to the
1698
+ * age-based legacy sweep. The probe and live-child list are injectable so
1699
+ * specs can prove dead/recycled verdicts deterministically.
1700
+ * @param {object} [args] - Options.
1701
+ * @param {string} [args.hostname] - Hostname whose local probe answers ownership, defaults to this process's hostname.
1702
+ * @param {import("./owner-liveness.js").BackgroundJobLiveChildIdentity[]} [args.liveChildren] - Runner identities this main observes as live, for the pid-recycling check.
1703
+ * @param {(pid: number) => boolean} [args.probe] - Process probe, injectable for tests.
1704
+ * @returns {Promise<import("./types.js").BackgroundJobHandoffOwnershipVerdict[]>} - One verdict per exact active handoff.
1705
+ */
1706
+ async snapshotHandedOffJobOwnership({hostname, liveChildren = [], probe} = {}) {
1707
+ await this.ensureReady()
1708
+
1709
+ return await this._withDb(async (db) => {
1710
+ const rows = await db
1711
+ .newQuery()
1712
+ .from(JOBS_TABLE)
1713
+ .where({status: "handed_off"})
1714
+ .order("created_at_ms ASC")
1715
+ .order("id ASC")
1716
+ .results()
1717
+ /** @type {import("./types.js").BackgroundJobHandoffOwnershipVerdict[]} */
1718
+ const verdicts = []
1719
+
1720
+ for (const row of rows) {
1721
+ const job = this._normalizeJobRow(row)
1722
+
1723
+ if (!job.handoffId || !job.workerId || typeof job.handedOffAtMs !== "number") continue
1724
+
1725
+ const record = new BackgroundJobRecord()
1726
+
1727
+ record.loadExistingRecord(row)
1728
+ verdicts.push({
1729
+ handedOffAtMs: job.handedOffAtMs,
1730
+ handoffId: job.handoffId,
1731
+ jobId: job.id,
1732
+ liveness: record.ownerIsAlive({hostname, liveChildren, probe}),
1733
+ workerId: job.workerId
1734
+ })
1735
+ }
1736
+
1737
+ return verdicts
1738
+ })
1739
+ }
1740
+
1741
+ /**
1742
+ * Settles every exact active handoff whose recorded runner process is
1743
+ * provably dead on this host, regardless of which generation created the
1744
+ * lease. Owners that answer `alive` (still running) or `unknown` (remote
1745
+ * host, identity recorded before hostnames/pids existed) stay untouched for
1746
+ * their existing recovery paths. Each settled row goes through the same
1747
+ * fenced orphan transition as the orphan sweep, so retries, terminal status,
1748
+ * `orphaned_at_ms`/`last_error` stamping, count transitions, schedule
1749
+ * ownership, and concurrency release all follow.
1750
+ * @param {object} [args] - Options.
1751
+ * @param {ReturnType<typeof JSON.parse>} [args.error] - Orphan reason recorded on settled rows.
1752
+ * @param {string} [args.hostname] - Hostname whose local probe answers ownership, defaults to this process's hostname.
1753
+ * @param {import("./owner-liveness.js").BackgroundJobLiveChildIdentity[]} [args.liveChildren] - Runner identities this main observes as live, for the pid-recycling check.
1754
+ * @param {(pid: number) => boolean} [args.probe] - Process probe, injectable for tests.
1755
+ * @returns {Promise<import("./types.js").BackgroundJobRow[]>} - Accepted orphan transitions.
1756
+ */
1757
+ async markOrphanedHandoffsByDeadOwner({error = "Job orphaned when its recorded runner process was found dead", hostname, liveChildren = [], probe} = {}) {
1758
+ await this.ensureReady()
1759
+
1760
+ const verdicts = await this.snapshotHandedOffJobOwnership({hostname, liveChildren, probe})
1761
+ const deadHandoffs = verdicts.filter((verdict) => verdict.liveness.status === "dead")
1762
+
1763
+ if (deadHandoffs.length === 0) return []
1764
+
1765
+ return await this.markOrphanedHandoffs({error, handoffs: deadHandoffs})
1766
+ }
1767
+
1694
1768
  /**
1695
1769
  * Runs mark failed.
1696
1770
  * @param {object} args - Options.
@@ -159,6 +159,9 @@
159
159
  * @property {string} workerId - Stable worker id that received the lease.
160
160
  * @property {number} handedOffAtMs - Time handed to the worker in ms.
161
161
  */
162
+ /**
163
+ * @typedef {BackgroundJobHandoffSnapshot & {liveness: import("./owner-liveness.js").BackgroundJobOwnerLivenessResult}} BackgroundJobHandoffOwnershipVerdict
164
+ */
162
165
  /**
163
166
  * @typedef {object} BackgroundJobHandoffRequest
164
167
  * @property {string} jobId - Job to claim.
@@ -882,7 +882,7 @@
882
882
  * behavior; the file backend is disabled by default and only selectable by
883
883
  * explicit configuration.
884
884
  * @typedef {object} AdvisoryLocksConfiguration
885
- * @property {{file?: {directory?: string, enabled?: boolean, leaseMs?: number, renewIntervalMs?: number}, lockServer?: {authToken?: string, connectTimeoutMs?: number, enabled?: boolean, host?: string, leaseMs?: number, port?: number, reprobeCooldownMs?: number, url?: string}}} [backends] - Backend enablement. `file.enabled` defaults to false; enabling it is the deployment's explicit single-host declaration. `lockServer` defaults to enabled when a host/port/url or the `VELOCIOUS_LOCK_SERVER_HOST`/`_PORT` env pair is configured.
885
+ * @property {{file?: {directory?: string, enabled?: boolean, leaseMs?: number, renewIntervalMs?: number}, lockServer?: {authToken?: string, connectTimeoutMs?: number, enabled?: boolean, host?: string, leaseMs?: number, mode?: "embedded" | "external" | "local", port?: number, reprobeCooldownMs?: number, socketPath?: string, url?: string}}} [backends] - Backend enablement. `file.enabled` defaults to false; enabling it is the deployment's explicit single-host declaration. `lockServer` defaults to enabled when a host/port/url or the `VELOCIOUS_LOCK_SERVER_HOST`/`_PORT` env pair is configured.
886
886
  * @property {import("./database/advisory-lock-backend-policy.js").AdvisoryLockBackendPolicyEntry[]} [policy] - Per-lock-name minimum/allowed/preferred tier rules (design §5).
887
887
  */
888
888
 
@@ -275,6 +275,14 @@ export default class VelociousConfiguration {
275
275
  _advisoryLockAcquisitions = new Map()
276
276
  _advisoryLockReentries = new Map()
277
277
 
278
+ /**
279
+ * Per-tier advisory-lock acquisition counters — the "how many locks are
280
+ * served by which tier" attribution surface (STC-218 scope 1). Keyed by
281
+ * tier so production reads tier attribution directly instead of ad-hoc
282
+ * processlist probing.
283
+ * @type {Map<import("./database/advisory-lock-backend-policy.js").AdvisoryLockTier, {count: number, lastServedAt: string}>} */
284
+ _advisoryLockTierCounts = new Map()
285
+
278
286
  /** @type {Map<string, number>} */
279
287
  _schemaCacheGenerationsByReuseKey = new Map()
280
288
 
@@ -4226,7 +4234,7 @@ export default class VelociousConfiguration {
4226
4234
  * Full backward compatibility: an app that configures nothing gets today's
4227
4235
  * driver-native-only behavior — the file backend is **disabled by default**
4228
4236
  * and only ever selectable by explicit configuration.
4229
- * @returns {{backends: {file: {directory?: string, enabled: boolean, leaseMs: number, renewIntervalMs: number}, lockServer: {authToken: string, connectTimeoutMs: number, enabled: boolean, leaseMs: number, reprobeCooldownMs: number, url: string}}, policy: import("./database/advisory-lock-backend-policy.js").AdvisoryLockBackendPolicyEntry[]}} - Advisory-lock configuration with defaults applied.
4237
+ * @returns {{backends: {file: {directory?: string, enabled: boolean, leaseMs: number, renewIntervalMs: number}, lockServer: {authToken: string, connectTimeoutMs: number, enabled: boolean, leaseMs: number, mode: "embedded" | "external" | "local", reprobeCooldownMs: number, socketPath: string, url: string}}, policy: import("./database/advisory-lock-backend-policy.js").AdvisoryLockBackendPolicyEntry[]}} - Advisory-lock configuration with defaults applied.
4230
4238
  */
4231
4239
  getAdvisoryLockConfig() {
4232
4240
  const configured = this._advisoryLocks || {}
@@ -4241,6 +4249,15 @@ export default class VelociousConfiguration {
4241
4249
  : (typeof envPort === "number" && Number.isFinite(envPort) ? envPort : 3006)
4242
4250
  const lockServerUrl = lockServerConfigured.url
4243
4251
  || `ws://${lockServerHost || "127.0.0.1"}:${lockServerPort}/websocket`
4252
+ // STC-715: `embedded` runs the lock-server engine in this process, `local`
4253
+ // spawns it as a child over stdio JSON-lines. Both default `enabled` to true
4254
+ // when the mode is named explicitly — no host/port/url required. `external`
4255
+ // (the default) keeps today's rule: enabled only via url/host/port config
4256
+ // or the env pair, so an app that configures nothing is unchanged.
4257
+ const configuredMode = lockServerConfigured.mode
4258
+ const lockServerMode = configuredMode === "embedded" || configuredMode === "local" || configuredMode === "external"
4259
+ ? configuredMode
4260
+ : "external"
4244
4261
 
4245
4262
  return {
4246
4263
  backends: {
@@ -4257,9 +4274,13 @@ export default class VelociousConfiguration {
4257
4274
  connectTimeoutMs: typeof lockServerConfigured.connectTimeoutMs === "number" ? lockServerConfigured.connectTimeoutMs : 3000,
4258
4275
  enabled: typeof lockServerConfigured.enabled === "boolean"
4259
4276
  ? lockServerConfigured.enabled
4260
- : Boolean(lockServerConfigured.url || lockServerConfigured.host || envHost || typeof lockServerConfigured.port === "number" || envPortRaw),
4277
+ : lockServerMode === "external"
4278
+ ? Boolean(lockServerConfigured.url || lockServerConfigured.host || envHost || typeof lockServerConfigured.port === "number" || envPortRaw)
4279
+ : true,
4261
4280
  leaseMs: typeof lockServerConfigured.leaseMs === "number" ? lockServerConfigured.leaseMs : 30_000,
4281
+ mode: lockServerMode,
4262
4282
  reprobeCooldownMs: typeof lockServerConfigured.reprobeCooldownMs === "number" ? lockServerConfigured.reprobeCooldownMs : 5000,
4283
+ socketPath: typeof lockServerConfigured.socketPath === "string" ? lockServerConfigured.socketPath : "",
4263
4284
  url: lockServerUrl
4264
4285
  }
4265
4286
  },
@@ -4329,11 +4350,13 @@ export default class VelociousConfiguration {
4329
4350
  authToken: config.backends.lockServer.authToken,
4330
4351
  connectTimeoutMs: config.backends.lockServer.connectTimeoutMs,
4331
4352
  leaseMs: config.backends.lockServer.leaseMs,
4353
+ mode: config.backends.lockServer.mode,
4332
4354
  onError: (/** @type {Error} */ error) => {
4333
4355
  this.getErrorEvents().emit("framework-error", {context: "advisory-lock-lock-server", error})
4334
4356
  this.getErrorEvents().emit("all-error", {context: "advisory-lock-lock-server", error, errorType: "framework-error"})
4335
4357
  },
4336
4358
  reprobeCooldownMs: config.backends.lockServer.reprobeCooldownMs,
4359
+ socketPath: config.backends.lockServer.socketPath,
4337
4360
  url: config.backends.lockServer.url
4338
4361
  })
4339
4362
 
@@ -4353,7 +4376,34 @@ export default class VelociousConfiguration {
4353
4376
  * @returns {void}
4354
4377
  */
4355
4378
  recordAdvisoryLockAcquisition({lockName, backend, fencingSupported}) {
4356
- this._advisoryLockAcquisitions.set(lockName, {backend, fencingSupported, servedAt: new Date().toISOString()})
4379
+ const servedAt = new Date().toISOString()
4380
+
4381
+ this._advisoryLockAcquisitions.set(lockName, {backend, fencingSupported, servedAt})
4382
+
4383
+ const tierCount = this._advisoryLockTierCounts.get(backend) ?? {count: 0, lastServedAt: servedAt}
4384
+
4385
+ tierCount.count += 1
4386
+ tierCount.lastServedAt = servedAt
4387
+ this._advisoryLockTierCounts.set(backend, tierCount)
4388
+ }
4389
+
4390
+ /**
4391
+ * The "how many locks are served by which tier" attribution surface
4392
+ * (STC-218 scope 1): per tier, the total acquisitions this process served
4393
+ * and when the tier last served one. Together with
4394
+ * `getAdvisoryLockBackendStatus(lockName)` this is what production reads to
4395
+ * attribute lock traffic to backends without ad-hoc probing.
4396
+ * @returns {Record<string, {count: number, lastServedAt: string}>} - Snapshot of the per-tier counters.
4397
+ */
4398
+ getAdvisoryLockBackendStats() {
4399
+ /** @type {Record<string, {count: number, lastServedAt: string}>} */
4400
+ const snapshot = {}
4401
+
4402
+ for (const [tier, counter] of this._advisoryLockTierCounts) {
4403
+ snapshot[tier] = {count: counter.count, lastServedAt: counter.lastServedAt}
4404
+ }
4405
+
4406
+ return snapshot
4357
4407
  }
4358
4408
 
4359
4409
  /**
@@ -1,8 +1,5 @@
1
1
  // @ts-check
2
2
 
3
- import WebSocket from "ws"
4
-
5
- import VelociousWebsocketClient from "../http-client/websocket-client.js"
6
3
  import {AdvisoryLockServiceUnavailableError} from "./advisory-lock-errors.js"
7
4
 
8
5
  /**
@@ -40,6 +37,13 @@ const LOCK_SERVER_CONNECTION_TYPE = "lock"
40
37
  *
41
38
  * Unavailability is cached for `reprobeCooldownMs` so a flapping server cannot
42
39
  * turn every waiting caller into a connection attempt (design §2).
40
+ *
41
+ * The transport is injected through `clientFactory` (STC-715): this module no
42
+ * longer imports `ws` itself, so it is safe for browser/Expo bundles. The Node
43
+ * handler injects the websocket transport for `external` mode, the spawned
44
+ * stdio/UDS client for `local` mode, and the in-process engine client for
45
+ * `embedded` mode — all three speak the same protocol surface through this
46
+ * adapter.
43
47
  */
44
48
  export default class AdvisoryLockLockServerBackend {
45
49
  /**
@@ -47,14 +51,14 @@ export default class AdvisoryLockLockServerBackend {
47
51
  * constructing this object never performs I/O.
48
52
  * @param {object} args - Backend options.
49
53
  * @param {string} [args.authToken] - Shared secret; the server fails closed without a matching token.
50
- * @param {(args: {authToken: string, url: string}) => LockServerClientLike} [args.clientFactory] - Transport factory (tests inject a fake).
54
+ * @param {(args: {authToken: string, url: string}) => LockServerClientLike} args.clientFactory - Transport factory injected by the environment handler (websocket for external mode, spawned child for local mode, in-process engine for embedded mode).
51
55
  * @param {number} [args.connectTimeoutMs] - Handshake timeout.
52
56
  * @param {number} [args.leaseMs] - Lease duration used until the server advertises its own.
53
57
  * @param {(error: Error) => void} [args.onError] - Async error reporter.
54
58
  * @param {number} [args.reprobeCooldownMs] - How long an unavailable server is not re-probed.
55
- * @param {string} args.url - Lock-server websocket URL.
59
+ * @param {string} args.url - Lock-server endpoint (websocket URL for external mode; informational for embedded/local modes).
56
60
  */
57
- constructor({authToken = "", clientFactory = defaultClientFactory, connectTimeoutMs = 3000, leaseMs = 30_000, onError, reprobeCooldownMs = 5000, url}) {
61
+ constructor({authToken = "", clientFactory, connectTimeoutMs = 3000, leaseMs = 30_000, onError, reprobeCooldownMs = 5000, url}) {
58
62
  this.url = url
59
63
  this.authToken = authToken
60
64
  this.clientFactory = clientFactory
@@ -117,6 +121,8 @@ export default class AdvisoryLockLockServerBackend {
117
121
  this.status.lastProbeAt = new Date().toISOString()
118
122
 
119
123
  try {
124
+ if (typeof this.clientFactory !== "function") throw new Error("The lock-server backend requires a clientFactory transport from the environment handler")
125
+
120
126
  const client = this.clientFactory({authToken: this.authToken, url: this.url})
121
127
 
122
128
  this.client = client
@@ -350,6 +356,24 @@ export default class AdvisoryLockLockServerBackend {
350
356
  return
351
357
  }
352
358
 
359
+ // Transport-side death (spawned child exited, embedded engine closed):
360
+ // fail every pending request now and start the re-probe cooldown instead
361
+ // of letting each request burn its own timeout. This is the crash-detection
362
+ // half of the local-mode lifecycle (STC-715); the cooldown is the restart
363
+ // backoff and the next acquisition spawns the child again.
364
+ if (body.type === "response" && body.transportClosed === true) {
365
+ const error = new AdvisoryLockServiceUnavailableError(
366
+ `Lock server ${this.url} transport closed: ${String(body.error || "transport closed")}`,
367
+ {lockName: "", reason: "transport-closed"}
368
+ )
369
+
370
+ this.markUnavailable(error)
371
+ this.teardown()
372
+ this.reportError(error)
373
+
374
+ return
375
+ }
376
+
353
377
  if (body.type === "lost") {
354
378
  const name = String(body.name ?? "")
355
379
  const hold = this.holds.get(name)
@@ -547,21 +571,3 @@ export default class AdvisoryLockLockServerBackend {
547
571
  if (this.onError) this.onError(normalized)
548
572
  }
549
573
  }
550
-
551
- /**
552
- * Default transport factory: the framework websocket client with the Node
553
- * `ws` implementation bound explicitly (no global mutation).
554
- * @param {{authToken: string, url: string}} args - Connection args.
555
- * @returns {LockServerClientLike} - Websocket client.
556
- */
557
- function defaultClientFactory({url}) {
558
- // Node's global WebSocket is not guaranteed to exist, so the implementation is
559
- // bound explicitly instead of mutating `globalThis`.
560
- const webSocketImplementation = /** @type {typeof globalThis.WebSocket} */ (/** @type {unknown} */ (WebSocket))
561
-
562
- return /** @type {LockServerClientLike} */ (/** @type {unknown} */ (new VelociousWebsocketClient({
563
- heartbeatIntervalMs: 10_000,
564
- url,
565
- webSocketImplementation
566
- })))
567
- }
@@ -0,0 +1,74 @@
1
+ // @ts-check
2
+
3
+ import AdvisoryLockLockServerEngine from "./advisory-lock-lock-server-engine.js"
4
+
5
+ /**
6
+ * @typedef {import("./advisory-lock-lock-server-backend.js").LockServerClientLike} LockServerClientLike
7
+ * @typedef {import("./advisory-lock-lock-server-backend.js").LockServerConnectionLike} LockServerConnectionLike
8
+ */
9
+
10
+ /**
11
+ * In-process transport factory for the `embedded` lock-server mode (STC-715).
12
+ * It wires the lock-server backend adapter to an `AdvisoryLockLockServerEngine`
13
+ * living in the same process over plain function calls — no sockets, no `ws`,
14
+ * no Node builtins — so browser/Expo bundles can run the lock-server tier.
15
+ *
16
+ * The engine mints the holder id for the connection exactly like the external
17
+ * daemon does over the websocket transport, so the adapter's lease/epoch/
18
+ * fencing behavior is identical across modes.
19
+ * @param {AdvisoryLockLockServerEngine} engine - Engine this transport connects to.
20
+ * @returns {(args: {authToken: string, url: string}) => LockServerClientLike} - Client factory for the backend adapter.
21
+ */
22
+ export function createEmbeddedLockServerClientFactory(engine) {
23
+ return () => {
24
+ /** @type {string | null} */
25
+ let connectionId = null
26
+ /** @type {((body: Record<string, unknown>) => void) | null} */
27
+ let onMessage = null
28
+
29
+ /** @type {LockServerClientLike} */
30
+ const client = {
31
+ async connect() {},
32
+
33
+ close() {
34
+ if (connectionId !== null) engine.connectionClose(connectionId)
35
+
36
+ connectionId = null
37
+ },
38
+
39
+ openConnection(_type, {onMessage: messageHandler}) {
40
+ onMessage = messageHandler
41
+
42
+ connectionId = engine.openConnection({
43
+ onClose: () => {
44
+ onMessage?.({error: "Embedded lock server engine is closed", status: "transport-closed", transportClosed: true, type: "response"})
45
+ },
46
+ send: (/** @type {Record<string, unknown>} */ body) => messageHandler(body)
47
+ })
48
+
49
+ // The engine answers the connection-open frame with a hello-ack, just
50
+ // like the external daemon does after verifying auth and protocol.
51
+ queueMicrotask(() => messageHandler({leaseMs: engine.leaseMs, protocolVersion: 2, type: "hello-ack"}))
52
+
53
+ /** @type {LockServerConnectionLike} */
54
+ const connection = {
55
+ close: () => {
56
+ if (connectionId !== null) engine.connectionClose(connectionId)
57
+
58
+ connectionId = null
59
+ },
60
+ ready: Promise.resolve(),
61
+ sendMessage: (/** @type {Record<string, unknown>} */ body) => {
62
+ if (connectionId === null) throw new Error("Embedded lock server connection is not open")
63
+
64
+ engine.handleMessage(connectionId, body)
65
+ }
66
+ }
67
+
68
+ return connection
69
+ }
70
+ }
71
+
72
+ return client
73
+ }
74
+ }