velocious 1.0.753 → 1.0.754

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. package/README.md +1 -1
  2. package/build/background-jobs/adapter.js +3 -3
  3. package/build/background-jobs/client.js +6 -5
  4. package/build/background-jobs/local-adapter.js +3 -3
  5. package/build/background-jobs/local-store.js +10 -5
  6. package/build/background-jobs/main.js +87 -8
  7. package/build/background-jobs/refusal-error.js +87 -0
  8. package/build/background-jobs/store.js +109 -10
  9. package/build/background-jobs/types.js +5 -0
  10. package/build/src/background-jobs/adapter.d.ts +15 -6
  11. package/build/src/background-jobs/adapter.d.ts.map +1 -1
  12. package/build/src/background-jobs/adapter.js +4 -4
  13. package/build/src/background-jobs/client.d.ts.map +1 -1
  14. package/build/src/background-jobs/client.js +7 -6
  15. package/build/src/background-jobs/local-adapter.d.ts +8 -4
  16. package/build/src/background-jobs/local-adapter.d.ts.map +1 -1
  17. package/build/src/background-jobs/local-adapter.js +4 -4
  18. package/build/src/background-jobs/local-store.d.ts +12 -5
  19. package/build/src/background-jobs/local-store.d.ts.map +1 -1
  20. package/build/src/background-jobs/local-store.js +10 -6
  21. package/build/src/background-jobs/main.d.ts +46 -2
  22. package/build/src/background-jobs/main.d.ts.map +1 -1
  23. package/build/src/background-jobs/main.js +85 -9
  24. package/build/src/background-jobs/refusal-error.d.ts +57 -0
  25. package/build/src/background-jobs/refusal-error.d.ts.map +1 -0
  26. package/build/src/background-jobs/refusal-error.js +80 -0
  27. package/build/src/background-jobs/store.d.ts +43 -9
  28. package/build/src/background-jobs/store.d.ts.map +1 -1
  29. package/build/src/background-jobs/store.js +99 -11
  30. package/build/src/background-jobs/types.d.ts +10 -0
  31. package/build/src/background-jobs/types.d.ts.map +1 -1
  32. package/build/src/background-jobs/types.js +6 -1
  33. package/package.json +1 -1
  34. package/scripts/verify-docker-dev-environment.js +3 -3
  35. package/src/background-jobs/adapter.js +3 -3
  36. package/src/background-jobs/client.js +6 -5
  37. package/src/background-jobs/local-adapter.js +3 -3
  38. package/src/background-jobs/local-store.js +10 -5
  39. package/src/background-jobs/main.js +87 -8
  40. package/src/background-jobs/refusal-error.js +87 -0
  41. package/src/background-jobs/store.js +109 -10
  42. package/src/background-jobs/types.js +5 -0
package/README.md CHANGED
@@ -130,7 +130,7 @@ Maintainers cutting a package release must follow the [Velocious release runbook
130
130
 
131
131
  # Docker development environment
132
132
 
133
- The checked-in root `Dockerfile` and `compose.yml` define one canonical `dev` service used by humans, CI, and agent systems alike (see [docs/docker-development-environment.md](docs/docker-development-environment.md)). The image is Ubuntu 26.04 LTS (pinned by digest) with Node.js 24.x from signed NodeSource, the universal apt coding/debugging baseline, the newest published retained provider CLIs, and owner-pinned native Qwen Code 0.23.3; it is source-independent — no project source is copied and no project dependencies are installed at image build time.
133
+ The checked-in root `Dockerfile` and `compose.yml` define one canonical `dev` service used by humans, CI, and agent systems alike (see [docs/docker-development-environment.md](docs/docker-development-environment.md)). The image is Ubuntu 26.04 LTS (pinned by digest) with Node.js 24.x from signed NodeSource, the universal apt coding/debugging baseline, the newest published retained provider CLIs, and owner-pinned native Qwen Code 0.28.0; it is source-independent — no project source is copied and no project dependencies are installed at image build time.
134
134
 
135
135
  Prerequisites: Docker with the Compose v2 plugin, and this repository checked out at `$DEV_HOME_PATH/velocious` (default `DEV_HOME_PATH`: `/home/dev`).
136
136
 
@@ -174,8 +174,8 @@ export default class BackgroundJobsAdapter {
174
174
 
175
175
  /**
176
176
  * Returns a handed-off job to the queue.
177
- * @param {{jobId: string, handoffId: string}} _args - Handoff release.
178
- * @returns {Promise<void>} - Resolves after the job is returned.
177
+ * @param {{deferLiveOwners?: boolean, handoffId: string, hostname?: string, jobId: string, liveChildren?: import("./owner-liveness.js").BackgroundJobLiveChildIdentity[], probe?: (pid: number) => boolean}} _args - Handoff release and live-owner controls.
178
+ * @returns {Promise<boolean>} - Whether the lease was returned to the queue.
179
179
  */
180
180
  async markReturnedToQueue(_args) { throw new Error("BackgroundJobsAdapter#markReturnedToQueue is not implemented") }
181
181
 
@@ -197,7 +197,7 @@ export default class BackgroundJobsAdapter {
197
197
  * Applies orphan failure semantics to unchanged exact handoff snapshots.
198
198
  * Adapters that return startup snapshots must implement the matching fenced
199
199
  * transition.
200
- * @param {{handoffs: import("./types.js").BackgroundJobHandoffSnapshot[], error: ReturnType<typeof JSON.parse>}} _args - Exact leases and orphan reason.
200
+ * @param {{deferLiveOwners?: boolean, error: ReturnType<typeof JSON.parse>, handoffs: import("./types.js").BackgroundJobHandoffSnapshot[], hostname?: string, liveChildren?: import("./owner-liveness.js").BackgroundJobLiveChildIdentity[], onLiveOwnerDeferred?: (handoffs: import("./types.js").BackgroundJobDeferredLiveOwnerHandoff[]) => void | Promise<void>, probe?: (pid: number) => boolean}} _args - Exact leases, orphan reason, and live-owner controls.
201
201
  * @returns {Promise<import("./types.js").BackgroundJobRow[]>} - Accepted transitions.
202
202
  */
203
203
  async markOrphanedHandoffs(_args) { return [] }
@@ -8,6 +8,7 @@ import BackgroundJobsSocketRequest from "./socket-request.js"
8
8
  import { DEFAULT_GENERATION_HANDSHAKE_TIMEOUT_MS, validateGenerationHandshakeTimeoutMs } from "./generation-handshake-timeout-error.js"
9
9
  import { BACKGROUND_JOB_ACTIVE_STATUSES, BACKGROUND_JOB_EXECUTION_MODES, BACKGROUND_JOB_STATUSES, BACKGROUND_JOB_TERMINAL_STATUSES } from "./job-semantics.js"
10
10
  import {readBackgroundJobsEndpoint} from "./endpoint-file.js"
11
+ import {backgroundJobRefusalError} from "./refusal-error.js"
11
12
 
12
13
  const DEFAULT_ENQUEUE_TIMEOUT_MS = 5000
13
14
  const BACKGROUND_JOB_WAKE_OUTCOMES = ["woken", "already_due", "handed_off", "not_found"]
@@ -222,7 +223,7 @@ export default class BackgroundJobsClient {
222
223
 
223
224
  if (message?.type === "enqueue-error") {
224
225
  attemptObservation.explicitlyRejected = true
225
- reject(new Error(message.error || "Failed to enqueue job"))
226
+ reject(backgroundJobRefusalError({fallbackMessage: "Failed to enqueue job", message}))
226
227
  }
227
228
  }
228
229
  })
@@ -296,7 +297,7 @@ export default class BackgroundJobsClient {
296
297
  }
297
298
 
298
299
  if (message?.type === "replace-scheduled-error") {
299
- reject(new Error(message.error || "Failed to replace scheduled job"))
300
+ reject(backgroundJobRefusalError({fallbackMessage: "Failed to replace scheduled job", message}))
300
301
  }
301
302
  }
302
303
  })
@@ -322,7 +323,7 @@ export default class BackgroundJobsClient {
322
323
  }
323
324
 
324
325
  if (message?.type === "cancel-scheduled-error") {
325
- reject(new Error(message.error || "Failed to cancel scheduled job"))
326
+ reject(backgroundJobRefusalError({fallbackMessage: "Failed to cancel scheduled job", message}))
326
327
  }
327
328
  }
328
329
  })
@@ -358,7 +359,7 @@ export default class BackgroundJobsClient {
358
359
  }
359
360
 
360
361
  if (message?.type === "get-scheduled-job-error") {
361
- reject(new Error(message.error || "Failed to read scheduled job"))
362
+ reject(backgroundJobRefusalError({fallbackMessage: "Failed to read scheduled job", message}))
362
363
  return
363
364
  }
364
365
 
@@ -395,7 +396,7 @@ export default class BackgroundJobsClient {
395
396
  }
396
397
 
397
398
  if (message?.type === "wake-scheduled-error") {
398
- reject(new Error(message.error || "Failed to wake scheduled job"))
399
+ reject(backgroundJobRefusalError({fallbackMessage: "Failed to wake scheduled job", message}))
399
400
  return
400
401
  }
401
402
 
@@ -149,10 +149,10 @@ export default class LocalBackgroundJobsAdapter extends BackgroundJobsAdapter {
149
149
 
150
150
  /**
151
151
  * Returns an exact active local handoff to the queue.
152
- * @param {{jobId: string, handoffId: string}} args - Handoff release.
153
- * @returns {Promise<void>} - Resolves after the fenced release.
152
+ * @param {{deferLiveOwners?: boolean, handoffId: string, hostname?: string, jobId: string, liveChildren?: import("./owner-liveness.js").BackgroundJobLiveChildIdentity[], probe?: (pid: number) => boolean}} args - Handoff release and live-owner controls.
153
+ * @returns {Promise<boolean>} - Whether the lease was returned to the queue.
154
154
  */
155
- async markReturnedToQueue(args) { await this.store.markReturnedToQueue(args) }
155
+ async markReturnedToQueue(args) { return await this.store.markReturnedToQueue(args) }
156
156
 
157
157
  /**
158
158
  * Acknowledges successful local job completion.
@@ -1068,17 +1068,20 @@ export default class LocalBackgroundJobsStore {
1068
1068
  }
1069
1069
 
1070
1070
  /**
1071
- * Returns an exact active handoff to the queue.
1072
- * @param {{jobId: string, handoffId: string}} args - Handoff release.
1073
- * @returns {Promise<void>} - Resolves after the fenced release.
1071
+ * Returns an exact active handoff to the queue. The live-owner controls are
1072
+ * accepted for adapter compatibility and answer "released": this store has no
1073
+ * separate runner process to probe — its jobs execute in the dispatching
1074
+ * process, so a lost handoff here has no live executor to defer to.
1075
+ * @param {{deferLiveOwners?: boolean, handoffId: string, hostname?: string, jobId: string, liveChildren?: import("./owner-liveness.js").BackgroundJobLiveChildIdentity[], probe?: (pid: number) => boolean}} args - Handoff release and live-owner controls.
1076
+ * @returns {Promise<boolean>} - Whether the lease was returned to the queue.
1074
1077
  */
1075
1078
  async markReturnedToQueue({jobId, handoffId}) {
1076
1079
  await this.ensureReady()
1077
1080
 
1078
- await this._withDb(async (connection) => await this._mutate(connection, async (db) => {
1081
+ return await this._withDb(async (connection) => await this._mutate(connection, async (db) => {
1079
1082
  const job = await this._getJob(db, jobId)
1080
1083
 
1081
- if (!this._acceptsHandoff(job, handoffId)) return
1084
+ if (!this._acceptsHandoff(job, handoffId)) return false
1082
1085
  await this._lockConcurrencyRow(db, job.concurrencyKey)
1083
1086
 
1084
1087
  const affectedRows = await this._updateAffectedRows(db, {
@@ -1095,6 +1098,8 @@ export default class LocalBackgroundJobsStore {
1095
1098
  })
1096
1099
 
1097
1100
  if (affectedRows === 1) await this._releaseConcurrency(db, job.concurrencyKey)
1101
+
1102
+ return affectedRows === 1
1098
1103
  }))
1099
1104
  }
1100
1105
 
@@ -43,6 +43,8 @@ const WORKER_RECONNECT_GRACE_MS = 30000
43
43
  const GENERATION_ORPHANED_AFTER_MS = 60 * 60 * 1000
44
44
  /** Prefix of the per-generation liveness advisory lock one live main holds for its whole generation. */
45
45
  const GENERATION_LIVENESS_LOCK_PREFIX = "background-jobs:generation-liveness"
46
+ /** Maximum deferred-lease job ids named in one live-owner deferral log line. */
47
+ const LIVE_OWNER_DEFERRAL_JOB_ID_LIMIT = 20
46
48
  const WORKER_RECONNECT_GRACE_VALIDATION_MESSAGE = `workerReconnectGraceMs must be an integer between 0 and ${MAX_TIMER_MS}`
47
49
 
48
50
  /**
@@ -930,7 +932,13 @@ export default class BackgroundJobsMain {
930
932
  * acquire that generation's liveness lock on a dedicated connection and hold
931
933
  * it across the fenced store transition. A busy lock means a live main of
932
934
  * that generation still supervises its rows (retired-and-draining included),
933
- * so nothing here may touch them.
935
+ * so nothing here may touch them. An idle lock only proves the main is gone,
936
+ * not the work: a runner child of the retired generation can still be
937
+ * executing the job while its parent main has exited, so the store reclaim
938
+ * defers every lease whose recorded runner process is still alive. Those rows
939
+ * stay with the live runner and are settled by the dead-owner pass when it
940
+ * exits, instead of being requeued into a second attempt whose first is still
941
+ * running and failing its own producer fences.
934
942
  * @param {object} args - Options.
935
943
  * @param {string} args.foreignGenerationId - Generation whose rows are candidates.
936
944
  * @param {import("./types.js").BackgroundJobHandoffSnapshot[]} args.handoffs - Exact snapshot rows.
@@ -946,8 +954,11 @@ export default class BackgroundJobsMain {
946
954
  if (!acquired) return []
947
955
 
948
956
  return await this.store.markOrphanedHandoffs({
957
+ deferLiveOwners: true,
949
958
  error: "Job orphaned when a new generation activated while its owning generation was not live",
950
- handoffs
959
+ handoffs,
960
+ liveChildren: [...this.liveChildIdentities.values()],
961
+ onLiveOwnerDeferred: (deferred) => this._reportLiveOwnerHandoffDeferral({context: "cross-generation-settlement", handoffs: deferred})
951
962
  })
952
963
  } finally {
953
964
  try {
@@ -1044,8 +1055,11 @@ export default class BackgroundJobsMain {
1044
1055
 
1045
1056
  try {
1046
1057
  orphanedJobs = await this.store.markOrphanedHandoffs({
1058
+ deferLiveOwners: true,
1047
1059
  error: "Job orphaned after its pre-restart worker did not reconnect",
1048
- handoffs
1060
+ handoffs,
1061
+ liveChildren: [...this.liveChildIdentities.values()],
1062
+ onLiveOwnerDeferred: (deferred) => this._reportLiveOwnerHandoffDeferral({context: "startup-handoff-reclaim", handoffs: deferred})
1049
1063
  })
1050
1064
  } catch (error) {
1051
1065
  this._reportStartupHandoffReclaimError(error)
@@ -1690,13 +1704,35 @@ export default class BackgroundJobsMain {
1690
1704
  * @returns {Promise<void>} - Resolves after the fenced transition.
1691
1705
  */
1692
1706
  async _releaseHandoff({handoffId, jobId, worker}) {
1693
- await this.store.markReturnedToQueue({handoffId, jobId})
1707
+ await this._returnLeaseToQueue({handoffId, jobId})
1694
1708
 
1695
1709
  const handoffs = this.workerHandoffs.get(worker)
1696
1710
 
1697
1711
  if (handoffs?.get(jobId) === handoffId) handoffs.delete(jobId)
1698
1712
  }
1699
1713
 
1714
+ /**
1715
+ * Returns one exact lease to the queue, deferring the transition while the
1716
+ * recorded runner process is provably alive. A lost worker socket is not a
1717
+ * lost job: reclaiming a lease a live runner still holds starts a second
1718
+ * attempt beside the first, and the superseded attempt's owned follow-up
1719
+ * enqueues then all fail their producer fence. The in-memory lease is dropped
1720
+ * either way — the durable row keeps its exact owner, and a runner that
1721
+ * reconnects re-adopts it.
1722
+ * @param {object} args - Options.
1723
+ * @param {string} args.handoffId - Handoff lease id.
1724
+ * @param {string} args.jobId - Job id.
1725
+ * @returns {Promise<boolean>} - Whether the lease was returned to the queue.
1726
+ */
1727
+ async _returnLeaseToQueue({handoffId, jobId}) {
1728
+ return await this.store.markReturnedToQueue({
1729
+ deferLiveOwners: true,
1730
+ handoffId,
1731
+ jobId,
1732
+ liveChildren: [...this.liveChildIdentities.values()]
1733
+ })
1734
+ }
1735
+
1700
1736
  /**
1701
1737
  * Forgets an in-memory lease that no longer exists durably — a reported lease or
1702
1738
  * one settled as orphaned — without relying on worker ids.
@@ -1771,6 +1807,32 @@ export default class BackgroundJobsMain {
1771
1807
  errorEvents.emit("all-error", {...payload, errorType: "framework-error"})
1772
1808
  }
1773
1809
 
1810
+ /**
1811
+ * Records the leases an absence-based reclaim deliberately left alone because
1812
+ * their recorded runner process is still alive. A deferral is the healthy
1813
+ * outcome — the work is still running — but it must be visible: without it a
1814
+ * lease that stays `handed_off` looks like a stranded job, and the reason a
1815
+ * requeue was skipped would exist nowhere.
1816
+ * @param {object} args - Options.
1817
+ * @param {"cross-generation-settlement" | "generation-orphan-sweep" | "startup-handoff-reclaim"} args.context - Reclaim that deferred the leases.
1818
+ * @param {import("./types.js").BackgroundJobDeferredLiveOwnerHandoff[]} args.handoffs - Deferred leases with their measured liveness.
1819
+ * @returns {void}
1820
+ */
1821
+ _reportLiveOwnerHandoffDeferral({context, handoffs}) {
1822
+ const boundedJobIds = handoffs.slice(0, LIVE_OWNER_DEFERRAL_JOB_ID_LIMIT).map(({jobId}) => jobId)
1823
+
1824
+ this.logger.info(() => [
1825
+ "Left background job leases with their live runners instead of reclaiming them",
1826
+ {
1827
+ context,
1828
+ deferredCount: handoffs.length,
1829
+ jobIds: boundedJobIds,
1830
+ jobIdsTruncatedCount: Math.max(0, handoffs.length - boundedJobIds.length),
1831
+ livenessReasons: [...new Set(handoffs.map(({livenessReason}) => livenessReason))].sort()
1832
+ }
1833
+ ])
1834
+ }
1835
+
1774
1836
  /**
1775
1837
  * Runs handle enqueue.
1776
1838
  * @param {object} args - Options.
@@ -1926,6 +1988,9 @@ export default class BackgroundJobsMain {
1926
1988
 
1927
1989
  /**
1928
1990
  * Returns safe validation failures and reports unexpected client mutations.
1991
+ * A safe refusal keeps its machine code and measured details on the wire so
1992
+ * the requesting process can rebuild the same classification instead of
1993
+ * guessing from the message text.
1929
1994
  * @param {object} args - Options.
1930
1995
  * @param {Record<string, ReturnType<typeof JSON.parse>>} args.context - Framework-error context.
1931
1996
  * @param {ReturnType<typeof JSON.parse>} args.error - Mutation failure.
@@ -1937,7 +2002,12 @@ export default class BackgroundJobsMain {
1937
2002
  */
1938
2003
  _handleClientMutationError({context, error, fallbackMessage, jsonSocket, logMessage, responseType}) {
1939
2004
  if (error instanceof VelociousError && error.safeToExpose) {
1940
- jsonSocket.send({type: responseType, error: error.message})
2005
+ jsonSocket.send({
2006
+ type: responseType,
2007
+ error: error.message,
2008
+ ...(typeof error.code === "string" ? {code: error.code} : {}),
2009
+ ...(error.details ? {details: error.details} : {})
2010
+ })
1941
2011
  return
1942
2012
  }
1943
2013
 
@@ -2584,12 +2654,15 @@ export default class BackgroundJobsMain {
2584
2654
 
2585
2655
  /**
2586
2656
  * Returns one exact lease and forgets it only after the adapter acknowledges
2587
- * the fenced transition or confirms it was already absent.
2657
+ * the fenced transition or confirms it was already absent. A lease a live
2658
+ * runner already accepted is deferred by that transition: the dispatch that
2659
+ * main could not confirm may in fact have landed, and requeueing it beside a
2660
+ * running child is the duplicate this recovery is meant to avoid.
2588
2661
  * @param {{handoffId: string, jobId: string}} args - Exact recovery fence.
2589
2662
  * @returns {Promise<void>} - Resolves after durable recovery settles.
2590
2663
  */
2591
2664
  async _recoverHandoff({handoffId, jobId}) {
2592
- await this.store.markReturnedToQueue({handoffId, jobId})
2665
+ await this._returnLeaseToQueue({handoffId, jobId})
2593
2666
 
2594
2667
  if (this.pendingHandoffRecoveries.get(handoffId) === jobId) {
2595
2668
  this.pendingHandoffRecoveries.delete(handoffId)
@@ -2758,7 +2831,13 @@ export default class BackgroundJobsMain {
2758
2831
  })
2759
2832
  const generationOrphanedJobs = handoffs.length === 0
2760
2833
  ? []
2761
- : await this.store.markOrphanedHandoffs({handoffs, error: "Job orphaned after its generation owner disappeared"})
2834
+ : await this.store.markOrphanedHandoffs({
2835
+ deferLiveOwners: true,
2836
+ error: "Job orphaned after its generation owner disappeared",
2837
+ handoffs,
2838
+ liveChildren: [...this.liveChildIdentities.values()],
2839
+ onLiveOwnerDeferred: (deferred) => this._reportLiveOwnerHandoffDeferral({context: "generation-orphan-sweep", handoffs: deferred})
2840
+ })
2762
2841
  // Cross-generation rows are invisible to the fenced snapshot above; the
2763
2842
  // dead-owner pass settles them from the recorded runner process instead
2764
2843
  // of an age rule, so a retired generation can never strand a lease. The
@@ -0,0 +1,87 @@
1
+ // @ts-check
2
+
3
+ /** Machine code the jobs main answers a failed producer fence with. */
4
+ export const BACKGROUND_JOB_PRODUCER_HANDOFF_NOT_OWNED_ERROR_CODE = "background-job-producer-handoff-not-owned"
5
+
6
+ /**
7
+ * A refusal the jobs main classified and named on the other side of the socket.
8
+ * The reply used to carry only its text, which erased the framework's own
9
+ * classification: consumers could not tell a producer-handoff loss from any other
10
+ * enqueue failure without matching an English sentence, and the report they filed
11
+ * named no cause.
12
+ */
13
+ export class BackgroundJobRefusalError extends Error {
14
+ /**
15
+ * Builds a refusal from the code and measured details the main sent.
16
+ * @param {object} args - Refusal.
17
+ * @param {string} args.code - Stable machine code of the refusal.
18
+ * @param {Record<string, string | number>} [args.details] - Measured details recorded with the refusal.
19
+ * @param {string} args.message - Text the main exposed for this refusal.
20
+ */
21
+ constructor({code, details, message}) {
22
+ super(message)
23
+
24
+ this.name = "BackgroundJobRefusalError"
25
+ this.code = code
26
+ /** @type {Readonly<Record<string, string | number>> | null} */
27
+ this.details = details ? Object.freeze({...details}) : null
28
+ }
29
+ }
30
+
31
+ /**
32
+ * The producer handoff behind an owned follow-up enqueue is no longer owned, so
33
+ * the main refused to create the follow-up. This is a lifecycle outcome of the
34
+ * jobs main reclaiming the producer, not a defect in the calling feature.
35
+ */
36
+ export class BackgroundJobProducerHandoffLostError extends BackgroundJobRefusalError {
37
+ /**
38
+ * Builds the refusal from the code and measured details the main sent.
39
+ * @param {object} args - Refusal.
40
+ * @param {Record<string, string | number>} [args.details] - Loss reason and the producer state it was measured from.
41
+ * @param {string} args.message - Text the main exposed for this refusal.
42
+ */
43
+ constructor({details, message}) {
44
+ super({code: BACKGROUND_JOB_PRODUCER_HANDOFF_NOT_OWNED_ERROR_CODE, details, message})
45
+
46
+ this.name = "BackgroundJobProducerHandoffLostError"
47
+ }
48
+ }
49
+
50
+ /**
51
+ * Rebuilds one classified refusal from the jobs main. A reply that carries no
52
+ * code keeps its previous plain-error shape so text-only consumers are
53
+ * unaffected.
54
+ * @param {object} args - Options.
55
+ * @param {ReturnType<typeof JSON.parse>} args.message - Refusal message from the main.
56
+ * @param {string} args.fallbackMessage - Text for a reply that carries none.
57
+ * @returns {Error} - Typed refusal when classified, otherwise a plain error with the same text.
58
+ */
59
+ export function backgroundJobRefusalError({fallbackMessage, message}) {
60
+ const text = typeof message?.error === "string" ? message.error : fallbackMessage
61
+ const code = typeof message?.code === "string" ? message.code : null
62
+ const details = backgroundJobRefusalDetails(message?.details)
63
+
64
+ if (!code) return new Error(text)
65
+ if (code === BACKGROUND_JOB_PRODUCER_HANDOFF_NOT_OWNED_ERROR_CODE) return new BackgroundJobProducerHandoffLostError({details, message: text})
66
+
67
+ return new BackgroundJobRefusalError({code, details, message: text})
68
+ }
69
+
70
+ /**
71
+ * Copies the transport-safe detail fields of a refusal, dropping anything that
72
+ * is not a primitive the main is allowed to expose.
73
+ * @param {ReturnType<typeof JSON.parse>} details - Candidate details payload.
74
+ * @returns {Record<string, string | number> | undefined} - Safe details, when any.
75
+ */
76
+ function backgroundJobRefusalDetails(details) {
77
+ if (!details || typeof details !== "object" || Array.isArray(details)) return undefined
78
+
79
+ /** @type {Record<string, string | number>} */
80
+ const safeDetails = {}
81
+
82
+ for (const [key, value] of Object.entries(details)) {
83
+ if (typeof value === "string" || typeof value === "number") safeDetails[key] = value
84
+ }
85
+
86
+ return Object.keys(safeDetails).length > 0 ? safeDetails : undefined
87
+ }
@@ -1019,11 +1019,49 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
1019
1019
 
1020
1020
  if (!owned) {
1021
1021
  throw VelociousError.safe("Background job producer handoff is no longer owned.", {
1022
- code: "background-job-producer-handoff-not-owned"
1022
+ code: "background-job-producer-handoff-not-owned",
1023
+ details: this._producerHandoffLossDetails(producer, producerProof)
1023
1024
  })
1024
1025
  }
1025
1026
  }
1026
1027
 
1028
+ /**
1029
+ * Names, from the row just read inside the fence's own transaction, why the
1030
+ * producer proof no longer matches. The refusal reaches consumers over the
1031
+ * jobs-main socket as text only, so without this the same lifecycle loss is
1032
+ * indistinguishable from an unexplained enqueue failure.
1033
+ * @param {import("./types.js").BackgroundJobRow | null} producer - Producer row, or null when it is gone.
1034
+ * @param {import("./types.js").BackgroundJobProducerProof} producerProof - Proof the producer execution holds.
1035
+ * @returns {Record<string, string | number>} - Measured loss reason and the state it was measured from.
1036
+ */
1037
+ _producerHandoffLossDetails(producer, producerProof) {
1038
+ if (!producer) {
1039
+ return {
1040
+ producerJobId: producerProof.jobId,
1041
+ producerStatus: "missing",
1042
+ reason: "producer_row_missing"
1043
+ }
1044
+ }
1045
+
1046
+ if (producer.status !== "handed_off") {
1047
+ return {
1048
+ producerJobId: producerProof.jobId,
1049
+ producerStatus: producer.status,
1050
+ reason: `producer_${producer.status}`
1051
+ }
1052
+ }
1053
+
1054
+ return {
1055
+ producerJobId: producerProof.jobId,
1056
+ producerStatus: producer.status,
1057
+ producerWorkerId: producer.workerId || "",
1058
+ proofWorkerId: producerProof.workerId,
1059
+ reason: producer.handoffId === producerProof.handoffId
1060
+ ? "producer_lease_restarted"
1061
+ : "producer_lease_replaced"
1062
+ }
1063
+ }
1064
+
1027
1065
  /**
1028
1066
  * Replaces the queued owner of a stable schedule key with a new one-off job.
1029
1067
  * A handed-off owner is left running and reported truthfully.
@@ -1810,18 +1848,43 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
1810
1848
  }
1811
1849
 
1812
1850
  /**
1813
- * Runs mark returned to queue.
1851
+ * Returns an active handoff to the queue when its owning worker went away,
1852
+ * unless the recorded runner process is still provably alive and executing the
1853
+ * job. Absence of the worker's socket is not absence of the work: reclaiming a
1854
+ * lease a live runner still holds starts a second attempt while the first keeps
1855
+ * running, and every owned follow-up enqueue from the superseded attempt then
1856
+ * fails its producer fence. A deferred lease is settled by the dead-owner pass
1857
+ * once the runner actually exits.
1814
1858
  * @param {object} args - Options.
1859
+ * @param {boolean} [args.deferLiveOwners] - Whether a provably alive recorded runner keeps the lease instead of being returned to the queue.
1860
+ * @param {string} [args.handoffId] - Handoff lease id.
1861
+ * @param {string} [args.hostname] - Hostname whose local probe answers runner ownership, defaults to this process's hostname.
1815
1862
  * @param {string} args.jobId - Job id.
1816
- * @param {string} args.handoffId - Handoff lease id.
1817
- * @returns {Promise<void>} - Resolves when updated.
1863
+ * @param {import("./owner-liveness.js").BackgroundJobLiveChildIdentity[]} [args.liveChildren] - Runner identities this caller observes as live, for the pid-recycling check.
1864
+ * @param {(pid: number) => boolean} [args.probe] - Process probe for the live-owner check, injectable for tests.
1865
+ * @returns {Promise<boolean>} - Whether the lease was returned to the queue.
1818
1866
  */
1819
- async markReturnedToQueue({jobId, handoffId}) {
1867
+ async markReturnedToQueue({deferLiveOwners = false, handoffId, hostname, jobId, liveChildren = [], probe = isProcessAlive}) {
1820
1868
  await this.ensureReady()
1821
1869
 
1822
- await this._serializedCountMutation(async (db) => {
1870
+ return await this._serializedCountMutation(async (db) => {
1823
1871
  const job = await this._getJobRowById(db, jobId)
1824
- if (!job || job.handoffId !== handoffId || job.status !== "handed_off") return
1872
+
1873
+ if (!job || job.handoffId !== handoffId || job.status !== "handed_off") return false
1874
+
1875
+ if (deferLiveOwners) {
1876
+ const liveness = evaluateBackgroundJobOwnerLiveness({
1877
+ childInstanceId: job.childInstanceId,
1878
+ childPid: job.childPid,
1879
+ currentHostname: hostname ?? os.hostname(),
1880
+ liveChildren,
1881
+ probe,
1882
+ runnerHostname: job.runnerHostname
1883
+ })
1884
+
1885
+ if (liveness.status === "alive") return false
1886
+ }
1887
+
1825
1888
  await this._lockConcurrencyRows(db, this._jobConcurrencyKeys(job))
1826
1889
  const affectedRows = await this._updateAffectedRows(db, {
1827
1890
  tableName: JOBS_TABLE,
@@ -1838,7 +1901,11 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
1838
1901
  if (affectedRows === 1) {
1839
1902
  await this._releaseJobConcurrency(db, job)
1840
1903
  await this._recordStatusTransition(db, "handed_off", "queued")
1904
+
1905
+ return true
1841
1906
  }
1907
+
1908
+ return false
1842
1909
  })
1843
1910
  }
1844
1911
 
@@ -1914,14 +1981,21 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
1914
1981
  * snapshot. The ordinary orphan failure path owns retries, terminal status,
1915
1982
  * count transitions, schedule ownership, and concurrency release.
1916
1983
  * @param {object} args - Options.
1917
- * @param {import("./types.js").BackgroundJobHandoffSnapshot[]} args.handoffs - Exact startup snapshots.
1984
+ * @param {(handoffs: import("./types.js").BackgroundJobDeferredLiveOwnerHandoff[]) => void | Promise<void>} [args.onLiveOwnerDeferred] - Observation hook for leases a live runner still owns, called after the reclaim transaction settles.
1918
1985
  * @param {ReturnType<typeof JSON.parse>} args.error - Orphan reason.
1986
+ * @param {import("./types.js").BackgroundJobHandoffSnapshot[]} args.handoffs - Exact startup snapshots.
1987
+ * @param {string} [args.hostname] - Hostname whose local probe answers runner ownership, defaults to this process's hostname.
1988
+ * @param {boolean} [args.deferLiveOwners] - Whether a lease whose recorded runner process is provably alive stays with that runner instead of being reclaimed. Use this for every absence-based reclaim; leave it off once death is proven.
1989
+ * @param {import("./owner-liveness.js").BackgroundJobLiveChildIdentity[]} [args.liveChildren] - Runner identities this caller observes as live, for the pid-recycling check.
1990
+ * @param {(pid: number) => boolean} [args.probe] - Process probe for the live-owner check, injectable for tests.
1919
1991
  * @returns {Promise<import("./types.js").BackgroundJobRow[]>} - Accepted transitions.
1920
1992
  */
1921
- async markOrphanedHandoffs({handoffs, error}) {
1993
+ async markOrphanedHandoffs({deferLiveOwners = false, error, handoffs, hostname, liveChildren = [], onLiveOwnerDeferred, probe = isProcessAlive}) {
1922
1994
  await this.ensureReady()
1923
1995
 
1924
- return await this._serializedCountMutation(async (db) => {
1996
+ /** @type {import("./types.js").BackgroundJobDeferredLiveOwnerHandoff[]} */
1997
+ const deferredLiveOwnerHandoffs = []
1998
+ const orphanedJobs = await this._serializedCountMutation(async (db) => {
1925
1999
  /** @type {BackgroundJobOrphanSelection[]} */
1926
2000
  const selections = []
1927
2001
 
@@ -1933,6 +2007,27 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
1933
2007
  if (job.workerId !== handoff.workerId) continue
1934
2008
  if (job.handedOffAtMs !== handoff.handedOffAtMs) continue
1935
2009
 
2010
+ if (deferLiveOwners) {
2011
+ const liveness = evaluateBackgroundJobOwnerLiveness({
2012
+ childInstanceId: job.childInstanceId,
2013
+ childPid: job.childPid,
2014
+ currentHostname: hostname ?? os.hostname(),
2015
+ liveChildren,
2016
+ probe,
2017
+ runnerHostname: job.runnerHostname
2018
+ })
2019
+
2020
+ if (liveness.status === "alive") {
2021
+ deferredLiveOwnerHandoffs.push({
2022
+ ...handoff,
2023
+ childPid: liveness.childPid,
2024
+ livenessReason: liveness.reason,
2025
+ runnerHostname: liveness.runnerHostname
2026
+ })
2027
+ continue
2028
+ }
2029
+ }
2030
+
1936
2031
  selections.push({
1937
2032
  conditions: {
1938
2033
  handed_off_at_ms: handoff.handedOffAtMs,
@@ -1948,6 +2043,10 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
1948
2043
 
1949
2044
  return await this._markOrphanSelections({db, selections})
1950
2045
  })
2046
+
2047
+ if (deferredLiveOwnerHandoffs.length > 0 && onLiveOwnerDeferred) await onLiveOwnerDeferred(deferredLiveOwnerHandoffs)
2048
+
2049
+ return orphanedJobs
1951
2050
  }
1952
2051
 
1953
2052
  /**
@@ -188,6 +188,11 @@
188
188
  /**
189
189
  * @typedef {BackgroundJobHandoffSnapshot & {liveness: import("./owner-liveness.js").BackgroundJobOwnerLivenessResult}} BackgroundJobHandoffOwnershipVerdict
190
190
  */
191
+ /**
192
+ * One exact lease an absence-based reclaim left with its runner because the
193
+ * recorded runner process is provably alive.
194
+ * @typedef {BackgroundJobHandoffSnapshot & {childPid: number | null, livenessReason: string, runnerHostname: string | null}} BackgroundJobDeferredLiveOwnerHandoff
195
+ */
191
196
  /**
192
197
  * @typedef {object} BackgroundJobHandoffRequest
193
198
  * @property {string} jobId - Job to claim.
@@ -192,13 +192,17 @@ export default class BackgroundJobsAdapter {
192
192
  }): Promise<boolean>;
193
193
  /**
194
194
  * Returns a handed-off job to the queue.
195
- * @param {{jobId: string, handoffId: string}} _args - Handoff release.
196
- * @returns {Promise<void>} - Resolves after the job is returned.
195
+ * @param {{deferLiveOwners?: boolean, handoffId: string, hostname?: string, jobId: string, liveChildren?: import("./owner-liveness.js").BackgroundJobLiveChildIdentity[], probe?: (pid: number) => boolean}} _args - Handoff release and live-owner controls.
196
+ * @returns {Promise<boolean>} - Whether the lease was returned to the queue.
197
197
  */
198
198
  markReturnedToQueue(_args: {
199
- jobId: string;
199
+ deferLiveOwners?: boolean;
200
200
  handoffId: string;
201
- }): Promise<void>;
201
+ hostname?: string;
202
+ jobId: string;
203
+ liveChildren?: import("./owner-liveness.js").BackgroundJobLiveChildIdentity[];
204
+ probe?: (pid: number) => boolean;
205
+ }): Promise<boolean>;
202
206
  /**
203
207
  * Finds active handoffs for a worker.
204
208
  * @param {{workerId: string}} _args - Worker identity.
@@ -220,12 +224,17 @@ export default class BackgroundJobsAdapter {
220
224
  * Applies orphan failure semantics to unchanged exact handoff snapshots.
221
225
  * Adapters that return startup snapshots must implement the matching fenced
222
226
  * transition.
223
- * @param {{handoffs: import("./types.js").BackgroundJobHandoffSnapshot[], error: ReturnType<typeof JSON.parse>}} _args - Exact leases and orphan reason.
227
+ * @param {{deferLiveOwners?: boolean, error: ReturnType<typeof JSON.parse>, handoffs: import("./types.js").BackgroundJobHandoffSnapshot[], hostname?: string, liveChildren?: import("./owner-liveness.js").BackgroundJobLiveChildIdentity[], onLiveOwnerDeferred?: (handoffs: import("./types.js").BackgroundJobDeferredLiveOwnerHandoff[]) => void | Promise<void>, probe?: (pid: number) => boolean}} _args - Exact leases, orphan reason, and live-owner controls.
224
228
  * @returns {Promise<import("./types.js").BackgroundJobRow[]>} - Accepted transitions.
225
229
  */
226
230
  markOrphanedHandoffs(_args: {
227
- handoffs: import("./types.js").BackgroundJobHandoffSnapshot[];
231
+ deferLiveOwners?: boolean;
228
232
  error: ReturnType<typeof JSON.parse>;
233
+ handoffs: import("./types.js").BackgroundJobHandoffSnapshot[];
234
+ hostname?: string;
235
+ liveChildren?: import("./owner-liveness.js").BackgroundJobLiveChildIdentity[];
236
+ onLiveOwnerDeferred?: (handoffs: import("./types.js").BackgroundJobDeferredLiveOwnerHandoff[]) => void | Promise<void>;
237
+ probe?: (pid: number) => boolean;
229
238
  }): Promise<import("./types.js").BackgroundJobRow[]>;
230
239
  /**
231
240
  * Evaluates recorded runner-process liveness for exact active handoffs.