velocious 1.0.752 → 1.0.754

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. package/README.md +1 -1
  2. package/build/background-jobs/adapter.js +3 -3
  3. package/build/background-jobs/client.js +6 -5
  4. package/build/background-jobs/endpoint-unavailable-error.js +21 -0
  5. package/build/background-jobs/local-adapter.js +3 -3
  6. package/build/background-jobs/local-store.js +10 -5
  7. package/build/background-jobs/main.js +87 -8
  8. package/build/background-jobs/refusal-error.js +87 -0
  9. package/build/background-jobs/socket-request.js +7 -0
  10. package/build/background-jobs/store.js +109 -10
  11. package/build/background-jobs/types.js +5 -0
  12. package/build/src/background-jobs/adapter.d.ts +15 -6
  13. package/build/src/background-jobs/adapter.d.ts.map +1 -1
  14. package/build/src/background-jobs/adapter.js +4 -4
  15. package/build/src/background-jobs/client.d.ts.map +1 -1
  16. package/build/src/background-jobs/client.js +7 -6
  17. package/build/src/background-jobs/endpoint-unavailable-error.d.ts +20 -0
  18. package/build/src/background-jobs/endpoint-unavailable-error.d.ts.map +1 -0
  19. package/build/src/background-jobs/endpoint-unavailable-error.js +20 -0
  20. package/build/src/background-jobs/local-adapter.d.ts +8 -4
  21. package/build/src/background-jobs/local-adapter.d.ts.map +1 -1
  22. package/build/src/background-jobs/local-adapter.js +4 -4
  23. package/build/src/background-jobs/local-store.d.ts +12 -5
  24. package/build/src/background-jobs/local-store.d.ts.map +1 -1
  25. package/build/src/background-jobs/local-store.js +10 -6
  26. package/build/src/background-jobs/main.d.ts +46 -2
  27. package/build/src/background-jobs/main.d.ts.map +1 -1
  28. package/build/src/background-jobs/main.js +85 -9
  29. package/build/src/background-jobs/refusal-error.d.ts +57 -0
  30. package/build/src/background-jobs/refusal-error.d.ts.map +1 -0
  31. package/build/src/background-jobs/refusal-error.js +80 -0
  32. package/build/src/background-jobs/socket-request.d.ts.map +1 -1
  33. package/build/src/background-jobs/socket-request.js +7 -1
  34. package/build/src/background-jobs/store.d.ts +43 -9
  35. package/build/src/background-jobs/store.d.ts.map +1 -1
  36. package/build/src/background-jobs/store.js +99 -11
  37. package/build/src/background-jobs/types.d.ts +10 -0
  38. package/build/src/background-jobs/types.d.ts.map +1 -1
  39. package/build/src/background-jobs/types.js +6 -1
  40. package/package.json +1 -1
  41. package/scripts/verify-docker-dev-environment.js +3 -3
  42. package/src/background-jobs/adapter.js +3 -3
  43. package/src/background-jobs/client.js +6 -5
  44. package/src/background-jobs/endpoint-unavailable-error.js +21 -0
  45. package/src/background-jobs/local-adapter.js +3 -3
  46. package/src/background-jobs/local-store.js +10 -5
  47. package/src/background-jobs/main.js +87 -8
  48. package/src/background-jobs/refusal-error.js +87 -0
  49. package/src/background-jobs/socket-request.js +7 -0
  50. package/src/background-jobs/store.js +109 -10
  51. package/src/background-jobs/types.js +5 -0
@@ -43,6 +43,8 @@ const WORKER_RECONNECT_GRACE_MS = 30000
43
43
  const GENERATION_ORPHANED_AFTER_MS = 60 * 60 * 1000
44
44
  /** Prefix of the per-generation liveness advisory lock one live main holds for its whole generation. */
45
45
  const GENERATION_LIVENESS_LOCK_PREFIX = "background-jobs:generation-liveness"
46
+ /** Maximum deferred-lease job ids named in one live-owner deferral log line. */
47
+ const LIVE_OWNER_DEFERRAL_JOB_ID_LIMIT = 20
46
48
  const WORKER_RECONNECT_GRACE_VALIDATION_MESSAGE = `workerReconnectGraceMs must be an integer between 0 and ${MAX_TIMER_MS}`
47
49
 
48
50
  /**
@@ -930,7 +932,13 @@ export default class BackgroundJobsMain {
930
932
  * acquire that generation's liveness lock on a dedicated connection and hold
931
933
  * it across the fenced store transition. A busy lock means a live main of
932
934
  * that generation still supervises its rows (retired-and-draining included),
933
- * so nothing here may touch them.
935
+ * so nothing here may touch them. An idle lock only proves the main is gone,
936
+ * not the work: a runner child of the retired generation can still be
937
+ * executing the job while its parent main has exited, so the store reclaim
938
+ * defers every lease whose recorded runner process is still alive. Those rows
939
+ * stay with the live runner and are settled by the dead-owner pass when it
940
+ * exits, instead of being requeued into a second attempt whose first is still
941
+ * running and failing its own producer fences.
934
942
  * @param {object} args - Options.
935
943
  * @param {string} args.foreignGenerationId - Generation whose rows are candidates.
936
944
  * @param {import("./types.js").BackgroundJobHandoffSnapshot[]} args.handoffs - Exact snapshot rows.
@@ -946,8 +954,11 @@ export default class BackgroundJobsMain {
946
954
  if (!acquired) return []
947
955
 
948
956
  return await this.store.markOrphanedHandoffs({
957
+ deferLiveOwners: true,
949
958
  error: "Job orphaned when a new generation activated while its owning generation was not live",
950
- handoffs
959
+ handoffs,
960
+ liveChildren: [...this.liveChildIdentities.values()],
961
+ onLiveOwnerDeferred: (deferred) => this._reportLiveOwnerHandoffDeferral({context: "cross-generation-settlement", handoffs: deferred})
951
962
  })
952
963
  } finally {
953
964
  try {
@@ -1044,8 +1055,11 @@ export default class BackgroundJobsMain {
1044
1055
 
1045
1056
  try {
1046
1057
  orphanedJobs = await this.store.markOrphanedHandoffs({
1058
+ deferLiveOwners: true,
1047
1059
  error: "Job orphaned after its pre-restart worker did not reconnect",
1048
- handoffs
1060
+ handoffs,
1061
+ liveChildren: [...this.liveChildIdentities.values()],
1062
+ onLiveOwnerDeferred: (deferred) => this._reportLiveOwnerHandoffDeferral({context: "startup-handoff-reclaim", handoffs: deferred})
1049
1063
  })
1050
1064
  } catch (error) {
1051
1065
  this._reportStartupHandoffReclaimError(error)
@@ -1690,13 +1704,35 @@ export default class BackgroundJobsMain {
1690
1704
  * @returns {Promise<void>} - Resolves after the fenced transition.
1691
1705
  */
1692
1706
  async _releaseHandoff({handoffId, jobId, worker}) {
1693
- await this.store.markReturnedToQueue({handoffId, jobId})
1707
+ await this._returnLeaseToQueue({handoffId, jobId})
1694
1708
 
1695
1709
  const handoffs = this.workerHandoffs.get(worker)
1696
1710
 
1697
1711
  if (handoffs?.get(jobId) === handoffId) handoffs.delete(jobId)
1698
1712
  }
1699
1713
 
1714
+ /**
1715
+ * Returns one exact lease to the queue, deferring the transition while the
1716
+ * recorded runner process is provably alive. A lost worker socket is not a
1717
+ * lost job: reclaiming a lease a live runner still holds starts a second
1718
+ * attempt beside the first, and the superseded attempt's owned follow-up
1719
+ * enqueues then all fail their producer fence. The in-memory lease is dropped
1720
+ * either way — the durable row keeps its exact owner, and a runner that
1721
+ * reconnects re-adopts it.
1722
+ * @param {object} args - Options.
1723
+ * @param {string} args.handoffId - Handoff lease id.
1724
+ * @param {string} args.jobId - Job id.
1725
+ * @returns {Promise<boolean>} - Whether the lease was returned to the queue.
1726
+ */
1727
+ async _returnLeaseToQueue({handoffId, jobId}) {
1728
+ return await this.store.markReturnedToQueue({
1729
+ deferLiveOwners: true,
1730
+ handoffId,
1731
+ jobId,
1732
+ liveChildren: [...this.liveChildIdentities.values()]
1733
+ })
1734
+ }
1735
+
1700
1736
  /**
1701
1737
  * Forgets an in-memory lease that no longer exists durably — a reported lease or
1702
1738
  * one settled as orphaned — without relying on worker ids.
@@ -1771,6 +1807,32 @@ export default class BackgroundJobsMain {
1771
1807
  errorEvents.emit("all-error", {...payload, errorType: "framework-error"})
1772
1808
  }
1773
1809
 
1810
+ /**
1811
+ * Records the leases an absence-based reclaim deliberately left alone because
1812
+ * their recorded runner process is still alive. A deferral is the healthy
1813
+ * outcome — the work is still running — but it must be visible: without it a
1814
+ * lease that stays `handed_off` looks like a stranded job, and the reason a
1815
+ * requeue was skipped would exist nowhere.
1816
+ * @param {object} args - Options.
1817
+ * @param {"cross-generation-settlement" | "generation-orphan-sweep" | "startup-handoff-reclaim"} args.context - Reclaim that deferred the leases.
1818
+ * @param {import("./types.js").BackgroundJobDeferredLiveOwnerHandoff[]} args.handoffs - Deferred leases with their measured liveness.
1819
+ * @returns {void}
1820
+ */
1821
+ _reportLiveOwnerHandoffDeferral({context, handoffs}) {
1822
+ const boundedJobIds = handoffs.slice(0, LIVE_OWNER_DEFERRAL_JOB_ID_LIMIT).map(({jobId}) => jobId)
1823
+
1824
+ this.logger.info(() => [
1825
+ "Left background job leases with their live runners instead of reclaiming them",
1826
+ {
1827
+ context,
1828
+ deferredCount: handoffs.length,
1829
+ jobIds: boundedJobIds,
1830
+ jobIdsTruncatedCount: Math.max(0, handoffs.length - boundedJobIds.length),
1831
+ livenessReasons: [...new Set(handoffs.map(({livenessReason}) => livenessReason))].sort()
1832
+ }
1833
+ ])
1834
+ }
1835
+
1774
1836
  /**
1775
1837
  * Runs handle enqueue.
1776
1838
  * @param {object} args - Options.
@@ -1926,6 +1988,9 @@ export default class BackgroundJobsMain {
1926
1988
 
1927
1989
  /**
1928
1990
  * Returns safe validation failures and reports unexpected client mutations.
1991
+ * A safe refusal keeps its machine code and measured details on the wire so
1992
+ * the requesting process can rebuild the same classification instead of
1993
+ * guessing from the message text.
1929
1994
  * @param {object} args - Options.
1930
1995
  * @param {Record<string, ReturnType<typeof JSON.parse>>} args.context - Framework-error context.
1931
1996
  * @param {ReturnType<typeof JSON.parse>} args.error - Mutation failure.
@@ -1937,7 +2002,12 @@ export default class BackgroundJobsMain {
1937
2002
  */
1938
2003
  _handleClientMutationError({context, error, fallbackMessage, jsonSocket, logMessage, responseType}) {
1939
2004
  if (error instanceof VelociousError && error.safeToExpose) {
1940
- jsonSocket.send({type: responseType, error: error.message})
2005
+ jsonSocket.send({
2006
+ type: responseType,
2007
+ error: error.message,
2008
+ ...(typeof error.code === "string" ? {code: error.code} : {}),
2009
+ ...(error.details ? {details: error.details} : {})
2010
+ })
1941
2011
  return
1942
2012
  }
1943
2013
 
@@ -2584,12 +2654,15 @@ export default class BackgroundJobsMain {
2584
2654
 
2585
2655
  /**
2586
2656
  * Returns one exact lease and forgets it only after the adapter acknowledges
2587
- * the fenced transition or confirms it was already absent.
2657
+ * the fenced transition or confirms it was already absent. A lease a live
2658
+ * runner already accepted is deferred by that transition: the dispatch that
2659
+ * main could not confirm may in fact have landed, and requeueing it beside a
2660
+ * running child is the duplicate this recovery is meant to avoid.
2588
2661
  * @param {{handoffId: string, jobId: string}} args - Exact recovery fence.
2589
2662
  * @returns {Promise<void>} - Resolves after durable recovery settles.
2590
2663
  */
2591
2664
  async _recoverHandoff({handoffId, jobId}) {
2592
- await this.store.markReturnedToQueue({handoffId, jobId})
2665
+ await this._returnLeaseToQueue({handoffId, jobId})
2593
2666
 
2594
2667
  if (this.pendingHandoffRecoveries.get(handoffId) === jobId) {
2595
2668
  this.pendingHandoffRecoveries.delete(handoffId)
@@ -2758,7 +2831,13 @@ export default class BackgroundJobsMain {
2758
2831
  })
2759
2832
  const generationOrphanedJobs = handoffs.length === 0
2760
2833
  ? []
2761
- : await this.store.markOrphanedHandoffs({handoffs, error: "Job orphaned after its generation owner disappeared"})
2834
+ : await this.store.markOrphanedHandoffs({
2835
+ deferLiveOwners: true,
2836
+ error: "Job orphaned after its generation owner disappeared",
2837
+ handoffs,
2838
+ liveChildren: [...this.liveChildIdentities.values()],
2839
+ onLiveOwnerDeferred: (deferred) => this._reportLiveOwnerHandoffDeferral({context: "generation-orphan-sweep", handoffs: deferred})
2840
+ })
2762
2841
  // Cross-generation rows are invisible to the fenced snapshot above; the
2763
2842
  // dead-owner pass settles them from the recorded runner process instead
2764
2843
  // of an age rule, so a retired generation can never strand a lease. The
@@ -0,0 +1,87 @@
1
+ // @ts-check
2
+
3
+ /** Machine code the jobs main answers a failed producer fence with. */
4
+ export const BACKGROUND_JOB_PRODUCER_HANDOFF_NOT_OWNED_ERROR_CODE = "background-job-producer-handoff-not-owned"
5
+
6
+ /**
7
+ * A refusal the jobs main classified and named on the other side of the socket.
8
+ * The reply used to carry only its text, which erased the framework's own
9
+ * classification: consumers could not tell a producer-handoff loss from any other
10
+ * enqueue failure without matching an English sentence, and the report they filed
11
+ * named no cause.
12
+ */
13
+ export class BackgroundJobRefusalError extends Error {
14
+ /**
15
+ * Builds a refusal from the code and measured details the main sent.
16
+ * @param {object} args - Refusal.
17
+ * @param {string} args.code - Stable machine code of the refusal.
18
+ * @param {Record<string, string | number>} [args.details] - Measured details recorded with the refusal.
19
+ * @param {string} args.message - Text the main exposed for this refusal.
20
+ */
21
+ constructor({code, details, message}) {
22
+ super(message)
23
+
24
+ this.name = "BackgroundJobRefusalError"
25
+ this.code = code
26
+ /** @type {Readonly<Record<string, string | number>> | null} */
27
+ this.details = details ? Object.freeze({...details}) : null
28
+ }
29
+ }
30
+
31
+ /**
32
+ * The producer handoff behind an owned follow-up enqueue is no longer owned, so
33
+ * the main refused to create the follow-up. This is a lifecycle outcome of the
34
+ * jobs main reclaiming the producer, not a defect in the calling feature.
35
+ */
36
+ export class BackgroundJobProducerHandoffLostError extends BackgroundJobRefusalError {
37
+ /**
38
+ * Builds the refusal from the code and measured details the main sent.
39
+ * @param {object} args - Refusal.
40
+ * @param {Record<string, string | number>} [args.details] - Loss reason and the producer state it was measured from.
41
+ * @param {string} args.message - Text the main exposed for this refusal.
42
+ */
43
+ constructor({details, message}) {
44
+ super({code: BACKGROUND_JOB_PRODUCER_HANDOFF_NOT_OWNED_ERROR_CODE, details, message})
45
+
46
+ this.name = "BackgroundJobProducerHandoffLostError"
47
+ }
48
+ }
49
+
50
+ /**
51
+ * Rebuilds one classified refusal from the jobs main. A reply that carries no
52
+ * code keeps its previous plain-error shape so text-only consumers are
53
+ * unaffected.
54
+ * @param {object} args - Options.
55
+ * @param {ReturnType<typeof JSON.parse>} args.message - Refusal message from the main.
56
+ * @param {string} args.fallbackMessage - Text for a reply that carries none.
57
+ * @returns {Error} - Typed refusal when classified, otherwise a plain error with the same text.
58
+ */
59
+ export function backgroundJobRefusalError({fallbackMessage, message}) {
60
+ const text = typeof message?.error === "string" ? message.error : fallbackMessage
61
+ const code = typeof message?.code === "string" ? message.code : null
62
+ const details = backgroundJobRefusalDetails(message?.details)
63
+
64
+ if (!code) return new Error(text)
65
+ if (code === BACKGROUND_JOB_PRODUCER_HANDOFF_NOT_OWNED_ERROR_CODE) return new BackgroundJobProducerHandoffLostError({details, message: text})
66
+
67
+ return new BackgroundJobRefusalError({code, details, message: text})
68
+ }
69
+
70
+ /**
71
+ * Copies the transport-safe detail fields of a refusal, dropping anything that
72
+ * is not a primitive the main is allowed to expose.
73
+ * @param {ReturnType<typeof JSON.parse>} details - Candidate details payload.
74
+ * @returns {Record<string, string | number> | undefined} - Safe details, when any.
75
+ */
76
+ function backgroundJobRefusalDetails(details) {
77
+ if (!details || typeof details !== "object" || Array.isArray(details)) return undefined
78
+
79
+ /** @type {Record<string, string | number>} */
80
+ const safeDetails = {}
81
+
82
+ for (const [key, value] of Object.entries(details)) {
83
+ if (typeof value === "string" || typeof value === "number") safeDetails[key] = value
84
+ }
85
+
86
+ return Object.keys(safeDetails).length > 0 ? safeDetails : undefined
87
+ }
@@ -4,6 +4,7 @@ import net from "net"
4
4
  import JsonSocket from "./json-socket.js"
5
5
  import BackgroundJobsGenerationHandshakeTimeoutError, { DEFAULT_GENERATION_HANDSHAKE_TIMEOUT_MS, validateGenerationHandshakeTimeoutMs } from "./generation-handshake-timeout-error.js"
6
6
  import BackgroundJobsGenerationRejectedError from "./generation-rejected-error.js"
7
+ import BackgroundJobsEndpointUnavailableError from "./endpoint-unavailable-error.js"
7
8
 
8
9
  export default class BackgroundJobsSocketRequest {
9
10
  /**
@@ -95,6 +96,12 @@ export default class BackgroundJobsSocketRequest {
95
96
  }
96
97
 
97
98
  jsonSocket.on("error", (error) => {
99
+ if (error && typeof error === "object" && "code" in error && error.code === "ECONNREFUSED") {
100
+ const connectionError = error instanceof Error ? error : new Error(String(error))
101
+ finish({}, () => reject(new BackgroundJobsEndpointUnavailableError({endpoint: `${this.host}:${this.port}`, role: this.role, cause: connectionError})))
102
+ return
103
+ }
104
+
98
105
  finish({}, () => reject(error))
99
106
  })
100
107
 
@@ -1019,11 +1019,49 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
1019
1019
 
1020
1020
  if (!owned) {
1021
1021
  throw VelociousError.safe("Background job producer handoff is no longer owned.", {
1022
- code: "background-job-producer-handoff-not-owned"
1022
+ code: "background-job-producer-handoff-not-owned",
1023
+ details: this._producerHandoffLossDetails(producer, producerProof)
1023
1024
  })
1024
1025
  }
1025
1026
  }
1026
1027
 
1028
+ /**
1029
+ * Names, from the row just read inside the fence's own transaction, why the
1030
+ * producer proof no longer matches. The refusal reaches consumers over the
1031
+ * jobs-main socket as text only, so without this the same lifecycle loss is
1032
+ * indistinguishable from an unexplained enqueue failure.
1033
+ * @param {import("./types.js").BackgroundJobRow | null} producer - Producer row, or null when it is gone.
1034
+ * @param {import("./types.js").BackgroundJobProducerProof} producerProof - Proof the producer execution holds.
1035
+ * @returns {Record<string, string | number>} - Measured loss reason and the state it was measured from.
1036
+ */
1037
+ _producerHandoffLossDetails(producer, producerProof) {
1038
+ if (!producer) {
1039
+ return {
1040
+ producerJobId: producerProof.jobId,
1041
+ producerStatus: "missing",
1042
+ reason: "producer_row_missing"
1043
+ }
1044
+ }
1045
+
1046
+ if (producer.status !== "handed_off") {
1047
+ return {
1048
+ producerJobId: producerProof.jobId,
1049
+ producerStatus: producer.status,
1050
+ reason: `producer_${producer.status}`
1051
+ }
1052
+ }
1053
+
1054
+ return {
1055
+ producerJobId: producerProof.jobId,
1056
+ producerStatus: producer.status,
1057
+ producerWorkerId: producer.workerId || "",
1058
+ proofWorkerId: producerProof.workerId,
1059
+ reason: producer.handoffId === producerProof.handoffId
1060
+ ? "producer_lease_restarted"
1061
+ : "producer_lease_replaced"
1062
+ }
1063
+ }
1064
+
1027
1065
  /**
1028
1066
  * Replaces the queued owner of a stable schedule key with a new one-off job.
1029
1067
  * A handed-off owner is left running and reported truthfully.
@@ -1810,18 +1848,43 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
1810
1848
  }
1811
1849
 
1812
1850
  /**
1813
- * Runs mark returned to queue.
1851
+ * Returns an active handoff to the queue when its owning worker went away,
1852
+ * unless the recorded runner process is still provably alive and executing the
1853
+ * job. Absence of the worker's socket is not absence of the work: reclaiming a
1854
+ * lease a live runner still holds starts a second attempt while the first keeps
1855
+ * running, and every owned follow-up enqueue from the superseded attempt then
1856
+ * fails its producer fence. A deferred lease is settled by the dead-owner pass
1857
+ * once the runner actually exits.
1814
1858
  * @param {object} args - Options.
1859
+ * @param {boolean} [args.deferLiveOwners] - Whether a provably alive recorded runner keeps the lease instead of being returned to the queue.
1860
+ * @param {string} [args.handoffId] - Handoff lease id.
1861
+ * @param {string} [args.hostname] - Hostname whose local probe answers runner ownership, defaults to this process's hostname.
1815
1862
  * @param {string} args.jobId - Job id.
1816
- * @param {string} args.handoffId - Handoff lease id.
1817
- * @returns {Promise<void>} - Resolves when updated.
1863
+ * @param {import("./owner-liveness.js").BackgroundJobLiveChildIdentity[]} [args.liveChildren] - Runner identities this caller observes as live, for the pid-recycling check.
1864
+ * @param {(pid: number) => boolean} [args.probe] - Process probe for the live-owner check, injectable for tests.
1865
+ * @returns {Promise<boolean>} - Whether the lease was returned to the queue.
1818
1866
  */
1819
- async markReturnedToQueue({jobId, handoffId}) {
1867
+ async markReturnedToQueue({deferLiveOwners = false, handoffId, hostname, jobId, liveChildren = [], probe = isProcessAlive}) {
1820
1868
  await this.ensureReady()
1821
1869
 
1822
- await this._serializedCountMutation(async (db) => {
1870
+ return await this._serializedCountMutation(async (db) => {
1823
1871
  const job = await this._getJobRowById(db, jobId)
1824
- if (!job || job.handoffId !== handoffId || job.status !== "handed_off") return
1872
+
1873
+ if (!job || job.handoffId !== handoffId || job.status !== "handed_off") return false
1874
+
1875
+ if (deferLiveOwners) {
1876
+ const liveness = evaluateBackgroundJobOwnerLiveness({
1877
+ childInstanceId: job.childInstanceId,
1878
+ childPid: job.childPid,
1879
+ currentHostname: hostname ?? os.hostname(),
1880
+ liveChildren,
1881
+ probe,
1882
+ runnerHostname: job.runnerHostname
1883
+ })
1884
+
1885
+ if (liveness.status === "alive") return false
1886
+ }
1887
+
1825
1888
  await this._lockConcurrencyRows(db, this._jobConcurrencyKeys(job))
1826
1889
  const affectedRows = await this._updateAffectedRows(db, {
1827
1890
  tableName: JOBS_TABLE,
@@ -1838,7 +1901,11 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
1838
1901
  if (affectedRows === 1) {
1839
1902
  await this._releaseJobConcurrency(db, job)
1840
1903
  await this._recordStatusTransition(db, "handed_off", "queued")
1904
+
1905
+ return true
1841
1906
  }
1907
+
1908
+ return false
1842
1909
  })
1843
1910
  }
1844
1911
 
@@ -1914,14 +1981,21 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
1914
1981
  * snapshot. The ordinary orphan failure path owns retries, terminal status,
1915
1982
  * count transitions, schedule ownership, and concurrency release.
1916
1983
  * @param {object} args - Options.
1917
- * @param {import("./types.js").BackgroundJobHandoffSnapshot[]} args.handoffs - Exact startup snapshots.
1984
+ * @param {(handoffs: import("./types.js").BackgroundJobDeferredLiveOwnerHandoff[]) => void | Promise<void>} [args.onLiveOwnerDeferred] - Observation hook for leases a live runner still owns, called after the reclaim transaction settles.
1918
1985
  * @param {ReturnType<typeof JSON.parse>} args.error - Orphan reason.
1986
+ * @param {import("./types.js").BackgroundJobHandoffSnapshot[]} args.handoffs - Exact startup snapshots.
1987
+ * @param {string} [args.hostname] - Hostname whose local probe answers runner ownership, defaults to this process's hostname.
1988
+ * @param {boolean} [args.deferLiveOwners] - Whether a lease whose recorded runner process is provably alive stays with that runner instead of being reclaimed. Use this for every absence-based reclaim; leave it off once death is proven.
1989
+ * @param {import("./owner-liveness.js").BackgroundJobLiveChildIdentity[]} [args.liveChildren] - Runner identities this caller observes as live, for the pid-recycling check.
1990
+ * @param {(pid: number) => boolean} [args.probe] - Process probe for the live-owner check, injectable for tests.
1919
1991
  * @returns {Promise<import("./types.js").BackgroundJobRow[]>} - Accepted transitions.
1920
1992
  */
1921
- async markOrphanedHandoffs({handoffs, error}) {
1993
+ async markOrphanedHandoffs({deferLiveOwners = false, error, handoffs, hostname, liveChildren = [], onLiveOwnerDeferred, probe = isProcessAlive}) {
1922
1994
  await this.ensureReady()
1923
1995
 
1924
- return await this._serializedCountMutation(async (db) => {
1996
+ /** @type {import("./types.js").BackgroundJobDeferredLiveOwnerHandoff[]} */
1997
+ const deferredLiveOwnerHandoffs = []
1998
+ const orphanedJobs = await this._serializedCountMutation(async (db) => {
1925
1999
  /** @type {BackgroundJobOrphanSelection[]} */
1926
2000
  const selections = []
1927
2001
 
@@ -1933,6 +2007,27 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
1933
2007
  if (job.workerId !== handoff.workerId) continue
1934
2008
  if (job.handedOffAtMs !== handoff.handedOffAtMs) continue
1935
2009
 
2010
+ if (deferLiveOwners) {
2011
+ const liveness = evaluateBackgroundJobOwnerLiveness({
2012
+ childInstanceId: job.childInstanceId,
2013
+ childPid: job.childPid,
2014
+ currentHostname: hostname ?? os.hostname(),
2015
+ liveChildren,
2016
+ probe,
2017
+ runnerHostname: job.runnerHostname
2018
+ })
2019
+
2020
+ if (liveness.status === "alive") {
2021
+ deferredLiveOwnerHandoffs.push({
2022
+ ...handoff,
2023
+ childPid: liveness.childPid,
2024
+ livenessReason: liveness.reason,
2025
+ runnerHostname: liveness.runnerHostname
2026
+ })
2027
+ continue
2028
+ }
2029
+ }
2030
+
1936
2031
  selections.push({
1937
2032
  conditions: {
1938
2033
  handed_off_at_ms: handoff.handedOffAtMs,
@@ -1948,6 +2043,10 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
1948
2043
 
1949
2044
  return await this._markOrphanSelections({db, selections})
1950
2045
  })
2046
+
2047
+ if (deferredLiveOwnerHandoffs.length > 0 && onLiveOwnerDeferred) await onLiveOwnerDeferred(deferredLiveOwnerHandoffs)
2048
+
2049
+ return orphanedJobs
1951
2050
  }
1952
2051
 
1953
2052
  /**
@@ -188,6 +188,11 @@
188
188
  /**
189
189
  * @typedef {BackgroundJobHandoffSnapshot & {liveness: import("./owner-liveness.js").BackgroundJobOwnerLivenessResult}} BackgroundJobHandoffOwnershipVerdict
190
190
  */
191
+ /**
192
+ * One exact lease an absence-based reclaim left with its runner because the
193
+ * recorded runner process is provably alive.
194
+ * @typedef {BackgroundJobHandoffSnapshot & {childPid: number | null, livenessReason: string, runnerHostname: string | null}} BackgroundJobDeferredLiveOwnerHandoff
195
+ */
191
196
  /**
192
197
  * @typedef {object} BackgroundJobHandoffRequest
193
198
  * @property {string} jobId - Job to claim.