velocious 1.0.753 → 1.0.754

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. package/README.md +1 -1
  2. package/build/background-jobs/adapter.js +3 -3
  3. package/build/background-jobs/client.js +6 -5
  4. package/build/background-jobs/local-adapter.js +3 -3
  5. package/build/background-jobs/local-store.js +10 -5
  6. package/build/background-jobs/main.js +87 -8
  7. package/build/background-jobs/refusal-error.js +87 -0
  8. package/build/background-jobs/store.js +109 -10
  9. package/build/background-jobs/types.js +5 -0
  10. package/build/src/background-jobs/adapter.d.ts +15 -6
  11. package/build/src/background-jobs/adapter.d.ts.map +1 -1
  12. package/build/src/background-jobs/adapter.js +4 -4
  13. package/build/src/background-jobs/client.d.ts.map +1 -1
  14. package/build/src/background-jobs/client.js +7 -6
  15. package/build/src/background-jobs/local-adapter.d.ts +8 -4
  16. package/build/src/background-jobs/local-adapter.d.ts.map +1 -1
  17. package/build/src/background-jobs/local-adapter.js +4 -4
  18. package/build/src/background-jobs/local-store.d.ts +12 -5
  19. package/build/src/background-jobs/local-store.d.ts.map +1 -1
  20. package/build/src/background-jobs/local-store.js +10 -6
  21. package/build/src/background-jobs/main.d.ts +46 -2
  22. package/build/src/background-jobs/main.d.ts.map +1 -1
  23. package/build/src/background-jobs/main.js +85 -9
  24. package/build/src/background-jobs/refusal-error.d.ts +57 -0
  25. package/build/src/background-jobs/refusal-error.d.ts.map +1 -0
  26. package/build/src/background-jobs/refusal-error.js +80 -0
  27. package/build/src/background-jobs/store.d.ts +43 -9
  28. package/build/src/background-jobs/store.d.ts.map +1 -1
  29. package/build/src/background-jobs/store.js +99 -11
  30. package/build/src/background-jobs/types.d.ts +10 -0
  31. package/build/src/background-jobs/types.d.ts.map +1 -1
  32. package/build/src/background-jobs/types.js +6 -1
  33. package/package.json +1 -1
  34. package/scripts/verify-docker-dev-environment.js +3 -3
  35. package/src/background-jobs/adapter.js +3 -3
  36. package/src/background-jobs/client.js +6 -5
  37. package/src/background-jobs/local-adapter.js +3 -3
  38. package/src/background-jobs/local-store.js +10 -5
  39. package/src/background-jobs/main.js +87 -8
  40. package/src/background-jobs/refusal-error.js +87 -0
  41. package/src/background-jobs/store.js +109 -10
  42. package/src/background-jobs/types.js +5 -0
@@ -0,0 +1,87 @@
1
+ // @ts-check
2
+
3
+ /** Machine code the jobs main answers a failed producer fence with. */
4
+ export const BACKGROUND_JOB_PRODUCER_HANDOFF_NOT_OWNED_ERROR_CODE = "background-job-producer-handoff-not-owned"
5
+
6
+ /**
7
+ * A refusal the jobs main classified and named on the other side of the socket.
8
+ * The reply used to carry only its text, which erased the framework's own
9
+ * classification: consumers could not tell a producer-handoff loss from any other
10
+ * enqueue failure without matching an English sentence, and the report they filed
11
+ * named no cause.
12
+ */
13
+ export class BackgroundJobRefusalError extends Error {
14
+ /**
15
+ * Builds a refusal from the code and measured details the main sent.
16
+ * @param {object} args - Refusal.
17
+ * @param {string} args.code - Stable machine code of the refusal.
18
+ * @param {Record<string, string | number>} [args.details] - Measured details recorded with the refusal.
19
+ * @param {string} args.message - Text the main exposed for this refusal.
20
+ */
21
+ constructor({code, details, message}) {
22
+ super(message)
23
+
24
+ this.name = "BackgroundJobRefusalError"
25
+ this.code = code
26
+ /** @type {Readonly<Record<string, string | number>> | null} */
27
+ this.details = details ? Object.freeze({...details}) : null
28
+ }
29
+ }
30
+
31
+ /**
32
+ * The producer handoff behind an owned follow-up enqueue is no longer owned, so
33
+ * the main refused to create the follow-up. This is a lifecycle outcome of the
34
+ * jobs main reclaiming the producer, not a defect in the calling feature.
35
+ */
36
+ export class BackgroundJobProducerHandoffLostError extends BackgroundJobRefusalError {
37
+ /**
38
+ * Builds the refusal from the code and measured details the main sent.
39
+ * @param {object} args - Refusal.
40
+ * @param {Record<string, string | number>} [args.details] - Loss reason and the producer state it was measured from.
41
+ * @param {string} args.message - Text the main exposed for this refusal.
42
+ */
43
+ constructor({details, message}) {
44
+ super({code: BACKGROUND_JOB_PRODUCER_HANDOFF_NOT_OWNED_ERROR_CODE, details, message})
45
+
46
+ this.name = "BackgroundJobProducerHandoffLostError"
47
+ }
48
+ }
49
+
50
+ /**
51
+ * Rebuilds one classified refusal from the jobs main. A reply that carries no
52
+ * code keeps its previous plain-error shape so text-only consumers are
53
+ * unaffected.
54
+ * @param {object} args - Options.
55
+ * @param {ReturnType<typeof JSON.parse>} args.message - Refusal message from the main.
56
+ * @param {string} args.fallbackMessage - Text for a reply that carries none.
57
+ * @returns {Error} - Typed refusal when classified, otherwise a plain error with the same text.
58
+ */
59
+ export function backgroundJobRefusalError({fallbackMessage, message}) {
60
+ const text = typeof message?.error === "string" ? message.error : fallbackMessage
61
+ const code = typeof message?.code === "string" ? message.code : null
62
+ const details = backgroundJobRefusalDetails(message?.details)
63
+
64
+ if (!code) return new Error(text)
65
+ if (code === BACKGROUND_JOB_PRODUCER_HANDOFF_NOT_OWNED_ERROR_CODE) return new BackgroundJobProducerHandoffLostError({details, message: text})
66
+
67
+ return new BackgroundJobRefusalError({code, details, message: text})
68
+ }
69
+
70
+ /**
71
+ * Copies the transport-safe detail fields of a refusal, dropping anything that
72
+ * is not a primitive the main is allowed to expose.
73
+ * @param {ReturnType<typeof JSON.parse>} details - Candidate details payload.
74
+ * @returns {Record<string, string | number> | undefined} - Safe details, when any.
75
+ */
76
+ function backgroundJobRefusalDetails(details) {
77
+ if (!details || typeof details !== "object" || Array.isArray(details)) return undefined
78
+
79
+ /** @type {Record<string, string | number>} */
80
+ const safeDetails = {}
81
+
82
+ for (const [key, value] of Object.entries(details)) {
83
+ if (typeof value === "string" || typeof value === "number") safeDetails[key] = value
84
+ }
85
+
86
+ return Object.keys(safeDetails).length > 0 ? safeDetails : undefined
87
+ }
@@ -1019,11 +1019,49 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
1019
1019
 
1020
1020
  if (!owned) {
1021
1021
  throw VelociousError.safe("Background job producer handoff is no longer owned.", {
1022
- code: "background-job-producer-handoff-not-owned"
1022
+ code: "background-job-producer-handoff-not-owned",
1023
+ details: this._producerHandoffLossDetails(producer, producerProof)
1023
1024
  })
1024
1025
  }
1025
1026
  }
1026
1027
 
1028
+ /**
1029
+ * Names, from the row just read inside the fence's own transaction, why the
1030
+ * producer proof no longer matches. The refusal reaches consumers over the
1031
+ * jobs-main socket as text only, so without this the same lifecycle loss is
1032
+ * indistinguishable from an unexplained enqueue failure.
1033
+ * @param {import("./types.js").BackgroundJobRow | null} producer - Producer row, or null when it is gone.
1034
+ * @param {import("./types.js").BackgroundJobProducerProof} producerProof - Proof the producer execution holds.
1035
+ * @returns {Record<string, string | number>} - Measured loss reason and the state it was measured from.
1036
+ */
1037
+ _producerHandoffLossDetails(producer, producerProof) {
1038
+ if (!producer) {
1039
+ return {
1040
+ producerJobId: producerProof.jobId,
1041
+ producerStatus: "missing",
1042
+ reason: "producer_row_missing"
1043
+ }
1044
+ }
1045
+
1046
+ if (producer.status !== "handed_off") {
1047
+ return {
1048
+ producerJobId: producerProof.jobId,
1049
+ producerStatus: producer.status,
1050
+ reason: `producer_${producer.status}`
1051
+ }
1052
+ }
1053
+
1054
+ return {
1055
+ producerJobId: producerProof.jobId,
1056
+ producerStatus: producer.status,
1057
+ producerWorkerId: producer.workerId || "",
1058
+ proofWorkerId: producerProof.workerId,
1059
+ reason: producer.handoffId === producerProof.handoffId
1060
+ ? "producer_lease_restarted"
1061
+ : "producer_lease_replaced"
1062
+ }
1063
+ }
1064
+
1027
1065
  /**
1028
1066
  * Replaces the queued owner of a stable schedule key with a new one-off job.
1029
1067
  * A handed-off owner is left running and reported truthfully.
@@ -1810,18 +1848,43 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
1810
1848
  }
1811
1849
 
1812
1850
  /**
1813
- * Runs mark returned to queue.
1851
+ * Returns an active handoff to the queue when its owning worker went away,
1852
+ * unless the recorded runner process is still provably alive and executing the
1853
+ * job. Absence of the worker's socket is not absence of the work: reclaiming a
1854
+ * lease a live runner still holds starts a second attempt while the first keeps
1855
+ * running, and every owned follow-up enqueue from the superseded attempt then
1856
+ * fails its producer fence. A deferred lease is settled by the dead-owner pass
1857
+ * once the runner actually exits.
1814
1858
  * @param {object} args - Options.
1859
+ * @param {boolean} [args.deferLiveOwners] - Whether a provably alive recorded runner keeps the lease instead of being returned to the queue.
1860
+ * @param {string} [args.handoffId] - Handoff lease id.
1861
+ * @param {string} [args.hostname] - Hostname whose local probe answers runner ownership, defaults to this process's hostname.
1815
1862
  * @param {string} args.jobId - Job id.
1816
- * @param {string} args.handoffId - Handoff lease id.
1817
- * @returns {Promise<void>} - Resolves when updated.
1863
+ * @param {import("./owner-liveness.js").BackgroundJobLiveChildIdentity[]} [args.liveChildren] - Runner identities this caller observes as live, for the pid-recycling check.
1864
+ * @param {(pid: number) => boolean} [args.probe] - Process probe for the live-owner check, injectable for tests.
1865
+ * @returns {Promise<boolean>} - Whether the lease was returned to the queue.
1818
1866
  */
1819
- async markReturnedToQueue({jobId, handoffId}) {
1867
+ async markReturnedToQueue({deferLiveOwners = false, handoffId, hostname, jobId, liveChildren = [], probe = isProcessAlive}) {
1820
1868
  await this.ensureReady()
1821
1869
 
1822
- await this._serializedCountMutation(async (db) => {
1870
+ return await this._serializedCountMutation(async (db) => {
1823
1871
  const job = await this._getJobRowById(db, jobId)
1824
- if (!job || job.handoffId !== handoffId || job.status !== "handed_off") return
1872
+
1873
+ if (!job || job.handoffId !== handoffId || job.status !== "handed_off") return false
1874
+
1875
+ if (deferLiveOwners) {
1876
+ const liveness = evaluateBackgroundJobOwnerLiveness({
1877
+ childInstanceId: job.childInstanceId,
1878
+ childPid: job.childPid,
1879
+ currentHostname: hostname ?? os.hostname(),
1880
+ liveChildren,
1881
+ probe,
1882
+ runnerHostname: job.runnerHostname
1883
+ })
1884
+
1885
+ if (liveness.status === "alive") return false
1886
+ }
1887
+
1825
1888
  await this._lockConcurrencyRows(db, this._jobConcurrencyKeys(job))
1826
1889
  const affectedRows = await this._updateAffectedRows(db, {
1827
1890
  tableName: JOBS_TABLE,
@@ -1838,7 +1901,11 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
1838
1901
  if (affectedRows === 1) {
1839
1902
  await this._releaseJobConcurrency(db, job)
1840
1903
  await this._recordStatusTransition(db, "handed_off", "queued")
1904
+
1905
+ return true
1841
1906
  }
1907
+
1908
+ return false
1842
1909
  })
1843
1910
  }
1844
1911
 
@@ -1914,14 +1981,21 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
1914
1981
  * snapshot. The ordinary orphan failure path owns retries, terminal status,
1915
1982
  * count transitions, schedule ownership, and concurrency release.
1916
1983
  * @param {object} args - Options.
1917
- * @param {import("./types.js").BackgroundJobHandoffSnapshot[]} args.handoffs - Exact startup snapshots.
1984
+ * @param {(handoffs: import("./types.js").BackgroundJobDeferredLiveOwnerHandoff[]) => void | Promise<void>} [args.onLiveOwnerDeferred] - Observation hook for leases a live runner still owns, called after the reclaim transaction settles.
1918
1985
  * @param {ReturnType<typeof JSON.parse>} args.error - Orphan reason.
1986
+ * @param {import("./types.js").BackgroundJobHandoffSnapshot[]} args.handoffs - Exact startup snapshots.
1987
+ * @param {string} [args.hostname] - Hostname whose local probe answers runner ownership, defaults to this process's hostname.
1988
+ * @param {boolean} [args.deferLiveOwners] - Whether a lease whose recorded runner process is provably alive stays with that runner instead of being reclaimed. Use this for every absence-based reclaim; leave it off once death is proven.
1989
+ * @param {import("./owner-liveness.js").BackgroundJobLiveChildIdentity[]} [args.liveChildren] - Runner identities this caller observes as live, for the pid-recycling check.
1990
+ * @param {(pid: number) => boolean} [args.probe] - Process probe for the live-owner check, injectable for tests.
1919
1991
  * @returns {Promise<import("./types.js").BackgroundJobRow[]>} - Accepted transitions.
1920
1992
  */
1921
- async markOrphanedHandoffs({handoffs, error}) {
1993
+ async markOrphanedHandoffs({deferLiveOwners = false, error, handoffs, hostname, liveChildren = [], onLiveOwnerDeferred, probe = isProcessAlive}) {
1922
1994
  await this.ensureReady()
1923
1995
 
1924
- return await this._serializedCountMutation(async (db) => {
1996
+ /** @type {import("./types.js").BackgroundJobDeferredLiveOwnerHandoff[]} */
1997
+ const deferredLiveOwnerHandoffs = []
1998
+ const orphanedJobs = await this._serializedCountMutation(async (db) => {
1925
1999
  /** @type {BackgroundJobOrphanSelection[]} */
1926
2000
  const selections = []
1927
2001
 
@@ -1933,6 +2007,27 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
1933
2007
  if (job.workerId !== handoff.workerId) continue
1934
2008
  if (job.handedOffAtMs !== handoff.handedOffAtMs) continue
1935
2009
 
2010
+ if (deferLiveOwners) {
2011
+ const liveness = evaluateBackgroundJobOwnerLiveness({
2012
+ childInstanceId: job.childInstanceId,
2013
+ childPid: job.childPid,
2014
+ currentHostname: hostname ?? os.hostname(),
2015
+ liveChildren,
2016
+ probe,
2017
+ runnerHostname: job.runnerHostname
2018
+ })
2019
+
2020
+ if (liveness.status === "alive") {
2021
+ deferredLiveOwnerHandoffs.push({
2022
+ ...handoff,
2023
+ childPid: liveness.childPid,
2024
+ livenessReason: liveness.reason,
2025
+ runnerHostname: liveness.runnerHostname
2026
+ })
2027
+ continue
2028
+ }
2029
+ }
2030
+
1936
2031
  selections.push({
1937
2032
  conditions: {
1938
2033
  handed_off_at_ms: handoff.handedOffAtMs,
@@ -1948,6 +2043,10 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
1948
2043
 
1949
2044
  return await this._markOrphanSelections({db, selections})
1950
2045
  })
2046
+
2047
+ if (deferredLiveOwnerHandoffs.length > 0 && onLiveOwnerDeferred) await onLiveOwnerDeferred(deferredLiveOwnerHandoffs)
2048
+
2049
+ return orphanedJobs
1951
2050
  }
1952
2051
 
1953
2052
  /**
@@ -188,6 +188,11 @@
188
188
  /**
189
189
  * @typedef {BackgroundJobHandoffSnapshot & {liveness: import("./owner-liveness.js").BackgroundJobOwnerLivenessResult}} BackgroundJobHandoffOwnershipVerdict
190
190
  */
191
+ /**
192
+ * One exact lease an absence-based reclaim left with its runner because the
193
+ * recorded runner process is provably alive.
194
+ * @typedef {BackgroundJobHandoffSnapshot & {childPid: number | null, livenessReason: string, runnerHostname: string | null}} BackgroundJobDeferredLiveOwnerHandoff
195
+ */
191
196
  /**
192
197
  * @typedef {object} BackgroundJobHandoffRequest
193
198
  * @property {string} jobId - Job to claim.