velocious 1.0.753 → 1.0.754
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/build/background-jobs/adapter.js +3 -3
- package/build/background-jobs/client.js +6 -5
- package/build/background-jobs/local-adapter.js +3 -3
- package/build/background-jobs/local-store.js +10 -5
- package/build/background-jobs/main.js +87 -8
- package/build/background-jobs/refusal-error.js +87 -0
- package/build/background-jobs/store.js +109 -10
- package/build/background-jobs/types.js +5 -0
- package/build/src/background-jobs/adapter.d.ts +15 -6
- package/build/src/background-jobs/adapter.d.ts.map +1 -1
- package/build/src/background-jobs/adapter.js +4 -4
- package/build/src/background-jobs/client.d.ts.map +1 -1
- package/build/src/background-jobs/client.js +7 -6
- package/build/src/background-jobs/local-adapter.d.ts +8 -4
- package/build/src/background-jobs/local-adapter.d.ts.map +1 -1
- package/build/src/background-jobs/local-adapter.js +4 -4
- package/build/src/background-jobs/local-store.d.ts +12 -5
- package/build/src/background-jobs/local-store.d.ts.map +1 -1
- package/build/src/background-jobs/local-store.js +10 -6
- package/build/src/background-jobs/main.d.ts +46 -2
- package/build/src/background-jobs/main.d.ts.map +1 -1
- package/build/src/background-jobs/main.js +85 -9
- package/build/src/background-jobs/refusal-error.d.ts +57 -0
- package/build/src/background-jobs/refusal-error.d.ts.map +1 -0
- package/build/src/background-jobs/refusal-error.js +80 -0
- package/build/src/background-jobs/store.d.ts +43 -9
- package/build/src/background-jobs/store.d.ts.map +1 -1
- package/build/src/background-jobs/store.js +99 -11
- package/build/src/background-jobs/types.d.ts +10 -0
- package/build/src/background-jobs/types.d.ts.map +1 -1
- package/build/src/background-jobs/types.js +6 -1
- package/package.json +1 -1
- package/scripts/verify-docker-dev-environment.js +3 -3
- package/src/background-jobs/adapter.js +3 -3
- package/src/background-jobs/client.js +6 -5
- package/src/background-jobs/local-adapter.js +3 -3
- package/src/background-jobs/local-store.js +10 -5
- package/src/background-jobs/main.js +87 -8
- package/src/background-jobs/refusal-error.js +87 -0
- package/src/background-jobs/store.js +109 -10
- package/src/background-jobs/types.js +5 -0
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
// @ts-check
|
|
2
|
+
|
|
3
|
+
/** Machine code the jobs main answers a failed producer fence with. */
|
|
4
|
+
export const BACKGROUND_JOB_PRODUCER_HANDOFF_NOT_OWNED_ERROR_CODE = "background-job-producer-handoff-not-owned"
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* A refusal the jobs main classified and named on the other side of the socket.
|
|
8
|
+
* The reply used to carry only its text, which erased the framework's own
|
|
9
|
+
* classification: consumers could not tell a producer-handoff loss from any other
|
|
10
|
+
* enqueue failure without matching an English sentence, and the report they filed
|
|
11
|
+
* named no cause.
|
|
12
|
+
*/
|
|
13
|
+
export class BackgroundJobRefusalError extends Error {
|
|
14
|
+
/**
|
|
15
|
+
* Builds a refusal from the code and measured details the main sent.
|
|
16
|
+
* @param {object} args - Refusal.
|
|
17
|
+
* @param {string} args.code - Stable machine code of the refusal.
|
|
18
|
+
* @param {Record<string, string | number>} [args.details] - Measured details recorded with the refusal.
|
|
19
|
+
* @param {string} args.message - Text the main exposed for this refusal.
|
|
20
|
+
*/
|
|
21
|
+
constructor({code, details, message}) {
|
|
22
|
+
super(message)
|
|
23
|
+
|
|
24
|
+
this.name = "BackgroundJobRefusalError"
|
|
25
|
+
this.code = code
|
|
26
|
+
/** @type {Readonly<Record<string, string | number>> | null} */
|
|
27
|
+
this.details = details ? Object.freeze({...details}) : null
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* The producer handoff behind an owned follow-up enqueue is no longer owned, so
|
|
33
|
+
* the main refused to create the follow-up. This is a lifecycle outcome of the
|
|
34
|
+
* jobs main reclaiming the producer, not a defect in the calling feature.
|
|
35
|
+
*/
|
|
36
|
+
export class BackgroundJobProducerHandoffLostError extends BackgroundJobRefusalError {
|
|
37
|
+
/**
|
|
38
|
+
* Builds the refusal from the code and measured details the main sent.
|
|
39
|
+
* @param {object} args - Refusal.
|
|
40
|
+
* @param {Record<string, string | number>} [args.details] - Loss reason and the producer state it was measured from.
|
|
41
|
+
* @param {string} args.message - Text the main exposed for this refusal.
|
|
42
|
+
*/
|
|
43
|
+
constructor({details, message}) {
|
|
44
|
+
super({code: BACKGROUND_JOB_PRODUCER_HANDOFF_NOT_OWNED_ERROR_CODE, details, message})
|
|
45
|
+
|
|
46
|
+
this.name = "BackgroundJobProducerHandoffLostError"
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Rebuilds one classified refusal from the jobs main. A reply that carries no
|
|
52
|
+
* code keeps its previous plain-error shape so text-only consumers are
|
|
53
|
+
* unaffected.
|
|
54
|
+
* @param {object} args - Options.
|
|
55
|
+
* @param {ReturnType<typeof JSON.parse>} args.message - Refusal message from the main.
|
|
56
|
+
* @param {string} args.fallbackMessage - Text for a reply that carries none.
|
|
57
|
+
* @returns {Error} - Typed refusal when classified, otherwise a plain error with the same text.
|
|
58
|
+
*/
|
|
59
|
+
export function backgroundJobRefusalError({fallbackMessage, message}) {
|
|
60
|
+
const text = typeof message?.error === "string" ? message.error : fallbackMessage
|
|
61
|
+
const code = typeof message?.code === "string" ? message.code : null
|
|
62
|
+
const details = backgroundJobRefusalDetails(message?.details)
|
|
63
|
+
|
|
64
|
+
if (!code) return new Error(text)
|
|
65
|
+
if (code === BACKGROUND_JOB_PRODUCER_HANDOFF_NOT_OWNED_ERROR_CODE) return new BackgroundJobProducerHandoffLostError({details, message: text})
|
|
66
|
+
|
|
67
|
+
return new BackgroundJobRefusalError({code, details, message: text})
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* Copies the transport-safe detail fields of a refusal, dropping anything that
|
|
72
|
+
* is not a primitive the main is allowed to expose.
|
|
73
|
+
* @param {ReturnType<typeof JSON.parse>} details - Candidate details payload.
|
|
74
|
+
* @returns {Record<string, string | number> | undefined} - Safe details, when any.
|
|
75
|
+
*/
|
|
76
|
+
function backgroundJobRefusalDetails(details) {
|
|
77
|
+
if (!details || typeof details !== "object" || Array.isArray(details)) return undefined
|
|
78
|
+
|
|
79
|
+
/** @type {Record<string, string | number>} */
|
|
80
|
+
const safeDetails = {}
|
|
81
|
+
|
|
82
|
+
for (const [key, value] of Object.entries(details)) {
|
|
83
|
+
if (typeof value === "string" || typeof value === "number") safeDetails[key] = value
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
return Object.keys(safeDetails).length > 0 ? safeDetails : undefined
|
|
87
|
+
}
|
|
@@ -1019,11 +1019,49 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
|
|
|
1019
1019
|
|
|
1020
1020
|
if (!owned) {
|
|
1021
1021
|
throw VelociousError.safe("Background job producer handoff is no longer owned.", {
|
|
1022
|
-
code: "background-job-producer-handoff-not-owned"
|
|
1022
|
+
code: "background-job-producer-handoff-not-owned",
|
|
1023
|
+
details: this._producerHandoffLossDetails(producer, producerProof)
|
|
1023
1024
|
})
|
|
1024
1025
|
}
|
|
1025
1026
|
}
|
|
1026
1027
|
|
|
1028
|
+
/**
|
|
1029
|
+
* Names, from the row just read inside the fence's own transaction, why the
|
|
1030
|
+
* producer proof no longer matches. The refusal reaches consumers over the
|
|
1031
|
+
* jobs-main socket as text only, so without this the same lifecycle loss is
|
|
1032
|
+
* indistinguishable from an unexplained enqueue failure.
|
|
1033
|
+
* @param {import("./types.js").BackgroundJobRow | null} producer - Producer row, or null when it is gone.
|
|
1034
|
+
* @param {import("./types.js").BackgroundJobProducerProof} producerProof - Proof the producer execution holds.
|
|
1035
|
+
* @returns {Record<string, string | number>} - Measured loss reason and the state it was measured from.
|
|
1036
|
+
*/
|
|
1037
|
+
_producerHandoffLossDetails(producer, producerProof) {
|
|
1038
|
+
if (!producer) {
|
|
1039
|
+
return {
|
|
1040
|
+
producerJobId: producerProof.jobId,
|
|
1041
|
+
producerStatus: "missing",
|
|
1042
|
+
reason: "producer_row_missing"
|
|
1043
|
+
}
|
|
1044
|
+
}
|
|
1045
|
+
|
|
1046
|
+
if (producer.status !== "handed_off") {
|
|
1047
|
+
return {
|
|
1048
|
+
producerJobId: producerProof.jobId,
|
|
1049
|
+
producerStatus: producer.status,
|
|
1050
|
+
reason: `producer_${producer.status}`
|
|
1051
|
+
}
|
|
1052
|
+
}
|
|
1053
|
+
|
|
1054
|
+
return {
|
|
1055
|
+
producerJobId: producerProof.jobId,
|
|
1056
|
+
producerStatus: producer.status,
|
|
1057
|
+
producerWorkerId: producer.workerId || "",
|
|
1058
|
+
proofWorkerId: producerProof.workerId,
|
|
1059
|
+
reason: producer.handoffId === producerProof.handoffId
|
|
1060
|
+
? "producer_lease_restarted"
|
|
1061
|
+
: "producer_lease_replaced"
|
|
1062
|
+
}
|
|
1063
|
+
}
|
|
1064
|
+
|
|
1027
1065
|
/**
|
|
1028
1066
|
* Replaces the queued owner of a stable schedule key with a new one-off job.
|
|
1029
1067
|
* A handed-off owner is left running and reported truthfully.
|
|
@@ -1810,18 +1848,43 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
|
|
|
1810
1848
|
}
|
|
1811
1849
|
|
|
1812
1850
|
/**
|
|
1813
|
-
*
|
|
1851
|
+
* Returns an active handoff to the queue when its owning worker went away,
|
|
1852
|
+
* unless the recorded runner process is still provably alive and executing the
|
|
1853
|
+
* job. Absence of the worker's socket is not absence of the work: reclaiming a
|
|
1854
|
+
* lease a live runner still holds starts a second attempt while the first keeps
|
|
1855
|
+
* running, and every owned follow-up enqueue from the superseded attempt then
|
|
1856
|
+
* fails its producer fence. A deferred lease is settled by the dead-owner pass
|
|
1857
|
+
* once the runner actually exits.
|
|
1814
1858
|
* @param {object} args - Options.
|
|
1859
|
+
* @param {boolean} [args.deferLiveOwners] - Whether a provably alive recorded runner keeps the lease instead of being returned to the queue.
|
|
1860
|
+
* @param {string} [args.handoffId] - Handoff lease id.
|
|
1861
|
+
* @param {string} [args.hostname] - Hostname whose local probe answers runner ownership, defaults to this process's hostname.
|
|
1815
1862
|
* @param {string} args.jobId - Job id.
|
|
1816
|
-
* @param {
|
|
1817
|
-
* @
|
|
1863
|
+
* @param {import("./owner-liveness.js").BackgroundJobLiveChildIdentity[]} [args.liveChildren] - Runner identities this caller observes as live, for the pid-recycling check.
|
|
1864
|
+
* @param {(pid: number) => boolean} [args.probe] - Process probe for the live-owner check, injectable for tests.
|
|
1865
|
+
* @returns {Promise<boolean>} - Whether the lease was returned to the queue.
|
|
1818
1866
|
*/
|
|
1819
|
-
async markReturnedToQueue({jobId,
|
|
1867
|
+
async markReturnedToQueue({deferLiveOwners = false, handoffId, hostname, jobId, liveChildren = [], probe = isProcessAlive}) {
|
|
1820
1868
|
await this.ensureReady()
|
|
1821
1869
|
|
|
1822
|
-
await this._serializedCountMutation(async (db) => {
|
|
1870
|
+
return await this._serializedCountMutation(async (db) => {
|
|
1823
1871
|
const job = await this._getJobRowById(db, jobId)
|
|
1824
|
-
|
|
1872
|
+
|
|
1873
|
+
if (!job || job.handoffId !== handoffId || job.status !== "handed_off") return false
|
|
1874
|
+
|
|
1875
|
+
if (deferLiveOwners) {
|
|
1876
|
+
const liveness = evaluateBackgroundJobOwnerLiveness({
|
|
1877
|
+
childInstanceId: job.childInstanceId,
|
|
1878
|
+
childPid: job.childPid,
|
|
1879
|
+
currentHostname: hostname ?? os.hostname(),
|
|
1880
|
+
liveChildren,
|
|
1881
|
+
probe,
|
|
1882
|
+
runnerHostname: job.runnerHostname
|
|
1883
|
+
})
|
|
1884
|
+
|
|
1885
|
+
if (liveness.status === "alive") return false
|
|
1886
|
+
}
|
|
1887
|
+
|
|
1825
1888
|
await this._lockConcurrencyRows(db, this._jobConcurrencyKeys(job))
|
|
1826
1889
|
const affectedRows = await this._updateAffectedRows(db, {
|
|
1827
1890
|
tableName: JOBS_TABLE,
|
|
@@ -1838,7 +1901,11 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
|
|
|
1838
1901
|
if (affectedRows === 1) {
|
|
1839
1902
|
await this._releaseJobConcurrency(db, job)
|
|
1840
1903
|
await this._recordStatusTransition(db, "handed_off", "queued")
|
|
1904
|
+
|
|
1905
|
+
return true
|
|
1841
1906
|
}
|
|
1907
|
+
|
|
1908
|
+
return false
|
|
1842
1909
|
})
|
|
1843
1910
|
}
|
|
1844
1911
|
|
|
@@ -1914,14 +1981,21 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
|
|
|
1914
1981
|
* snapshot. The ordinary orphan failure path owns retries, terminal status,
|
|
1915
1982
|
* count transitions, schedule ownership, and concurrency release.
|
|
1916
1983
|
* @param {object} args - Options.
|
|
1917
|
-
* @param {import("./types.js").
|
|
1984
|
+
* @param {(handoffs: import("./types.js").BackgroundJobDeferredLiveOwnerHandoff[]) => void | Promise<void>} [args.onLiveOwnerDeferred] - Observation hook for leases a live runner still owns, called after the reclaim transaction settles.
|
|
1918
1985
|
* @param {ReturnType<typeof JSON.parse>} args.error - Orphan reason.
|
|
1986
|
+
* @param {import("./types.js").BackgroundJobHandoffSnapshot[]} args.handoffs - Exact startup snapshots.
|
|
1987
|
+
* @param {string} [args.hostname] - Hostname whose local probe answers runner ownership, defaults to this process's hostname.
|
|
1988
|
+
* @param {boolean} [args.deferLiveOwners] - Whether a lease whose recorded runner process is provably alive stays with that runner instead of being reclaimed. Use this for every absence-based reclaim; leave it off once death is proven.
|
|
1989
|
+
* @param {import("./owner-liveness.js").BackgroundJobLiveChildIdentity[]} [args.liveChildren] - Runner identities this caller observes as live, for the pid-recycling check.
|
|
1990
|
+
* @param {(pid: number) => boolean} [args.probe] - Process probe for the live-owner check, injectable for tests.
|
|
1919
1991
|
* @returns {Promise<import("./types.js").BackgroundJobRow[]>} - Accepted transitions.
|
|
1920
1992
|
*/
|
|
1921
|
-
async markOrphanedHandoffs({handoffs,
|
|
1993
|
+
async markOrphanedHandoffs({deferLiveOwners = false, error, handoffs, hostname, liveChildren = [], onLiveOwnerDeferred, probe = isProcessAlive}) {
|
|
1922
1994
|
await this.ensureReady()
|
|
1923
1995
|
|
|
1924
|
-
|
|
1996
|
+
/** @type {import("./types.js").BackgroundJobDeferredLiveOwnerHandoff[]} */
|
|
1997
|
+
const deferredLiveOwnerHandoffs = []
|
|
1998
|
+
const orphanedJobs = await this._serializedCountMutation(async (db) => {
|
|
1925
1999
|
/** @type {BackgroundJobOrphanSelection[]} */
|
|
1926
2000
|
const selections = []
|
|
1927
2001
|
|
|
@@ -1933,6 +2007,27 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
|
|
|
1933
2007
|
if (job.workerId !== handoff.workerId) continue
|
|
1934
2008
|
if (job.handedOffAtMs !== handoff.handedOffAtMs) continue
|
|
1935
2009
|
|
|
2010
|
+
if (deferLiveOwners) {
|
|
2011
|
+
const liveness = evaluateBackgroundJobOwnerLiveness({
|
|
2012
|
+
childInstanceId: job.childInstanceId,
|
|
2013
|
+
childPid: job.childPid,
|
|
2014
|
+
currentHostname: hostname ?? os.hostname(),
|
|
2015
|
+
liveChildren,
|
|
2016
|
+
probe,
|
|
2017
|
+
runnerHostname: job.runnerHostname
|
|
2018
|
+
})
|
|
2019
|
+
|
|
2020
|
+
if (liveness.status === "alive") {
|
|
2021
|
+
deferredLiveOwnerHandoffs.push({
|
|
2022
|
+
...handoff,
|
|
2023
|
+
childPid: liveness.childPid,
|
|
2024
|
+
livenessReason: liveness.reason,
|
|
2025
|
+
runnerHostname: liveness.runnerHostname
|
|
2026
|
+
})
|
|
2027
|
+
continue
|
|
2028
|
+
}
|
|
2029
|
+
}
|
|
2030
|
+
|
|
1936
2031
|
selections.push({
|
|
1937
2032
|
conditions: {
|
|
1938
2033
|
handed_off_at_ms: handoff.handedOffAtMs,
|
|
@@ -1948,6 +2043,10 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
|
|
|
1948
2043
|
|
|
1949
2044
|
return await this._markOrphanSelections({db, selections})
|
|
1950
2045
|
})
|
|
2046
|
+
|
|
2047
|
+
if (deferredLiveOwnerHandoffs.length > 0 && onLiveOwnerDeferred) await onLiveOwnerDeferred(deferredLiveOwnerHandoffs)
|
|
2048
|
+
|
|
2049
|
+
return orphanedJobs
|
|
1951
2050
|
}
|
|
1952
2051
|
|
|
1953
2052
|
/**
|
|
@@ -188,6 +188,11 @@
|
|
|
188
188
|
/**
|
|
189
189
|
* @typedef {BackgroundJobHandoffSnapshot & {liveness: import("./owner-liveness.js").BackgroundJobOwnerLivenessResult}} BackgroundJobHandoffOwnershipVerdict
|
|
190
190
|
*/
|
|
191
|
+
/**
|
|
192
|
+
* One exact lease an absence-based reclaim left with its runner because the
|
|
193
|
+
* recorded runner process is provably alive.
|
|
194
|
+
* @typedef {BackgroundJobHandoffSnapshot & {childPid: number | null, livenessReason: string, runnerHostname: string | null}} BackgroundJobDeferredLiveOwnerHandoff
|
|
195
|
+
*/
|
|
191
196
|
/**
|
|
192
197
|
* @typedef {object} BackgroundJobHandoffRequest
|
|
193
198
|
* @property {string} jobId - Job to claim.
|