velocious 1.0.753 → 1.0.754
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/build/background-jobs/adapter.js +3 -3
- package/build/background-jobs/client.js +6 -5
- package/build/background-jobs/local-adapter.js +3 -3
- package/build/background-jobs/local-store.js +10 -5
- package/build/background-jobs/main.js +87 -8
- package/build/background-jobs/refusal-error.js +87 -0
- package/build/background-jobs/store.js +109 -10
- package/build/background-jobs/types.js +5 -0
- package/build/src/background-jobs/adapter.d.ts +15 -6
- package/build/src/background-jobs/adapter.d.ts.map +1 -1
- package/build/src/background-jobs/adapter.js +4 -4
- package/build/src/background-jobs/client.d.ts.map +1 -1
- package/build/src/background-jobs/client.js +7 -6
- package/build/src/background-jobs/local-adapter.d.ts +8 -4
- package/build/src/background-jobs/local-adapter.d.ts.map +1 -1
- package/build/src/background-jobs/local-adapter.js +4 -4
- package/build/src/background-jobs/local-store.d.ts +12 -5
- package/build/src/background-jobs/local-store.d.ts.map +1 -1
- package/build/src/background-jobs/local-store.js +10 -6
- package/build/src/background-jobs/main.d.ts +46 -2
- package/build/src/background-jobs/main.d.ts.map +1 -1
- package/build/src/background-jobs/main.js +85 -9
- package/build/src/background-jobs/refusal-error.d.ts +57 -0
- package/build/src/background-jobs/refusal-error.d.ts.map +1 -0
- package/build/src/background-jobs/refusal-error.js +80 -0
- package/build/src/background-jobs/store.d.ts +43 -9
- package/build/src/background-jobs/store.d.ts.map +1 -1
- package/build/src/background-jobs/store.js +99 -11
- package/build/src/background-jobs/types.d.ts +10 -0
- package/build/src/background-jobs/types.d.ts.map +1 -1
- package/build/src/background-jobs/types.js +6 -1
- package/package.json +1 -1
- package/scripts/verify-docker-dev-environment.js +3 -3
- package/src/background-jobs/adapter.js +3 -3
- package/src/background-jobs/client.js +6 -5
- package/src/background-jobs/local-adapter.js +3 -3
- package/src/background-jobs/local-store.js +10 -5
- package/src/background-jobs/main.js +87 -8
- package/src/background-jobs/refusal-error.js +87 -0
- package/src/background-jobs/store.js +109 -10
- package/src/background-jobs/types.js +5 -0
package/README.md
CHANGED
|
@@ -130,7 +130,7 @@ Maintainers cutting a package release must follow the [Velocious release runbook
|
|
|
130
130
|
|
|
131
131
|
# Docker development environment
|
|
132
132
|
|
|
133
|
-
The checked-in root `Dockerfile` and `compose.yml` define one canonical `dev` service used by humans, CI, and agent systems alike (see [docs/docker-development-environment.md](docs/docker-development-environment.md)). The image is Ubuntu 26.04 LTS (pinned by digest) with Node.js 24.x from signed NodeSource, the universal apt coding/debugging baseline, the newest published retained provider CLIs, and owner-pinned native Qwen Code 0.
|
|
133
|
+
The checked-in root `Dockerfile` and `compose.yml` define one canonical `dev` service used by humans, CI, and agent systems alike (see [docs/docker-development-environment.md](docs/docker-development-environment.md)). The image is Ubuntu 26.04 LTS (pinned by digest) with Node.js 24.x from signed NodeSource, the universal apt coding/debugging baseline, the newest published retained provider CLIs, and owner-pinned native Qwen Code 0.28.0; it is source-independent — no project source is copied and no project dependencies are installed at image build time.
|
|
134
134
|
|
|
135
135
|
Prerequisites: Docker with the Compose v2 plugin, and this repository checked out at `$DEV_HOME_PATH/velocious` (default `DEV_HOME_PATH`: `/home/dev`).
|
|
136
136
|
|
|
@@ -174,8 +174,8 @@ export default class BackgroundJobsAdapter {
|
|
|
174
174
|
|
|
175
175
|
/**
|
|
176
176
|
* Returns a handed-off job to the queue.
|
|
177
|
-
* @param {{jobId: string,
|
|
178
|
-
* @returns {Promise<
|
|
177
|
+
* @param {{deferLiveOwners?: boolean, handoffId: string, hostname?: string, jobId: string, liveChildren?: import("./owner-liveness.js").BackgroundJobLiveChildIdentity[], probe?: (pid: number) => boolean}} _args - Handoff release and live-owner controls.
|
|
178
|
+
* @returns {Promise<boolean>} - Whether the lease was returned to the queue.
|
|
179
179
|
*/
|
|
180
180
|
async markReturnedToQueue(_args) { throw new Error("BackgroundJobsAdapter#markReturnedToQueue is not implemented") }
|
|
181
181
|
|
|
@@ -197,7 +197,7 @@ export default class BackgroundJobsAdapter {
|
|
|
197
197
|
* Applies orphan failure semantics to unchanged exact handoff snapshots.
|
|
198
198
|
* Adapters that return startup snapshots must implement the matching fenced
|
|
199
199
|
* transition.
|
|
200
|
-
* @param {{handoffs: import("./types.js").BackgroundJobHandoffSnapshot[],
|
|
200
|
+
* @param {{deferLiveOwners?: boolean, error: ReturnType<typeof JSON.parse>, handoffs: import("./types.js").BackgroundJobHandoffSnapshot[], hostname?: string, liveChildren?: import("./owner-liveness.js").BackgroundJobLiveChildIdentity[], onLiveOwnerDeferred?: (handoffs: import("./types.js").BackgroundJobDeferredLiveOwnerHandoff[]) => void | Promise<void>, probe?: (pid: number) => boolean}} _args - Exact leases, orphan reason, and live-owner controls.
|
|
201
201
|
* @returns {Promise<import("./types.js").BackgroundJobRow[]>} - Accepted transitions.
|
|
202
202
|
*/
|
|
203
203
|
async markOrphanedHandoffs(_args) { return [] }
|
|
@@ -8,6 +8,7 @@ import BackgroundJobsSocketRequest from "./socket-request.js"
|
|
|
8
8
|
import { DEFAULT_GENERATION_HANDSHAKE_TIMEOUT_MS, validateGenerationHandshakeTimeoutMs } from "./generation-handshake-timeout-error.js"
|
|
9
9
|
import { BACKGROUND_JOB_ACTIVE_STATUSES, BACKGROUND_JOB_EXECUTION_MODES, BACKGROUND_JOB_STATUSES, BACKGROUND_JOB_TERMINAL_STATUSES } from "./job-semantics.js"
|
|
10
10
|
import {readBackgroundJobsEndpoint} from "./endpoint-file.js"
|
|
11
|
+
import {backgroundJobRefusalError} from "./refusal-error.js"
|
|
11
12
|
|
|
12
13
|
const DEFAULT_ENQUEUE_TIMEOUT_MS = 5000
|
|
13
14
|
const BACKGROUND_JOB_WAKE_OUTCOMES = ["woken", "already_due", "handed_off", "not_found"]
|
|
@@ -222,7 +223,7 @@ export default class BackgroundJobsClient {
|
|
|
222
223
|
|
|
223
224
|
if (message?.type === "enqueue-error") {
|
|
224
225
|
attemptObservation.explicitlyRejected = true
|
|
225
|
-
reject(
|
|
226
|
+
reject(backgroundJobRefusalError({fallbackMessage: "Failed to enqueue job", message}))
|
|
226
227
|
}
|
|
227
228
|
}
|
|
228
229
|
})
|
|
@@ -296,7 +297,7 @@ export default class BackgroundJobsClient {
|
|
|
296
297
|
}
|
|
297
298
|
|
|
298
299
|
if (message?.type === "replace-scheduled-error") {
|
|
299
|
-
reject(
|
|
300
|
+
reject(backgroundJobRefusalError({fallbackMessage: "Failed to replace scheduled job", message}))
|
|
300
301
|
}
|
|
301
302
|
}
|
|
302
303
|
})
|
|
@@ -322,7 +323,7 @@ export default class BackgroundJobsClient {
|
|
|
322
323
|
}
|
|
323
324
|
|
|
324
325
|
if (message?.type === "cancel-scheduled-error") {
|
|
325
|
-
reject(
|
|
326
|
+
reject(backgroundJobRefusalError({fallbackMessage: "Failed to cancel scheduled job", message}))
|
|
326
327
|
}
|
|
327
328
|
}
|
|
328
329
|
})
|
|
@@ -358,7 +359,7 @@ export default class BackgroundJobsClient {
|
|
|
358
359
|
}
|
|
359
360
|
|
|
360
361
|
if (message?.type === "get-scheduled-job-error") {
|
|
361
|
-
reject(
|
|
362
|
+
reject(backgroundJobRefusalError({fallbackMessage: "Failed to read scheduled job", message}))
|
|
362
363
|
return
|
|
363
364
|
}
|
|
364
365
|
|
|
@@ -395,7 +396,7 @@ export default class BackgroundJobsClient {
|
|
|
395
396
|
}
|
|
396
397
|
|
|
397
398
|
if (message?.type === "wake-scheduled-error") {
|
|
398
|
-
reject(
|
|
399
|
+
reject(backgroundJobRefusalError({fallbackMessage: "Failed to wake scheduled job", message}))
|
|
399
400
|
return
|
|
400
401
|
}
|
|
401
402
|
|
|
@@ -149,10 +149,10 @@ export default class LocalBackgroundJobsAdapter extends BackgroundJobsAdapter {
|
|
|
149
149
|
|
|
150
150
|
/**
|
|
151
151
|
* Returns an exact active local handoff to the queue.
|
|
152
|
-
* @param {{jobId: string,
|
|
153
|
-
* @returns {Promise<
|
|
152
|
+
* @param {{deferLiveOwners?: boolean, handoffId: string, hostname?: string, jobId: string, liveChildren?: import("./owner-liveness.js").BackgroundJobLiveChildIdentity[], probe?: (pid: number) => boolean}} args - Handoff release and live-owner controls.
|
|
153
|
+
* @returns {Promise<boolean>} - Whether the lease was returned to the queue.
|
|
154
154
|
*/
|
|
155
|
-
async markReturnedToQueue(args) { await this.store.markReturnedToQueue(args) }
|
|
155
|
+
async markReturnedToQueue(args) { return await this.store.markReturnedToQueue(args) }
|
|
156
156
|
|
|
157
157
|
/**
|
|
158
158
|
* Acknowledges successful local job completion.
|
|
@@ -1068,17 +1068,20 @@ export default class LocalBackgroundJobsStore {
|
|
|
1068
1068
|
}
|
|
1069
1069
|
|
|
1070
1070
|
/**
|
|
1071
|
-
* Returns an exact active handoff to the queue.
|
|
1072
|
-
*
|
|
1073
|
-
*
|
|
1071
|
+
* Returns an exact active handoff to the queue. The live-owner controls are
|
|
1072
|
+
* accepted for adapter compatibility and answer "released": this store has no
|
|
1073
|
+
* separate runner process to probe — its jobs execute in the dispatching
|
|
1074
|
+
* process, so a lost handoff here has no live executor to defer to.
|
|
1075
|
+
* @param {{deferLiveOwners?: boolean, handoffId: string, hostname?: string, jobId: string, liveChildren?: import("./owner-liveness.js").BackgroundJobLiveChildIdentity[], probe?: (pid: number) => boolean}} args - Handoff release and live-owner controls.
|
|
1076
|
+
* @returns {Promise<boolean>} - Whether the lease was returned to the queue.
|
|
1074
1077
|
*/
|
|
1075
1078
|
async markReturnedToQueue({jobId, handoffId}) {
|
|
1076
1079
|
await this.ensureReady()
|
|
1077
1080
|
|
|
1078
|
-
await this._withDb(async (connection) => await this._mutate(connection, async (db) => {
|
|
1081
|
+
return await this._withDb(async (connection) => await this._mutate(connection, async (db) => {
|
|
1079
1082
|
const job = await this._getJob(db, jobId)
|
|
1080
1083
|
|
|
1081
|
-
if (!this._acceptsHandoff(job, handoffId)) return
|
|
1084
|
+
if (!this._acceptsHandoff(job, handoffId)) return false
|
|
1082
1085
|
await this._lockConcurrencyRow(db, job.concurrencyKey)
|
|
1083
1086
|
|
|
1084
1087
|
const affectedRows = await this._updateAffectedRows(db, {
|
|
@@ -1095,6 +1098,8 @@ export default class LocalBackgroundJobsStore {
|
|
|
1095
1098
|
})
|
|
1096
1099
|
|
|
1097
1100
|
if (affectedRows === 1) await this._releaseConcurrency(db, job.concurrencyKey)
|
|
1101
|
+
|
|
1102
|
+
return affectedRows === 1
|
|
1098
1103
|
}))
|
|
1099
1104
|
}
|
|
1100
1105
|
|
|
@@ -43,6 +43,8 @@ const WORKER_RECONNECT_GRACE_MS = 30000
|
|
|
43
43
|
const GENERATION_ORPHANED_AFTER_MS = 60 * 60 * 1000
|
|
44
44
|
/** Prefix of the per-generation liveness advisory lock one live main holds for its whole generation. */
|
|
45
45
|
const GENERATION_LIVENESS_LOCK_PREFIX = "background-jobs:generation-liveness"
|
|
46
|
+
/** Maximum deferred-lease job ids named in one live-owner deferral log line. */
|
|
47
|
+
const LIVE_OWNER_DEFERRAL_JOB_ID_LIMIT = 20
|
|
46
48
|
const WORKER_RECONNECT_GRACE_VALIDATION_MESSAGE = `workerReconnectGraceMs must be an integer between 0 and ${MAX_TIMER_MS}`
|
|
47
49
|
|
|
48
50
|
/**
|
|
@@ -930,7 +932,13 @@ export default class BackgroundJobsMain {
|
|
|
930
932
|
* acquire that generation's liveness lock on a dedicated connection and hold
|
|
931
933
|
* it across the fenced store transition. A busy lock means a live main of
|
|
932
934
|
* that generation still supervises its rows (retired-and-draining included),
|
|
933
|
-
* so nothing here may touch them.
|
|
935
|
+
* so nothing here may touch them. An idle lock only proves the main is gone,
|
|
936
|
+
* not the work: a runner child of the retired generation can still be
|
|
937
|
+
* executing the job while its parent main has exited, so the store reclaim
|
|
938
|
+
* defers every lease whose recorded runner process is still alive. Those rows
|
|
939
|
+
* stay with the live runner and are settled by the dead-owner pass when it
|
|
940
|
+
* exits, instead of being requeued into a second attempt whose first is still
|
|
941
|
+
* running and failing its own producer fences.
|
|
934
942
|
* @param {object} args - Options.
|
|
935
943
|
* @param {string} args.foreignGenerationId - Generation whose rows are candidates.
|
|
936
944
|
* @param {import("./types.js").BackgroundJobHandoffSnapshot[]} args.handoffs - Exact snapshot rows.
|
|
@@ -946,8 +954,11 @@ export default class BackgroundJobsMain {
|
|
|
946
954
|
if (!acquired) return []
|
|
947
955
|
|
|
948
956
|
return await this.store.markOrphanedHandoffs({
|
|
957
|
+
deferLiveOwners: true,
|
|
949
958
|
error: "Job orphaned when a new generation activated while its owning generation was not live",
|
|
950
|
-
handoffs
|
|
959
|
+
handoffs,
|
|
960
|
+
liveChildren: [...this.liveChildIdentities.values()],
|
|
961
|
+
onLiveOwnerDeferred: (deferred) => this._reportLiveOwnerHandoffDeferral({context: "cross-generation-settlement", handoffs: deferred})
|
|
951
962
|
})
|
|
952
963
|
} finally {
|
|
953
964
|
try {
|
|
@@ -1044,8 +1055,11 @@ export default class BackgroundJobsMain {
|
|
|
1044
1055
|
|
|
1045
1056
|
try {
|
|
1046
1057
|
orphanedJobs = await this.store.markOrphanedHandoffs({
|
|
1058
|
+
deferLiveOwners: true,
|
|
1047
1059
|
error: "Job orphaned after its pre-restart worker did not reconnect",
|
|
1048
|
-
handoffs
|
|
1060
|
+
handoffs,
|
|
1061
|
+
liveChildren: [...this.liveChildIdentities.values()],
|
|
1062
|
+
onLiveOwnerDeferred: (deferred) => this._reportLiveOwnerHandoffDeferral({context: "startup-handoff-reclaim", handoffs: deferred})
|
|
1049
1063
|
})
|
|
1050
1064
|
} catch (error) {
|
|
1051
1065
|
this._reportStartupHandoffReclaimError(error)
|
|
@@ -1690,13 +1704,35 @@ export default class BackgroundJobsMain {
|
|
|
1690
1704
|
* @returns {Promise<void>} - Resolves after the fenced transition.
|
|
1691
1705
|
*/
|
|
1692
1706
|
async _releaseHandoff({handoffId, jobId, worker}) {
|
|
1693
|
-
await this.
|
|
1707
|
+
await this._returnLeaseToQueue({handoffId, jobId})
|
|
1694
1708
|
|
|
1695
1709
|
const handoffs = this.workerHandoffs.get(worker)
|
|
1696
1710
|
|
|
1697
1711
|
if (handoffs?.get(jobId) === handoffId) handoffs.delete(jobId)
|
|
1698
1712
|
}
|
|
1699
1713
|
|
|
1714
|
+
/**
|
|
1715
|
+
* Returns one exact lease to the queue, deferring the transition while the
|
|
1716
|
+
* recorded runner process is provably alive. A lost worker socket is not a
|
|
1717
|
+
* lost job: reclaiming a lease a live runner still holds starts a second
|
|
1718
|
+
* attempt beside the first, and the superseded attempt's owned follow-up
|
|
1719
|
+
* enqueues then all fail their producer fence. The in-memory lease is dropped
|
|
1720
|
+
* either way — the durable row keeps its exact owner, and a runner that
|
|
1721
|
+
* reconnects re-adopts it.
|
|
1722
|
+
* @param {object} args - Options.
|
|
1723
|
+
* @param {string} args.handoffId - Handoff lease id.
|
|
1724
|
+
* @param {string} args.jobId - Job id.
|
|
1725
|
+
* @returns {Promise<boolean>} - Whether the lease was returned to the queue.
|
|
1726
|
+
*/
|
|
1727
|
+
async _returnLeaseToQueue({handoffId, jobId}) {
|
|
1728
|
+
return await this.store.markReturnedToQueue({
|
|
1729
|
+
deferLiveOwners: true,
|
|
1730
|
+
handoffId,
|
|
1731
|
+
jobId,
|
|
1732
|
+
liveChildren: [...this.liveChildIdentities.values()]
|
|
1733
|
+
})
|
|
1734
|
+
}
|
|
1735
|
+
|
|
1700
1736
|
/**
|
|
1701
1737
|
* Forgets an in-memory lease that no longer exists durably — a reported lease or
|
|
1702
1738
|
* one settled as orphaned — without relying on worker ids.
|
|
@@ -1771,6 +1807,32 @@ export default class BackgroundJobsMain {
|
|
|
1771
1807
|
errorEvents.emit("all-error", {...payload, errorType: "framework-error"})
|
|
1772
1808
|
}
|
|
1773
1809
|
|
|
1810
|
+
/**
|
|
1811
|
+
* Records the leases an absence-based reclaim deliberately left alone because
|
|
1812
|
+
* their recorded runner process is still alive. A deferral is the healthy
|
|
1813
|
+
* outcome — the work is still running — but it must be visible: without it a
|
|
1814
|
+
* lease that stays `handed_off` looks like a stranded job, and the reason a
|
|
1815
|
+
* requeue was skipped would exist nowhere.
|
|
1816
|
+
* @param {object} args - Options.
|
|
1817
|
+
* @param {"cross-generation-settlement" | "generation-orphan-sweep" | "startup-handoff-reclaim"} args.context - Reclaim that deferred the leases.
|
|
1818
|
+
* @param {import("./types.js").BackgroundJobDeferredLiveOwnerHandoff[]} args.handoffs - Deferred leases with their measured liveness.
|
|
1819
|
+
* @returns {void}
|
|
1820
|
+
*/
|
|
1821
|
+
_reportLiveOwnerHandoffDeferral({context, handoffs}) {
|
|
1822
|
+
const boundedJobIds = handoffs.slice(0, LIVE_OWNER_DEFERRAL_JOB_ID_LIMIT).map(({jobId}) => jobId)
|
|
1823
|
+
|
|
1824
|
+
this.logger.info(() => [
|
|
1825
|
+
"Left background job leases with their live runners instead of reclaiming them",
|
|
1826
|
+
{
|
|
1827
|
+
context,
|
|
1828
|
+
deferredCount: handoffs.length,
|
|
1829
|
+
jobIds: boundedJobIds,
|
|
1830
|
+
jobIdsTruncatedCount: Math.max(0, handoffs.length - boundedJobIds.length),
|
|
1831
|
+
livenessReasons: [...new Set(handoffs.map(({livenessReason}) => livenessReason))].sort()
|
|
1832
|
+
}
|
|
1833
|
+
])
|
|
1834
|
+
}
|
|
1835
|
+
|
|
1774
1836
|
/**
|
|
1775
1837
|
* Runs handle enqueue.
|
|
1776
1838
|
* @param {object} args - Options.
|
|
@@ -1926,6 +1988,9 @@ export default class BackgroundJobsMain {
|
|
|
1926
1988
|
|
|
1927
1989
|
/**
|
|
1928
1990
|
* Returns safe validation failures and reports unexpected client mutations.
|
|
1991
|
+
* A safe refusal keeps its machine code and measured details on the wire so
|
|
1992
|
+
* the requesting process can rebuild the same classification instead of
|
|
1993
|
+
* guessing from the message text.
|
|
1929
1994
|
* @param {object} args - Options.
|
|
1930
1995
|
* @param {Record<string, ReturnType<typeof JSON.parse>>} args.context - Framework-error context.
|
|
1931
1996
|
* @param {ReturnType<typeof JSON.parse>} args.error - Mutation failure.
|
|
@@ -1937,7 +2002,12 @@ export default class BackgroundJobsMain {
|
|
|
1937
2002
|
*/
|
|
1938
2003
|
_handleClientMutationError({context, error, fallbackMessage, jsonSocket, logMessage, responseType}) {
|
|
1939
2004
|
if (error instanceof VelociousError && error.safeToExpose) {
|
|
1940
|
-
jsonSocket.send({
|
|
2005
|
+
jsonSocket.send({
|
|
2006
|
+
type: responseType,
|
|
2007
|
+
error: error.message,
|
|
2008
|
+
...(typeof error.code === "string" ? {code: error.code} : {}),
|
|
2009
|
+
...(error.details ? {details: error.details} : {})
|
|
2010
|
+
})
|
|
1941
2011
|
return
|
|
1942
2012
|
}
|
|
1943
2013
|
|
|
@@ -2584,12 +2654,15 @@ export default class BackgroundJobsMain {
|
|
|
2584
2654
|
|
|
2585
2655
|
/**
|
|
2586
2656
|
* Returns one exact lease and forgets it only after the adapter acknowledges
|
|
2587
|
-
* the fenced transition or confirms it was already absent.
|
|
2657
|
+
* the fenced transition or confirms it was already absent. A lease a live
|
|
2658
|
+
* runner already accepted is deferred by that transition: the dispatch that
|
|
2659
|
+
* main could not confirm may in fact have landed, and requeueing it beside a
|
|
2660
|
+
* running child is the duplicate this recovery is meant to avoid.
|
|
2588
2661
|
* @param {{handoffId: string, jobId: string}} args - Exact recovery fence.
|
|
2589
2662
|
* @returns {Promise<void>} - Resolves after durable recovery settles.
|
|
2590
2663
|
*/
|
|
2591
2664
|
async _recoverHandoff({handoffId, jobId}) {
|
|
2592
|
-
await this.
|
|
2665
|
+
await this._returnLeaseToQueue({handoffId, jobId})
|
|
2593
2666
|
|
|
2594
2667
|
if (this.pendingHandoffRecoveries.get(handoffId) === jobId) {
|
|
2595
2668
|
this.pendingHandoffRecoveries.delete(handoffId)
|
|
@@ -2758,7 +2831,13 @@ export default class BackgroundJobsMain {
|
|
|
2758
2831
|
})
|
|
2759
2832
|
const generationOrphanedJobs = handoffs.length === 0
|
|
2760
2833
|
? []
|
|
2761
|
-
: await this.store.markOrphanedHandoffs({
|
|
2834
|
+
: await this.store.markOrphanedHandoffs({
|
|
2835
|
+
deferLiveOwners: true,
|
|
2836
|
+
error: "Job orphaned after its generation owner disappeared",
|
|
2837
|
+
handoffs,
|
|
2838
|
+
liveChildren: [...this.liveChildIdentities.values()],
|
|
2839
|
+
onLiveOwnerDeferred: (deferred) => this._reportLiveOwnerHandoffDeferral({context: "generation-orphan-sweep", handoffs: deferred})
|
|
2840
|
+
})
|
|
2762
2841
|
// Cross-generation rows are invisible to the fenced snapshot above; the
|
|
2763
2842
|
// dead-owner pass settles them from the recorded runner process instead
|
|
2764
2843
|
// of an age rule, so a retired generation can never strand a lease. The
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
// @ts-check
|
|
2
|
+
|
|
3
|
+
/** Machine code the jobs main answers a failed producer fence with. */
|
|
4
|
+
export const BACKGROUND_JOB_PRODUCER_HANDOFF_NOT_OWNED_ERROR_CODE = "background-job-producer-handoff-not-owned"
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* A refusal the jobs main classified and named on the other side of the socket.
|
|
8
|
+
* The reply used to carry only its text, which erased the framework's own
|
|
9
|
+
* classification: consumers could not tell a producer-handoff loss from any other
|
|
10
|
+
* enqueue failure without matching an English sentence, and the report they filed
|
|
11
|
+
* named no cause.
|
|
12
|
+
*/
|
|
13
|
+
export class BackgroundJobRefusalError extends Error {
|
|
14
|
+
/**
|
|
15
|
+
* Builds a refusal from the code and measured details the main sent.
|
|
16
|
+
* @param {object} args - Refusal.
|
|
17
|
+
* @param {string} args.code - Stable machine code of the refusal.
|
|
18
|
+
* @param {Record<string, string | number>} [args.details] - Measured details recorded with the refusal.
|
|
19
|
+
* @param {string} args.message - Text the main exposed for this refusal.
|
|
20
|
+
*/
|
|
21
|
+
constructor({code, details, message}) {
|
|
22
|
+
super(message)
|
|
23
|
+
|
|
24
|
+
this.name = "BackgroundJobRefusalError"
|
|
25
|
+
this.code = code
|
|
26
|
+
/** @type {Readonly<Record<string, string | number>> | null} */
|
|
27
|
+
this.details = details ? Object.freeze({...details}) : null
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* The producer handoff behind an owned follow-up enqueue is no longer owned, so
|
|
33
|
+
* the main refused to create the follow-up. This is a lifecycle outcome of the
|
|
34
|
+
* jobs main reclaiming the producer, not a defect in the calling feature.
|
|
35
|
+
*/
|
|
36
|
+
export class BackgroundJobProducerHandoffLostError extends BackgroundJobRefusalError {
|
|
37
|
+
/**
|
|
38
|
+
* Builds the refusal from the code and measured details the main sent.
|
|
39
|
+
* @param {object} args - Refusal.
|
|
40
|
+
* @param {Record<string, string | number>} [args.details] - Loss reason and the producer state it was measured from.
|
|
41
|
+
* @param {string} args.message - Text the main exposed for this refusal.
|
|
42
|
+
*/
|
|
43
|
+
constructor({details, message}) {
|
|
44
|
+
super({code: BACKGROUND_JOB_PRODUCER_HANDOFF_NOT_OWNED_ERROR_CODE, details, message})
|
|
45
|
+
|
|
46
|
+
this.name = "BackgroundJobProducerHandoffLostError"
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Rebuilds one classified refusal from the jobs main. A reply that carries no
|
|
52
|
+
* code keeps its previous plain-error shape so text-only consumers are
|
|
53
|
+
* unaffected.
|
|
54
|
+
* @param {object} args - Options.
|
|
55
|
+
* @param {ReturnType<typeof JSON.parse>} args.message - Refusal message from the main.
|
|
56
|
+
* @param {string} args.fallbackMessage - Text for a reply that carries none.
|
|
57
|
+
* @returns {Error} - Typed refusal when classified, otherwise a plain error with the same text.
|
|
58
|
+
*/
|
|
59
|
+
export function backgroundJobRefusalError({fallbackMessage, message}) {
|
|
60
|
+
const text = typeof message?.error === "string" ? message.error : fallbackMessage
|
|
61
|
+
const code = typeof message?.code === "string" ? message.code : null
|
|
62
|
+
const details = backgroundJobRefusalDetails(message?.details)
|
|
63
|
+
|
|
64
|
+
if (!code) return new Error(text)
|
|
65
|
+
if (code === BACKGROUND_JOB_PRODUCER_HANDOFF_NOT_OWNED_ERROR_CODE) return new BackgroundJobProducerHandoffLostError({details, message: text})
|
|
66
|
+
|
|
67
|
+
return new BackgroundJobRefusalError({code, details, message: text})
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* Copies the transport-safe detail fields of a refusal, dropping anything that
|
|
72
|
+
* is not a primitive the main is allowed to expose.
|
|
73
|
+
* @param {ReturnType<typeof JSON.parse>} details - Candidate details payload.
|
|
74
|
+
* @returns {Record<string, string | number> | undefined} - Safe details, when any.
|
|
75
|
+
*/
|
|
76
|
+
function backgroundJobRefusalDetails(details) {
|
|
77
|
+
if (!details || typeof details !== "object" || Array.isArray(details)) return undefined
|
|
78
|
+
|
|
79
|
+
/** @type {Record<string, string | number>} */
|
|
80
|
+
const safeDetails = {}
|
|
81
|
+
|
|
82
|
+
for (const [key, value] of Object.entries(details)) {
|
|
83
|
+
if (typeof value === "string" || typeof value === "number") safeDetails[key] = value
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
return Object.keys(safeDetails).length > 0 ? safeDetails : undefined
|
|
87
|
+
}
|
|
@@ -1019,11 +1019,49 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
|
|
|
1019
1019
|
|
|
1020
1020
|
if (!owned) {
|
|
1021
1021
|
throw VelociousError.safe("Background job producer handoff is no longer owned.", {
|
|
1022
|
-
code: "background-job-producer-handoff-not-owned"
|
|
1022
|
+
code: "background-job-producer-handoff-not-owned",
|
|
1023
|
+
details: this._producerHandoffLossDetails(producer, producerProof)
|
|
1023
1024
|
})
|
|
1024
1025
|
}
|
|
1025
1026
|
}
|
|
1026
1027
|
|
|
1028
|
+
/**
|
|
1029
|
+
* Names, from the row just read inside the fence's own transaction, why the
|
|
1030
|
+
* producer proof no longer matches. The refusal reaches consumers over the
|
|
1031
|
+
* jobs-main socket as text only, so without this the same lifecycle loss is
|
|
1032
|
+
* indistinguishable from an unexplained enqueue failure.
|
|
1033
|
+
* @param {import("./types.js").BackgroundJobRow | null} producer - Producer row, or null when it is gone.
|
|
1034
|
+
* @param {import("./types.js").BackgroundJobProducerProof} producerProof - Proof the producer execution holds.
|
|
1035
|
+
* @returns {Record<string, string | number>} - Measured loss reason and the state it was measured from.
|
|
1036
|
+
*/
|
|
1037
|
+
_producerHandoffLossDetails(producer, producerProof) {
|
|
1038
|
+
if (!producer) {
|
|
1039
|
+
return {
|
|
1040
|
+
producerJobId: producerProof.jobId,
|
|
1041
|
+
producerStatus: "missing",
|
|
1042
|
+
reason: "producer_row_missing"
|
|
1043
|
+
}
|
|
1044
|
+
}
|
|
1045
|
+
|
|
1046
|
+
if (producer.status !== "handed_off") {
|
|
1047
|
+
return {
|
|
1048
|
+
producerJobId: producerProof.jobId,
|
|
1049
|
+
producerStatus: producer.status,
|
|
1050
|
+
reason: `producer_${producer.status}`
|
|
1051
|
+
}
|
|
1052
|
+
}
|
|
1053
|
+
|
|
1054
|
+
return {
|
|
1055
|
+
producerJobId: producerProof.jobId,
|
|
1056
|
+
producerStatus: producer.status,
|
|
1057
|
+
producerWorkerId: producer.workerId || "",
|
|
1058
|
+
proofWorkerId: producerProof.workerId,
|
|
1059
|
+
reason: producer.handoffId === producerProof.handoffId
|
|
1060
|
+
? "producer_lease_restarted"
|
|
1061
|
+
: "producer_lease_replaced"
|
|
1062
|
+
}
|
|
1063
|
+
}
|
|
1064
|
+
|
|
1027
1065
|
/**
|
|
1028
1066
|
* Replaces the queued owner of a stable schedule key with a new one-off job.
|
|
1029
1067
|
* A handed-off owner is left running and reported truthfully.
|
|
@@ -1810,18 +1848,43 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
|
|
|
1810
1848
|
}
|
|
1811
1849
|
|
|
1812
1850
|
/**
|
|
1813
|
-
*
|
|
1851
|
+
* Returns an active handoff to the queue when its owning worker went away,
|
|
1852
|
+
* unless the recorded runner process is still provably alive and executing the
|
|
1853
|
+
* job. Absence of the worker's socket is not absence of the work: reclaiming a
|
|
1854
|
+
* lease a live runner still holds starts a second attempt while the first keeps
|
|
1855
|
+
* running, and every owned follow-up enqueue from the superseded attempt then
|
|
1856
|
+
* fails its producer fence. A deferred lease is settled by the dead-owner pass
|
|
1857
|
+
* once the runner actually exits.
|
|
1814
1858
|
* @param {object} args - Options.
|
|
1859
|
+
* @param {boolean} [args.deferLiveOwners] - Whether a provably alive recorded runner keeps the lease instead of being returned to the queue.
|
|
1860
|
+
* @param {string} [args.handoffId] - Handoff lease id.
|
|
1861
|
+
* @param {string} [args.hostname] - Hostname whose local probe answers runner ownership, defaults to this process's hostname.
|
|
1815
1862
|
* @param {string} args.jobId - Job id.
|
|
1816
|
-
* @param {
|
|
1817
|
-
* @
|
|
1863
|
+
* @param {import("./owner-liveness.js").BackgroundJobLiveChildIdentity[]} [args.liveChildren] - Runner identities this caller observes as live, for the pid-recycling check.
|
|
1864
|
+
* @param {(pid: number) => boolean} [args.probe] - Process probe for the live-owner check, injectable for tests.
|
|
1865
|
+
* @returns {Promise<boolean>} - Whether the lease was returned to the queue.
|
|
1818
1866
|
*/
|
|
1819
|
-
async markReturnedToQueue({jobId,
|
|
1867
|
+
async markReturnedToQueue({deferLiveOwners = false, handoffId, hostname, jobId, liveChildren = [], probe = isProcessAlive}) {
|
|
1820
1868
|
await this.ensureReady()
|
|
1821
1869
|
|
|
1822
|
-
await this._serializedCountMutation(async (db) => {
|
|
1870
|
+
return await this._serializedCountMutation(async (db) => {
|
|
1823
1871
|
const job = await this._getJobRowById(db, jobId)
|
|
1824
|
-
|
|
1872
|
+
|
|
1873
|
+
if (!job || job.handoffId !== handoffId || job.status !== "handed_off") return false
|
|
1874
|
+
|
|
1875
|
+
if (deferLiveOwners) {
|
|
1876
|
+
const liveness = evaluateBackgroundJobOwnerLiveness({
|
|
1877
|
+
childInstanceId: job.childInstanceId,
|
|
1878
|
+
childPid: job.childPid,
|
|
1879
|
+
currentHostname: hostname ?? os.hostname(),
|
|
1880
|
+
liveChildren,
|
|
1881
|
+
probe,
|
|
1882
|
+
runnerHostname: job.runnerHostname
|
|
1883
|
+
})
|
|
1884
|
+
|
|
1885
|
+
if (liveness.status === "alive") return false
|
|
1886
|
+
}
|
|
1887
|
+
|
|
1825
1888
|
await this._lockConcurrencyRows(db, this._jobConcurrencyKeys(job))
|
|
1826
1889
|
const affectedRows = await this._updateAffectedRows(db, {
|
|
1827
1890
|
tableName: JOBS_TABLE,
|
|
@@ -1838,7 +1901,11 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
|
|
|
1838
1901
|
if (affectedRows === 1) {
|
|
1839
1902
|
await this._releaseJobConcurrency(db, job)
|
|
1840
1903
|
await this._recordStatusTransition(db, "handed_off", "queued")
|
|
1904
|
+
|
|
1905
|
+
return true
|
|
1841
1906
|
}
|
|
1907
|
+
|
|
1908
|
+
return false
|
|
1842
1909
|
})
|
|
1843
1910
|
}
|
|
1844
1911
|
|
|
@@ -1914,14 +1981,21 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
|
|
|
1914
1981
|
* snapshot. The ordinary orphan failure path owns retries, terminal status,
|
|
1915
1982
|
* count transitions, schedule ownership, and concurrency release.
|
|
1916
1983
|
* @param {object} args - Options.
|
|
1917
|
-
* @param {import("./types.js").
|
|
1984
|
+
* @param {(handoffs: import("./types.js").BackgroundJobDeferredLiveOwnerHandoff[]) => void | Promise<void>} [args.onLiveOwnerDeferred] - Observation hook for leases a live runner still owns, called after the reclaim transaction settles.
|
|
1918
1985
|
* @param {ReturnType<typeof JSON.parse>} args.error - Orphan reason.
|
|
1986
|
+
* @param {import("./types.js").BackgroundJobHandoffSnapshot[]} args.handoffs - Exact startup snapshots.
|
|
1987
|
+
* @param {string} [args.hostname] - Hostname whose local probe answers runner ownership, defaults to this process's hostname.
|
|
1988
|
+
* @param {boolean} [args.deferLiveOwners] - Whether a lease whose recorded runner process is provably alive stays with that runner instead of being reclaimed. Use this for every absence-based reclaim; leave it off once death is proven.
|
|
1989
|
+
* @param {import("./owner-liveness.js").BackgroundJobLiveChildIdentity[]} [args.liveChildren] - Runner identities this caller observes as live, for the pid-recycling check.
|
|
1990
|
+
* @param {(pid: number) => boolean} [args.probe] - Process probe for the live-owner check, injectable for tests.
|
|
1919
1991
|
* @returns {Promise<import("./types.js").BackgroundJobRow[]>} - Accepted transitions.
|
|
1920
1992
|
*/
|
|
1921
|
-
async markOrphanedHandoffs({handoffs,
|
|
1993
|
+
async markOrphanedHandoffs({deferLiveOwners = false, error, handoffs, hostname, liveChildren = [], onLiveOwnerDeferred, probe = isProcessAlive}) {
|
|
1922
1994
|
await this.ensureReady()
|
|
1923
1995
|
|
|
1924
|
-
|
|
1996
|
+
/** @type {import("./types.js").BackgroundJobDeferredLiveOwnerHandoff[]} */
|
|
1997
|
+
const deferredLiveOwnerHandoffs = []
|
|
1998
|
+
const orphanedJobs = await this._serializedCountMutation(async (db) => {
|
|
1925
1999
|
/** @type {BackgroundJobOrphanSelection[]} */
|
|
1926
2000
|
const selections = []
|
|
1927
2001
|
|
|
@@ -1933,6 +2007,27 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
|
|
|
1933
2007
|
if (job.workerId !== handoff.workerId) continue
|
|
1934
2008
|
if (job.handedOffAtMs !== handoff.handedOffAtMs) continue
|
|
1935
2009
|
|
|
2010
|
+
if (deferLiveOwners) {
|
|
2011
|
+
const liveness = evaluateBackgroundJobOwnerLiveness({
|
|
2012
|
+
childInstanceId: job.childInstanceId,
|
|
2013
|
+
childPid: job.childPid,
|
|
2014
|
+
currentHostname: hostname ?? os.hostname(),
|
|
2015
|
+
liveChildren,
|
|
2016
|
+
probe,
|
|
2017
|
+
runnerHostname: job.runnerHostname
|
|
2018
|
+
})
|
|
2019
|
+
|
|
2020
|
+
if (liveness.status === "alive") {
|
|
2021
|
+
deferredLiveOwnerHandoffs.push({
|
|
2022
|
+
...handoff,
|
|
2023
|
+
childPid: liveness.childPid,
|
|
2024
|
+
livenessReason: liveness.reason,
|
|
2025
|
+
runnerHostname: liveness.runnerHostname
|
|
2026
|
+
})
|
|
2027
|
+
continue
|
|
2028
|
+
}
|
|
2029
|
+
}
|
|
2030
|
+
|
|
1936
2031
|
selections.push({
|
|
1937
2032
|
conditions: {
|
|
1938
2033
|
handed_off_at_ms: handoff.handedOffAtMs,
|
|
@@ -1948,6 +2043,10 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
|
|
|
1948
2043
|
|
|
1949
2044
|
return await this._markOrphanSelections({db, selections})
|
|
1950
2045
|
})
|
|
2046
|
+
|
|
2047
|
+
if (deferredLiveOwnerHandoffs.length > 0 && onLiveOwnerDeferred) await onLiveOwnerDeferred(deferredLiveOwnerHandoffs)
|
|
2048
|
+
|
|
2049
|
+
return orphanedJobs
|
|
1951
2050
|
}
|
|
1952
2051
|
|
|
1953
2052
|
/**
|
|
@@ -188,6 +188,11 @@
|
|
|
188
188
|
/**
|
|
189
189
|
* @typedef {BackgroundJobHandoffSnapshot & {liveness: import("./owner-liveness.js").BackgroundJobOwnerLivenessResult}} BackgroundJobHandoffOwnershipVerdict
|
|
190
190
|
*/
|
|
191
|
+
/**
|
|
192
|
+
* One exact lease an absence-based reclaim left with its runner because the
|
|
193
|
+
* recorded runner process is provably alive.
|
|
194
|
+
* @typedef {BackgroundJobHandoffSnapshot & {childPid: number | null, livenessReason: string, runnerHostname: string | null}} BackgroundJobDeferredLiveOwnerHandoff
|
|
195
|
+
*/
|
|
191
196
|
/**
|
|
192
197
|
* @typedef {object} BackgroundJobHandoffRequest
|
|
193
198
|
* @property {string} jobId - Job to claim.
|
|
@@ -192,13 +192,17 @@ export default class BackgroundJobsAdapter {
|
|
|
192
192
|
}): Promise<boolean>;
|
|
193
193
|
/**
|
|
194
194
|
* Returns a handed-off job to the queue.
|
|
195
|
-
* @param {{jobId: string,
|
|
196
|
-
* @returns {Promise<
|
|
195
|
+
* @param {{deferLiveOwners?: boolean, handoffId: string, hostname?: string, jobId: string, liveChildren?: import("./owner-liveness.js").BackgroundJobLiveChildIdentity[], probe?: (pid: number) => boolean}} _args - Handoff release and live-owner controls.
|
|
196
|
+
* @returns {Promise<boolean>} - Whether the lease was returned to the queue.
|
|
197
197
|
*/
|
|
198
198
|
markReturnedToQueue(_args: {
|
|
199
|
-
|
|
199
|
+
deferLiveOwners?: boolean;
|
|
200
200
|
handoffId: string;
|
|
201
|
-
|
|
201
|
+
hostname?: string;
|
|
202
|
+
jobId: string;
|
|
203
|
+
liveChildren?: import("./owner-liveness.js").BackgroundJobLiveChildIdentity[];
|
|
204
|
+
probe?: (pid: number) => boolean;
|
|
205
|
+
}): Promise<boolean>;
|
|
202
206
|
/**
|
|
203
207
|
* Finds active handoffs for a worker.
|
|
204
208
|
* @param {{workerId: string}} _args - Worker identity.
|
|
@@ -220,12 +224,17 @@ export default class BackgroundJobsAdapter {
|
|
|
220
224
|
* Applies orphan failure semantics to unchanged exact handoff snapshots.
|
|
221
225
|
* Adapters that return startup snapshots must implement the matching fenced
|
|
222
226
|
* transition.
|
|
223
|
-
* @param {{handoffs: import("./types.js").BackgroundJobHandoffSnapshot[],
|
|
227
|
+
* @param {{deferLiveOwners?: boolean, error: ReturnType<typeof JSON.parse>, handoffs: import("./types.js").BackgroundJobHandoffSnapshot[], hostname?: string, liveChildren?: import("./owner-liveness.js").BackgroundJobLiveChildIdentity[], onLiveOwnerDeferred?: (handoffs: import("./types.js").BackgroundJobDeferredLiveOwnerHandoff[]) => void | Promise<void>, probe?: (pid: number) => boolean}} _args - Exact leases, orphan reason, and live-owner controls.
|
|
224
228
|
* @returns {Promise<import("./types.js").BackgroundJobRow[]>} - Accepted transitions.
|
|
225
229
|
*/
|
|
226
230
|
markOrphanedHandoffs(_args: {
|
|
227
|
-
|
|
231
|
+
deferLiveOwners?: boolean;
|
|
228
232
|
error: ReturnType<typeof JSON.parse>;
|
|
233
|
+
handoffs: import("./types.js").BackgroundJobHandoffSnapshot[];
|
|
234
|
+
hostname?: string;
|
|
235
|
+
liveChildren?: import("./owner-liveness.js").BackgroundJobLiveChildIdentity[];
|
|
236
|
+
onLiveOwnerDeferred?: (handoffs: import("./types.js").BackgroundJobDeferredLiveOwnerHandoff[]) => void | Promise<void>;
|
|
237
|
+
probe?: (pid: number) => boolean;
|
|
229
238
|
}): Promise<import("./types.js").BackgroundJobRow[]>;
|
|
230
239
|
/**
|
|
231
240
|
* Evaluates recorded runner-process liveness for exact active handoffs.
|