velocious 1.0.662 → 1.0.663
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/build/background-jobs/adapter.js +9 -0
- package/build/background-jobs/job-runner.js +3 -1
- package/build/background-jobs/local-adapter.js +7 -0
- package/build/background-jobs/local-store.js +113 -3
- package/build/background-jobs/main.js +35 -1
- package/build/background-jobs/pooled-runner-child.js +38 -0
- package/build/background-jobs/status-reporter.js +97 -1
- package/build/background-jobs/store.js +130 -4
- package/build/background-jobs/types.js +6 -1
- package/build/background-jobs/web/controller.js +4 -0
- package/build/background-jobs/worker.js +59 -0
- package/build/src/background-jobs/adapter.d.ts +17 -0
- package/build/src/background-jobs/adapter.d.ts.map +1 -1
- package/build/src/background-jobs/adapter.js +9 -1
- package/build/src/background-jobs/job-runner.d.ts +3 -1
- package/build/src/background-jobs/job-runner.d.ts.map +1 -1
- package/build/src/background-jobs/job-runner.js +5 -2
- package/build/src/background-jobs/local-adapter.d.ts +13 -0
- package/build/src/background-jobs/local-adapter.d.ts.map +1 -1
- package/build/src/background-jobs/local-adapter.js +7 -1
- package/build/src/background-jobs/local-store.d.ts +38 -0
- package/build/src/background-jobs/local-store.d.ts.map +1 -1
- package/build/src/background-jobs/local-store.js +117 -4
- package/build/src/background-jobs/main.d.ts +14 -0
- package/build/src/background-jobs/main.d.ts.map +1 -1
- package/build/src/background-jobs/main.js +35 -2
- package/build/src/background-jobs/pooled-runner-child.js +38 -1
- package/build/src/background-jobs/status-reporter.d.ts +51 -1
- package/build/src/background-jobs/status-reporter.d.ts.map +1 -1
- package/build/src/background-jobs/status-reporter.js +89 -2
- package/build/src/background-jobs/store.d.ts +48 -0
- package/build/src/background-jobs/store.d.ts.map +1 -1
- package/build/src/background-jobs/store.js +128 -5
- package/build/src/background-jobs/types.d.ts +34 -2
- package/build/src/background-jobs/types.d.ts.map +1 -1
- package/build/src/background-jobs/types.js +7 -2
- package/build/src/background-jobs/web/controller.d.ts.map +1 -1
- package/build/src/background-jobs/web/controller.js +5 -1
- package/build/src/background-jobs/worker.d.ts +20 -0
- package/build/src/background-jobs/worker.d.ts.map +1 -1
- package/build/src/background-jobs/worker.js +58 -1
- package/package.json +1 -1
- package/src/background-jobs/adapter.js +9 -0
- package/src/background-jobs/job-runner.js +3 -1
- package/src/background-jobs/local-adapter.js +7 -0
- package/src/background-jobs/local-store.js +113 -3
- package/src/background-jobs/main.js +35 -1
- package/src/background-jobs/pooled-runner-child.js +38 -0
- package/src/background-jobs/status-reporter.js +97 -1
- package/src/background-jobs/store.js +130 -4
- package/src/background-jobs/types.js +6 -1
- package/src/background-jobs/web/controller.js +4 -0
- package/src/background-jobs/worker.js +59 -0
package/README.md
CHANGED
|
@@ -2614,7 +2614,7 @@ actual state sources must still agree.
|
|
|
2614
2614
|
|
|
2615
2615
|
`maxConcurrentInlineJobs` (default: `4`) caps how many `executionMode: "inline"` jobs a single `background-jobs-worker` process runs in parallel. Concurrency is at the JS event-loop level: every job in flight shares the worker's process and DB connection pool, so the cap should fit the pool, not the CPU count. Forking remains the right tool when you need memory isolation across long-running jobs or want to use more cores; select it with `executionMode: "forked"`.
|
|
2616
2616
|
|
|
2617
|
-
New jobs default to `executionMode: "pooled"`: a worker runs them in warm, reusable Node child runners. `pooledRunnerCount` (default: `4`) bounds this independent per-worker pool, and `pooledRunnerConcurrency` (default: `1`) sets how many jobs each child runs at once on its own event loop, so total pooled capacity is `pooledRunnerCount × pooledRunnerConcurrency` — raise concurrency for I/O-bound jobs to get high throughput from a bounded, isolated set of processes. Workers advertise that exact free-slot count and the main consumes one slot per durable handoff, allowing one readiness notification to fill the pool. If an initialized child exits unexpectedly, the worker immediately advertises the freed capacity while failure reports retry; the replacement is spawned lazily by the next dispatch, and a pre-startup crash does not trigger a respawn loop. `pooledRunnerCount`, `pooledRunnerConcurrency`, and `pooledRunnerMaxJobs` must be finite positive integers; the RSS and lifetime limits must be finite positive numbers. A child is recycled after an acknowledged job when it reaches `pooledRunnerMaxJobs` (default: `100`), `pooledRunnerMaxRssBytes` (default: `536870912`, or 512 MiB), or `pooledRunnerMaxLifetimeMs` (default: `3600000`, or one hour). `execution_mode` is the single source of truth for a job's runtime — pooled rows persist as `execution_mode = "pooled"` directly. See [execution modes and pooled runners](docs/background-jobs.md#execution-modes-and-pooled-runners).
|
|
2617
|
+
New jobs default to `executionMode: "pooled"`: a worker runs them in warm, reusable Node child runners. `pooledRunnerCount` (default: `4`) bounds this independent per-worker pool, and `pooledRunnerConcurrency` (default: `1`) sets how many jobs each child runs at once on its own event loop, so total pooled capacity is `pooledRunnerCount × pooledRunnerConcurrency` — raise concurrency for I/O-bound jobs to get high throughput from a bounded, isolated set of processes. Workers advertise that exact free-slot count and the main consumes one slot per durable handoff, allowing one readiness notification to fill the pool. If an initialized child exits unexpectedly, the worker immediately advertises the freed capacity while failure reports retry; the replacement is spawned lazily by the next dispatch, and a pre-startup crash does not trigger a respawn loop. `pooledRunnerCount`, `pooledRunnerConcurrency`, and `pooledRunnerMaxJobs` must be finite positive integers; the RSS and lifetime limits must be finite positive numbers. A child is recycled after an acknowledged job when it reaches `pooledRunnerMaxJobs` (default: `100`), `pooledRunnerMaxRssBytes` (default: `536870912`, or 512 MiB), or `pooledRunnerMaxLifetimeMs` (default: `3600000`, or one hour). `execution_mode` is the single source of truth for a job's runtime — pooled rows persist as `execution_mode = "pooled"` directly. Pooled jobs also persist acceptance evidence (`child_received_at_ms`, `child_started_at_ms`, `child_instance_id`, `child_pid`) so a handed-off job can be told apart from one whose runner never picked it up. See [execution modes and pooled runners](docs/background-jobs.md#execution-modes-and-pooled-runners).
|
|
2618
2618
|
|
|
2619
2619
|
Cold pooled jobs share one atomic model-bootstrap phase. If that phase fails, all
|
|
2620
2620
|
waiting jobs receive the failure; a later job in the surviving child cannot run
|
|
@@ -129,6 +129,15 @@ export default class BackgroundJobsAdapter {
|
|
|
129
129
|
*/
|
|
130
130
|
async markCompleted(_args) { throw new Error("BackgroundJobsAdapter#markCompleted is not implemented") }
|
|
131
131
|
|
|
132
|
+
/**
|
|
133
|
+
* Records pooled-child acceptance evidence (received/started timestamps plus
|
|
134
|
+
* runner identity) for a handed-off job, fenced by its active handoff lease.
|
|
135
|
+
* Only the fields supplied are written.
|
|
136
|
+
* @param {{jobId: string, handoffId?: string, workerId?: string, handedOffAtMs?: number, receivedAtMs?: number, startedAtMs?: number, childInstanceId?: string, childPid?: number}} _args - Acceptance report.
|
|
137
|
+
* @returns {Promise<boolean>} - Whether the fenced report was accepted.
|
|
138
|
+
*/
|
|
139
|
+
async markChildAccepted(_args) { throw new Error("BackgroundJobsAdapter#markChildAccepted is not implemented") }
|
|
140
|
+
|
|
132
141
|
/**
|
|
133
142
|
* Returns a handed-off job to its schedule.
|
|
134
143
|
* @param {{jobId: string, delayMs: number, handoffId?: string, workerId?: string, handedOffAtMs?: number}} _args - Reschedule report.
|
|
@@ -82,10 +82,11 @@ function runnerProcessTitle(JobClass, payload) {
|
|
|
82
82
|
* @param {object} [options] - Runner options.
|
|
83
83
|
* @param {boolean} [options.closeConnections] - Whether to gracefully close framework connections after the job.
|
|
84
84
|
* @param {boolean} [options.manageProcessTitle] - Whether to set the per-job process title and restore it afterwards. Off for concurrent pooled runners, where interleaved snapshot/restore of the single process-wide `process.title` would corrupt it; the pooled child owns an aggregate title instead.
|
|
85
|
+
* @param {() => void} [options.onPerformStart] - Observation hook fired once, immediately before perform runs (with its connection acquired). Pooled runners use it to report that the job actually started. Must not throw into the job.
|
|
85
86
|
* @param {string} [options.processType] - Generic application process type.
|
|
86
87
|
* @returns {Promise<"completed" | "rescheduled">} - Acknowledged outcome.
|
|
87
88
|
*/
|
|
88
|
-
export default async function runJobPayload(payload, {closeConnections = true, manageProcessTitle = true, processType = "background-jobs-runner"} = {}) {
|
|
89
|
+
export default async function runJobPayload(payload, {closeConnections = true, manageProcessTitle = true, onPerformStart, processType = "background-jobs-runner"} = {}) {
|
|
89
90
|
const configuration = await configurationResolver()
|
|
90
91
|
configuration.setCurrent()
|
|
91
92
|
await configuration.initialize({type: processType})
|
|
@@ -119,6 +120,7 @@ export default async function runJobPayload(payload, {closeConnections = true, m
|
|
|
119
120
|
try {
|
|
120
121
|
await runWithBackgroundJobPayload(payload, async () => {
|
|
121
122
|
await configuration.withConnections({databaseIdentifiers: JobClass.databaseIdentifiers, name: `Background job runner: ${payload.jobName}`}, async () => {
|
|
123
|
+
if (onPerformStart) onPerformStart()
|
|
122
124
|
await perform.apply(jobInstance, jobArgs)
|
|
123
125
|
})
|
|
124
126
|
})
|
|
@@ -133,6 +133,13 @@ export default class LocalBackgroundJobsAdapter extends BackgroundJobsAdapter {
|
|
|
133
133
|
*/
|
|
134
134
|
async markCompleted(args) { return await this.store.markCompleted(args) }
|
|
135
135
|
|
|
136
|
+
/**
|
|
137
|
+
* Records pooled-child acceptance evidence for a local handoff.
|
|
138
|
+
* @param {{jobId: string, handoffId?: string, receivedAtMs?: number, startedAtMs?: number, childInstanceId?: string, childPid?: number}} args - Acceptance report.
|
|
139
|
+
* @returns {Promise<boolean>} - Whether accepted.
|
|
140
|
+
*/
|
|
141
|
+
async markChildAccepted(args) { return await this.store.markChildAccepted(args) }
|
|
142
|
+
|
|
136
143
|
/**
|
|
137
144
|
* Acknowledges an explicit local reschedule.
|
|
138
145
|
* @param {{jobId: string, delayMs: number, handoffId?: string}} args - Reschedule report.
|
|
@@ -49,7 +49,11 @@ const EXPECTED_JOB_COLUMNS = [
|
|
|
49
49
|
"failed_at_ms",
|
|
50
50
|
"last_error",
|
|
51
51
|
"concurrency_key",
|
|
52
|
-
"max_concurrency"
|
|
52
|
+
"max_concurrency",
|
|
53
|
+
"child_received_at_ms",
|
|
54
|
+
"child_started_at_ms",
|
|
55
|
+
"child_instance_id",
|
|
56
|
+
"child_pid"
|
|
53
57
|
]
|
|
54
58
|
const EXPECTED_CONCURRENCY_COLUMNS = ["concurrency_key", "max_concurrency", "active_count"]
|
|
55
59
|
/** @type {WeakMap<import("../configuration.js").default, Map<string, Promise<void>>>} */
|
|
@@ -202,6 +206,7 @@ export default class LocalBackgroundJobsStore {
|
|
|
202
206
|
await db.createTable(this._jobsTableData())
|
|
203
207
|
changed = true
|
|
204
208
|
} else {
|
|
209
|
+
if (await this._ensureJobColumns(db)) changed = true
|
|
205
210
|
await this._assertColumns(db, LOCAL_BACKGROUND_JOBS_TABLE, EXPECTED_JOB_COLUMNS)
|
|
206
211
|
}
|
|
207
212
|
|
|
@@ -232,6 +237,45 @@ export default class LocalBackgroundJobsStore {
|
|
|
232
237
|
return changed
|
|
233
238
|
}
|
|
234
239
|
|
|
240
|
+
/**
|
|
241
|
+
* Idempotently adds columns from the current jobs table definition that an
|
|
242
|
+
* existing local table is missing, so an upgraded framework finds a
|
|
243
|
+
* compatible schema instead of failing the column assertion.
|
|
244
|
+
* @param {import("../database/drivers/base.js").default} db - Local SQLite connection.
|
|
245
|
+
* @returns {Promise<boolean>} - Whether a column was added.
|
|
246
|
+
*/
|
|
247
|
+
async _ensureJobColumns(db) {
|
|
248
|
+
db.clearSchemaCache()
|
|
249
|
+
const table = await db.getTableByNameOrFail(LOCAL_BACKGROUND_JOBS_TABLE)
|
|
250
|
+
const tableData = new TableData(LOCAL_BACKGROUND_JOBS_TABLE)
|
|
251
|
+
let added = false
|
|
252
|
+
|
|
253
|
+
for (const column of this._jobsTableData().getColumns()) {
|
|
254
|
+
if (await table.getColumnByName(column.getName())) continue
|
|
255
|
+
if (column.getPrimaryKey()) continue
|
|
256
|
+
|
|
257
|
+
const columnArgs = /** @type {{null: boolean, maxLength?: number}} */ ({null: column.getNull() !== false})
|
|
258
|
+
const maxLength = column.getMaxLength()
|
|
259
|
+
|
|
260
|
+
if (typeof maxLength === "number") columnArgs.maxLength = maxLength
|
|
261
|
+
|
|
262
|
+
const type = column.getType()
|
|
263
|
+
if (type === "string") tableData.string(column.getName(), columnArgs)
|
|
264
|
+
else if (type === "text") tableData.text(column.getName(), columnArgs)
|
|
265
|
+
else if (type === "bigint") tableData.bigint(column.getName(), columnArgs)
|
|
266
|
+
else if (type === "integer") tableData.integer(column.getName(), columnArgs)
|
|
267
|
+
else if (type === "boolean") tableData.boolean(column.getName(), columnArgs)
|
|
268
|
+
else continue
|
|
269
|
+
added = true
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
if (!added) return false
|
|
273
|
+
|
|
274
|
+
for (const sql of await db.alterTableSQLs(tableData)) await db.query(sql)
|
|
275
|
+
db.clearSchemaCache()
|
|
276
|
+
return true
|
|
277
|
+
}
|
|
278
|
+
|
|
235
279
|
/**
|
|
236
280
|
* Builds the migration ledger table definition.
|
|
237
281
|
* @returns {TableData} - Migration ledger table.
|
|
@@ -272,6 +316,10 @@ export default class LocalBackgroundJobsStore {
|
|
|
272
316
|
table.text("last_error", {null: true})
|
|
273
317
|
table.string("concurrency_key", {null: true})
|
|
274
318
|
table.integer("max_concurrency", {null: true})
|
|
319
|
+
table.bigint("child_received_at_ms", {null: true})
|
|
320
|
+
table.bigint("child_started_at_ms", {null: true})
|
|
321
|
+
table.string("child_instance_id", {null: true})
|
|
322
|
+
table.integer("child_pid", {null: true})
|
|
275
323
|
table.addIndex(new TableIndex(["status", "scheduled_at_ms", "created_at_ms", "id"], {name: LOCAL_BACKGROUND_JOBS_INDEX_NAMES[0]}))
|
|
276
324
|
table.addIndex(new TableIndex(["queue", "status", "created_at_ms"], {name: LOCAL_BACKGROUND_JOBS_INDEX_NAMES[1]}))
|
|
277
325
|
table.addIndex(new TableIndex(["args_digest"], {name: LOCAL_BACKGROUND_JOBS_INDEX_NAMES[2]}))
|
|
@@ -708,7 +756,7 @@ export default class LocalBackgroundJobsStore {
|
|
|
708
756
|
const handedOffAtMs = this.clock.now()
|
|
709
757
|
const affectedRows = await this._updateAffectedRows(db, {
|
|
710
758
|
conditions: {id: jobId, status: "queued"},
|
|
711
|
-
data: {handed_off_at_ms: handedOffAtMs, handoff_id: handoffId, status: "handed_off", worker_id: workerId || "local"},
|
|
759
|
+
data: {...this._clearedChildAcceptanceData(), handed_off_at_ms: handedOffAtMs, handoff_id: handoffId, status: "handed_off", worker_id: workerId || "local"},
|
|
712
760
|
tableName: LOCAL_BACKGROUND_JOBS_TABLE
|
|
713
761
|
})
|
|
714
762
|
|
|
@@ -763,6 +811,7 @@ export default class LocalBackgroundJobsStore {
|
|
|
763
811
|
const affectedRows = await this._updateAffectedRows(db, {
|
|
764
812
|
conditions: {handoff_id: handoffId, id: jobId, status: "handed_off"},
|
|
765
813
|
data: {
|
|
814
|
+
...this._clearedChildAcceptanceData(),
|
|
766
815
|
handed_off_at_ms: null,
|
|
767
816
|
handoff_id: null,
|
|
768
817
|
scheduled_at_ms: this.clock.now(),
|
|
@@ -802,6 +851,43 @@ export default class LocalBackgroundJobsStore {
|
|
|
802
851
|
}))
|
|
803
852
|
}
|
|
804
853
|
|
|
854
|
+
/**
|
|
855
|
+
* Records pooled-child acceptance evidence for an active handoff. Only the
|
|
856
|
+
* fields supplied are written, fenced by the exact active handoff lease.
|
|
857
|
+
* @param {object} args - Acceptance report.
|
|
858
|
+
* @param {string} args.jobId - Job id.
|
|
859
|
+
* @param {string} [args.handoffId] - Handoff lease id.
|
|
860
|
+
* @param {number} [args.receivedAtMs] - Epoch ms the runner child received the job.
|
|
861
|
+
* @param {number} [args.startedAtMs] - Epoch ms the job's perform started in the child.
|
|
862
|
+
* @param {string} [args.childInstanceId] - Stable pooled child identity.
|
|
863
|
+
* @param {number} [args.childPid] - Pooled child OS pid.
|
|
864
|
+
* @returns {Promise<boolean>} - Whether the lease won.
|
|
865
|
+
*/
|
|
866
|
+
async markChildAccepted({jobId, handoffId, receivedAtMs, startedAtMs, childInstanceId, childPid}) {
|
|
867
|
+
await this.ensureReady()
|
|
868
|
+
|
|
869
|
+
return await this._withDb(async (connection) => await this._mutate(connection, async (db) => {
|
|
870
|
+
const job = await this._getJob(db, jobId)
|
|
871
|
+
|
|
872
|
+
if (!this._acceptsHandoff(job, handoffId)) return false
|
|
873
|
+
|
|
874
|
+
const data = {}
|
|
875
|
+
if (typeof receivedAtMs === "number") data.child_received_at_ms = receivedAtMs
|
|
876
|
+
if (typeof startedAtMs === "number") data.child_started_at_ms = startedAtMs
|
|
877
|
+
if (typeof childInstanceId === "string") data.child_instance_id = childInstanceId
|
|
878
|
+
if (typeof childPid === "number") data.child_pid = childPid
|
|
879
|
+
if (Object.keys(data).length === 0) return false
|
|
880
|
+
|
|
881
|
+
const affectedRows = await this._updateAffectedRows(db, {
|
|
882
|
+
conditions: {handoff_id: handoffId, id: jobId, status: "handed_off"},
|
|
883
|
+
data,
|
|
884
|
+
tableName: LOCAL_BACKGROUND_JOBS_TABLE
|
|
885
|
+
})
|
|
886
|
+
|
|
887
|
+
return affectedRows === 1
|
|
888
|
+
}))
|
|
889
|
+
}
|
|
890
|
+
|
|
805
891
|
/**
|
|
806
892
|
* Applies a fenced reschedule without consuming an attempt.
|
|
807
893
|
* @param {{jobId: string, handoffId?: string, delayMs: number}} args - Reschedule report.
|
|
@@ -819,6 +905,7 @@ export default class LocalBackgroundJobsStore {
|
|
|
819
905
|
const affectedRows = await this._updateAffectedRows(db, {
|
|
820
906
|
conditions: {handoff_id: handoffId, id: jobId, status: "handed_off"},
|
|
821
907
|
data: {
|
|
908
|
+
...this._clearedChildAcceptanceData(),
|
|
822
909
|
handed_off_at_ms: null,
|
|
823
910
|
handoff_id: null,
|
|
824
911
|
scheduled_at_ms: rescheduledBackgroundJobAtMs(delayMs, this.clock.now()),
|
|
@@ -917,7 +1004,9 @@ export default class LocalBackgroundJobsStore {
|
|
|
917
1004
|
}
|
|
918
1005
|
|
|
919
1006
|
if (willRetry) {
|
|
920
|
-
|
|
1007
|
+
// A retry starts a fresh handoff with a possibly different runner, so the
|
|
1008
|
+
// previous child's acceptance evidence must not leak into the next attempt.
|
|
1009
|
+
Object.assign(data, {scheduled_at_ms: nowMs + retryDelayMs(attempts), ...this._clearedChildAcceptanceData()})
|
|
921
1010
|
} else {
|
|
922
1011
|
Object.assign(data, {failed_at_ms: nowMs})
|
|
923
1012
|
}
|
|
@@ -934,6 +1023,7 @@ export default class LocalBackgroundJobsStore {
|
|
|
934
1023
|
|
|
935
1024
|
return {
|
|
936
1025
|
...job,
|
|
1026
|
+
...(willRetry ? this._clearedChildAcceptanceRow() : {}),
|
|
937
1027
|
attempts,
|
|
938
1028
|
failedAtMs: willRetry ? job.failedAtMs : nowMs,
|
|
939
1029
|
handedOffAtMs: null,
|
|
@@ -945,6 +1035,22 @@ export default class LocalBackgroundJobsStore {
|
|
|
945
1035
|
}
|
|
946
1036
|
}
|
|
947
1037
|
|
|
1038
|
+
/**
|
|
1039
|
+
* Returns the database data that clears pooled-child acceptance evidence.
|
|
1040
|
+
* @returns {Record<string, ReturnType<typeof JSON.parse>>} - Cleared acceptance columns.
|
|
1041
|
+
*/
|
|
1042
|
+
_clearedChildAcceptanceData() {
|
|
1043
|
+
return {child_instance_id: null, child_pid: null, child_received_at_ms: null, child_started_at_ms: null}
|
|
1044
|
+
}
|
|
1045
|
+
|
|
1046
|
+
/**
|
|
1047
|
+
* Returns the row-shape counterpart of the cleared acceptance columns.
|
|
1048
|
+
* @returns {Pick<import("./types.js").BackgroundJobRow, "childInstanceId" | "childPid" | "childReceivedAtMs" | "childStartedAtMs">} - Cleared acceptance fields.
|
|
1049
|
+
*/
|
|
1050
|
+
_clearedChildAcceptanceRow() {
|
|
1051
|
+
return {childInstanceId: null, childPid: null, childReceivedAtMs: null, childStartedAtMs: null}
|
|
1052
|
+
}
|
|
1053
|
+
|
|
948
1054
|
/**
|
|
949
1055
|
* Ensures that a durable concurrency counter exists with the required cap.
|
|
950
1056
|
* @param {import("../database/drivers/base.js").default} db - Local SQLite connection.
|
|
@@ -1106,6 +1212,10 @@ export default class LocalBackgroundJobsStore {
|
|
|
1106
1212
|
return {
|
|
1107
1213
|
args: parsedArgs,
|
|
1108
1214
|
attempts: this._numberOrNull(row.attempts),
|
|
1215
|
+
childInstanceId: row.child_instance_id === null || row.child_instance_id === undefined ? null : String(row.child_instance_id),
|
|
1216
|
+
childPid: this._numberOrNull(row.child_pid),
|
|
1217
|
+
childReceivedAtMs: this._numberOrNull(row.child_received_at_ms),
|
|
1218
|
+
childStartedAtMs: this._numberOrNull(row.child_started_at_ms),
|
|
1109
1219
|
completedAtMs: this._numberOrNull(row.completed_at_ms),
|
|
1110
1220
|
concurrencyKey: row.concurrency_key === null || row.concurrency_key === undefined ? null : String(row.concurrency_key),
|
|
1111
1221
|
createdAtMs: this._numberOrNull(row.created_at_ms),
|
|
@@ -1219,6 +1219,11 @@ export default class BackgroundJobsMain {
|
|
|
1219
1219
|
}
|
|
1220
1220
|
return
|
|
1221
1221
|
}
|
|
1222
|
+
if (message?.type === "job-accepted") {
|
|
1223
|
+
await this._handleJobAccepted({jsonSocket, message})
|
|
1224
|
+
return
|
|
1225
|
+
}
|
|
1226
|
+
|
|
1222
1227
|
if (message?.type === "job-complete") {
|
|
1223
1228
|
await this._handleJobComplete({jsonSocket, message})
|
|
1224
1229
|
return
|
|
@@ -1234,6 +1239,35 @@ export default class BackgroundJobsMain {
|
|
|
1234
1239
|
}
|
|
1235
1240
|
}
|
|
1236
1241
|
|
|
1242
|
+
/**
|
|
1243
|
+
* Persists pooled-child acceptance evidence for an active handoff. The
|
|
1244
|
+
* report is diagnostic: a stale lease (job already reclaimed or terminal)
|
|
1245
|
+
* answers the same `job-updated` acknowledgement as an accepted report, and
|
|
1246
|
+
* only a store failure answers `job-update-error` so the worker can retry.
|
|
1247
|
+
* @param {object} args - Options.
|
|
1248
|
+
* @param {JsonSocket} args.jsonSocket - JSON socket.
|
|
1249
|
+
* @param {import("./types.js").BackgroundJobAcceptedMessage} args.message - Message.
|
|
1250
|
+
* @returns {Promise<void>} - Resolves when handled.
|
|
1251
|
+
*/
|
|
1252
|
+
async _handleJobAccepted({jsonSocket, message}) {
|
|
1253
|
+
try {
|
|
1254
|
+
await this.store.markChildAccepted({
|
|
1255
|
+
childInstanceId: message.childInstanceId,
|
|
1256
|
+
childPid: message.childPid,
|
|
1257
|
+
handedOffAtMs: message.handedOffAtMs,
|
|
1258
|
+
handoffId: message.handoffId,
|
|
1259
|
+
jobId: message.jobId,
|
|
1260
|
+
receivedAtMs: message.receivedAtMs,
|
|
1261
|
+
startedAtMs: message.startedAtMs,
|
|
1262
|
+
workerId: message.workerId
|
|
1263
|
+
})
|
|
1264
|
+
jsonSocket.send({type: "job-updated", jobId: message.jobId})
|
|
1265
|
+
} catch (error) {
|
|
1266
|
+
this._reportJobUpdateFailure({error, jobId: message.jobId, stage: "background-job-accepted"})
|
|
1267
|
+
jsonSocket.send({type: "job-update-error", jobId: message.jobId, error: "Failed to update job"})
|
|
1268
|
+
}
|
|
1269
|
+
}
|
|
1270
|
+
|
|
1237
1271
|
/**
|
|
1238
1272
|
* Requires the complete durable lease identity before a generation-mode
|
|
1239
1273
|
* reporter can mutate a job. Legacy reporters keep their permissive protocol.
|
|
@@ -1241,7 +1275,7 @@ export default class BackgroundJobsMain {
|
|
|
1241
1275
|
* @returns {boolean} - Whether the report lacks its exact generation lease.
|
|
1242
1276
|
*/
|
|
1243
1277
|
_generationReportIsInvalid(message) {
|
|
1244
|
-
if (message?.type !== "job-complete" && message?.type !== "job-failed" && message?.type !== "job-reschedule") return false
|
|
1278
|
+
if (message?.type !== "job-accepted" && message?.type !== "job-complete" && message?.type !== "job-failed" && message?.type !== "job-reschedule") return false
|
|
1245
1279
|
const generationId = this.generationId
|
|
1246
1280
|
if (!generationId) return false
|
|
1247
1281
|
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
// @ts-check
|
|
2
2
|
|
|
3
|
+
import { randomUUID } from "node:crypto"
|
|
3
4
|
import runJobPayload, { BackgroundJobPerformedFailure } from "./job-runner.js"
|
|
4
5
|
import { closeRunnerConnections, closeRunnerFrameworkConnections, currentConfigurationOrNull } from "./runner-graceful-shutdown.js"
|
|
5
6
|
import setRunnerProcessTitle from "./runner-process-title.js"
|
|
@@ -7,6 +8,8 @@ import PooledRunnerBrokerIdentity from "./pooled-runner-broker-identity.js"
|
|
|
7
8
|
import { runWithSharedTransactionBrokerConfig } from "../testing/shared-transaction-proxy-driver.js"
|
|
8
9
|
|
|
9
10
|
const BASE_PROCESS_TITLE = "velocious background-jobs-runner"
|
|
11
|
+
/** Stable identity of this pooled child process for the life of the process. */
|
|
12
|
+
const childInstanceId = randomUUID()
|
|
10
13
|
|
|
11
14
|
setRunnerProcessTitle()
|
|
12
15
|
|
|
@@ -57,6 +60,39 @@ function updateProcessTitle() {
|
|
|
57
60
|
process.title = count > 0 ? `${BASE_PROCESS_TITLE}: ${count} ${count === 1 ? "job" : "jobs"}` : BASE_PROCESS_TITLE
|
|
58
61
|
}
|
|
59
62
|
|
|
63
|
+
/**
|
|
64
|
+
* Reports one acceptance observation (job received / perform started) to the
|
|
65
|
+
* worker over IPC. The message carries the job's exact handoff lease so the
|
|
66
|
+
* worker can persist it fenced without any other lookup. Send failures are
|
|
67
|
+
* swallowed: a dead IPC channel is terminal for this child (the disconnect
|
|
68
|
+
* handler owns shutdown), and losing acceptance evidence must never fail the
|
|
69
|
+
* job itself.
|
|
70
|
+
* @param {"job-received" | "job-started"} type - Observation kind.
|
|
71
|
+
* @param {import("./types.js").BackgroundJobPayload & {id: string}} payload - Job payload carrying the handoff lease.
|
|
72
|
+
* @param {number} observedAtMs - Epoch ms of the observation.
|
|
73
|
+
* @returns {void}
|
|
74
|
+
*/
|
|
75
|
+
function sendChildAcceptance(type, payload, observedAtMs) {
|
|
76
|
+
if (!process.send) return
|
|
77
|
+
|
|
78
|
+
const message = {
|
|
79
|
+
childInstanceId,
|
|
80
|
+
childPid: process.pid,
|
|
81
|
+
handedOffAtMs: payload.handedOffAtMs,
|
|
82
|
+
handoffId: payload.handoffId,
|
|
83
|
+
jobId: payload.id,
|
|
84
|
+
type,
|
|
85
|
+
workerId: payload.workerId,
|
|
86
|
+
...(type === "job-received" ? {receivedAtMs: observedAtMs} : {startedAtMs: observedAtMs})
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
try {
|
|
90
|
+
process.send(message)
|
|
91
|
+
} catch {
|
|
92
|
+
// The IPC channel is already gone; the disconnect handler owns shutdown.
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
|
|
60
96
|
/**
|
|
61
97
|
* Checks whether an IPC value is a runnable pooled job message.
|
|
62
98
|
* @param {ReturnType<typeof JSON.parse>} message - IPC message.
|
|
@@ -114,6 +150,7 @@ async function runJob(payload, sharedTransactionBroker) {
|
|
|
114
150
|
return await runJobPayload(payload, {
|
|
115
151
|
closeConnections: false,
|
|
116
152
|
manageProcessTitle: false,
|
|
153
|
+
onPerformStart: () => sendChildAcceptance("job-started", payload, Date.now()),
|
|
117
154
|
processType: "background-jobs-pooled-runner"
|
|
118
155
|
})
|
|
119
156
|
})
|
|
@@ -143,6 +180,7 @@ function handleMessage(message) {
|
|
|
143
180
|
|
|
144
181
|
runningJobIds.add(message.payload.id)
|
|
145
182
|
updateProcessTitle()
|
|
183
|
+
sendChildAcceptance("job-received", message.payload, Date.now())
|
|
146
184
|
void runJob(message.payload, message.sharedTransactionBroker || {expected: false})
|
|
147
185
|
}
|
|
148
186
|
|
|
@@ -92,6 +92,102 @@ export default class BackgroundJobsStatusReporter {
|
|
|
92
92
|
})
|
|
93
93
|
}
|
|
94
94
|
|
|
95
|
+
/**
|
|
96
|
+
* Runs report child accepted.
|
|
97
|
+
* @param {object} args - Options.
|
|
98
|
+
* @param {string} args.jobId - Job id.
|
|
99
|
+
* @param {string} [args.handoffId] - Handoff lease id.
|
|
100
|
+
* @param {string} [args.workerId] - Worker id.
|
|
101
|
+
* @param {number} [args.handedOffAtMs] - Handed off timestamp.
|
|
102
|
+
* @param {number} [args.receivedAtMs] - Epoch ms the runner child received the job.
|
|
103
|
+
* @param {number} [args.startedAtMs] - Epoch ms the job's perform started in the child.
|
|
104
|
+
* @param {string} [args.childInstanceId] - Stable pooled child identity.
|
|
105
|
+
* @param {number} [args.childPid] - Pooled child OS pid.
|
|
106
|
+
* @returns {Promise<void>} - Resolves when reported.
|
|
107
|
+
*/
|
|
108
|
+
async reportChildAccepted({jobId, handoffId, workerId, handedOffAtMs, receivedAtMs, startedAtMs, childInstanceId, childPid}) {
|
|
109
|
+
const config = this.configuration.getBackgroundJobsConfig()
|
|
110
|
+
const host = this.host || config.host
|
|
111
|
+
const port = typeof this.port === "number" ? this.port : config.port
|
|
112
|
+
const {generationId} = this.configuration.resolveBackgroundJobsGenerationConfig({
|
|
113
|
+
generationId: this.explicitGenerationId,
|
|
114
|
+
sourceName: "BackgroundJobsStatusReporter"
|
|
115
|
+
})
|
|
116
|
+
|
|
117
|
+
await timeout({timeout: this.attemptTimeoutMs}, async ({control}) => {
|
|
118
|
+
const request = new BackgroundJobsSocketRequest({host, port, role: "reporter", generationHandshakeTimeoutMs: this.generationHandshakeTimeoutMs, generationId})
|
|
119
|
+
|
|
120
|
+
this._lastRequest = request
|
|
121
|
+
|
|
122
|
+
await request.run({
|
|
123
|
+
signal: control.signal,
|
|
124
|
+
onConnect: (jsonSocket) => {
|
|
125
|
+
jsonSocket.send({
|
|
126
|
+
type: "job-accepted",
|
|
127
|
+
jobId,
|
|
128
|
+
handoffId,
|
|
129
|
+
workerId,
|
|
130
|
+
handedOffAtMs,
|
|
131
|
+
receivedAtMs,
|
|
132
|
+
startedAtMs,
|
|
133
|
+
childInstanceId,
|
|
134
|
+
childPid
|
|
135
|
+
})
|
|
136
|
+
},
|
|
137
|
+
onMessage: ({message, resolve, reject}) => {
|
|
138
|
+
if (message?.type === "job-updated" && message.jobId === jobId) {
|
|
139
|
+
resolve(undefined)
|
|
140
|
+
return
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
if (message?.type === "job-update-error" && message.jobId === jobId) {
|
|
144
|
+
reject(new BackgroundJobUpdateError(message.error || "Job update failed"))
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
})
|
|
148
|
+
})
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
/**
|
|
152
|
+
* Runs report child accepted with retry. Acceptance evidence is diagnostic
|
|
153
|
+
* rather than terminal, so a transient main/DB failure retries only until
|
|
154
|
+
* `maxDurationMs` elapses and then gives up instead of stranding the report.
|
|
155
|
+
* @param {object} args - Options.
|
|
156
|
+
* @param {string} args.jobId - Job id.
|
|
157
|
+
* @param {string} [args.handoffId] - Handoff lease id.
|
|
158
|
+
* @param {string} [args.workerId] - Worker id.
|
|
159
|
+
* @param {number} [args.handedOffAtMs] - Handed off timestamp.
|
|
160
|
+
* @param {number} [args.receivedAtMs] - Epoch ms the runner child received the job.
|
|
161
|
+
* @param {number} [args.startedAtMs] - Epoch ms the job's perform started in the child.
|
|
162
|
+
* @param {string} [args.childInstanceId] - Stable pooled child identity.
|
|
163
|
+
* @param {number} [args.childPid] - Pooled child OS pid.
|
|
164
|
+
* @param {number} args.maxDurationMs - Max duration for retries.
|
|
165
|
+
* @returns {Promise<void>} - Resolves when reported or the budget elapses.
|
|
166
|
+
*/
|
|
167
|
+
async reportChildAcceptedWithRetry({jobId, handoffId, workerId, handedOffAtMs, receivedAtMs, startedAtMs, childInstanceId, childPid, maxDurationMs}) {
|
|
168
|
+
let attempt = 0
|
|
169
|
+
const startTime = Date.now()
|
|
170
|
+
|
|
171
|
+
while (true) {
|
|
172
|
+
try {
|
|
173
|
+
await this.reportChildAccepted({jobId, handoffId, workerId, handedOffAtMs, receivedAtMs, startedAtMs, childInstanceId, childPid})
|
|
174
|
+
return
|
|
175
|
+
} catch (error) {
|
|
176
|
+
attempt += 1
|
|
177
|
+
const delaySeconds = Math.min(30, 0.5 * attempt)
|
|
178
|
+
|
|
179
|
+
this.logger.debug(() => ["Background job child-acceptance report failed, retrying", error])
|
|
180
|
+
|
|
181
|
+
if (Date.now() - startTime >= maxDurationMs) {
|
|
182
|
+
this.logger.warn(() => ["Background job child-acceptance report timed out, giving up", error])
|
|
183
|
+
throw error
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
await wait(delaySeconds)
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
|
|
95
191
|
/**
|
|
96
192
|
* Runs report with retry.
|
|
97
193
|
* @param {object} args - Options.
|
|
@@ -104,7 +200,7 @@ export default class BackgroundJobsStatusReporter {
|
|
|
104
200
|
* @param {string} [args.workerId] - Worker id.
|
|
105
201
|
* @param {import("./types.js").PooledRunnerFailure} [args.runnerFailure] - Pooled-child process failure provenance.
|
|
106
202
|
* @param {number} [args.maxDurationMs] - Max duration for retries.
|
|
107
|
-
* @param {boolean} [args.retryPersistErrors] - Retry a `BackgroundJobUpdateError` (main's `job-update-error`, i.e. a transient DB failure while persisting the terminal status) instead of throwing immediately. Off by default so short-lived forked/spawned runners keep failing loudly and exit non-zero to be reclaimed; on for the long-lived worker, which cannot exit
|
|
203
|
+
* @param {boolean} [args.retryPersistErrors] - Retry a `BackgroundJobUpdateError` (main's `job-update-error`, i.e. a transient DB failure while persisting the terminal status) instead of throwing immediately. Off by default so short-lived forked/spawned runners keep failing loudly and exit non-zero to be reclaimed; on for the long-lived worker, which cannot exit to trigger orphan reclaim and would otherwise drop the completion and strand the row in `handed_off`.
|
|
108
204
|
* @returns {Promise<void>} - Resolves when reported.
|
|
109
205
|
*/
|
|
110
206
|
async reportWithRetry({jobId, status, delayMs, error, handoffId, handedOffAtMs, workerId, runnerFailure, maxDurationMs, retryPersistErrors = false}) {
|