velocious 1.0.611 → 1.0.613
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/build/background-jobs/adapter.js +3 -1
- package/build/background-jobs/json-socket.js +4 -0
- package/build/background-jobs/local-adapter.js +1 -1
- package/build/background-jobs/local-store.js +2 -3
- package/build/background-jobs/main.js +166 -12
- package/build/background-jobs/pooled-runner-child.js +1 -0
- package/build/background-jobs/store.js +2 -3
- package/build/background-jobs/types.js +6 -0
- package/build/background-jobs/worker.js +65 -27
- package/build/src/background-jobs/adapter.d.ts +4 -5
- package/build/src/background-jobs/adapter.d.ts.map +1 -1
- package/build/src/background-jobs/adapter.js +4 -2
- package/build/src/background-jobs/json-socket.d.ts +4 -0
- package/build/src/background-jobs/json-socket.d.ts.map +1 -1
- package/build/src/background-jobs/json-socket.js +5 -1
- package/build/src/background-jobs/local-adapter.d.ts +2 -5
- package/build/src/background-jobs/local-adapter.d.ts.map +1 -1
- package/build/src/background-jobs/local-adapter.js +2 -2
- package/build/src/background-jobs/local-store.d.ts +2 -5
- package/build/src/background-jobs/local-store.d.ts.map +1 -1
- package/build/src/background-jobs/local-store.js +3 -4
- package/build/src/background-jobs/main.d.ts +90 -1
- package/build/src/background-jobs/main.d.ts.map +1 -1
- package/build/src/background-jobs/main.js +161 -15
- package/build/src/background-jobs/pooled-runner-child.js +3 -1
- package/build/src/background-jobs/store.d.ts +3 -1
- package/build/src/background-jobs/store.d.ts.map +1 -1
- package/build/src/background-jobs/store.js +3 -3
- package/build/src/background-jobs/types.d.ts +20 -0
- package/build/src/background-jobs/types.d.ts.map +1 -1
- package/build/src/background-jobs/types.js +7 -1
- package/build/src/background-jobs/worker.d.ts +23 -10
- package/build/src/background-jobs/worker.d.ts.map +1 -1
- package/build/src/background-jobs/worker.js +67 -29
- package/package.json +1 -1
- package/src/background-jobs/adapter.js +3 -1
- package/src/background-jobs/json-socket.js +4 -0
- package/src/background-jobs/local-adapter.js +1 -1
- package/src/background-jobs/local-store.js +2 -3
- package/src/background-jobs/main.js +166 -12
- package/src/background-jobs/pooled-runner-child.js +1 -0
- package/src/background-jobs/store.js +2 -3
- package/src/background-jobs/types.js +6 -0
- package/src/background-jobs/worker.js +65 -27
package/package.json
CHANGED
|
@@ -83,7 +83,9 @@ export default class BackgroundJobsAdapter {
|
|
|
83
83
|
|
|
84
84
|
/**
|
|
85
85
|
* Starts a job by claiming its durable handoff.
|
|
86
|
-
*
|
|
86
|
+
* When `handoffId` is supplied, the adapter must persist and return that exact
|
|
87
|
+
* id so the caller can fence an ambiguous commit acknowledgement.
|
|
88
|
+
* @param {import("./types.js").BackgroundJobHandoffRequest} _args - Handoff request.
|
|
87
89
|
* @returns {Promise<import("./types.js").BackgroundJobHandoff | null>} - Claimed handoff.
|
|
88
90
|
*/
|
|
89
91
|
async markHandedOff(_args) { throw new Error("BackgroundJobsAdapter#markHandedOff is not implemented") }
|
|
@@ -32,6 +32,10 @@ export default class JsonSocket extends EventEmitter {
|
|
|
32
32
|
this.availablePooledSlots = 0
|
|
33
33
|
/** Whether the worker/main pair uses consumable pooled-capacity credits. */
|
|
34
34
|
this.usesPooledCapacityCredits = false
|
|
35
|
+
/** Whether this worker has permanently stopped accepting new handoffs. */
|
|
36
|
+
this.isDraining = false
|
|
37
|
+
/** Monotonic generation of the worker's latest readiness advertisement. */
|
|
38
|
+
this.readinessVersion = 0
|
|
35
39
|
/**
|
|
36
40
|
* Narrows the runtime value to the documented type.
|
|
37
41
|
* @type {boolean} */
|
|
@@ -107,7 +107,7 @@ export default class LocalBackgroundJobsAdapter extends BackgroundJobsAdapter {
|
|
|
107
107
|
|
|
108
108
|
/**
|
|
109
109
|
* Claims one queued local job.
|
|
110
|
-
* @param {
|
|
110
|
+
* @param {import("./types.js").BackgroundJobHandoffRequest} args - Claim request. A supplied handoff id is persisted exactly.
|
|
111
111
|
* @returns {Promise<import("./types.js").BackgroundJobHandoff | null>} - Handoff.
|
|
112
112
|
*/
|
|
113
113
|
async markHandedOff(args) { return await this.store.markHandedOff(args) }
|
|
@@ -693,10 +693,10 @@ export default class LocalBackgroundJobsStore {
|
|
|
693
693
|
|
|
694
694
|
/**
|
|
695
695
|
* Atomically reserves concurrency and claims one queued job.
|
|
696
|
-
* @param {
|
|
696
|
+
* @param {import("./types.js").BackgroundJobHandoffRequest} args - Claim request. A supplied handoff id is persisted exactly.
|
|
697
697
|
* @returns {Promise<import("./types.js").BackgroundJobHandoff | null>} - Fenced claim.
|
|
698
698
|
*/
|
|
699
|
-
async markHandedOff({jobId, workerId}) {
|
|
699
|
+
async markHandedOff({jobId, handoffId = new UUID(4).format(), workerId}) {
|
|
700
700
|
await this.ensureReady()
|
|
701
701
|
|
|
702
702
|
return await this._withDb(async (connection) => await this._mutate(connection, async (db) => {
|
|
@@ -706,7 +706,6 @@ export default class LocalBackgroundJobsStore {
|
|
|
706
706
|
if (job.concurrencyKey && !(await this._reserveConcurrency(db, job.concurrencyKey))) return null
|
|
707
707
|
|
|
708
708
|
const handedOffAtMs = this.clock.now()
|
|
709
|
-
const handoffId = new UUID(4).format()
|
|
710
709
|
const affectedRows = await this._updateAffectedRows(db, {
|
|
711
710
|
conditions: {id: jobId, status: "queued"},
|
|
712
711
|
data: {handed_off_at_ms: handedOffAtMs, handoff_id: handoffId, status: "handed_off", worker_id: workerId || "local"},
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
// @ts-check
|
|
2
2
|
|
|
3
|
+
import { randomUUID } from "crypto"
|
|
3
4
|
import net from "net"
|
|
4
5
|
import JsonSocket from "./json-socket.js"
|
|
5
6
|
import BackgroundJobsScheduler from "./scheduler.js"
|
|
@@ -92,6 +93,12 @@ export default class BackgroundJobsMain {
|
|
|
92
93
|
* Active durable handoffs keyed by the exact worker socket that received them.
|
|
93
94
|
* @type {Map<JsonSocket, Map<string, string>>} */
|
|
94
95
|
this.workerHandoffs = new Map()
|
|
96
|
+
/**
|
|
97
|
+
* Exact caller-generated leases whose claim outcome was ambiguous or whose
|
|
98
|
+
* pre-dispatch release has not yet been acknowledged. Retained until a
|
|
99
|
+
* fenced return succeeds (including an exact no-op).
|
|
100
|
+
* @type {Map<string, string>} */
|
|
101
|
+
this.pendingHandoffRecoveries = new Map()
|
|
95
102
|
/**
|
|
96
103
|
* Handoff-adoption queries started by worker hello messages. Shutdown must
|
|
97
104
|
* wait for these before closing the configuration's database pools.
|
|
@@ -667,6 +674,7 @@ export default class BackgroundJobsMain {
|
|
|
667
674
|
* @returns {void}
|
|
668
675
|
*/
|
|
669
676
|
_handleWorkerReady({jsonSocket, message}) {
|
|
677
|
+
jsonSocket.readinessVersion += 1
|
|
670
678
|
jsonSocket.acceptsSpawnedJobs = message.acceptsSpawned !== false && message.acceptsForked !== false
|
|
671
679
|
jsonSocket.acceptsForkedJobs = message.acceptsForked !== false
|
|
672
680
|
jsonSocket.acceptsPooledJobs = message.acceptsPooled === true
|
|
@@ -676,7 +684,7 @@ export default class BackgroundJobsMain {
|
|
|
676
684
|
? availablePooledSlots
|
|
677
685
|
: 0
|
|
678
686
|
jsonSocket.acceptsInlineJobs = message.acceptsInline !== false
|
|
679
|
-
if (jsonSocket.supportsHandoffIdReporting) {
|
|
687
|
+
if (jsonSocket.supportsHandoffIdReporting && !jsonSocket.isDraining) {
|
|
680
688
|
this.readyWorkers.add(jsonSocket)
|
|
681
689
|
} else {
|
|
682
690
|
this.readyWorkers.delete(jsonSocket)
|
|
@@ -694,6 +702,7 @@ export default class BackgroundJobsMain {
|
|
|
694
702
|
// The worker is shutting down gracefully. Stop dispatching new jobs
|
|
695
703
|
// to it but keep the connection in `workers` so any in-flight job
|
|
696
704
|
// it's still draining can report its result.
|
|
705
|
+
jsonSocket.isDraining = true
|
|
697
706
|
this.readyWorkers.delete(jsonSocket)
|
|
698
707
|
}
|
|
699
708
|
|
|
@@ -1230,6 +1239,8 @@ export default class BackgroundJobsMain {
|
|
|
1230
1239
|
* Runs clear error retry timer.
|
|
1231
1240
|
* @returns {void} */
|
|
1232
1241
|
_clearErrorRetryTimer() {
|
|
1242
|
+
if (this.pendingHandoffRecoveries.size > 0) return
|
|
1243
|
+
|
|
1233
1244
|
for (const worker of this.workerHandoffs.keys()) {
|
|
1234
1245
|
if (!this.workers.has(worker)) return
|
|
1235
1246
|
}
|
|
@@ -1296,12 +1307,20 @@ export default class BackgroundJobsMain {
|
|
|
1296
1307
|
}
|
|
1297
1308
|
|
|
1298
1309
|
/**
|
|
1299
|
-
* Retries failed disconnected-socket releases before
|
|
1310
|
+
* Retries failed pre-dispatch and disconnected-socket releases before
|
|
1311
|
+
* draining queued work.
|
|
1300
1312
|
* @returns {Promise<void>} - Resolves after retry work.
|
|
1301
1313
|
*/
|
|
1302
1314
|
async _retryAfterError() {
|
|
1303
1315
|
if (this._stopped) return
|
|
1304
1316
|
|
|
1317
|
+
try {
|
|
1318
|
+
await this._retryPendingHandoffRecoveries()
|
|
1319
|
+
} catch {
|
|
1320
|
+
this._scheduleErrorRetry()
|
|
1321
|
+
return
|
|
1322
|
+
}
|
|
1323
|
+
|
|
1305
1324
|
try {
|
|
1306
1325
|
for (const worker of this.workerHandoffs.keys()) {
|
|
1307
1326
|
if (!this.workers.has(worker)) await this._releaseWorkerHandoffs(worker)
|
|
@@ -1328,30 +1347,46 @@ export default class BackgroundJobsMain {
|
|
|
1328
1347
|
const worker = this.readyWorkerForJob(job)
|
|
1329
1348
|
if (!worker) return
|
|
1330
1349
|
|
|
1331
|
-
this.
|
|
1350
|
+
const admission = this._consumeWorkerAdmission({job, worker})
|
|
1351
|
+
const requestedHandoffId = randomUUID()
|
|
1352
|
+
let handoff
|
|
1332
1353
|
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1354
|
+
try {
|
|
1355
|
+
handoff = await this.store.markHandedOff({handoffId: requestedHandoffId, jobId: job.id, workerId: worker.workerId})
|
|
1356
|
+
} catch (error) {
|
|
1357
|
+
this._rememberHandoffRecovery({handoffId: requestedHandoffId, jobId: job.id})
|
|
1358
|
+
this._restoreWorkerAdmission({...admission, worker})
|
|
1337
1359
|
|
|
1338
|
-
|
|
1360
|
+
try {
|
|
1361
|
+
await this._recoverHandoff({handoffId: requestedHandoffId, jobId: job.id})
|
|
1362
|
+
} catch (recoveryError) {
|
|
1363
|
+
this._reportHandoffRecoveryError({error: recoveryError, handoffId: requestedHandoffId, jobId: job.id})
|
|
1364
|
+
}
|
|
1365
|
+
|
|
1366
|
+
throw error
|
|
1367
|
+
}
|
|
1339
1368
|
|
|
1340
1369
|
if (!handoff) {
|
|
1341
|
-
|
|
1342
|
-
if (this.workers.has(worker)) this.readyWorkers.add(worker)
|
|
1370
|
+
this._restoreWorkerAdmission({...admission, worker})
|
|
1343
1371
|
continue
|
|
1344
1372
|
}
|
|
1345
1373
|
|
|
1346
1374
|
const handoffs = this.workerHandoffs.get(worker)
|
|
1347
1375
|
|
|
1348
|
-
if (!handoffs || !this.workers.has(worker)) {
|
|
1349
|
-
|
|
1376
|
+
if (!handoffs || !this.workers.has(worker) || worker.isDraining) {
|
|
1377
|
+
this._rememberHandoffRecovery({handoffId: handoff.handoffId, jobId: job.id})
|
|
1378
|
+
try {
|
|
1379
|
+
await this._recoverHandoff({handoffId: handoff.handoffId, jobId: job.id})
|
|
1380
|
+
} catch (recoveryError) {
|
|
1381
|
+
this._reportHandoffRecoveryError({error: recoveryError, handoffId: handoff.handoffId, jobId: job.id})
|
|
1382
|
+
throw recoveryError
|
|
1383
|
+
}
|
|
1350
1384
|
this._notifyEnqueued()
|
|
1351
1385
|
this._redrainQueued = true
|
|
1352
1386
|
continue
|
|
1353
1387
|
}
|
|
1354
1388
|
|
|
1389
|
+
this._finalizeWorkerAdmission({...admission, job, worker})
|
|
1355
1390
|
handoffs.set(job.id, handoff.handoffId)
|
|
1356
1391
|
|
|
1357
1392
|
try {
|
|
@@ -1382,6 +1417,125 @@ export default class BackgroundJobsMain {
|
|
|
1382
1417
|
}
|
|
1383
1418
|
}
|
|
1384
1419
|
|
|
1420
|
+
/**
|
|
1421
|
+
* Consumes one advertised worker admission while persistence is in flight.
|
|
1422
|
+
* @param {object} args - Admission details.
|
|
1423
|
+
* @param {import("./types.js").BackgroundJobRow} args.job - Selected job.
|
|
1424
|
+
* @param {JsonSocket} args.worker - Selected worker socket.
|
|
1425
|
+
* @returns {{pooledCreditConsumed: boolean, readinessVersion: number}} - Reversible admission debit.
|
|
1426
|
+
*/
|
|
1427
|
+
_consumeWorkerAdmission({job, worker}) {
|
|
1428
|
+
let pooledCreditConsumed = false
|
|
1429
|
+
|
|
1430
|
+
this.readyWorkers.delete(worker)
|
|
1431
|
+
|
|
1432
|
+
if (job.executionMode === "pooled" && worker.usesPooledCapacityCredits && worker.availablePooledSlots > 0) {
|
|
1433
|
+
pooledCreditConsumed = true
|
|
1434
|
+
worker.availablePooledSlots -= 1
|
|
1435
|
+
if (worker.availablePooledSlots > 0) this.readyWorkers.add(worker)
|
|
1436
|
+
}
|
|
1437
|
+
|
|
1438
|
+
return {pooledCreditConsumed, readinessVersion: worker.readinessVersion}
|
|
1439
|
+
}
|
|
1440
|
+
|
|
1441
|
+
/**
|
|
1442
|
+
* Restores an admission that never reached a worker. A newer readiness
|
|
1443
|
+
* advertisement is already authoritative, so its pooled count is not changed.
|
|
1444
|
+
* @param {object} args - Admission details.
|
|
1445
|
+
* @param {boolean} args.pooledCreditConsumed - Whether a pooled credit was debited.
|
|
1446
|
+
* @param {number} args.readinessVersion - Readiness generation at debit time.
|
|
1447
|
+
* @param {JsonSocket} args.worker - Selected worker socket.
|
|
1448
|
+
* @returns {void}
|
|
1449
|
+
*/
|
|
1450
|
+
_restoreWorkerAdmission({pooledCreditConsumed, readinessVersion, worker}) {
|
|
1451
|
+
if (this._stopped || !this.workers.has(worker) || worker.isDraining) return
|
|
1452
|
+
|
|
1453
|
+
if (pooledCreditConsumed && worker.readinessVersion === readinessVersion) {
|
|
1454
|
+
worker.availablePooledSlots += 1
|
|
1455
|
+
}
|
|
1456
|
+
|
|
1457
|
+
if (worker.supportsHandoffIdReporting) this.readyWorkers.add(worker)
|
|
1458
|
+
}
|
|
1459
|
+
|
|
1460
|
+
/**
|
|
1461
|
+
* Applies a successful pooled admission to a readiness advertisement that
|
|
1462
|
+
* arrived while persistence was in flight and replaced the earlier debit.
|
|
1463
|
+
* @param {object} args - Admission details.
|
|
1464
|
+
* @param {import("./types.js").BackgroundJobRow} args.job - Selected job.
|
|
1465
|
+
* @param {boolean} args.pooledCreditConsumed - Whether a pooled credit was debited.
|
|
1466
|
+
* @param {number} args.readinessVersion - Readiness generation at debit time.
|
|
1467
|
+
* @param {JsonSocket} args.worker - Selected worker socket.
|
|
1468
|
+
* @returns {void}
|
|
1469
|
+
*/
|
|
1470
|
+
_finalizeWorkerAdmission({job, pooledCreditConsumed, readinessVersion, worker}) {
|
|
1471
|
+
if (!pooledCreditConsumed || job.executionMode !== "pooled") return
|
|
1472
|
+
if (worker.readinessVersion === readinessVersion || !worker.usesPooledCapacityCredits) return
|
|
1473
|
+
if (worker.availablePooledSlots <= 0) return
|
|
1474
|
+
|
|
1475
|
+
worker.availablePooledSlots -= 1
|
|
1476
|
+
if (worker.availablePooledSlots === 0) this.readyWorkers.delete(worker)
|
|
1477
|
+
}
|
|
1478
|
+
|
|
1479
|
+
/**
|
|
1480
|
+
* Retains an exact lease for idempotent pre-dispatch recovery.
|
|
1481
|
+
* @param {{handoffId: string, jobId: string}} args - Exact recovery fence.
|
|
1482
|
+
* @returns {void}
|
|
1483
|
+
*/
|
|
1484
|
+
_rememberHandoffRecovery({handoffId, jobId}) {
|
|
1485
|
+
this.pendingHandoffRecoveries.set(handoffId, jobId)
|
|
1486
|
+
}
|
|
1487
|
+
|
|
1488
|
+
/**
|
|
1489
|
+
* Returns one exact lease and forgets it only after the adapter acknowledges
|
|
1490
|
+
* the fenced transition or confirms it was already absent.
|
|
1491
|
+
* @param {{handoffId: string, jobId: string}} args - Exact recovery fence.
|
|
1492
|
+
* @returns {Promise<void>} - Resolves after durable recovery settles.
|
|
1493
|
+
*/
|
|
1494
|
+
async _recoverHandoff({handoffId, jobId}) {
|
|
1495
|
+
await this.store.markReturnedToQueue({handoffId, jobId})
|
|
1496
|
+
|
|
1497
|
+
if (this.pendingHandoffRecoveries.get(handoffId) === jobId) {
|
|
1498
|
+
this.pendingHandoffRecoveries.delete(handoffId)
|
|
1499
|
+
}
|
|
1500
|
+
}
|
|
1501
|
+
|
|
1502
|
+
/**
|
|
1503
|
+
* Replays retained exact-ID recoveries through the dispatcher's existing
|
|
1504
|
+
* transient-error retry lifecycle.
|
|
1505
|
+
* @returns {Promise<void>} - Resolves after every retained recovery settles.
|
|
1506
|
+
*/
|
|
1507
|
+
async _retryPendingHandoffRecoveries() {
|
|
1508
|
+
for (const [handoffId, jobId] of [...this.pendingHandoffRecoveries]) {
|
|
1509
|
+
try {
|
|
1510
|
+
await this._recoverHandoff({handoffId, jobId})
|
|
1511
|
+
} catch (error) {
|
|
1512
|
+
this._reportHandoffRecoveryError({error, handoffId, jobId})
|
|
1513
|
+
throw error
|
|
1514
|
+
}
|
|
1515
|
+
}
|
|
1516
|
+
}
|
|
1517
|
+
|
|
1518
|
+
/**
|
|
1519
|
+
* Surfaces a failed exact-ID recovery without dropping its retry ledger entry.
|
|
1520
|
+
* @param {object} args - Recovery failure.
|
|
1521
|
+
* @param {ReturnType<typeof JSON.parse>} args.error - Adapter failure.
|
|
1522
|
+
* @param {string} args.handoffId - Exact lease fence.
|
|
1523
|
+
* @param {string} args.jobId - Job id.
|
|
1524
|
+
* @returns {void}
|
|
1525
|
+
*/
|
|
1526
|
+
_reportHandoffRecoveryError({error, handoffId, jobId}) {
|
|
1527
|
+
const normalizedError = error instanceof Error ? error : new Error(String(error))
|
|
1528
|
+
const payload = {
|
|
1529
|
+
context: {handoffId, jobId, stage: "background-job-handoff-admission-recovery"},
|
|
1530
|
+
error: normalizedError
|
|
1531
|
+
}
|
|
1532
|
+
const errorEvents = this.configuration.getErrorEvents()
|
|
1533
|
+
|
|
1534
|
+
this.logger.error(() => ["Failed to recover an ambiguous background job handoff:", normalizedError])
|
|
1535
|
+
errorEvents.emit("framework-error", payload)
|
|
1536
|
+
errorEvents.emit("all-error", {...payload, errorType: "framework-error"})
|
|
1537
|
+
}
|
|
1538
|
+
|
|
1385
1539
|
/**
|
|
1386
1540
|
* Runs next available job for ready workers.
|
|
1387
1541
|
* @returns {Promise<import("./types.js").BackgroundJobRow | null>} - Next queued job matching ready worker capacity.
|
|
@@ -141,3 +141,4 @@ function handleMessage(message) {
|
|
|
141
141
|
process.on("message", (message) => handleMessage(message))
|
|
142
142
|
process.once("disconnect", () => void shutdownRunner(0))
|
|
143
143
|
for (const signal of ["SIGTERM", "SIGINT"]) process.once(signal, () => void shutdownRunner(1))
|
|
144
|
+
if (process.send) process.send({type: "ready"})
|
|
@@ -916,10 +916,11 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
|
|
|
916
916
|
* Runs mark handed off.
|
|
917
917
|
* @param {object} args - Options.
|
|
918
918
|
* @param {string} args.jobId - Job id.
|
|
919
|
+
* @param {string} [args.handoffId] - Caller-selected exact lease id. Generated for legacy direct callers when omitted.
|
|
919
920
|
* @param {string} [args.workerId] - Worker id.
|
|
920
921
|
* @returns {Promise<import("./types.js").BackgroundJobHandoff | null>} - Claimed handoff lease, or null when no longer queued.
|
|
921
922
|
*/
|
|
922
|
-
async markHandedOff({jobId, workerId}) {
|
|
923
|
+
async markHandedOff({jobId, handoffId = randomUUID(), workerId}) {
|
|
923
924
|
await this.ensureReady()
|
|
924
925
|
|
|
925
926
|
const handedOffAtMs = Date.now()
|
|
@@ -928,8 +929,6 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
|
|
|
928
929
|
const queuedJob = await this._getJobRowById(db, jobId)
|
|
929
930
|
if (!queuedJob || queuedJob.status !== "queued") return null
|
|
930
931
|
if (queuedJob.concurrencyKey && !(await this._reserveConcurrency(db, queuedJob.concurrencyKey))) return null
|
|
931
|
-
const handoffId = randomUUID()
|
|
932
|
-
|
|
933
932
|
const affectedRows = await this._updateAffectedRows(db, {
|
|
934
933
|
tableName: JOBS_TABLE,
|
|
935
934
|
data: {
|
|
@@ -43,6 +43,12 @@
|
|
|
43
43
|
* @property {string} handoffId - Unique handoff lease id.
|
|
44
44
|
* @property {number} handedOffAtMs - Time handed to a worker in ms.
|
|
45
45
|
*/
|
|
46
|
+
/**
|
|
47
|
+
* @typedef {object} BackgroundJobHandoffRequest
|
|
48
|
+
* @property {string} jobId - Job to claim.
|
|
49
|
+
* @property {string} [handoffId] - Exact caller-selected lease id. Adapters must persist and return this id when supplied; built-in adapters generate one when omitted for legacy direct callers.
|
|
50
|
+
* @property {string} [workerId] - Worker claiming the job.
|
|
51
|
+
*/
|
|
46
52
|
/**
|
|
47
53
|
* @typedef {object} BackgroundJobOptions
|
|
48
54
|
* @property {BackgroundJobExecutionMode} [executionMode] - How the job should run. Node defaults to `"pooled"` (a warm, reused local runner process). Browser/Expo local dispatch defaults to and only accepts `"inline"`. `"forked"` runs a Node job in a fresh `child_process.fork()` child, and `"spawned"` in a detached CLI runner.
|
|
@@ -194,8 +194,10 @@ export default class BackgroundJobsWorker {
|
|
|
194
194
|
this.inflightPooledJobs = new Set()
|
|
195
195
|
/** @type {Set<import("node:child_process").ChildProcess>} */
|
|
196
196
|
this.pooledChildren = new Set()
|
|
197
|
-
/** @type {Map<import("node:child_process").ChildProcess, {createdAtMs: number, jobsRun: number, inflight: Map<string, {payload: import("./types.js").BackgroundJobPayload & {id: string}, resolve?: (value: void) => void, timeoutTimer?: ReturnType<typeof setTimeout> | null}>, lastDispatchSeq: number, retiring: boolean, settling?: boolean, timeoutSigkillTimer?: ReturnType<typeof setTimeout> | null}>} */
|
|
197
|
+
/** @type {Map<import("node:child_process").ChildProcess, {createdAtMs: number, jobsRun: number, inflight: Map<string, {payload: import("./types.js").BackgroundJobPayload & {id: string}, resolve?: (value: void) => void, pooledJob?: Promise<void>, timeoutTimer?: ReturnType<typeof setTimeout> | null}>, lastDispatchSeq: number, retiring: boolean, started?: boolean, settling?: boolean, timeoutSigkillTimer?: ReturnType<typeof setTimeout> | null}>} */
|
|
198
198
|
this.pooledChildStates = new Map()
|
|
199
|
+
/** @type {WeakSet<Promise<void>>} */
|
|
200
|
+
this._pooledStartupFailureJobs = new WeakSet()
|
|
199
201
|
// Monotonic dispatch counter for round-robin child selection: each dispatch stamps
|
|
200
202
|
// the chosen child, and selection prefers the child dispatched least recently.
|
|
201
203
|
this._pooledDispatchSeq = 0
|
|
@@ -600,16 +602,16 @@ export default class BackgroundJobsWorker {
|
|
|
600
602
|
}
|
|
601
603
|
|
|
602
604
|
/**
|
|
603
|
-
*
|
|
604
|
-
*
|
|
605
|
-
*
|
|
605
|
+
* Advertises current worker capacity unless the worker is draining.
|
|
606
|
+
* @param {object} [options] - Advertisement options.
|
|
607
|
+
* @param {boolean} [options.revokePooledAdmission] - Revoke pooled credits while preserving other execution modes.
|
|
606
608
|
* @returns {void}
|
|
607
609
|
*/
|
|
608
|
-
_sendReadyIfRunning() {
|
|
610
|
+
_sendReadyIfRunning({revokePooledAdmission = false} = {}) {
|
|
609
611
|
if (this.shouldStop) return
|
|
610
612
|
if (!this.jsonSocket) return
|
|
611
613
|
|
|
612
|
-
const readyMessage = this._readyMessage()
|
|
614
|
+
const readyMessage = this._readyMessage({revokePooledAdmission})
|
|
613
615
|
|
|
614
616
|
if (!readyMessage) return
|
|
615
617
|
this.jsonSocket.send(readyMessage)
|
|
@@ -617,21 +619,24 @@ export default class BackgroundJobsWorker {
|
|
|
617
619
|
|
|
618
620
|
/**
|
|
619
621
|
* Runs ready message.
|
|
622
|
+
* @param {object} [options] - Advertisement options.
|
|
623
|
+
* @param {boolean} [options.revokePooledAdmission] - Revoke pooled credits while preserving other execution modes.
|
|
620
624
|
* @returns {import("./types.js").BackgroundJobSocketMessage | null} - Ready message or null when the worker has no capacity.
|
|
621
625
|
*/
|
|
622
|
-
_readyMessage() {
|
|
626
|
+
_readyMessage({revokePooledAdmission = false} = {}) {
|
|
623
627
|
const acceptsProcessJob = this.inflightProcessJobs.size < this.maxConcurrentForkedJobs
|
|
624
628
|
const acceptsInline = this.inflightInlineJobs.size < this.maxConcurrentInlineJobs
|
|
625
|
-
const
|
|
629
|
+
const availablePooledSlots = revokePooledAdmission ? 0 : this._availablePooledSlots()
|
|
630
|
+
const acceptsPooled = availablePooledSlots > 0
|
|
626
631
|
|
|
627
|
-
if (!acceptsProcessJob && !acceptsInline && !acceptsPooled) return null
|
|
632
|
+
if (!revokePooledAdmission && !acceptsProcessJob && !acceptsInline && !acceptsPooled) return null
|
|
628
633
|
|
|
629
634
|
return {
|
|
630
635
|
type: "ready",
|
|
631
636
|
acceptsForked: acceptsProcessJob,
|
|
632
637
|
acceptsInline,
|
|
633
638
|
acceptsPooled,
|
|
634
|
-
availablePooledSlots
|
|
639
|
+
availablePooledSlots,
|
|
635
640
|
acceptsSpawned: acceptsProcessJob
|
|
636
641
|
}
|
|
637
642
|
}
|
|
@@ -646,7 +651,7 @@ export default class BackgroundJobsWorker {
|
|
|
646
651
|
let inflight
|
|
647
652
|
inflight = pooledJob.finally(() => {
|
|
648
653
|
this.inflightPooledJobs.delete(inflight)
|
|
649
|
-
if (!this.shouldStop) this._sendReadyIfRunning()
|
|
654
|
+
if (!this.shouldStop && !this._pooledStartupFailureJobs.has(pooledJob)) this._sendReadyIfRunning()
|
|
650
655
|
})
|
|
651
656
|
this.inflightPooledJobs.add(inflight)
|
|
652
657
|
}
|
|
@@ -689,16 +694,22 @@ export default class BackgroundJobsWorker {
|
|
|
689
694
|
// Stamp the round-robin cursor so the next dispatch prefers a different child.
|
|
690
695
|
state.lastDispatchSeq = ++this._pooledDispatchSeq
|
|
691
696
|
|
|
692
|
-
|
|
693
|
-
|
|
697
|
+
/**
|
|
698
|
+
* Resolves the pooled job promise.
|
|
699
|
+
* @type {(value: void) => void}
|
|
700
|
+
*/
|
|
701
|
+
let resolvePooledJob = () => {}
|
|
702
|
+
const pooledJob = new Promise((resolve) => { resolvePooledJob = resolve })
|
|
703
|
+
const timeoutTimer = this._armPooledJobTimeout({child, payload})
|
|
694
704
|
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
705
|
+
state.inflight.set(payload.id, {payload, resolve: resolvePooledJob, pooledJob, timeoutTimer})
|
|
706
|
+
try {
|
|
707
|
+
child.send({type: "job", payload, sharedTransactionBroker: this._pooledJobSharedTransactionBrokerConfig()})
|
|
708
|
+
} catch (error) {
|
|
709
|
+
void this._handlePooledChildFailure({child, error})
|
|
710
|
+
}
|
|
711
|
+
|
|
712
|
+
return pooledJob
|
|
702
713
|
}
|
|
703
714
|
|
|
704
715
|
/**
|
|
@@ -769,7 +780,8 @@ export default class BackgroundJobsWorker {
|
|
|
769
780
|
* (SIGTERM, then SIGKILL after the grace) — a hung JS job cannot be cancelled
|
|
770
781
|
* any other way. The non-clean exit flows through `_handlePooledChildFailure`,
|
|
771
782
|
* which reports every in-flight job on the child failed (so they requeue) and
|
|
772
|
-
* drops it from tracking;
|
|
783
|
+
* drops it from tracking; the failure path immediately re-advertises the
|
|
784
|
+
* resulting capacity once the runner has completed startup.
|
|
773
785
|
* @param {object} args - Options.
|
|
774
786
|
* @param {import("node:child_process").ChildProcess} args.child - Pooled child.
|
|
775
787
|
* @param {string} args.jobId - Job id that overran.
|
|
@@ -810,7 +822,7 @@ export default class BackgroundJobsWorker {
|
|
|
810
822
|
})
|
|
811
823
|
this.pooledChildren.add(child)
|
|
812
824
|
this.inflightProcessChildren.add(child)
|
|
813
|
-
this.pooledChildStates.set(child, {createdAtMs: Date.now(), jobsRun: 0, inflight: new Map(), lastDispatchSeq: 0, retiring: false})
|
|
825
|
+
this.pooledChildStates.set(child, {createdAtMs: Date.now(), jobsRun: 0, inflight: new Map(), lastDispatchSeq: 0, retiring: false, started: false})
|
|
814
826
|
child.on("message", (message) => this._handlePooledChildMessage({child, message}))
|
|
815
827
|
child.once("exit", (code, signal) => this._handlePooledChildFailure({child, error: new Error(`Pooled background job runner exited: code=${code} signal=${signal || "none"}`)}))
|
|
816
828
|
child.once("error", (error) => this._handlePooledChildFailure({child, error}))
|
|
@@ -829,7 +841,12 @@ export default class BackgroundJobsWorker {
|
|
|
829
841
|
if (!message || typeof message !== "object") return
|
|
830
842
|
const record = /** @type {{type?: ReturnType<typeof JSON.parse>, jobId?: ReturnType<typeof JSON.parse>, acknowledged?: ReturnType<typeof JSON.parse>, rssBytes?: ReturnType<typeof JSON.parse>, error?: ReturnType<typeof JSON.parse>}} */ (message)
|
|
831
843
|
const state = this.pooledChildStates.get(child)
|
|
844
|
+
if (record.type === "ready") {
|
|
845
|
+
if (state) state.started = true
|
|
846
|
+
return
|
|
847
|
+
}
|
|
832
848
|
if (record.type !== "job-outcome" || !state || state.settling || typeof record.jobId !== "string") return
|
|
849
|
+
state.started = true
|
|
833
850
|
const entry = state.inflight.get(record.jobId)
|
|
834
851
|
if (!entry) return
|
|
835
852
|
|
|
@@ -907,9 +924,11 @@ export default class BackgroundJobsWorker {
|
|
|
907
924
|
/**
|
|
908
925
|
* Removes an exited/unhealthy pooled child and reports every job that was
|
|
909
926
|
* in-flight on it as failed — a process-level crash's blast radius is the
|
|
910
|
-
* child's whole in-flight set.
|
|
911
|
-
*
|
|
912
|
-
*
|
|
927
|
+
* child's whole in-flight set. Once the child has completed startup, its
|
|
928
|
+
* freed capacity is advertised immediately; the replacement itself is still
|
|
929
|
+
* spawned lazily by the next dispatch. A child that exits before its startup
|
|
930
|
+
* handshake does not re-announce, avoiding a tight respawn loop on startup
|
|
931
|
+
* failure.
|
|
913
932
|
* @param {object} args - Failure details.
|
|
914
933
|
* @param {import("node:child_process").ChildProcess} args.child - Pooled child.
|
|
915
934
|
* @param {ReturnType<typeof JSON.parse>} args.error - Failure.
|
|
@@ -934,7 +953,7 @@ export default class BackgroundJobsWorker {
|
|
|
934
953
|
if (state) state.inflight.clear()
|
|
935
954
|
this.pooledChildStates.delete(child)
|
|
936
955
|
|
|
937
|
-
|
|
956
|
+
const failureReports = entries.map(async (entry) => {
|
|
938
957
|
await this._reportJobResult({
|
|
939
958
|
jobId: entry.payload.id,
|
|
940
959
|
status: "failed",
|
|
@@ -944,7 +963,26 @@ export default class BackgroundJobsWorker {
|
|
|
944
963
|
workerId: entry.payload.workerId || this.workerId
|
|
945
964
|
})
|
|
946
965
|
if (entry.resolve) entry.resolve(undefined)
|
|
947
|
-
})
|
|
966
|
+
})
|
|
967
|
+
|
|
968
|
+
// Start every fallback report before announcing capacity so the main cannot
|
|
969
|
+
// observe a replacement slot before the failed jobs' reports are in flight.
|
|
970
|
+
// The report promises remain tracked below; a slow retry must not hold the
|
|
971
|
+
// newly freed runner capacity hostage.
|
|
972
|
+
if (state && state.started !== false) {
|
|
973
|
+
this._sendReadyIfRunning()
|
|
974
|
+
} else if (state) {
|
|
975
|
+
for (const entry of entries) {
|
|
976
|
+
if (entry.pooledJob) this._pooledStartupFailureJobs.add(entry.pooledJob)
|
|
977
|
+
}
|
|
978
|
+
// A previous ready message may still have unconsumed pooled credits at the
|
|
979
|
+
// main. Revoke them authoritatively without suppressing valid inline or
|
|
980
|
+
// process-runner readiness; otherwise queued jobs can trigger a startup
|
|
981
|
+
// crash loop using the stale credits.
|
|
982
|
+
this._sendReadyIfRunning({revokePooledAdmission: true})
|
|
983
|
+
}
|
|
984
|
+
|
|
985
|
+
await Promise.allSettled(failureReports)
|
|
948
986
|
}
|
|
949
987
|
|
|
950
988
|
/**
|