velocious 1.0.695 → 1.0.697
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/build/background-jobs/adapter.js +1 -1
- package/build/background-jobs/client.js +1 -1
- package/build/background-jobs/job-record.js +23 -0
- package/build/background-jobs/local-adapter.js +1 -1
- package/build/background-jobs/local-store.js +10 -5
- package/build/background-jobs/main.js +6 -1
- package/build/background-jobs/owner-liveness.js +92 -0
- package/build/background-jobs/pooled-runner-child.js +5 -1
- package/build/background-jobs/status-reporter.js +7 -4
- package/build/background-jobs/store.js +13 -5
- package/build/background-jobs/types.js +2 -1
- package/build/background-jobs/worker.js +4 -2
- package/build/database/migrations-ledger.js +96 -8
- package/build/src/background-jobs/adapter.d.ts +2 -1
- package/build/src/background-jobs/adapter.d.ts.map +1 -1
- package/build/src/background-jobs/adapter.js +2 -2
- package/build/src/background-jobs/client.js +2 -2
- package/build/src/background-jobs/job-record.d.ts +16 -0
- package/build/src/background-jobs/job-record.d.ts.map +1 -1
- package/build/src/background-jobs/job-record.js +23 -1
- package/build/src/background-jobs/local-adapter.d.ts +2 -1
- package/build/src/background-jobs/local-adapter.d.ts.map +1 -1
- package/build/src/background-jobs/local-adapter.js +2 -2
- package/build/src/background-jobs/local-store.d.ts +5 -3
- package/build/src/background-jobs/local-store.d.ts.map +1 -1
- package/build/src/background-jobs/local-store.js +12 -6
- package/build/src/background-jobs/main.d.ts.map +1 -1
- package/build/src/background-jobs/main.js +7 -2
- package/build/src/background-jobs/owner-liveness.d.ts +71 -0
- package/build/src/background-jobs/owner-liveness.d.ts.map +1 -0
- package/build/src/background-jobs/owner-liveness.js +78 -0
- package/build/src/background-jobs/pooled-runner-child.js +6 -2
- package/build/src/background-jobs/status-reporter.d.ts +6 -2
- package/build/src/background-jobs/status-reporter.d.ts.map +1 -1
- package/build/src/background-jobs/status-reporter.js +8 -5
- package/build/src/background-jobs/store.d.ts +5 -3
- package/build/src/background-jobs/store.d.ts.map +1 -1
- package/build/src/background-jobs/store.js +15 -6
- package/build/src/background-jobs/types.d.ts +7 -1
- package/build/src/background-jobs/types.d.ts.map +1 -1
- package/build/src/background-jobs/types.js +3 -2
- package/build/src/background-jobs/worker.d.ts +2 -1
- package/build/src/background-jobs/worker.d.ts.map +1 -1
- package/build/src/background-jobs/worker.js +6 -4
- package/build/src/database/migrations-ledger.d.ts +34 -1
- package/build/src/database/migrations-ledger.d.ts.map +1 -1
- package/build/src/database/migrations-ledger.js +82 -9
- package/package.json +1 -1
- package/src/background-jobs/adapter.js +1 -1
- package/src/background-jobs/client.js +1 -1
- package/src/background-jobs/job-record.js +23 -0
- package/src/background-jobs/local-adapter.js +1 -1
- package/src/background-jobs/local-store.js +10 -5
- package/src/background-jobs/main.js +6 -1
- package/src/background-jobs/owner-liveness.js +92 -0
- package/src/background-jobs/pooled-runner-child.js +5 -1
- package/src/background-jobs/status-reporter.js +7 -4
- package/src/background-jobs/store.js +13 -5
- package/src/background-jobs/types.js +2 -1
- package/src/background-jobs/worker.js +4 -2
- package/src/database/migrations-ledger.js +96 -8
package/README.md
CHANGED
|
@@ -2508,7 +2508,7 @@ creation across generation processes.
|
|
|
2508
2508
|
|
|
2509
2509
|
Jobs can opt into cross-worker durable concurrency limits by pairing a non-empty `concurrencyKey` with a positive-integer `maxConcurrency` in their background-job options, or by deriving the key in a hydrated job instance's non-static `concurrencyKey()` method. Explicit enqueue options win. The first cap registered for a key is stable; conflicting caps are rejected. See [durable concurrency limits](docs/background-jobs.md#durable-concurrency-limits).
|
|
2510
2510
|
|
|
2511
|
-
Production apps can listen for `background-job-failed` (or its `all-error` mirror) to report accepted failed attempts, including retry and terminal-state metadata. Process-level pooled-runner failures also carry one shared `context.runnerFailure` snapshot for every affected job, with active handoff identities, child instance/PID, runner/worker lifecycle, exit code/signal, an exact typed shutdown reason and request/observation timestamps, bounded in-flight job IDs, and an explicit nullable OOM verdict. The deprecated `terminationReason` compatibility category remains present, while new consumers should use authoritative `shutdownReason`. Parent timeout/stop/retirement requests are recorded before signalling; normal drained retirement is permitted only with zero tracked jobs. Listen for `background-job-orphaned` to react to a specific job the main process reclaimed after its worker died mid-run — e.g. enqueue a targeted recovery for the work it left behind, instead of only polling for the aftermath. Orphan handlers run before the sweep waits for reclaimed jobs to be dispatched, so a stalled dispatcher does not delay application recovery. An orphan without a delivered runner-failure report has unknown shutdown cause; do not infer OOM, deploy, timeout, signal, or normal retirement from acceptance evidence alone. See [docs/background-jobs.md](docs/background-jobs.md#failure-events).
|
|
2511
|
+
Production apps can listen for `background-job-failed` (or its `all-error` mirror) to report accepted failed attempts, including retry and terminal-state metadata. Process-level pooled-runner failures also carry one shared `context.runnerFailure` snapshot for every affected job, with active handoff identities, child instance/PID, runner/worker lifecycle, exit code/signal, an exact typed shutdown reason and request/observation timestamps, bounded in-flight job IDs, and an explicit nullable OOM verdict. Every handed-off job also records the accepting runner's `child_instance_id`, `child_pid`, `runner_hostname` and start time; `BackgroundJobRecord#ownerIsAlive()` reports whether that recorded local process still exists (`alive`, `dead`, or `unknown` for remote/unrecorded owners), so claim ownership can be checked without polling. The deprecated `terminationReason` compatibility category remains present, while new consumers should use authoritative `shutdownReason`. Parent timeout/stop/retirement requests are recorded before signalling; normal drained retirement is permitted only with zero tracked jobs. Listen for `background-job-orphaned` to react to a specific job the main process reclaimed after its worker died mid-run — e.g. enqueue a targeted recovery for the work it left behind, instead of only polling for the aftermath. The orphan payload carries the dead owner's `childInstanceId`, `childPid`, `runnerHostname` and `workerId`, so a handler can release exactly that owner's claims on the event it already receives. Orphan handlers run before the sweep waits for reclaimed jobs to be dispatched, so a stalled dispatcher does not delay application recovery. An orphan without a delivered runner-failure report has unknown shutdown cause; do not infer OOM, deploy, timeout, signal, or normal retirement from acceptance evidence alone. See [docs/background-jobs.md](docs/background-jobs.md#failure-events) and [runner identity and owner liveness](docs/background-jobs.md#runner-identity-and-owner-liveness).
|
|
2512
2512
|
|
|
2513
2513
|
## Setup
|
|
2514
2514
|
|
|
@@ -148,7 +148,7 @@ export default class BackgroundJobsAdapter {
|
|
|
148
148
|
* Records pooled-child acceptance evidence (received/started timestamps plus
|
|
149
149
|
* runner identity) for a handed-off job, fenced by its active handoff lease.
|
|
150
150
|
* Only the fields supplied are written.
|
|
151
|
-
* @param {{jobId: string, handoffId?: string, workerId?: string, handedOffAtMs?: number, receivedAtMs?: number, startedAtMs?: number, childInstanceId?: string, childPid?: number}} _args - Acceptance report.
|
|
151
|
+
* @param {{jobId: string, handoffId?: string, workerId?: string, handedOffAtMs?: number, receivedAtMs?: number, startedAtMs?: number, childInstanceId?: string, childPid?: number, runnerHostname?: string}} _args - Acceptance report.
|
|
152
152
|
* @returns {Promise<boolean>} - Whether the fenced report was accepted.
|
|
153
153
|
*/
|
|
154
154
|
async markChildAccepted(_args) { throw new Error("BackgroundJobsAdapter#markChildAccepted is not implemented") }
|
|
@@ -26,7 +26,7 @@ const BACKGROUND_JOB_NULLABLE_NUMBER_FIELDS = [
|
|
|
26
26
|
"scheduledAtMs",
|
|
27
27
|
"timeoutMs"
|
|
28
28
|
]
|
|
29
|
-
const BACKGROUND_JOB_NULLABLE_STRING_FIELDS = ["childInstanceId", "concurrencyKey", "handoffId", "lastError", "scheduleKey", "workerId"]
|
|
29
|
+
const BACKGROUND_JOB_NULLABLE_STRING_FIELDS = ["childInstanceId", "concurrencyKey", "handoffId", "lastError", "runnerHostname", "scheduleKey", "workerId"]
|
|
30
30
|
|
|
31
31
|
/**
|
|
32
32
|
* Checks a required nullable number from a normalized wire row.
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
// @ts-check
|
|
2
2
|
|
|
3
3
|
import DatabaseRecord from "../database/record/index.js"
|
|
4
|
+
import { evaluateBackgroundJobOwnerLiveness } from "./owner-liveness.js"
|
|
4
5
|
|
|
5
6
|
export default class BackgroundJobRecord extends DatabaseRecord {
|
|
6
7
|
static tableName() {
|
|
@@ -10,4 +11,26 @@ export default class BackgroundJobRecord extends DatabaseRecord {
|
|
|
10
11
|
static primaryKey() {
|
|
11
12
|
return "id"
|
|
12
13
|
}
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Evaluates whether the process that accepted this job is still alive.
|
|
17
|
+
*
|
|
18
|
+
* Local ownership is answered from the recorded runner hostname and OS pid.
|
|
19
|
+
* Jobs accepted on another host, or jobs recorded before runner hostnames
|
|
20
|
+
* existed, answer `unknown`: remote liveness needs a heartbeat or instance
|
|
21
|
+
* registry, which this helper deliberately does not invent.
|
|
22
|
+
* @param {object} [args] - Options.
|
|
23
|
+
* @param {string} [args.hostname] - Hostname to compare against, defaults to this process's hostname.
|
|
24
|
+
* @param {(pid: number) => boolean} [args.probe] - Process probe, injectable for tests.
|
|
25
|
+
* @returns {import("./owner-liveness.js").BackgroundJobOwnerLivenessResult} - Liveness verdict.
|
|
26
|
+
*/
|
|
27
|
+
ownerIsAlive({hostname, probe} = {}) {
|
|
28
|
+
return evaluateBackgroundJobOwnerLiveness({
|
|
29
|
+
childInstanceId: /** @type {string | null} */ (this.readAttribute("childInstanceId")),
|
|
30
|
+
childPid: /** @type {number | null} */ (this.readAttribute("childPid")),
|
|
31
|
+
currentHostname: hostname,
|
|
32
|
+
probe,
|
|
33
|
+
runnerHostname: /** @type {string | null} */ (this.readAttribute("runnerHostname"))
|
|
34
|
+
})
|
|
35
|
+
}
|
|
13
36
|
}
|
|
@@ -163,7 +163,7 @@ export default class LocalBackgroundJobsAdapter extends BackgroundJobsAdapter {
|
|
|
163
163
|
|
|
164
164
|
/**
|
|
165
165
|
* Records pooled-child acceptance evidence for a local handoff.
|
|
166
|
-
* @param {{jobId: string, handoffId?: string, receivedAtMs?: number, startedAtMs?: number, childInstanceId?: string, childPid?: number}} args - Acceptance report.
|
|
166
|
+
* @param {{jobId: string, handoffId?: string, receivedAtMs?: number, startedAtMs?: number, childInstanceId?: string, childPid?: number, runnerHostname?: string}} args - Acceptance report.
|
|
167
167
|
* @returns {Promise<boolean>} - Whether accepted.
|
|
168
168
|
*/
|
|
169
169
|
async markChildAccepted(args) { return await this.store.markChildAccepted(args) }
|
|
@@ -63,7 +63,8 @@ const EXPECTED_JOB_COLUMNS = [
|
|
|
63
63
|
"child_received_at_ms",
|
|
64
64
|
"child_started_at_ms",
|
|
65
65
|
"child_instance_id",
|
|
66
|
-
"child_pid"
|
|
66
|
+
"child_pid",
|
|
67
|
+
"runner_hostname"
|
|
67
68
|
]
|
|
68
69
|
const EXPECTED_CONCURRENCY_COLUMNS = ["concurrency_key", "max_concurrency", "active_count"]
|
|
69
70
|
const EXPECTED_SCHEDULE_KEY_COLUMNS = ["schedule_key", "job_id"]
|
|
@@ -333,6 +334,7 @@ export default class LocalBackgroundJobsStore {
|
|
|
333
334
|
table.bigint("child_started_at_ms", {null: true})
|
|
334
335
|
table.string("child_instance_id", {null: true})
|
|
335
336
|
table.integer("child_pid", {null: true})
|
|
337
|
+
table.string("runner_hostname", {null: true})
|
|
336
338
|
table.addIndex(new TableIndex(["status", "scheduled_at_ms", "created_at_ms", "id"], {name: LOCAL_BACKGROUND_JOBS_INDEX_NAMES[0]}))
|
|
337
339
|
table.addIndex(new TableIndex(["queue", "status", "created_at_ms"], {name: LOCAL_BACKGROUND_JOBS_INDEX_NAMES[1]}))
|
|
338
340
|
table.addIndex(new TableIndex(["args_digest"], {name: LOCAL_BACKGROUND_JOBS_INDEX_NAMES[2]}))
|
|
@@ -1098,9 +1100,10 @@ export default class LocalBackgroundJobsStore {
|
|
|
1098
1100
|
* @param {number} [args.startedAtMs] - Epoch ms the job's perform started in the child.
|
|
1099
1101
|
* @param {string} [args.childInstanceId] - Stable pooled child identity.
|
|
1100
1102
|
* @param {number} [args.childPid] - Pooled child OS pid.
|
|
1103
|
+
* @param {string} [args.runnerHostname] - Hostname of the machine running the pooled child.
|
|
1101
1104
|
* @returns {Promise<boolean>} - Whether the lease won.
|
|
1102
1105
|
*/
|
|
1103
|
-
async markChildAccepted({jobId, handoffId, receivedAtMs, startedAtMs, childInstanceId, childPid}) {
|
|
1106
|
+
async markChildAccepted({jobId, handoffId, receivedAtMs, startedAtMs, childInstanceId, childPid, runnerHostname}) {
|
|
1104
1107
|
await this.ensureReady()
|
|
1105
1108
|
|
|
1106
1109
|
return await this._withDb(async (connection) => await this._mutate(connection, async (db) => {
|
|
@@ -1113,6 +1116,7 @@ export default class LocalBackgroundJobsStore {
|
|
|
1113
1116
|
if (typeof startedAtMs === "number") data.child_started_at_ms = startedAtMs
|
|
1114
1117
|
if (typeof childInstanceId === "string") data.child_instance_id = childInstanceId
|
|
1115
1118
|
if (typeof childPid === "number") data.child_pid = childPid
|
|
1119
|
+
if (typeof runnerHostname === "string") data.runner_hostname = runnerHostname
|
|
1116
1120
|
if (Object.keys(data).length === 0) return false
|
|
1117
1121
|
|
|
1118
1122
|
const affectedRows = await this._updateAffectedRows(db, {
|
|
@@ -1279,15 +1283,15 @@ export default class LocalBackgroundJobsStore {
|
|
|
1279
1283
|
* @returns {Record<string, ReturnType<typeof JSON.parse>>} - Cleared acceptance columns.
|
|
1280
1284
|
*/
|
|
1281
1285
|
_clearedChildAcceptanceData() {
|
|
1282
|
-
return {child_instance_id: null, child_pid: null, child_received_at_ms: null, child_started_at_ms: null}
|
|
1286
|
+
return {child_instance_id: null, child_pid: null, child_received_at_ms: null, child_started_at_ms: null, runner_hostname: null}
|
|
1283
1287
|
}
|
|
1284
1288
|
|
|
1285
1289
|
/**
|
|
1286
1290
|
* Returns the row-shape counterpart of the cleared acceptance columns.
|
|
1287
|
-
* @returns {Pick<import("./types.js").BackgroundJobRow, "childInstanceId" | "childPid" | "childReceivedAtMs" | "childStartedAtMs">} - Cleared acceptance fields.
|
|
1291
|
+
* @returns {Pick<import("./types.js").BackgroundJobRow, "childInstanceId" | "childPid" | "childReceivedAtMs" | "childStartedAtMs" | "runnerHostname">} - Cleared acceptance fields.
|
|
1288
1292
|
*/
|
|
1289
1293
|
_clearedChildAcceptanceRow() {
|
|
1290
|
-
return {childInstanceId: null, childPid: null, childReceivedAtMs: null, childStartedAtMs: null}
|
|
1294
|
+
return {childInstanceId: null, childPid: null, childReceivedAtMs: null, childStartedAtMs: null, runnerHostname: null}
|
|
1291
1295
|
}
|
|
1292
1296
|
|
|
1293
1297
|
/**
|
|
@@ -1583,6 +1587,7 @@ export default class LocalBackgroundJobsStore {
|
|
|
1583
1587
|
args: parsedArgs,
|
|
1584
1588
|
attempts: this._numberOrNull(row.attempts),
|
|
1585
1589
|
childInstanceId: row.child_instance_id === null || row.child_instance_id === undefined ? null : String(row.child_instance_id),
|
|
1590
|
+
runnerHostname: row.runner_hostname === null || row.runner_hostname === undefined ? null : String(row.runner_hostname),
|
|
1586
1591
|
childPid: this._numberOrNull(row.child_pid),
|
|
1587
1592
|
childReceivedAtMs: this._numberOrNull(row.child_received_at_ms),
|
|
1588
1593
|
childStartedAtMs: this._numberOrNull(row.child_started_at_ms),
|
|
@@ -1272,6 +1272,7 @@ export default class BackgroundJobsMain {
|
|
|
1272
1272
|
handoffId: message.handoffId,
|
|
1273
1273
|
jobId: message.jobId,
|
|
1274
1274
|
receivedAtMs: message.receivedAtMs,
|
|
1275
|
+
runnerHostname: message.runnerHostname,
|
|
1275
1276
|
startedAtMs: message.startedAtMs,
|
|
1276
1277
|
workerId: message.workerId
|
|
1277
1278
|
})
|
|
@@ -1862,14 +1863,18 @@ export default class BackgroundJobsMain {
|
|
|
1862
1863
|
const payload = {
|
|
1863
1864
|
context: {
|
|
1864
1865
|
attempts: job.attempts,
|
|
1866
|
+
childInstanceId: job.childInstanceId,
|
|
1867
|
+
childPid: job.childPid,
|
|
1865
1868
|
jobArgs: job.args,
|
|
1866
1869
|
jobId: job.id,
|
|
1867
1870
|
jobName: job.jobName,
|
|
1868
1871
|
maxRetries: job.maxRetries,
|
|
1872
|
+
runnerHostname: job.runnerHostname,
|
|
1869
1873
|
stage: "background-job-orphaned",
|
|
1870
1874
|
status: job.status,
|
|
1871
1875
|
terminal: job.status === "failed" || job.status === "orphaned",
|
|
1872
|
-
willRetry: job.status === "queued"
|
|
1876
|
+
willRetry: job.status === "queued",
|
|
1877
|
+
workerId: job.workerId
|
|
1873
1878
|
},
|
|
1874
1879
|
error: normalizedError
|
|
1875
1880
|
}
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
// @ts-check
|
|
2
|
+
|
|
3
|
+
import os from "node:os"
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Liveness verdict for the process recorded on a background job.
|
|
7
|
+
* @typedef {"alive" | "dead" | "unknown"} BackgroundJobOwnerLivenessStatus
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Verdict describing whether the process that accepted a background job can
|
|
12
|
+
* still be running.
|
|
13
|
+
* @typedef {object} BackgroundJobOwnerLivenessResult
|
|
14
|
+
* @property {string | null} childInstanceId - Recorded pooled-child identity, when available.
|
|
15
|
+
* @property {number | null} childPid - Recorded OS pid, when available.
|
|
16
|
+
* @property {string} reason - Machine-readable reason for the verdict.
|
|
17
|
+
* @property {string | null} runnerHostname - Recorded runner hostname, when available.
|
|
18
|
+
* @property {BackgroundJobOwnerLivenessStatus} status - Liveness verdict.
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* Probes a local pid without signalling the target process. `EPERM` means the
|
|
23
|
+
* process exists but belongs to another user, so it counts as alive; `ESRCH`
|
|
24
|
+
* means it is gone. Any other error is unexpected and is rethrown rather than
|
|
25
|
+
* silently reported as "dead".
|
|
26
|
+
* @param {number} pid - Process id to probe.
|
|
27
|
+
* @returns {boolean} - Whether a process with that pid exists.
|
|
28
|
+
*/
|
|
29
|
+
export function isProcessAlive(pid) {
|
|
30
|
+
if (!Number.isInteger(pid) || pid <= 0) return false
|
|
31
|
+
|
|
32
|
+
try {
|
|
33
|
+
process.kill(pid, 0)
|
|
34
|
+
return true
|
|
35
|
+
} catch (error) {
|
|
36
|
+
const code = /** @type {{code?: string}} */ (error).code
|
|
37
|
+
|
|
38
|
+
if (code === "EPERM") return true
|
|
39
|
+
if (code === "ESRCH") return false
|
|
40
|
+
|
|
41
|
+
throw error
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* Evaluates whether the process recorded on a background job is still alive.
|
|
47
|
+
*
|
|
48
|
+
* Only the local host can be answered from a PID: when the job was accepted on
|
|
49
|
+
* another machine the result is `unknown`, because liveness there needs a
|
|
50
|
+
* heartbeat or instance registry rather than a local probe. A pid alone can be
|
|
51
|
+
* recycled by the OS after a process exits, so treat `alive` as "a process with
|
|
52
|
+
* that pid exists now", not proof that it is the same process; pair it with
|
|
53
|
+
* `childStartedAtMs` when that distinction matters.
|
|
54
|
+
* @param {object} [args] - Options.
|
|
55
|
+
* @param {string | null} [args.childInstanceId] - Recorded pooled-child identity.
|
|
56
|
+
* @param {number | null} [args.childPid] - Recorded OS pid.
|
|
57
|
+
* @param {string} [args.currentHostname] - Hostname of this process, defaults to `os.hostname()`.
|
|
58
|
+
* @param {(pid: number) => boolean} [args.probe] - Process probe, injectable for tests.
|
|
59
|
+
* @param {string | null} [args.runnerHostname] - Hostname recorded with the job.
|
|
60
|
+
* @returns {BackgroundJobOwnerLivenessResult} - Liveness verdict with its reason.
|
|
61
|
+
*/
|
|
62
|
+
export function evaluateBackgroundJobOwnerLiveness({
|
|
63
|
+
childInstanceId = null,
|
|
64
|
+
childPid = null,
|
|
65
|
+
currentHostname = os.hostname(),
|
|
66
|
+
probe = isProcessAlive,
|
|
67
|
+
runnerHostname = null
|
|
68
|
+
} = {}) {
|
|
69
|
+
const owner = {
|
|
70
|
+
childInstanceId: typeof childInstanceId === "string" ? childInstanceId : null,
|
|
71
|
+
childPid: typeof childPid === "number" ? childPid : null,
|
|
72
|
+
runnerHostname: typeof runnerHostname === "string" ? runnerHostname : null
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
if (!owner.runnerHostname) {
|
|
76
|
+
return {...owner, reason: "runner_hostname_unrecorded", status: "unknown"}
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
if (owner.runnerHostname !== currentHostname) {
|
|
80
|
+
return {...owner, reason: "remote_runner_host", status: "unknown"}
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
if (!Number.isInteger(owner.childPid) || owner.childPid === null || owner.childPid <= 0) {
|
|
84
|
+
return {...owner, reason: "child_pid_unrecorded", status: "unknown"}
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
const probedPid = /** @type {number} */ (owner.childPid)
|
|
88
|
+
|
|
89
|
+
return probe(probedPid)
|
|
90
|
+
? {...owner, reason: "local_pid_alive", status: "alive"}
|
|
91
|
+
: {...owner, reason: "local_pid_missing", status: "dead"}
|
|
92
|
+
}
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
import { randomUUID } from "node:crypto"
|
|
4
4
|
import { readFileSync } from "node:fs"
|
|
5
|
+
import os from "node:os"
|
|
5
6
|
import v8 from "node:v8"
|
|
6
7
|
import timeout from "awaitery/build/timeout.js"
|
|
7
8
|
import runJobPayload, { BackgroundJobPerformedFailure } from "./job-runner.js"
|
|
@@ -16,6 +17,8 @@ const BASE_PROCESS_TITLE = "velocious background-jobs-runner"
|
|
|
16
17
|
const SHUTDOWN_OBSERVATION_SEND_TIMEOUT_MS = 100
|
|
17
18
|
/** Stable identity of this pooled child process for the life of the process. */
|
|
18
19
|
const childInstanceId = randomUUID()
|
|
20
|
+
/** Hostname (the container id under Docker) of the machine running this pooled child. */
|
|
21
|
+
const runnerHostname = os.hostname()
|
|
19
22
|
/** Sampling cadence for the memory observation sampler. */
|
|
20
23
|
const MEMORY_OBSERVATION_INTERVAL_MS = 10000
|
|
21
24
|
/** Bound on in-flight job ids carried in a memory observation. */
|
|
@@ -167,6 +170,7 @@ function sendChildAcceptance(type, payload, observedAtMs) {
|
|
|
167
170
|
handedOffAtMs: payload.handedOffAtMs,
|
|
168
171
|
handoffId: payload.handoffId,
|
|
169
172
|
jobId: payload.id,
|
|
173
|
+
runnerHostname,
|
|
170
174
|
type,
|
|
171
175
|
workerId: payload.workerId,
|
|
172
176
|
...(type === "job-received" ? {receivedAtMs: observedAtMs} : {startedAtMs: observedAtMs})
|
|
@@ -399,4 +403,4 @@ process.on("message", (message) => handleMessage(message))
|
|
|
399
403
|
process.once("disconnect", () => void shutdownRunner({exitCode: 0, reason: "ipc_disconnect"}))
|
|
400
404
|
process.once("SIGTERM", () => void shutdownRunner({exitCode: 1, reason: "signal_sigterm", signal: "SIGTERM"}))
|
|
401
405
|
process.once("SIGINT", () => void shutdownRunner({exitCode: 1, reason: "signal_sigint", signal: "SIGINT"}))
|
|
402
|
-
if (process.send) process.send({childInstanceId, childPid: process.pid, type: "ready"})
|
|
406
|
+
if (process.send) process.send({childInstanceId, childPid: process.pid, runnerHostname, type: "ready"})
|
|
@@ -103,9 +103,10 @@ export default class BackgroundJobsStatusReporter {
|
|
|
103
103
|
* @param {number} [args.startedAtMs] - Epoch ms the job's perform started in the child.
|
|
104
104
|
* @param {string} [args.childInstanceId] - Stable pooled child identity.
|
|
105
105
|
* @param {number} [args.childPid] - Pooled child OS pid.
|
|
106
|
+
* @param {string} [args.runnerHostname] - Hostname of the machine running the pooled child.
|
|
106
107
|
* @returns {Promise<void>} - Resolves when reported.
|
|
107
108
|
*/
|
|
108
|
-
async reportChildAccepted({jobId, handoffId, workerId, handedOffAtMs, receivedAtMs, startedAtMs, childInstanceId, childPid}) {
|
|
109
|
+
async reportChildAccepted({jobId, handoffId, workerId, handedOffAtMs, receivedAtMs, startedAtMs, childInstanceId, childPid, runnerHostname}) {
|
|
109
110
|
const config = this.configuration.getBackgroundJobsConfig()
|
|
110
111
|
const host = this.host || config.host
|
|
111
112
|
const port = typeof this.port === "number" ? this.port : config.port
|
|
@@ -131,7 +132,8 @@ export default class BackgroundJobsStatusReporter {
|
|
|
131
132
|
receivedAtMs,
|
|
132
133
|
startedAtMs,
|
|
133
134
|
childInstanceId,
|
|
134
|
-
childPid
|
|
135
|
+
childPid,
|
|
136
|
+
runnerHostname
|
|
135
137
|
})
|
|
136
138
|
},
|
|
137
139
|
onMessage: ({message, resolve, reject}) => {
|
|
@@ -161,16 +163,17 @@ export default class BackgroundJobsStatusReporter {
|
|
|
161
163
|
* @param {number} [args.startedAtMs] - Epoch ms the job's perform started in the child.
|
|
162
164
|
* @param {string} [args.childInstanceId] - Stable pooled child identity.
|
|
163
165
|
* @param {number} [args.childPid] - Pooled child OS pid.
|
|
166
|
+
* @param {string} [args.runnerHostname] - Hostname of the machine running the pooled child.
|
|
164
167
|
* @param {number} args.maxDurationMs - Max duration for retries.
|
|
165
168
|
* @returns {Promise<void>} - Resolves when reported or the budget elapses.
|
|
166
169
|
*/
|
|
167
|
-
async reportChildAcceptedWithRetry({jobId, handoffId, workerId, handedOffAtMs, receivedAtMs, startedAtMs, childInstanceId, childPid, maxDurationMs}) {
|
|
170
|
+
async reportChildAcceptedWithRetry({jobId, handoffId, workerId, handedOffAtMs, receivedAtMs, startedAtMs, childInstanceId, childPid, runnerHostname, maxDurationMs}) {
|
|
168
171
|
let attempt = 0
|
|
169
172
|
const startTime = Date.now()
|
|
170
173
|
|
|
171
174
|
while (true) {
|
|
172
175
|
try {
|
|
173
|
-
await this.reportChildAccepted({jobId, handoffId, workerId, handedOffAtMs, receivedAtMs, startedAtMs, childInstanceId, childPid})
|
|
176
|
+
await this.reportChildAccepted({jobId, handoffId, workerId, handedOffAtMs, receivedAtMs, startedAtMs, childInstanceId, childPid, runnerHostname})
|
|
174
177
|
return
|
|
175
178
|
} catch (error) {
|
|
176
179
|
attempt += 1
|
|
@@ -1437,9 +1437,10 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
|
|
|
1437
1437
|
* @param {number} [args.startedAtMs] - Epoch ms the job's perform started in the child.
|
|
1438
1438
|
* @param {string} [args.childInstanceId] - Stable pooled child identity.
|
|
1439
1439
|
* @param {number} [args.childPid] - Pooled child OS pid.
|
|
1440
|
+
* @param {string} [args.runnerHostname] - Hostname of the machine running the pooled child.
|
|
1440
1441
|
* @returns {Promise<boolean>} - Whether the fenced report was accepted.
|
|
1441
1442
|
*/
|
|
1442
|
-
async markChildAccepted({jobId, handoffId, workerId, handedOffAtMs, receivedAtMs, startedAtMs, childInstanceId, childPid}) {
|
|
1443
|
+
async markChildAccepted({jobId, handoffId, workerId, handedOffAtMs, receivedAtMs, startedAtMs, childInstanceId, childPid, runnerHostname}) {
|
|
1443
1444
|
await this.ensureReady()
|
|
1444
1445
|
|
|
1445
1446
|
return await this._serializedConnectionMutation(async (db) => {
|
|
@@ -1453,6 +1454,7 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
|
|
|
1453
1454
|
if (typeof startedAtMs === "number") data.child_started_at_ms = startedAtMs
|
|
1454
1455
|
if (typeof childInstanceId === "string") data.child_instance_id = childInstanceId
|
|
1455
1456
|
if (typeof childPid === "number") data.child_pid = childPid
|
|
1457
|
+
if (typeof runnerHostname === "string") data.runner_hostname = runnerHostname
|
|
1456
1458
|
if (Object.keys(data).length === 0) return false
|
|
1457
1459
|
|
|
1458
1460
|
const affectedRows = await this._updateAffectedRows(db, {
|
|
@@ -1470,15 +1472,15 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
|
|
|
1470
1472
|
* @returns {Record<string, ReturnType<typeof JSON.parse>>} - Cleared acceptance columns.
|
|
1471
1473
|
*/
|
|
1472
1474
|
_clearedChildAcceptanceData() {
|
|
1473
|
-
return {child_instance_id: null, child_pid: null, child_received_at_ms: null, child_started_at_ms: null}
|
|
1475
|
+
return {child_instance_id: null, child_pid: null, child_received_at_ms: null, child_started_at_ms: null, runner_hostname: null}
|
|
1474
1476
|
}
|
|
1475
1477
|
|
|
1476
1478
|
/**
|
|
1477
1479
|
* Returns the row-shape counterpart of the cleared acceptance columns.
|
|
1478
|
-
* @returns {Pick<import("./types.js").BackgroundJobRow, "childInstanceId" | "childPid" | "childReceivedAtMs" | "childStartedAtMs">} - Cleared acceptance fields.
|
|
1480
|
+
* @returns {Pick<import("./types.js").BackgroundJobRow, "childInstanceId" | "childPid" | "childReceivedAtMs" | "childStartedAtMs" | "runnerHostname">} - Cleared acceptance fields.
|
|
1479
1481
|
*/
|
|
1480
1482
|
_clearedChildAcceptanceRow() {
|
|
1481
|
-
return {childInstanceId: null, childPid: null, childReceivedAtMs: null, childStartedAtMs: null}
|
|
1483
|
+
return {childInstanceId: null, childPid: null, childReceivedAtMs: null, childStartedAtMs: null, runnerHostname: null}
|
|
1482
1484
|
}
|
|
1483
1485
|
|
|
1484
1486
|
/**
|
|
@@ -2275,6 +2277,7 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
|
|
|
2275
2277
|
table.bigint("child_started_at_ms", {null: true})
|
|
2276
2278
|
table.string("child_instance_id", {null: true})
|
|
2277
2279
|
table.integer("child_pid", {null: true})
|
|
2280
|
+
table.string("runner_hostname", {null: true})
|
|
2278
2281
|
|
|
2279
2282
|
await db.createTable(table)
|
|
2280
2283
|
}
|
|
@@ -2411,6 +2414,10 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
|
|
|
2411
2414
|
tableData.integer("child_pid", {null: true})
|
|
2412
2415
|
added = true
|
|
2413
2416
|
}
|
|
2417
|
+
if (!(await table.getColumnByName("runner_hostname"))) {
|
|
2418
|
+
tableData.string("runner_hostname", {null: true})
|
|
2419
|
+
added = true
|
|
2420
|
+
}
|
|
2414
2421
|
|
|
2415
2422
|
if (added) {
|
|
2416
2423
|
for (const sql of await db.alterTableSQLs(tableData)) await db.query(sql)
|
|
@@ -3219,7 +3226,8 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
|
|
|
3219
3226
|
childReceivedAtMs: this._normalizeNumber(row.child_received_at_ms),
|
|
3220
3227
|
childStartedAtMs: this._normalizeNumber(row.child_started_at_ms),
|
|
3221
3228
|
childInstanceId: row.child_instance_id ? String(row.child_instance_id) : null,
|
|
3222
|
-
childPid: this._normalizeNumber(row.child_pid)
|
|
3229
|
+
childPid: this._normalizeNumber(row.child_pid),
|
|
3230
|
+
runnerHostname: row.runner_hostname ? String(row.runner_hostname) : null
|
|
3223
3231
|
}
|
|
3224
3232
|
}
|
|
3225
3233
|
|
|
@@ -223,6 +223,7 @@
|
|
|
223
223
|
* @property {number | null} childStartedAtMs - Epoch ms when the job's perform started in the pooled child, or null when it never started.
|
|
224
224
|
* @property {string | null} childInstanceId - Stable identity of the pooled child process that accepted the job, or null.
|
|
225
225
|
* @property {number | null} childPid - OS pid of the pooled child process that accepted the job, or null.
|
|
226
|
+
* @property {string | null} runnerHostname - Hostname reported by the pooled child process that accepted the job, or null.
|
|
226
227
|
*/
|
|
227
228
|
/**
|
|
228
229
|
* @typedef {"queued" | "handed_off" | null} BackgroundJobReplacementPreviousStatus
|
|
@@ -292,7 +293,7 @@
|
|
|
292
293
|
* @typedef {{type: "schedule-woken", jobId: string | null, outcome: BackgroundJobWakeOutcome}} BackgroundJobScheduleWokenMessage
|
|
293
294
|
* @typedef {{type: "wake-scheduled-error", error?: string}} BackgroundJobWakeScheduledErrorMessage
|
|
294
295
|
* @typedef {{type: "job", payload: BackgroundJobPayload}} BackgroundJobJobMessage
|
|
295
|
-
* @typedef {{type: "job-accepted", jobId: string, handoffId?: string, workerId?: string, handedOffAtMs?: number, receivedAtMs?: number, startedAtMs?: number, childInstanceId?: string, childPid?: number}} BackgroundJobAcceptedMessage
|
|
296
|
+
* @typedef {{type: "job-accepted", jobId: string, handoffId?: string, workerId?: string, handedOffAtMs?: number, receivedAtMs?: number, startedAtMs?: number, childInstanceId?: string, childPid?: number, runnerHostname?: string}} BackgroundJobAcceptedMessage
|
|
296
297
|
* @typedef {{type: "job-complete", jobId: string, handoffId?: string, workerId?: string, handedOffAtMs?: number}} BackgroundJobCompleteMessage
|
|
297
298
|
* @typedef {{type: "job-failed", jobId: string, error?: ReturnType<typeof JSON.parse>, handoffId?: string, workerId?: string, handedOffAtMs?: number, runnerFailure?: PooledRunnerFailure}} BackgroundJobFailedMessage
|
|
298
299
|
* @typedef {{type: "job-reschedule", jobId: string, delayMs: number, handoffId?: string, workerId?: string, handedOffAtMs?: number}} BackgroundJobRescheduleMessage
|
|
@@ -93,7 +93,7 @@ function positiveInteger(value) {
|
|
|
93
93
|
* one job. The child carries its exact handoff lease so the worker can forward
|
|
94
94
|
* the report without depending on its in-flight entry still existing.
|
|
95
95
|
* @param {ReturnType<typeof JSON.parse>} message - IPC message.
|
|
96
|
-
* @returns {message is {type: "job-received" | "job-started", jobId: string, handoffId?: string, workerId?: string, handedOffAtMs?: number, receivedAtMs?: number, startedAtMs?: number, childInstanceId?: string, childPid?: number}} - Whether this is a valid acceptance message.
|
|
96
|
+
* @returns {message is {type: "job-received" | "job-started", jobId: string, handoffId?: string, workerId?: string, handedOffAtMs?: number, receivedAtMs?: number, startedAtMs?: number, childInstanceId?: string, childPid?: number, runnerHostname?: string}} - Whether this is a valid acceptance message.
|
|
97
97
|
*/
|
|
98
98
|
function isChildAcceptanceMessage(message) {
|
|
99
99
|
if (!message || typeof message !== "object") return false
|
|
@@ -108,6 +108,7 @@ function isChildAcceptanceMessage(message) {
|
|
|
108
108
|
&& (record.startedAtMs === undefined || typeof record.startedAtMs === "number")
|
|
109
109
|
&& (record.childInstanceId === undefined || typeof record.childInstanceId === "string")
|
|
110
110
|
&& (record.childPid === undefined || Number.isInteger(record.childPid))
|
|
111
|
+
&& (record.runnerHostname === undefined || typeof record.runnerHostname === "string")
|
|
111
112
|
}
|
|
112
113
|
|
|
113
114
|
/**
|
|
@@ -1510,7 +1511,7 @@ export default class BackgroundJobsWorker {
|
|
|
1510
1511
|
* or outcome that already settled the worker's in-flight entry cannot lose
|
|
1511
1512
|
* the fencing. A report that never lands degrades phase diagnostics for that
|
|
1512
1513
|
* job only — it must never block or fail the job itself.
|
|
1513
|
-
* @param {{type: "job-received" | "job-started", jobId: string, handoffId?: string, workerId?: string, handedOffAtMs?: number, receivedAtMs?: number, startedAtMs?: number, childInstanceId?: string, childPid?: number}} message - Validated child acceptance message.
|
|
1514
|
+
* @param {{type: "job-received" | "job-started", jobId: string, handoffId?: string, workerId?: string, handedOffAtMs?: number, receivedAtMs?: number, startedAtMs?: number, childInstanceId?: string, childPid?: number, runnerHostname?: string}} message - Validated child acceptance message.
|
|
1514
1515
|
* @returns {void}
|
|
1515
1516
|
*/
|
|
1516
1517
|
_reportChildAccepted(message) {
|
|
@@ -1525,6 +1526,7 @@ export default class BackgroundJobsWorker {
|
|
|
1525
1526
|
startedAtMs: message.startedAtMs,
|
|
1526
1527
|
childInstanceId: message.childInstanceId,
|
|
1527
1528
|
childPid: message.childPid,
|
|
1529
|
+
runnerHostname: message.runnerHostname,
|
|
1528
1530
|
maxDurationMs: CHILD_ACCEPTANCE_REPORT_MAX_DURATION_MS
|
|
1529
1531
|
}).catch((error) => {
|
|
1530
1532
|
console.error("Background job child-acceptance reporting failed:", error)
|
|
@@ -4,6 +4,11 @@ import {digg} from "diggerize"
|
|
|
4
4
|
import TableData from "./table-data/index.js"
|
|
5
5
|
|
|
6
6
|
const TABLE_NAME = "schema_migrations"
|
|
7
|
+
/** SQL Server "There is already an object named ..." — raised when two sessions race to create the same table. */
|
|
8
|
+
const MSSQL_OBJECT_ALREADY_EXISTS_ERROR_NUMBER = 2714
|
|
9
|
+
/** SQL Server deadlock victim — raised when concurrent sessions deadlock on the same catalog reads. */
|
|
10
|
+
const MSSQL_DEADLOCK_VICTIM_ERROR_NUMBER = 1205
|
|
11
|
+
|
|
7
12
|
|
|
8
13
|
/**
|
|
9
14
|
* Single owner of the `schema_migrations` ledger shape and the only place that reads
|
|
@@ -35,24 +40,107 @@ export default class MigrationsLedger {
|
|
|
35
40
|
return Boolean(table)
|
|
36
41
|
}
|
|
37
42
|
|
|
43
|
+
/**
|
|
44
|
+
* Whether the error means a concurrent session already created the ledger table.
|
|
45
|
+
*
|
|
46
|
+
* Several processes bootstrap the same database at the same time (for example every
|
|
47
|
+
* CI test shard), and the MSSQL `IF NOT EXISTS (...) BEGIN CREATE TABLE ... END` guard
|
|
48
|
+
* is NOT atomic: two sessions can both evaluate the check as false and then both
|
|
49
|
+
* create, so the loser is rejected. Walks the wrapped-error cause chain because
|
|
50
|
+
* `db.query` re-wraps the driver's `RequestError` in a plain `Error`, and matches the
|
|
51
|
+
* stable numeric error number instead of the localized message text.
|
|
52
|
+
* @param {unknown} error - Error thrown while creating the ledger table.
|
|
53
|
+
* @returns {boolean} - True if the table was created by a concurrent session.
|
|
54
|
+
*/
|
|
55
|
+
static isConcurrentCreationError(error) {
|
|
56
|
+
let current = error
|
|
57
|
+
|
|
58
|
+
while (current instanceof Error) {
|
|
59
|
+
const errorNumber = /** @type {{number?: unknown}} */ (current).number
|
|
60
|
+
|
|
61
|
+
if (typeof errorNumber === "number" && errorNumber === MSSQL_OBJECT_ALREADY_EXISTS_ERROR_NUMBER) return true
|
|
62
|
+
|
|
63
|
+
current = current.cause
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
return false
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* Whether the error is a transient deadlock that should be retried.
|
|
71
|
+
*
|
|
72
|
+
* Concurrent boots also deadlock on the `INFORMATION_SCHEMA` metadata scan that
|
|
73
|
+
* `getTableByName` performs on SQL Server; the victim transaction rolled back cleanly,
|
|
74
|
+
* so retrying it is safe. Matched by numeric error number, never by message text.
|
|
75
|
+
* @param {unknown} error - Error thrown while inspecting or creating the ledger table.
|
|
76
|
+
* @returns {boolean} - True if the error is a retryable deadlock.
|
|
77
|
+
*/
|
|
78
|
+
static isDeadlockError(error) {
|
|
79
|
+
let current = error
|
|
80
|
+
|
|
81
|
+
while (current instanceof Error) {
|
|
82
|
+
const errorNumber = /** @type {{number?: unknown}} */ (current).number
|
|
83
|
+
|
|
84
|
+
if (typeof errorNumber === "number" && errorNumber === MSSQL_DEADLOCK_VICTIM_ERROR_NUMBER) return true
|
|
85
|
+
|
|
86
|
+
current = current.cause
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
return false
|
|
90
|
+
}
|
|
91
|
+
|
|
38
92
|
/**
|
|
39
93
|
* Creates the ledger table if it does not exist. This is the single definition of
|
|
40
94
|
* the `schema_migrations` table shape.
|
|
95
|
+
*
|
|
96
|
+
* Safe under concurrency: a session that loses the create race (or deadlocks on the
|
|
97
|
+
* catalog read) converges on the existing table instead of failing its caller. That
|
|
98
|
+
* matters because this runs on every boot of a database configured for migrations
|
|
99
|
+
* (see `Migrator#createMigrationsTableForDatabase` and
|
|
100
|
+
* `migrateRequireContextForDatabase`), so a crash here takes down a whole test shard
|
|
101
|
+
* or deployment over a table that provably already exists.
|
|
41
102
|
* @param {import("./drivers/base.js").default} db - Database that should contain the ledger table.
|
|
103
|
+
* @param {{maxAttempts?: number}} [args] - Retry budget for transient catalog deadlocks.
|
|
42
104
|
* @returns {Promise<void>}
|
|
43
105
|
*/
|
|
44
|
-
static async ensureTable(db) {
|
|
45
|
-
|
|
106
|
+
static async ensureTable(db, {maxAttempts = 3} = {}) {
|
|
107
|
+
let attempt = 0
|
|
46
108
|
|
|
47
|
-
|
|
109
|
+
while (true) {
|
|
110
|
+
attempt++
|
|
48
111
|
|
|
49
|
-
|
|
112
|
+
try {
|
|
113
|
+
if (await MigrationsLedger.tableExists(db)) return
|
|
50
114
|
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
115
|
+
const tableData = new TableData(TABLE_NAME, {ifNotExists: true})
|
|
116
|
+
|
|
117
|
+
tableData.string("version", {null: false, primaryKey: true})
|
|
54
118
|
|
|
55
|
-
|
|
119
|
+
for (const sql of await db.createTableSql(tableData)) {
|
|
120
|
+
await db.query(sql)
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
db.clearSchemaCache()
|
|
124
|
+
|
|
125
|
+
return
|
|
126
|
+
} catch (error) {
|
|
127
|
+
if (MigrationsLedger.isConcurrentCreationError(error)) {
|
|
128
|
+
// Another session created the ledger between our check and our create. The
|
|
129
|
+
// table exists with exactly this shape, so the goal state is already reached.
|
|
130
|
+
db.clearSchemaCache()
|
|
131
|
+
|
|
132
|
+
return
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
if (MigrationsLedger.isDeadlockError(error) && attempt < maxAttempts) {
|
|
136
|
+
db.clearSchemaCache()
|
|
137
|
+
|
|
138
|
+
continue
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
throw error
|
|
142
|
+
}
|
|
143
|
+
}
|
|
56
144
|
}
|
|
57
145
|
|
|
58
146
|
/**
|
|
@@ -149,7 +149,7 @@ export default class BackgroundJobsAdapter {
|
|
|
149
149
|
* Records pooled-child acceptance evidence (received/started timestamps plus
|
|
150
150
|
* runner identity) for a handed-off job, fenced by its active handoff lease.
|
|
151
151
|
* Only the fields supplied are written.
|
|
152
|
-
* @param {{jobId: string, handoffId?: string, workerId?: string, handedOffAtMs?: number, receivedAtMs?: number, startedAtMs?: number, childInstanceId?: string, childPid?: number}} _args - Acceptance report.
|
|
152
|
+
* @param {{jobId: string, handoffId?: string, workerId?: string, handedOffAtMs?: number, receivedAtMs?: number, startedAtMs?: number, childInstanceId?: string, childPid?: number, runnerHostname?: string}} _args - Acceptance report.
|
|
153
153
|
* @returns {Promise<boolean>} - Whether the fenced report was accepted.
|
|
154
154
|
*/
|
|
155
155
|
markChildAccepted(_args: {
|
|
@@ -161,6 +161,7 @@ export default class BackgroundJobsAdapter {
|
|
|
161
161
|
startedAtMs?: number;
|
|
162
162
|
childInstanceId?: string;
|
|
163
163
|
childPid?: number;
|
|
164
|
+
runnerHostname?: string;
|
|
164
165
|
}): Promise<boolean>;
|
|
165
166
|
/**
|
|
166
167
|
* Returns a handed-off job to its schedule.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"adapter.d.ts","sourceRoot":"","sources":["../../../src/background-jobs/adapter.js"],"names":[],"mappings":"AAEA;;;;GAIG;AACH,MAAM,CAAC,OAAO,OAAO,qBAAqB;IACxC;;;;;OAKG;IACH,gCAAgC,IAFnB,OAAO,CAE+B;IAEnD;;;;;OAKG;IACH,+BAA+B,IAFlB,OAAO,CAE8B;IAElD;;;OAGG;IACG,WAAW,IAFJ,OAAO,CAAC,IAAI,CAAC,CAEqE;IAE/F;;;OAGG;IACG,KAAK,IAFE,OAAO,CAAC,IAAI,CAAC,CAEV;IAEhB;;;OAGG;IACG,MAAM,IAFC,OAAO,CAAC,OAAO,YAAY,EAAE,oBAAoB,CAAC,CAI9D;IAED;;;;;OAKG;IACG,qBAAqB,CAAC,KAAK,EAHtB;QAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,6BAA6B,EAAE,OAAO,CAAC,CAAA;KAG7C,GAFpB,OAAO,CAAC,IAAI,CAAC,CAEW;IAErC;;;OAGG;IACG,yBAAyB,IAFlB,OAAO,CAAC,IAAI,CAAC,CAEiG;IAE3H;;;;OAIG;IACG,0BAA0B,IAFnB,OAAO,CAAC,OAAO,YAAY,EAAE,sCAAsC,CAAC,CAIhF;IAED;;;;OAIG;IACG,OAAO,CAAC,KAAK,EAHR;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,KAAK,CAAC,UAAU,CAAC,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;QAAC,OAAO,CAAC,EAAE,OAAO,YAAY,EAAE,oBAAoB,CAAA;KAGzG,GAFN,OAAO,CAAC,MAAM,CAAC,CAEgE;IAE5F;;;;OAIG;IACG,uBAAuB,CAAC,KAAK,EAHxB;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,KAAK,CAAC,UAAU,CAAC,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;QAAC,OAAO,CAAC,EAAE,OAAO,YAAY,EAAE,oBAAoB,CAAC;QAAC,oBAAoB,CAAC,EAAE,MAAM,CAAC;QAAC,aAAa,EAAE,OAAO,YAAY,EAAE,0BAA0B,CAAA;KAGxL,GAFtB,OAAO,CAAC,MAAM,CAAC,CAEgG;IAE5H;;;;OAIG;IACG,gBAAgB,CAAC,KAAK,EAHjB;QAAC,WAAW,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,KAAK,CAAC,UAAU,CAAC,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;QAAC,OAAO,CAAC,EAAE,OAAO,YAAY,EAAE,oBAAoB,CAAA;KAGrH,GAFf,OAAO,CAAC,OAAO,YAAY,EAAE,8BAA8B,CAAC,CAEqC;IAE9G;;;;OAIG;IACG,eAAe,CAAC,YAAY,EAHvB,MAGuB,GAFrB,OAAO,CAAC,OAAO,YAAY,EAAE,+BAA+B,CAAC,CAEyC;IAEnH;;;;;OAKG;IACG,eAAe,CAAC,YAAY,EAJvB,MAIuB,EAAE,QAAQ,AAHzC,CACA,EADQ;QAAC,qBAAqB,CAAC,EAAE,OAAO,CAAA;KAGM,GAFpC,OAAO,CAAC,OAAO,YAAY,EAAE,kCAAkC,CAAC,CAEqD;IAElI;;;;OAIG;IACG,aAAa,CAAC,YAAY,EAHrB,MAGqB,GAFnB,OAAO,CAAC,OAAO,YAAY,EAAE,uBAAuB,CAAC,CAE6C;IAE/G;;;;OAIG;IACG,gBAAgB,CAAC,KAAK,AAHzB,CACA,EADQ;QAAC,aAAa,CAAC,EAAE,OAAO,YAAY,EAAE,0BAA0B,GAAG,OAAO,YAAY,EAAE,0BAA0B,EAAE,CAAA;KAG9F,GAFpB,OAAO,CAAC,OAAO,YAAY,EAAE,gBAAgB,GAAG,IAAI,CAAC,CAEiD;IAEnH;;;OAGG;IACG,gBAAgB,IAFT,OAAO,CAAC,OAAO,YAAY,EAAE,gBAAgB,GAAG,IAAI,CAAC,CAEuC;IAEzG;;;;OAIG;IACG,MAAM,CAAC,MAAM,EAHR,MAGQ,GAFN,OAAO,CAAC,OAAO,YAAY,EAAE,gBAAgB,GAAG,IAAI,CAAC,CAEyB;IAE3F;;;;;;OAMG;IACG,aAAa,CAAC,KAAK,EAHd,OAAO,YAAY,EAAE,2BAGP,GAFZ,OAAO,CAAC,OAAO,YAAY,EAAE,oBAAoB,GAAG,IAAI,CAAC,CAEkC;IAExG;;;;OAIG;IACG,aAAa,CAAC,KAAK,EAHd;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;QAAC,aAAa,CAAC,EAAE,MAAM,CAAA;KAG/D,GAFZ,OAAO,CAAC,OAAO,CAAC,CAE2E;IAExG;;;;;;OAMG;IACG,iBAAiB,CAAC,KAAK,EAHlB;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;QAAC,aAAa,CAAC,EAAE,MAAM,CAAC;QAAC,YAAY,CAAC,EAAE,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAC;QAAC,eAAe,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;
|
|
1
|
+
{"version":3,"file":"adapter.d.ts","sourceRoot":"","sources":["../../../src/background-jobs/adapter.js"],"names":[],"mappings":"AAEA;;;;GAIG;AACH,MAAM,CAAC,OAAO,OAAO,qBAAqB;IACxC;;;;;OAKG;IACH,gCAAgC,IAFnB,OAAO,CAE+B;IAEnD;;;;;OAKG;IACH,+BAA+B,IAFlB,OAAO,CAE8B;IAElD;;;OAGG;IACG,WAAW,IAFJ,OAAO,CAAC,IAAI,CAAC,CAEqE;IAE/F;;;OAGG;IACG,KAAK,IAFE,OAAO,CAAC,IAAI,CAAC,CAEV;IAEhB;;;OAGG;IACG,MAAM,IAFC,OAAO,CAAC,OAAO,YAAY,EAAE,oBAAoB,CAAC,CAI9D;IAED;;;;;OAKG;IACG,qBAAqB,CAAC,KAAK,EAHtB;QAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,6BAA6B,EAAE,OAAO,CAAC,CAAA;KAG7C,GAFpB,OAAO,CAAC,IAAI,CAAC,CAEW;IAErC;;;OAGG;IACG,yBAAyB,IAFlB,OAAO,CAAC,IAAI,CAAC,CAEiG;IAE3H;;;;OAIG;IACG,0BAA0B,IAFnB,OAAO,CAAC,OAAO,YAAY,EAAE,sCAAsC,CAAC,CAIhF;IAED;;;;OAIG;IACG,OAAO,CAAC,KAAK,EAHR;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,KAAK,CAAC,UAAU,CAAC,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;QAAC,OAAO,CAAC,EAAE,OAAO,YAAY,EAAE,oBAAoB,CAAA;KAGzG,GAFN,OAAO,CAAC,MAAM,CAAC,CAEgE;IAE5F;;;;OAIG;IACG,uBAAuB,CAAC,KAAK,EAHxB;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,KAAK,CAAC,UAAU,CAAC,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;QAAC,OAAO,CAAC,EAAE,OAAO,YAAY,EAAE,oBAAoB,CAAC;QAAC,oBAAoB,CAAC,EAAE,MAAM,CAAC;QAAC,aAAa,EAAE,OAAO,YAAY,EAAE,0BAA0B,CAAA;KAGxL,GAFtB,OAAO,CAAC,MAAM,CAAC,CAEgG;IAE5H;;;;OAIG;IACG,gBAAgB,CAAC,KAAK,EAHjB;QAAC,WAAW,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,KAAK,CAAC,UAAU,CAAC,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;QAAC,OAAO,CAAC,EAAE,OAAO,YAAY,EAAE,oBAAoB,CAAA;KAGrH,GAFf,OAAO,CAAC,OAAO,YAAY,EAAE,8BAA8B,CAAC,CAEqC;IAE9G;;;;OAIG;IACG,eAAe,CAAC,YAAY,EAHvB,MAGuB,GAFrB,OAAO,CAAC,OAAO,YAAY,EAAE,+BAA+B,CAAC,CAEyC;IAEnH;;;;;OAKG;IACG,eAAe,CAAC,YAAY,EAJvB,MAIuB,EAAE,QAAQ,AAHzC,CACA,EADQ;QAAC,qBAAqB,CAAC,EAAE,OAAO,CAAA;KAGM,GAFpC,OAAO,CAAC,OAAO,YAAY,EAAE,kCAAkC,CAAC,CAEqD;IAElI;;;;OAIG;IACG,aAAa,CAAC,YAAY,EAHrB,MAGqB,GAFnB,OAAO,CAAC,OAAO,YAAY,EAAE,uBAAuB,CAAC,CAE6C;IAE/G;;;;OAIG;IACG,gBAAgB,CAAC,KAAK,AAHzB,CACA,EADQ;QAAC,aAAa,CAAC,EAAE,OAAO,YAAY,EAAE,0BAA0B,GAAG,OAAO,YAAY,EAAE,0BAA0B,EAAE,CAAA;KAG9F,GAFpB,OAAO,CAAC,OAAO,YAAY,EAAE,gBAAgB,GAAG,IAAI,CAAC,CAEiD;IAEnH;;;OAGG;IACG,gBAAgB,IAFT,OAAO,CAAC,OAAO,YAAY,EAAE,gBAAgB,GAAG,IAAI,CAAC,CAEuC;IAEzG;;;;OAIG;IACG,MAAM,CAAC,MAAM,EAHR,MAGQ,GAFN,OAAO,CAAC,OAAO,YAAY,EAAE,gBAAgB,GAAG,IAAI,CAAC,CAEyB;IAE3F;;;;;;OAMG;IACG,aAAa,CAAC,KAAK,EAHd,OAAO,YAAY,EAAE,2BAGP,GAFZ,OAAO,CAAC,OAAO,YAAY,EAAE,oBAAoB,GAAG,IAAI,CAAC,CAEkC;IAExG;;;;OAIG;IACG,aAAa,CAAC,KAAK,EAHd;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;QAAC,aAAa,CAAC,EAAE,MAAM,CAAA;KAG/D,GAFZ,OAAO,CAAC,OAAO,CAAC,CAE2E;IAExG;;;;;;OAMG;IACG,iBAAiB,CAAC,KAAK,EAHlB;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;QAAC,aAAa,CAAC,EAAE,MAAM,CAAC;QAAC,YAAY,CAAC,EAAE,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAC;QAAC,eAAe,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;QAAC,cAAc,CAAC,EAAE,MAAM,CAAA;KAG9K,GAFhB,OAAO,CAAC,OAAO,CAAC,CAEmF;IAEhH;;;;OAIG;IACG,eAAe,CAAC,KAAK,EAHhB;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;QAAC,aAAa,CAAC,EAAE,MAAM,CAAA;KAG9E,GAFd,OAAO,CAAC,OAAO,CAAC,CAE+E;IAE5G;;;;OAIG;IACG,mBAAmB,CAAC,KAAK,EAHpB;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAGb,GAFlB,OAAO,CAAC,IAAI,CAAC,CAE0F;IAEpH;;;;OAIG;IACG,sBAAsB,CAAC,KAAK,EAHvB;QAAC,QAAQ,EAAE,MAAM,CAAA;KAGM,GAFrB,OAAO,CAAC,KAAK,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAC,CAAC,CAAC,CAE2D;IAE1H;;;;OAIG;IACG,qBAAqB,IAFd,OAAO,CAAC,OAAO,YAAY,EAAE,4BAA4B,EAAE,CAAC,CAE9B;IAE3C;;;;;;OAMG;IACG,oBAAoB,CAAC,KAAK,EAHrB;QAAC,QAAQ,EAAE,OAAO,YAAY,EAAE,4BAA4B,EAAE,CAAC;QAAC,KAAK,EAAE,UAAU,CAAC,OAAO,IAAI,CAAC,KAAK,CAAC,CAAA;KAG/E,GAFnB,OAAO,CAAC,OAAO,YAAY,EAAE,gBAAgB,EAAE,CAAC,CAEd;IAE/C;;;;OAIG;IACG,UAAU,CAAC,KAAK,EAHX;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,UAAU,CAAC,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;QAAC,aAAa,CAAC,EAAE,MAAM,CAAA;KAGxG,GAFT,OAAO,CAAC,OAAO,YAAY,EAAE,gBAAgB,GAAG,IAAI,CAAC,CAEgC;IAElG;;;;OAIG;IACG,gBAAgB,CAAC,KAAK,AAHzB,CACA,EADQ;QAAC,eAAe,CAAC,EAAE,MAAM,CAAA;KAGH,GAFpB,OAAO,CAAC,OAAO,YAAY,EAAE,gBAAgB,EAAE,CAAC,CAEsD;IAEnH;;;;OAIG;IACG,iBAAiB,CAAC,KAAK,AAH1B,CACA,EADQ;QAAC,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAA;KAGzD,GAFrB,OAAO,CAAC,MAAM,CAAC,CAEyF;CACtH"}
|