velocious 1.0.723 → 1.0.725
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/build/background-jobs/forked-runner-child.js +44 -16
- package/build/background-jobs/store.js +56 -20
- package/build/background-jobs/worker.js +195 -39
- package/build/database/drivers/mysql/index.js +29 -0
- package/build/src/background-jobs/forked-runner-child.d.ts +14 -1
- package/build/src/background-jobs/forked-runner-child.d.ts.map +1 -1
- package/build/src/background-jobs/forked-runner-child.js +41 -16
- package/build/src/background-jobs/store.d.ts +21 -7
- package/build/src/background-jobs/store.d.ts.map +1 -1
- package/build/src/background-jobs/store.js +58 -20
- package/build/src/background-jobs/worker.d.ts +130 -28
- package/build/src/background-jobs/worker.d.ts.map +1 -1
- package/build/src/background-jobs/worker.js +185 -39
- package/build/src/database/drivers/mysql/index.d.ts.map +1 -1
- package/build/src/database/drivers/mysql/index.js +27 -1
- package/package.json +1 -1
- package/src/background-jobs/forked-runner-child.js +44 -16
- package/src/background-jobs/store.js +56 -20
- package/src/background-jobs/worker.js +195 -39
- package/src/database/drivers/mysql/index.js +29 -0
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
// @ts-check
|
|
2
2
|
|
|
3
3
|
import {createHash, randomUUID} from "crypto"
|
|
4
|
+
import os from "node:os"
|
|
4
5
|
import BackgroundJobsAdapter from "./adapter.js"
|
|
5
6
|
import Logger from "../logger.js"
|
|
6
7
|
import TableData from "../database/table-data/index.js"
|
|
7
8
|
import VelociousError from "../velocious-error.js"
|
|
8
9
|
import BackgroundJobRecord from "./job-record.js"
|
|
9
10
|
import normalizeBackgroundJobError from "./normalize-error.js"
|
|
11
|
+
import {evaluateBackgroundJobOwnerLiveness, isProcessAlive} from "./owner-liveness.js"
|
|
10
12
|
import { coordinateSharedTransactionConnection } from "../testing/shared-transaction-connection-coordinator.js"
|
|
11
13
|
import stableJsonStringify from "../utils/stable-json.js"
|
|
12
14
|
import {
|
|
@@ -51,6 +53,7 @@ import {
|
|
|
51
53
|
* BackgroundJobOrphanSelection type.
|
|
52
54
|
* @typedef {object} BackgroundJobOrphanSelection
|
|
53
55
|
* @property {Record<string, ReturnType<typeof JSON.parse>>} conditions - Exact update fence.
|
|
56
|
+
* @property {ReturnType<typeof JSON.parse>} error - Orphan reason for this selection.
|
|
54
57
|
* @property {import("./types.js").BackgroundJobRow} job - Selected active handoff.
|
|
55
58
|
*/
|
|
56
59
|
|
|
@@ -1679,11 +1682,12 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
|
|
|
1679
1682
|
status: "handed_off",
|
|
1680
1683
|
worker_id: handoff.workerId
|
|
1681
1684
|
},
|
|
1685
|
+
error,
|
|
1682
1686
|
job
|
|
1683
1687
|
})
|
|
1684
1688
|
}
|
|
1685
1689
|
|
|
1686
|
-
return await this._markOrphanSelections({db,
|
|
1690
|
+
return await this._markOrphanSelections({db, selections})
|
|
1687
1691
|
})
|
|
1688
1692
|
}
|
|
1689
1693
|
|
|
@@ -1714,29 +1718,65 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
|
|
|
1714
1718
|
}
|
|
1715
1719
|
|
|
1716
1720
|
/**
|
|
1717
|
-
*
|
|
1721
|
+
* Reaps `handed_off` jobs whose executing child is gone. A live child is
|
|
1722
|
+
* never reaped: a job that is still running (or whose child is still
|
|
1723
|
+
* starting) keeps its lease and its concurrency slot. A child recorded on
|
|
1724
|
+
* this host is reaped only when the handoff age exceeds the job's effective
|
|
1725
|
+
* timeout AND the pid probe proves the child is gone — never on age alone,
|
|
1726
|
+
* because a pid that still exists may be the same child mid-startup. Rows
|
|
1727
|
+
* whose liveness cannot be answered (remote runner host, or no recorded pid)
|
|
1728
|
+
* stay under the bounded age sweep, which is the recovery path for crashes
|
|
1729
|
+
* that cannot report (container restart, OOM kill).
|
|
1718
1730
|
* @param {object} [args] - Options.
|
|
1719
|
-
* @param {number} [args.orphanedAfterMs] - Mark jobs orphaned after this duration.
|
|
1731
|
+
* @param {number} [args.orphanedAfterMs] - Mark jobs orphaned after this duration when liveness is unknown.
|
|
1732
|
+
* @param {(pid: number) => boolean} [args.livenessProbe] - Process probe for local children, injectable for tests.
|
|
1733
|
+
* @param {string} [args.currentHostname] - Hostname of this process, defaults to `os.hostname()`.
|
|
1720
1734
|
* @returns {Promise<import("./types.js").BackgroundJobRow[]>} - The jobs this sweep marked orphaned.
|
|
1721
1735
|
*/
|
|
1722
|
-
async markOrphanedJobs({orphanedAfterMs = ORPHANED_AFTER_MS} = {}) {
|
|
1736
|
+
async markOrphanedJobs({orphanedAfterMs = ORPHANED_AFTER_MS, livenessProbe = isProcessAlive, currentHostname = os.hostname()} = {}) {
|
|
1723
1737
|
await this.ensureReady()
|
|
1724
1738
|
|
|
1725
1739
|
return await this._serializedCountMutation(async (db) => {
|
|
1726
|
-
const
|
|
1727
|
-
const
|
|
1740
|
+
const now = this.clock.now()
|
|
1741
|
+
const rows = await db
|
|
1728
1742
|
.newQuery()
|
|
1729
1743
|
.from(JOBS_TABLE)
|
|
1730
1744
|
.where({status: "handed_off"})
|
|
1731
|
-
.
|
|
1732
|
-
|
|
1733
|
-
const rows = await query.results()
|
|
1745
|
+
.results()
|
|
1734
1746
|
|
|
1735
1747
|
/** @type {BackgroundJobOrphanSelection[]} */
|
|
1736
1748
|
const selections = []
|
|
1737
1749
|
|
|
1738
1750
|
for (const row of rows) {
|
|
1739
1751
|
const job = this._normalizeJobRow(row)
|
|
1752
|
+
// A handed-off row without a handoff timestamp cannot be aged; it stays
|
|
1753
|
+
// for startup reconciliation rather than being reaped blindly.
|
|
1754
|
+
if (job.handedOffAtMs === null) continue
|
|
1755
|
+
const ageMs = now - job.handedOffAtMs
|
|
1756
|
+
const liveness = evaluateBackgroundJobOwnerLiveness({
|
|
1757
|
+
childInstanceId: job.childInstanceId,
|
|
1758
|
+
childPid: job.childPid,
|
|
1759
|
+
currentHostname,
|
|
1760
|
+
probe: livenessProbe,
|
|
1761
|
+
runnerHostname: job.runnerHostname
|
|
1762
|
+
})
|
|
1763
|
+
|
|
1764
|
+
if (liveness.status === "alive") continue
|
|
1765
|
+
|
|
1766
|
+
/** @type {ReturnType<typeof JSON.parse>} */
|
|
1767
|
+
let error
|
|
1768
|
+
|
|
1769
|
+
if (liveness.status === "dead") {
|
|
1770
|
+
const effectiveTimeoutMs = job.timeoutMs ?? this.configuration.getBackgroundJobsConfig().jobTimeoutMs ?? orphanedAfterMs
|
|
1771
|
+
|
|
1772
|
+
if (ageMs <= effectiveTimeoutMs) continue
|
|
1773
|
+
error = `Job orphaned: handoff child ${liveness.reason} after ${ageMs}ms (timeout ${effectiveTimeoutMs}ms)`
|
|
1774
|
+
} else {
|
|
1775
|
+
// Matches the original sweep boundary: `handed_off_at_ms <= now - orphanedAfterMs`,
|
|
1776
|
+
// i.e. reap when the age reaches the configured window.
|
|
1777
|
+
if (ageMs < orphanedAfterMs) continue
|
|
1778
|
+
error = "Job orphaned after timeout"
|
|
1779
|
+
}
|
|
1740
1780
|
|
|
1741
1781
|
// Fence the reclaim on the exact handoff this sweep selected, using its
|
|
1742
1782
|
// `handed_off_at_ms` rather than its `handoff_id`. Two reasons:
|
|
@@ -1749,19 +1789,16 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
|
|
|
1749
1789
|
// `handed_off_at_ms` (always "now"), so this stale cutoff-era
|
|
1750
1790
|
// timestamp no longer matches and we won't fail/orphan — or
|
|
1751
1791
|
// wrongly release the concurrency reservation of — that new lease.
|
|
1752
|
-
// `handed_off_at_ms` is always set on a handed-off row
|
|
1753
|
-
//
|
|
1792
|
+
// `handed_off_at_ms` is always set on a handed-off row, so it is a
|
|
1793
|
+
// reliable null-safe lease pin.
|
|
1754
1794
|
selections.push({
|
|
1755
1795
|
conditions: {id: job.id, status: "handed_off", handed_off_at_ms: job.handedOffAtMs},
|
|
1796
|
+
error,
|
|
1756
1797
|
job
|
|
1757
1798
|
})
|
|
1758
1799
|
}
|
|
1759
1800
|
|
|
1760
|
-
return await this._markOrphanSelections({
|
|
1761
|
-
db,
|
|
1762
|
-
error: "Job orphaned after timeout",
|
|
1763
|
-
selections
|
|
1764
|
-
})
|
|
1801
|
+
return await this._markOrphanSelections({db, selections})
|
|
1765
1802
|
})
|
|
1766
1803
|
}
|
|
1767
1804
|
|
|
@@ -1770,15 +1807,14 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
|
|
|
1770
1807
|
* delta for the accepted rows.
|
|
1771
1808
|
* @param {object} args - Options.
|
|
1772
1809
|
* @param {import("../database/drivers/base.js").default} args.db - Transaction connection.
|
|
1773
|
-
* @param {
|
|
1774
|
-
* @param {BackgroundJobOrphanSelection[]} args.selections - Selected handoffs and exact fences.
|
|
1810
|
+
* @param {BackgroundJobOrphanSelection[]} args.selections - Selected handoffs, per-row orphan reasons and exact fences.
|
|
1775
1811
|
* @returns {Promise<import("./types.js").BackgroundJobRow[]>} - Accepted transitions.
|
|
1776
1812
|
*/
|
|
1777
|
-
async _markOrphanSelections({db,
|
|
1813
|
+
async _markOrphanSelections({db, selections}) {
|
|
1778
1814
|
/** @type {import("./types.js").BackgroundJobRow[]} */
|
|
1779
1815
|
const orphanedJobs = []
|
|
1780
1816
|
|
|
1781
|
-
for (const {conditions, job} of selections) {
|
|
1817
|
+
for (const {conditions, error, job} of selections) {
|
|
1782
1818
|
const orphanedJob = await this._applyFailure({
|
|
1783
1819
|
conditions,
|
|
1784
1820
|
db,
|
|
@@ -17,12 +17,24 @@ import BackgroundJobsGenerationHandshakeTimeoutError, { DEFAULT_GENERATION_HANDS
|
|
|
17
17
|
import { POOLED_RUNNER_INFLIGHT_JOB_ID_LIMIT, boundedPooledRunnerInflightJobIds, isPooledChildShutdownReason, isPooledChildShutdownSignal } from "./pooled-runner-shutdown.js"
|
|
18
18
|
|
|
19
19
|
/**
|
|
20
|
-
* Per-forked-child
|
|
21
|
-
*
|
|
20
|
+
* Per-forked-child bookkeeping, shared by the wall-clock backstop, the runner's
|
|
21
|
+
* own job-outcome report, and the exit/error handlers.
|
|
22
|
+
* @typedef {object} ForkedJobState
|
|
22
23
|
* @property {boolean} timedOut - Whether the timeout fired and the child was terminated.
|
|
23
24
|
* @property {number | null} timeoutMs - The armed timeout in ms, or null when disabled.
|
|
24
25
|
* @property {ReturnType<typeof setTimeout> | null} timer - The pending timeout timer, cleared on exit.
|
|
25
26
|
* @property {ReturnType<typeof setTimeout> | null} sigkillTimer - The pending SIGKILL grace timer, cleared on exit.
|
|
27
|
+
* @property {boolean} outcomeReported - Whether the runner reported its job outcome over IPC.
|
|
28
|
+
* @property {boolean} outcomeAcknowledged - Whether that report confirmed the outcome was recorded.
|
|
29
|
+
* @property {boolean} settled - Whether the worker's in-flight entry for this job has been freed.
|
|
30
|
+
* @property {(value: void) => void} resolve - Resolver that frees the in-flight entry; a placeholder until the promise that owns the entry is created.
|
|
31
|
+
*/
|
|
32
|
+
/**
|
|
33
|
+
* One live runner child, kept so a shutdown that runs out of drain time can name
|
|
34
|
+
* the exact process it is still waiting on instead of a bare count.
|
|
35
|
+
* @typedef {object} ProcessJobHolder
|
|
36
|
+
* @property {string} jobId - Job the child was handed.
|
|
37
|
+
* @property {number} startedAtMs - When the worker handed it that job.
|
|
26
38
|
*/
|
|
27
39
|
/**
|
|
28
40
|
* @typedef {object} PooledJobEntry
|
|
@@ -123,6 +135,24 @@ function isChildAcceptanceMessage(message) {
|
|
|
123
135
|
&& (record.runnerHostname === undefined || typeof record.runnerHostname === "string")
|
|
124
136
|
}
|
|
125
137
|
|
|
138
|
+
/**
|
|
139
|
+
* Checks whether an IPC value is a forked runner's job-outcome report. The forked
|
|
140
|
+
* child sends exactly one of these, once its own terminal report has settled, and
|
|
141
|
+
* says whether that report was acknowledged — so the worker can tell a runner that
|
|
142
|
+
* has finished its job from one that is still running it.
|
|
143
|
+
* @param {ReturnType<typeof JSON.parse>} message - IPC message.
|
|
144
|
+
* @returns {message is {acknowledged: boolean, error?: string, jobId?: string, type: "job-reported"}} - Whether this is a job-outcome report.
|
|
145
|
+
*/
|
|
146
|
+
function isForkedChildJobReport(message) {
|
|
147
|
+
if (!message || typeof message !== "object") return false
|
|
148
|
+
const record = /** @type {{acknowledged?: ReturnType<typeof JSON.parse>, error?: ReturnType<typeof JSON.parse>, jobId?: ReturnType<typeof JSON.parse>, type?: ReturnType<typeof JSON.parse>}} */ (message)
|
|
149
|
+
|
|
150
|
+
return record.type === "job-reported"
|
|
151
|
+
&& typeof record.acknowledged === "boolean"
|
|
152
|
+
&& (record.jobId === undefined || typeof record.jobId === "string")
|
|
153
|
+
&& (record.error === undefined || typeof record.error === "string")
|
|
154
|
+
}
|
|
155
|
+
|
|
126
156
|
/**
|
|
127
157
|
* Checks whether an IPC value is the child's bounded pre-teardown observation.
|
|
128
158
|
* @param {ReturnType<typeof JSON.parse>} message - IPC message.
|
|
@@ -366,6 +396,13 @@ export default class BackgroundJobsWorker {
|
|
|
366
396
|
* @type {Set<import("node:child_process").ChildProcess>}
|
|
367
397
|
*/
|
|
368
398
|
this.inflightProcessChildren = new Set()
|
|
399
|
+
/**
|
|
400
|
+
* Which job each live runner child was handed. A shutdown that runs out of
|
|
401
|
+
* drain time names the exact process it is still waiting on (pid, job, how
|
|
402
|
+
* long it has been held) from this map instead of reporting only a count.
|
|
403
|
+
* @type {Map<import("node:child_process").ChildProcess, ProcessJobHolder>}
|
|
404
|
+
*/
|
|
405
|
+
this.processChildJobs = new Map()
|
|
369
406
|
/** @type {Set<Promise<void>>} */
|
|
370
407
|
this.inflightPooledJobs = new Set()
|
|
371
408
|
/** @type {Map<string, Array<import("./types.js").BackgroundJobPayload & {id: string}>>} */
|
|
@@ -551,9 +588,9 @@ export default class BackgroundJobsWorker {
|
|
|
551
588
|
}
|
|
552
589
|
}
|
|
553
590
|
|
|
554
|
-
await this._drainInflight(this.inflightInlineJobs, timeoutMs)
|
|
555
|
-
await this._drainInflight(this.inflightPooledJobs, timeoutMs)
|
|
556
|
-
await this._drainInflight(this.inflightProcessJobs, timeoutMs)
|
|
591
|
+
await this._drainInflight(this.inflightInlineJobs, timeoutMs, {phase: "inline-jobs"})
|
|
592
|
+
await this._drainInflight(this.inflightPooledJobs, timeoutMs, {phase: "pooled-jobs"})
|
|
593
|
+
await this._drainInflight(this.inflightProcessJobs, timeoutMs, {phase: "process-jobs", describeHolders: () => this._describeProcessJobHolders()})
|
|
557
594
|
await this._terminateProcessChildren()
|
|
558
595
|
// Give in-flight result reports (now decoupled from job slots) a bounded
|
|
559
596
|
// chance to land before the socket closes.
|
|
@@ -645,12 +682,17 @@ export default class BackgroundJobsWorker {
|
|
|
645
682
|
|
|
646
683
|
/**
|
|
647
684
|
* Waits for a set of in-flight job promises to settle, optionally bounded by
|
|
648
|
-
* `timeoutMs`.
|
|
685
|
+
* `timeoutMs`. When a bounded window runs out with entries still held, the drain
|
|
686
|
+
* names what it is waiting on instead of returning silently: an unexplained wait
|
|
687
|
+
* is what turned one wedged worker into a 28-minute investigation.
|
|
649
688
|
* @param {Set<Promise<void>>} inflight - In-flight job promises.
|
|
650
689
|
* @param {number} [timeoutMs] - Max wait in ms; unbounded when omitted.
|
|
690
|
+
* @param {object} [options] - Drain options.
|
|
691
|
+
* @param {string} [options.phase] - Drain phase named in a timeout report.
|
|
692
|
+
* @param {(() => ProcessJobHolder[])} [options.describeHolders] - Names the live processes this phase is waiting on.
|
|
651
693
|
* @returns {Promise<void>} - Resolves when settled or the timeout elapses.
|
|
652
694
|
*/
|
|
653
|
-
async _drainInflight(inflight, timeoutMs) {
|
|
695
|
+
async _drainInflight(inflight, timeoutMs, {phase = "inflight", describeHolders} = {}) {
|
|
654
696
|
if (inflight.size === 0) return
|
|
655
697
|
|
|
656
698
|
const drain = Promise.allSettled([...inflight])
|
|
@@ -661,11 +703,51 @@ export default class BackgroundJobsWorker {
|
|
|
661
703
|
|
|
662
704
|
await Promise.race([drain, timeout])
|
|
663
705
|
clearTimeout(timer)
|
|
706
|
+
|
|
707
|
+
if (inflight.size > 0) {
|
|
708
|
+
this._logDrainTimeout({phase, timeoutMs, pendingCount: inflight.size, holders: describeHolders ? describeHolders() : []})
|
|
709
|
+
}
|
|
664
710
|
} else {
|
|
665
711
|
await drain
|
|
666
712
|
}
|
|
667
713
|
}
|
|
668
714
|
|
|
715
|
+
/**
|
|
716
|
+
* Names every live runner child the worker still holds a job for, so a drain
|
|
717
|
+
* that ran out of time says which process it is waiting on, for which job, and
|
|
718
|
+
* for how long.
|
|
719
|
+
* @returns {ProcessJobHolder[]} - Held job descriptors, oldest first.
|
|
720
|
+
*/
|
|
721
|
+
_describeProcessJobHolders() {
|
|
722
|
+
return [...this.processChildJobs.values()].sort((left, right) => left.startedAtMs - right.startedAtMs)
|
|
723
|
+
}
|
|
724
|
+
|
|
725
|
+
/**
|
|
726
|
+
* Logs one compact drain-timeout event to the worker's own stderr (the surface
|
|
727
|
+
* that reaches the production log), naming the held processes and how long each
|
|
728
|
+
* has been in flight.
|
|
729
|
+
* @param {object} args - Event fields.
|
|
730
|
+
* @param {string} args.phase - Drain phase that ran out of time.
|
|
731
|
+
* @param {number} args.timeoutMs - The bound that expired.
|
|
732
|
+
* @param {number} args.pendingCount - Entries still unsettled when it expired.
|
|
733
|
+
* @param {ProcessJobHolder[]} args.holders - Held processes named by the phase.
|
|
734
|
+
* @returns {void}
|
|
735
|
+
*/
|
|
736
|
+
_logDrainTimeout({phase, timeoutMs, pendingCount, holders}) {
|
|
737
|
+
const nowMs = Date.now()
|
|
738
|
+
|
|
739
|
+
console.error(JSON.stringify({
|
|
740
|
+
event: "worker-shutdown-drain-timeout",
|
|
741
|
+
phase,
|
|
742
|
+
timeoutMs,
|
|
743
|
+
pendingCount,
|
|
744
|
+
held: holders.map((holder) => ({
|
|
745
|
+
jobId: holder.jobId,
|
|
746
|
+
heldMs: nowMs - holder.startedAtMs
|
|
747
|
+
}))
|
|
748
|
+
}))
|
|
749
|
+
}
|
|
750
|
+
|
|
669
751
|
/**
|
|
670
752
|
* Terminates any process runner children still alive after the drain window so
|
|
671
753
|
* they don't outlive the worker as orphans. SIGTERM lets the runner close its
|
|
@@ -2147,12 +2229,13 @@ export default class BackgroundJobsWorker {
|
|
|
2147
2229
|
/**
|
|
2148
2230
|
* Runs fork job.
|
|
2149
2231
|
* @param {import("./types.js").BackgroundJobPayload & {id: string}} payload - Payload.
|
|
2150
|
-
* @returns {Promise<void>} - Resolves when the
|
|
2232
|
+
* @returns {Promise<void>} - Resolves when the runner reports its job outcome or the fork fails.
|
|
2151
2233
|
*/
|
|
2152
2234
|
_forkJob(payload) {
|
|
2153
2235
|
const child = this._createForkedChild()
|
|
2154
2236
|
|
|
2155
2237
|
this.inflightProcessChildren.add(child)
|
|
2238
|
+
this.processChildJobs.set(child, {jobId: payload.id, startedAtMs: Date.now()})
|
|
2156
2239
|
|
|
2157
2240
|
const finished = this._waitForForkedChild({child, payload})
|
|
2158
2241
|
|
|
@@ -2183,46 +2266,108 @@ export default class BackgroundJobsWorker {
|
|
|
2183
2266
|
* @param {object} args - Options.
|
|
2184
2267
|
* @param {import("node:child_process").ChildProcess} args.child - Forked child process.
|
|
2185
2268
|
* @param {import("./types.js").BackgroundJobPayload & {id: string}} args.payload - Payload.
|
|
2186
|
-
* @returns {Promise<void>} - Resolves when the child exits.
|
|
2269
|
+
* @returns {Promise<void>} - Resolves when the runner reports its job outcome or the child exits.
|
|
2187
2270
|
*/
|
|
2188
2271
|
_waitForForkedChild({child, payload}) {
|
|
2189
|
-
|
|
2272
|
+
/** @type {ForkedJobState} */
|
|
2273
|
+
const state = {
|
|
2274
|
+
timedOut: false,
|
|
2275
|
+
timeoutMs: null,
|
|
2276
|
+
timer: null,
|
|
2277
|
+
sigkillTimer: null,
|
|
2278
|
+
outcomeReported: false,
|
|
2279
|
+
outcomeAcknowledged: false,
|
|
2280
|
+
settled: false,
|
|
2281
|
+
resolve: () => {}
|
|
2282
|
+
}
|
|
2190
2283
|
|
|
2191
2284
|
return new Promise((resolve) => {
|
|
2285
|
+
state.resolve = resolve
|
|
2286
|
+
this._armForkedJobTimeout({child, payload, state})
|
|
2287
|
+
|
|
2288
|
+
child.once("message", (message) => {
|
|
2289
|
+
this._handleForkedChildJobReport({child, message, payload, state})
|
|
2290
|
+
})
|
|
2192
2291
|
child.once("exit", (code, signal) => {
|
|
2193
|
-
this._clearForkedJobTimeout(
|
|
2194
|
-
this._handleForkedChildExit({child, code, signal, payload,
|
|
2292
|
+
this._clearForkedJobTimeout(state)
|
|
2293
|
+
this._handleForkedChildExit({child, code, signal, payload, state})
|
|
2195
2294
|
})
|
|
2196
2295
|
child.once("error", (error) => {
|
|
2197
|
-
this._clearForkedJobTimeout(
|
|
2198
|
-
this._handleForkedChildError({child, error, payload,
|
|
2296
|
+
this._clearForkedJobTimeout(state)
|
|
2297
|
+
this._handleForkedChildError({child, error, payload, state})
|
|
2199
2298
|
})
|
|
2200
2299
|
})
|
|
2201
2300
|
}
|
|
2202
2301
|
|
|
2302
|
+
/**
|
|
2303
|
+
* Handles one forked runner's job-outcome report. The runner has finished the job
|
|
2304
|
+
* it was handed, so the worker frees the slot here instead of waiting for the
|
|
2305
|
+
* process to exit: a runner that holds no job has no reason to exit, and a
|
|
2306
|
+
* shutdown drain that waits for one never finishes. The wall-clock backstop stays
|
|
2307
|
+
* armed, so a runner that reports and then lingers is still terminated.
|
|
2308
|
+
* @param {object} args - Options.
|
|
2309
|
+
* @param {import("node:child_process").ChildProcess} args.child - Forked child process.
|
|
2310
|
+
* @param {ReturnType<typeof JSON.parse>} args.message - IPC message.
|
|
2311
|
+
* @param {import("./types.js").BackgroundJobPayload & {id: string}} args.payload - Job payload.
|
|
2312
|
+
* @param {ForkedJobState} args.state - Shared bookkeeping for this runner.
|
|
2313
|
+
* @returns {void}
|
|
2314
|
+
*/
|
|
2315
|
+
_handleForkedChildJobReport({child, message, payload, state}) {
|
|
2316
|
+
if (!isForkedChildJobReport(message)) return
|
|
2317
|
+
|
|
2318
|
+
state.outcomeReported = true
|
|
2319
|
+
state.outcomeAcknowledged = message.acknowledged === true
|
|
2320
|
+
|
|
2321
|
+
if (!state.outcomeAcknowledged) {
|
|
2322
|
+
// The runner stayed alive but could not confirm its own terminal report, so
|
|
2323
|
+
// reclaim the job exactly as the pooled lane does before freeing the slot.
|
|
2324
|
+
this._reportForkedChildFailure({
|
|
2325
|
+
payload,
|
|
2326
|
+
error: new Error(message.error || "Forked background job runner terminal report was not acknowledged")
|
|
2327
|
+
})
|
|
2328
|
+
}
|
|
2329
|
+
|
|
2330
|
+
this._settleForkedJob({child, state})
|
|
2331
|
+
}
|
|
2332
|
+
|
|
2333
|
+
/**
|
|
2334
|
+
* Frees the worker's in-flight entry for one forked job exactly once, so neither
|
|
2335
|
+
* a job-outcome report nor a later exit can settle it twice.
|
|
2336
|
+
* @param {object} args - Options.
|
|
2337
|
+
* @param {import("node:child_process").ChildProcess} args.child - Forked child process.
|
|
2338
|
+
* @param {ForkedJobState} args.state - Shared bookkeeping for this runner.
|
|
2339
|
+
* @returns {void}
|
|
2340
|
+
*/
|
|
2341
|
+
_settleForkedJob({child, state}) {
|
|
2342
|
+
if (state.settled) return
|
|
2343
|
+
|
|
2344
|
+
state.settled = true
|
|
2345
|
+
this.processChildJobs.delete(child)
|
|
2346
|
+
state.resolve(undefined)
|
|
2347
|
+
}
|
|
2348
|
+
|
|
2203
2349
|
/**
|
|
2204
2350
|
* Arms a wall-clock backstop for a forked job runner. A forked job still
|
|
2205
2351
|
* running after `jobTimeoutMs` is terminated (SIGTERM, then SIGKILL after the
|
|
2206
2352
|
* grace) so a single genuinely-hung runner can't pin a draining worker — and
|
|
2207
|
-
* its full-app boot and database connections — indefinitely.
|
|
2208
|
-
*
|
|
2209
|
-
* timeout-specific failure. When no timeout is configured the
|
|
2210
|
-
* and behavior is unchanged.
|
|
2353
|
+
* its full-app boot and database connections — indefinitely. The armed timers are
|
|
2354
|
+
* recorded on the shared state the outcome/exit/error handlers use to cancel them
|
|
2355
|
+
* and to report a timeout-specific failure. When no timeout is configured the
|
|
2356
|
+
* timer stays null and behavior is unchanged.
|
|
2211
2357
|
* @param {object} args - Options.
|
|
2212
2358
|
* @param {import("node:child_process").ChildProcess} args.child - Forked child process.
|
|
2213
2359
|
* @param {import("./types.js").BackgroundJobPayload & {id: string}} args.payload - Job payload.
|
|
2214
|
-
* @
|
|
2360
|
+
* @param {ForkedJobState} args.state - Shared bookkeeping for this runner.
|
|
2361
|
+
* @returns {void}
|
|
2215
2362
|
*/
|
|
2216
|
-
_armForkedJobTimeout({child, payload}) {
|
|
2363
|
+
_armForkedJobTimeout({child, payload, state}) {
|
|
2217
2364
|
const timeoutMs = this._resolveJobTimeoutMs(payload.options)
|
|
2218
|
-
/** @type {ForkedJobTimeoutState} */
|
|
2219
|
-
const state = {timedOut: false, timeoutMs, timer: null, sigkillTimer: null}
|
|
2220
2365
|
|
|
2221
|
-
|
|
2366
|
+
state.timeoutMs = timeoutMs
|
|
2222
2367
|
|
|
2223
|
-
|
|
2368
|
+
if (!(typeof timeoutMs === "number" && timeoutMs > 0)) return
|
|
2224
2369
|
|
|
2225
|
-
|
|
2370
|
+
state.timer = setTimeout(() => this._onForkedJobTimeout({child, state}), timeoutMs)
|
|
2226
2371
|
}
|
|
2227
2372
|
|
|
2228
2373
|
/**
|
|
@@ -2251,11 +2396,13 @@ export default class BackgroundJobsWorker {
|
|
|
2251
2396
|
/**
|
|
2252
2397
|
* Fired when a forked runner overruns its timeout. Sends SIGTERM for a clean
|
|
2253
2398
|
* shutdown, then SIGKILL after the grace for a runner that ignores it. The
|
|
2254
|
-
* resulting non-clean exit flows through `_handleForkedChildExit`, which
|
|
2255
|
-
* the
|
|
2399
|
+
* resulting non-clean exit flows through `_handleForkedChildExit`, which reports
|
|
2400
|
+
* the job failed unless the runner had already recorded its outcome — the
|
|
2401
|
+
* backstop still reaps a runner that reported and then lingered, without turning
|
|
2402
|
+
* that reap into a second terminal result for a settled job.
|
|
2256
2403
|
* @param {object} args - Options.
|
|
2257
2404
|
* @param {import("node:child_process").ChildProcess} args.child - Forked child process.
|
|
2258
|
-
* @param {
|
|
2405
|
+
* @param {ForkedJobState} args.state - Shared bookkeeping for this runner.
|
|
2259
2406
|
* @returns {void}
|
|
2260
2407
|
*/
|
|
2261
2408
|
_onForkedJobTimeout({child, state}) {
|
|
@@ -2279,7 +2426,7 @@ export default class BackgroundJobsWorker {
|
|
|
2279
2426
|
/**
|
|
2280
2427
|
* Cancels any pending timeout/SIGKILL timers for a forked runner that has
|
|
2281
2428
|
* exited (or errored) so they never fire against a gone or reused child.
|
|
2282
|
-
* @param {
|
|
2429
|
+
* @param {ForkedJobState} state - Shared bookkeeping for this runner.
|
|
2283
2430
|
* @returns {void}
|
|
2284
2431
|
*/
|
|
2285
2432
|
_clearForkedJobTimeout(state) {
|
|
@@ -2301,22 +2448,25 @@ export default class BackgroundJobsWorker {
|
|
|
2301
2448
|
* @param {number | null} args.code - Exit code.
|
|
2302
2449
|
* @param {keyof typeof import("node:os").constants.signals | null} args.signal - Exit signal.
|
|
2303
2450
|
* @param {import("./types.js").BackgroundJobPayload & {id: string}} args.payload - Payload.
|
|
2304
|
-
* @param {
|
|
2305
|
-
* @param {ForkedJobTimeoutState} [args.timeoutState] - Timeout state, when the runner had a wall-clock backstop.
|
|
2451
|
+
* @param {ForkedJobState} args.state - Shared bookkeeping for this runner.
|
|
2306
2452
|
* @returns {void}
|
|
2307
2453
|
*/
|
|
2308
|
-
_handleForkedChildExit({child, code, signal, payload,
|
|
2454
|
+
_handleForkedChildExit({child, code, signal, payload, state}) {
|
|
2309
2455
|
this.inflightProcessChildren.delete(child)
|
|
2310
2456
|
|
|
2311
2457
|
// Free the worker slot as soon as the child is gone — never gate it on the
|
|
2312
2458
|
// failure report. A hung/slow report must not leak the slot; enough leaked
|
|
2313
2459
|
// slots drive `acceptsForked` to false and silently wedge the worker.
|
|
2314
|
-
|
|
2460
|
+
this._settleForkedJob({child, state})
|
|
2315
2461
|
|
|
2316
2462
|
if (this._forkedChildExitedCleanly({code, signal})) return
|
|
2463
|
+
// The runner has already reported this job's outcome, so the outcome handler
|
|
2464
|
+
// owned the terminal result (or reclaimed it). A later non-clean exit is
|
|
2465
|
+
// teardown noise, not a second terminal result for the same job.
|
|
2466
|
+
if (state.outcomeReported) return
|
|
2317
2467
|
|
|
2318
|
-
const error =
|
|
2319
|
-
? new Error(`Forked background job runner timed out after ${
|
|
2468
|
+
const error = state.timedOut
|
|
2469
|
+
? new Error(`Forked background job runner timed out after ${state.timeoutMs}ms and was terminated: code=${code} signal=${signal || "none"}`)
|
|
2320
2470
|
: new Error(`Forked background job runner exited before reporting: code=${code} signal=${signal || "none"}`)
|
|
2321
2471
|
|
|
2322
2472
|
this._reportForkedChildFailure({payload, error})
|
|
@@ -2339,14 +2489,17 @@ export default class BackgroundJobsWorker {
|
|
|
2339
2489
|
* @param {import("node:child_process").ChildProcess} args.child - Forked child process.
|
|
2340
2490
|
* @param {Error} args.error - Child process error.
|
|
2341
2491
|
* @param {import("./types.js").BackgroundJobPayload & {id: string}} args.payload - Payload.
|
|
2342
|
-
* @param {
|
|
2492
|
+
* @param {ForkedJobState} args.state - Shared bookkeeping for this runner.
|
|
2343
2493
|
* @returns {void}
|
|
2344
2494
|
*/
|
|
2345
|
-
_handleForkedChildError({child, error, payload,
|
|
2495
|
+
_handleForkedChildError({child, error, payload, state}) {
|
|
2346
2496
|
this.inflightProcessChildren.delete(child)
|
|
2347
2497
|
// Free the slot first (see _handleForkedChildExit) — reporting is best-effort.
|
|
2348
|
-
|
|
2498
|
+
this._settleForkedJob({child, state})
|
|
2349
2499
|
console.error("Background jobs forked runner error:", error)
|
|
2500
|
+
|
|
2501
|
+
if (state.outcomeReported) return
|
|
2502
|
+
|
|
2350
2503
|
this._reportForkedChildFailure({payload, error})
|
|
2351
2504
|
}
|
|
2352
2505
|
|
|
@@ -2386,7 +2539,7 @@ export default class BackgroundJobsWorker {
|
|
|
2386
2539
|
|
|
2387
2540
|
/**
|
|
2388
2541
|
* Runs spawn job.
|
|
2389
|
-
* @param {import("./types.js").BackgroundJobPayload} payload - Payload.
|
|
2542
|
+
* @param {import("./types.js").BackgroundJobPayload & {id: string}} payload - Payload.
|
|
2390
2543
|
* @returns {Promise<void>} - Resolves when the spawned runner exits or spawn fails.
|
|
2391
2544
|
*/
|
|
2392
2545
|
_spawnJob(payload) {
|
|
@@ -2405,14 +2558,17 @@ export default class BackgroundJobsWorker {
|
|
|
2405
2558
|
})
|
|
2406
2559
|
|
|
2407
2560
|
this.inflightProcessChildren.add(child)
|
|
2561
|
+
this.processChildJobs.set(child, {jobId: payload.id, startedAtMs: Date.now()})
|
|
2408
2562
|
|
|
2409
2563
|
const finished = new Promise((resolve) => {
|
|
2410
2564
|
child.once("exit", () => {
|
|
2411
2565
|
this.inflightProcessChildren.delete(child)
|
|
2566
|
+
this.processChildJobs.delete(child)
|
|
2412
2567
|
resolve(undefined)
|
|
2413
2568
|
})
|
|
2414
2569
|
child.once("error", (error) => {
|
|
2415
2570
|
this.inflightProcessChildren.delete(child)
|
|
2571
|
+
this.processChildJobs.delete(child)
|
|
2416
2572
|
console.error("Background jobs spawned runner error:", error)
|
|
2417
2573
|
resolve(undefined)
|
|
2418
2574
|
})
|
|
@@ -34,6 +34,33 @@ import parseInnodbDeadlockSummary from "./deadlock-diagnostic-parser.js"
|
|
|
34
34
|
*/
|
|
35
35
|
const MYSQL_INDEFINITE_LOCK_TIMEOUT_SECONDS = 60 * 60 * 24 * 365
|
|
36
36
|
const INNODB_DEADLOCK_CAPTURE_TIMEOUT_MS = 250
|
|
37
|
+
const INFORMATION_SCHEMA_STRING_LITERAL_PATTERN = /^[A-Za-z0-9 _.,:/@+-]*$/
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* `information_schema.COLUMNS` reports a string column default as its SQL literal
|
|
41
|
+
* (quoted, e.g. `'normal'`), while `SHOW FULL COLUMNS` reports the bare value
|
|
42
|
+
* (`normal`). Column consumers key on the `SHOW FULL COLUMNS` shape: the schema
|
|
43
|
+
* cloner's simple-default pattern rejects the quoted form, so a batch-read string
|
|
44
|
+
* default would be dropped from the cloned column and an insert that omits it then
|
|
45
|
+
* fails with ER_NO_DEFAULT_FOR_FIELD on the tenant copy. Normalize the batch read
|
|
46
|
+
* to the same shape the per-table read produces.
|
|
47
|
+
* @param {unknown} defaultValue - Raw `COLUMN_DEFAULT` value from the batch read.
|
|
48
|
+
* @returns {unknown} - The default in the `SHOW FULL COLUMNS` shape.
|
|
49
|
+
*/
|
|
50
|
+
function normalizeInformationSchemaDefault(defaultValue) {
|
|
51
|
+
if (typeof defaultValue !== "string") return defaultValue
|
|
52
|
+
|
|
53
|
+
if (defaultValue.length >= 2 && defaultValue.startsWith("'") && defaultValue.endsWith("'")) {
|
|
54
|
+
const innerValue = defaultValue.slice(1, -1)
|
|
55
|
+
|
|
56
|
+
// Only plain string literals are unquoted; expression defaults (e.g.
|
|
57
|
+
// DEFAULT '''UUID()''') keep their stored form, which SHOW FULL COLUMNS
|
|
58
|
+
// also reports verbatim.
|
|
59
|
+
if (INFORMATION_SCHEMA_STRING_LITERAL_PATTERN.test(innerValue)) return innerValue
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
return defaultValue
|
|
63
|
+
}
|
|
37
64
|
|
|
38
65
|
export default class VelociousDatabaseDriversMysql extends Base{
|
|
39
66
|
/** @type {import("mysql").Pool | undefined} */
|
|
@@ -807,6 +834,8 @@ export default class VelociousDatabaseDriversMysql extends Base{
|
|
|
807
834
|
|
|
808
835
|
if (!table) continue
|
|
809
836
|
|
|
837
|
+
row.Default = normalizeInformationSchemaDefault(row.Default)
|
|
838
|
+
|
|
810
839
|
const columns = columnsByTable.get(tableName) || []
|
|
811
840
|
|
|
812
841
|
columns.push(new Column(table, /** @type {Record<string, ReturnType<typeof JSON.parse>>} */ (row)))
|