velocious 1.0.723 → 1.0.725

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,12 +1,14 @@
1
1
  // @ts-check
2
2
 
3
3
  import {createHash, randomUUID} from "crypto"
4
+ import os from "node:os"
4
5
  import BackgroundJobsAdapter from "./adapter.js"
5
6
  import Logger from "../logger.js"
6
7
  import TableData from "../database/table-data/index.js"
7
8
  import VelociousError from "../velocious-error.js"
8
9
  import BackgroundJobRecord from "./job-record.js"
9
10
  import normalizeBackgroundJobError from "./normalize-error.js"
11
+ import {evaluateBackgroundJobOwnerLiveness, isProcessAlive} from "./owner-liveness.js"
10
12
  import { coordinateSharedTransactionConnection } from "../testing/shared-transaction-connection-coordinator.js"
11
13
  import stableJsonStringify from "../utils/stable-json.js"
12
14
  import {
@@ -51,6 +53,7 @@ import {
51
53
  * BackgroundJobOrphanSelection type.
52
54
  * @typedef {object} BackgroundJobOrphanSelection
53
55
  * @property {Record<string, ReturnType<typeof JSON.parse>>} conditions - Exact update fence.
56
+ * @property {ReturnType<typeof JSON.parse>} error - Orphan reason for this selection.
54
57
  * @property {import("./types.js").BackgroundJobRow} job - Selected active handoff.
55
58
  */
56
59
 
@@ -1679,11 +1682,12 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
1679
1682
  status: "handed_off",
1680
1683
  worker_id: handoff.workerId
1681
1684
  },
1685
+ error,
1682
1686
  job
1683
1687
  })
1684
1688
  }
1685
1689
 
1686
- return await this._markOrphanSelections({db, error, selections})
1690
+ return await this._markOrphanSelections({db, selections})
1687
1691
  })
1688
1692
  }
1689
1693
 
@@ -1714,29 +1718,65 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
1714
1718
  }
1715
1719
 
1716
1720
  /**
1717
- * Runs mark orphaned jobs.
1721
+ * Reaps `handed_off` jobs whose executing child is gone. A live child is
1722
+ * never reaped: a job that is still running (or whose child is still
1723
+ * starting) keeps its lease and its concurrency slot. A child recorded on
1724
+ * this host is reaped only when the handoff age exceeds the job's effective
1725
+ * timeout AND the pid probe proves the child is gone — never on age alone,
1726
+ * because a pid that still exists may be the same child mid-startup. Rows
1727
+ * whose liveness cannot be answered (remote runner host, or no recorded pid)
1728
+ * stay under the bounded age sweep, which is the recovery path for crashes
1729
+ * that cannot report (container restart, OOM kill).
1718
1730
  * @param {object} [args] - Options.
1719
- * @param {number} [args.orphanedAfterMs] - Mark jobs orphaned after this duration.
1731
+ * @param {number} [args.orphanedAfterMs] - Mark jobs orphaned after this duration when liveness is unknown.
1732
+ * @param {(pid: number) => boolean} [args.livenessProbe] - Process probe for local children, injectable for tests.
1733
+ * @param {string} [args.currentHostname] - Hostname of this process, defaults to `os.hostname()`.
1720
1734
  * @returns {Promise<import("./types.js").BackgroundJobRow[]>} - The jobs this sweep marked orphaned.
1721
1735
  */
1722
- async markOrphanedJobs({orphanedAfterMs = ORPHANED_AFTER_MS} = {}) {
1736
+ async markOrphanedJobs({orphanedAfterMs = ORPHANED_AFTER_MS, livenessProbe = isProcessAlive, currentHostname = os.hostname()} = {}) {
1723
1737
  await this.ensureReady()
1724
1738
 
1725
1739
  return await this._serializedCountMutation(async (db) => {
1726
- const cutoff = this.clock.now() - orphanedAfterMs
1727
- const query = db
1740
+ const now = this.clock.now()
1741
+ const rows = await db
1728
1742
  .newQuery()
1729
1743
  .from(JOBS_TABLE)
1730
1744
  .where({status: "handed_off"})
1731
- .where(`handed_off_at_ms <= ${db.quote(cutoff)}`)
1732
-
1733
- const rows = await query.results()
1745
+ .results()
1734
1746
 
1735
1747
  /** @type {BackgroundJobOrphanSelection[]} */
1736
1748
  const selections = []
1737
1749
 
1738
1750
  for (const row of rows) {
1739
1751
  const job = this._normalizeJobRow(row)
1752
+ // A handed-off row without a handoff timestamp cannot be aged; it stays
1753
+ // for startup reconciliation rather than being reaped blindly.
1754
+ if (job.handedOffAtMs === null) continue
1755
+ const ageMs = now - job.handedOffAtMs
1756
+ const liveness = evaluateBackgroundJobOwnerLiveness({
1757
+ childInstanceId: job.childInstanceId,
1758
+ childPid: job.childPid,
1759
+ currentHostname,
1760
+ probe: livenessProbe,
1761
+ runnerHostname: job.runnerHostname
1762
+ })
1763
+
1764
+ if (liveness.status === "alive") continue
1765
+
1766
+ /** @type {ReturnType<typeof JSON.parse>} */
1767
+ let error
1768
+
1769
+ if (liveness.status === "dead") {
1770
+ const effectiveTimeoutMs = job.timeoutMs ?? this.configuration.getBackgroundJobsConfig().jobTimeoutMs ?? orphanedAfterMs
1771
+
1772
+ if (ageMs <= effectiveTimeoutMs) continue
1773
+ error = `Job orphaned: handoff child ${liveness.reason} after ${ageMs}ms (timeout ${effectiveTimeoutMs}ms)`
1774
+ } else {
1775
+ // Matches the original sweep boundary: `handed_off_at_ms <= now - orphanedAfterMs`,
1776
+ // i.e. reap when the age reaches the configured window.
1777
+ if (ageMs < orphanedAfterMs) continue
1778
+ error = "Job orphaned after timeout"
1779
+ }
1740
1780
 
1741
1781
  // Fence the reclaim on the exact handoff this sweep selected, using its
1742
1782
  // `handed_off_at_ms` rather than its `handoff_id`. Two reasons:
@@ -1749,19 +1789,16 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
1749
1789
  // `handed_off_at_ms` (always "now"), so this stale cutoff-era
1750
1790
  // timestamp no longer matches and we won't fail/orphan — or
1751
1791
  // wrongly release the concurrency reservation of — that new lease.
1752
- // `handed_off_at_ms` is always set on a handed-off row (and the SELECT
1753
- // required it `<= cutoff`), so it is a reliable null-safe lease pin.
1792
+ // `handed_off_at_ms` is always set on a handed-off row, so it is a
1793
+ // reliable null-safe lease pin.
1754
1794
  selections.push({
1755
1795
  conditions: {id: job.id, status: "handed_off", handed_off_at_ms: job.handedOffAtMs},
1796
+ error,
1756
1797
  job
1757
1798
  })
1758
1799
  }
1759
1800
 
1760
- return await this._markOrphanSelections({
1761
- db,
1762
- error: "Job orphaned after timeout",
1763
- selections
1764
- })
1801
+ return await this._markOrphanSelections({db, selections})
1765
1802
  })
1766
1803
  }
1767
1804
 
@@ -1770,15 +1807,14 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
1770
1807
  * delta for the accepted rows.
1771
1808
  * @param {object} args - Options.
1772
1809
  * @param {import("../database/drivers/base.js").default} args.db - Transaction connection.
1773
- * @param {ReturnType<typeof JSON.parse>} args.error - Orphan reason.
1774
- * @param {BackgroundJobOrphanSelection[]} args.selections - Selected handoffs and exact fences.
1810
+ * @param {BackgroundJobOrphanSelection[]} args.selections - Selected handoffs, per-row orphan reasons and exact fences.
1775
1811
  * @returns {Promise<import("./types.js").BackgroundJobRow[]>} - Accepted transitions.
1776
1812
  */
1777
- async _markOrphanSelections({db, error, selections}) {
1813
+ async _markOrphanSelections({db, selections}) {
1778
1814
  /** @type {import("./types.js").BackgroundJobRow[]} */
1779
1815
  const orphanedJobs = []
1780
1816
 
1781
- for (const {conditions, job} of selections) {
1817
+ for (const {conditions, error, job} of selections) {
1782
1818
  const orphanedJob = await this._applyFailure({
1783
1819
  conditions,
1784
1820
  db,
@@ -17,12 +17,24 @@ import BackgroundJobsGenerationHandshakeTimeoutError, { DEFAULT_GENERATION_HANDS
17
17
  import { POOLED_RUNNER_INFLIGHT_JOB_ID_LIMIT, boundedPooledRunnerInflightJobIds, isPooledChildShutdownReason, isPooledChildShutdownSignal } from "./pooled-runner-shutdown.js"
18
18
 
19
19
  /**
20
- * Per-forked-child timeout bookkeeping.
21
- * @typedef {object} ForkedJobTimeoutState
20
+ * Per-forked-child bookkeeping, shared by the wall-clock backstop, the runner's
21
+ * own job-outcome report, and the exit/error handlers.
22
+ * @typedef {object} ForkedJobState
22
23
  * @property {boolean} timedOut - Whether the timeout fired and the child was terminated.
23
24
  * @property {number | null} timeoutMs - The armed timeout in ms, or null when disabled.
24
25
  * @property {ReturnType<typeof setTimeout> | null} timer - The pending timeout timer, cleared on exit.
25
26
  * @property {ReturnType<typeof setTimeout> | null} sigkillTimer - The pending SIGKILL grace timer, cleared on exit.
27
+ * @property {boolean} outcomeReported - Whether the runner reported its job outcome over IPC.
28
+ * @property {boolean} outcomeAcknowledged - Whether that report confirmed the outcome was recorded.
29
+ * @property {boolean} settled - Whether the worker's in-flight entry for this job has been freed.
30
+ * @property {(value: void) => void} resolve - Resolver that frees the in-flight entry; a placeholder until the promise that owns the entry is created.
31
+ */
32
+ /**
33
+ * One live runner child, kept so a shutdown that runs out of drain time can name
34
+ * the exact process it is still waiting on instead of a bare count.
35
+ * @typedef {object} ProcessJobHolder
36
+ * @property {string} jobId - Job the child was handed.
37
+ * @property {number} startedAtMs - When the worker handed it that job.
26
38
  */
27
39
  /**
28
40
  * @typedef {object} PooledJobEntry
@@ -123,6 +135,24 @@ function isChildAcceptanceMessage(message) {
123
135
  && (record.runnerHostname === undefined || typeof record.runnerHostname === "string")
124
136
  }
125
137
 
138
+ /**
139
+ * Checks whether an IPC value is a forked runner's job-outcome report. The forked
140
+ * child sends exactly one of these, once its own terminal report has settled, and
141
+ * says whether that report was acknowledged — so the worker can tell a runner that
142
+ * has finished its job from one that is still running it.
143
+ * @param {ReturnType<typeof JSON.parse>} message - IPC message.
144
+ * @returns {message is {acknowledged: boolean, error?: string, jobId?: string, type: "job-reported"}} - Whether this is a job-outcome report.
145
+ */
146
+ function isForkedChildJobReport(message) {
147
+ if (!message || typeof message !== "object") return false
148
+ const record = /** @type {{acknowledged?: ReturnType<typeof JSON.parse>, error?: ReturnType<typeof JSON.parse>, jobId?: ReturnType<typeof JSON.parse>, type?: ReturnType<typeof JSON.parse>}} */ (message)
149
+
150
+ return record.type === "job-reported"
151
+ && typeof record.acknowledged === "boolean"
152
+ && (record.jobId === undefined || typeof record.jobId === "string")
153
+ && (record.error === undefined || typeof record.error === "string")
154
+ }
155
+
126
156
  /**
127
157
  * Checks whether an IPC value is the child's bounded pre-teardown observation.
128
158
  * @param {ReturnType<typeof JSON.parse>} message - IPC message.
@@ -366,6 +396,13 @@ export default class BackgroundJobsWorker {
366
396
  * @type {Set<import("node:child_process").ChildProcess>}
367
397
  */
368
398
  this.inflightProcessChildren = new Set()
399
+ /**
400
+ * Which job each live runner child was handed. A shutdown that runs out of
401
+ * drain time names the exact process it is still waiting on (pid, job, how
402
+ * long it has been held) from this map instead of reporting only a count.
403
+ * @type {Map<import("node:child_process").ChildProcess, ProcessJobHolder>}
404
+ */
405
+ this.processChildJobs = new Map()
369
406
  /** @type {Set<Promise<void>>} */
370
407
  this.inflightPooledJobs = new Set()
371
408
  /** @type {Map<string, Array<import("./types.js").BackgroundJobPayload & {id: string}>>} */
@@ -551,9 +588,9 @@ export default class BackgroundJobsWorker {
551
588
  }
552
589
  }
553
590
 
554
- await this._drainInflight(this.inflightInlineJobs, timeoutMs)
555
- await this._drainInflight(this.inflightPooledJobs, timeoutMs)
556
- await this._drainInflight(this.inflightProcessJobs, timeoutMs)
591
+ await this._drainInflight(this.inflightInlineJobs, timeoutMs, {phase: "inline-jobs"})
592
+ await this._drainInflight(this.inflightPooledJobs, timeoutMs, {phase: "pooled-jobs"})
593
+ await this._drainInflight(this.inflightProcessJobs, timeoutMs, {phase: "process-jobs", describeHolders: () => this._describeProcessJobHolders()})
557
594
  await this._terminateProcessChildren()
558
595
  // Give in-flight result reports (now decoupled from job slots) a bounded
559
596
  // chance to land before the socket closes.
@@ -645,12 +682,17 @@ export default class BackgroundJobsWorker {
645
682
 
646
683
  /**
647
684
  * Waits for a set of in-flight job promises to settle, optionally bounded by
648
- * `timeoutMs`.
685
+ * `timeoutMs`. When a bounded window runs out with entries still held, the drain
686
+ * names what it is waiting on instead of returning silently: an unexplained wait
687
+ * is what turned one wedged worker into a 28-minute investigation.
649
688
  * @param {Set<Promise<void>>} inflight - In-flight job promises.
650
689
  * @param {number} [timeoutMs] - Max wait in ms; unbounded when omitted.
690
+ * @param {object} [options] - Drain options.
691
+ * @param {string} [options.phase] - Drain phase named in a timeout report.
692
+ * @param {(() => ProcessJobHolder[])} [options.describeHolders] - Names the live processes this phase is waiting on.
651
693
  * @returns {Promise<void>} - Resolves when settled or the timeout elapses.
652
694
  */
653
- async _drainInflight(inflight, timeoutMs) {
695
+ async _drainInflight(inflight, timeoutMs, {phase = "inflight", describeHolders} = {}) {
654
696
  if (inflight.size === 0) return
655
697
 
656
698
  const drain = Promise.allSettled([...inflight])
@@ -661,11 +703,51 @@ export default class BackgroundJobsWorker {
661
703
 
662
704
  await Promise.race([drain, timeout])
663
705
  clearTimeout(timer)
706
+
707
+ if (inflight.size > 0) {
708
+ this._logDrainTimeout({phase, timeoutMs, pendingCount: inflight.size, holders: describeHolders ? describeHolders() : []})
709
+ }
664
710
  } else {
665
711
  await drain
666
712
  }
667
713
  }
668
714
 
715
+ /**
716
+ * Names every live runner child the worker still holds a job for, so a drain
717
+ * that ran out of time says which process it is waiting on, for which job, and
718
+ * for how long.
719
+ * @returns {ProcessJobHolder[]} - Held job descriptors, oldest first.
720
+ */
721
+ _describeProcessJobHolders() {
722
+ return [...this.processChildJobs.values()].sort((left, right) => left.startedAtMs - right.startedAtMs)
723
+ }
724
+
725
+ /**
726
+ * Logs one compact drain-timeout event to the worker's own stderr (the surface
727
+ * that reaches the production log), naming the held processes and how long each
728
+ * has been in flight.
729
+ * @param {object} args - Event fields.
730
+ * @param {string} args.phase - Drain phase that ran out of time.
731
+ * @param {number} args.timeoutMs - The bound that expired.
732
+ * @param {number} args.pendingCount - Entries still unsettled when it expired.
733
+ * @param {ProcessJobHolder[]} args.holders - Held processes named by the phase.
734
+ * @returns {void}
735
+ */
736
+ _logDrainTimeout({phase, timeoutMs, pendingCount, holders}) {
737
+ const nowMs = Date.now()
738
+
739
+ console.error(JSON.stringify({
740
+ event: "worker-shutdown-drain-timeout",
741
+ phase,
742
+ timeoutMs,
743
+ pendingCount,
744
+ held: holders.map((holder) => ({
745
+ jobId: holder.jobId,
746
+ heldMs: nowMs - holder.startedAtMs
747
+ }))
748
+ }))
749
+ }
750
+
669
751
  /**
670
752
  * Terminates any process runner children still alive after the drain window so
671
753
  * they don't outlive the worker as orphans. SIGTERM lets the runner close its
@@ -2147,12 +2229,13 @@ export default class BackgroundJobsWorker {
2147
2229
  /**
2148
2230
  * Runs fork job.
2149
2231
  * @param {import("./types.js").BackgroundJobPayload & {id: string}} payload - Payload.
2150
- * @returns {Promise<void>} - Resolves when the forked runner exits or fork fails.
2232
+ * @returns {Promise<void>} - Resolves when the runner reports its job outcome or the fork fails.
2151
2233
  */
2152
2234
  _forkJob(payload) {
2153
2235
  const child = this._createForkedChild()
2154
2236
 
2155
2237
  this.inflightProcessChildren.add(child)
2238
+ this.processChildJobs.set(child, {jobId: payload.id, startedAtMs: Date.now()})
2156
2239
 
2157
2240
  const finished = this._waitForForkedChild({child, payload})
2158
2241
 
@@ -2183,46 +2266,108 @@ export default class BackgroundJobsWorker {
2183
2266
  * @param {object} args - Options.
2184
2267
  * @param {import("node:child_process").ChildProcess} args.child - Forked child process.
2185
2268
  * @param {import("./types.js").BackgroundJobPayload & {id: string}} args.payload - Payload.
2186
- * @returns {Promise<void>} - Resolves when the child exits.
2269
+ * @returns {Promise<void>} - Resolves when the runner reports its job outcome or the child exits.
2187
2270
  */
2188
2271
  _waitForForkedChild({child, payload}) {
2189
- const timeoutState = this._armForkedJobTimeout({child, payload})
2272
+ /** @type {ForkedJobState} */
2273
+ const state = {
2274
+ timedOut: false,
2275
+ timeoutMs: null,
2276
+ timer: null,
2277
+ sigkillTimer: null,
2278
+ outcomeReported: false,
2279
+ outcomeAcknowledged: false,
2280
+ settled: false,
2281
+ resolve: () => {}
2282
+ }
2190
2283
 
2191
2284
  return new Promise((resolve) => {
2285
+ state.resolve = resolve
2286
+ this._armForkedJobTimeout({child, payload, state})
2287
+
2288
+ child.once("message", (message) => {
2289
+ this._handleForkedChildJobReport({child, message, payload, state})
2290
+ })
2192
2291
  child.once("exit", (code, signal) => {
2193
- this._clearForkedJobTimeout(timeoutState)
2194
- this._handleForkedChildExit({child, code, signal, payload, resolve, timeoutState})
2292
+ this._clearForkedJobTimeout(state)
2293
+ this._handleForkedChildExit({child, code, signal, payload, state})
2195
2294
  })
2196
2295
  child.once("error", (error) => {
2197
- this._clearForkedJobTimeout(timeoutState)
2198
- this._handleForkedChildError({child, error, payload, resolve})
2296
+ this._clearForkedJobTimeout(state)
2297
+ this._handleForkedChildError({child, error, payload, state})
2199
2298
  })
2200
2299
  })
2201
2300
  }
2202
2301
 
2302
+ /**
2303
+ * Handles one forked runner's job-outcome report. The runner has finished the job
2304
+ * it was handed, so the worker frees the slot here instead of waiting for the
2305
+ * process to exit: a runner that holds no job has no reason to exit, and a
2306
+ * shutdown drain that waits for one never finishes. The wall-clock backstop stays
2307
+ * armed, so a runner that reports and then lingers is still terminated.
2308
+ * @param {object} args - Options.
2309
+ * @param {import("node:child_process").ChildProcess} args.child - Forked child process.
2310
+ * @param {ReturnType<typeof JSON.parse>} args.message - IPC message.
2311
+ * @param {import("./types.js").BackgroundJobPayload & {id: string}} args.payload - Job payload.
2312
+ * @param {ForkedJobState} args.state - Shared bookkeeping for this runner.
2313
+ * @returns {void}
2314
+ */
2315
+ _handleForkedChildJobReport({child, message, payload, state}) {
2316
+ if (!isForkedChildJobReport(message)) return
2317
+
2318
+ state.outcomeReported = true
2319
+ state.outcomeAcknowledged = message.acknowledged === true
2320
+
2321
+ if (!state.outcomeAcknowledged) {
2322
+ // The runner stayed alive but could not confirm its own terminal report, so
2323
+ // reclaim the job exactly as the pooled lane does before freeing the slot.
2324
+ this._reportForkedChildFailure({
2325
+ payload,
2326
+ error: new Error(message.error || "Forked background job runner terminal report was not acknowledged")
2327
+ })
2328
+ }
2329
+
2330
+ this._settleForkedJob({child, state})
2331
+ }
2332
+
2333
+ /**
2334
+ * Frees the worker's in-flight entry for one forked job exactly once, so neither
2335
+ * a job-outcome report nor a later exit can settle it twice.
2336
+ * @param {object} args - Options.
2337
+ * @param {import("node:child_process").ChildProcess} args.child - Forked child process.
2338
+ * @param {ForkedJobState} args.state - Shared bookkeeping for this runner.
2339
+ * @returns {void}
2340
+ */
2341
+ _settleForkedJob({child, state}) {
2342
+ if (state.settled) return
2343
+
2344
+ state.settled = true
2345
+ this.processChildJobs.delete(child)
2346
+ state.resolve(undefined)
2347
+ }
2348
+
2203
2349
  /**
2204
2350
  * Arms a wall-clock backstop for a forked job runner. A forked job still
2205
2351
  * running after `jobTimeoutMs` is terminated (SIGTERM, then SIGKILL after the
2206
2352
  * grace) so a single genuinely-hung runner can't pin a draining worker — and
2207
- * its full-app boot and database connections — indefinitely. Returns a state
2208
- * object the exit/error handlers use to cancel the timer and to report a
2209
- * timeout-specific failure. When no timeout is configured the timer is null
2210
- * and behavior is unchanged.
2353
+ * its full-app boot and database connections — indefinitely. The armed timers are
2354
+ * recorded on the shared state the outcome/exit/error handlers use to cancel them
2355
+ * and to report a timeout-specific failure. When no timeout is configured the
2356
+ * timer stays null and behavior is unchanged.
2211
2357
  * @param {object} args - Options.
2212
2358
  * @param {import("node:child_process").ChildProcess} args.child - Forked child process.
2213
2359
  * @param {import("./types.js").BackgroundJobPayload & {id: string}} args.payload - Job payload.
2214
- * @returns {ForkedJobTimeoutState} - Timeout state.
2360
+ * @param {ForkedJobState} args.state - Shared bookkeeping for this runner.
2361
+ * @returns {void}
2215
2362
  */
2216
- _armForkedJobTimeout({child, payload}) {
2363
+ _armForkedJobTimeout({child, payload, state}) {
2217
2364
  const timeoutMs = this._resolveJobTimeoutMs(payload.options)
2218
- /** @type {ForkedJobTimeoutState} */
2219
- const state = {timedOut: false, timeoutMs, timer: null, sigkillTimer: null}
2220
2365
 
2221
- if (!(typeof timeoutMs === "number" && timeoutMs > 0)) return state
2366
+ state.timeoutMs = timeoutMs
2222
2367
 
2223
- state.timer = setTimeout(() => this._onForkedJobTimeout({child, state}), timeoutMs)
2368
+ if (!(typeof timeoutMs === "number" && timeoutMs > 0)) return
2224
2369
 
2225
- return state
2370
+ state.timer = setTimeout(() => this._onForkedJobTimeout({child, state}), timeoutMs)
2226
2371
  }
2227
2372
 
2228
2373
  /**
@@ -2251,11 +2396,13 @@ export default class BackgroundJobsWorker {
2251
2396
  /**
2252
2397
  * Fired when a forked runner overruns its timeout. Sends SIGTERM for a clean
2253
2398
  * shutdown, then SIGKILL after the grace for a runner that ignores it. The
2254
- * resulting non-clean exit flows through `_handleForkedChildExit`, which frees
2255
- * the slot and reports the job failed.
2399
+ * resulting non-clean exit flows through `_handleForkedChildExit`, which reports
2400
+ * the job failed unless the runner had already recorded its outcome — the
2401
+ * backstop still reaps a runner that reported and then lingered, without turning
2402
+ * that reap into a second terminal result for a settled job.
2256
2403
  * @param {object} args - Options.
2257
2404
  * @param {import("node:child_process").ChildProcess} args.child - Forked child process.
2258
- * @param {ForkedJobTimeoutState} args.state - Timeout state.
2405
+ * @param {ForkedJobState} args.state - Shared bookkeeping for this runner.
2259
2406
  * @returns {void}
2260
2407
  */
2261
2408
  _onForkedJobTimeout({child, state}) {
@@ -2279,7 +2426,7 @@ export default class BackgroundJobsWorker {
2279
2426
  /**
2280
2427
  * Cancels any pending timeout/SIGKILL timers for a forked runner that has
2281
2428
  * exited (or errored) so they never fire against a gone or reused child.
2282
- * @param {ForkedJobTimeoutState} state - Timeout state.
2429
+ * @param {ForkedJobState} state - Shared bookkeeping for this runner.
2283
2430
  * @returns {void}
2284
2431
  */
2285
2432
  _clearForkedJobTimeout(state) {
@@ -2301,22 +2448,25 @@ export default class BackgroundJobsWorker {
2301
2448
  * @param {number | null} args.code - Exit code.
2302
2449
  * @param {keyof typeof import("node:os").constants.signals | null} args.signal - Exit signal.
2303
2450
  * @param {import("./types.js").BackgroundJobPayload & {id: string}} args.payload - Payload.
2304
- * @param {(value: void) => void} args.resolve - Promise resolver.
2305
- * @param {ForkedJobTimeoutState} [args.timeoutState] - Timeout state, when the runner had a wall-clock backstop.
2451
+ * @param {ForkedJobState} args.state - Shared bookkeeping for this runner.
2306
2452
  * @returns {void}
2307
2453
  */
2308
- _handleForkedChildExit({child, code, signal, payload, resolve, timeoutState}) {
2454
+ _handleForkedChildExit({child, code, signal, payload, state}) {
2309
2455
  this.inflightProcessChildren.delete(child)
2310
2456
 
2311
2457
  // Free the worker slot as soon as the child is gone — never gate it on the
2312
2458
  // failure report. A hung/slow report must not leak the slot; enough leaked
2313
2459
  // slots drive `acceptsForked` to false and silently wedge the worker.
2314
- resolve(undefined)
2460
+ this._settleForkedJob({child, state})
2315
2461
 
2316
2462
  if (this._forkedChildExitedCleanly({code, signal})) return
2463
+ // The runner has already reported this job's outcome, so the outcome handler
2464
+ // owned the terminal result (or reclaimed it). A later non-clean exit is
2465
+ // teardown noise, not a second terminal result for the same job.
2466
+ if (state.outcomeReported) return
2317
2467
 
2318
- const error = timeoutState?.timedOut
2319
- ? new Error(`Forked background job runner timed out after ${timeoutState.timeoutMs}ms and was terminated: code=${code} signal=${signal || "none"}`)
2468
+ const error = state.timedOut
2469
+ ? new Error(`Forked background job runner timed out after ${state.timeoutMs}ms and was terminated: code=${code} signal=${signal || "none"}`)
2320
2470
  : new Error(`Forked background job runner exited before reporting: code=${code} signal=${signal || "none"}`)
2321
2471
 
2322
2472
  this._reportForkedChildFailure({payload, error})
@@ -2339,14 +2489,17 @@ export default class BackgroundJobsWorker {
2339
2489
  * @param {import("node:child_process").ChildProcess} args.child - Forked child process.
2340
2490
  * @param {Error} args.error - Child process error.
2341
2491
  * @param {import("./types.js").BackgroundJobPayload & {id: string}} args.payload - Payload.
2342
- * @param {(value: void) => void} args.resolve - Promise resolver.
2492
+ * @param {ForkedJobState} args.state - Shared bookkeeping for this runner.
2343
2493
  * @returns {void}
2344
2494
  */
2345
- _handleForkedChildError({child, error, payload, resolve}) {
2495
+ _handleForkedChildError({child, error, payload, state}) {
2346
2496
  this.inflightProcessChildren.delete(child)
2347
2497
  // Free the slot first (see _handleForkedChildExit) — reporting is best-effort.
2348
- resolve(undefined)
2498
+ this._settleForkedJob({child, state})
2349
2499
  console.error("Background jobs forked runner error:", error)
2500
+
2501
+ if (state.outcomeReported) return
2502
+
2350
2503
  this._reportForkedChildFailure({payload, error})
2351
2504
  }
2352
2505
 
@@ -2386,7 +2539,7 @@ export default class BackgroundJobsWorker {
2386
2539
 
2387
2540
  /**
2388
2541
  * Runs spawn job.
2389
- * @param {import("./types.js").BackgroundJobPayload} payload - Payload.
2542
+ * @param {import("./types.js").BackgroundJobPayload & {id: string}} payload - Payload.
2390
2543
  * @returns {Promise<void>} - Resolves when the spawned runner exits or spawn fails.
2391
2544
  */
2392
2545
  _spawnJob(payload) {
@@ -2405,14 +2558,17 @@ export default class BackgroundJobsWorker {
2405
2558
  })
2406
2559
 
2407
2560
  this.inflightProcessChildren.add(child)
2561
+ this.processChildJobs.set(child, {jobId: payload.id, startedAtMs: Date.now()})
2408
2562
 
2409
2563
  const finished = new Promise((resolve) => {
2410
2564
  child.once("exit", () => {
2411
2565
  this.inflightProcessChildren.delete(child)
2566
+ this.processChildJobs.delete(child)
2412
2567
  resolve(undefined)
2413
2568
  })
2414
2569
  child.once("error", (error) => {
2415
2570
  this.inflightProcessChildren.delete(child)
2571
+ this.processChildJobs.delete(child)
2416
2572
  console.error("Background jobs spawned runner error:", error)
2417
2573
  resolve(undefined)
2418
2574
  })
@@ -34,6 +34,33 @@ import parseInnodbDeadlockSummary from "./deadlock-diagnostic-parser.js"
34
34
  */
35
35
  const MYSQL_INDEFINITE_LOCK_TIMEOUT_SECONDS = 60 * 60 * 24 * 365
36
36
  const INNODB_DEADLOCK_CAPTURE_TIMEOUT_MS = 250
37
+ const INFORMATION_SCHEMA_STRING_LITERAL_PATTERN = /^[A-Za-z0-9 _.,:/@+-]*$/
38
+
39
+ /**
40
+ * `information_schema.COLUMNS` reports a string column default as its SQL literal
41
+ * (quoted, e.g. `'normal'`), while `SHOW FULL COLUMNS` reports the bare value
42
+ * (`normal`). Column consumers key on the `SHOW FULL COLUMNS` shape: the schema
43
+ * cloner's simple-default pattern rejects the quoted form, so a batch-read string
44
+ * default would be dropped from the cloned column and an insert that omits it then
45
+ * fails with ER_NO_DEFAULT_FOR_FIELD on the tenant copy. Normalize the batch read
46
+ * to the same shape the per-table read produces.
47
+ * @param {unknown} defaultValue - Raw `COLUMN_DEFAULT` value from the batch read.
48
+ * @returns {unknown} - The default in the `SHOW FULL COLUMNS` shape.
49
+ */
50
+ function normalizeInformationSchemaDefault(defaultValue) {
51
+ if (typeof defaultValue !== "string") return defaultValue
52
+
53
+ if (defaultValue.length >= 2 && defaultValue.startsWith("'") && defaultValue.endsWith("'")) {
54
+ const innerValue = defaultValue.slice(1, -1)
55
+
56
+ // Only plain string literals are unquoted; expression defaults (e.g.
57
+ // DEFAULT '''UUID()''') keep their stored form, which SHOW FULL COLUMNS
58
+ // also reports verbatim.
59
+ if (INFORMATION_SCHEMA_STRING_LITERAL_PATTERN.test(innerValue)) return innerValue
60
+ }
61
+
62
+ return defaultValue
63
+ }
37
64
 
38
65
  export default class VelociousDatabaseDriversMysql extends Base{
39
66
  /** @type {import("mysql").Pool | undefined} */
@@ -807,6 +834,8 @@ export default class VelociousDatabaseDriversMysql extends Base{
807
834
 
808
835
  if (!table) continue
809
836
 
837
+ row.Default = normalizeInformationSchemaDefault(row.Default)
838
+
810
839
  const columns = columnsByTable.get(tableName) || []
811
840
 
812
841
  columns.push(new Column(table, /** @type {Record<string, ReturnType<typeof JSON.parse>>} */ (row)))