velocious 1.0.747 → 1.0.748
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/build/background-jobs/pooled-runner-child.js +41 -4
- package/build/background-jobs/pooled-runner-shutdown.js +92 -0
- package/build/background-jobs/types.js +16 -0
- package/build/background-jobs/worker.js +56 -3
- package/build/frontend-model-controller.js +41 -0
- package/build/frontend-models/query.js +42 -0
- package/build/src/background-jobs/pooled-runner-child.js +39 -5
- package/build/src/background-jobs/pooled-runner-shutdown.d.ts +22 -0
- package/build/src/background-jobs/pooled-runner-shutdown.d.ts.map +1 -1
- package/build/src/background-jobs/pooled-runner-shutdown.js +87 -1
- package/build/src/background-jobs/types.d.ts +47 -0
- package/build/src/background-jobs/types.d.ts.map +1 -1
- package/build/src/background-jobs/types.js +17 -1
- package/build/src/background-jobs/worker.d.ts +16 -0
- package/build/src/background-jobs/worker.d.ts.map +1 -1
- package/build/src/background-jobs/worker.js +56 -4
- package/build/src/frontend-model-controller.d.ts +4 -0
- package/build/src/frontend-model-controller.d.ts.map +1 -1
- package/build/src/frontend-model-controller.js +34 -1
- package/build/src/frontend-models/query.d.ts +4 -0
- package/build/src/frontend-models/query.d.ts.map +1 -1
- package/build/src/frontend-models/query.js +38 -1
- package/build/src/sync/sync-client.d.ts.map +1 -1
- package/build/src/sync/sync-client.js +2 -1
- package/build/src/sync/sync-offline-session.d.ts.map +1 -1
- package/build/src/sync/sync-offline-session.js +9 -3
- package/build/sync/sync-client.js +1 -0
- package/build/sync/sync-offline-session.js +9 -1
- package/package.json +1 -1
- package/src/background-jobs/pooled-runner-child.js +41 -4
- package/src/background-jobs/pooled-runner-shutdown.js +92 -0
- package/src/background-jobs/types.js +16 -0
- package/src/background-jobs/worker.js +56 -3
- package/src/frontend-model-controller.js +41 -0
- package/src/frontend-models/query.js +42 -0
- package/src/sync/sync-client.js +1 -0
- package/src/sync/sync-offline-session.js +9 -1
package/README.md
CHANGED
|
@@ -2512,7 +2512,7 @@ creation across generation processes.
|
|
|
2512
2512
|
|
|
2513
2513
|
Jobs can opt into cross-worker durable concurrency limits by pairing a non-empty `concurrencyKey` with a positive-integer `maxConcurrency` in their background-job options, or by deriving the key in a hydrated job instance's non-static `concurrencyKey()` method. Explicit enqueue options win. The first cap registered for a key is stable; conflicting caps are rejected. A key's counter row is removed when its last live job settles, so keys that name a one-off unit of work do not accumulate rows forever; queue-derived lane counters are kept. See [durable concurrency limits](docs/background-jobs.md#durable-concurrency-limits).
|
|
2514
2514
|
|
|
2515
|
-
Production apps can listen for `background-job-failed` (or its `all-error` mirror) to report accepted failed attempts, including retry and terminal-state metadata. Process-level pooled-runner failures also carry one shared `context.runnerFailure` snapshot for every affected job, with active handoff identities, child instance/PID, runner/worker lifecycle, exit code/signal, an exact typed shutdown reason and request/observation timestamps, bounded in-flight job IDs,
|
|
2515
|
+
Production apps can listen for `background-job-failed` (or its `all-error` mirror) to report accepted failed attempts, including retry and terminal-state metadata. Process-level pooled-runner failures also carry one shared `context.runnerFailure` snapshot for every affected job, with active handoff identities, child instance/PID, runner/worker lifecycle, exit code/signal, an exact typed shutdown reason and request/observation timestamps, bounded in-flight job IDs, an explicit nullable OOM verdict, and the bounded `processFault` the child reported over IPC before it exited (`type` `uncaughtException`/`unhandledRejection`, `message`, `name`, `stack`, and a JSON `valuePreview` for a rejected non-`Error` value), so an unhandled rejection inside a runner names itself instead of surfacing as an unexplained runner exit. Every handed-off job also records the accepting runner's `child_instance_id`, `child_pid`, `runner_hostname` and start time; `BackgroundJobRecord#ownerIsAlive()` reports whether that recorded local process still exists (`alive`, `dead`, or `unknown` for remote/unrecorded owners), so claim ownership can be checked without polling. The deprecated `terminationReason` compatibility category remains present, while new consumers should use authoritative `shutdownReason`. Parent timeout/stop/retirement requests are recorded before signalling; normal drained retirement is permitted only with zero tracked jobs. Listen for `background-job-orphaned` to react to a specific job the main process reclaimed after its worker died mid-run — e.g. enqueue a targeted recovery for the work it left behind, instead of only polling for the aftermath. The orphan payload carries the dead owner's `childInstanceId`, `childPid`, `runnerHostname` and `workerId`, so a handler can release exactly that owner's claims on the event it already receives. Orphan handlers run before the sweep waits for reclaimed jobs to be dispatched, so a stalled dispatcher does not delay application recovery. An orphan without a delivered runner-failure report has unknown shutdown cause; do not infer OOM, deploy, timeout, signal, or normal retirement from acceptance evidence alone. See [docs/background-jobs.md](docs/background-jobs.md#failure-events) and [runner identity and owner liveness](docs/background-jobs.md#runner-identity-and-owner-liveness).
|
|
2516
2516
|
|
|
2517
2517
|
## Setup
|
|
2518
2518
|
|
|
@@ -6,7 +6,7 @@ import os from "node:os"
|
|
|
6
6
|
import v8 from "node:v8"
|
|
7
7
|
import timeout from "awaitery/build/timeout.js"
|
|
8
8
|
import runJobPayload, { BackgroundJobPerformedFailure } from "./job-runner.js"
|
|
9
|
-
import { boundedPooledRunnerInflightJobIds, isPooledChildShutdownReason, isPooledChildShutdownSignal } from "./pooled-runner-shutdown.js"
|
|
9
|
+
import { boundedPooledRunnerInflightJobIds, isPooledChildShutdownReason, isPooledChildShutdownSignal, pooledChildProcessFault } from "./pooled-runner-shutdown.js"
|
|
10
10
|
import { closeRunnerConnections, closeRunnerFrameworkConnections, currentConfigurationOrNull } from "./runner-graceful-shutdown.js"
|
|
11
11
|
import setRunnerProcessTitle from "./runner-process-title.js"
|
|
12
12
|
import PooledRunnerBrokerIdentity from "./pooled-runner-broker-identity.js"
|
|
@@ -51,6 +51,13 @@ function readPeakRssBytes() {
|
|
|
51
51
|
|
|
52
52
|
setRunnerProcessTitle()
|
|
53
53
|
|
|
54
|
+
/**
|
|
55
|
+
* Process-level fault observed by this child, carried on the shutdown observation so
|
|
56
|
+
* the worker reports the fault itself instead of only the exit it caused.
|
|
57
|
+
* @type {import("./types.js").PooledChildProcessFault | undefined}
|
|
58
|
+
*/
|
|
59
|
+
let processFault
|
|
60
|
+
|
|
54
61
|
/** @type {Promise<void> | undefined} */
|
|
55
62
|
let shutdownPromise
|
|
56
63
|
|
|
@@ -73,7 +80,16 @@ function shutdownRunner({exitCode, reason, shutdownRequestedAtMs = null, signal
|
|
|
73
80
|
|
|
74
81
|
shutdownPromise = (async () => {
|
|
75
82
|
await sendShutdownObservation({reason, shutdownRequestedAtMs, signal})
|
|
76
|
-
|
|
83
|
+
|
|
84
|
+
try {
|
|
85
|
+
await closeRunnerConnections(currentConfigurationOrNull())
|
|
86
|
+
} catch (error) {
|
|
87
|
+
// A teardown failure must not escape into the process-level handler: it would
|
|
88
|
+
// report a fault that only happened while closing as the cause of a shutdown
|
|
89
|
+
// whose reason this child already sent.
|
|
90
|
+
console.error("Pooled background job runner could not close its connections:", error)
|
|
91
|
+
}
|
|
92
|
+
|
|
77
93
|
process.exit(exitCode)
|
|
78
94
|
})()
|
|
79
95
|
|
|
@@ -147,6 +163,7 @@ async function sendShutdownObservation({reason, shutdownRequestedAtMs, signal})
|
|
|
147
163
|
/** @type {import("./types.js").PooledChildShutdownObservation & {type: "shutdown-observation"}} */
|
|
148
164
|
const message = {
|
|
149
165
|
childInstanceId,
|
|
166
|
+
fault: processFault ?? null,
|
|
150
167
|
...boundedInflight,
|
|
151
168
|
reason,
|
|
152
169
|
shutdownObservedAtMs: Date.now(),
|
|
@@ -176,6 +193,23 @@ async function sendShutdownObservation({reason, shutdownRequestedAtMs, signal})
|
|
|
176
193
|
}
|
|
177
194
|
}
|
|
178
195
|
|
|
196
|
+
/**
|
|
197
|
+
* Records one process-level fault and takes the child down through the same bounded
|
|
198
|
+
* shutdown path as a signalled or parent-requested exit, so the worker receives the
|
|
199
|
+
* fault over IPC before the process is gone. Node terminates the process on an
|
|
200
|
+
* unhandled rejection regardless, and this child's stdio is ignored by the worker
|
|
201
|
+
* fork, so the only open question is whether the worker learns why the child died:
|
|
202
|
+
* without this handler the exit is recorded as runner loss and the whole shard is
|
|
203
|
+
* retried with zero reported failures.
|
|
204
|
+
* @param {import("./types.js").PooledChildProcessFaultType} type - Process event that observed the fault.
|
|
205
|
+
* @param {ReturnType<typeof JSON.parse>} value - Thrown error or rejected value.
|
|
206
|
+
* @returns {void}
|
|
207
|
+
*/
|
|
208
|
+
function reportProcessFault(type, value) {
|
|
209
|
+
processFault = pooledChildProcessFault(type, value)
|
|
210
|
+
void shutdownRunner({exitCode: 1, reason: "process_error"})
|
|
211
|
+
}
|
|
212
|
+
|
|
179
213
|
/**
|
|
180
214
|
* Reports one acceptance observation (job received / perform started) to the
|
|
181
215
|
* worker over IPC. The message carries the job's exact handoff lease so the
|
|
@@ -359,8 +393,9 @@ function sendOutcome({jobId, acknowledged, status, error}) {
|
|
|
359
393
|
* outcome. A single job's unexpected failure reports that job for reclamation
|
|
360
394
|
* (`acknowledged: false`) but does NOT take down the child — its concurrent
|
|
361
395
|
* siblings keep running. Only a process-level fault (which escapes every
|
|
362
|
-
* per-job try/catch) ends the child,
|
|
363
|
-
*
|
|
396
|
+
* per-job try/catch) ends the child, and `reportProcessFault` names that fault
|
|
397
|
+
* on the shutdown observation so the worker reclaims the whole in-flight set
|
|
398
|
+
* with the cause instead of an unexplained exit.
|
|
364
399
|
* @param {import("./types.js").BackgroundJobPayload & {id: string}} payload - Job payload.
|
|
365
400
|
* @param {import("../testing/shared-transaction-proxy-driver.js").SharedTransactionBrokerJobConfig} sharedTransactionBroker - Per-job broker configuration.
|
|
366
401
|
* @returns {Promise<void>} - Resolves after reporting.
|
|
@@ -435,4 +470,6 @@ process.on("message", (message) => handleMessage(message))
|
|
|
435
470
|
process.once("disconnect", () => void shutdownRunner({exitCode: 0, reason: "ipc_disconnect"}))
|
|
436
471
|
process.once("SIGTERM", () => void shutdownRunner({exitCode: 1, reason: "signal_sigterm", signal: "SIGTERM"}))
|
|
437
472
|
process.once("SIGINT", () => void shutdownRunner({exitCode: 1, reason: "signal_sigint", signal: "SIGINT"}))
|
|
473
|
+
process.once("uncaughtException", (error) => reportProcessFault("uncaughtException", error))
|
|
474
|
+
process.once("unhandledRejection", (reason) => reportProcessFault("unhandledRejection", reason))
|
|
438
475
|
if (process.send) process.send({childInstanceId, childPid: process.pid, runnerHostname, type: "ready"})
|
|
@@ -4,6 +4,12 @@ import { constants } from "node:os"
|
|
|
4
4
|
|
|
5
5
|
/** Maximum job ids copied into a pooled-child shutdown observation or failure snapshot. */
|
|
6
6
|
export const POOLED_RUNNER_INFLIGHT_JOB_ID_LIMIT = 100
|
|
7
|
+
/** Maximum characters of a pooled-child fault message or error name crossing IPC. */
|
|
8
|
+
export const POOLED_CHILD_PROCESS_FAULT_MESSAGE_LIMIT = 1000
|
|
9
|
+
/** Maximum characters of a pooled-child fault stack crossing IPC. */
|
|
10
|
+
export const POOLED_CHILD_PROCESS_FAULT_STACK_LIMIT = 4000
|
|
11
|
+
/** Maximum characters of the preview of a rejected value that is not an Error. */
|
|
12
|
+
export const POOLED_CHILD_PROCESS_FAULT_VALUE_LIMIT = 500
|
|
7
13
|
|
|
8
14
|
/** @type {readonly import("./types.js").PooledChildShutdownReason[]} */
|
|
9
15
|
export const POOLED_CHILD_SHUTDOWN_REASONS = Object.freeze([
|
|
@@ -20,6 +26,12 @@ export const POOLED_CHILD_SHUTDOWN_REASONS = Object.freeze([
|
|
|
20
26
|
"unexpected_exit"
|
|
21
27
|
])
|
|
22
28
|
|
|
29
|
+
/** @type {readonly import("./types.js").PooledChildProcessFaultType[]} */
|
|
30
|
+
export const POOLED_CHILD_PROCESS_FAULT_TYPES = Object.freeze([
|
|
31
|
+
"uncaughtException",
|
|
32
|
+
"unhandledRejection"
|
|
33
|
+
])
|
|
34
|
+
|
|
23
35
|
/**
|
|
24
36
|
* Checks one dynamic IPC reason against the complete shutdown contract.
|
|
25
37
|
* @param {ReturnType<typeof JSON.parse>} value - Candidate shutdown reason.
|
|
@@ -54,3 +66,83 @@ export function boundedPooledRunnerInflightJobIds(jobIds) {
|
|
|
54
66
|
inflightJobIdsTruncatedCount: sortedJobIds.length - inflightJobIds.length
|
|
55
67
|
}
|
|
56
68
|
}
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* Bounds one dynamic fault text for the IPC/report contract: a non-text value
|
|
72
|
+
* becomes null, an over-long value is cut with a visible marker so a reader can
|
|
73
|
+
* tell a truncated record from the whole fault.
|
|
74
|
+
* @param {ReturnType<typeof JSON.parse>} value - Candidate text.
|
|
75
|
+
* @param {number} limit - Maximum characters.
|
|
76
|
+
* @returns {string | null} - Bounded text, or null when the value is not text.
|
|
77
|
+
*/
|
|
78
|
+
function boundedPooledChildFaultText(value, limit) {
|
|
79
|
+
if (typeof value !== "string") return null
|
|
80
|
+
if (value.length <= limit) return value
|
|
81
|
+
|
|
82
|
+
return `${value.slice(0, limit - 1)}…`
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* Checks one fault field against its bound.
|
|
87
|
+
* @param {ReturnType<typeof JSON.parse>} value - Candidate field.
|
|
88
|
+
* @param {number} limit - Maximum characters.
|
|
89
|
+
* @returns {boolean} - Whether the field is null or bounded text.
|
|
90
|
+
*/
|
|
91
|
+
function isBoundedPooledChildFaultText(value, limit) {
|
|
92
|
+
return value === null || (typeof value === "string" && value.length <= limit)
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* Previews a rejected value that is not an Error. `Promise.reject({})` — the shape a
|
|
97
|
+
* real incident printed — stringifies to `[object Object]` and names nothing, so a
|
|
98
|
+
* structured value is previewed as JSON instead.
|
|
99
|
+
* @param {ReturnType<typeof JSON.parse>} value - Rejected value.
|
|
100
|
+
* @returns {string | null} - Bounded preview, or null when nothing renderable was given.
|
|
101
|
+
*/
|
|
102
|
+
function previewPooledChildFaultValue(value) {
|
|
103
|
+
if (typeof value === "string" || value === null) return boundedPooledChildFaultText(value, POOLED_CHILD_PROCESS_FAULT_VALUE_LIMIT)
|
|
104
|
+
|
|
105
|
+
try {
|
|
106
|
+
const preview = JSON.stringify(value)
|
|
107
|
+
|
|
108
|
+
return boundedPooledChildFaultText(preview === undefined ? String(value) : preview, POOLED_CHILD_PROCESS_FAULT_VALUE_LIMIT)
|
|
109
|
+
} catch {
|
|
110
|
+
return boundedPooledChildFaultText(String(value), POOLED_CHILD_PROCESS_FAULT_VALUE_LIMIT)
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* Describes one pooled-child process-level fault as the bounded record that crosses
|
|
116
|
+
* IPC, so the worker reports the fault itself instead of only the exit it caused.
|
|
117
|
+
* @param {import("./types.js").PooledChildProcessFaultType} type - Process event that observed the fault.
|
|
118
|
+
* @param {ReturnType<typeof JSON.parse>} value - Thrown error or rejected value.
|
|
119
|
+
* @returns {import("./types.js").PooledChildProcessFault} - Bounded fault record.
|
|
120
|
+
*/
|
|
121
|
+
export function pooledChildProcessFault(type, value) {
|
|
122
|
+
const faultError = value instanceof Error ? value : null
|
|
123
|
+
const valuePreview = faultError ? null : previewPooledChildFaultValue(value)
|
|
124
|
+
|
|
125
|
+
return {
|
|
126
|
+
message: boundedPooledChildFaultText(faultError ? faultError.message : valuePreview, POOLED_CHILD_PROCESS_FAULT_MESSAGE_LIMIT),
|
|
127
|
+
name: faultError ? boundedPooledChildFaultText(faultError.name, POOLED_CHILD_PROCESS_FAULT_MESSAGE_LIMIT) : null,
|
|
128
|
+
stack: faultError ? boundedPooledChildFaultText(faultError.stack, POOLED_CHILD_PROCESS_FAULT_STACK_LIMIT) : null,
|
|
129
|
+
type,
|
|
130
|
+
valuePreview
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/**
|
|
135
|
+
* Checks one dynamic IPC value against the process-fault contract.
|
|
136
|
+
* @param {ReturnType<typeof JSON.parse>} value - Candidate fault record.
|
|
137
|
+
* @returns {value is import("./types.js").PooledChildProcessFault} - Whether the value is a bounded fault record.
|
|
138
|
+
*/
|
|
139
|
+
export function isPooledChildProcessFault(value) {
|
|
140
|
+
if (!value || typeof value !== "object") return false
|
|
141
|
+
const record = /** @type {{message?: ReturnType<typeof JSON.parse>, name?: ReturnType<typeof JSON.parse>, stack?: ReturnType<typeof JSON.parse>, type?: ReturnType<typeof JSON.parse>, valuePreview?: ReturnType<typeof JSON.parse>}} */ (value)
|
|
142
|
+
|
|
143
|
+
return POOLED_CHILD_PROCESS_FAULT_TYPES.some((type) => type === record.type)
|
|
144
|
+
&& isBoundedPooledChildFaultText(record.message, POOLED_CHILD_PROCESS_FAULT_MESSAGE_LIMIT)
|
|
145
|
+
&& isBoundedPooledChildFaultText(record.name, POOLED_CHILD_PROCESS_FAULT_MESSAGE_LIMIT)
|
|
146
|
+
&& isBoundedPooledChildFaultText(record.stack, POOLED_CHILD_PROCESS_FAULT_STACK_LIMIT)
|
|
147
|
+
&& isBoundedPooledChildFaultText(record.valuePreview, POOLED_CHILD_PROCESS_FAULT_VALUE_LIMIT)
|
|
148
|
+
}
|
|
@@ -12,6 +12,7 @@
|
|
|
12
12
|
/** @typedef {"exit" | "process-error" | "ipc-send" | "child-silence"} PooledRunnerFailureOrigin */
|
|
13
13
|
/** @typedef {"starting" | "running" | "retiring"} PooledRunnerLifecycleState */
|
|
14
14
|
/** @typedef {"parent_retire_drained" | "job_timeout" | "worker_stop" | "child_silence" | "signal_sigterm" | "signal_sigint" | "signal_sigkill" | "signal_other" | "ipc_disconnect" | "process_error" | "unexpected_exit"} PooledChildShutdownReason */
|
|
15
|
+
/** @typedef {"uncaughtException" | "unhandledRejection"} PooledChildProcessFaultType */
|
|
15
16
|
/**
|
|
16
17
|
* @deprecated Use PooledChildShutdownReason for exact shutdown provenance.
|
|
17
18
|
* @typedef {"unexpected" | "job-timeout" | "worker-shutdown-timeout"} PooledRunnerTerminationReason
|
|
@@ -45,6 +46,7 @@
|
|
|
45
46
|
* @property {number} inflightJobIdsTruncatedCount - In-flight ids omitted from the bounded snapshot.
|
|
46
47
|
* @property {boolean | null} oomKilled - False when the observed exit rules OOM out; null when an unexpected SIGKILL cannot be distinguished from an OOM kill without supervisor/kernel evidence.
|
|
47
48
|
* @property {PooledRunnerFailureOrigin} origin - Worker observation that initiated failure handling.
|
|
49
|
+
* @property {PooledChildProcessFault | null} processFault - Process-level fault the child reported over IPC before exiting, or null when it reported none.
|
|
48
50
|
* @property {number} runnerAgeMs - Child age when failure handling started.
|
|
49
51
|
* @property {number} runnerCreatedAtMs - Child creation timestamp.
|
|
50
52
|
* @property {boolean} runnerDetached - Whether the runner owned a detached process group.
|
|
@@ -62,10 +64,24 @@
|
|
|
62
64
|
* @property {BackgroundJobsWorkerLifecycleState} workerLifecycle - Parent worker lifecycle immediately before recovery.
|
|
63
65
|
* @property {number} workerPid - Parent worker process id.
|
|
64
66
|
*/
|
|
67
|
+
/**
|
|
68
|
+
* One process-level fault observed by a pooled runner child and transmitted to the
|
|
69
|
+
* worker before the child exits. The child's stdio is ignored by the worker fork, so
|
|
70
|
+
* without this record the only surviving evidence of an `uncaughtException` or an
|
|
71
|
+
* `unhandledRejection` is the exit code and signal, and every such exit is attributed
|
|
72
|
+
* to whatever job happened to be in flight.
|
|
73
|
+
* @typedef {object} PooledChildProcessFault
|
|
74
|
+
* @property {string | null} message - Fault message, bounded.
|
|
75
|
+
* @property {string | null} name - Error name when the fault value carried one.
|
|
76
|
+
* @property {string | null} stack - Bounded stack when the fault value was an Error.
|
|
77
|
+
* @property {PooledChildProcessFaultType} type - Process event that observed the fault.
|
|
78
|
+
* @property {string | null} valuePreview - Bounded preview of a rejected value that was not an Error.
|
|
79
|
+
*/
|
|
65
80
|
/**
|
|
66
81
|
* Best-effort observation sent before a pooled child closes its resources.
|
|
67
82
|
* @typedef {object} PooledChildShutdownObservation
|
|
68
83
|
* @property {string} childInstanceId - Stable pooled-child identity.
|
|
84
|
+
* @property {PooledChildProcessFault | null} fault - Process-level fault that caused this shutdown, or null for a requested/signalled/IPC shutdown.
|
|
69
85
|
* @property {string[]} inflightJobIds - Bounded in-flight durable job ids.
|
|
70
86
|
* @property {number} inflightJobIdsTruncatedCount - In-flight ids omitted from the bounded snapshot.
|
|
71
87
|
* @property {PooledChildShutdownReason} reason - Shutdown reason observed by the child.
|
|
@@ -15,7 +15,7 @@ import { runWithBackgroundJobPayload } from "./execution-context.js"
|
|
|
15
15
|
import { createGenerationWorkerId } from "./generation-identity.js"
|
|
16
16
|
import { isProcessAlive } from "./owner-liveness.js"
|
|
17
17
|
import BackgroundJobsGenerationHandshakeTimeoutError, { DEFAULT_GENERATION_HANDSHAKE_TIMEOUT_MS, validateGenerationHandshakeTimeoutMs } from "./generation-handshake-timeout-error.js"
|
|
18
|
-
import { POOLED_RUNNER_INFLIGHT_JOB_ID_LIMIT, boundedPooledRunnerInflightJobIds, isPooledChildShutdownReason, isPooledChildShutdownSignal } from "./pooled-runner-shutdown.js"
|
|
18
|
+
import { POOLED_RUNNER_INFLIGHT_JOB_ID_LIMIT, boundedPooledRunnerInflightJobIds, isPooledChildProcessFault, isPooledChildShutdownReason, isPooledChildShutdownSignal } from "./pooled-runner-shutdown.js"
|
|
19
19
|
|
|
20
20
|
/**
|
|
21
21
|
* Per-forked-child bookkeeping, shared by the wall-clock backstop, the runner's
|
|
@@ -160,10 +160,13 @@ function isForkedChildJobReport(message) {
|
|
|
160
160
|
*/
|
|
161
161
|
function isChildShutdownObservationMessage(message) {
|
|
162
162
|
if (!message || typeof message !== "object") return false
|
|
163
|
-
const record = /** @type {{childInstanceId?: ReturnType<typeof JSON.parse>, inflightJobIds?: ReturnType<typeof JSON.parse>, inflightJobIdsTruncatedCount?: ReturnType<typeof JSON.parse>, reason?: ReturnType<typeof JSON.parse>, shutdownObservedAtMs?: ReturnType<typeof JSON.parse>, shutdownRequestedAtMs?: ReturnType<typeof JSON.parse>, signal?: ReturnType<typeof JSON.parse>, type?: ReturnType<typeof JSON.parse>}} */ (message)
|
|
163
|
+
const record = /** @type {{childInstanceId?: ReturnType<typeof JSON.parse>, fault?: ReturnType<typeof JSON.parse>, inflightJobIds?: ReturnType<typeof JSON.parse>, inflightJobIdsTruncatedCount?: ReturnType<typeof JSON.parse>, reason?: ReturnType<typeof JSON.parse>, shutdownObservedAtMs?: ReturnType<typeof JSON.parse>, shutdownRequestedAtMs?: ReturnType<typeof JSON.parse>, signal?: ReturnType<typeof JSON.parse>, type?: ReturnType<typeof JSON.parse>}} */ (message)
|
|
164
164
|
|
|
165
165
|
return record.type === "shutdown-observation"
|
|
166
166
|
&& typeof record.childInstanceId === "string"
|
|
167
|
+
// A child predating the fault contract sends no fault field at all; its shutdown
|
|
168
|
+
// provenance must still be accepted.
|
|
169
|
+
&& (record.fault === undefined || record.fault === null || isPooledChildProcessFault(record.fault))
|
|
167
170
|
&& Array.isArray(record.inflightJobIds)
|
|
168
171
|
&& record.inflightJobIds.length <= POOLED_RUNNER_INFLIGHT_JOB_ID_LIMIT
|
|
169
172
|
&& record.inflightJobIds.every((jobId) => typeof jobId === "string")
|
|
@@ -176,6 +179,23 @@ function isChildShutdownObservationMessage(message) {
|
|
|
176
179
|
&& isPooledChildShutdownSignal(record.signal)
|
|
177
180
|
}
|
|
178
181
|
|
|
182
|
+
/**
|
|
183
|
+
* Names the failure reported for every job lost with a pooled child. When the child
|
|
184
|
+
* transmitted its process-level fault, that fault is part of the report: the exit
|
|
185
|
+
* line alone is what makes a real framework fault read as infrastructure loss and
|
|
186
|
+
* re-triggers a whole shard with zero reported failures. The exit line is kept so the
|
|
187
|
+
* observed code and signal remain readable in the same record.
|
|
188
|
+
* @param {object} args - Failure details.
|
|
189
|
+
* @param {Error} args.error - Worker-side failure observed for the child.
|
|
190
|
+
* @param {import("./types.js").PooledChildProcessFault | null} args.fault - Fault the child transmitted before exiting.
|
|
191
|
+
* @returns {Error} - Error to report for the lost jobs.
|
|
192
|
+
*/
|
|
193
|
+
function pooledChildFailureError({error, fault}) {
|
|
194
|
+
if (!fault) return error
|
|
195
|
+
|
|
196
|
+
return new Error(`${error.message} — ${fault.type}: ${fault.message ?? fault.valuePreview ?? "unspecified fault"}`)
|
|
197
|
+
}
|
|
198
|
+
|
|
179
199
|
/**
|
|
180
200
|
* Checks whether an IPC value is a pooled child's bounded memory observation.
|
|
181
201
|
* The pooled child's stdio is ignored by the worker fork, so this observation
|
|
@@ -2102,14 +2122,16 @@ export default class BackgroundJobsWorker {
|
|
|
2102
2122
|
const runnerFailure = state
|
|
2103
2123
|
? this._pooledRunnerFailure({child, exitCode, origin, signal, state})
|
|
2104
2124
|
: undefined
|
|
2125
|
+
if (runnerFailure?.processFault) this._logPooledChildProcessFault({child, runnerFailure})
|
|
2105
2126
|
if (state) state.inflight.clear()
|
|
2106
2127
|
this.pooledChildStates.delete(child)
|
|
2107
2128
|
|
|
2129
|
+
const failureError = pooledChildFailureError({error, fault: runnerFailure?.processFault ?? null})
|
|
2108
2130
|
const failureReports = entries.map(async (entry) => {
|
|
2109
2131
|
await this._reportJobResult({
|
|
2110
2132
|
jobId: entry.payload.id,
|
|
2111
2133
|
status: "failed",
|
|
2112
|
-
error,
|
|
2134
|
+
error: failureError,
|
|
2113
2135
|
handoffId: entry.payload.handoffId,
|
|
2114
2136
|
handedOffAtMs: entry.payload.handedOffAtMs,
|
|
2115
2137
|
runnerFailure,
|
|
@@ -2211,6 +2233,7 @@ export default class BackgroundJobsWorker {
|
|
|
2211
2233
|
...boundedInflight,
|
|
2212
2234
|
oomKilled: signal === "SIGKILL" && shutdownReason === "signal_sigkill" ? null : false,
|
|
2213
2235
|
origin,
|
|
2236
|
+
processFault: observation?.fault ?? null,
|
|
2214
2237
|
runnerAgeMs: Math.max(0, Date.now() - state.createdAtMs),
|
|
2215
2238
|
runnerCreatedAtMs: state.createdAtMs,
|
|
2216
2239
|
runnerDetached: false,
|
|
@@ -2230,6 +2253,36 @@ export default class BackgroundJobsWorker {
|
|
|
2230
2253
|
})
|
|
2231
2254
|
}
|
|
2232
2255
|
|
|
2256
|
+
/**
|
|
2257
|
+
* Logs one compact JSON line naming the process-level fault a pooled child
|
|
2258
|
+
* transmitted over IPC before it exited — the same stderr surface the memory
|
|
2259
|
+
* observations use, which reaches the production log while the child's own stdio
|
|
2260
|
+
* is ignored. A child that dies of an unhandled rejection otherwise looks exactly
|
|
2261
|
+
* like an infrastructure exit, so the shard is retried with zero reported failures
|
|
2262
|
+
* and the fault is never attributed.
|
|
2263
|
+
* @param {object} args - Event details.
|
|
2264
|
+
* @param {import("node:child_process").ChildProcess} args.child - Pooled child.
|
|
2265
|
+
* @param {import("./types.js").PooledRunnerFailure} args.runnerFailure - Shared failure provenance carrying the fault.
|
|
2266
|
+
* @returns {void}
|
|
2267
|
+
*/
|
|
2268
|
+
_logPooledChildProcessFault({child, runnerFailure}) {
|
|
2269
|
+
const fault = runnerFailure.processFault
|
|
2270
|
+
if (!fault) return
|
|
2271
|
+
|
|
2272
|
+
console.error(JSON.stringify({
|
|
2273
|
+
event: "pooled-child-process-fault",
|
|
2274
|
+
childInstanceId: runnerFailure.childInstanceId,
|
|
2275
|
+
childPid: runnerFailure.runnerPid ?? child.pid ?? null,
|
|
2276
|
+
exitCode: runnerFailure.exitCode,
|
|
2277
|
+
faultMessage: fault.message,
|
|
2278
|
+
faultName: fault.name,
|
|
2279
|
+
faultType: fault.type,
|
|
2280
|
+
jobs: runnerFailure.inflightJobIds.length,
|
|
2281
|
+
shutdownReason: runnerFailure.shutdownReason,
|
|
2282
|
+
valuePreview: fault.valuePreview
|
|
2283
|
+
}))
|
|
2284
|
+
}
|
|
2285
|
+
|
|
2233
2286
|
/**
|
|
2234
2287
|
* Runs run job inline.
|
|
2235
2288
|
* @param {import("./types.js").BackgroundJobPayload} payload - Payload.
|
|
@@ -83,6 +83,11 @@ import {RansackQueryError, normalizeRansackGroup, parseRansackSort} from "./util
|
|
|
83
83
|
|
|
84
84
|
const ATTACHMENT_OWNER_KEY = "__attachmentOwner"
|
|
85
85
|
|
|
86
|
+
/** Maximum number of commands accepted in one frontend-model API batch. */
|
|
87
|
+
export const FRONTEND_MODEL_MAX_BATCH_REQUESTS = 100
|
|
88
|
+
/** Maximum UTF-8 byte size of a frontend-model API request payload. */
|
|
89
|
+
export const FRONTEND_MODEL_MAX_REQUEST_BYTES = 256 * 1024
|
|
90
|
+
|
|
86
91
|
/**
|
|
87
92
|
* Runs normalize frontend model preload.
|
|
88
93
|
* @param {import("./database/query/index.js").NestedPreloadRecord | string | string[] | boolean | undefined | null} preload - Preload shorthand.
|
|
@@ -211,6 +216,24 @@ function frontendModelQueryError(message) {
|
|
|
211
216
|
return VelociousError.safe(message, {code: "frontend-model-query-error"})
|
|
212
217
|
}
|
|
213
218
|
|
|
219
|
+
/**
|
|
220
|
+
* Checks transport-level frontend-model request admission limits.
|
|
221
|
+
* @param {Record<string, ReturnType<typeof JSON.parse>>} params - Decoded request params.
|
|
222
|
+
* @returns {void}
|
|
223
|
+
*/
|
|
224
|
+
function assertFrontendModelRequestLimits(params) {
|
|
225
|
+
const serializedParams = JSON.stringify(params)
|
|
226
|
+
const requestBytes = Buffer.byteLength(serializedParams || "", "utf8")
|
|
227
|
+
|
|
228
|
+
if (requestBytes > FRONTEND_MODEL_MAX_REQUEST_BYTES) {
|
|
229
|
+
throw frontendModelQueryError(`Frontend-model request payload exceeds the maximum size of ${FRONTEND_MODEL_MAX_REQUEST_BYTES} bytes`)
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
if (Array.isArray(params.requests) && params.requests.length > FRONTEND_MODEL_MAX_BATCH_REQUESTS) {
|
|
233
|
+
throw frontendModelQueryError(`Frontend-model request batch exceeds the maximum of ${FRONTEND_MODEL_MAX_BATCH_REQUESTS} requests`)
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
|
|
214
237
|
/**
|
|
215
238
|
* Throws a client-safe frontend-model query error for typed query parser errors.
|
|
216
239
|
* @param {ReturnType<typeof JSON.parse>} error - Error raised while normalizing client query params.
|
|
@@ -4431,6 +4454,24 @@ export default class FrontendModelController extends Controller {
|
|
|
4431
4454
|
}
|
|
4432
4455
|
|
|
4433
4456
|
const params = /** @type {Record<string, ReturnType<typeof JSON.parse>>} */ (deserializeFrontendModelTransportValue(this.params()))
|
|
4457
|
+
try {
|
|
4458
|
+
assertFrontendModelRequestLimits(params)
|
|
4459
|
+
} catch (error) {
|
|
4460
|
+
const errorContext = this.frontendModelEndpointErrorContext({
|
|
4461
|
+
action: "frontendApi",
|
|
4462
|
+
error
|
|
4463
|
+
})
|
|
4464
|
+
|
|
4465
|
+
await this.frontendModelLogEndpointError({error, errorContext})
|
|
4466
|
+
await this.render({
|
|
4467
|
+
json: /** @type {Record<string, ReturnType<typeof JSON.parse>>} */ (serializeFrontendModelTransportValue(
|
|
4468
|
+
await this.frontendModelClientErrorPayloadForError(error, errorContext),
|
|
4469
|
+
this.transportSerializationOptions()
|
|
4470
|
+
))
|
|
4471
|
+
})
|
|
4472
|
+
return
|
|
4473
|
+
}
|
|
4474
|
+
|
|
4434
4475
|
const requests = Array.isArray(params.requests) ? params.requests : [params]
|
|
4435
4476
|
/**
|
|
4436
4477
|
* Responses.
|
|
@@ -140,6 +140,42 @@ const FRONTEND_MODEL_INDEX_PAYLOAD_KEYS = new Set([
|
|
|
140
140
|
"withCount"
|
|
141
141
|
])
|
|
142
142
|
|
|
143
|
+
/** Maximum nesting depth accepted in a frontend-model query shape. */
|
|
144
|
+
export const FRONTEND_MODEL_QUERY_MAX_DEPTH = 8
|
|
145
|
+
/** Maximum number of object keys accepted in one frontend-model query shape. */
|
|
146
|
+
export const FRONTEND_MODEL_QUERY_MAX_KEYS = 256
|
|
147
|
+
|
|
148
|
+
/**
|
|
149
|
+
* Checks the resource cost of a nested frontend-model query shape before any
|
|
150
|
+
* query normalization or database work starts.
|
|
151
|
+
* @param {ReturnType<typeof JSON.parse>} value - Query shape.
|
|
152
|
+
* @param {string} label - Query field name for the error message.
|
|
153
|
+
* @param {number} [depth] - Current nesting depth.
|
|
154
|
+
* @param {{keys: number}} [state] - Shared key counter.
|
|
155
|
+
* @returns {void}
|
|
156
|
+
*/
|
|
157
|
+
function assertFrontendModelQueryShape(value, label, depth = 0, state = {keys: 0}) {
|
|
158
|
+
if (!Array.isArray(value) && !isPlainObject(value)) return
|
|
159
|
+
|
|
160
|
+
if (depth > FRONTEND_MODEL_QUERY_MAX_DEPTH) {
|
|
161
|
+
throw frontendModelQueryError(`${label} exceeds the maximum nesting depth of ${FRONTEND_MODEL_QUERY_MAX_DEPTH}`)
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
if (Array.isArray(value)) {
|
|
165
|
+
for (const entry of value) assertFrontendModelQueryShape(entry, label, depth + 1, state)
|
|
166
|
+
return
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
for (const entry of Object.values(value)) {
|
|
170
|
+
state.keys += 1
|
|
171
|
+
if (state.keys > FRONTEND_MODEL_QUERY_MAX_KEYS) {
|
|
172
|
+
throw frontendModelQueryError(`${label} exceeds the maximum of ${FRONTEND_MODEL_QUERY_MAX_KEYS} keys`)
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
assertFrontendModelQueryShape(entry, label, depth + 1, state)
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
|
|
143
179
|
/**
|
|
144
180
|
* Builds a query descriptor error.
|
|
145
181
|
* @param {string} message - Error message.
|
|
@@ -167,6 +203,12 @@ export function assertFrontendModelIndexPayload(payload) {
|
|
|
167
203
|
}
|
|
168
204
|
}
|
|
169
205
|
|
|
206
|
+
for (const queryKey of ["where", "joins", "preload", "group", "sort", "pluck"]) {
|
|
207
|
+
if (Object.hasOwn(payload, queryKey)) {
|
|
208
|
+
assertFrontendModelQueryShape(payload[queryKey], queryKey)
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
|
|
170
212
|
return payload
|
|
171
213
|
}
|
|
172
214
|
|