velocious 1.0.662 → 1.0.664

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (102) hide show
  1. package/README.md +3 -3
  2. package/build/background-jobs/adapter.js +9 -0
  3. package/build/background-jobs/client.js +36 -6
  4. package/build/background-jobs/job-runner.js +3 -1
  5. package/build/background-jobs/local-adapter.js +7 -0
  6. package/build/background-jobs/local-store.js +113 -3
  7. package/build/background-jobs/main.js +35 -1
  8. package/build/background-jobs/pooled-runner-child.js +38 -0
  9. package/build/background-jobs/status-reporter.js +97 -1
  10. package/build/background-jobs/store.js +130 -4
  11. package/build/background-jobs/types.js +6 -1
  12. package/build/background-jobs/web/controller.js +4 -0
  13. package/build/background-jobs/worker.js +59 -0
  14. package/build/configuration-types.js +3 -3
  15. package/build/database/query/alter-table-base.js +11 -1
  16. package/build/src/background-jobs/adapter.d.ts +17 -0
  17. package/build/src/background-jobs/adapter.d.ts.map +1 -1
  18. package/build/src/background-jobs/adapter.js +9 -1
  19. package/build/src/background-jobs/client.d.ts.map +1 -1
  20. package/build/src/background-jobs/client.js +35 -6
  21. package/build/src/background-jobs/job-runner.d.ts +3 -1
  22. package/build/src/background-jobs/job-runner.d.ts.map +1 -1
  23. package/build/src/background-jobs/job-runner.js +5 -2
  24. package/build/src/background-jobs/local-adapter.d.ts +13 -0
  25. package/build/src/background-jobs/local-adapter.d.ts.map +1 -1
  26. package/build/src/background-jobs/local-adapter.js +7 -1
  27. package/build/src/background-jobs/local-store.d.ts +38 -0
  28. package/build/src/background-jobs/local-store.d.ts.map +1 -1
  29. package/build/src/background-jobs/local-store.js +117 -4
  30. package/build/src/background-jobs/main.d.ts +14 -0
  31. package/build/src/background-jobs/main.d.ts.map +1 -1
  32. package/build/src/background-jobs/main.js +35 -2
  33. package/build/src/background-jobs/pooled-runner-child.js +38 -1
  34. package/build/src/background-jobs/status-reporter.d.ts +51 -1
  35. package/build/src/background-jobs/status-reporter.d.ts.map +1 -1
  36. package/build/src/background-jobs/status-reporter.js +89 -2
  37. package/build/src/background-jobs/store.d.ts +48 -0
  38. package/build/src/background-jobs/store.d.ts.map +1 -1
  39. package/build/src/background-jobs/store.js +128 -5
  40. package/build/src/background-jobs/types.d.ts +34 -2
  41. package/build/src/background-jobs/types.d.ts.map +1 -1
  42. package/build/src/background-jobs/types.js +7 -2
  43. package/build/src/background-jobs/web/controller.d.ts.map +1 -1
  44. package/build/src/background-jobs/web/controller.js +5 -1
  45. package/build/src/background-jobs/worker.d.ts +20 -0
  46. package/build/src/background-jobs/worker.d.ts.map +1 -1
  47. package/build/src/background-jobs/worker.js +58 -1
  48. package/build/src/configuration-types.d.ts +8 -4
  49. package/build/src/configuration-types.d.ts.map +1 -1
  50. package/build/src/configuration-types.js +4 -4
  51. package/build/src/database/query/alter-table-base.d.ts.map +1 -1
  52. package/build/src/database/query/alter-table-base.js +12 -2
  53. package/build/src/sync/sync-api-client.d.ts +36 -10
  54. package/build/src/sync/sync-api-client.d.ts.map +1 -1
  55. package/build/src/sync/sync-api-client.js +58 -14
  56. package/build/src/sync/sync-client-types.d.ts +41 -2
  57. package/build/src/sync/sync-client-types.d.ts.map +1 -1
  58. package/build/src/sync/sync-client-types.js +20 -3
  59. package/build/src/sync/sync-client.d.ts +112 -3
  60. package/build/src/sync/sync-client.d.ts.map +1 -1
  61. package/build/src/sync/sync-client.js +306 -60
  62. package/build/src/sync/sync-publisher-types.d.ts +36 -3
  63. package/build/src/sync/sync-publisher-types.d.ts.map +1 -1
  64. package/build/src/sync/sync-publisher-types.js +16 -2
  65. package/build/src/sync/sync-publisher.d.ts +12 -0
  66. package/build/src/sync/sync-publisher.d.ts.map +1 -1
  67. package/build/src/sync/sync-publisher.js +68 -4
  68. package/build/src/sync/sync-realtime-bridge.d.ts +11 -5
  69. package/build/src/sync/sync-realtime-bridge.d.ts.map +1 -1
  70. package/build/src/sync/sync-realtime-bridge.js +37 -13
  71. package/build/src/sync/sync-scope-store.d.ts +32 -0
  72. package/build/src/sync/sync-scope-store.d.ts.map +1 -1
  73. package/build/src/sync/sync-scope-store.js +74 -4
  74. package/build/sync/sync-api-client.js +58 -13
  75. package/build/sync/sync-client-types.js +22 -2
  76. package/build/sync/sync-client.js +333 -63
  77. package/build/sync/sync-publisher-types.js +17 -1
  78. package/build/sync/sync-publisher.js +79 -3
  79. package/build/sync/sync-realtime-bridge.js +37 -12
  80. package/build/sync/sync-scope-store.js +79 -3
  81. package/package.json +1 -1
  82. package/src/background-jobs/adapter.js +9 -0
  83. package/src/background-jobs/client.js +36 -6
  84. package/src/background-jobs/job-runner.js +3 -1
  85. package/src/background-jobs/local-adapter.js +7 -0
  86. package/src/background-jobs/local-store.js +113 -3
  87. package/src/background-jobs/main.js +35 -1
  88. package/src/background-jobs/pooled-runner-child.js +38 -0
  89. package/src/background-jobs/status-reporter.js +97 -1
  90. package/src/background-jobs/store.js +130 -4
  91. package/src/background-jobs/types.js +6 -1
  92. package/src/background-jobs/web/controller.js +4 -0
  93. package/src/background-jobs/worker.js +59 -0
  94. package/src/configuration-types.js +3 -3
  95. package/src/database/query/alter-table-base.js +11 -1
  96. package/src/sync/sync-api-client.js +58 -13
  97. package/src/sync/sync-client-types.js +22 -2
  98. package/src/sync/sync-client.js +333 -63
  99. package/src/sync/sync-publisher-types.js +17 -1
  100. package/src/sync/sync-publisher.js +79 -3
  101. package/src/sync/sync-realtime-bridge.js +37 -12
  102. package/src/sync/sync-scope-store.js +79 -3
package/README.md CHANGED
@@ -15,7 +15,7 @@
15
15
  * Controllers and views for HTTP endpoints
16
16
  * Frontend-model transport for creating, updating, querying, and subscribing to query-filtered lifecycle events over HTTP/WebSocket, including committed counter-cache parent updates, structured per-attribute validation error responses, immutable per-operation remote request context, registration-local tenant subscription partitioning, and one-budget WebSocket startup controls (see [docs/frontend-models.md](docs/frontend-models.md), [docs/remote-request-context.md](docs/remote-request-context.md), and [docs/websocket-channels.md](docs/websocket-channels.md))
17
17
  * Client-side offline sync mutation logs and frontend-model optimistic queueing primitives (see the [shared-resource sync developer guide](docs/shared-resource-sync-guide.md) and [offline sync architecture](docs/offline-sync.md))
18
- * Declarative client sync scopes with per-scope cursors, automatic mutation tracking, opt-in durable base-version conflict replay, realtime delivery, and immutable-handle project clients whose local database state plus remote pull/replay/realtime request context stay tenant-bound through reconnect (see [docs/sync-client.md](docs/sync-client.md), [docs/remote-request-context.md](docs/remote-request-context.md), and [docs/offline-sync.md](docs/offline-sync.md))
18
+ * Declarative client sync scopes with per-scope cursors, generation-fenced identity replacement and selected-scope cache reset, automatic mutation tracking, opt-in durable base-version conflict replay, realtime delivery and reconnect gap closure, computed server-publish scope metadata, and immutable-handle project clients whose local database state plus remote pull/replay/realtime request context stay tenant-bound through reconnect (see [docs/sync-client.md](docs/sync-client.md), [docs/remote-request-context.md](docs/remote-request-context.md), and [docs/offline-sync.md](docs/offline-sync.md))
19
19
  * Reactive `useLiveQuery(Model.where(...))` queries for default databases plus immutable-handle tenant live-query sources whose committed events and refreshes stay on the captured physical tenant (see [docs/live-queries.md](docs/live-queries.md))
20
20
  * Server-side sync envelope replay orchestration for app-owned sync receivers, including allowlisted authoritative conflict snapshots that retain submitted aliases in conflict metadata while keying `serverModel` by canonical model attributes (see [docs/sync-envelope-replay-service.md](docs/sync-envelope-replay-service.md))
21
21
  * Self-sustaining sync feeds: upstream imports triggered by the changes pull itself, with framework-owned coalescing and throttling (see [docs/sync-upstream-imports.md](docs/sync-upstream-imports.md))
@@ -2614,7 +2614,7 @@ actual state sources must still agree.
2614
2614
 
2615
2615
  `maxConcurrentInlineJobs` (default: `4`) caps how many `executionMode: "inline"` jobs a single `background-jobs-worker` process runs in parallel. Concurrency is at the JS event-loop level: every job in flight shares the worker's process and DB connection pool, so the cap should fit the pool, not the CPU count. Forking remains the right tool when you need memory isolation across long-running jobs or want to use more cores; select it with `executionMode: "forked"`.
2616
2616
 
2617
- New jobs default to `executionMode: "pooled"`: a worker runs them in warm, reusable Node child runners. `pooledRunnerCount` (default: `4`) bounds this independent per-worker pool, and `pooledRunnerConcurrency` (default: `1`) sets how many jobs each child runs at once on its own event loop, so total pooled capacity is `pooledRunnerCount × pooledRunnerConcurrency` — raise concurrency for I/O-bound jobs to get high throughput from a bounded, isolated set of processes. Workers advertise that exact free-slot count and the main consumes one slot per durable handoff, allowing one readiness notification to fill the pool. If an initialized child exits unexpectedly, the worker immediately advertises the freed capacity while failure reports retry; the replacement is spawned lazily by the next dispatch, and a pre-startup crash does not trigger a respawn loop. `pooledRunnerCount`, `pooledRunnerConcurrency`, and `pooledRunnerMaxJobs` must be finite positive integers; the RSS and lifetime limits must be finite positive numbers. A child is recycled after an acknowledged job when it reaches `pooledRunnerMaxJobs` (default: `100`), `pooledRunnerMaxRssBytes` (default: `536870912`, or 512 MiB), or `pooledRunnerMaxLifetimeMs` (default: `3600000`, or one hour). `execution_mode` is the single source of truth for a job's runtime — pooled rows persist as `execution_mode = "pooled"` directly. See [execution modes and pooled runners](docs/background-jobs.md#execution-modes-and-pooled-runners).
2617
+ New jobs default to `executionMode: "pooled"`: a worker runs them in warm, reusable Node child runners. `pooledRunnerCount` (default: `4`) bounds this independent per-worker pool, and `pooledRunnerConcurrency` (default: `1`) sets how many jobs each child runs at once on its own event loop, so total pooled capacity is `pooledRunnerCount × pooledRunnerConcurrency` — raise concurrency for I/O-bound jobs to get high throughput from a bounded, isolated set of processes. Workers advertise that exact free-slot count and the main consumes one slot per durable handoff, allowing one readiness notification to fill the pool. If an initialized child exits unexpectedly, the worker immediately advertises the freed capacity while failure reports retry; the replacement is spawned lazily by the next dispatch, and a pre-startup crash does not trigger a respawn loop. `pooledRunnerCount`, `pooledRunnerConcurrency`, and `pooledRunnerMaxJobs` must be finite positive integers; the RSS and lifetime limits must be finite positive numbers. A child is recycled after an acknowledged job when it reaches `pooledRunnerMaxJobs` (default: `100`), `pooledRunnerMaxRssBytes` (default: `536870912`, or 512 MiB), or `pooledRunnerMaxLifetimeMs` (default: `3600000`, or one hour). `execution_mode` is the single source of truth for a job's runtime — pooled rows persist as `execution_mode = "pooled"` directly. Pooled jobs also persist acceptance evidence (`child_received_at_ms`, `child_started_at_ms`, `child_instance_id`, `child_pid`) so a handed-off job can be told apart from one whose runner never picked it up. See [execution modes and pooled runners](docs/background-jobs.md#execution-modes-and-pooled-runners).
2618
2618
 
2619
2619
  Cold pooled jobs share one atomic model-bootstrap phase. If that phase fails, all
2620
2620
  waiting jobs receive the failure; a later job in the surviving child cannot run
@@ -2729,7 +2729,7 @@ Set `deduplicateWhileQueued: true` to coalesce an enqueue onto the earliest iden
2729
2729
 
2730
2730
  Use `options: {idempotencyKey}` when producer replay must converge on the original durable job across every state and even after terminal-job pruning. Ownership is scoped to the resolved job class name, resolved queue, and key; reusing that scope with changed canonical arguments or behavior-affecting options fails. This is distinct from queued-only deduplication, and ownership rows are intentionally retained until a future explicit reconciliation/deletion policy. See [durable idempotent enqueue](docs/background-jobs.md#durable-idempotent-enqueue).
2731
2731
 
2732
- The Node producer rejects and destroys its one-shot socket when the main closes before acknowledging or when an enqueue acknowledgement stalls for 5 seconds. Because the main may already have committed the job, this is an ambiguous outcome. A call made by an executing generation-owned job automatically makes one recovery attempt with its exact internal producer proof and per-call invocation identity, but never after a pre-send generation failure or explicit enqueue rejection. Ordinary enqueues are not retried automatically: replay one with the same durable `idempotencyKey` to recover the original job id without creating a duplicate. Direct `BackgroundJobsClient` users can set a different bounded `enqueueTimeoutMs` constructor option. See [durable idempotent enqueue](docs/background-jobs.md#durable-idempotent-enqueue).
2732
+ The Node producer gives connection/handshake and post-send acknowledgement separate bounded phases, so connection or generation-handshake latency does not consume the 5-second `enqueued` acknowledgement budget. It rejects and destroys its one-shot socket when either phase stalls or when the main closes before acknowledging. Because the main may already have committed the job, a post-send failure is an ambiguous outcome. A call made by an executing generation-owned job automatically makes one recovery attempt with its exact internal producer proof and per-call invocation identity, but never after a pre-send generation failure or explicit enqueue rejection. Ordinary enqueues are not retried automatically: replay one with the same durable `idempotencyKey` to recover the original job id without creating a duplicate. Direct `BackgroundJobsClient` users can set a different bounded `enqueueTimeoutMs` constructor option for both phases. See [durable idempotent enqueue](docs/background-jobs.md#durable-idempotent-enqueue).
2733
2733
 
2734
2734
  Select a non-default runtime explicitly with `options: {executionMode: "inline" | "forked" | "spawned"}`.
2735
2735
 
@@ -129,6 +129,15 @@ export default class BackgroundJobsAdapter {
129
129
  */
130
130
  async markCompleted(_args) { throw new Error("BackgroundJobsAdapter#markCompleted is not implemented") }
131
131
 
132
+ /**
133
+ * Records pooled-child acceptance evidence (received/started timestamps plus
134
+ * runner identity) for a handed-off job, fenced by its active handoff lease.
135
+ * Only the fields supplied are written.
136
+ * @param {{jobId: string, handoffId?: string, workerId?: string, handedOffAtMs?: number, receivedAtMs?: number, startedAtMs?: number, childInstanceId?: string, childPid?: number}} _args - Acceptance report.
137
+ * @returns {Promise<boolean>} - Whether the fenced report was accepted.
138
+ */
139
+ async markChildAccepted(_args) { throw new Error("BackgroundJobsAdapter#markChildAccepted is not implemented") }
140
+
132
141
  /**
133
142
  * Returns a handed-off job to its schedule.
134
143
  * @param {{jobId: string, delayMs: number, handoffId?: string, workerId?: string, handedOffAtMs?: number}} _args - Reschedule report.
@@ -66,18 +66,44 @@ export default class BackgroundJobsClient {
66
66
  */
67
67
  const enqueueAttempt = async (attemptAcknowledgement) => {
68
68
  const request = await this._request()
69
-
70
- return await timeout({
71
- errorMessage: `Background job enqueue acknowledgement timed out after ${this.enqueueTimeoutMs}ms`,
69
+ const requestAbortController = new AbortController()
70
+ const timeoutErrorMessage = `Background job enqueue acknowledgement timed out after ${this.enqueueTimeoutMs}ms`
71
+ /**
72
+ * Resolves the pre-send phase when the mutation has entered the socket.
73
+ * @type {() => void}
74
+ */
75
+ let markRequestSent = () => {}
76
+ const requestSent = new Promise((resolve) => {
77
+ markRequestSent = () => resolve(undefined)
78
+ })
79
+ /**
80
+ * Applies the configured deadline independently to one request phase.
81
+ * @template T
82
+ * @param {() => Promise<T>} callback - Phase work.
83
+ * @returns {Promise<T>} - Phase result.
84
+ */
85
+ const withEnqueueTimeout = async (callback) => await timeout({
86
+ errorMessage: timeoutErrorMessage,
72
87
  timeout: this.enqueueTimeoutMs
73
- }, async ({control}) => await request.run({
74
- signal: control.signal,
88
+ }, async ({control}) => {
89
+ const abortRequest = () => requestAbortController.abort(control.signal.reason)
90
+
91
+ control.signal.addEventListener("abort", abortRequest)
92
+ try {
93
+ return await callback()
94
+ } finally {
95
+ control.signal.removeEventListener("abort", abortRequest)
96
+ }
97
+ })
98
+ const requestPromise = request.run({
99
+ signal: requestAbortController.signal,
75
100
  onConnect: (jsonSocket) => {
76
101
  jsonSocket.send(message)
77
102
  if (attemptAcknowledgement) {
78
103
  attemptAcknowledgement.generationFenced = Boolean(request.generationId)
79
104
  attemptAcknowledgement.requestSent = true
80
105
  }
106
+ markRequestSent()
81
107
  },
82
108
  onMessage: ({message, resolve, reject}) => {
83
109
  if (message?.type === "enqueued") {
@@ -90,7 +116,11 @@ export default class BackgroundJobsClient {
90
116
  reject(new Error(message.error || "Failed to enqueue job"))
91
117
  }
92
118
  }
93
- }))
119
+ })
120
+
121
+ await withEnqueueTimeout(async () => await Promise.race([requestSent, requestPromise]))
122
+
123
+ return await withEnqueueTimeout(async () => await requestPromise)
94
124
  }
95
125
 
96
126
  try {
@@ -82,10 +82,11 @@ function runnerProcessTitle(JobClass, payload) {
82
82
  * @param {object} [options] - Runner options.
83
83
  * @param {boolean} [options.closeConnections] - Whether to gracefully close framework connections after the job.
84
84
  * @param {boolean} [options.manageProcessTitle] - Whether to set the per-job process title and restore it afterwards. Off for concurrent pooled runners, where interleaved snapshot/restore of the single process-wide `process.title` would corrupt it; the pooled child owns an aggregate title instead.
85
+ * @param {() => void} [options.onPerformStart] - Observation hook fired once, immediately before perform runs (with its connection acquired). Pooled runners use it to report that the job actually started. Must not throw into the job.
85
86
  * @param {string} [options.processType] - Generic application process type.
86
87
  * @returns {Promise<"completed" | "rescheduled">} - Acknowledged outcome.
87
88
  */
88
- export default async function runJobPayload(payload, {closeConnections = true, manageProcessTitle = true, processType = "background-jobs-runner"} = {}) {
89
+ export default async function runJobPayload(payload, {closeConnections = true, manageProcessTitle = true, onPerformStart, processType = "background-jobs-runner"} = {}) {
89
90
  const configuration = await configurationResolver()
90
91
  configuration.setCurrent()
91
92
  await configuration.initialize({type: processType})
@@ -119,6 +120,7 @@ export default async function runJobPayload(payload, {closeConnections = true, m
119
120
  try {
120
121
  await runWithBackgroundJobPayload(payload, async () => {
121
122
  await configuration.withConnections({databaseIdentifiers: JobClass.databaseIdentifiers, name: `Background job runner: ${payload.jobName}`}, async () => {
123
+ if (onPerformStart) onPerformStart()
122
124
  await perform.apply(jobInstance, jobArgs)
123
125
  })
124
126
  })
@@ -133,6 +133,13 @@ export default class LocalBackgroundJobsAdapter extends BackgroundJobsAdapter {
133
133
  */
134
134
  async markCompleted(args) { return await this.store.markCompleted(args) }
135
135
 
136
+ /**
137
+ * Records pooled-child acceptance evidence for a local handoff.
138
+ * @param {{jobId: string, handoffId?: string, receivedAtMs?: number, startedAtMs?: number, childInstanceId?: string, childPid?: number}} args - Acceptance report.
139
+ * @returns {Promise<boolean>} - Whether accepted.
140
+ */
141
+ async markChildAccepted(args) { return await this.store.markChildAccepted(args) }
142
+
136
143
  /**
137
144
  * Acknowledges an explicit local reschedule.
138
145
  * @param {{jobId: string, delayMs: number, handoffId?: string}} args - Reschedule report.
@@ -49,7 +49,11 @@ const EXPECTED_JOB_COLUMNS = [
49
49
  "failed_at_ms",
50
50
  "last_error",
51
51
  "concurrency_key",
52
- "max_concurrency"
52
+ "max_concurrency",
53
+ "child_received_at_ms",
54
+ "child_started_at_ms",
55
+ "child_instance_id",
56
+ "child_pid"
53
57
  ]
54
58
  const EXPECTED_CONCURRENCY_COLUMNS = ["concurrency_key", "max_concurrency", "active_count"]
55
59
  /** @type {WeakMap<import("../configuration.js").default, Map<string, Promise<void>>>} */
@@ -202,6 +206,7 @@ export default class LocalBackgroundJobsStore {
202
206
  await db.createTable(this._jobsTableData())
203
207
  changed = true
204
208
  } else {
209
+ if (await this._ensureJobColumns(db)) changed = true
205
210
  await this._assertColumns(db, LOCAL_BACKGROUND_JOBS_TABLE, EXPECTED_JOB_COLUMNS)
206
211
  }
207
212
 
@@ -232,6 +237,45 @@ export default class LocalBackgroundJobsStore {
232
237
  return changed
233
238
  }
234
239
 
240
+ /**
241
+ * Idempotently adds columns from the current jobs table definition that an
242
+ * existing local table is missing, so an upgraded framework finds a
243
+ * compatible schema instead of failing the column assertion.
244
+ * @param {import("../database/drivers/base.js").default} db - Local SQLite connection.
245
+ * @returns {Promise<boolean>} - Whether a column was added.
246
+ */
247
+ async _ensureJobColumns(db) {
248
+ db.clearSchemaCache()
249
+ const table = await db.getTableByNameOrFail(LOCAL_BACKGROUND_JOBS_TABLE)
250
+ const tableData = new TableData(LOCAL_BACKGROUND_JOBS_TABLE)
251
+ let added = false
252
+
253
+ for (const column of this._jobsTableData().getColumns()) {
254
+ if (await table.getColumnByName(column.getName())) continue
255
+ if (column.getPrimaryKey()) continue
256
+
257
+ const columnArgs = /** @type {{null: boolean, maxLength?: number}} */ ({null: column.getNull() !== false})
258
+ const maxLength = column.getMaxLength()
259
+
260
+ if (typeof maxLength === "number") columnArgs.maxLength = maxLength
261
+
262
+ const type = column.getType()
263
+ if (type === "string") tableData.string(column.getName(), columnArgs)
264
+ else if (type === "text") tableData.text(column.getName(), columnArgs)
265
+ else if (type === "bigint") tableData.bigint(column.getName(), columnArgs)
266
+ else if (type === "integer") tableData.integer(column.getName(), columnArgs)
267
+ else if (type === "boolean") tableData.boolean(column.getName(), columnArgs)
268
+ else continue
269
+ added = true
270
+ }
271
+
272
+ if (!added) return false
273
+
274
+ for (const sql of await db.alterTableSQLs(tableData)) await db.query(sql)
275
+ db.clearSchemaCache()
276
+ return true
277
+ }
278
+
235
279
  /**
236
280
  * Builds the migration ledger table definition.
237
281
  * @returns {TableData} - Migration ledger table.
@@ -272,6 +316,10 @@ export default class LocalBackgroundJobsStore {
272
316
  table.text("last_error", {null: true})
273
317
  table.string("concurrency_key", {null: true})
274
318
  table.integer("max_concurrency", {null: true})
319
+ table.bigint("child_received_at_ms", {null: true})
320
+ table.bigint("child_started_at_ms", {null: true})
321
+ table.string("child_instance_id", {null: true})
322
+ table.integer("child_pid", {null: true})
275
323
  table.addIndex(new TableIndex(["status", "scheduled_at_ms", "created_at_ms", "id"], {name: LOCAL_BACKGROUND_JOBS_INDEX_NAMES[0]}))
276
324
  table.addIndex(new TableIndex(["queue", "status", "created_at_ms"], {name: LOCAL_BACKGROUND_JOBS_INDEX_NAMES[1]}))
277
325
  table.addIndex(new TableIndex(["args_digest"], {name: LOCAL_BACKGROUND_JOBS_INDEX_NAMES[2]}))
@@ -708,7 +756,7 @@ export default class LocalBackgroundJobsStore {
708
756
  const handedOffAtMs = this.clock.now()
709
757
  const affectedRows = await this._updateAffectedRows(db, {
710
758
  conditions: {id: jobId, status: "queued"},
711
- data: {handed_off_at_ms: handedOffAtMs, handoff_id: handoffId, status: "handed_off", worker_id: workerId || "local"},
759
+ data: {...this._clearedChildAcceptanceData(), handed_off_at_ms: handedOffAtMs, handoff_id: handoffId, status: "handed_off", worker_id: workerId || "local"},
712
760
  tableName: LOCAL_BACKGROUND_JOBS_TABLE
713
761
  })
714
762
 
@@ -763,6 +811,7 @@ export default class LocalBackgroundJobsStore {
763
811
  const affectedRows = await this._updateAffectedRows(db, {
764
812
  conditions: {handoff_id: handoffId, id: jobId, status: "handed_off"},
765
813
  data: {
814
+ ...this._clearedChildAcceptanceData(),
766
815
  handed_off_at_ms: null,
767
816
  handoff_id: null,
768
817
  scheduled_at_ms: this.clock.now(),
@@ -802,6 +851,43 @@ export default class LocalBackgroundJobsStore {
802
851
  }))
803
852
  }
804
853
 
854
+ /**
855
+ * Records pooled-child acceptance evidence for an active handoff. Only the
856
+ * fields supplied are written, fenced by the exact active handoff lease.
857
+ * @param {object} args - Acceptance report.
858
+ * @param {string} args.jobId - Job id.
859
+ * @param {string} [args.handoffId] - Handoff lease id.
860
+ * @param {number} [args.receivedAtMs] - Epoch ms the runner child received the job.
861
+ * @param {number} [args.startedAtMs] - Epoch ms the job's perform started in the child.
862
+ * @param {string} [args.childInstanceId] - Stable pooled child identity.
863
+ * @param {number} [args.childPid] - Pooled child OS pid.
864
+ * @returns {Promise<boolean>} - Whether the lease won.
865
+ */
866
+ async markChildAccepted({jobId, handoffId, receivedAtMs, startedAtMs, childInstanceId, childPid}) {
867
+ await this.ensureReady()
868
+
869
+ return await this._withDb(async (connection) => await this._mutate(connection, async (db) => {
870
+ const job = await this._getJob(db, jobId)
871
+
872
+ if (!this._acceptsHandoff(job, handoffId)) return false
873
+
874
+ const data = {}
875
+ if (typeof receivedAtMs === "number") data.child_received_at_ms = receivedAtMs
876
+ if (typeof startedAtMs === "number") data.child_started_at_ms = startedAtMs
877
+ if (typeof childInstanceId === "string") data.child_instance_id = childInstanceId
878
+ if (typeof childPid === "number") data.child_pid = childPid
879
+ if (Object.keys(data).length === 0) return false
880
+
881
+ const affectedRows = await this._updateAffectedRows(db, {
882
+ conditions: {handoff_id: handoffId, id: jobId, status: "handed_off"},
883
+ data,
884
+ tableName: LOCAL_BACKGROUND_JOBS_TABLE
885
+ })
886
+
887
+ return affectedRows === 1
888
+ }))
889
+ }
890
+
805
891
  /**
806
892
  * Applies a fenced reschedule without consuming an attempt.
807
893
  * @param {{jobId: string, handoffId?: string, delayMs: number}} args - Reschedule report.
@@ -819,6 +905,7 @@ export default class LocalBackgroundJobsStore {
819
905
  const affectedRows = await this._updateAffectedRows(db, {
820
906
  conditions: {handoff_id: handoffId, id: jobId, status: "handed_off"},
821
907
  data: {
908
+ ...this._clearedChildAcceptanceData(),
822
909
  handed_off_at_ms: null,
823
910
  handoff_id: null,
824
911
  scheduled_at_ms: rescheduledBackgroundJobAtMs(delayMs, this.clock.now()),
@@ -917,7 +1004,9 @@ export default class LocalBackgroundJobsStore {
917
1004
  }
918
1005
 
919
1006
  if (willRetry) {
920
- Object.assign(data, {scheduled_at_ms: nowMs + retryDelayMs(attempts)})
1007
+ // A retry starts a fresh handoff with a possibly different runner, so the
1008
+ // previous child's acceptance evidence must not leak into the next attempt.
1009
+ Object.assign(data, {scheduled_at_ms: nowMs + retryDelayMs(attempts), ...this._clearedChildAcceptanceData()})
921
1010
  } else {
922
1011
  Object.assign(data, {failed_at_ms: nowMs})
923
1012
  }
@@ -934,6 +1023,7 @@ export default class LocalBackgroundJobsStore {
934
1023
 
935
1024
  return {
936
1025
  ...job,
1026
+ ...(willRetry ? this._clearedChildAcceptanceRow() : {}),
937
1027
  attempts,
938
1028
  failedAtMs: willRetry ? job.failedAtMs : nowMs,
939
1029
  handedOffAtMs: null,
@@ -945,6 +1035,22 @@ export default class LocalBackgroundJobsStore {
945
1035
  }
946
1036
  }
947
1037
 
1038
+ /**
1039
+ * Returns the database data that clears pooled-child acceptance evidence.
1040
+ * @returns {Record<string, ReturnType<typeof JSON.parse>>} - Cleared acceptance columns.
1041
+ */
1042
+ _clearedChildAcceptanceData() {
1043
+ return {child_instance_id: null, child_pid: null, child_received_at_ms: null, child_started_at_ms: null}
1044
+ }
1045
+
1046
+ /**
1047
+ * Returns the row-shape counterpart of the cleared acceptance columns.
1048
+ * @returns {Pick<import("./types.js").BackgroundJobRow, "childInstanceId" | "childPid" | "childReceivedAtMs" | "childStartedAtMs">} - Cleared acceptance fields.
1049
+ */
1050
+ _clearedChildAcceptanceRow() {
1051
+ return {childInstanceId: null, childPid: null, childReceivedAtMs: null, childStartedAtMs: null}
1052
+ }
1053
+
948
1054
  /**
949
1055
  * Ensures that a durable concurrency counter exists with the required cap.
950
1056
  * @param {import("../database/drivers/base.js").default} db - Local SQLite connection.
@@ -1106,6 +1212,10 @@ export default class LocalBackgroundJobsStore {
1106
1212
  return {
1107
1213
  args: parsedArgs,
1108
1214
  attempts: this._numberOrNull(row.attempts),
1215
+ childInstanceId: row.child_instance_id === null || row.child_instance_id === undefined ? null : String(row.child_instance_id),
1216
+ childPid: this._numberOrNull(row.child_pid),
1217
+ childReceivedAtMs: this._numberOrNull(row.child_received_at_ms),
1218
+ childStartedAtMs: this._numberOrNull(row.child_started_at_ms),
1109
1219
  completedAtMs: this._numberOrNull(row.completed_at_ms),
1110
1220
  concurrencyKey: row.concurrency_key === null || row.concurrency_key === undefined ? null : String(row.concurrency_key),
1111
1221
  createdAtMs: this._numberOrNull(row.created_at_ms),
@@ -1219,6 +1219,11 @@ export default class BackgroundJobsMain {
1219
1219
  }
1220
1220
  return
1221
1221
  }
1222
+ if (message?.type === "job-accepted") {
1223
+ await this._handleJobAccepted({jsonSocket, message})
1224
+ return
1225
+ }
1226
+
1222
1227
  if (message?.type === "job-complete") {
1223
1228
  await this._handleJobComplete({jsonSocket, message})
1224
1229
  return
@@ -1234,6 +1239,35 @@ export default class BackgroundJobsMain {
1234
1239
  }
1235
1240
  }
1236
1241
 
1242
+ /**
1243
+ * Persists pooled-child acceptance evidence for an active handoff. The
1244
+ * report is diagnostic: a stale lease (job already reclaimed or terminal)
1245
+ * answers the same `job-updated` acknowledgement as an accepted report, and
1246
+ * only a store failure answers `job-update-error` so the worker can retry.
1247
+ * @param {object} args - Options.
1248
+ * @param {JsonSocket} args.jsonSocket - JSON socket.
1249
+ * @param {import("./types.js").BackgroundJobAcceptedMessage} args.message - Message.
1250
+ * @returns {Promise<void>} - Resolves when handled.
1251
+ */
1252
+ async _handleJobAccepted({jsonSocket, message}) {
1253
+ try {
1254
+ await this.store.markChildAccepted({
1255
+ childInstanceId: message.childInstanceId,
1256
+ childPid: message.childPid,
1257
+ handedOffAtMs: message.handedOffAtMs,
1258
+ handoffId: message.handoffId,
1259
+ jobId: message.jobId,
1260
+ receivedAtMs: message.receivedAtMs,
1261
+ startedAtMs: message.startedAtMs,
1262
+ workerId: message.workerId
1263
+ })
1264
+ jsonSocket.send({type: "job-updated", jobId: message.jobId})
1265
+ } catch (error) {
1266
+ this._reportJobUpdateFailure({error, jobId: message.jobId, stage: "background-job-accepted"})
1267
+ jsonSocket.send({type: "job-update-error", jobId: message.jobId, error: "Failed to update job"})
1268
+ }
1269
+ }
1270
+
1237
1271
  /**
1238
1272
  * Requires the complete durable lease identity before a generation-mode
1239
1273
  * reporter can mutate a job. Legacy reporters keep their permissive protocol.
@@ -1241,7 +1275,7 @@ export default class BackgroundJobsMain {
1241
1275
  * @returns {boolean} - Whether the report lacks its exact generation lease.
1242
1276
  */
1243
1277
  _generationReportIsInvalid(message) {
1244
- if (message?.type !== "job-complete" && message?.type !== "job-failed" && message?.type !== "job-reschedule") return false
1278
+ if (message?.type !== "job-accepted" && message?.type !== "job-complete" && message?.type !== "job-failed" && message?.type !== "job-reschedule") return false
1245
1279
  const generationId = this.generationId
1246
1280
  if (!generationId) return false
1247
1281
 
@@ -1,5 +1,6 @@
1
1
  // @ts-check
2
2
 
3
+ import { randomUUID } from "node:crypto"
3
4
  import runJobPayload, { BackgroundJobPerformedFailure } from "./job-runner.js"
4
5
  import { closeRunnerConnections, closeRunnerFrameworkConnections, currentConfigurationOrNull } from "./runner-graceful-shutdown.js"
5
6
  import setRunnerProcessTitle from "./runner-process-title.js"
@@ -7,6 +8,8 @@ import PooledRunnerBrokerIdentity from "./pooled-runner-broker-identity.js"
7
8
  import { runWithSharedTransactionBrokerConfig } from "../testing/shared-transaction-proxy-driver.js"
8
9
 
9
10
  const BASE_PROCESS_TITLE = "velocious background-jobs-runner"
11
+ /** Stable identity of this pooled child process for the life of the process. */
12
+ const childInstanceId = randomUUID()
10
13
 
11
14
  setRunnerProcessTitle()
12
15
 
@@ -57,6 +60,39 @@ function updateProcessTitle() {
57
60
  process.title = count > 0 ? `${BASE_PROCESS_TITLE}: ${count} ${count === 1 ? "job" : "jobs"}` : BASE_PROCESS_TITLE
58
61
  }
59
62
 
63
+ /**
64
+ * Reports one acceptance observation (job received / perform started) to the
65
+ * worker over IPC. The message carries the job's exact handoff lease so the
66
+ * worker can persist it fenced without any other lookup. Send failures are
67
+ * swallowed: a dead IPC channel is terminal for this child (the disconnect
68
+ * handler owns shutdown), and losing acceptance evidence must never fail the
69
+ * job itself.
70
+ * @param {"job-received" | "job-started"} type - Observation kind.
71
+ * @param {import("./types.js").BackgroundJobPayload & {id: string}} payload - Job payload carrying the handoff lease.
72
+ * @param {number} observedAtMs - Epoch ms of the observation.
73
+ * @returns {void}
74
+ */
75
+ function sendChildAcceptance(type, payload, observedAtMs) {
76
+ if (!process.send) return
77
+
78
+ const message = {
79
+ childInstanceId,
80
+ childPid: process.pid,
81
+ handedOffAtMs: payload.handedOffAtMs,
82
+ handoffId: payload.handoffId,
83
+ jobId: payload.id,
84
+ type,
85
+ workerId: payload.workerId,
86
+ ...(type === "job-received" ? {receivedAtMs: observedAtMs} : {startedAtMs: observedAtMs})
87
+ }
88
+
89
+ try {
90
+ process.send(message)
91
+ } catch {
92
+ // The IPC channel is already gone; the disconnect handler owns shutdown.
93
+ }
94
+ }
95
+
60
96
  /**
61
97
  * Checks whether an IPC value is a runnable pooled job message.
62
98
  * @param {ReturnType<typeof JSON.parse>} message - IPC message.
@@ -114,6 +150,7 @@ async function runJob(payload, sharedTransactionBroker) {
114
150
  return await runJobPayload(payload, {
115
151
  closeConnections: false,
116
152
  manageProcessTitle: false,
153
+ onPerformStart: () => sendChildAcceptance("job-started", payload, Date.now()),
117
154
  processType: "background-jobs-pooled-runner"
118
155
  })
119
156
  })
@@ -143,6 +180,7 @@ function handleMessage(message) {
143
180
 
144
181
  runningJobIds.add(message.payload.id)
145
182
  updateProcessTitle()
183
+ sendChildAcceptance("job-received", message.payload, Date.now())
146
184
  void runJob(message.payload, message.sharedTransactionBroker || {expected: false})
147
185
  }
148
186
 
@@ -92,6 +92,102 @@ export default class BackgroundJobsStatusReporter {
92
92
  })
93
93
  }
94
94
 
95
+ /**
96
+ * Runs report child accepted.
97
+ * @param {object} args - Options.
98
+ * @param {string} args.jobId - Job id.
99
+ * @param {string} [args.handoffId] - Handoff lease id.
100
+ * @param {string} [args.workerId] - Worker id.
101
+ * @param {number} [args.handedOffAtMs] - Handed off timestamp.
102
+ * @param {number} [args.receivedAtMs] - Epoch ms the runner child received the job.
103
+ * @param {number} [args.startedAtMs] - Epoch ms the job's perform started in the child.
104
+ * @param {string} [args.childInstanceId] - Stable pooled child identity.
105
+ * @param {number} [args.childPid] - Pooled child OS pid.
106
+ * @returns {Promise<void>} - Resolves when reported.
107
+ */
108
+ async reportChildAccepted({jobId, handoffId, workerId, handedOffAtMs, receivedAtMs, startedAtMs, childInstanceId, childPid}) {
109
+ const config = this.configuration.getBackgroundJobsConfig()
110
+ const host = this.host || config.host
111
+ const port = typeof this.port === "number" ? this.port : config.port
112
+ const {generationId} = this.configuration.resolveBackgroundJobsGenerationConfig({
113
+ generationId: this.explicitGenerationId,
114
+ sourceName: "BackgroundJobsStatusReporter"
115
+ })
116
+
117
+ await timeout({timeout: this.attemptTimeoutMs}, async ({control}) => {
118
+ const request = new BackgroundJobsSocketRequest({host, port, role: "reporter", generationHandshakeTimeoutMs: this.generationHandshakeTimeoutMs, generationId})
119
+
120
+ this._lastRequest = request
121
+
122
+ await request.run({
123
+ signal: control.signal,
124
+ onConnect: (jsonSocket) => {
125
+ jsonSocket.send({
126
+ type: "job-accepted",
127
+ jobId,
128
+ handoffId,
129
+ workerId,
130
+ handedOffAtMs,
131
+ receivedAtMs,
132
+ startedAtMs,
133
+ childInstanceId,
134
+ childPid
135
+ })
136
+ },
137
+ onMessage: ({message, resolve, reject}) => {
138
+ if (message?.type === "job-updated" && message.jobId === jobId) {
139
+ resolve(undefined)
140
+ return
141
+ }
142
+
143
+ if (message?.type === "job-update-error" && message.jobId === jobId) {
144
+ reject(new BackgroundJobUpdateError(message.error || "Job update failed"))
145
+ }
146
+ }
147
+ })
148
+ })
149
+ }
150
+
151
+ /**
152
+ * Runs report child accepted with retry. Acceptance evidence is diagnostic
153
+ * rather than terminal, so a transient main/DB failure retries only until
154
+ * `maxDurationMs` elapses and then gives up instead of stranding the report.
155
+ * @param {object} args - Options.
156
+ * @param {string} args.jobId - Job id.
157
+ * @param {string} [args.handoffId] - Handoff lease id.
158
+ * @param {string} [args.workerId] - Worker id.
159
+ * @param {number} [args.handedOffAtMs] - Handed off timestamp.
160
+ * @param {number} [args.receivedAtMs] - Epoch ms the runner child received the job.
161
+ * @param {number} [args.startedAtMs] - Epoch ms the job's perform started in the child.
162
+ * @param {string} [args.childInstanceId] - Stable pooled child identity.
163
+ * @param {number} [args.childPid] - Pooled child OS pid.
164
+ * @param {number} args.maxDurationMs - Max duration for retries.
165
+ * @returns {Promise<void>} - Resolves when reported or the budget elapses.
166
+ */
167
+ async reportChildAcceptedWithRetry({jobId, handoffId, workerId, handedOffAtMs, receivedAtMs, startedAtMs, childInstanceId, childPid, maxDurationMs}) {
168
+ let attempt = 0
169
+ const startTime = Date.now()
170
+
171
+ while (true) {
172
+ try {
173
+ await this.reportChildAccepted({jobId, handoffId, workerId, handedOffAtMs, receivedAtMs, startedAtMs, childInstanceId, childPid})
174
+ return
175
+ } catch (error) {
176
+ attempt += 1
177
+ const delaySeconds = Math.min(30, 0.5 * attempt)
178
+
179
+ this.logger.debug(() => ["Background job child-acceptance report failed, retrying", error])
180
+
181
+ if (Date.now() - startTime >= maxDurationMs) {
182
+ this.logger.warn(() => ["Background job child-acceptance report timed out, giving up", error])
183
+ throw error
184
+ }
185
+
186
+ await wait(delaySeconds)
187
+ }
188
+ }
189
+ }
190
+
95
191
  /**
96
192
  * Runs report with retry.
97
193
  * @param {object} args - Options.
@@ -104,7 +200,7 @@ export default class BackgroundJobsStatusReporter {
104
200
  * @param {string} [args.workerId] - Worker id.
105
201
  * @param {import("./types.js").PooledRunnerFailure} [args.runnerFailure] - Pooled-child process failure provenance.
106
202
  * @param {number} [args.maxDurationMs] - Max duration for retries.
107
- * @param {boolean} [args.retryPersistErrors] - Retry a `BackgroundJobUpdateError` (main's `job-update-error`, i.e. a transient DB failure while persisting the terminal status) instead of throwing immediately. Off by default so short-lived forked/spawned runners keep failing loudly and exit non-zero to be reclaimed; on for the long-lived worker, which cannot exit-to-reclaim and would otherwise strand the job in `handed_off`.
203
+ * @param {boolean} [args.retryPersistErrors] - Retry a `BackgroundJobUpdateError` (main's `job-update-error`, i.e. a transient DB failure while persisting the terminal status) instead of throwing immediately. Off by default so short-lived forked/spawned runners keep failing loudly and exit non-zero to be reclaimed; on for the long-lived worker, which cannot exit to trigger orphan reclaim and would otherwise drop the completion and strand the row in `handed_off`.
108
204
  * @returns {Promise<void>} - Resolves when reported.
109
205
  */
110
206
  async reportWithRetry({jobId, status, delayMs, error, handoffId, handedOffAtMs, workerId, runnerFailure, maxDurationMs, retryPersistErrors = false}) {