velocious 1.0.661 → 1.0.663

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. package/README.md +12 -1
  2. package/build/application.js +1 -1
  3. package/build/background-jobs/adapter.js +9 -0
  4. package/build/background-jobs/job-runner.js +3 -1
  5. package/build/background-jobs/local-adapter.js +7 -0
  6. package/build/background-jobs/local-store.js +113 -3
  7. package/build/background-jobs/main.js +35 -1
  8. package/build/background-jobs/pooled-runner-child.js +38 -0
  9. package/build/background-jobs/status-reporter.js +97 -1
  10. package/build/background-jobs/store.js +130 -4
  11. package/build/background-jobs/types.js +6 -1
  12. package/build/background-jobs/web/controller.js +4 -0
  13. package/build/background-jobs/worker.js +59 -0
  14. package/build/configuration-types.js +3 -1
  15. package/build/configuration.js +33 -0
  16. package/build/http-server/client/errors.js +27 -0
  17. package/build/http-server/client/index.js +26 -1
  18. package/build/http-server/client/request-buffer/index.js +38 -6
  19. package/build/http-server/client/request-runner.js +8 -1
  20. package/build/http-server/client/response.js +11 -0
  21. package/build/routes/resolver.js +1 -1
  22. package/build/src/application.js +2 -2
  23. package/build/src/background-jobs/adapter.d.ts +17 -0
  24. package/build/src/background-jobs/adapter.d.ts.map +1 -1
  25. package/build/src/background-jobs/adapter.js +9 -1
  26. package/build/src/background-jobs/job-runner.d.ts +3 -1
  27. package/build/src/background-jobs/job-runner.d.ts.map +1 -1
  28. package/build/src/background-jobs/job-runner.js +5 -2
  29. package/build/src/background-jobs/local-adapter.d.ts +13 -0
  30. package/build/src/background-jobs/local-adapter.d.ts.map +1 -1
  31. package/build/src/background-jobs/local-adapter.js +7 -1
  32. package/build/src/background-jobs/local-store.d.ts +38 -0
  33. package/build/src/background-jobs/local-store.d.ts.map +1 -1
  34. package/build/src/background-jobs/local-store.js +117 -4
  35. package/build/src/background-jobs/main.d.ts +14 -0
  36. package/build/src/background-jobs/main.d.ts.map +1 -1
  37. package/build/src/background-jobs/main.js +35 -2
  38. package/build/src/background-jobs/pooled-runner-child.js +38 -1
  39. package/build/src/background-jobs/status-reporter.d.ts +51 -1
  40. package/build/src/background-jobs/status-reporter.d.ts.map +1 -1
  41. package/build/src/background-jobs/status-reporter.js +89 -2
  42. package/build/src/background-jobs/store.d.ts +48 -0
  43. package/build/src/background-jobs/store.d.ts.map +1 -1
  44. package/build/src/background-jobs/store.js +128 -5
  45. package/build/src/background-jobs/types.d.ts +34 -2
  46. package/build/src/background-jobs/types.d.ts.map +1 -1
  47. package/build/src/background-jobs/types.js +7 -2
  48. package/build/src/background-jobs/web/controller.d.ts.map +1 -1
  49. package/build/src/background-jobs/web/controller.js +5 -1
  50. package/build/src/background-jobs/worker.d.ts +20 -0
  51. package/build/src/background-jobs/worker.d.ts.map +1 -1
  52. package/build/src/background-jobs/worker.js +58 -1
  53. package/build/src/configuration-types.d.ts +13 -3
  54. package/build/src/configuration-types.d.ts.map +1 -1
  55. package/build/src/configuration-types.js +4 -2
  56. package/build/src/configuration.d.ts +14 -2
  57. package/build/src/configuration.d.ts.map +1 -1
  58. package/build/src/configuration.js +31 -1
  59. package/build/src/http-server/client/errors.d.ts +25 -0
  60. package/build/src/http-server/client/errors.d.ts.map +1 -0
  61. package/build/src/http-server/client/errors.js +26 -0
  62. package/build/src/http-server/client/index.d.ts +5 -0
  63. package/build/src/http-server/client/index.d.ts.map +1 -1
  64. package/build/src/http-server/client/index.js +26 -2
  65. package/build/src/http-server/client/request-buffer/index.d.ts +13 -0
  66. package/build/src/http-server/client/request-buffer/index.d.ts.map +1 -1
  67. package/build/src/http-server/client/request-buffer/index.js +34 -8
  68. package/build/src/http-server/client/request-runner.d.ts.map +1 -1
  69. package/build/src/http-server/client/request-runner.js +10 -2
  70. package/build/src/http-server/client/response.d.ts.map +1 -1
  71. package/build/src/http-server/client/response.js +8 -1
  72. package/build/src/routes/resolver.js +2 -2
  73. package/package.json +1 -1
  74. package/src/application.js +1 -1
  75. package/src/background-jobs/adapter.js +9 -0
  76. package/src/background-jobs/job-runner.js +3 -1
  77. package/src/background-jobs/local-adapter.js +7 -0
  78. package/src/background-jobs/local-store.js +113 -3
  79. package/src/background-jobs/main.js +35 -1
  80. package/src/background-jobs/pooled-runner-child.js +38 -0
  81. package/src/background-jobs/status-reporter.js +97 -1
  82. package/src/background-jobs/store.js +130 -4
  83. package/src/background-jobs/types.js +6 -1
  84. package/src/background-jobs/web/controller.js +4 -0
  85. package/src/background-jobs/worker.js +59 -0
  86. package/src/configuration-types.js +3 -1
  87. package/src/configuration.js +33 -0
  88. package/src/http-server/client/errors.js +27 -0
  89. package/src/http-server/client/index.js +26 -1
  90. package/src/http-server/client/request-buffer/index.js +38 -6
  91. package/src/http-server/client/request-runner.js +8 -1
  92. package/src/http-server/client/response.js +11 -0
  93. package/src/routes/resolver.js +1 -1
package/README.md CHANGED
@@ -34,6 +34,7 @@
34
34
  * Cross-process broadcast bus for `broadcastToChannel` via `velocious beacon`, including background job runner processes (see [docs/beacon.md](docs/beacon.md))
35
35
  * Rails-style application process initializer teardown with immutable process identity, reverse idempotent shutdown, and explicit HTTP/background-job ownership (see [docs/application-process-lifecycle.md](docs/application-process-lifecycle.md))
36
36
  * Configurable HTTP server worker handlers plus backpressured, descriptor-only file responses with completion callbacks (see [docs/http-server.md](docs/http-server.md))
37
+ * Explicit database-free in-process HTTP applications with optional request and buffered-response byte limits (see [docs/http-server.md](docs/http-server.md#database-free-applications))
37
38
  * Default-on buffered HTTP response compression with Brotli/gzip content negotiation, global and per-response opt-outs, and HEAD-correct representation headers (see [docs/http-server.md](docs/http-server.md#response-compression))
38
39
  * Background jobs with Node SQL/TCP workers plus a Browser/Expo local SQLite store and in-process dispatcher, including failure events, authorized database-scoped dashboard counts, and an opt-in release-scoped main/worker generation protocol with acknowledged activation, asynchronous retirement, and retired-main recovery. Production compliance additionally requires downstream supervisor retention/activation ordering and release pins (see [docs/background-jobs.md](docs/background-jobs.md), [docs/local-background-jobs.md](docs/local-background-jobs.md), and [docs/background-jobs-dashboard.md](docs/background-jobs-dashboard.md))
39
40
  * Durable one-off background-job scheduling with exact epoch timestamps (see [docs/scheduled-background-job-enqueue.md](docs/scheduled-background-job-enqueue.md))
@@ -2613,7 +2614,7 @@ actual state sources must still agree.
2613
2614
 
2614
2615
  `maxConcurrentInlineJobs` (default: `4`) caps how many `executionMode: "inline"` jobs a single `background-jobs-worker` process runs in parallel. Concurrency is at the JS event-loop level: every job in flight shares the worker's process and DB connection pool, so the cap should fit the pool, not the CPU count. Forking remains the right tool when you need memory isolation across long-running jobs or want to use more cores; select it with `executionMode: "forked"`.
2615
2616
 
2616
- New jobs default to `executionMode: "pooled"`: a worker runs them in warm, reusable Node child runners. `pooledRunnerCount` (default: `4`) bounds this independent per-worker pool, and `pooledRunnerConcurrency` (default: `1`) sets how many jobs each child runs at once on its own event loop, so total pooled capacity is `pooledRunnerCount × pooledRunnerConcurrency` — raise concurrency for I/O-bound jobs to get high throughput from a bounded, isolated set of processes. Workers advertise that exact free-slot count and the main consumes one slot per durable handoff, allowing one readiness notification to fill the pool. If an initialized child exits unexpectedly, the worker immediately advertises the freed capacity while failure reports retry; the replacement is spawned lazily by the next dispatch, and a pre-startup crash does not trigger a respawn loop. `pooledRunnerCount`, `pooledRunnerConcurrency`, and `pooledRunnerMaxJobs` must be finite positive integers; the RSS and lifetime limits must be finite positive numbers. A child is recycled after an acknowledged job when it reaches `pooledRunnerMaxJobs` (default: `100`), `pooledRunnerMaxRssBytes` (default: `536870912`, or 512 MiB), or `pooledRunnerMaxLifetimeMs` (default: `3600000`, or one hour). `execution_mode` is the single source of truth for a job's runtime — pooled rows persist as `execution_mode = "pooled"` directly. See [execution modes and pooled runners](docs/background-jobs.md#execution-modes-and-pooled-runners).
2617
+ New jobs default to `executionMode: "pooled"`: a worker runs them in warm, reusable Node child runners. `pooledRunnerCount` (default: `4`) bounds this independent per-worker pool, and `pooledRunnerConcurrency` (default: `1`) sets how many jobs each child runs at once on its own event loop, so total pooled capacity is `pooledRunnerCount × pooledRunnerConcurrency` — raise concurrency for I/O-bound jobs to get high throughput from a bounded, isolated set of processes. Workers advertise that exact free-slot count and the main consumes one slot per durable handoff, allowing one readiness notification to fill the pool. If an initialized child exits unexpectedly, the worker immediately advertises the freed capacity while failure reports retry; the replacement is spawned lazily by the next dispatch, and a pre-startup crash does not trigger a respawn loop. `pooledRunnerCount`, `pooledRunnerConcurrency`, and `pooledRunnerMaxJobs` must be finite positive integers; the RSS and lifetime limits must be finite positive numbers. A child is recycled after an acknowledged job when it reaches `pooledRunnerMaxJobs` (default: `100`), `pooledRunnerMaxRssBytes` (default: `536870912`, or 512 MiB), or `pooledRunnerMaxLifetimeMs` (default: `3600000`, or one hour). `execution_mode` is the single source of truth for a job's runtime — pooled rows persist as `execution_mode = "pooled"` directly. Pooled jobs also persist acceptance evidence (`child_received_at_ms`, `child_started_at_ms`, `child_instance_id`, `child_pid`) so a handed-off job can be told apart from one whose runner never picked it up. See [execution modes and pooled runners](docs/background-jobs.md#execution-modes-and-pooled-runners).
2617
2618
 
2618
2619
  Cold pooled jobs share one atomic model-bootstrap phase. If that phase fails, all
2619
2620
  waiting jobs receive the failure; a later job in the surviving child cannot run
@@ -2907,6 +2908,16 @@ When the server runs in the `development` environment, Velocious watches applica
2907
2908
 
2908
2909
  Starting the HTTP server creates `tmp/server.lock` under the configured application directory before Beacon, workers, or the TCP listener start. A second server for the same app fails fast with the lock owner details instead of partially starting. Normal shutdown removes the lock; stale locks with a dead local PID are reclaimed automatically, while locks from another host or unreadable metadata should be removed manually only after confirming no server is running. See [docs/http-server.md](docs/http-server.md#server-lock).
2909
2910
 
2911
+ Set `database: false` for an explicitly database-free application; its HTTP and
2912
+ initializer lifecycle remains intact without creating a default pool or an
2913
+ implicit route connection checkout. Optional
2914
+ `httpServer.maxRequestBodyBytes` and
2915
+ `httpServer.maxBufferedResponseBodyBytes` positive byte limits reject oversized
2916
+ request bodies with a connection-closing `413` and oversized buffered responses
2917
+ with a reported, empty `500`. Both remain unbounded when omitted, and streamed
2918
+ `sendFile` responses are unaffected. The request limit is enforced while
2919
+ unframed multipart bodies accumulate. See [docs/http-server.md](docs/http-server.md#database-free-applications).
2920
+
2910
2921
  Buffered string and `Uint8Array` responses are compressed with Brotli (`br`) or gzip by default whenever request negotiation and response eligibility allow — no opt-in is required. Disable compression globally with `httpServer.compression: false` or `httpServer.compression: {enabled: false}`, and tune it with `threshold`/`brotliQuality`/`gzipLevel` overrides. Negotiation honors `Accept-Encoding` q-values, wildcards, and identity semantics (empty `406` when no acceptable representation exists), combines repeated `Accept-Encoding` header fields in wire order, and advertises a framework-owned `Vary: Accept-Encoding` on every framework-selected representation (transformed, identity, `406`, and file) so caches key on the field. Streamed `sendFile` responses are never buffered or re-encoded: they are sent identity, so when the client forbids identity the server answers with the same empty `406` without opening or streaming the file. Transformation also skips already-encoded or `no-transform` responses, server-sent events, partial/range responses, bodyless statuses, and non-allowlisted content types. Transformation is additionally excluded automatically for credentialed traffic and validator-carrying responses — requests with `Authorization`/`Cookie` and responses with `Set-Cookie`, `ETag`, `Digest`, or `Content-Digest` are never compressed (compression-oracle protection, and validators stay application-owned). Controllers opt out per response with `response.disableCompression()`, and HEAD requests compute GET-equivalent representation headers without emitting a body. See [docs/http-server.md](docs/http-server.md#response-compression).
2911
2922
 
2912
2923
  # Authorization (CanCan-style)
@@ -66,7 +66,7 @@ export default class VelociousApplication {
66
66
 
67
67
  this.configuration.setRoutes(routes)
68
68
 
69
- if (!this.configuration.isDatabasePoolInitialized()) {
69
+ if (this.configuration.database !== false && !this.configuration.isDatabasePoolInitialized()) {
70
70
  await this.configuration.initializeDatabasePool()
71
71
  }
72
72
 
@@ -129,6 +129,15 @@ export default class BackgroundJobsAdapter {
129
129
  */
130
130
  async markCompleted(_args) { throw new Error("BackgroundJobsAdapter#markCompleted is not implemented") }
131
131
 
132
+ /**
133
+ * Records pooled-child acceptance evidence (received/started timestamps plus
134
+ * runner identity) for a handed-off job, fenced by its active handoff lease.
135
+ * Only the fields supplied are written.
136
+ * @param {{jobId: string, handoffId?: string, workerId?: string, handedOffAtMs?: number, receivedAtMs?: number, startedAtMs?: number, childInstanceId?: string, childPid?: number}} _args - Acceptance report.
137
+ * @returns {Promise<boolean>} - Whether the fenced report was accepted.
138
+ */
139
+ async markChildAccepted(_args) { throw new Error("BackgroundJobsAdapter#markChildAccepted is not implemented") }
140
+
132
141
  /**
133
142
  * Returns a handed-off job to its schedule.
134
143
  * @param {{jobId: string, delayMs: number, handoffId?: string, workerId?: string, handedOffAtMs?: number}} _args - Reschedule report.
@@ -82,10 +82,11 @@ function runnerProcessTitle(JobClass, payload) {
82
82
  * @param {object} [options] - Runner options.
83
83
  * @param {boolean} [options.closeConnections] - Whether to gracefully close framework connections after the job.
84
84
  * @param {boolean} [options.manageProcessTitle] - Whether to set the per-job process title and restore it afterwards. Off for concurrent pooled runners, where interleaved snapshot/restore of the single process-wide `process.title` would corrupt it; the pooled child owns an aggregate title instead.
85
+ * @param {() => void} [options.onPerformStart] - Observation hook fired once, immediately before perform runs (with its connection acquired). Pooled runners use it to report that the job actually started. Must not throw into the job.
85
86
  * @param {string} [options.processType] - Generic application process type.
86
87
  * @returns {Promise<"completed" | "rescheduled">} - Acknowledged outcome.
87
88
  */
88
- export default async function runJobPayload(payload, {closeConnections = true, manageProcessTitle = true, processType = "background-jobs-runner"} = {}) {
89
+ export default async function runJobPayload(payload, {closeConnections = true, manageProcessTitle = true, onPerformStart, processType = "background-jobs-runner"} = {}) {
89
90
  const configuration = await configurationResolver()
90
91
  configuration.setCurrent()
91
92
  await configuration.initialize({type: processType})
@@ -119,6 +120,7 @@ export default async function runJobPayload(payload, {closeConnections = true, m
119
120
  try {
120
121
  await runWithBackgroundJobPayload(payload, async () => {
121
122
  await configuration.withConnections({databaseIdentifiers: JobClass.databaseIdentifiers, name: `Background job runner: ${payload.jobName}`}, async () => {
123
+ if (onPerformStart) onPerformStart()
122
124
  await perform.apply(jobInstance, jobArgs)
123
125
  })
124
126
  })
@@ -133,6 +133,13 @@ export default class LocalBackgroundJobsAdapter extends BackgroundJobsAdapter {
133
133
  */
134
134
  async markCompleted(args) { return await this.store.markCompleted(args) }
135
135
 
136
+ /**
137
+ * Records pooled-child acceptance evidence for a local handoff.
138
+ * @param {{jobId: string, handoffId?: string, receivedAtMs?: number, startedAtMs?: number, childInstanceId?: string, childPid?: number}} args - Acceptance report.
139
+ * @returns {Promise<boolean>} - Whether accepted.
140
+ */
141
+ async markChildAccepted(args) { return await this.store.markChildAccepted(args) }
142
+
136
143
  /**
137
144
  * Acknowledges an explicit local reschedule.
138
145
  * @param {{jobId: string, delayMs: number, handoffId?: string}} args - Reschedule report.
@@ -49,7 +49,11 @@ const EXPECTED_JOB_COLUMNS = [
49
49
  "failed_at_ms",
50
50
  "last_error",
51
51
  "concurrency_key",
52
- "max_concurrency"
52
+ "max_concurrency",
53
+ "child_received_at_ms",
54
+ "child_started_at_ms",
55
+ "child_instance_id",
56
+ "child_pid"
53
57
  ]
54
58
  const EXPECTED_CONCURRENCY_COLUMNS = ["concurrency_key", "max_concurrency", "active_count"]
55
59
  /** @type {WeakMap<import("../configuration.js").default, Map<string, Promise<void>>>} */
@@ -202,6 +206,7 @@ export default class LocalBackgroundJobsStore {
202
206
  await db.createTable(this._jobsTableData())
203
207
  changed = true
204
208
  } else {
209
+ if (await this._ensureJobColumns(db)) changed = true
205
210
  await this._assertColumns(db, LOCAL_BACKGROUND_JOBS_TABLE, EXPECTED_JOB_COLUMNS)
206
211
  }
207
212
 
@@ -232,6 +237,45 @@ export default class LocalBackgroundJobsStore {
232
237
  return changed
233
238
  }
234
239
 
240
+ /**
241
+ * Idempotently adds columns from the current jobs table definition that an
242
+ * existing local table is missing, so an upgraded framework finds a
243
+ * compatible schema instead of failing the column assertion.
244
+ * @param {import("../database/drivers/base.js").default} db - Local SQLite connection.
245
+ * @returns {Promise<boolean>} - Whether a column was added.
246
+ */
247
+ async _ensureJobColumns(db) {
248
+ db.clearSchemaCache()
249
+ const table = await db.getTableByNameOrFail(LOCAL_BACKGROUND_JOBS_TABLE)
250
+ const tableData = new TableData(LOCAL_BACKGROUND_JOBS_TABLE)
251
+ let added = false
252
+
253
+ for (const column of this._jobsTableData().getColumns()) {
254
+ if (await table.getColumnByName(column.getName())) continue
255
+ if (column.getPrimaryKey()) continue
256
+
257
+ const columnArgs = /** @type {{null: boolean, maxLength?: number}} */ ({null: column.getNull() !== false})
258
+ const maxLength = column.getMaxLength()
259
+
260
+ if (typeof maxLength === "number") columnArgs.maxLength = maxLength
261
+
262
+ const type = column.getType()
263
+ if (type === "string") tableData.string(column.getName(), columnArgs)
264
+ else if (type === "text") tableData.text(column.getName(), columnArgs)
265
+ else if (type === "bigint") tableData.bigint(column.getName(), columnArgs)
266
+ else if (type === "integer") tableData.integer(column.getName(), columnArgs)
267
+ else if (type === "boolean") tableData.boolean(column.getName(), columnArgs)
268
+ else continue
269
+ added = true
270
+ }
271
+
272
+ if (!added) return false
273
+
274
+ for (const sql of await db.alterTableSQLs(tableData)) await db.query(sql)
275
+ db.clearSchemaCache()
276
+ return true
277
+ }
278
+
235
279
  /**
236
280
  * Builds the migration ledger table definition.
237
281
  * @returns {TableData} - Migration ledger table.
@@ -272,6 +316,10 @@ export default class LocalBackgroundJobsStore {
272
316
  table.text("last_error", {null: true})
273
317
  table.string("concurrency_key", {null: true})
274
318
  table.integer("max_concurrency", {null: true})
319
+ table.bigint("child_received_at_ms", {null: true})
320
+ table.bigint("child_started_at_ms", {null: true})
321
+ table.string("child_instance_id", {null: true})
322
+ table.integer("child_pid", {null: true})
275
323
  table.addIndex(new TableIndex(["status", "scheduled_at_ms", "created_at_ms", "id"], {name: LOCAL_BACKGROUND_JOBS_INDEX_NAMES[0]}))
276
324
  table.addIndex(new TableIndex(["queue", "status", "created_at_ms"], {name: LOCAL_BACKGROUND_JOBS_INDEX_NAMES[1]}))
277
325
  table.addIndex(new TableIndex(["args_digest"], {name: LOCAL_BACKGROUND_JOBS_INDEX_NAMES[2]}))
@@ -708,7 +756,7 @@ export default class LocalBackgroundJobsStore {
708
756
  const handedOffAtMs = this.clock.now()
709
757
  const affectedRows = await this._updateAffectedRows(db, {
710
758
  conditions: {id: jobId, status: "queued"},
711
- data: {handed_off_at_ms: handedOffAtMs, handoff_id: handoffId, status: "handed_off", worker_id: workerId || "local"},
759
+ data: {...this._clearedChildAcceptanceData(), handed_off_at_ms: handedOffAtMs, handoff_id: handoffId, status: "handed_off", worker_id: workerId || "local"},
712
760
  tableName: LOCAL_BACKGROUND_JOBS_TABLE
713
761
  })
714
762
 
@@ -763,6 +811,7 @@ export default class LocalBackgroundJobsStore {
763
811
  const affectedRows = await this._updateAffectedRows(db, {
764
812
  conditions: {handoff_id: handoffId, id: jobId, status: "handed_off"},
765
813
  data: {
814
+ ...this._clearedChildAcceptanceData(),
766
815
  handed_off_at_ms: null,
767
816
  handoff_id: null,
768
817
  scheduled_at_ms: this.clock.now(),
@@ -802,6 +851,43 @@ export default class LocalBackgroundJobsStore {
802
851
  }))
803
852
  }
804
853
 
854
+ /**
855
+ * Records pooled-child acceptance evidence for an active handoff. Only the
856
+ * fields supplied are written, fenced by the exact active handoff lease.
857
+ * @param {object} args - Acceptance report.
858
+ * @param {string} args.jobId - Job id.
859
+ * @param {string} [args.handoffId] - Handoff lease id.
860
+ * @param {number} [args.receivedAtMs] - Epoch ms the runner child received the job.
861
+ * @param {number} [args.startedAtMs] - Epoch ms the job's perform started in the child.
862
+ * @param {string} [args.childInstanceId] - Stable pooled child identity.
863
+ * @param {number} [args.childPid] - Pooled child OS pid.
864
+ * @returns {Promise<boolean>} - Whether the lease won.
865
+ */
866
+ async markChildAccepted({jobId, handoffId, receivedAtMs, startedAtMs, childInstanceId, childPid}) {
867
+ await this.ensureReady()
868
+
869
+ return await this._withDb(async (connection) => await this._mutate(connection, async (db) => {
870
+ const job = await this._getJob(db, jobId)
871
+
872
+ if (!this._acceptsHandoff(job, handoffId)) return false
873
+
874
+ const data = {}
875
+ if (typeof receivedAtMs === "number") data.child_received_at_ms = receivedAtMs
876
+ if (typeof startedAtMs === "number") data.child_started_at_ms = startedAtMs
877
+ if (typeof childInstanceId === "string") data.child_instance_id = childInstanceId
878
+ if (typeof childPid === "number") data.child_pid = childPid
879
+ if (Object.keys(data).length === 0) return false
880
+
881
+ const affectedRows = await this._updateAffectedRows(db, {
882
+ conditions: {handoff_id: handoffId, id: jobId, status: "handed_off"},
883
+ data,
884
+ tableName: LOCAL_BACKGROUND_JOBS_TABLE
885
+ })
886
+
887
+ return affectedRows === 1
888
+ }))
889
+ }
890
+
805
891
  /**
806
892
  * Applies a fenced reschedule without consuming an attempt.
807
893
  * @param {{jobId: string, handoffId?: string, delayMs: number}} args - Reschedule report.
@@ -819,6 +905,7 @@ export default class LocalBackgroundJobsStore {
819
905
  const affectedRows = await this._updateAffectedRows(db, {
820
906
  conditions: {handoff_id: handoffId, id: jobId, status: "handed_off"},
821
907
  data: {
908
+ ...this._clearedChildAcceptanceData(),
822
909
  handed_off_at_ms: null,
823
910
  handoff_id: null,
824
911
  scheduled_at_ms: rescheduledBackgroundJobAtMs(delayMs, this.clock.now()),
@@ -917,7 +1004,9 @@ export default class LocalBackgroundJobsStore {
917
1004
  }
918
1005
 
919
1006
  if (willRetry) {
920
- Object.assign(data, {scheduled_at_ms: nowMs + retryDelayMs(attempts)})
1007
+ // A retry starts a fresh handoff with a possibly different runner, so the
1008
+ // previous child's acceptance evidence must not leak into the next attempt.
1009
+ Object.assign(data, {scheduled_at_ms: nowMs + retryDelayMs(attempts), ...this._clearedChildAcceptanceData()})
921
1010
  } else {
922
1011
  Object.assign(data, {failed_at_ms: nowMs})
923
1012
  }
@@ -934,6 +1023,7 @@ export default class LocalBackgroundJobsStore {
934
1023
 
935
1024
  return {
936
1025
  ...job,
1026
+ ...(willRetry ? this._clearedChildAcceptanceRow() : {}),
937
1027
  attempts,
938
1028
  failedAtMs: willRetry ? job.failedAtMs : nowMs,
939
1029
  handedOffAtMs: null,
@@ -945,6 +1035,22 @@ export default class LocalBackgroundJobsStore {
945
1035
  }
946
1036
  }
947
1037
 
1038
+ /**
1039
+ * Returns the database data that clears pooled-child acceptance evidence.
1040
+ * @returns {Record<string, ReturnType<typeof JSON.parse>>} - Cleared acceptance columns.
1041
+ */
1042
+ _clearedChildAcceptanceData() {
1043
+ return {child_instance_id: null, child_pid: null, child_received_at_ms: null, child_started_at_ms: null}
1044
+ }
1045
+
1046
+ /**
1047
+ * Returns the row-shape counterpart of the cleared acceptance columns.
1048
+ * @returns {Pick<import("./types.js").BackgroundJobRow, "childInstanceId" | "childPid" | "childReceivedAtMs" | "childStartedAtMs">} - Cleared acceptance fields.
1049
+ */
1050
+ _clearedChildAcceptanceRow() {
1051
+ return {childInstanceId: null, childPid: null, childReceivedAtMs: null, childStartedAtMs: null}
1052
+ }
1053
+
948
1054
  /**
949
1055
  * Ensures that a durable concurrency counter exists with the required cap.
950
1056
  * @param {import("../database/drivers/base.js").default} db - Local SQLite connection.
@@ -1106,6 +1212,10 @@ export default class LocalBackgroundJobsStore {
1106
1212
  return {
1107
1213
  args: parsedArgs,
1108
1214
  attempts: this._numberOrNull(row.attempts),
1215
+ childInstanceId: row.child_instance_id === null || row.child_instance_id === undefined ? null : String(row.child_instance_id),
1216
+ childPid: this._numberOrNull(row.child_pid),
1217
+ childReceivedAtMs: this._numberOrNull(row.child_received_at_ms),
1218
+ childStartedAtMs: this._numberOrNull(row.child_started_at_ms),
1109
1219
  completedAtMs: this._numberOrNull(row.completed_at_ms),
1110
1220
  concurrencyKey: row.concurrency_key === null || row.concurrency_key === undefined ? null : String(row.concurrency_key),
1111
1221
  createdAtMs: this._numberOrNull(row.created_at_ms),
@@ -1219,6 +1219,11 @@ export default class BackgroundJobsMain {
1219
1219
  }
1220
1220
  return
1221
1221
  }
1222
+ if (message?.type === "job-accepted") {
1223
+ await this._handleJobAccepted({jsonSocket, message})
1224
+ return
1225
+ }
1226
+
1222
1227
  if (message?.type === "job-complete") {
1223
1228
  await this._handleJobComplete({jsonSocket, message})
1224
1229
  return
@@ -1234,6 +1239,35 @@ export default class BackgroundJobsMain {
1234
1239
  }
1235
1240
  }
1236
1241
 
1242
+ /**
1243
+ * Persists pooled-child acceptance evidence for an active handoff. The
1244
+ * report is diagnostic: a stale lease (job already reclaimed or terminal)
1245
+ * answers the same `job-updated` acknowledgement as an accepted report, and
1246
+ * only a store failure answers `job-update-error` so the worker can retry.
1247
+ * @param {object} args - Options.
1248
+ * @param {JsonSocket} args.jsonSocket - JSON socket.
1249
+ * @param {import("./types.js").BackgroundJobAcceptedMessage} args.message - Message.
1250
+ * @returns {Promise<void>} - Resolves when handled.
1251
+ */
1252
+ async _handleJobAccepted({jsonSocket, message}) {
1253
+ try {
1254
+ await this.store.markChildAccepted({
1255
+ childInstanceId: message.childInstanceId,
1256
+ childPid: message.childPid,
1257
+ handedOffAtMs: message.handedOffAtMs,
1258
+ handoffId: message.handoffId,
1259
+ jobId: message.jobId,
1260
+ receivedAtMs: message.receivedAtMs,
1261
+ startedAtMs: message.startedAtMs,
1262
+ workerId: message.workerId
1263
+ })
1264
+ jsonSocket.send({type: "job-updated", jobId: message.jobId})
1265
+ } catch (error) {
1266
+ this._reportJobUpdateFailure({error, jobId: message.jobId, stage: "background-job-accepted"})
1267
+ jsonSocket.send({type: "job-update-error", jobId: message.jobId, error: "Failed to update job"})
1268
+ }
1269
+ }
1270
+
1237
1271
  /**
1238
1272
  * Requires the complete durable lease identity before a generation-mode
1239
1273
  * reporter can mutate a job. Legacy reporters keep their permissive protocol.
@@ -1241,7 +1275,7 @@ export default class BackgroundJobsMain {
1241
1275
  * @returns {boolean} - Whether the report lacks its exact generation lease.
1242
1276
  */
1243
1277
  _generationReportIsInvalid(message) {
1244
- if (message?.type !== "job-complete" && message?.type !== "job-failed" && message?.type !== "job-reschedule") return false
1278
+ if (message?.type !== "job-accepted" && message?.type !== "job-complete" && message?.type !== "job-failed" && message?.type !== "job-reschedule") return false
1245
1279
  const generationId = this.generationId
1246
1280
  if (!generationId) return false
1247
1281
 
@@ -1,5 +1,6 @@
1
1
  // @ts-check
2
2
 
3
+ import { randomUUID } from "node:crypto"
3
4
  import runJobPayload, { BackgroundJobPerformedFailure } from "./job-runner.js"
4
5
  import { closeRunnerConnections, closeRunnerFrameworkConnections, currentConfigurationOrNull } from "./runner-graceful-shutdown.js"
5
6
  import setRunnerProcessTitle from "./runner-process-title.js"
@@ -7,6 +8,8 @@ import PooledRunnerBrokerIdentity from "./pooled-runner-broker-identity.js"
7
8
  import { runWithSharedTransactionBrokerConfig } from "../testing/shared-transaction-proxy-driver.js"
8
9
 
9
10
  const BASE_PROCESS_TITLE = "velocious background-jobs-runner"
11
+ /** Stable identity of this pooled child process for the life of the process. */
12
+ const childInstanceId = randomUUID()
10
13
 
11
14
  setRunnerProcessTitle()
12
15
 
@@ -57,6 +60,39 @@ function updateProcessTitle() {
57
60
  process.title = count > 0 ? `${BASE_PROCESS_TITLE}: ${count} ${count === 1 ? "job" : "jobs"}` : BASE_PROCESS_TITLE
58
61
  }
59
62
 
63
+ /**
64
+ * Reports one acceptance observation (job received / perform started) to the
65
+ * worker over IPC. The message carries the job's exact handoff lease so the
66
+ * worker can persist it fenced without any other lookup. Send failures are
67
+ * swallowed: a dead IPC channel is terminal for this child (the disconnect
68
+ * handler owns shutdown), and losing acceptance evidence must never fail the
69
+ * job itself.
70
+ * @param {"job-received" | "job-started"} type - Observation kind.
71
+ * @param {import("./types.js").BackgroundJobPayload & {id: string}} payload - Job payload carrying the handoff lease.
72
+ * @param {number} observedAtMs - Epoch ms of the observation.
73
+ * @returns {void}
74
+ */
75
+ function sendChildAcceptance(type, payload, observedAtMs) {
76
+ if (!process.send) return
77
+
78
+ const message = {
79
+ childInstanceId,
80
+ childPid: process.pid,
81
+ handedOffAtMs: payload.handedOffAtMs,
82
+ handoffId: payload.handoffId,
83
+ jobId: payload.id,
84
+ type,
85
+ workerId: payload.workerId,
86
+ ...(type === "job-received" ? {receivedAtMs: observedAtMs} : {startedAtMs: observedAtMs})
87
+ }
88
+
89
+ try {
90
+ process.send(message)
91
+ } catch {
92
+ // The IPC channel is already gone; the disconnect handler owns shutdown.
93
+ }
94
+ }
95
+
60
96
  /**
61
97
  * Checks whether an IPC value is a runnable pooled job message.
62
98
  * @param {ReturnType<typeof JSON.parse>} message - IPC message.
@@ -114,6 +150,7 @@ async function runJob(payload, sharedTransactionBroker) {
114
150
  return await runJobPayload(payload, {
115
151
  closeConnections: false,
116
152
  manageProcessTitle: false,
153
+ onPerformStart: () => sendChildAcceptance("job-started", payload, Date.now()),
117
154
  processType: "background-jobs-pooled-runner"
118
155
  })
119
156
  })
@@ -143,6 +180,7 @@ function handleMessage(message) {
143
180
 
144
181
  runningJobIds.add(message.payload.id)
145
182
  updateProcessTitle()
183
+ sendChildAcceptance("job-received", message.payload, Date.now())
146
184
  void runJob(message.payload, message.sharedTransactionBroker || {expected: false})
147
185
  }
148
186
 
@@ -92,6 +92,102 @@ export default class BackgroundJobsStatusReporter {
92
92
  })
93
93
  }
94
94
 
95
+ /**
96
+ * Runs report child accepted.
97
+ * @param {object} args - Options.
98
+ * @param {string} args.jobId - Job id.
99
+ * @param {string} [args.handoffId] - Handoff lease id.
100
+ * @param {string} [args.workerId] - Worker id.
101
+ * @param {number} [args.handedOffAtMs] - Handed off timestamp.
102
+ * @param {number} [args.receivedAtMs] - Epoch ms the runner child received the job.
103
+ * @param {number} [args.startedAtMs] - Epoch ms the job's perform started in the child.
104
+ * @param {string} [args.childInstanceId] - Stable pooled child identity.
105
+ * @param {number} [args.childPid] - Pooled child OS pid.
106
+ * @returns {Promise<void>} - Resolves when reported.
107
+ */
108
+ async reportChildAccepted({jobId, handoffId, workerId, handedOffAtMs, receivedAtMs, startedAtMs, childInstanceId, childPid}) {
109
+ const config = this.configuration.getBackgroundJobsConfig()
110
+ const host = this.host || config.host
111
+ const port = typeof this.port === "number" ? this.port : config.port
112
+ const {generationId} = this.configuration.resolveBackgroundJobsGenerationConfig({
113
+ generationId: this.explicitGenerationId,
114
+ sourceName: "BackgroundJobsStatusReporter"
115
+ })
116
+
117
+ await timeout({timeout: this.attemptTimeoutMs}, async ({control}) => {
118
+ const request = new BackgroundJobsSocketRequest({host, port, role: "reporter", generationHandshakeTimeoutMs: this.generationHandshakeTimeoutMs, generationId})
119
+
120
+ this._lastRequest = request
121
+
122
+ await request.run({
123
+ signal: control.signal,
124
+ onConnect: (jsonSocket) => {
125
+ jsonSocket.send({
126
+ type: "job-accepted",
127
+ jobId,
128
+ handoffId,
129
+ workerId,
130
+ handedOffAtMs,
131
+ receivedAtMs,
132
+ startedAtMs,
133
+ childInstanceId,
134
+ childPid
135
+ })
136
+ },
137
+ onMessage: ({message, resolve, reject}) => {
138
+ if (message?.type === "job-updated" && message.jobId === jobId) {
139
+ resolve(undefined)
140
+ return
141
+ }
142
+
143
+ if (message?.type === "job-update-error" && message.jobId === jobId) {
144
+ reject(new BackgroundJobUpdateError(message.error || "Job update failed"))
145
+ }
146
+ }
147
+ })
148
+ })
149
+ }
150
+
151
+ /**
152
+ * Runs report child accepted with retry. Acceptance evidence is diagnostic
153
+ * rather than terminal, so a transient main/DB failure retries only until
154
+ * `maxDurationMs` elapses and then gives up instead of stranding the report.
155
+ * @param {object} args - Options.
156
+ * @param {string} args.jobId - Job id.
157
+ * @param {string} [args.handoffId] - Handoff lease id.
158
+ * @param {string} [args.workerId] - Worker id.
159
+ * @param {number} [args.handedOffAtMs] - Handed off timestamp.
160
+ * @param {number} [args.receivedAtMs] - Epoch ms the runner child received the job.
161
+ * @param {number} [args.startedAtMs] - Epoch ms the job's perform started in the child.
162
+ * @param {string} [args.childInstanceId] - Stable pooled child identity.
163
+ * @param {number} [args.childPid] - Pooled child OS pid.
164
+ * @param {number} args.maxDurationMs - Max duration for retries.
165
+ * @returns {Promise<void>} - Resolves when reported or the budget elapses.
166
+ */
167
+ async reportChildAcceptedWithRetry({jobId, handoffId, workerId, handedOffAtMs, receivedAtMs, startedAtMs, childInstanceId, childPid, maxDurationMs}) {
168
+ let attempt = 0
169
+ const startTime = Date.now()
170
+
171
+ while (true) {
172
+ try {
173
+ await this.reportChildAccepted({jobId, handoffId, workerId, handedOffAtMs, receivedAtMs, startedAtMs, childInstanceId, childPid})
174
+ return
175
+ } catch (error) {
176
+ attempt += 1
177
+ const delaySeconds = Math.min(30, 0.5 * attempt)
178
+
179
+ this.logger.debug(() => ["Background job child-acceptance report failed, retrying", error])
180
+
181
+ if (Date.now() - startTime >= maxDurationMs) {
182
+ this.logger.warn(() => ["Background job child-acceptance report timed out, giving up", error])
183
+ throw error
184
+ }
185
+
186
+ await wait(delaySeconds)
187
+ }
188
+ }
189
+ }
190
+
95
191
  /**
96
192
  * Runs report with retry.
97
193
  * @param {object} args - Options.
@@ -104,7 +200,7 @@ export default class BackgroundJobsStatusReporter {
104
200
  * @param {string} [args.workerId] - Worker id.
105
201
  * @param {import("./types.js").PooledRunnerFailure} [args.runnerFailure] - Pooled-child process failure provenance.
106
202
  * @param {number} [args.maxDurationMs] - Max duration for retries.
107
- * @param {boolean} [args.retryPersistErrors] - Retry a `BackgroundJobUpdateError` (main's `job-update-error`, i.e. a transient DB failure while persisting the terminal status) instead of throwing immediately. Off by default so short-lived forked/spawned runners keep failing loudly and exit non-zero to be reclaimed; on for the long-lived worker, which cannot exit-to-reclaim and would otherwise strand the job in `handed_off`.
203
+ * @param {boolean} [args.retryPersistErrors] - Retry a `BackgroundJobUpdateError` (main's `job-update-error`, i.e. a transient DB failure while persisting the terminal status) instead of throwing immediately. Off by default so short-lived forked/spawned runners keep failing loudly and exit non-zero to be reclaimed; on for the long-lived worker, which cannot exit to trigger orphan reclaim and would otherwise drop the completion and strand the row in `handed_off`.
108
204
  * @returns {Promise<void>} - Resolves when reported.
109
205
  */
110
206
  async reportWithRetry({jobId, status, delayMs, error, handoffId, handedOffAtMs, workerId, runnerFailure, maxDurationMs, retryPersistErrors = false}) {