velocious 1.0.630 → 1.0.632

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (197) hide show
  1. package/README.md +48 -5
  2. package/build/application.js +30 -10
  3. package/build/background-jobs/adapter.js +8 -0
  4. package/build/background-jobs/client.js +11 -2
  5. package/build/background-jobs/forked-runner-child.js +26 -30
  6. package/build/background-jobs/generation-handshake-timeout-error.js +32 -0
  7. package/build/background-jobs/generation-identity.js +160 -0
  8. package/build/background-jobs/job-runner.js +5 -7
  9. package/build/background-jobs/lifecycle-client.js +127 -0
  10. package/build/background-jobs/lifecycle-control-server.js +310 -0
  11. package/build/background-jobs/main.js +617 -124
  12. package/build/background-jobs/pooled-runner-child.js +17 -9
  13. package/build/background-jobs/runner-graceful-shutdown.js +26 -10
  14. package/build/background-jobs/socket-request.js +44 -3
  15. package/build/background-jobs/sql-adapter.js +6 -0
  16. package/build/background-jobs/status-reporter.js +11 -2
  17. package/build/background-jobs/types.js +7 -2
  18. package/build/background-jobs/worker.js +340 -48
  19. package/build/cli/commands/background-jobs/activate.js +7 -0
  20. package/build/cli/commands/background-jobs/retire.js +7 -0
  21. package/build/cli/index.js +1 -1
  22. package/build/configuration-types.js +12 -0
  23. package/build/configuration.js +276 -42
  24. package/build/database/drivers/base.js +41 -14
  25. package/build/environment-handlers/base.js +19 -1
  26. package/build/environment-handlers/node/cli/commands/background-jobs-activate.js +19 -0
  27. package/build/environment-handlers/node/cli/commands/background-jobs-main.js +71 -11
  28. package/build/environment-handlers/node/cli/commands/background-jobs-retire.js +19 -0
  29. package/build/environment-handlers/node/cli/commands/background-jobs-worker.js +53 -12
  30. package/build/environment-handlers/node/cli/commands/server.js +7 -2
  31. package/build/environment-handlers/node.js +26 -6
  32. package/build/error-reporting/request-details.js +28 -31
  33. package/build/frontend-model-controller.js +19 -13
  34. package/build/http-server/client/index.js +5 -13
  35. package/build/http-server/client/request-runner.js +26 -9
  36. package/build/http-server/client/request-timing.js +19 -0
  37. package/build/http-server/client/websocket-session.js +54 -14
  38. package/build/http-server/worker-handler/worker-thread.js +13 -6
  39. package/build/initializer.js +19 -1
  40. package/build/log-redactor.js +341 -0
  41. package/build/routes/resolver.js +32 -4
  42. package/build/src/application.d.ts +7 -0
  43. package/build/src/application.d.ts.map +1 -1
  44. package/build/src/application.js +32 -12
  45. package/build/src/background-jobs/adapter.d.ts +7 -0
  46. package/build/src/background-jobs/adapter.d.ts.map +1 -1
  47. package/build/src/background-jobs/adapter.js +8 -1
  48. package/build/src/background-jobs/client.d.ts +7 -1
  49. package/build/src/background-jobs/client.d.ts.map +1 -1
  50. package/build/src/background-jobs/client.js +12 -3
  51. package/build/src/background-jobs/forked-runner-child.js +25 -31
  52. package/build/src/background-jobs/generation-handshake-timeout-error.d.ts +25 -0
  53. package/build/src/background-jobs/generation-handshake-timeout-error.d.ts.map +1 -0
  54. package/build/src/background-jobs/generation-handshake-timeout-error.js +29 -0
  55. package/build/src/background-jobs/generation-identity.d.ts +73 -0
  56. package/build/src/background-jobs/generation-identity.d.ts.map +1 -0
  57. package/build/src/background-jobs/generation-identity.js +135 -0
  58. package/build/src/background-jobs/job-runner.d.ts +3 -1
  59. package/build/src/background-jobs/job-runner.d.ts.map +1 -1
  60. package/build/src/background-jobs/job-runner.js +6 -9
  61. package/build/src/background-jobs/lifecycle-client.d.ts +48 -0
  62. package/build/src/background-jobs/lifecycle-client.d.ts.map +1 -0
  63. package/build/src/background-jobs/lifecycle-client.js +126 -0
  64. package/build/src/background-jobs/lifecycle-control-server.d.ts +104 -0
  65. package/build/src/background-jobs/lifecycle-control-server.d.ts.map +1 -0
  66. package/build/src/background-jobs/lifecycle-control-server.js +307 -0
  67. package/build/src/background-jobs/main.d.ts +199 -18
  68. package/build/src/background-jobs/main.d.ts.map +1 -1
  69. package/build/src/background-jobs/main.js +612 -128
  70. package/build/src/background-jobs/pooled-runner-child.js +18 -11
  71. package/build/src/background-jobs/runner-graceful-shutdown.d.ts +9 -0
  72. package/build/src/background-jobs/runner-graceful-shutdown.d.ts.map +1 -1
  73. package/build/src/background-jobs/runner-graceful-shutdown.js +27 -13
  74. package/build/src/background-jobs/socket-request.d.ts +7 -1
  75. package/build/src/background-jobs/socket-request.d.ts.map +1 -1
  76. package/build/src/background-jobs/socket-request.js +42 -4
  77. package/build/src/background-jobs/sql-adapter.d.ts +5 -0
  78. package/build/src/background-jobs/sql-adapter.d.ts.map +1 -1
  79. package/build/src/background-jobs/sql-adapter.js +6 -1
  80. package/build/src/background-jobs/status-reporter.d.ts +7 -1
  81. package/build/src/background-jobs/status-reporter.d.ts.map +1 -1
  82. package/build/src/background-jobs/status-reporter.js +12 -3
  83. package/build/src/background-jobs/types.d.ts +21 -3
  84. package/build/src/background-jobs/types.d.ts.map +1 -1
  85. package/build/src/background-jobs/types.js +8 -3
  86. package/build/src/background-jobs/worker.d.ts +82 -3
  87. package/build/src/background-jobs/worker.d.ts.map +1 -1
  88. package/build/src/background-jobs/worker.js +329 -49
  89. package/build/src/cli/commands/background-jobs/activate.d.ts +5 -0
  90. package/build/src/cli/commands/background-jobs/activate.d.ts.map +1 -0
  91. package/build/src/cli/commands/background-jobs/activate.js +7 -0
  92. package/build/src/cli/commands/background-jobs/retire.d.ts +5 -0
  93. package/build/src/cli/commands/background-jobs/retire.d.ts.map +1 -0
  94. package/build/src/cli/commands/background-jobs/retire.js +7 -0
  95. package/build/src/cli/index.js +2 -2
  96. package/build/src/configuration-types.d.ts +38 -0
  97. package/build/src/configuration-types.d.ts.map +1 -1
  98. package/build/src/configuration-types.js +12 -1
  99. package/build/src/configuration.d.ts +101 -2
  100. package/build/src/configuration.d.ts.map +1 -1
  101. package/build/src/configuration.js +240 -32
  102. package/build/src/database/drivers/base.d.ts +5 -1
  103. package/build/src/database/drivers/base.d.ts.map +1 -1
  104. package/build/src/database/drivers/base.js +42 -16
  105. package/build/src/environment-handlers/base.d.ts +12 -0
  106. package/build/src/environment-handlers/base.d.ts.map +1 -1
  107. package/build/src/environment-handlers/base.js +17 -1
  108. package/build/src/environment-handlers/node/cli/commands/background-jobs-activate.d.ts +5 -0
  109. package/build/src/environment-handlers/node/cli/commands/background-jobs-activate.d.ts.map +1 -0
  110. package/build/src/environment-handlers/node/cli/commands/background-jobs-activate.js +17 -0
  111. package/build/src/environment-handlers/node/cli/commands/background-jobs-main.d.ts +45 -0
  112. package/build/src/environment-handlers/node/cli/commands/background-jobs-main.d.ts.map +1 -1
  113. package/build/src/environment-handlers/node/cli/commands/background-jobs-main.js +67 -10
  114. package/build/src/environment-handlers/node/cli/commands/background-jobs-retire.d.ts +5 -0
  115. package/build/src/environment-handlers/node/cli/commands/background-jobs-retire.d.ts.map +1 -0
  116. package/build/src/environment-handlers/node/cli/commands/background-jobs-retire.js +17 -0
  117. package/build/src/environment-handlers/node/cli/commands/background-jobs-worker.d.ts +36 -0
  118. package/build/src/environment-handlers/node/cli/commands/background-jobs-worker.d.ts.map +1 -1
  119. package/build/src/environment-handlers/node/cli/commands/background-jobs-worker.js +51 -11
  120. package/build/src/environment-handlers/node/cli/commands/server.d.ts.map +1 -1
  121. package/build/src/environment-handlers/node/cli/commands/server.js +8 -3
  122. package/build/src/environment-handlers/node.d.ts +12 -0
  123. package/build/src/environment-handlers/node.d.ts.map +1 -1
  124. package/build/src/environment-handlers/node.js +19 -1
  125. package/build/src/error-reporting/request-details.d.ts +8 -1
  126. package/build/src/error-reporting/request-details.d.ts.map +1 -1
  127. package/build/src/error-reporting/request-details.js +30 -32
  128. package/build/src/frontend-model-controller.d.ts.map +1 -1
  129. package/build/src/frontend-model-controller.js +20 -14
  130. package/build/src/http-server/client/index.d.ts.map +1 -1
  131. package/build/src/http-server/client/index.js +5 -12
  132. package/build/src/http-server/client/request-runner.d.ts.map +1 -1
  133. package/build/src/http-server/client/request-runner.js +23 -10
  134. package/build/src/http-server/client/request-timing.d.ts +13 -0
  135. package/build/src/http-server/client/request-timing.d.ts.map +1 -1
  136. package/build/src/http-server/client/request-timing.js +19 -1
  137. package/build/src/http-server/client/websocket-session.d.ts +9 -2
  138. package/build/src/http-server/client/websocket-session.d.ts.map +1 -1
  139. package/build/src/http-server/client/websocket-session.js +46 -15
  140. package/build/src/http-server/worker-handler/worker-thread.d.ts.map +1 -1
  141. package/build/src/http-server/worker-handler/worker-thread.js +13 -6
  142. package/build/src/initializer.d.ts +14 -1
  143. package/build/src/initializer.d.ts.map +1 -1
  144. package/build/src/initializer.js +18 -2
  145. package/build/src/log-redactor.d.ts +97 -0
  146. package/build/src/log-redactor.d.ts.map +1 -0
  147. package/build/src/log-redactor.js +297 -0
  148. package/build/src/routes/resolver.d.ts +7 -0
  149. package/build/src/routes/resolver.d.ts.map +1 -1
  150. package/build/src/routes/resolver.js +26 -5
  151. package/build/src/utils/shutdown-lifecycle.d.ts +13 -0
  152. package/build/src/utils/shutdown-lifecycle.d.ts.map +1 -1
  153. package/build/src/utils/shutdown-lifecycle.js +35 -26
  154. package/build/tsconfig.tsbuildinfo +1 -1
  155. package/build/utils/shutdown-lifecycle.js +33 -30
  156. package/package.json +1 -1
  157. package/src/application.js +30 -10
  158. package/src/background-jobs/adapter.js +8 -0
  159. package/src/background-jobs/client.js +11 -2
  160. package/src/background-jobs/forked-runner-child.js +26 -30
  161. package/src/background-jobs/generation-handshake-timeout-error.js +32 -0
  162. package/src/background-jobs/generation-identity.js +160 -0
  163. package/src/background-jobs/job-runner.js +5 -7
  164. package/src/background-jobs/lifecycle-client.js +127 -0
  165. package/src/background-jobs/lifecycle-control-server.js +310 -0
  166. package/src/background-jobs/main.js +617 -124
  167. package/src/background-jobs/pooled-runner-child.js +17 -9
  168. package/src/background-jobs/runner-graceful-shutdown.js +26 -10
  169. package/src/background-jobs/socket-request.js +44 -3
  170. package/src/background-jobs/sql-adapter.js +6 -0
  171. package/src/background-jobs/status-reporter.js +11 -2
  172. package/src/background-jobs/types.js +7 -2
  173. package/src/background-jobs/worker.js +340 -48
  174. package/src/cli/commands/background-jobs/activate.js +7 -0
  175. package/src/cli/commands/background-jobs/retire.js +7 -0
  176. package/src/cli/index.js +1 -1
  177. package/src/configuration-types.js +12 -0
  178. package/src/configuration.js +276 -42
  179. package/src/database/drivers/base.js +41 -14
  180. package/src/environment-handlers/base.js +19 -1
  181. package/src/environment-handlers/node/cli/commands/background-jobs-activate.js +19 -0
  182. package/src/environment-handlers/node/cli/commands/background-jobs-main.js +71 -11
  183. package/src/environment-handlers/node/cli/commands/background-jobs-retire.js +19 -0
  184. package/src/environment-handlers/node/cli/commands/background-jobs-worker.js +53 -12
  185. package/src/environment-handlers/node/cli/commands/server.js +7 -2
  186. package/src/environment-handlers/node.js +26 -6
  187. package/src/error-reporting/request-details.js +28 -31
  188. package/src/frontend-model-controller.js +19 -13
  189. package/src/http-server/client/index.js +5 -13
  190. package/src/http-server/client/request-runner.js +26 -9
  191. package/src/http-server/client/request-timing.js +19 -0
  192. package/src/http-server/client/websocket-session.js +54 -14
  193. package/src/http-server/worker-handler/worker-thread.js +13 -6
  194. package/src/initializer.js +19 -1
  195. package/src/log-redactor.js +341 -0
  196. package/src/routes/resolver.js +32 -4
  197. package/src/utils/shutdown-lifecycle.js +33 -30
package/README.md CHANGED
@@ -32,11 +32,12 @@
32
32
  * Per-record ability checks via `.abilities(...)` on frontend queries + `record.can(action)` (see [docs/abilities.md](docs/abilities.md))
33
33
  * Translated model attributes with current-locale relationship sorting (see [docs/translations.md](docs/translations.md))
34
34
  * Cross-process broadcast bus for `broadcastToChannel` via `velocious beacon`, including background job runner processes (see [docs/beacon.md](docs/beacon.md))
35
+ * Rails-style application process initializer teardown with immutable process identity, reverse idempotent shutdown, and explicit HTTP/background-job ownership (see [docs/application-process-lifecycle.md](docs/application-process-lifecycle.md))
35
36
  * Configurable HTTP server worker handlers plus backpressured, descriptor-only file responses with completion callbacks (see [docs/http-server.md](docs/http-server.md))
36
37
  * Default-on buffered HTTP response compression with Brotli/gzip content negotiation, global and per-response opt-outs, and HEAD-correct representation headers (see [docs/http-server.md](docs/http-server.md#response-compression))
37
- * Background jobs with Node SQL/TCP workers plus a Browser/Expo local SQLite store and in-process dispatcher, including failure events and authorized database-scoped dashboard count snapshots/deltas. Release-directory integrations have a required release-scoped jobs-main/worker generation and asynchronous retirement compliance target that current startup adoption does not implement end to end (see [docs/background-jobs.md](docs/background-jobs.md), [docs/local-background-jobs.md](docs/local-background-jobs.md), and [docs/background-jobs-dashboard.md](docs/background-jobs-dashboard.md))
38
+ * Background jobs with Node SQL/TCP workers plus a Browser/Expo local SQLite store and in-process dispatcher, including failure events, authorized database-scoped dashboard counts, and an opt-in release-scoped main/worker generation protocol with acknowledged activation, asynchronous retirement, and retired-main recovery. Production compliance additionally requires downstream supervisor retention/activation ordering and release pins (see [docs/background-jobs.md](docs/background-jobs.md), [docs/local-background-jobs.md](docs/local-background-jobs.md), and [docs/background-jobs-dashboard.md](docs/background-jobs-dashboard.md))
38
39
  * Durable one-off background-job scheduling with exact epoch timestamps (see [docs/scheduled-background-job-enqueue.md](docs/scheduled-background-job-enqueue.md))
39
- * Rails-style request and database query logging (see [docs/logging.md](docs/logging.md))
40
+ * Rails-style request and database query logging with structured credential redaction (see [docs/logging.md](docs/logging.md))
40
41
  * EJS-backed mailers with delivery, queueing, and payload rendering support (see [docs/mailers.md](docs/mailers.md))
41
42
  * Trusted reverse proxy handling for `request.remoteAddress()` (see [docs/trusted-proxies.md](docs/trusted-proxies.md))
42
43
  * In-process driver schema metadata caching (see [docs/schema-metadata-cache.md](docs/schema-metadata-cache.md))
@@ -60,6 +61,12 @@ npx velocious init
60
61
 
61
62
  By default, Velocious looks for your configuration in `src/config/configuration.js`. If you keep the configuration elsewhere, make sure your app imports it early and calls `configuration.setCurrent()`.
62
63
 
64
+ Application initializers may implement `teardown()` and inspect their frozen
65
+ `getProcessContext()` value. Long-lived process owners call
66
+ `configuration.shutdown()` before framework connection cleanup; see the
67
+ [application process lifecycle guide](docs/application-process-lifecycle.md) for
68
+ promise identity, errors, process types, and pooled/forked runner semantics.
69
+
63
70
  # Node SQLite driver
64
71
 
65
72
  Projects using `velocious/build/src/database/drivers/sqlite/index.js` must install its optional peer dependencies:
@@ -1908,7 +1915,8 @@ const configuration = new Configuration({
1908
1915
  console: false, // disable console output
1909
1916
  file: true, // enable file output
1910
1917
  directory: "/custom/logs", // optional, defaults to "<project>/log" in Node
1911
- filePath: "/tmp/app.log" // optional explicit path
1918
+ filePath: "/tmp/app.log", // optional explicit path
1919
+ sensitiveNames: ["integrationPin"] // optional app-specific additions
1912
1920
  }
1913
1921
  })
1914
1922
  ```
@@ -1969,6 +1977,8 @@ Model queries use operation names such as `Task Load`, `Task Count`, `Task Pluck
1969
1977
 
1970
1978
  Query logging defaults to off in the `test` environment to keep CI output quiet and is skipped when no output emits `info`. Override it with `logging: {queryLogging: true}` when a test build should write SQL timing logs, and use the normal logging output settings to send those logs to console or file.
1971
1979
 
1980
+ - **Credential redaction**: Request headers, parsed body/params, nested arrays, URL queries, WebSocket authentication params, rendered SQL diagnostics, and request/frontend-model errors are redacted before formatting and output fan-out. Defaults match common authorization, authentication, credential, password, secret, token, API-key, cookie/session, and base64-content name variants case-insensitively. Add application names with `logging.sensitiveNames`; entries must be non-blank strings. Exact request-scoped values are replaced in SQL/error text while safe fields, SQL shape, timing, source lines, error class/backtrace, and correlation metadata remain visible. Import `LOG_REDACTION_MARKER` from `velocious/build/src/log-redactor.js` when code needs to compare the deterministic marker. See [logging and credential redaction](docs/logging.md#credential-redaction).
1981
+
1972
1982
  ## Listen for framework errors
1973
1983
 
1974
1984
  Velocious emits framework errors (including uncaught controller action errors) on the configuration error event bus:
@@ -2321,6 +2331,11 @@ the new main. Deploy and HTTP/WebSocket drain completion are independent of this
2321
2331
  potentially hours-long lifecycle. See [release-generation
2322
2332
  draining](docs/background-jobs.md#release-generation-draining).
2323
2333
 
2334
+ Velocious provides the opt-in generation protocol; production still requires a
2335
+ supervisor that preserves old generation units and release pins, and a deploy
2336
+ coordinator that retires the old generation before activating the healthy
2337
+ candidate without waiting for retired work to finish.
2338
+
2324
2339
  Jobs can opt into cross-worker durable concurrency limits by pairing a non-empty `concurrencyKey` with a positive-integer `maxConcurrency` in their background-job options, or by deriving the key in a hydrated job instance's non-static `concurrencyKey()` method. Explicit enqueue options win. The first cap registered for a key is stable; conflicting caps are rejected. See [durable concurrency limits](docs/background-jobs.md#durable-concurrency-limits).
2325
2340
 
2326
2341
  Production apps can listen for `background-job-failed` (or its `all-error` mirror) to report accepted failed attempts, including retry and terminal-state metadata, and for `background-job-orphaned` to react to a specific job the main process reclaimed after its worker died mid-run — e.g. enqueue a targeted recovery for the work it left behind, instead of only polling for the aftermath. Orphan handlers run before the sweep waits for reclaimed jobs to be dispatched, so a stalled dispatcher does not delay application recovery. See [docs/background-jobs.md](docs/background-jobs.md#failure-events).
@@ -2365,7 +2380,11 @@ export default new Configuration({
2365
2380
  pooledRunnerMaxRssBytes: 536870912,
2366
2381
  pooledRunnerMaxLifetimeMs: 3600000,
2367
2382
  dispatchStrategy: "beacon",
2368
- jobTimeoutMs: null
2383
+ jobTimeoutMs: null,
2384
+ // Release-directory deployments opt in with one exact id and local socket:
2385
+ generationId: "release-20260828.1",
2386
+ initialGenerationState: "candidate",
2387
+ lifecycleSocketPath: "/srv/app/releases/20260828.1/run/background-jobs.sock"
2369
2388
  }
2370
2389
  })
2371
2390
  ```
@@ -2428,8 +2447,32 @@ VELOCIOUS_BACKGROUND_JOBS_DISPATCH_STRATEGY=beacon
2428
2447
  VELOCIOUS_BACKGROUND_JOBS_POLL_INTERVAL_MS=1000
2429
2448
  VELOCIOUS_BACKGROUND_JOBS_WORKER_SHUTDOWN_TIMEOUT_MS=indefinite
2430
2449
  VELOCIOUS_BACKGROUND_JOBS_JOB_TIMEOUT_MS=5400000
2450
+ # Opt-in release generation values (omit all three for exact legacy behavior):
2451
+ VELOCIOUS_BACKGROUND_JOBS_GENERATION_ID=release-20260828.1
2452
+ VELOCIOUS_BACKGROUND_JOBS_INITIAL_GENERATION_STATE=candidate
2453
+ VELOCIOUS_BACKGROUND_JOBS_LIFECYCLE_SOCKET_PATH=/srv/app/releases/20260828.1/run/background-jobs.sock
2431
2454
  ```
2432
2455
 
2456
+ Activate or retire that exact generation with one acknowledged local request:
2457
+
2458
+ ```sh
2459
+ npx velocious background-jobs:activate --generation release-20260828.1 --socket /srv/app/releases/20260828.1/run/background-jobs.sock
2460
+ npx velocious background-jobs:retire --generation release-20260828.1 --socket /srv/app/releases/20260828.1/run/background-jobs.sock
2461
+ ```
2462
+
2463
+ Each lifecycle command sends one request with no retry and has a hard 10000ms
2464
+ deadline; `--timeout-ms` accepts 1 through 25000ms. Generation-aware workers,
2465
+ clients, and reporters require their hello acknowledgement before readiness or
2466
+ mutation and bound it to 4000ms by default.
2467
+
2468
+ Generation ids supplied through config, environment, API, or CLI must be
2469
+ identical and match `^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$`; invalid or conflicting
2470
+ identity fails before listening. Omit generation settings to preserve legacy
2471
+ worker ids, protocol, disconnect recovery, and custom-adapter compatibility.
2472
+ An ID-only configuration derives `candidate`; that default does not conflict
2473
+ with an explicit API/CLI `active` or `retired` recovery state, while multiple
2474
+ actual state sources must still agree.
2475
+
2433
2476
  `VELOCIOUS_BACKGROUND_JOBS_WORKER_SHUTDOWN_TIMEOUT_MS` (default: `indefinite`) bounds how long a `background-jobs-worker` waits for in-flight jobs on `SIGTERM`/`SIGINT` before terminating any forked or spawned child runners still running. The default waits for jobs to finish and never interrupts a running job; a positive finite cap is a per-worker shutdown control for an explicitly requested process stop, not the normal deploy-completion mechanism. During release retirement, the old jobs-main and workers may drain for hours after deploy returns. See [docs/background-jobs.md](docs/background-jobs.md#worker-shutdown-and-process-job-draining).
2434
2477
 
2435
2478
  `maxConcurrentInlineJobs` (default: `4`) caps how many `executionMode: "inline"` jobs a single `background-jobs-worker` process runs in parallel. Concurrency is at the JS event-loop level: every job in flight shares the worker's process and DB connection pool, so the cap should fit the pool, not the CPU count. Forking remains the right tool when you need memory isolation across long-running jobs or want to use more cores; select it with `executionMode: "forked"`.
@@ -2637,7 +2680,7 @@ Each job must define exactly one of `every` or `cron`. Cron times are evaluated
2637
2680
 
2638
2681
  ## Persistence and retries
2639
2682
 
2640
- Jobs are persisted in the configured database (`backgroundJobs.databaseIdentifier`) in an internal `background_jobs` table. When a worker picks a job, the main generates a unique lease id before asking the adapter to mark the job handed off, and the worker reports completion or failure back to the main process. If that persistence call has an ambiguous result, only the exact caller-generated lease is conditionally returned; failed recovery is retained for the dispatch error-retry path, so worker admission and concurrency do not remain stranded and a newer lease is never reclaimed. Custom adapters must persist a supplied `markHandedOff({handoffId})` exactly; built-in adapters continue generating one for legacy direct callers that omit it. If a worker socket disconnects unexpectedly, only the leases handed to that exact socket are immediately returned to the queue; late reports are fenced by lease id so they cannot mutate a newer attempt. This recovery is at-least-once and may repeat application side effects if the disconnected attempt had already started them. Gracefully draining workers keep their leases while they finish. Startup reconnection/adoption is an abnormal crash/legacy-recovery facility, not the normal deploy topology: during ordinary release retirement the old main remains alive and owns its old workers, and they must not reconnect to the new main. A production integration that restarts jobs-main every deploy and depends on worker adoption is not compliant with the release-generation contract. See [release-generation draining](docs/background-jobs.md#release-generation-draining) and [worker disconnect recovery](docs/background-jobs.md#worker-disconnect-recovery).
2683
+ Jobs are persisted in the configured database (`backgroundJobs.databaseIdentifier`) in an internal `background_jobs` table. When a worker picks a job, the main generates a unique lease id before asking the adapter to mark the job handed off, and the worker reports completion or failure back to the main process. If that persistence call has an ambiguous result, only the exact caller-generated lease is conditionally returned; failed recovery is retained for the dispatch error-retry path, so worker admission and concurrency do not remain stranded and a newer lease is never reclaimed. Custom adapters must persist a supplied `markHandedOff({handoffId})` exactly; built-in adapters continue generating one for legacy direct callers that omit it. A legacy worker disconnect returns only that socket's leases immediately. Generation mode instead preserves the exact leases through reconnect grace for the same qualified worker, then returns them to the global queue on expiry. Late reports are fenced by generation-qualified worker id, lease id, and handoff time so they cannot mutate a newer attempt. This recovery is at-least-once and may repeat application side effects if the disconnected attempt had already started them. A release-retiring worker revokes readiness but retains heartbeat, its unchanged old endpoint, exact-generation reconnect, accepted work, child execution, durable reports, and acknowledgements until its drain settles; retiring/retired mains reject new identities and never grant reconnecting workers readiness. Startup reconnection/adoption is an abnormal crash/legacy-recovery facility, not the normal deploy topology: during ordinary release retirement the old main remains alive and owns its old workers, and they must not reconnect to the new main. A production integration that restarts jobs-main on every deploy and depends on worker adoption is not compliant with the release-generation contract. See [release-generation draining](docs/background-jobs.md#release-generation-draining) and [worker disconnect recovery](docs/background-jobs.md#worker-disconnect-recovery).
2641
2684
 
2642
2685
  Failed jobs are re-queued with backoff and retried up to 10 times by default (10s, 1m, 10m, 1h, then +1h per retry). You can override the retry limit per job:
2643
2686
 
@@ -9,6 +9,7 @@ import SyncPublisher from "./sync/sync-publisher.js"
9
9
  import SyncWebsocketChannel from "./sync/sync-websocket-channel.js"
10
10
  import websocketEventsHost from "./http-server/websocket-events-host.js"
11
11
  import restArgsError from "./utils/rest-args-error.js"
12
+ import { runShutdownSteps } from "./utils/shutdown-lifecycle.js"
12
13
 
13
14
  /**
14
15
  * HttpServerConfiguration type.
@@ -40,6 +41,8 @@ export default class VelociousApplication {
40
41
  * Stores the http server lock value.
41
42
  * @type {HttpServerLock | undefined} */
42
43
  this.httpServerLock = undefined
44
+ /** @type {Promise<void> | undefined} */
45
+ this._stopPromise = undefined
43
46
  }
44
47
 
45
48
  /**
@@ -186,17 +189,34 @@ export default class VelociousApplication {
186
189
  * Runs stop.
187
190
  * @returns {Promise<void>} - Resolves when complete.
188
191
  */
189
- async stop() {
190
- await this.logger.debug("Stopping server")
192
+ stop() {
193
+ if (!this._stopPromise) this._stopPromise = this._stop()
191
194
 
192
- try {
193
- await this.httpServer?.stop()
194
- this.configuration._httpServerInstance = undefined
195
- await this.configuration.disconnectBeacon()
196
- await this.configuration.closeDatabaseConnections()
197
- } finally {
198
- await this.releaseHttpServerLock()
199
- }
195
+ return this._stopPromise
196
+ }
197
+
198
+ /**
199
+ * Stops application and framework resources.
200
+ * @returns {Promise<void>} - Resolves after every application and framework close succeeds.
201
+ */
202
+ async _stop() {
203
+ await runShutdownSteps({
204
+ message: "Application and framework shutdown failed",
205
+ steps: [
206
+ async () => await this.logger.debug("Stopping server"),
207
+ async () => {
208
+ try {
209
+ await this.httpServer?.stop()
210
+ } finally {
211
+ this.configuration._httpServerInstance = undefined
212
+ }
213
+ },
214
+ async () => await this.configuration.shutdown(),
215
+ async () => await this.configuration.disconnectBeacon(),
216
+ async () => await this.configuration.closeDatabaseConnections(),
217
+ async () => await this.releaseHttpServerLock()
218
+ ]
219
+ })
200
220
  }
201
221
 
202
222
  /**
@@ -6,6 +6,14 @@
6
6
  * separate concerns.
7
7
  */
8
8
  export default class BackgroundJobsAdapter {
9
+ /**
10
+ * Declares exact durable fencing support for release-scoped generations.
11
+ * Third-party adapters must override this only after implementing the full
12
+ * snapshot, owner, report, and recovery contract.
13
+ * @returns {boolean} - Whether generation mode is supported.
14
+ */
15
+ supportsReleaseScopedGenerations() { return false }
16
+
9
17
  /**
10
18
  * Ensures the adapter can accept work.
11
19
  * @returns {Promise<void>} - Resolves when ready.
@@ -3,6 +3,7 @@
3
3
  import timeout from "awaitery/build/timeout.js"
4
4
  import configurationResolver from "../configuration-resolver.js"
5
5
  import BackgroundJobsSocketRequest from "./socket-request.js"
6
+ import { DEFAULT_GENERATION_HANDSHAKE_TIMEOUT_MS, validateGenerationHandshakeTimeoutMs } from "./generation-handshake-timeout-error.js"
6
7
 
7
8
  const DEFAULT_ENQUEUE_TIMEOUT_MS = 5000
8
9
 
@@ -12,10 +13,14 @@ export default class BackgroundJobsClient {
12
13
  * @param {object} [args] - Options.
13
14
  * @param {import("../configuration.js").default} [args.configuration] - Configuration.
14
15
  * @param {number} [args.enqueueTimeoutMs] - Maximum time to wait for an enqueue acknowledgement in milliseconds (default: 5000).
16
+ * @param {number} [args.generationHandshakeTimeoutMs] - Maximum time to wait for generation acknowledgement (default: 4000).
17
+ * @param {string} [args.generationId] - Explicit release generation identity.
15
18
  */
16
- constructor({configuration, enqueueTimeoutMs = DEFAULT_ENQUEUE_TIMEOUT_MS} = {}) {
19
+ constructor({configuration, enqueueTimeoutMs = DEFAULT_ENQUEUE_TIMEOUT_MS, generationHandshakeTimeoutMs = DEFAULT_GENERATION_HANDSHAKE_TIMEOUT_MS, generationId} = {}) {
17
20
  this.configurationPromise = configuration ? Promise.resolve(configuration) : configurationResolver()
18
21
  this.enqueueTimeoutMs = enqueueTimeoutMs
22
+ this.generationHandshakeTimeoutMs = validateGenerationHandshakeTimeoutMs(generationHandshakeTimeoutMs)
23
+ this.explicitGenerationId = generationId
19
24
  }
20
25
 
21
26
  /**
@@ -25,8 +30,12 @@ export default class BackgroundJobsClient {
25
30
  async _request() {
26
31
  const configuration = await this.configurationPromise
27
32
  const {host, port} = configuration.getBackgroundJobsConfig()
33
+ const {generationId} = configuration.resolveBackgroundJobsGenerationConfig({
34
+ generationId: this.explicitGenerationId,
35
+ sourceName: "BackgroundJobsClient"
36
+ })
28
37
 
29
- return new BackgroundJobsSocketRequest({host, port, role: "client"})
38
+ return new BackgroundJobsSocketRequest({host, port, role: "client", generationHandshakeTimeoutMs: this.generationHandshakeTimeoutMs, generationId})
30
39
  }
31
40
 
32
41
  /**
@@ -10,8 +10,8 @@ import setRunnerProcessTitle from "./runner-process-title.js"
10
10
  // which jobs are running, how many of each, and which are eating resources.
11
11
  setRunnerProcessTitle()
12
12
 
13
- let finishing = false
14
- let shuttingDown = false
13
+ /** @type {Promise<void> | undefined} */
14
+ let shutdownPromise
15
15
 
16
16
  /**
17
17
  * Closes the runner's connections — releasing any advisory lock a killed-mid-job
@@ -21,12 +21,15 @@ let shuttingDown = false
21
21
  * @param {number} exitCode - Process exit code.
22
22
  * @returns {Promise<void>}
23
23
  */
24
- async function shutdownRunner(exitCode) {
25
- if (shuttingDown || finishing) return
26
- shuttingDown = true
24
+ function shutdownRunner(exitCode) {
25
+ if (shutdownPromise) return shutdownPromise
27
26
 
28
- await closeRunnerConnections(currentConfigurationOrNull())
29
- process.exit(exitCode)
27
+ shutdownPromise = (async () => {
28
+ await closeRunnerConnections(currentConfigurationOrNull())
29
+ process.exit(exitCode)
30
+ })()
31
+
32
+ return shutdownPromise
30
33
  }
31
34
 
32
35
  /**
@@ -45,25 +48,13 @@ function isJobMessage(message) {
45
48
  /**
46
49
  * Runs finish.
47
50
  * @param {number} exitCode - Process exit code.
48
- * @returns {void}
51
+ * @returns {Promise<void>}
49
52
  */
50
- function finish(exitCode) {
51
- finishing = true
52
- process.exitCode = exitCode
53
-
54
- if (process.connected && process.disconnect) {
55
- process.disconnect()
53
+ async function finish(exitCode) {
54
+ if (process.send) {
55
+ await new Promise((resolve) => process.send?.({type: "job-reported"}, () => resolve(undefined)))
56
56
  }
57
-
58
- setImmediate(() => process.exit(exitCode))
59
- }
60
-
61
- /**
62
- * Runs report job finished.
63
- * @returns {void}
64
- */
65
- function reportJobFinished() {
66
- if (process.send) process.send({type: "job-reported"})
57
+ await shutdownRunner(exitCode)
67
58
  }
68
59
 
69
60
  /**
@@ -79,7 +70,10 @@ async function runJobMessage(message) {
79
70
  // The per-job process title (and its restore) is set inside runJobPayload,
80
71
  // which reads the job class's `static processTitle`. This process boots with
81
72
  // the base "velocious background-jobs-runner" title set at module load above.
82
- await runJobPayload(message.payload, {closeConnections: false})
73
+ await runJobPayload(message.payload, {
74
+ closeConnections: false,
75
+ processType: "background-jobs-forked-runner"
76
+ })
83
77
  }
84
78
 
85
79
  /**
@@ -88,15 +82,17 @@ async function runJobMessage(message) {
88
82
  * @returns {Promise<void>} - Resolves after completion is reported.
89
83
  */
90
84
  async function handleJobMessage(message) {
85
+ let exitCode
86
+
91
87
  try {
92
88
  await runJobMessage(message)
93
- reportJobFinished()
94
- finish(0)
89
+ exitCode = 0
95
90
  } catch (error) {
96
- reportJobFinished()
97
91
  console.error("Forked background job runner failed:", error)
98
- finish(1)
92
+ exitCode = 1
99
93
  }
94
+
95
+ await finish(exitCode)
100
96
  }
101
97
 
102
98
  for (const signal of ["SIGTERM", "SIGINT"]) {
@@ -104,7 +100,7 @@ for (const signal of ["SIGTERM", "SIGINT"]) {
104
100
  }
105
101
 
106
102
  process.once("disconnect", () => {
107
- if (!finishing) void shutdownRunner(0)
103
+ void shutdownRunner(0)
108
104
  })
109
105
 
110
106
  process.once("message", (message) => {
@@ -0,0 +1,32 @@
1
+ // @ts-check
2
+
3
+ export const DEFAULT_GENERATION_HANDSHAKE_TIMEOUT_MS = 4000
4
+
5
+ /** Actionable failure for an unacknowledged generation hello. */
6
+ export default class BackgroundJobsGenerationHandshakeTimeoutError extends Error {
7
+ /**
8
+ * Creates an actionable generation-handshake deadline failure.
9
+ * @param {object} args - Timeout context.
10
+ * @param {string} args.endpoint - Main endpoint.
11
+ * @param {string} args.generationId - Expected generation.
12
+ * @param {"worker" | "client" | "reporter"} args.role - Initiating peer role.
13
+ * @param {number} args.timeoutMs - Handshake deadline.
14
+ */
15
+ constructor({endpoint, generationId, role, timeoutMs}) {
16
+ super(`Background jobs ${role} generation handshake for ${generationId} timed out after ${timeoutMs}ms at ${endpoint}`)
17
+ this.name = "BackgroundJobsGenerationHandshakeTimeoutError"
18
+ }
19
+ }
20
+
21
+ /**
22
+ * Validates a generation handshake deadline.
23
+ * @param {number} timeoutMs - Candidate deadline.
24
+ * @returns {number} - Valid deadline.
25
+ */
26
+ export function validateGenerationHandshakeTimeoutMs(timeoutMs) {
27
+ if (!Number.isInteger(timeoutMs) || timeoutMs < 1 || timeoutMs > 2_147_483_647) {
28
+ throw new TypeError("generationHandshakeTimeoutMs must be an integer between 1 and 2147483647")
29
+ }
30
+
31
+ return timeoutMs
32
+ }
@@ -0,0 +1,160 @@
1
+ // @ts-check
2
+
3
+ import { utf8ByteLength } from "../utils/utf8-byte-length.js"
4
+
5
+ /**
6
+ * @typedef {object} GenerationValueSource
7
+ * @property {string} name - Human-readable source name.
8
+ * @property {boolean} present - Whether the source was explicitly supplied.
9
+ * @property {ReturnType<typeof JSON.parse> | undefined} value - Supplied value.
10
+ */
11
+
12
+ const GENERATION_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/
13
+ const WORKER_INSTANCE_ID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i
14
+ const INITIAL_GENERATION_STATES = new Set(["candidate", "active", "retired"])
15
+
16
+ /**
17
+ * Validates one release generation identifier.
18
+ * @param {ReturnType<typeof JSON.parse> | undefined} value - Candidate value.
19
+ * @param {string} [sourceName] - Source label for failures.
20
+ * @returns {string} - Valid generation id.
21
+ */
22
+ export function validateGenerationId(value, sourceName = "background jobs generationId") {
23
+ if (typeof value !== "string" || !GENERATION_ID_PATTERN.test(value)) {
24
+ throw new TypeError(`${sourceName} must match ${GENERATION_ID_PATTERN}`)
25
+ }
26
+
27
+ return value
28
+ }
29
+
30
+ /**
31
+ * Resolves explicitly present generation identity sources without precedence.
32
+ * @param {GenerationValueSource[]} sources - Identity sources.
33
+ * @returns {string | undefined} - Identical resolved identity or legacy unset.
34
+ */
35
+ export function resolveGenerationId(sources) {
36
+ const presentSources = sources.filter((source) => source.present)
37
+
38
+ if (presentSources.length === 0) return undefined
39
+
40
+ const values = presentSources.map((source) => validateGenerationId(source.value, source.name))
41
+ const generationId = values[0]
42
+
43
+ if (values.some((value) => value !== generationId)) {
44
+ const names = presentSources.map((source) => source.name).join(", ")
45
+
46
+ throw new Error(`Conflicting background jobs generation identities from: ${names}`)
47
+ }
48
+
49
+ return generationId
50
+ }
51
+
52
+ /**
53
+ * Resolves the boot lifecycle state.
54
+ * @param {GenerationValueSource[]} sources - State sources.
55
+ * @param {string | undefined} generationId - Resolved generation identity.
56
+ * @returns {import("./types.js").BackgroundJobsGenerationInitialState | "active"} - Boot state.
57
+ */
58
+ export function resolveInitialGenerationState(sources, generationId) {
59
+ const presentSources = sources.filter((source) => source.present)
60
+
61
+ if (presentSources.length === 0) return generationId ? "candidate" : "active"
62
+ if (!generationId) throw new Error("backgroundJobs.initialGenerationState requires backgroundJobs.generationId")
63
+
64
+ const values = presentSources.map((source) => {
65
+ if (typeof source.value !== "string" || !INITIAL_GENERATION_STATES.has(source.value)) {
66
+ throw new TypeError(`${source.name} must be "candidate", "active", or "retired"`)
67
+ }
68
+
69
+ return /** @type {import("./types.js").BackgroundJobsGenerationInitialState} */ (source.value)
70
+ })
71
+ const state = values[0]
72
+
73
+ if (values.some((value) => value !== state)) {
74
+ const names = presentSources.map((source) => source.name).join(", ")
75
+
76
+ throw new Error(`Conflicting background jobs initialGenerationState values from: ${names}`)
77
+ }
78
+
79
+ return state
80
+ }
81
+
82
+ /**
83
+ * Resolves the optional release-local lifecycle socket path.
84
+ * @param {GenerationValueSource[]} sources - Path sources.
85
+ * @param {string | undefined} generationId - Resolved generation identity.
86
+ * @returns {string | undefined} - Absolute Unix socket path.
87
+ */
88
+ export function resolveLifecycleSocketPath(sources, generationId) {
89
+ const presentSources = sources.filter((source) => source.present)
90
+
91
+ if (presentSources.length === 0) return undefined
92
+ if (!generationId) throw new Error("backgroundJobs.lifecycleSocketPath requires backgroundJobs.generationId")
93
+
94
+ const values = presentSources.map((source) => {
95
+ if (typeof source.value !== "string" || !source.value.startsWith("/") || source.value.includes("\0")) {
96
+ throw new TypeError(`${source.name} must be an absolute Unix socket path`)
97
+ }
98
+
99
+ if (utf8ByteLength(source.value) > 103) {
100
+ throw new TypeError(`${source.name} must be at most 103 UTF-8 bytes for portable Unix socket support`)
101
+ }
102
+
103
+ return source.value
104
+ })
105
+ const socketPath = values[0]
106
+
107
+ if (values.some((value) => value !== socketPath)) {
108
+ const names = presentSources.map((source) => source.name).join(", ")
109
+
110
+ throw new Error(`Conflicting background jobs lifecycle socket paths from: ${names}`)
111
+ }
112
+
113
+ return socketPath
114
+ }
115
+
116
+ /**
117
+ * Creates the exact durable worker owner token.
118
+ * @param {object} args - Owner parts.
119
+ * @param {string} args.generationId - Release generation.
120
+ * @param {string} args.workerInstanceId - Worker process UUID.
121
+ * @returns {string} - Generation-qualified durable worker id.
122
+ */
123
+ export function createGenerationWorkerId({generationId, workerInstanceId}) {
124
+ validateGenerationId(generationId)
125
+ if (!WORKER_INSTANCE_ID_PATTERN.test(workerInstanceId)) throw new TypeError("workerInstanceId must be a UUID")
126
+
127
+ return `${generationId}:${workerInstanceId}`
128
+ }
129
+
130
+ /**
131
+ * Parses a generation-qualified durable worker id.
132
+ * @param {ReturnType<typeof JSON.parse>} workerId - Durable worker id.
133
+ * @returns {{generationId: string, workerInstanceId: string} | null} - Parsed owner or null.
134
+ */
135
+ export function parseGenerationWorkerId(workerId) {
136
+ if (typeof workerId !== "string") return null
137
+
138
+ const separatorIndex = workerId.indexOf(":")
139
+ if (separatorIndex < 1 || separatorIndex !== workerId.lastIndexOf(":")) return null
140
+
141
+ const generationId = workerId.slice(0, separatorIndex)
142
+ const workerInstanceId = workerId.slice(separatorIndex + 1)
143
+
144
+ if (!GENERATION_ID_PATTERN.test(generationId) || !WORKER_INSTANCE_ID_PATTERN.test(workerInstanceId)) return null
145
+
146
+ return {generationId, workerInstanceId}
147
+ }
148
+
149
+ /**
150
+ * Checks exact parsed generation ownership.
151
+ * @param {object} args - Ownership query.
152
+ * @param {string} args.generationId - Expected generation.
153
+ * @param {ReturnType<typeof JSON.parse>} args.workerId - Durable worker id.
154
+ * @returns {boolean} - Whether the parsed owner belongs to the generation.
155
+ */
156
+ export function workerIdBelongsToGeneration({generationId, workerId}) {
157
+ const parsed = parseGenerationWorkerId(workerId)
158
+
159
+ return parsed?.generationId === generationId
160
+ }
@@ -4,6 +4,7 @@ import configurationResolver from "../configuration-resolver.js"
4
4
  import BackgroundJobRegistry from "./job-registry.js"
5
5
  import BackgroundJobsStatusReporter from "./status-reporter.js"
6
6
  import BackgroundJobRescheduleSignal from "./reschedule-signal.js"
7
+ import { closeRunnerConnections } from "./runner-graceful-shutdown.js"
7
8
 
8
9
  const BEACON_READY_TIMEOUT_MS = 5000
9
10
 
@@ -80,12 +81,13 @@ function runnerProcessTitle(JobClass, payload) {
80
81
  * @param {object} [options] - Runner options.
81
82
  * @param {boolean} [options.closeConnections] - Whether to gracefully close framework connections after the job.
82
83
  * @param {boolean} [options.manageProcessTitle] - Whether to set the per-job process title and restore it afterwards. Off for concurrent pooled runners, where interleaved snapshot/restore of the single process-wide `process.title` would corrupt it; the pooled child owns an aggregate title instead.
84
+ * @param {string} [options.processType] - Generic application process type.
83
85
  * @returns {Promise<"completed" | "rescheduled">} - Acknowledged outcome.
84
86
  */
85
- export default async function runJobPayload(payload, {closeConnections = true, manageProcessTitle = true} = {}) {
87
+ export default async function runJobPayload(payload, {closeConnections = true, manageProcessTitle = true, processType = "background-jobs-runner"} = {}) {
86
88
  const configuration = await configurationResolver()
87
89
  configuration.setCurrent()
88
- await configuration.initialize({type: "background-jobs-runner"})
90
+ await configuration.initialize({type: processType})
89
91
  await connectBeacon(configuration)
90
92
  const reporter = new BackgroundJobsStatusReporter({configuration})
91
93
 
@@ -167,11 +169,7 @@ export default async function runJobPayload(payload, {closeConnections = true, m
167
169
  // one) doesn't misreport a finished job as still running.
168
170
  if (manageProcessTitle) process.title = previousTitle
169
171
  if (closeConnections) {
170
- try {
171
- await configuration.disconnectBeacon()
172
- } finally {
173
- await configuration.closeDatabaseConnections()
174
- }
172
+ await closeRunnerConnections(configuration)
175
173
  }
176
174
  }
177
175
  }