velocious 1.0.519 → 1.0.520
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +34 -0
- package/build/background-jobs/json-socket.js +12 -0
- package/build/background-jobs/main.js +78 -23
- package/build/background-jobs/store.js +87 -5
- package/build/background-jobs/types.js +4 -2
- package/build/background-jobs/worker.js +113 -23
- package/build/configuration-types.js +11 -2
- package/build/configuration.js +1 -1
- package/build/environment-handlers/node/cli/commands/generate/base-models.js +1 -1
- package/build/jobs/prune-terminal-background-jobs.js +67 -0
- package/build/src/background-jobs/json-socket.d.ts +12 -0
- package/build/src/background-jobs/json-socket.d.ts.map +1 -1
- package/build/src/background-jobs/json-socket.js +13 -1
- package/build/src/background-jobs/main.d.ts +18 -9
- package/build/src/background-jobs/main.d.ts.map +1 -1
- package/build/src/background-jobs/main.js +73 -26
- package/build/src/background-jobs/store.d.ts +16 -0
- package/build/src/background-jobs/store.d.ts.map +1 -1
- package/build/src/background-jobs/store.js +75 -5
- package/build/src/background-jobs/types.d.ts +14 -3
- package/build/src/background-jobs/types.d.ts.map +1 -1
- package/build/src/background-jobs/types.js +5 -3
- package/build/src/background-jobs/worker.d.ts +55 -7
- package/build/src/background-jobs/worker.d.ts.map +1 -1
- package/build/src/background-jobs/worker.js +108 -22
- package/build/src/configuration-types.d.ts +35 -4
- package/build/src/configuration-types.d.ts.map +1 -1
- package/build/src/configuration-types.js +12 -3
- package/build/src/configuration.d.ts +4 -2
- package/build/src/configuration.d.ts.map +1 -1
- package/build/src/configuration.js +2 -2
- package/build/src/environment-handlers/node/cli/commands/generate/base-models.js +2 -2
- package/build/src/jobs/prune-terminal-background-jobs.d.ts +23 -0
- package/build/src/jobs/prune-terminal-background-jobs.d.ts.map +1 -0
- package/build/src/jobs/prune-terminal-background-jobs.js +61 -0
- package/build/tsconfig.tsbuildinfo +1 -1
- package/package.json +1 -1
- package/src/background-jobs/json-socket.js +12 -0
- package/src/background-jobs/main.js +78 -23
- package/src/background-jobs/store.js +87 -5
- package/src/background-jobs/types.js +4 -2
- package/src/background-jobs/worker.js +113 -23
- package/src/configuration-types.js +11 -2
- package/src/configuration.js +1 -1
- package/src/environment-handlers/node/cli/commands/generate/base-models.js +1 -1
- package/src/jobs/prune-terminal-background-jobs.js +67 -0
|
@@ -12,6 +12,10 @@ import {fileURLToPath} from "node:url"
|
|
|
12
12
|
/** Grace period after SIGTERM before a lingering process runner is SIGKILLed. */
|
|
13
13
|
const FORKED_CHILD_SIGKILL_GRACE_MS = 5000
|
|
14
14
|
const FORKED_RUNNER_ENTRY_PATH = fileURLToPath(new URL("./forked-runner-child.js", import.meta.url))
|
|
15
|
+
/** How often the worker sends a liveness heartbeat to the main. */
|
|
16
|
+
const HEARTBEAT_INTERVAL_MS = 15000
|
|
17
|
+
/** TCP keepalive so a half-open connection to the main surfaces as a close. */
|
|
18
|
+
const SOCKET_KEEPALIVE_MS = 10000
|
|
15
19
|
/**
|
|
16
20
|
* Execution modes.
|
|
17
21
|
* @type {import("./types.js").BackgroundJobExecutionMode[]} */
|
|
@@ -27,8 +31,9 @@ export default class BackgroundJobsWorker {
|
|
|
27
31
|
* @param {number} [args.maxConcurrentForkedJobs] - Override the process runner concurrency cap from `configuration.getBackgroundJobsConfig()`.
|
|
28
32
|
* @param {number} [args.maxConcurrentInlineJobs] - Override the inline-job concurrency cap from `configuration.getBackgroundJobsConfig()`.
|
|
29
33
|
* @param {number} [args.forkedChildSigkillGraceMs] - Override the grace period between SIGTERM and SIGKILL when reaping lingering process runners on stop.
|
|
34
|
+
* @param {number} [args.heartbeatIntervalMs] - Override the liveness heartbeat interval (default 15000ms).
|
|
30
35
|
*/
|
|
31
|
-
constructor({configuration, host, port, maxConcurrentForkedJobs, maxConcurrentInlineJobs, forkedChildSigkillGraceMs} = {}) {
|
|
36
|
+
constructor({configuration, host, port, maxConcurrentForkedJobs, maxConcurrentInlineJobs, forkedChildSigkillGraceMs, heartbeatIntervalMs} = {}) {
|
|
32
37
|
/**
|
|
33
38
|
* Narrows the runtime value to the documented type.
|
|
34
39
|
* @type {Promise<import("../configuration.js").default>} */
|
|
@@ -74,6 +79,21 @@ export default class BackgroundJobsWorker {
|
|
|
74
79
|
: FORKED_CHILD_SIGKILL_GRACE_MS
|
|
75
80
|
this.shouldStop = false
|
|
76
81
|
this.workerId = randomUUID()
|
|
82
|
+
this.heartbeatIntervalMs = typeof heartbeatIntervalMs === "number" && heartbeatIntervalMs >= 1
|
|
83
|
+
? heartbeatIntervalMs
|
|
84
|
+
: HEARTBEAT_INTERVAL_MS
|
|
85
|
+
/**
|
|
86
|
+
* Narrows the runtime value to the documented type.
|
|
87
|
+
* @type {ReturnType<typeof setInterval> | undefined} */
|
|
88
|
+
this._heartbeatTimer = undefined
|
|
89
|
+
/**
|
|
90
|
+
* In-flight job-result reports to the main. Reporting is decoupled from the
|
|
91
|
+
* job/child slot (freeing the slot never waits on a report) and retried
|
|
92
|
+
* durably, so a transient main/DB outage cannot leak slots or lose a
|
|
93
|
+
* terminal report. Tracked so a graceful `stop()` can drain them.
|
|
94
|
+
* @type {Set<Promise<void>>}
|
|
95
|
+
*/
|
|
96
|
+
this.inflightReports = new Set()
|
|
77
97
|
/**
|
|
78
98
|
* Narrows the runtime value to the documented type.
|
|
79
99
|
* @type {JsonSocket | undefined} */
|
|
@@ -154,6 +174,7 @@ export default class BackgroundJobsWorker {
|
|
|
154
174
|
async stop({timeoutMs} = {}) {
|
|
155
175
|
if (this.shouldStop) return
|
|
156
176
|
this.shouldStop = true
|
|
177
|
+
this._stopHeartbeat()
|
|
157
178
|
|
|
158
179
|
// Announce drain so main stops dispatching but keeps the connection
|
|
159
180
|
// open until we close it ourselves below.
|
|
@@ -168,6 +189,9 @@ export default class BackgroundJobsWorker {
|
|
|
168
189
|
await this._drainInflight(this.inflightInlineJobs, timeoutMs)
|
|
169
190
|
await this._drainInflight(this.inflightProcessJobs, timeoutMs)
|
|
170
191
|
await this._terminateProcessChildren()
|
|
192
|
+
// Give in-flight result reports (now decoupled from job slots) a bounded
|
|
193
|
+
// chance to land before the socket closes.
|
|
194
|
+
await this._drainInflight(this.inflightReports, timeoutMs)
|
|
171
195
|
|
|
172
196
|
if (this.jsonSocket) this.jsonSocket.close()
|
|
173
197
|
if (this.configuration) {
|
|
@@ -238,6 +262,7 @@ export default class BackgroundJobsWorker {
|
|
|
238
262
|
const host = this.host || config.host
|
|
239
263
|
const port = typeof this.port === "number" ? this.port : config.port
|
|
240
264
|
const socket = net.createConnection({host, port})
|
|
265
|
+
socket.setKeepAlive(true, SOCKET_KEEPALIVE_MS)
|
|
241
266
|
const jsonSocket = new JsonSocket(socket)
|
|
242
267
|
this.jsonSocket = jsonSocket
|
|
243
268
|
|
|
@@ -256,16 +281,51 @@ export default class BackgroundJobsWorker {
|
|
|
256
281
|
})
|
|
257
282
|
|
|
258
283
|
jsonSocket.on("close", () => {
|
|
284
|
+
this._stopHeartbeat()
|
|
259
285
|
if (this.shouldStop) return
|
|
260
286
|
setTimeout(() => { void this._connect() }, 1000)
|
|
261
287
|
})
|
|
262
288
|
|
|
263
289
|
socket.on("connect", () => {
|
|
264
|
-
jsonSocket.send({type: "hello", role: "worker", supportsHandoffIdReporting: true, workerId: this.workerId})
|
|
290
|
+
jsonSocket.send({type: "hello", role: "worker", supportsHandoffIdReporting: true, supportsHeartbeat: true, workerId: this.workerId})
|
|
265
291
|
this._sendReadyIfRunning()
|
|
292
|
+
this._startHeartbeat()
|
|
266
293
|
})
|
|
267
294
|
}
|
|
268
295
|
|
|
296
|
+
/**
|
|
297
|
+
* Sends periodic liveness heartbeats to the main so a wedged or silent worker
|
|
298
|
+
* can be detected and dropped there (its leases released) instead of freezing
|
|
299
|
+
* the queue until a human notices.
|
|
300
|
+
* @returns {void}
|
|
301
|
+
*/
|
|
302
|
+
_startHeartbeat() {
|
|
303
|
+
this._stopHeartbeat()
|
|
304
|
+
|
|
305
|
+
this._heartbeatTimer = setInterval(() => {
|
|
306
|
+
if (this.shouldStop || !this.jsonSocket) return
|
|
307
|
+
|
|
308
|
+
try {
|
|
309
|
+
this.jsonSocket.send({type: "heartbeat", workerId: this.workerId})
|
|
310
|
+
} catch {
|
|
311
|
+
// Socket is closing/closed; the close handler drives reconnect.
|
|
312
|
+
}
|
|
313
|
+
}, this.heartbeatIntervalMs)
|
|
314
|
+
|
|
315
|
+
if (typeof this._heartbeatTimer.unref === "function") this._heartbeatTimer.unref()
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
/**
|
|
319
|
+
* Stops the liveness heartbeat timer.
|
|
320
|
+
* @returns {void}
|
|
321
|
+
*/
|
|
322
|
+
_stopHeartbeat() {
|
|
323
|
+
if (this._heartbeatTimer) {
|
|
324
|
+
clearInterval(this._heartbeatTimer)
|
|
325
|
+
this._heartbeatTimer = undefined
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
|
|
269
329
|
/**
|
|
270
330
|
* Runs handle job.
|
|
271
331
|
* @param {import("./types.js").BackgroundJobPayload} payload - Payload.
|
|
@@ -394,9 +454,12 @@ export default class BackgroundJobsWorker {
|
|
|
394
454
|
* @returns {Promise<void>} - Resolves when complete (success or failure reported).
|
|
395
455
|
*/
|
|
396
456
|
async _runInlineJobAndReport(payload) {
|
|
457
|
+
// Report in the background so freeing this inline slot never waits on the
|
|
458
|
+
// report. Reporting is durable (retried until it lands), so a transient
|
|
459
|
+
// main/DB outage neither wedges the slot nor loses the terminal result.
|
|
397
460
|
try {
|
|
398
461
|
await this._runJobInline(payload)
|
|
399
|
-
|
|
462
|
+
this._reportJobResultInBackground({
|
|
400
463
|
jobId: payload.id,
|
|
401
464
|
status: "completed",
|
|
402
465
|
handoffId: payload.handoffId,
|
|
@@ -404,7 +467,7 @@ export default class BackgroundJobsWorker {
|
|
|
404
467
|
workerId: payload.workerId || this.workerId
|
|
405
468
|
})
|
|
406
469
|
} catch (error) {
|
|
407
|
-
|
|
470
|
+
this._reportJobResultInBackground({
|
|
408
471
|
jobId: payload.id,
|
|
409
472
|
status: "failed",
|
|
410
473
|
error,
|
|
@@ -522,10 +585,10 @@ export default class BackgroundJobsWorker {
|
|
|
522
585
|
_waitForForkedChild({child, payload}) {
|
|
523
586
|
return new Promise((resolve) => {
|
|
524
587
|
child.once("exit", (code, signal) => {
|
|
525
|
-
|
|
588
|
+
this._handleForkedChildExit({child, code, signal, payload, resolve})
|
|
526
589
|
})
|
|
527
590
|
child.once("error", (error) => {
|
|
528
|
-
|
|
591
|
+
this._handleForkedChildError({child, error, payload, resolve})
|
|
529
592
|
})
|
|
530
593
|
})
|
|
531
594
|
}
|
|
@@ -538,22 +601,22 @@ export default class BackgroundJobsWorker {
|
|
|
538
601
|
* @param {keyof typeof import("node:os").constants.signals | null} args.signal - Exit signal.
|
|
539
602
|
* @param {import("./types.js").BackgroundJobPayload & {id: string}} args.payload - Payload.
|
|
540
603
|
* @param {(value: void) => void} args.resolve - Promise resolver.
|
|
541
|
-
* @returns {
|
|
604
|
+
* @returns {void}
|
|
542
605
|
*/
|
|
543
|
-
|
|
606
|
+
_handleForkedChildExit({child, code, signal, payload, resolve}) {
|
|
544
607
|
this.inflightProcessChildren.delete(child)
|
|
545
608
|
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
609
|
+
// Free the worker slot as soon as the child is gone — never gate it on the
|
|
610
|
+
// failure report. A hung/slow report must not leak the slot; enough leaked
|
|
611
|
+
// slots drive `acceptsForked` to false and silently wedge the worker.
|
|
612
|
+
resolve(undefined)
|
|
613
|
+
|
|
614
|
+
if (this._forkedChildExitedCleanly({code, signal})) return
|
|
550
615
|
|
|
551
|
-
|
|
616
|
+
this._reportForkedChildFailure({
|
|
552
617
|
payload,
|
|
553
618
|
error: new Error(`Forked background job runner exited before reporting: code=${code} signal=${signal || "none"}`)
|
|
554
619
|
})
|
|
555
|
-
|
|
556
|
-
resolve(undefined)
|
|
557
620
|
}
|
|
558
621
|
|
|
559
622
|
/**
|
|
@@ -574,13 +637,14 @@ export default class BackgroundJobsWorker {
|
|
|
574
637
|
* @param {Error} args.error - Child process error.
|
|
575
638
|
* @param {import("./types.js").BackgroundJobPayload & {id: string}} args.payload - Payload.
|
|
576
639
|
* @param {(value: void) => void} args.resolve - Promise resolver.
|
|
577
|
-
* @returns {
|
|
640
|
+
* @returns {void}
|
|
578
641
|
*/
|
|
579
|
-
|
|
642
|
+
_handleForkedChildError({child, error, payload, resolve}) {
|
|
580
643
|
this.inflightProcessChildren.delete(child)
|
|
581
|
-
|
|
582
|
-
await this._reportForkedChildFailure({payload, error})
|
|
644
|
+
// Free the slot first (see _handleForkedChildExit) — reporting is best-effort.
|
|
583
645
|
resolve(undefined)
|
|
646
|
+
console.error("Background jobs forked runner error:", error)
|
|
647
|
+
this._reportForkedChildFailure({payload, error})
|
|
584
648
|
}
|
|
585
649
|
|
|
586
650
|
/**
|
|
@@ -595,7 +659,7 @@ export default class BackgroundJobsWorker {
|
|
|
595
659
|
child.send({type: "job", payload})
|
|
596
660
|
} catch (error) {
|
|
597
661
|
child.kill("SIGTERM")
|
|
598
|
-
|
|
662
|
+
this._reportForkedChildFailure({payload, error})
|
|
599
663
|
}
|
|
600
664
|
}
|
|
601
665
|
|
|
@@ -604,10 +668,10 @@ export default class BackgroundJobsWorker {
|
|
|
604
668
|
* @param {object} args - Options.
|
|
605
669
|
* @param {import("./types.js").BackgroundJobPayload & {id: string}} args.payload - Payload.
|
|
606
670
|
* @param {?} args.error - Error.
|
|
607
|
-
* @returns {
|
|
671
|
+
* @returns {void}
|
|
608
672
|
*/
|
|
609
|
-
|
|
610
|
-
|
|
673
|
+
_reportForkedChildFailure({payload, error}) {
|
|
674
|
+
this._reportJobResultInBackground({
|
|
611
675
|
jobId: payload.id,
|
|
612
676
|
status: "failed",
|
|
613
677
|
error,
|
|
@@ -682,4 +746,30 @@ export default class BackgroundJobsWorker {
|
|
|
682
746
|
console.error("Background job status reporting failed:", reportError)
|
|
683
747
|
}
|
|
684
748
|
}
|
|
749
|
+
|
|
750
|
+
/**
|
|
751
|
+
* Fires a durable job-result report without blocking the caller (so freeing a
|
|
752
|
+
* job/child slot never waits on the report). The report is tracked so a
|
|
753
|
+
* graceful `stop()` can drain in-flight reports before closing the socket.
|
|
754
|
+
* @param {object} args - Options.
|
|
755
|
+
* @param {string} args.jobId - Job id.
|
|
756
|
+
* @param {"completed" | "failed"} args.status - Status.
|
|
757
|
+
* @param {?} [args.error] - Error.
|
|
758
|
+
* @param {string} [args.handoffId] - Handoff lease id.
|
|
759
|
+
* @param {number} [args.handedOffAtMs] - Handed off timestamp.
|
|
760
|
+
* @param {string} [args.workerId] - Worker id.
|
|
761
|
+
* @returns {void}
|
|
762
|
+
*/
|
|
763
|
+
_reportJobResultInBackground({jobId, status, error, handoffId, handedOffAtMs, workerId}) {
|
|
764
|
+
/**
|
|
765
|
+
* Defines report.
|
|
766
|
+
* @type {Promise<void>} */
|
|
767
|
+
let report
|
|
768
|
+
|
|
769
|
+
report = this._reportJobResult({jobId, status, error, handoffId, handedOffAtMs, workerId}).finally(() => {
|
|
770
|
+
this.inflightReports.delete(report)
|
|
771
|
+
})
|
|
772
|
+
|
|
773
|
+
this.inflightReports.add(report)
|
|
774
|
+
}
|
|
685
775
|
}
|
|
@@ -194,8 +194,17 @@
|
|
|
194
194
|
* jobs older than this many ms. `null` or `<= 0` disables (keeps them for
|
|
195
195
|
* debugging). Default: `2592000000` (30 days).
|
|
196
196
|
* @property {number} [batchSize] - Rows deleted per batch. Default: `1000`.
|
|
197
|
-
* @property {number} [sweepIntervalMs] - How often the
|
|
198
|
-
*
|
|
197
|
+
* @property {number} [sweepIntervalMs] - How often the retention sweep runs.
|
|
198
|
+
* Default: `3600000` (1 hour).
|
|
199
|
+
*/
|
|
200
|
+
/**
|
|
201
|
+
* Fully-resolved retention config as returned by `getBackgroundJobsConfig()`
|
|
202
|
+
* (every field defaulted), as opposed to the partial user-provided input.
|
|
203
|
+
* @typedef {object} ResolvedBackgroundJobsRetentionConfiguration
|
|
204
|
+
* @property {number | null} completedTtlMs - Resolved completed-job TTL in ms (`null` disables).
|
|
205
|
+
* @property {number | null} failedTtlMs - Resolved failed/orphaned TTL in ms (`null` disables).
|
|
206
|
+
* @property {number} batchSize - Resolved delete batch size.
|
|
207
|
+
* @property {number} sweepIntervalMs - Resolved sweep interval in ms.
|
|
199
208
|
*/
|
|
200
209
|
|
|
201
210
|
/**
|
package/build/configuration.js
CHANGED
|
@@ -1260,7 +1260,7 @@ export default class VelociousConfiguration {
|
|
|
1260
1260
|
|
|
1261
1261
|
/**
|
|
1262
1262
|
* Runs get background jobs config.
|
|
1263
|
-
* @returns {Required<import("./configuration-types.js").BackgroundJobsConfiguration>} - Background jobs configuration.
|
|
1263
|
+
* @returns {Required<import("./configuration-types.js").BackgroundJobsConfiguration> & {retention: import("./configuration-types.js").ResolvedBackgroundJobsRetentionConfiguration}} - Background jobs configuration.
|
|
1264
1264
|
*/
|
|
1265
1265
|
getBackgroundJobsConfig() {
|
|
1266
1266
|
const envHost = process.env.VELOCIOUS_BACKGROUND_JOBS_HOST
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
// @ts-check
|
|
2
|
+
|
|
3
|
+
import BackgroundJobsStore from "../background-jobs/store.js"
|
|
4
|
+
import Configuration from "../configuration.js"
|
|
5
|
+
import VelociousJob from "../background-jobs/job.js"
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Built-in job that prunes terminal `background_jobs` rows past their retention
|
|
9
|
+
* window so the table does not grow unbounded. The main process registers this
|
|
10
|
+
* on the normal background-jobs scheduler when `backgroundJobs.retention` is
|
|
11
|
+
* enabled, so it runs as an ordinary scheduled/queued job — visible in the job
|
|
12
|
+
* tables and dispatched to a worker — rather than a hidden in-process timer.
|
|
13
|
+
* @augments {VelociousJob<[]>}
|
|
14
|
+
*/
|
|
15
|
+
export default class PruneTerminalBackgroundJobsJob extends VelociousJob {
|
|
16
|
+
/**
|
|
17
|
+
* Reserved job name that an application job cannot shadow. The registry loads
|
|
18
|
+
* app `src/jobs` first and skips duplicate built-in names, so if this used the
|
|
19
|
+
* default class-name identity an app class named `PruneTerminalBackgroundJobsJob`
|
|
20
|
+
* would be dispatched instead. A `:`-namespaced name can never collide with a
|
|
21
|
+
* default (class-name) identity, since class names cannot contain `:`.
|
|
22
|
+
* @returns {string} - Reserved job name.
|
|
23
|
+
*/
|
|
24
|
+
static jobName() {
|
|
25
|
+
return "velocious:prune-terminal-background-jobs"
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Builds the scheduler configuration for this job from a resolved retention
|
|
30
|
+
* config, or returns `null` when retention is fully disabled (nothing to
|
|
31
|
+
* prune, so nothing to schedule). `maxConcurrency: 1` keeps runs from
|
|
32
|
+
* overlapping, and `deduplicateWhileQueued` stops the interval scheduler from
|
|
33
|
+
* piling up redundant queued rows when a prune is slow or no worker is free.
|
|
34
|
+
* @param {import("../configuration-types.js").ResolvedBackgroundJobsRetentionConfiguration} retention - Resolved retention config.
|
|
35
|
+
* @returns {import("../configuration-types.js").ScheduledBackgroundJobConfiguration | null} - Scheduler config for the prune job, or null when retention is disabled.
|
|
36
|
+
*/
|
|
37
|
+
static scheduleConfiguration(retention) {
|
|
38
|
+
const prunesCompleted = typeof retention.completedTtlMs === "number" && retention.completedTtlMs > 0
|
|
39
|
+
const prunesFailed = typeof retention.failedTtlMs === "number" && retention.failedTtlMs > 0
|
|
40
|
+
|
|
41
|
+
if (!prunesCompleted && !prunesFailed) {
|
|
42
|
+
return null
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
return {
|
|
46
|
+
class: this,
|
|
47
|
+
every: retention.sweepIntervalMs,
|
|
48
|
+
options: {concurrencyKey: "velocious-prune-terminal-background-jobs", maxConcurrency: 1, deduplicateWhileQueued: true}
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* Prunes terminal job rows past their retention window.
|
|
54
|
+
* @returns {Promise<void>}
|
|
55
|
+
*/
|
|
56
|
+
async perform() {
|
|
57
|
+
const configuration = Configuration.current()
|
|
58
|
+
const config = configuration.getBackgroundJobsConfig()
|
|
59
|
+
const store = new BackgroundJobsStore({configuration, databaseIdentifier: config.databaseIdentifier})
|
|
60
|
+
|
|
61
|
+
await store.pruneTerminalJobs({
|
|
62
|
+
completedTtlMs: config.retention.completedTtlMs,
|
|
63
|
+
failedTtlMs: config.retention.failedTtlMs,
|
|
64
|
+
batchSize: config.retention.batchSize
|
|
65
|
+
})
|
|
66
|
+
}
|
|
67
|
+
}
|
|
@@ -26,6 +26,18 @@ export default class JsonSocket extends JsonSocket_base {
|
|
|
26
26
|
* Narrows the runtime value to the documented type.
|
|
27
27
|
* @type {boolean} */
|
|
28
28
|
acceptsInlineJobs: boolean;
|
|
29
|
+
/**
|
|
30
|
+
* Whether this worker advertised heartbeat support in its hello. Only
|
|
31
|
+
* heartbeat-capable workers are subject to the main's stale-liveness
|
|
32
|
+
* eviction; a legacy worker (e.g. mid rolling deploy) is exempt so its
|
|
33
|
+
* active leases are not released while it is still running them.
|
|
34
|
+
* @type {boolean} */
|
|
35
|
+
supportsHeartbeat: boolean;
|
|
36
|
+
/**
|
|
37
|
+
* Last time (ms) the main saw any message from this worker socket; used by
|
|
38
|
+
* the main's liveness sweep to drop a wedged/silent worker.
|
|
39
|
+
* @type {number | undefined} */
|
|
40
|
+
lastSeenAt: number | undefined;
|
|
29
41
|
buffer: string;
|
|
30
42
|
/**
|
|
31
43
|
* Runs on data.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"json-socket.d.ts","sourceRoot":"","sources":["../../../src/background-jobs/json-socket.js"],"names":[],"mappings":";AAIA;IACE;;;OAGG;IACH,oBAFW,OAAO,KAAK,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"json-socket.d.ts","sourceRoot":"","sources":["../../../src/background-jobs/json-socket.js"],"names":[],"mappings":";AAIA;IACE;;;OAGG;IACH,oBAFW,OAAO,KAAK,EAAE,MAAM,EA0C9B;IAtCC,kCAAoB;IACpB;;oCAEgC;IAChC,UADU,MAAM,GAAG,SAAS,CACH;IACzB;;yBAEqB;IACrB,4BADU,OAAO,CACsB;IACvC;;yBAEqB;IACrB,oBADU,OAAO,CACa;IAC9B;;yBAEqB;IACrB,mBADU,OAAO,CACY;IAC7B;;yBAEqB;IACrB,mBADU,OAAO,CACY;IAC7B;;;;;yBAKqB;IACrB,mBADU,OAAO,CACa;IAC9B;;;oCAGgC;IAChC,YADU,MAAM,GAAG,SAAS,CACD;IAC3B,eAAgB;IAOlB;;;;OAIG;IACH,eAHW,MAAM,GACJ,IAAI,CAqBhB;IAED;;;;OAIG;IACH,cAHW,OAAC,GACC,IAAI,CAIhB;IAED;;;OAGG;IACH,SAFa,IAAI,CAIhB;CACF"}
|
|
@@ -28,6 +28,18 @@ export default class JsonSocket extends EventEmitter {
|
|
|
28
28
|
* Narrows the runtime value to the documented type.
|
|
29
29
|
* @type {boolean} */
|
|
30
30
|
this.acceptsInlineJobs = true;
|
|
31
|
+
/**
|
|
32
|
+
* Whether this worker advertised heartbeat support in its hello. Only
|
|
33
|
+
* heartbeat-capable workers are subject to the main's stale-liveness
|
|
34
|
+
* eviction; a legacy worker (e.g. mid rolling deploy) is exempt so its
|
|
35
|
+
* active leases are not released while it is still running them.
|
|
36
|
+
* @type {boolean} */
|
|
37
|
+
this.supportsHeartbeat = false;
|
|
38
|
+
/**
|
|
39
|
+
* Last time (ms) the main saw any message from this worker socket; used by
|
|
40
|
+
* the main's liveness sweep to drop a wedged/silent worker.
|
|
41
|
+
* @type {number | undefined} */
|
|
42
|
+
this.lastSeenAt = undefined;
|
|
31
43
|
this.buffer = "";
|
|
32
44
|
this.socket.setEncoding("utf8");
|
|
33
45
|
this.socket.on("data", (chunk) => this._onData(String(chunk)));
|
|
@@ -74,4 +86,4 @@ export default class JsonSocket extends EventEmitter {
|
|
|
74
86
|
this.socket.end();
|
|
75
87
|
}
|
|
76
88
|
}
|
|
77
|
-
//# sourceMappingURL=data:application/json;base64,
|
|
89
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoianNvbi1zb2NrZXQuanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi9zcmMvYmFja2dyb3VuZC1qb2JzL2pzb24tc29ja2V0LmpzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLFlBQVk7QUFFWixPQUFPLFlBQVksTUFBTSwyQkFBMkIsQ0FBQTtBQUVwRCxNQUFNLENBQUMsT0FBTyxPQUFPLFVBQVcsU0FBUSxZQUFZO0lBQ2xEOzs7T0FHRztJQUNILFlBQVksTUFBTTtRQUNoQixLQUFLLEVBQUUsQ0FBQTtRQUNQLElBQUksQ0FBQyxNQUFNLEdBQUcsTUFBTSxDQUFBO1FBQ3BCOzt3Q0FFZ0M7UUFDaEMsSUFBSSxDQUFDLFFBQVEsR0FBRyxTQUFTLENBQUE7UUFDekI7OzZCQUVxQjtRQUNyQixJQUFJLENBQUMsMEJBQTBCLEdBQUcsS0FBSyxDQUFBO1FBQ3ZDOzs2QkFFcUI7UUFDckIsSUFBSSxDQUFDLGtCQUFrQixHQUFHLElBQUksQ0FBQTtRQUM5Qjs7NkJBRXFCO1FBQ3JCLElBQUksQ0FBQyxpQkFBaUIsR0FBRyxJQUFJLENBQUE7UUFDN0I7OzZCQUVxQjtRQUNyQixJQUFJLENBQUMsaUJBQWlCLEdBQUcsSUFBSSxDQUFBO1FBQzdCOzs7Ozs2QkFLcUI7UUFDckIsSUFBSSxDQUFDLGlCQUFpQixHQUFHLEtBQUssQ0FBQTtRQUM5Qjs7O3dDQUdnQztRQUNoQyxJQUFJLENBQUMsVUFBVSxHQUFHLFNBQVMsQ0FBQTtRQUMzQixJQUFJLENBQUMsTUFBTSxHQUFHLEVBQUUsQ0FBQTtRQUNoQixJQUFJLENBQUMsTUFBTSxDQUFDLFdBQVcsQ0FBQyxNQUFNLENBQUMsQ0FBQTtRQUMvQixJQUFJLENBQUMsTUFBTSxDQUFDLEVBQUUsQ0FBQyxNQUFNLEVBQUUsQ0FBQyxLQUFLLEVBQUUsRUFBRSxDQUFDLElBQUksQ0FBQyxPQUFPLENBQUMsTUFBTSxDQUFDLEtBQUssQ0FBQyxDQUFDLENBQUMsQ0FBQTtRQUM5RCxJQUFJLENBQUMsTUFBTSxDQUFDLEVBQUUsQ0FBQyxPQUFPLEVBQUUsR0FBRyxFQUFFLENBQUMsSUFBSSxDQUFDLElBQUksQ0FBQyxPQUFPLENBQUMsQ0FBQyxDQUFBO1FBQ2pELElBQUksQ0FBQyxNQUFNLENBQUMsRUFBRSxDQUFDLE9BQU8sRUFBRSxDQUFDLEtBQUssRUFBRSxFQUFFLENBQUMsSUFBSSxDQUFDLElBQUksQ0FBQyxPQUFPLEVBQUUsS0FBSyxDQUFDLENBQUMsQ0FBQTtJQUMvRCxDQUFDO0lBRUQ7Ozs7T0FJRztJQUNILE9BQU8sQ0FBQyxLQUFLO1FBQ1gsSUFBSSxDQUFDLE1BQU0sSUFBSSxLQUFLLENBQUE7UUFFcEIsT0FBTyxJQUFJLEVBQUUsQ0FBQztZQUNaLE1BQU0sWUFBWSxHQUFHLElBQUksQ0FBQyxNQUFNLENBQUMsT0FBTyxDQUFDLElBQUksQ0FBQyxDQUFBO1lBQzlDLElBQUksWUFBWSxLQUFLLENBQUMsQ0FBQztnQkFBRSxNQUFLO1lBRTlCLE1BQU0sSUFBSSxHQUFHLElBQUksQ0FBQyxNQUFNLENBQUMsS0FBSyxDQUFDLENBQUMsRUFBRSxZQUFZLENBQUMsQ0FBQyxJQUFJLEVBQUUsQ0FBQTtZQUN0RCxJQUFJLENBQUMsTUFBTSxHQUFHLElBQUksQ0FBQyxNQUFNLENBQUMsS0FBSyxDQUFDLFlBQVksR0FBRyxDQUFDLENBQUMsQ0FBQTtZQUVqRCxJQUFJLENBQUMsSUFBSTtnQkFBRSxTQUFRO1lBRW5CLElBQUksQ0FBQztnQkFDSCxNQUFNLE9BQU8sR0FBRyxJQUFJLENBQUMsS0FBSyxDQUFDLElBQUksQ0FBQyxDQUFBO2dCQUNoQyxJQUFJLENBQUMsSUFBSSxDQUFDLFNBQVMsRUFBRSxPQUFPLENBQUMsQ0FBQTtZQUMvQixDQUFDO1lBQUMsT0FBTyxLQUFLLEVBQUUsQ0FBQztnQkFDZixJQUFJLENBQUMsSUFBSSxDQUFDLE9BQU8sRUFBRSxLQUFLLENBQUMsQ0FBQTtZQUMzQixDQUFDO1FBQ0gsQ0FBQztJQUNILENBQUM7SUFFRDs7OztPQUlHO0lBQ0gsSUFBSSxDQUFDLE9BQU87UUFDVixJQUFJLENBQUMsTUFBTSxDQUFDLEtBQUssQ0FBQyxHQUFHLElBQUksQ0FBQyxTQUFTLENBQUMsT0FBTyxDQUFDLElBQUksQ0FBQyxDQUFBO0lBQ25ELENBQUM7SUFFRDs7O09BR0c7SUFDSCxLQUFLO1FBQ0gsSUFBSSxDQUFDLE1BQU0sQ0FBQyxHQUFHLEVBQUUsQ0FBQTtJQUNuQixDQUFDO0NBQ0YiLCJzb3VyY2VzQ29udGVudCI6WyIvLyBAdHMtY2hlY2tcblxuaW1wb3J0IEV2ZW50RW1pdHRlciBmcm9tIFwiLi4vdXRpbHMvZXZlbnQtZW1pdHRlci5qc1wiXG5cbmV4cG9ydCBkZWZhdWx0IGNsYXNzIEpzb25Tb2NrZXQgZXh0ZW5kcyBFdmVudEVtaXR0ZXIge1xuICAvKipcbiAgICogUnVucyBjb25zdHJ1Y3Rvci5cbiAgICogQHBhcmFtIHtpbXBvcnQoXCJuZXRcIikuU29ja2V0fSBzb2NrZXQgLSBTb2NrZXQgaW5zdGFuY2UuXG4gICAqL1xuICBjb25zdHJ1Y3Rvcihzb2NrZXQpIHtcbiAgICBzdXBlcigpXG4gICAgdGhpcy5zb2NrZXQgPSBzb2NrZXRcbiAgICAvKipcbiAgICAgKiBOYXJyb3dzIHRoZSBydW50aW1lIHZhbHVlIHRvIHRoZSBkb2N1bWVudGVkIHR5cGUuXG4gICAgICogQHR5cGUge3N0cmluZyB8IHVuZGVmaW5lZH0gKi9cbiAgICB0aGlzLndvcmtlcklkID0gdW5kZWZpbmVkXG4gICAgLyoqXG4gICAgICogTmFycm93cyB0aGUgcnVudGltZSB2YWx1ZSB0byB0aGUgZG9jdW1lbnRlZCB0eXBlLlxuICAgICAqIEB0eXBlIHtib29sZWFufSAqL1xuICAgIHRoaXMuc3VwcG9ydHNIYW5kb2ZmSWRSZXBvcnRpbmcgPSBmYWxzZVxuICAgIC8qKlxuICAgICAqIE5hcnJvd3MgdGhlIHJ1bnRpbWUgdmFsdWUgdG8gdGhlIGRvY3VtZW50ZWQgdHlwZS5cbiAgICAgKiBAdHlwZSB7Ym9vbGVhbn0gKi9cbiAgICB0aGlzLmFjY2VwdHNTcGF3bmVkSm9icyA9IHRydWVcbiAgICAvKipcbiAgICAgKiBOYXJyb3dzIHRoZSBydW50aW1lIHZhbHVlIHRvIHRoZSBkb2N1bWVudGVkIHR5cGUuXG4gICAgICogQHR5cGUge2Jvb2xlYW59ICovXG4gICAgdGhpcy5hY2NlcHRzRm9ya2VkSm9icyA9IHRydWVcbiAgICAvKipcbiAgICAgKiBOYXJyb3dzIHRoZSBydW50aW1lIHZhbHVlIHRvIHRoZSBkb2N1bWVudGVkIHR5cGUuXG4gICAgICogQHR5cGUge2Jvb2xlYW59ICovXG4gICAgdGhpcy5hY2NlcHRzSW5saW5lSm9icyA9IHRydWVcbiAgICAvKipcbiAgICAgKiBXaGV0aGVyIHRoaXMgd29ya2VyIGFkdmVydGlzZWQgaGVhcnRiZWF0IHN1cHBvcnQgaW4gaXRzIGhlbGxvLiBPbmx5XG4gICAgICogaGVhcnRiZWF0LWNhcGFibGUgd29ya2VycyBhcmUgc3ViamVjdCB0byB0aGUgbWFpbidzIHN0YWxlLWxpdmVuZXNzXG4gICAgICogZXZpY3Rpb247IGEgbGVnYWN5IHdvcmtlciAoZS5nLiBtaWQgcm9sbGluZyBkZXBsb3kpIGlzIGV4ZW1wdCBzbyBpdHNcbiAgICAgKiBhY3RpdmUgbGVhc2VzIGFyZSBub3QgcmVsZWFzZWQgd2hpbGUgaXQgaXMgc3RpbGwgcnVubmluZyB0aGVtLlxuICAgICAqIEB0eXBlIHtib29sZWFufSAqL1xuICAgIHRoaXMuc3VwcG9ydHNIZWFydGJlYXQgPSBmYWxzZVxuICAgIC8qKlxuICAgICAqIExhc3QgdGltZSAobXMpIHRoZSBtYWluIHNhdyBhbnkgbWVzc2FnZSBmcm9tIHRoaXMgd29ya2VyIHNvY2tldDsgdXNlZCBieVxuICAgICAqIHRoZSBtYWluJ3MgbGl2ZW5lc3Mgc3dlZXAgdG8gZHJvcCBhIHdlZGdlZC9zaWxlbnQgd29ya2VyLlxuICAgICAqIEB0eXBlIHtudW1iZXIgfCB1bmRlZmluZWR9ICovXG4gICAgdGhpcy5sYXN0U2VlbkF0ID0gdW5kZWZpbmVkXG4gICAgdGhpcy5idWZmZXIgPSBcIlwiXG4gICAgdGhpcy5zb2NrZXQuc2V0RW5jb2RpbmcoXCJ1dGY4XCIpXG4gICAgdGhpcy5zb2NrZXQub24oXCJkYXRhXCIsIChjaHVuaykgPT4gdGhpcy5fb25EYXRhKFN0cmluZyhjaHVuaykpKVxuICAgIHRoaXMuc29ja2V0Lm9uKFwiY2xvc2VcIiwgKCkgPT4gdGhpcy5lbWl0KFwiY2xvc2VcIikpXG4gICAgdGhpcy5zb2NrZXQub24oXCJlcnJvclwiLCAoZXJyb3IpID0+IHRoaXMuZW1pdChcImVycm9yXCIsIGVycm9yKSlcbiAgfVxuXG4gIC8qKlxuICAgKiBSdW5zIG9uIGRhdGEuXG4gICAqIEBwYXJhbSB7c3RyaW5nfSBjaHVuayAtIERhdGEgY2h1bmsuXG4gICAqIEByZXR1cm5zIHt2b2lkfVxuICAgKi9cbiAgX29uRGF0YShjaHVuaykge1xuICAgIHRoaXMuYnVmZmVyICs9IGNodW5rXG5cbiAgICB3aGlsZSAodHJ1ZSkge1xuICAgICAgY29uc3QgbmV3bGluZUluZGV4ID0gdGhpcy5idWZmZXIuaW5kZXhPZihcIlxcblwiKVxuICAgICAgaWYgKG5ld2xpbmVJbmRleCA9PT0gLTEpIGJyZWFrXG5cbiAgICAgIGNvbnN0IGxpbmUgPSB0aGlzLmJ1ZmZlci5zbGljZSgwLCBuZXdsaW5lSW5kZXgpLnRyaW0oKVxuICAgICAgdGhpcy5idWZmZXIgPSB0aGlzLmJ1ZmZlci5zbGljZShuZXdsaW5lSW5kZXggKyAxKVxuXG4gICAgICBpZiAoIWxpbmUpIGNvbnRpbnVlXG5cbiAgICAgIHRyeSB7XG4gICAgICAgIGNvbnN0IG1lc3NhZ2UgPSBKU09OLnBhcnNlKGxpbmUpXG4gICAgICAgIHRoaXMuZW1pdChcIm1lc3NhZ2VcIiwgbWVzc2FnZSlcbiAgICAgIH0gY2F0Y2ggKGVycm9yKSB7XG4gICAgICAgIHRoaXMuZW1pdChcImVycm9yXCIsIGVycm9yKVxuICAgICAgfVxuICAgIH1cbiAgfVxuXG4gIC8qKlxuICAgKiBSdW5zIHNlbmQuXG4gICAqIEBwYXJhbSB7P30gbWVzc2FnZSAtIE1lc3NhZ2UgdG8gc2VuZC5cbiAgICogQHJldHVybnMge3ZvaWR9XG4gICAqL1xuICBzZW5kKG1lc3NhZ2UpIHtcbiAgICB0aGlzLnNvY2tldC53cml0ZShgJHtKU09OLnN0cmluZ2lmeShtZXNzYWdlKX1cXG5gKVxuICB9XG5cbiAgLyoqXG4gICAqIFJ1bnMgY2xvc2UuXG4gICAqIEByZXR1cm5zIHt2b2lkfVxuICAgKi9cbiAgY2xvc2UoKSB7XG4gICAgdGhpcy5zb2NrZXQuZW5kKClcbiAgfVxufVxuIl19
|
|
@@ -5,18 +5,24 @@ export default class BackgroundJobsMain {
|
|
|
5
5
|
* @param {import("../configuration.js").default} args.configuration - Configuration.
|
|
6
6
|
* @param {string} [args.host] - Hostname.
|
|
7
7
|
* @param {number} [args.port] - Port.
|
|
8
|
+
* @param {number} [args.workerStaleTimeoutMs] - Override how long a silent worker may go before being dropped (default 60000ms).
|
|
9
|
+
* @param {number} [args.workerLivenessSweepMs] - Override how often stale workers are swept for (default 15000ms).
|
|
8
10
|
*/
|
|
9
|
-
constructor({ configuration, host, port }: {
|
|
11
|
+
constructor({ configuration, host, port, workerStaleTimeoutMs, workerLivenessSweepMs }: {
|
|
10
12
|
configuration: import("../configuration.js").default;
|
|
11
13
|
host?: string | undefined;
|
|
12
14
|
port?: number | undefined;
|
|
15
|
+
workerStaleTimeoutMs?: number | undefined;
|
|
16
|
+
workerLivenessSweepMs?: number | undefined;
|
|
13
17
|
});
|
|
14
18
|
configuration: import("../configuration.js").default;
|
|
15
19
|
host: string;
|
|
16
20
|
port: number;
|
|
17
21
|
dispatchStrategy: import("../configuration-types.js").BackgroundJobsDispatchStrategy;
|
|
18
22
|
pollIntervalMs: number;
|
|
19
|
-
retention: import("../configuration-types.js").BackgroundJobsRetentionConfiguration;
|
|
23
|
+
retention: import("../configuration-types.js").BackgroundJobsRetentionConfiguration & import("../configuration-types.js").ResolvedBackgroundJobsRetentionConfiguration;
|
|
24
|
+
workerStaleTimeoutMs: number;
|
|
25
|
+
workerLivenessSweepMs: number;
|
|
20
26
|
store: BackgroundJobsStore;
|
|
21
27
|
logger: Logger;
|
|
22
28
|
/**
|
|
@@ -53,8 +59,8 @@ export default class BackgroundJobsMain {
|
|
|
53
59
|
_orphanTimer: ReturnType<typeof setTimeout> | undefined;
|
|
54
60
|
/**
|
|
55
61
|
* Narrows the runtime value to the documented type.
|
|
56
|
-
* @type {ReturnType<typeof
|
|
57
|
-
|
|
62
|
+
* @type {ReturnType<typeof setInterval> | undefined} */
|
|
63
|
+
_workerStaleTimer: ReturnType<typeof setInterval> | undefined;
|
|
58
64
|
/**
|
|
59
65
|
* Narrows the runtime value to the documented type.
|
|
60
66
|
* @type {BackgroundJobsScheduler | undefined} */
|
|
@@ -466,12 +472,15 @@ export default class BackgroundJobsMain {
|
|
|
466
472
|
_armScheduledTimer(): Promise<void>;
|
|
467
473
|
_sweepOrphans(): Promise<void>;
|
|
468
474
|
/**
|
|
469
|
-
*
|
|
470
|
-
*
|
|
471
|
-
*
|
|
472
|
-
*
|
|
475
|
+
* Drops workers that have gone silent past `workerStaleTimeoutMs` (no
|
|
476
|
+
* heartbeat, ready, or report). A wedged worker keeps its socket open, so the
|
|
477
|
+
* `close`-based cleanup never fires and its in-flight leases — and the whole
|
|
478
|
+
* queue — stay stuck until a human notices. Releasing the lost worker's
|
|
479
|
+
* leases lets its jobs run elsewhere and stops dispatch to it; the worker's
|
|
480
|
+
* own process lifecycle is the supervisor's concern.
|
|
481
|
+
* @returns {Promise<void>} - Resolves after the sweep.
|
|
473
482
|
*/
|
|
474
|
-
|
|
483
|
+
_sweepStaleWorkers(): Promise<void>;
|
|
475
484
|
}
|
|
476
485
|
/**
|
|
477
486
|
* WorkerExecutionModeCapability type.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"main.d.ts","sourceRoot":"","sources":["../../../src/background-jobs/main.js"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"main.d.ts","sourceRoot":"","sources":["../../../src/background-jobs/main.js"],"names":[],"mappings":"AA8CA;IACE;;;;;;;;OAQG;IACH,wFANG;QAAoD,aAAa,EAAzD,OAAO,qBAAqB,EAAE,OAAO;QACvB,IAAI;QACJ,IAAI;QACJ,oBAAoB;QACpB,qBAAqB;KAC7C,EAsEA;IApEC,qDAAkC;IAElC,aAA+B;IAC/B,aAAyD;IACzD,qFAA+C;IAC/C,uBAA2C;IAC3C,uKAAiC;IAGjC,6BAAkJ;IAClJ,8BAAuJ;IACvJ,2BAAoG;IACpG,eAA8B;IAC9B;;iCAE6B;IAC7B,SADU,GAAG,CAAC,UAAU,CAAC,CACD;IACxB;;iCAE6B;IAC7B,cADU,GAAG,CAAC,UAAU,CAAC,CACI;IAC7B;;sDAEkD;IAClD,gBADU,GAAG,CAAC,UAAU,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CACf;IAC/B;;wCAEoC;IACpC,QADU,GAAG,CAAC,MAAM,GAAG,SAAS,CACT;IACvB;;2DAEuD;IACvD,YADU,UAAU,CAAC,OAAO,UAAU,CAAC,GAAG,SAAS,CACxB;IAC3B;;2DAEuD;IACvD,iBADU,UAAU,CAAC,OAAO,UAAU,CAAC,GAAG,SAAS,CACnB;IAChC;;2DAEuD;IACvD,kBADU,UAAU,CAAC,OAAO,UAAU,CAAC,GAAG,SAAS,CAClB;IACjC;;2DAEuD;IACvD,cADU,UAAU,CAAC,OAAO,UAAU,CAAC,GAAG,SAAS,CACtB;IAC7B;;4DAEwD;IACxD,mBADU,UAAU,CAAC,OAAO,WAAW,CAAC,GAAG,SAAS,CAClB;IAClC;;qDAEiD;IACjD,WADU,uBAAuB,GAAG,SAAS,CACnB;IAC1B,mBAAsB;IACtB,wBAA2B;IAC3B,kBAAqB;IACrB;;0CAEsC;IACtC,oBADU,CAAC,MAAM,IAAI,CAAC,GAAG,SAAS,CACC;IACnC;;2DAEuD;IACvD,uBADU,CAAC,CAAC,GAAG,IAAI,EAAE,KAAK,CAAC,OAAC,CAAC,KAAK,IAAI,CAAC,GAAG,SAAS,CACb;IACtC;;sHAEkH;IAClH,eADU,OAAO,qBAAqB,EAAE,OAAO,GAAG,OAAO,gCAAgC,EAAE,OAAO,GAAG,SAAS,CAChF;IAGhC;;;OAGG;IACH,SAFa,OAAO,CAAC,IAAI,CAAC,CAqEzB;IAED;;;OAGG;IACH,QAFa,OAAO,CAAC,IAAI,CAAC,CAWzB;IAED;;yBAEqB;IACrB,iBADa,IAAI,CAKhB;IAED;;yBAEqB;IACrB,gBADa,IAAI,CAYhB;IAED;;yBAEqB;IACrB,6BADa,IAAI,CAYhB;IAED;;kCAE8B;IAC9B,wBADa,OAAO,CAAC,IAAI,CAAC,CAOzB;IAED;;kCAE8B;IAC9B,sCADa,OAAO,CAAC,IAAI,CAAC,CAOzB;IAED;;kCAE8B;IAC9B,gBADa,OAAO,CAAC,IAAI,CAAC,CAOzB;IAED;;;OAGG;IACH,WAFa,MAAM,CAIlB;IAED;;;;;;;;;;OAUG;IACH,0BAFa,IAAI,CA2BhB;IAED;;;;;;OAMG;IACH,mBAFa,IAAI,CAiBhB;IAED;;;;OAIG;IACH,0BAHW,OAAO,KAAK,EAAE,MAAM,GAClB,IAAI,CA0BhB;IAED;;;;;;;OAOG;IACH,oDALG;QAAyB,UAAU,EAA3B,UAAU;QAC4C,OAAO,EAA7D,OAAO,YAAY,EAAE,0BAA0B;QACW,IAAI,EAA9D,OAAO,YAAY,EAAE,uBAAuB,GAAG,IAAI;KAC3D,GAAU,OAAO,YAAY,EAAE,uBAAuB,GAAG,IAAI,CAS/D;IAED;;;;;;OAMG;IACH,sDAJG;QAAyB,UAAU,EAA3B,UAAU;QAC4C,OAAO,EAA7D,OAAO,YAAY,EAAE,0BAA0B;KACvD,GAAU,OAAO,YAAY,EAAE,uBAAuB,GAAG,IAAI,CAe/D;IAED;;;;;;OAMG;IACH,oDAJG;QAAyB,UAAU,EAA3B,UAAU;QAC4C,OAAO,EAA7D,OAAO,YAAY,EAAE,0BAA0B;KACvD,GAAU,IAAI,CAMhB;IAED;;;;;;OAMG;IACH,oDAJG;QAAyB,UAAU,EAA3B,UAAU;QAC4C,OAAO,EAA7D,OAAO,YAAY,EAAE,0BAA0B;KACvD,GAAU,IAAI,CAsBhB;IAED;;;;;;OAMG;IACH,sDAJG;QAAyB,UAAU,EAA3B,UAAU;QAC4C,OAAO,EAA7D,OAAO,YAAY,EAAE,0BAA0B;KACvD,GAAU,IAAI,CAWhB;IAED;;;;;;OAMG;IACH,4CAJG;QAAyB,UAAU,EAA3B,UAAU;QAC2C,OAAO,EAA5D,OAAO,YAAY,EAAE,yBAAyB;KACtD,GAAU,IAAI,CAYhB;IAED;;;;;OAKG;IACH,sCAHG;QAAyB,UAAU,EAA3B,UAAU;KAClB,GAAU,IAAI,CAOhB;IAED;;;;OAIG;IACH,kCAHW,UAAU,GACR,OAAO,CAAC,IAAI,CAAC,CAiBzB;IAED;;;;OAIG;IACH,+BAHW,UAAU,GACR,OAAO,CAAC,IAAI,CAAC,CAiBzB;IAED;;;;;;;OAOG;IACH,8CALG;QAAqB,SAAS,EAAtB,MAAM;QACO,KAAK,EAAlB,MAAM;QACW,MAAM,EAAvB,UAAU;KAClB,GAAU,OAAO,CAAC,IAAI,CAAC,CAQzB;IAED;;;;;;OAMG;IACH,qCAJG;QAAqB,SAAS,EAAtB,MAAM;QACO,KAAK,EAAlB,MAAM;KACd,GAAU,IAAI,CAUhB;IAED;;;;OAIG;IACH,kCAHW,OAAC,GACC,IAAI,CAUhB;IAED;;;;;;OAMG;IACH,wCAJG;QAAyB,UAAU,EAA3B,UAAU;QAC6C,OAAO,EAA9D,OAAO,YAAY,EAAE,2BAA2B;KACxD,GAAU,OAAO,CAAC,IAAI,CAAC,CAiBzB;IAED;;;;;;OAMG;IACH,4CAJG;QAAyB,UAAU,EAA3B,UAAU;QAC8C,OAAO,EAA/D,OAAO,YAAY,EAAE,4BAA4B;KACzD,GAAU,OAAO,CAAC,IAAI,CAAC,CAkBzB;IAED;;;;;;OAMG;IACH,0CAJG;QAAyB,UAAU,EAA3B,UAAU;QAC4C,OAAO,EAA7D,OAAO,YAAY,EAAE,0BAA0B;KACvD,GAAU,OAAO,CAAC,IAAI,CAAC,CAkCzB;IAED;;;;OAIG;IACH,6EAHW;QAAC,KAAK,EAAE,OAAC,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,aAAa,CAAC,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,OAAO,YAAY,EAAE,gBAAgB,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAC,GACnH,IAAI,CAyBhB;IAED;;;;OAIG;IACH,8BAHW,OAAC,GACC,KAAK,CAMjB;IAED;;;;OAIG;IACH,gCAHW,OAAC,GACC,KAAK,CASjB;IAED;;;;OAIG;IACH,kCAHW,OAAC,GACC,MAAM,CAMlB;IAED;;;;OAIG;IACH,yBAHW,OAAC,GACC,KAAK,IAAI,MAAM,CAI3B;IAED;;;;;;OAMG;IACH,oDAJG;QAAgB,KAAK,EAAb,OAAC;QACW,eAAe,EAA3B,KAAK;KACb,GAAU,IAAI,CAIhB;IAED;;;;;;;;;;;;;;;;OAgBG;IACH,UAFa,OAAO,CAAC,IAAI,CAAC,CAQzB;IAED;;;OAGG;IACH,eAFa,OAAO,CAQnB;IAED;;;;;OAKG;IACH,0BAHG;QAAsB,OAAO,EAArB,OAAO;KACf,GAAU,OAAO,CAAC,IAAI,CAAC,CAOzB;IAED;;;OAGG;IACH,6BAFa,OAAO,CAAC,IAAI,CAAC,CAYzB;IAED;;yBAEqB;IACrB,yBADa,IAAI,CAUhB;IAED;;;OAGG;IACH,+BAFa,OAAO,CAOnB;IAED;;;OAGG;IACH,mBAFa,OAAO,CAAC,OAAO,CAAC,CAQ5B;IAED;;;OAGG;IACH,iBAFa,OAAO,CAAC,OAAO,CAAC,CAW5B;IAED;;;OAGG;IACH,6BAFa,OAAO,CAAC,OAAO,CAAC,CAU5B;IAED;;;;;;OAMG;IACH,uBAFa,IAAI,CAWhB;IAED;;;OAGG;IACH,oBAFa,OAAO,CAAC,IAAI,CAAC,CAgBzB;IAED;;;;OAIG;IACH,cAFa,OAAO,CAAC,IAAI,CAAC,CAwDzB;IAED;;;OAGG;IACH,mCAFa,OAAO,CAAC,OAAO,YAAY,EAAE,gBAAgB,GAAG,IAAI,CAAC,CASjE;IAED;;;OAGG;IACH,6BAFa,OAAO,YAAY,EAAE,0BAA0B,EAAE,CAU7D;IAED;;;;;;OAMG;IACH,uDAJG;QAAmE,cAAc,EAAzE,GAAG,CAAC,OAAO,YAAY,EAAE,0BAA0B,CAAC;QACnC,MAAM,EAAvB,UAAU;KAClB,GAAU,IAAI,CAQhB;IAED;;;;OAIG;IACH,uBAHW,OAAO,YAAY,EAAE,gBAAgB,GACnC,UAAU,GAAG,SAAS,CAMlC;IAED;;;;;;OAMG;IACH,mCAJG;QAAoD,GAAG,EAA/C,OAAO,YAAY,EAAE,gBAAgB;QACpB,MAAM,EAAvB,UAAU;KAClB,GAAU,OAAO,CAUnB;IAED;;;;;;OAMG;IACH,sBAFa,OAAO,CAAC,IAAI,CAAC,CAoBzB;IAED,+BAcC;IAED;;;;;;;;OAQG;IACH,sBAFa,OAAO,CAAC,IAAI,CAAC,CAgCzB;CACF;;;;;;;;mBAtnCa,OAAO,YAAY,EAAE,0BAA0B;;;;aAC/C,CAAC,MAAM,EAAE,UAAU,KAAK,OAAO;;gCA3Bb,YAAY;mBACzB,cAAc;uBAHV,kBAAkB;gBADzB,KAAK;oCAEe,gBAAgB"}
|