vona-module-a-queue 5.1.18 → 5.1.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +154 -8
- package/dist/index.js.map +1 -1
- package/dist/service/queue.d.ts +9 -1
- package/dist/service/queue.d.ts.map +1 -1
- package/package.json +2 -2
- package/src/service/queue.ts +227 -30
package/dist/index.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { BeanInfo, BeanBase, uuidv4, deepExtend, instanceDesp, beanFullNameFromOnionName, BeanSimple, BeanScopeBase, createBeanDecorator } from 'vona';
|
|
1
|
+
import { BeanInfo, BeanBase, getTestAsyncCompletion, withoutTestAsyncCompletion, uuidv4, deepExtend, instanceDesp, beanFullNameFromOnionName, BeanSimple, BeanScopeBase, createBeanDecorator } from 'vona';
|
|
2
2
|
import { __decorate } from 'tslib';
|
|
3
3
|
import * as Bull from 'bullmq';
|
|
4
4
|
import { Service, Scope } from 'vona-module-a-bean';
|
|
@@ -12,6 +12,8 @@ let ServiceQueue = class ServiceQueue extends BeanBase {
|
|
|
12
12
|
this._workers = {};
|
|
13
13
|
this._queues = {};
|
|
14
14
|
this._queueCallbacks = {};
|
|
15
|
+
this._ownedJobs = new Map();
|
|
16
|
+
this._ownedHandles = new Map();
|
|
15
17
|
}
|
|
16
18
|
push(info) {
|
|
17
19
|
if (!info.options?.dbInfo) throw new Error('should specify the options.dbInfo');
|
|
@@ -49,6 +51,69 @@ let ServiceQueue = class ServiceQueue extends BeanBase {
|
|
|
49
51
|
}
|
|
50
52
|
this._queues = {};
|
|
51
53
|
}
|
|
54
|
+
async releaseOwnedInstanceHandles(owner) {
|
|
55
|
+
if (owner.app !== this.app) throw new Error('Queue completion owner belongs to another app');
|
|
56
|
+
owner.assertSettled();
|
|
57
|
+
for (const job of this._ownedJobs.values()) {
|
|
58
|
+
if (job.owner === owner) throw new Error(`Owned queue job is still pending: ${job.jobId}`);
|
|
59
|
+
}
|
|
60
|
+
const handles = this._ownedHandles.get(owner);
|
|
61
|
+
if (!handles) return;
|
|
62
|
+
for (const [queueKey, entry] of handles) {
|
|
63
|
+
if (entry.prefix !== this._getPrefix() || entry.worker && this._workers[queueKey] !== entry.worker || entry.queue && this._queues[queueKey] !== entry.queue) {
|
|
64
|
+
throw new Error(`Owned queue handles changed: ${queueKey}`);
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
for (const [queueKey, entry] of handles) {
|
|
68
|
+
if (entry.worker) {
|
|
69
|
+
await entry.worker.worker.close();
|
|
70
|
+
delete this._workers[queueKey];
|
|
71
|
+
entry.worker = undefined;
|
|
72
|
+
}
|
|
73
|
+
if (entry.queue) {
|
|
74
|
+
await entry.queue.queue.close();
|
|
75
|
+
await entry.queue.queueEvents.close();
|
|
76
|
+
delete this._queues[queueKey];
|
|
77
|
+
entry.queue = undefined;
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
this._ownedHandles.delete(owner);
|
|
81
|
+
}
|
|
82
|
+
_getOwnedHandles(info, queueKey) {
|
|
83
|
+
const owner = getTestAsyncCompletion(this.app);
|
|
84
|
+
// Global/default handles belong to the runner, even during an owned operation.
|
|
85
|
+
if (!owner || info.options?.instanceName !== owner.instanceName) return;
|
|
86
|
+
let handles = this._ownedHandles.get(owner);
|
|
87
|
+
let entry = handles?.get(queueKey);
|
|
88
|
+
if (this._workers[queueKey] && this._workers[queueKey] !== entry?.worker || this._queues[queueKey] && this._queues[queueKey] !== entry?.queue || entry && entry.prefix !== this._getPrefix()) {
|
|
89
|
+
throw new Error(`Cannot share preexisting owned-instance queue handles: ${queueKey}`);
|
|
90
|
+
}
|
|
91
|
+
if (!handles) {
|
|
92
|
+
handles = new Map();
|
|
93
|
+
this._ownedHandles.set(owner, handles);
|
|
94
|
+
}
|
|
95
|
+
if (!entry) {
|
|
96
|
+
entry = {
|
|
97
|
+
prefix: this._getPrefix()
|
|
98
|
+
};
|
|
99
|
+
handles.set(queueKey, entry);
|
|
100
|
+
}
|
|
101
|
+
return entry;
|
|
102
|
+
}
|
|
103
|
+
_ownedJobKey(prefix, queueKey, jobId) {
|
|
104
|
+
return `${prefix}||${queueKey}||${jobId}`;
|
|
105
|
+
}
|
|
106
|
+
_completeOwnedJob(prefix, queueKey, jobId, error) {
|
|
107
|
+
const key = this._ownedJobKey(prefix, queueKey, jobId);
|
|
108
|
+
const job = this._ownedJobs.get(key);
|
|
109
|
+
if (!job) return;
|
|
110
|
+
job.terminal = {
|
|
111
|
+
error
|
|
112
|
+
};
|
|
113
|
+
if (job.executions) return;
|
|
114
|
+
this._ownedJobs.delete(key);
|
|
115
|
+
job.ticket.complete(error);
|
|
116
|
+
}
|
|
52
117
|
_getPrefix() {
|
|
53
118
|
return `${getRedisClientKeyPrefix('bull', this.app)}queue`;
|
|
54
119
|
}
|
|
@@ -75,7 +140,7 @@ let ServiceQueue = class ServiceQueue extends BeanBase {
|
|
|
75
140
|
connection: connectionWorker
|
|
76
141
|
});
|
|
77
142
|
_worker.workerOptions = _workerOptions;
|
|
78
|
-
_worker.worker = new Bull.Worker(queueKey, async job => {
|
|
143
|
+
_worker.worker = withoutTestAsyncCompletion(() => new Bull.Worker(queueKey, async job => {
|
|
79
144
|
// concurrency
|
|
80
145
|
if (queueConfig?.concurrency) {
|
|
81
146
|
return await this._performTask(job);
|
|
@@ -91,7 +156,7 @@ let ServiceQueue = class ServiceQueue extends BeanBase {
|
|
|
91
156
|
redlock: _worker.redlock,
|
|
92
157
|
lockTTL: _lockTTL
|
|
93
158
|
});
|
|
94
|
-
}, _workerOptions);
|
|
159
|
+
}, _workerOptions));
|
|
95
160
|
_worker.worker.on('failed', (_job, err) => {
|
|
96
161
|
this.app.handleError(err);
|
|
97
162
|
});
|
|
@@ -134,17 +199,20 @@ let ServiceQueue = class ServiceQueue extends BeanBase {
|
|
|
134
199
|
connection: connectionEvents
|
|
135
200
|
};
|
|
136
201
|
_queue.queueEventsOptions = _queueEventsOptions;
|
|
137
|
-
_queue.queueEvents = new Bull.QueueEvents(queueKey, _queueEventsOptions);
|
|
202
|
+
_queue.queueEvents = withoutTestAsyncCompletion(() => new Bull.QueueEvents(queueKey, _queueEventsOptions));
|
|
138
203
|
_queue.queueEvents.on('completed', ({
|
|
139
204
|
jobId,
|
|
140
205
|
returnvalue
|
|
141
206
|
}) => {
|
|
207
|
+
this._completeOwnedJob(prefix, queueKey, jobId);
|
|
142
208
|
this._callCallback(jobId, undefined, returnvalue);
|
|
143
209
|
});
|
|
210
|
+
// QueueEvents reports terminal failure; Worker 'failed' also fires before retries.
|
|
144
211
|
_queue.queueEvents.on('failed', ({
|
|
145
212
|
jobId,
|
|
146
213
|
failedReason
|
|
147
214
|
}) => {
|
|
215
|
+
this._completeOwnedJob(prefix, queueKey, jobId, new Error(failedReason));
|
|
148
216
|
this._callCallback(jobId, failedReason, undefined);
|
|
149
217
|
});
|
|
150
218
|
// ok
|
|
@@ -153,9 +221,11 @@ let ServiceQueue = class ServiceQueue extends BeanBase {
|
|
|
153
221
|
_ensureWorker(info) {
|
|
154
222
|
// queueKey
|
|
155
223
|
const queueKey = this._combineQueueKey(info);
|
|
224
|
+
const ownedHandles = this._getOwnedHandles(info, queueKey);
|
|
156
225
|
// worker
|
|
157
226
|
if (!this._workers[queueKey]) {
|
|
158
227
|
this._workers[queueKey] = this._createWorker(info, queueKey);
|
|
228
|
+
if (ownedHandles) ownedHandles.worker = this._workers[queueKey];
|
|
159
229
|
}
|
|
160
230
|
}
|
|
161
231
|
_ensureQueue(info) {
|
|
@@ -163,9 +233,11 @@ let ServiceQueue = class ServiceQueue extends BeanBase {
|
|
|
163
233
|
this._ensureWorker(info);
|
|
164
234
|
// queueKey
|
|
165
235
|
const queueKey = this._combineQueueKey(info);
|
|
236
|
+
const ownedHandles = this._getOwnedHandles(info, queueKey);
|
|
166
237
|
// queue
|
|
167
238
|
if (!this._queues[queueKey]) {
|
|
168
239
|
this._queues[queueKey] = this._createQueue(info, queueKey);
|
|
240
|
+
if (ownedHandles) ownedHandles.queue = this._queues[queueKey];
|
|
169
241
|
}
|
|
170
242
|
// ok
|
|
171
243
|
return this._queues[queueKey];
|
|
@@ -190,6 +262,11 @@ let ServiceQueue = class ServiceQueue extends BeanBase {
|
|
|
190
262
|
}
|
|
191
263
|
}
|
|
192
264
|
async _queuePush(info, isAsync) {
|
|
265
|
+
const owner = isAsync ? getTestAsyncCompletion(this.app) : undefined;
|
|
266
|
+
let ticket;
|
|
267
|
+
let ownedJob;
|
|
268
|
+
let addAttempted = false;
|
|
269
|
+
let callbackJobId;
|
|
193
270
|
const telemetry = this.$scope.telemetry.service.telemetry;
|
|
194
271
|
const span = telemetry.enabled ? telemetry.startSpan(`queue send ${String(info.queueName)}`, {
|
|
195
272
|
kind: 3,
|
|
@@ -209,19 +286,43 @@ let ServiceQueue = class ServiceQueue extends BeanBase {
|
|
|
209
286
|
}
|
|
210
287
|
let callback;
|
|
211
288
|
try {
|
|
289
|
+
ticket = owner?.reserve(`queue ${String(info.queueName)}`);
|
|
290
|
+
if (owner) {
|
|
291
|
+
owner.assertScope(info.options);
|
|
292
|
+
if (info.queueName !== 'a-orm:doubleDelete') {
|
|
293
|
+
throw new Error(`Unsupported owned queue: ${String(info.queueName)}`);
|
|
294
|
+
}
|
|
295
|
+
this._assertOwnedDatasource(info, owner);
|
|
296
|
+
}
|
|
212
297
|
// queue config
|
|
213
298
|
const queueConfig = this.bean.onion.queue.getOnionOptions(info.queueName);
|
|
214
299
|
// queueConfig.options: queue/worker/job/limiter
|
|
215
300
|
const jobOptionsBase = queueConfig?.options?.job;
|
|
216
|
-
// queue
|
|
217
|
-
const queueQueue = this._ensureQueue(info);
|
|
218
|
-
const queue = queueQueue.queue;
|
|
219
301
|
// job
|
|
220
|
-
const jobId = info.options?.jobOptions?.jobId || uuidv4();
|
|
302
|
+
const jobId = owner ? uuidv4() : info.options?.jobOptions?.jobId || uuidv4();
|
|
221
303
|
const jobName = info.options?.jobName || jobId;
|
|
222
304
|
const jobOptions = deepExtend({
|
|
223
305
|
jobId
|
|
224
306
|
}, jobOptionsBase, info.options?.jobOptions);
|
|
307
|
+
if (owner) {
|
|
308
|
+
if (jobOptions.jobId !== jobId || jobOptions.repeat || jobOptions.deduplication || jobOptions.debounce) {
|
|
309
|
+
throw new Error('Owned queue jobs require a local UUID without repeat or deduplication');
|
|
310
|
+
}
|
|
311
|
+
const prefix = this._getPrefix();
|
|
312
|
+
const queueKey = this._combineQueueKey(info);
|
|
313
|
+
ownedJob = {
|
|
314
|
+
owner,
|
|
315
|
+
ticket: ticket,
|
|
316
|
+
prefix,
|
|
317
|
+
queueKey,
|
|
318
|
+
jobId,
|
|
319
|
+
executions: 0
|
|
320
|
+
};
|
|
321
|
+
this._ownedJobs.set(this._ownedJobKey(prefix, queueKey, jobId), ownedJob);
|
|
322
|
+
}
|
|
323
|
+
// Register ownership before creating handles or awaiting QueueEvents readiness.
|
|
324
|
+
const queueQueue = this._ensureQueue(info);
|
|
325
|
+
const queue = queueQueue.queue;
|
|
225
326
|
// should not change info, hold original info.options?.jobName, info.options?.jobOptions
|
|
226
327
|
// info = deepExtend({}, info, { options: { jobName, jobOptions } });
|
|
227
328
|
if (!isAsync) {
|
|
@@ -229,6 +330,7 @@ let ServiceQueue = class ServiceQueue extends BeanBase {
|
|
|
229
330
|
return undefined;
|
|
230
331
|
}
|
|
231
332
|
await this._queueEventsReady(queueQueue);
|
|
333
|
+
callbackJobId = jobId;
|
|
232
334
|
callback = new Promise((resolve, reject) => {
|
|
233
335
|
this._queueCallbacks[jobId] = {
|
|
234
336
|
info,
|
|
@@ -238,8 +340,26 @@ let ServiceQueue = class ServiceQueue extends BeanBase {
|
|
|
238
340
|
}
|
|
239
341
|
};
|
|
240
342
|
});
|
|
343
|
+
// Observe terminal rejection even if queue.add rejects before we can await callback.
|
|
344
|
+
if (owner) void callback.catch(() => {});
|
|
345
|
+
addAttempted = true;
|
|
241
346
|
await queue.add(jobName, info, jobOptions);
|
|
242
347
|
} catch (err) {
|
|
348
|
+
if (owner && ticket) {
|
|
349
|
+
if (addAttempted) {
|
|
350
|
+
// A transport error cannot prove the job was not enqueued. Keep ownership
|
|
351
|
+
// pending until its actual terminal event, but retain the send failure.
|
|
352
|
+
void owner.track(`queue add ${ownedJob?.jobId}`, async () => {
|
|
353
|
+
throw err;
|
|
354
|
+
}).catch(() => {});
|
|
355
|
+
} else {
|
|
356
|
+
if (ownedJob) {
|
|
357
|
+
this._ownedJobs.delete(this._ownedJobKey(ownedJob.prefix, ownedJob.queueKey, ownedJob.jobId));
|
|
358
|
+
}
|
|
359
|
+
if (callbackJobId) delete this._queueCallbacks[callbackJobId];
|
|
360
|
+
ticket.complete(err);
|
|
361
|
+
}
|
|
362
|
+
}
|
|
243
363
|
if (span) telemetry.recordException(span, err);
|
|
244
364
|
throw err;
|
|
245
365
|
} finally {
|
|
@@ -257,6 +377,32 @@ let ServiceQueue = class ServiceQueue extends BeanBase {
|
|
|
257
377
|
return `${instanceName}||${beanFullNameFromOnionName(info.queueName, 'queue')}`;
|
|
258
378
|
}
|
|
259
379
|
async _performTask(job) {
|
|
380
|
+
const ownedJob = job.id ? this._ownedJobs.get(this._ownedJobKey(job.prefix, job.queueName, job.id)) : undefined;
|
|
381
|
+
if (!ownedJob) return await withoutTestAsyncCompletion(() => this._performTaskInner(job));
|
|
382
|
+
ownedJob.executions++;
|
|
383
|
+
try {
|
|
384
|
+
return await ownedJob.owner.bind(async () => {
|
|
385
|
+
ownedJob.owner.assertScope(job.data.options);
|
|
386
|
+
if (this._combineQueueKey(job.data) !== ownedJob.queueKey) {
|
|
387
|
+
throw new Error('Owned queue execution key does not match its registered scope');
|
|
388
|
+
}
|
|
389
|
+
this._assertOwnedDatasource(job.data, ownedJob.owner);
|
|
390
|
+
return await this._performTaskInner(job);
|
|
391
|
+
});
|
|
392
|
+
} finally {
|
|
393
|
+
ownedJob.executions--;
|
|
394
|
+
if (ownedJob.terminal) {
|
|
395
|
+
this._completeOwnedJob(ownedJob.prefix, ownedJob.queueKey, ownedJob.jobId, ownedJob.terminal.error);
|
|
396
|
+
}
|
|
397
|
+
}
|
|
398
|
+
}
|
|
399
|
+
_assertOwnedDatasource(info, owner) {
|
|
400
|
+
// This supported queue executes against data.clientName, not just newCtx's dbInfo.
|
|
401
|
+
if (info.queueName !== 'a-orm:doubleDelete' || info.data?.clientName !== owner.clientName) {
|
|
402
|
+
throw new Error('Owned queue datasource does not match its completion owner');
|
|
403
|
+
}
|
|
404
|
+
}
|
|
405
|
+
async _performTaskInner(job) {
|
|
260
406
|
const info = job.data;
|
|
261
407
|
const telemetry = this.$scope.telemetry.service.telemetry;
|
|
262
408
|
const parent = telemetry.extractCarrier(info.options?.telemetry);
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sources":["../src/service/queue.ts","../src/bean/hmr.queue.ts","../src/bean/startup.loadQueueWorkers.ts","../src/config/config.ts","../src/.metadata/this.ts","../src/monkey.ts","../src/.metadata/index.ts","../src/lib/beanQueueBase.ts","../src/lib/queue.ts"],"sourcesContent":[null,null,null,null,null,null,null,null,null],"names":["ServiceQueue","BeanBase","constructor","args","_workers","_queues","_queueCallbacks","push","info","options","dbInfo","Error","_queuePush","pushAsync","loadQueueWorkers","instanceName","queueItem","bean","onion","queue","getOnionsEnabledCached","queueName","name","data","undefined","_ensureWorker","clearWorkers","queueKey","_worker","worker","close","clearQueues","_queue","queueEvents","_getPrefix","getRedisClientKeyPrefix","app","_createWorker","prefix","queueConfig","getOnionOptions","workerOptions","redlockOptions","redlock","_lockTTL","lockTTL","$scope","config","concurrency","service","create","connectionWorker","redis","get","_workerOptions","Object","assign","scope","connection","Bull","Worker","job","_performTask","queueNameSub","_lockResource","lock","on","_job","err","handleError","message","includes","workerInner","running","run","catch","_createQueue","queueOptions","connectionQueue","_queueOptions","Queue","connectionEvents","_queueEventsOptions","queueEventsOptions","QueueEvents","jobId","returnvalue","_callCallback","failedReason","_combineQueueKey","_ensureQueue","getQueue","_getQueue","ctx","_callback","callback","isAsync","telemetry","span","enabled","startSpan","String","kind","attributes","injectCarrier","createContext","jobOptionsBase","queueQueue","jobOptions","uuidv4","jobName","deepExtend","add","_queueEventsReady","Promise","resolve","reject","recordException","end","queueEventsReady","waitUntilReady","instanceDesp","beanFullNameFromOnionName","parent","extractCarrier","attemptsMade","getOnionSlice","execute","executor","newCtx","beanFullName","beanOptions","beanInstance","_getBean","locale","tz","extraData","state","context","transaction","withSpan","getRepeatKey","repeat","endDate","Date","getTime","pattern","suffix","every","prepareJobInfo","prepareGeneralInfo","__decorate","Service","__z_BeanInfo","module","HmrQueue","reload","_beanOptions","Hmr","StartupLoadQueueWorkers","meta","isTest","Startup","instance","after","_app","removeOnComplete","count","removeOnFail","lockDuration","maxStalledCount","stalledInterval","__ThisModule__","Monkey","BeanSimple","appClose","scopeSelf","appClosed","ScopeModuleAQueue","BeanScopeBase","Scope","BeanQueueBase","$onionName","createBeanDecorator"],"mappings":";;;;;;;;AAqBO,IAAMA,YAAY,GAAlB,MAAMA,YAAa,SAAQC,QAAQ,CAAA;AAAAC,EAAAA,WAAAA,CAAA,GAAAC,IAAA,EAAA;AAAA,IAAA,KAAA,CAAA,GAAAA,IAAA,CAAA;IAAA,IAAA,CACxCC,QAAQ,GAAgB,EAAE;IAAA,IAAA,CAC1BC,OAAO,GAAiB,EAAE;IAAA,IAAA,CAC1BC,eAAe,GAAoB,EAAE;AAAA,EAAA;EAErCC,IAAIA,CAAOC,IAA4B,EAAA;AACrC,IAAA,IAAI,CAACA,IAAI,CAACC,OAAO,EAAEC,MAAM,EAAE,MAAM,IAAIC,KAAK,CAAC,mCAAmC,CAAC;AAC/E,IAAA,OAAO,IAAI,CAACC,UAAU,CAACJ,IAAI,EAAE,KAAK,CAAC;AACrC,EAAA;AAEA;EACAK,SAASA,CAAeL,IAA4B,EAAA;AAClD,IAAA,IAAI,CAACA,IAAI,CAACC,OAAO,EAAEC,MAAM,EAAE,MAAM,IAAIC,KAAK,CAAC,mCAAmC,CAAC;AAC/E,IAAA,OAAO,IAAI,CAACC,UAAU,CAACJ,IAAI,EAAE,IAAI,CAAC;AACpC,EAAA;EAEAM,gBAAgBA,CAACC,YAAuD,EAAA;AACtE,IAAA,KAAK,MAAMC,SAAS,IAAI,IAAI,CAACC,IAAI,CAACC,KAAK,CAACC,KAAK,CAACC,sBAAsB,EAAE,EAAE;AACtE,MAAA,MAAMZ,IAAI,GAA8B;QACtCa,SAAS,EAAEL,SAAS,CAACM,IAAa;AAClCC,QAAAA,IAAI,EAAEC,SAAgB;AACtBf,QAAAA,OAAO,EAAE;AACPM,UAAAA;AACD;OACF;AACD,MAAA,IAAI,CAACU,aAAa,CAACjB,IAAI,CAAC;AAC1B,IAAA;AACF,EAAA;EAEA,MAAMkB,YAAYA,GAAA;AAChB,IAAA,KAAK,MAAMC,QAAQ,IAAI,IAAI,CAACvB,QAAQ,EAAE;AACpC,MAAA,MAAMwB,OAAO,GAAG,IAAI,CAACxB,QAAQ,CAACuB,QAAQ,CAAC;AACvC,MAAA,MAAMC,OAAO,CAACC,MAAM,CAACC,KAAK,EAAE;AAC9B,IAAA;AACA,IAAA,IAAI,CAAC1B,QAAQ,GAAG,EAAE;AACpB,EAAA;EAEA,MAAM2B,WAAWA,GAAA;AACf,IAAA,KAAK,MAAMJ,QAAQ,IAAI,IAAI,CAACtB,OAAO,EAAE;AACnC,MAAA,MAAM2B,MAAM,GAAG,IAAI,CAAC3B,OAAO,CAACsB,QAAQ,CAAC;AACrC,MAAA,MAAMK,MAAM,CAACb,KAAK,CAACW,KAAK,EAAE;AAC1B,MAAA,MAAME,MAAM,CAACC,WAAW,CAACH,KAAK,EAAE;AAClC,IAAA;AACA,IAAA,IAAI,CAACzB,OAAO,GAAG,EAAE;AACnB,EAAA;AAEA6B,EAAAA,UAAUA,GAAA;IACR,OAAO,CAAA,EAAGC,uBAAuB,CAAC,MAAM,EAAE,IAAI,CAACC,GAAG,CAAC,CAAA,KAAA,CAAO;AAC5D,EAAA;AAEAC,EAAAA,aAAaA,CAAO7B,IAA4B,EAAEmB,QAAgB,EAAA;AAChE,IAAA,MAAMS,GAAG,GAAG,IAAI,CAACA,GAAG;AACpB;IACA,MAAMR,OAAO,GAAG,EAAgB;AAChC;AACA,IAAA,MAAMU,MAAM,GAAG,IAAI,CAACJ,UAAU,EAAE;AAChC;AACA,IAAA,MAAMK,WAAW,GAAGH,GAAG,CAACnB,IAAI,CAACC,KAAK,CAACC,KAAK,CAACqB,eAAe,CAAChC,IAAI,CAACa,SAAS,CAAC;AACxE;AACA,IAAA,MAAMoB,aAAa,GAAGF,WAAW,EAAE9B,OAAO,EAAEoB,MAAM;AAClD,IAAA,MAAMa,cAAc,GAAGH,WAAW,EAAE9B,OAAO,EAAEkC,OAAO;AACpD,IAAA,MAAMC,QAAQ,GAAGF,cAAc,EAAEG,OAAO,IAAI,IAAI,CAACC,MAAM,CAACH,OAAO,CAACI,MAAM,CAACF,OAAO;AAE9E;AACA,IAAA,IAAI,CAACN,WAAW,EAAES,WAAW,EAAE;AAC7BpB,MAAAA,OAAO,CAACe,OAAO,GAAG,IAAI,CAACG,MAAM,CAACH,OAAO,CAACM,OAAO,CAACN,OAAO,CAACO,MAAM,CAACR,cAAc,CAAC;AAC9E,IAAA;AAEA;IACA,MAAMS,gBAAgB,GAAGf,GAAG,CAACnB,IAAI,CAACmC,KAAK,CAACC,GAAG,CAAC,OAAO,CAAC;AACpD,IAAA,MAAMC,cAAc,GAAGC,MAAM,CAACC,MAAM,CAAC,EAAE,EAAE,IAAI,CAACC,KAAK,CAACV,MAAM,CAAClB,MAAM,EAAEY,aAAa,EAAE;MAChFH,MAAM;AACNoB,MAAAA,UAAU,EAAEP;AACb,KAAA,CAAC;IACFvB,OAAO,CAACa,aAAa,GAAGa,cAAc;AACtC1B,IAAAA,OAAO,CAACC,MAAM,GAAG,IAAI8B,IAAI,CAACC,MAAM,CAC9BjC,QAAQ,EACR,MAAMkC,GAAG,IAAG;AACV;MACA,IAAItB,WAAW,EAAES,WAAW,EAAE;AAC5B,QAAA,OAAO,MAAM,IAAI,CAACc,YAAY,CAACD,GAAG,CAAC;AACrC,MAAA;AACA;AACA,MAAA,MAAMrD,IAAI,GAAGqD,GAAG,CAACtC,IAA8B;AAC/C,MAAA,MAAMwC,YAAY,GAAGvD,IAAI,CAACC,OAAO,EAAEsD,YAAY;AAC/C,MAAA,MAAMC,aAAa,GAAG,CAAA,MAAA,EAASrC,QAAQ,CAAA,EAAGoC,YAAY,GAAG,CAAA,CAAA,EAAIA,YAAY,CAAA,CAAE,GAAG,EAAE,CAAA,CAAE;AAClF,MAAA,OAAO,MAAM,IAAI,CAACjB,MAAM,CAACH,OAAO,CAACM,OAAO,CAACN,OAAO,CAACsB,IAAI,CACnDD,aAAa,EACb,YAAW;AACT,QAAA,OAAO,MAAM,IAAI,CAACF,YAAY,CAACD,GAAG,CAAC;AACrC,MAAA,CAAC,EACD;AACE;QACAlB,OAAO,EAAEf,OAAO,CAACe,OAAO;AACxBE,QAAAA,OAAO,EAAED;AACV,OAAA,CACF;IACH,CAAC,EACDU,cAAc,CACf;IAED1B,OAAO,CAACC,MAAM,CAACqC,EAAE,CAAC,QAAQ,EAAE,CAACC,IAAI,EAAEC,GAAG,KAAI;AACxC,MAAA,IAAI,CAAChC,GAAG,CAACiC,WAAW,CAACD,GAAG,CAAC;AAC3B,IAAA,CAAC,CAAC;IAEFxC,OAAO,CAACC,MAAM,CAACqC,EAAE,CAAC,OAAO,EAAEE,GAAG,IAAG;AAC/B,MAAA,IAAIA,GAAG,CAACE,OAAO,IAAIF,GAAG,CAACE,OAAO,CAACC,QAAQ,CAAC,sBAAsB,CAAC,EAAE;AAC/D,QAAA,MAAMC,WAAW,GAAG5C,OAAO,CAACC,MAAa;AACzC,QAAA,IAAI,CAAC2C,WAAW,CAACC,OAAO,EAAE;UACxB7C,OAAO,CAACC,MAAM,CAAC6C,GAAG,EAAE,CAACC,KAAK,CAACP,GAAG,IAAG;AAC/B,YAAA,IAAI,CAAChC,GAAG,CAACiC,WAAW,CAACD,GAAG,CAAC;AAC3B,UAAA,CAAC,CAAC;AACJ,QAAA;AACF,MAAA;AACF,IAAA,CAAC,CAAC;AAEF;AACA,IAAA,OAAOxC,OAAO;AAChB,EAAA;AAEAgD,EAAAA,YAAYA,CAAOpE,IAA4B,EAAEmB,QAAgB,EAAA;AAC/D,IAAA,MAAMS,GAAG,GAAG,IAAI,CAACA,GAAG;AACpB;IACA,MAAMJ,MAAM,GAAG,EAAiB;AAChC;AACA,IAAA,MAAMM,MAAM,GAAG,IAAI,CAACJ,UAAU,EAAE;AAChC;AACA,IAAA,MAAMK,WAAW,GAAGH,GAAG,CAACnB,IAAI,CAACC,KAAK,CAACC,KAAK,CAACqB,eAAe,CAAChC,IAAI,CAACa,SAAS,CAAC;AACxE;AACA,IAAA,MAAMwD,YAAY,GAAGtC,WAAW,EAAE9B,OAAO,EAAEU,KAAK;AAEhD;IACA,MAAM2D,eAAe,GAAG1C,GAAG,CAACnB,IAAI,CAACmC,KAAK,CAACC,GAAG,CAAC,OAAO,CAAC;IACnD,MAAM0B,aAAa,GAAGxB,MAAM,CAACC,MAAM,CAAC,EAAE,EAAEqB,YAAY,EAAE;MAAEvC,MAAM;AAAEoB,MAAAA,UAAU,EAAEoB;AAAe,KAAE,CAAC;IAC9F9C,MAAM,CAACe,MAAM,GAAGR,WAAW;IAC3BP,MAAM,CAACvB,OAAO,GAAGsE,aAAa;IAC9B/C,MAAM,CAACb,KAAK,GAAG,IAAIwC,IAAI,CAACqB,KAAK,CAACrD,QAAQ,EAAEoD,aAAa,CAAC;AAEtD;IACA,MAAME,gBAAgB,GAAG7C,GAAG,CAACnB,IAAI,CAACmC,KAAK,CAACC,GAAG,CAAC,OAAO,CAAC;AACpD,IAAA,MAAM6B,mBAAmB,GAAG;MAAE5C,MAAM;AAAEoB,MAAAA,UAAU,EAAEuB;KAA6C;IAC/FjD,MAAM,CAACmD,kBAAkB,GAAGD,mBAAmB;IAC/ClD,MAAM,CAACC,WAAW,GAAG,IAAI0B,IAAI,CAACyB,WAAW,CAACzD,QAAQ,EAAEuD,mBAAmB,CAAC;AACxElD,IAAAA,MAAM,CAACC,WAAW,CAACiC,EAAE,CAAC,WAAW,EAAE,CAAC;MAAEmB,KAAK;AAAEC,MAAAA;AAAW,KAAE,KAAI;MAC5D,IAAI,CAACC,aAAa,CAACF,KAAK,EAAE7D,SAAS,EAAE8D,WAAW,CAAC;AACnD,IAAA,CAAC,CAAC;AACFtD,IAAAA,MAAM,CAACC,WAAW,CAACiC,EAAE,CAAC,QAAQ,EAAE,CAAC;MAAEmB,KAAK;AAAEG,MAAAA;AAAY,KAAE,KAAI;MAC1D,IAAI,CAACD,aAAa,CAACF,KAAK,EAAEG,YAAY,EAAEhE,SAAS,CAAC;AACpD,IAAA,CAAC,CAAC;AAEF;AACA,IAAA,OAAOQ,MAAM;AACf,EAAA;EAEAP,aAAaA,CAAOjB,IAA4B,EAAA;AAC9C;AACA,IAAA,MAAMmB,QAAQ,GAAG,IAAI,CAAC8D,gBAAgB,CAACjF,IAAI,CAAC;AAC5C;AACA,IAAA,IAAI,CAAC,IAAI,CAACJ,QAAQ,CAACuB,QAAQ,CAAC,EAAE;AAC5B,MAAA,IAAI,CAACvB,QAAQ,CAACuB,QAAQ,CAAC,GAAG,IAAI,CAACU,aAAa,CAAC7B,IAAI,EAAEmB,QAAQ,CAAC;AAC9D,IAAA;AACF,EAAA;EAEA+D,YAAYA,CAAOlF,IAA4B,EAAA;AAC7C;AACA,IAAA,IAAI,CAACiB,aAAa,CAACjB,IAAI,CAAC;AACxB;AACA,IAAA,MAAMmB,QAAQ,GAAG,IAAI,CAAC8D,gBAAgB,CAACjF,IAAI,CAAC;AAC5C;AACA,IAAA,IAAI,CAAC,IAAI,CAACH,OAAO,CAACsB,QAAQ,CAAC,EAAE;AAC3B,MAAA,IAAI,CAACtB,OAAO,CAACsB,QAAQ,CAAC,GAAG,IAAI,CAACiD,YAAY,CAACpE,IAAI,EAAEmB,QAAQ,CAAC;AAC5D,IAAA;AACA;AACA,IAAA,OAAO,IAAI,CAACtB,OAAO,CAACsB,QAAQ,CAAC;AAC/B,EAAA;AAEAgE,EAAAA,QAAQA,CAACtE,SAA6B,EAAEN,YAAuD,EAAA;IAC7F,OAAO,IAAI,CAAC6E,SAAS,CAAC;MACpBvE,SAAS;AACTE,MAAAA,IAAI,EAAEC,SAAgB;AACtBf,MAAAA,OAAO,EAAE;AACPM,QAAAA,YAAY,EAAEA,YAAY,IAAI,IAAI,CAAC8E,GAAG,CAAC9E;AACxC;AACF,KAAA,CAAC;AACJ,EAAA;EAEA6E,SAASA,CAAOpF,IAA4B,EAAA;AAC1C,IAAA,OAAO,IAAI,CAACkF,YAAY,CAAClF,IAAI,CAAC,CAACW,KAAK;AACtC,EAAA;AAEAoE,EAAAA,aAAaA,CACXF,KAAsB,EACtBG,YAAgC,EAChCjE,IAAsB,EAAA;AAEtB,IAAA,MAAMuE,SAAS,GAAG,IAAI,CAACxF,eAAe,CAAC+E,KAAK,CAAC;AAC7C,IAAA,IAAIS,SAAS,EAAE;AACb,MAAA,OAAO,IAAI,CAACxF,eAAe,CAAC+E,KAAK,CAAC;AAClCS,MAAAA,SAAS,CAACC,QAAQ,CAACP,YAAY,GAAG,IAAI7E,KAAK,CAAC6E,YAAY,CAAC,GAAGhE,SAAS,EAAED,IAAI,CAAC;AAC9E,IAAA;AACF,EAAA;AAEA,EAAA,MAAMX,UAAUA,CAAeJ,IAA4B,EAAEwF,OAAgB,EAAA;IAC3E,MAAMC,SAAS,GAAG,IAAI,CAACnD,MAAM,CAACmD,SAAS,CAAChD,OAAO,CAACgD,SAAS;AACzD,IAAA,MAAMC,IAAI,GAAGD,SAAS,CAACE,OAAO,GAC1BF,SAAS,CAACG,SAAS,CAAC,CAAA,WAAA,EAAcC,MAAM,CAAC7F,IAAI,CAACa,SAAS,CAAC,EAAE,EAAE;AAC1DiF,MAAAA,IAAI,EAAE,CAAC;AACPC,MAAAA,UAAU,EAAE;AACV,QAAA,0BAA0B,EAAE,MAAM;AAClC,QAAA,4BAA4B,EAAEF,MAAM,CAAC7F,IAAI,CAACa,SAAS;AACpD;KACF,CAAC,GACFG,SAAS;AACb,IAAA,IAAI0E,IAAI,EAAE;AACR1F,MAAAA,IAAI,GAAG;AACL,QAAA,GAAGA,IAAI;AACPC,QAAAA,OAAO,EAAE;UACP,GAAGD,IAAI,CAACC,OAAO;UACfwF,SAAS,EAAEA,SAAS,CAACO,aAAa,CAACP,SAAS,CAACQ,aAAa,CAACP,IAAI,CAAC;AACjE;OACF;AACH,IAAA;AACA,IAAA,IAAIH,QAAqC;IACzC,IAAI;AACF;AACA,MAAA,MAAMxD,WAAW,GAAG,IAAI,CAACtB,IAAI,CAACC,KAAK,CAACC,KAAK,CAACqB,eAAe,CAAChC,IAAI,CAACa,SAAS,CAAC;AACzE;AACA,MAAA,MAAMqF,cAAc,GAAGnE,WAAW,EAAE9B,OAAO,EAAEoD,GAAG;AAChD;AACA,MAAA,MAAM8C,UAAU,GAAG,IAAI,CAACjB,YAAY,CAAClF,IAAI,CAAC;AAC1C,MAAA,MAAMW,KAAK,GAAGwF,UAAU,CAACxF,KAAK;AAC9B;AACA,MAAA,MAAMkE,KAAK,GAAG7E,IAAI,CAACC,OAAO,EAAEmG,UAAU,EAAEvB,KAAK,IAAIwB,MAAM,EAAE;MACzD,MAAMC,OAAO,GAAGtG,IAAI,CAACC,OAAO,EAAEqG,OAAO,IAAIzB,KAAK;MAC9C,MAAMuB,UAAU,GAAGG,UAAU,CAAC;AAAE1B,QAAAA;OAAO,EAAEqB,cAAc,EAAElG,IAAI,CAACC,OAAO,EAAEmG,UAAU,CAAC;AAClF;AACA;MACA,IAAI,CAACZ,OAAO,EAAE;QACZ,MAAM7E,KAAK,CAAC6F,GAAG,CAACF,OAAO,EAAEtG,IAAI,EAAEoG,UAAU,CAAC;AAC1C,QAAA,OAAOpF,SAAgB;AACzB,MAAA;AACA,MAAA,MAAM,IAAI,CAACyF,iBAAiB,CAACN,UAAU,CAAC;MACxCZ,QAAQ,GAAG,IAAImB,OAAO,CAAC,CAACC,OAAO,EAAEC,MAAM,KAAI;AACzC,QAAA,IAAI,CAAC9G,eAAe,CAAC+E,KAAK,CAAC,GAAG;UAC5B7E,IAAI;AACJuF,UAAAA,QAAQ,EAAEA,CAAC3B,GAAG,EAAE7C,IAAI,KAAI;AACtB,YAAA,IAAI6C,GAAG,EAAE,OAAOgD,MAAM,CAAChD,GAAG,CAAC;YAC3B+C,OAAO,CAAC5F,IAAyB,CAAC;AACpC,UAAA;SACD;AACH,MAAA,CAAC,CAAC;MACF,MAAMJ,KAAK,CAAC6F,GAAG,CAACF,OAAO,EAAEtG,IAAI,EAAEoG,UAAU,CAAC;IAC5C,CAAC,CAAC,OAAOxC,GAAG,EAAE;MACZ,IAAI8B,IAAI,EAAED,SAAS,CAACoB,eAAe,CAACnB,IAAI,EAAE9B,GAAG,CAAC;AAC9C,MAAA,MAAMA,GAAG;AACX,IAAA,CAAC,SAAS;MACR8B,IAAI,EAAEoB,GAAG,EAAE;AACb,IAAA;AACA,IAAA,OAAO,MAAMvB,QAAS;AACxB,EAAA;EAEA,MAAMkB,iBAAiBA,CAACN,UAAuB,EAAA;IAC7C,IAAIA,UAAU,CAACY,gBAAgB,EAAE;AACjC,IAAA,MAAMZ,UAAU,CAAC1E,WAAW,CAACuF,cAAc,EAAE;IAC7Cb,UAAU,CAACY,gBAAgB,GAAG,IAAI;AACpC,EAAA;EAEA9B,gBAAgBA,CAAOjF,IAA4B,EAAA;IACjD,MAAMO,YAAY,GAAG0G,YAAY,CAACjH,IAAI,CAACC,OAAO,EAAEM,YAAY,CAAC;IAC7D,OAAO,CAAA,EAAGA,YAAY,CAAA,EAAA,EAAK2G,yBAAyB,CAAClH,IAAI,CAACa,SAAS,EAAE,OAAO,CAAC,CAAA,CAAE;AACjF,EAAA;EAEA,MAAMyC,YAAYA,CAAeD,GAA+B,EAAA;AAC9D,IAAA,MAAMrD,IAAI,GAAGqD,GAAG,CAACtC,IAAI;IACrB,MAAM0E,SAAS,GAAG,IAAI,CAACnD,MAAM,CAACmD,SAAS,CAAChD,OAAO,CAACgD,SAAS;IACzD,MAAM0B,MAAM,GAAG1B,SAAS,CAAC2B,cAAc,CAACpH,IAAI,CAACC,OAAO,EAAEwF,SAAS,CAAC;AAChE,IAAA,MAAMC,IAAI,GAAGD,SAAS,CAACE,OAAO,GAC1BF,SAAS,CAACG,SAAS,CACjB,CAAA,cAAA,EAAiBC,MAAM,CAAC7F,IAAI,CAACa,SAAS,CAAC,EAAE,EACzC;AACEiF,MAAAA,IAAI,EAAE,CAAC;AACPC,MAAAA,UAAU,EAAE;AACV,QAAA,0BAA0B,EAAE,SAAS;AACrC,QAAA,4BAA4B,EAAEF,MAAM,CAAC7F,IAAI,CAACa,SAAS,CAAC;QACpD,+BAA+B,EAAEwC,GAAG,CAACgE;AACtC;AACF,KAAA,EACDF,MAAM,CACP,GACDnG,SAAS;AACb;AACA,IAAA,MAAMR,SAAS,GAAG,IAAI,CAACC,IAAI,CAACC,KAAK,CAACC,KAAK,CAAC2G,aAAa,CAACtH,IAAI,CAACa,SAAS,CAAC;AACrE,IAAA,MAAMkB,WAAW,GAAG,IAAI,CAACtB,IAAI,CAACC,KAAK,CAACC,KAAK,CAACqB,eAAe,CAAChC,IAAI,CAACa,SAAS,CAAC;AACzE,IAAA,MAAM0G,OAAO,GAAGA,MACd,IAAI,CAAC9G,IAAI,CAAC+G,QAAQ,CAACC,MAAM,CACvB,YAAW;AACT,MAAA,MAAMC,YAAY,GAAGlH,SAAS,CAACmH,WAAW,CAACD,YAAY;MACvD,MAAME,YAAY,GAAwB,IAAI,CAAChG,GAAG,CAACnB,IAAI,CAACoH,QAAQ,CAACH,YAAmB,CAAC;AACrF,MAAA,OAAO,MAAME,YAAY,CAACL,OAAO,CAACvH,IAAI,CAACe,IAAI,EAAEf,IAAI,CAACC,OAAO,EAAEoD,GAAG,CAAC;AACjE,IAAA,CAAC,EACD;AACEnD,MAAAA,MAAM,EAAEF,IAAI,CAACC,OAAO,EAAEC,MAAM;AAC5B4H,MAAAA,MAAM,EAAE9H,IAAI,CAACC,OAAO,EAAE6H,MAAM;AAC5BC,MAAAA,EAAE,EAAE/H,IAAI,CAACC,OAAO,EAAE8H,EAAE;AACpBxH,MAAAA,YAAY,EAAEP,IAAI,CAACC,OAAO,EAAEM,YAAY;AACxCyH,MAAAA,SAAS,EAAE;AACT,QAAA,GAAGhI,IAAI,CAACC,OAAO,EAAE+H,SAAS;AAC1BC,QAAAA,KAAK,EAAE;AACL,UAAA,GAAGjI,IAAI,CAACC,OAAO,EAAE+H,SAAS,EAAEC,KAAK;UACjCxC,SAAS,EAAEC,IAAI,GACX;YAAEwC,OAAO,EAAEzC,SAAS,CAACQ,aAAa,CAACP,IAAI,EAAEyB,MAAM,CAAC;AAAEzB,YAAAA;AAAI,WAAE,GACxD1E;AACL;OACF;MACDmH,WAAW,EAAEpG,WAAW,EAAEoG;AAC3B,KAAA,CACF;IACH,IAAI;AACF,MAAA,OAAO,OAAOzC,IAAI,GAAGD,SAAS,CAAC2C,QAAQ,CAAC1C,IAAI,EAAE6B,OAAO,CAAC,GAAGA,OAAO,EAAE,CAAC;IACrE,CAAC,CAAC,OAAO3D,GAAG,EAAE;MACZ,IAAI8B,IAAI,EAAED,SAAS,CAACoB,eAAe,CAACnB,IAAI,EAAE9B,GAAG,CAAC;AAC9C,MAAA,MAAMA,GAAG;AACX,IAAA,CAAC,SAAS;MACR8B,IAAI,EAAEoB,GAAG,EAAE;AACb,IAAA;AACF,EAAA;AAEAuB,EAAAA,YAAYA,CAAC/B,OAAe,EAAEgC,MAA0B,EAAA;AACtD,IAAA,MAAMC,OAAO,GAAGD,MAAM,CAACC,OAAO,GAAG,IAAIC,IAAI,CAACF,MAAM,CAACC,OAAO,CAAC,CAACE,OAAO,EAAE,GAAG,EAAE;AACxE,IAAA,MAAMV,EAAE,GAAGO,MAAM,CAACP,EAAE,IAAI,EAAE;AAC1B,IAAA,MAAMW,OAAO,GAAGJ,MAAM,CAACI,OAAO;IAC9B,MAAMC,MAAM,GAAGD,OAAO,IAAI7C,MAAM,CAACyC,MAAM,CAACM,KAAK,CAAC,IAAI,EAAE;IACpD,OAAO,CAAA,EAAGtC,OAAO,CAAA,CAAA,EAAIiC,OAAO,IAAIR,EAAE,CAAA,CAAA,EAAIY,MAAM,CAAA,CAAE;AAChD,EAAA;AAEAE,EAAAA,cAAcA,CACZhI,SAA6B,EAC7BE,IAAU,EACVd,OAA2B,EAAA;AAE3BA,IAAAA,OAAO,GAAG,IAAI,CAACqC,MAAM,CAACkF,QAAQ,CAAC/E,OAAO,CAAC+E,QAAQ,CAACsB,kBAAkB,CAAC7I,OAAO,CAAC;AAC3E;IACA,OAAO;MAAEY,SAAS;MAAEE,IAAI;AAAEd,MAAAA;KAAS;AACrC,EAAA;;AAvVWT,YAAY,GAAAuJ,UAAA,CAAA,CADxBC,OAAO,EAAE,EAAAC,QAAA,CAAA;EAAAC,MAAA,EAAA;AAAA,CAAA,CAAA,CACG,EAAA1J,YAAY,CAwVxB;;ACtWM,IAAM2J,QAAQ,GAAd,MAAMA,QAAS,SAAQ1J,QAAQ,CAAA;EACpC,MAAM2J,MAAMA,CAACC,YAAuC,EAAA;IAClD,IAAI,CAACzH,GAAG,CAACnB,IAAI,CAACY,MAAM,CAAC+H,MAAM,EAAE;AAC/B,EAAA;;AAHWD,QAAQ,GAAAJ,UAAA,CAAA,CADpBO,GAAG,EAAE,EAAAL,QAAA,CAAA;EAAAC,MAAA,EAAA;AAAA,CAAA,CAAA,CACO,EAAAC,QAAQ,CAIpB;;ACLM,IAAMI,uBAAuB,GAA7B,MAAMA,uBAAwB,SAAQ9J,QAAQ,CAAA;EACnD,MAAM8H,OAAOA,GAAA;AACX;IACA,IAAI,CAAC,IAAI,CAAC3F,GAAG,CAAC4H,IAAI,CAACC,MAAM,EAAE;AACzB,MAAA,IAAI,CAACxG,KAAK,CAACR,OAAO,CAAC9B,KAAK,CAACL,gBAAgB,CAAC,IAAI,CAAC+E,GAAG,CAAC9E,YAAY,CAAC;AAClE,IAAA;AACF,EAAA;;AANWgJ,uBAAuB,GAAAR,UAAA,CAAA,CADnCW,OAAO,CAAC;AAAEC,EAAAA,QAAQ,EAAE,IAAI;AAAEC,EAAAA,KAAK,EAAE;AAAI,CAAE,CAAC,EAAAX,QAAA,CAAA;EAAAC,MAAA,EAAA;AAAA,CAAA,CAAA,CAC5B,EAAAK,uBAAuB,CAOnC;;ACVK,SAAUhH,MAAMA,CAACsH,IAAqB,EAAA;EAC1C,OAAO;AACLxI,IAAAA,MAAM,EAAE;AACNyI,MAAAA,gBAAgB,EAAE;AAAEC,QAAAA,KAAK,EAAE;OAAG;AAC9BC,MAAAA,YAAY,EAAE;AAAED,QAAAA,KAAK,EAAE;OAAG;MAC1BE,YAAY,EAAE,EAAE,GAAG,IAAI;AACvBC,MAAAA,eAAe,EAAE,IAAI;MACrBC,eAAe,EAAE,EAAE,GAAG;AACD;GACxB;AACH;;ACbO,MAAMC,cAAc,GAAG,SAAS;;ACMjC,MAAOC,MAAO,SAAQC,UAAU,CAAA;EACpC,MAAMC,QAAQA,GAAA;IACZ,MAAMC,SAAS,GAAG,IAAI,CAAC5I,GAAG,CAACqB,KAAK,CAACmH,cAAc,CAAC;IAChD,MAAMI,SAAS,CAAC/H,OAAO,CAAC9B,KAAK,CAACO,YAAY,EAAE;AAC9C,EAAA;EAEA,MAAMuJ,SAASA,GAAA;IACb,MAAMD,SAAS,GAAG,IAAI,CAAC5I,GAAG,CAACqB,KAAK,CAACmH,cAAc,CAAC;IAChD,MAAMI,SAAS,CAAC/H,OAAO,CAAC9B,KAAK,CAACY,WAAW,EAAE;AAC7C,EAAA;AACD;;ACwFM,IAAMmJ,iBAAiB,GAAvB,MAAMA,iBAAkB,SAAQC,aAAa,CAAA;AAAvCD,iBAAiB,GAAA3B,UAAA,CAAA,CAD7B6B,KAAK,EAAE,EAAA3B,QAAA,CAAA;EAAAC,MAAA,EAAA;AAAA,CAAA,CAAA,CACK,EAAAwB,iBAAiB,CAAyB;AA0BvD;;AC5HM,MAAOG,aAAgD,SAAQpL,QAAQ,CAAA;AAC3E,EAAA,MAAMY,SAASA,CAACU,IAAU,EAAEd,OAA2B,EAAA;AACrD,IAAA,OAAO,MAAM,IAAI,CAACqC,MAAM,CAAC3B,KAAK,CAAC8B,OAAO,CAAC9B,KAAK,CAACN,SAAS,CACpD,IAAI,CAACiC,MAAM,CAAC3B,KAAK,CAAC8B,OAAO,CAAC9B,KAAK,CAACkI,cAAc,CAAC,IAAI,CAACiC,UAAiB,EAAE/J,IAAI,EAAEd,OAAO,CAAC,CACtF;AACH,EAAA;AAEAF,EAAAA,IAAIA,CAACgB,IAAU,EAAEd,OAA2B,EAAA;AAC1C,IAAA,OAAO,IAAI,CAACqC,MAAM,CAAC3B,KAAK,CAAC8B,OAAO,CAAC9B,KAAK,CAACZ,IAAI,CACzC,IAAI,CAACuC,MAAM,CAAC3B,KAAK,CAAC8B,OAAO,CAAC9B,KAAK,CAACkI,cAAc,CAAC,IAAI,CAACiC,UAAiB,EAAE/J,IAAI,EAAEd,OAAO,CAAC,CACtF;AACH,EAAA;EAEAkF,QAAQA,CAAC5E,YAAoC,EAAA;AAC3C,IAAA,OAAO,IAAI,CAAC+B,MAAM,CAAC3B,KAAK,CAAC8B,OAAO,CAAC9B,KAAK,CAACwE,QAAQ,CAC7C,IAAI,CAAC2F,UAAgC,EACrCvK,YAAY,CACb;AACH,EAAA;AACD;;ACrBK,SAAUiE,KAAKA,CAACvE,OAAgC,EAAA;AACpD,EAAA,OAAO8K,mBAAmB,CAAC,OAAO,EAAE9K,OAAO,CAAC;AAC9C;;"}
|
|
1
|
+
{"version":3,"file":"index.js","sources":["../src/service/queue.ts","../src/bean/hmr.queue.ts","../src/bean/startup.loadQueueWorkers.ts","../src/config/config.ts","../src/.metadata/this.ts","../src/monkey.ts","../src/.metadata/index.ts","../src/lib/beanQueueBase.ts","../src/lib/queue.ts"],"sourcesContent":[null,null,null,null,null,null,null,null,null],"names":["ServiceQueue","BeanBase","constructor","args","_workers","_queues","_queueCallbacks","_ownedJobs","Map","_ownedHandles","push","info","options","dbInfo","Error","_queuePush","pushAsync","loadQueueWorkers","instanceName","queueItem","bean","onion","queue","getOnionsEnabledCached","queueName","name","data","undefined","_ensureWorker","clearWorkers","queueKey","_worker","worker","close","clearQueues","_queue","queueEvents","releaseOwnedInstanceHandles","owner","app","assertSettled","job","values","jobId","handles","get","entry","prefix","_getPrefix","delete","_getOwnedHandles","getTestAsyncCompletion","set","_ownedJobKey","_completeOwnedJob","error","key","terminal","executions","ticket","complete","getRedisClientKeyPrefix","_createWorker","queueConfig","getOnionOptions","workerOptions","redlockOptions","redlock","_lockTTL","lockTTL","$scope","config","concurrency","service","create","connectionWorker","redis","_workerOptions","Object","assign","scope","connection","withoutTestAsyncCompletion","Bull","Worker","_performTask","queueNameSub","_lockResource","lock","on","_job","err","handleError","message","includes","workerInner","running","run","catch","_createQueue","queueOptions","connectionQueue","_queueOptions","Queue","connectionEvents","_queueEventsOptions","queueEventsOptions","QueueEvents","returnvalue","_callCallback","failedReason","_combineQueueKey","ownedHandles","_ensureQueue","getQueue","_getQueue","ctx","_callback","callback","isAsync","ownedJob","addAttempted","callbackJobId","telemetry","span","enabled","startSpan","String","kind","attributes","injectCarrier","createContext","reserve","assertScope","_assertOwnedDatasource","jobOptionsBase","uuidv4","jobOptions","jobName","deepExtend","repeat","deduplication","debounce","queueQueue","add","_queueEventsReady","Promise","resolve","reject","track","recordException","end","queueEventsReady","waitUntilReady","instanceDesp","beanFullNameFromOnionName","id","_performTaskInner","bind","clientName","parent","extractCarrier","attemptsMade","getOnionSlice","execute","executor","newCtx","beanFullName","beanOptions","beanInstance","_getBean","locale","tz","extraData","state","context","transaction","withSpan","getRepeatKey","endDate","Date","getTime","pattern","suffix","every","prepareJobInfo","prepareGeneralInfo","__decorate","Service","__z_BeanInfo","module","HmrQueue","reload","_beanOptions","Hmr","StartupLoadQueueWorkers","meta","isTest","Startup","instance","after","_app","removeOnComplete","count","removeOnFail","lockDuration","maxStalledCount","stalledInterval","__ThisModule__","Monkey","BeanSimple","appClose","scopeSelf","appClosed","ScopeModuleAQueue","BeanScopeBase","Scope","BeanQueueBase","$onionName","createBeanDecorator"],"mappings":";;;;;;;;AA6CO,IAAMA,YAAY,GAAlB,MAAMA,YAAa,SAAQC,QAAQ,CAAA;AAAAC,EAAAA,WAAAA,CAAA,GAAAC,IAAA,EAAA;AAAA,IAAA,KAAA,CAAA,GAAAA,IAAA,CAAA;IAAA,IAAA,CACxCC,QAAQ,GAAgB,EAAE;IAAA,IAAA,CAC1BC,OAAO,GAAiB,EAAE;IAAA,IAAA,CAC1BC,eAAe,GAAoB,EAAE;AAAA,IAAA,IAAA,CAC7BC,UAAU,GAAG,IAAIC,GAAG,EAA0B;AAAA,IAAA,IAAA,CAC9CC,aAAa,GAAG,IAAID,GAAG,EAAwD;AAAA,EAAA;EAEvFE,IAAIA,CAAOC,IAA4B,EAAA;AACrC,IAAA,IAAI,CAACA,IAAI,CAACC,OAAO,EAAEC,MAAM,EAAE,MAAM,IAAIC,KAAK,CAAC,mCAAmC,CAAC;AAC/E,IAAA,OAAO,IAAI,CAACC,UAAU,CAACJ,IAAI,EAAE,KAAK,CAAC;AACrC,EAAA;AAEA;EACAK,SAASA,CAAeL,IAA4B,EAAA;AAClD,IAAA,IAAI,CAACA,IAAI,CAACC,OAAO,EAAEC,MAAM,EAAE,MAAM,IAAIC,KAAK,CAAC,mCAAmC,CAAC;AAC/E,IAAA,OAAO,IAAI,CAACC,UAAU,CAACJ,IAAI,EAAE,IAAI,CAAC;AACpC,EAAA;EAEAM,gBAAgBA,CAACC,YAAuD,EAAA;AACtE,IAAA,KAAK,MAAMC,SAAS,IAAI,IAAI,CAACC,IAAI,CAACC,KAAK,CAACC,KAAK,CAACC,sBAAsB,EAAE,EAAE;AACtE,MAAA,MAAMZ,IAAI,GAA8B;QACtCa,SAAS,EAAEL,SAAS,CAACM,IAAa;AAClCC,QAAAA,IAAI,EAAEC,SAAgB;AACtBf,QAAAA,OAAO,EAAE;AACPM,UAAAA;AACD;OACF;AACD,MAAA,IAAI,CAACU,aAAa,CAACjB,IAAI,CAAC;AAC1B,IAAA;AACF,EAAA;EAEA,MAAMkB,YAAYA,GAAA;AAChB,IAAA,KAAK,MAAMC,QAAQ,IAAI,IAAI,CAAC1B,QAAQ,EAAE;AACpC,MAAA,MAAM2B,OAAO,GAAG,IAAI,CAAC3B,QAAQ,CAAC0B,QAAQ,CAAC;AACvC,MAAA,MAAMC,OAAO,CAACC,MAAM,CAACC,KAAK,EAAE;AAC9B,IAAA;AACA,IAAA,IAAI,CAAC7B,QAAQ,GAAG,EAAE;AACpB,EAAA;EAEA,MAAM8B,WAAWA,GAAA;AACf,IAAA,KAAK,MAAMJ,QAAQ,IAAI,IAAI,CAACzB,OAAO,EAAE;AACnC,MAAA,MAAM8B,MAAM,GAAG,IAAI,CAAC9B,OAAO,CAACyB,QAAQ,CAAC;AACrC,MAAA,MAAMK,MAAM,CAACb,KAAK,CAACW,KAAK,EAAE;AAC1B,MAAA,MAAME,MAAM,CAACC,WAAW,CAACH,KAAK,EAAE;AAClC,IAAA;AACA,IAAA,IAAI,CAAC5B,OAAO,GAAG,EAAE;AACnB,EAAA;EAEA,MAAMgC,2BAA2BA,CAACC,KAA0B,EAAA;AAC1D,IAAA,IAAIA,KAAK,CAACC,GAAG,KAAK,IAAI,CAACA,GAAG,EAAE,MAAM,IAAIzB,KAAK,CAAC,+CAA+C,CAAC;IAC5FwB,KAAK,CAACE,aAAa,EAAE;IACrB,KAAK,MAAMC,GAAG,IAAI,IAAI,CAAClC,UAAU,CAACmC,MAAM,EAAE,EAAE;AAC1C,MAAA,IAAID,GAAG,CAACH,KAAK,KAAKA,KAAK,EAAE,MAAM,IAAIxB,KAAK,CAAC,CAAA,kCAAA,EAAqC2B,GAAG,CAACE,KAAK,EAAE,CAAC;AAC5F,IAAA;IACA,MAAMC,OAAO,GAAG,IAAI,CAACnC,aAAa,CAACoC,GAAG,CAACP,KAAK,CAAC;IAC7C,IAAI,CAACM,OAAO,EAAE;IACd,KAAK,MAAM,CAACd,QAAQ,EAAEgB,KAAK,CAAC,IAAIF,OAAO,EAAE;AACvC,MAAA,IACEE,KAAK,CAACC,MAAM,KAAK,IAAI,CAACC,UAAU,EAAE,IACjCF,KAAK,CAACd,MAAM,IAAI,IAAI,CAAC5B,QAAQ,CAAC0B,QAAQ,CAAC,KAAKgB,KAAK,CAACd,MAAO,IACzDc,KAAK,CAACxB,KAAK,IAAI,IAAI,CAACjB,OAAO,CAACyB,QAAQ,CAAC,KAAKgB,KAAK,CAACxB,KAAM,EACvD;AACA,QAAA,MAAM,IAAIR,KAAK,CAAC,CAAA,6BAAA,EAAgCgB,QAAQ,EAAE,CAAC;AAC7D,MAAA;AACF,IAAA;IACA,KAAK,MAAM,CAACA,QAAQ,EAAEgB,KAAK,CAAC,IAAIF,OAAO,EAAE;MACvC,IAAIE,KAAK,CAACd,MAAM,EAAE;QAChB,MAAMc,KAAK,CAACd,MAAM,CAACA,MAAM,CAACC,KAAK,EAAE;AACjC,QAAA,OAAO,IAAI,CAAC7B,QAAQ,CAAC0B,QAAQ,CAAC;QAC9BgB,KAAK,CAACd,MAAM,GAAGL,SAAS;AAC1B,MAAA;MACA,IAAImB,KAAK,CAACxB,KAAK,EAAE;QACf,MAAMwB,KAAK,CAACxB,KAAK,CAACA,KAAK,CAACW,KAAK,EAAE;QAC/B,MAAMa,KAAK,CAACxB,KAAK,CAACc,WAAW,CAACH,KAAK,EAAE;AACrC,QAAA,OAAO,IAAI,CAAC5B,OAAO,CAACyB,QAAQ,CAAC;QAC7BgB,KAAK,CAACxB,KAAK,GAAGK,SAAS;AACzB,MAAA;AACF,IAAA;AACA,IAAA,IAAI,CAAClB,aAAa,CAACwC,MAAM,CAACX,KAAK,CAAC;AAClC,EAAA;AAEQY,EAAAA,gBAAgBA,CAAOvC,IAA4B,EAAEmB,QAAgB,EAAA;AAC3E,IAAA,MAAMQ,KAAK,GAAGa,sBAAsB,CAAC,IAAI,CAACZ,GAAG,CAAC;AAC9C;AACA,IAAA,IAAI,CAACD,KAAK,IAAI3B,IAAI,CAACC,OAAO,EAAEM,YAAY,KAAKoB,KAAK,CAACpB,YAAY,EAAE;IACjE,IAAI0B,OAAO,GAAG,IAAI,CAACnC,aAAa,CAACoC,GAAG,CAACP,KAAK,CAAC;AAC3C,IAAA,IAAIQ,KAAK,GAAGF,OAAO,EAAEC,GAAG,CAACf,QAAQ,CAAC;IAClC,IACG,IAAI,CAAC1B,QAAQ,CAAC0B,QAAQ,CAAC,IAAI,IAAI,CAAC1B,QAAQ,CAAC0B,QAAQ,CAAC,KAAKgB,KAAK,EAAEd,MAAM,IACpE,IAAI,CAAC3B,OAAO,CAACyB,QAAQ,CAAC,IAAI,IAAI,CAACzB,OAAO,CAACyB,QAAQ,CAAC,KAAKgB,KAAK,EAAExB,KAAM,IAClEwB,KAAK,IAAIA,KAAK,CAACC,MAAM,KAAK,IAAI,CAACC,UAAU,EAAG,EAC7C;AACA,MAAA,MAAM,IAAIlC,KAAK,CAAC,CAAA,uDAAA,EAA0DgB,QAAQ,EAAE,CAAC;AACvF,IAAA;IACA,IAAI,CAACc,OAAO,EAAE;AACZA,MAAAA,OAAO,GAAG,IAAIpC,GAAG,EAAE;MACnB,IAAI,CAACC,aAAa,CAAC2C,GAAG,CAACd,KAAK,EAAEM,OAAO,CAAC;AACxC,IAAA;IACA,IAAI,CAACE,KAAK,EAAE;AACVA,MAAAA,KAAK,GAAG;AAAEC,QAAAA,MAAM,EAAE,IAAI,CAACC,UAAU;OAAI;AACrCJ,MAAAA,OAAO,CAACQ,GAAG,CAACtB,QAAQ,EAAEgB,KAAK,CAAC;AAC9B,IAAA;AACA,IAAA,OAAOA,KAAK;AACd,EAAA;AAEQO,EAAAA,YAAYA,CAACN,MAAc,EAAEjB,QAAgB,EAAEa,KAAa,EAAA;AAClE,IAAA,OAAO,GAAGI,MAAM,CAAA,EAAA,EAAKjB,QAAQ,CAAA,EAAA,EAAKa,KAAK,CAAA,CAAE;AAC3C,EAAA;EAEQW,iBAAiBA,CAACP,MAAc,EAAEjB,QAAgB,EAAEa,KAAa,EAAEY,KAAa,EAAA;IACtF,MAAMC,GAAG,GAAG,IAAI,CAACH,YAAY,CAACN,MAAM,EAAEjB,QAAQ,EAAEa,KAAK,CAAC;IACtD,MAAMF,GAAG,GAAG,IAAI,CAAClC,UAAU,CAACsC,GAAG,CAACW,GAAG,CAAC;IACpC,IAAI,CAACf,GAAG,EAAE;IACVA,GAAG,CAACgB,QAAQ,GAAG;AAAEF,MAAAA;KAAO;IACxB,IAAId,GAAG,CAACiB,UAAU,EAAE;AACpB,IAAA,IAAI,CAACnD,UAAU,CAAC0C,MAAM,CAACO,GAAG,CAAC;AAC3Bf,IAAAA,GAAG,CAACkB,MAAM,CAACC,QAAQ,CAACL,KAAK,CAAC;AAC5B,EAAA;AAEAP,EAAAA,UAAUA,GAAA;IACR,OAAO,CAAA,EAAGa,uBAAuB,CAAC,MAAM,EAAE,IAAI,CAACtB,GAAG,CAAC,CAAA,KAAA,CAAO;AAC5D,EAAA;AAEAuB,EAAAA,aAAaA,CAAOnD,IAA4B,EAAEmB,QAAgB,EAAA;AAChE,IAAA,MAAMS,GAAG,GAAG,IAAI,CAACA,GAAG;AACpB;IACA,MAAMR,OAAO,GAAG,EAAgB;AAChC;AACA,IAAA,MAAMgB,MAAM,GAAG,IAAI,CAACC,UAAU,EAAE;AAChC;AACA,IAAA,MAAMe,WAAW,GAAGxB,GAAG,CAACnB,IAAI,CAACC,KAAK,CAACC,KAAK,CAAC0C,eAAe,CAACrD,IAAI,CAACa,SAAS,CAAC;AACxE;AACA,IAAA,MAAMyC,aAAa,GAAGF,WAAW,EAAEnD,OAAO,EAAEoB,MAAM;AAClD,IAAA,MAAMkC,cAAc,GAAGH,WAAW,EAAEnD,OAAO,EAAEuD,OAAO;AACpD,IAAA,MAAMC,QAAQ,GAAGF,cAAc,EAAEG,OAAO,IAAI,IAAI,CAACC,MAAM,CAACH,OAAO,CAACI,MAAM,CAACF,OAAO;AAE9E;AACA,IAAA,IAAI,CAACN,WAAW,EAAES,WAAW,EAAE;AAC7BzC,MAAAA,OAAO,CAACoC,OAAO,GAAG,IAAI,CAACG,MAAM,CAACH,OAAO,CAACM,OAAO,CAACN,OAAO,CAACO,MAAM,CAACR,cAAc,CAAC;AAC9E,IAAA;AAEA;IACA,MAAMS,gBAAgB,GAAGpC,GAAG,CAACnB,IAAI,CAACwD,KAAK,CAAC/B,GAAG,CAAC,OAAO,CAAC;AACpD,IAAA,MAAMgC,cAAc,GAAGC,MAAM,CAACC,MAAM,CAAC,EAAE,EAAE,IAAI,CAACC,KAAK,CAACT,MAAM,CAACvC,MAAM,EAAEiC,aAAa,EAAE;MAChFlB,MAAM;AACNkC,MAAAA,UAAU,EAAEN;AACb,KAAA,CAAC;IACF5C,OAAO,CAACkC,aAAa,GAAGY,cAAc;AACtC9C,IAAAA,OAAO,CAACC,MAAM,GAAGkD,0BAA0B,CACzC,MACE,IAAIC,IAAI,CAACC,MAAM,CACbtD,QAAQ,EACR,MAAMW,GAAG,IAAG;AACV;MACA,IAAIsB,WAAW,EAAES,WAAW,EAAE;AAC5B,QAAA,OAAO,MAAM,IAAI,CAACa,YAAY,CAAC5C,GAAG,CAAC;AACrC,MAAA;AACA;AACA,MAAA,MAAM9B,IAAI,GAAG8B,GAAG,CAACf,IAA8B;AAC/C,MAAA,MAAM4D,YAAY,GAAG3E,IAAI,CAACC,OAAO,EAAE0E,YAAY;AAC/C,MAAA,MAAMC,aAAa,GAAG,CAAA,MAAA,EAASzD,QAAQ,CAAA,EAAGwD,YAAY,GAAG,CAAA,CAAA,EAAIA,YAAY,CAAA,CAAE,GAAG,EAAE,CAAA,CAAE;AAClF,MAAA,OAAO,MAAM,IAAI,CAAChB,MAAM,CAACH,OAAO,CAACM,OAAO,CAACN,OAAO,CAACqB,IAAI,CACnDD,aAAa,EACb,YAAW;AACT,QAAA,OAAO,MAAM,IAAI,CAACF,YAAY,CAAC5C,GAAG,CAAC;AACrC,MAAA,CAAC,EACD;AACE;QACA0B,OAAO,EAAEpC,OAAO,CAACoC,OAAO;AACxBE,QAAAA,OAAO,EAAED;AACV,OAAA,CACF;IACH,CAAC,EACDS,cAAc,CACf,CACJ;IAED9C,OAAO,CAACC,MAAM,CAACyD,EAAE,CAAC,QAAQ,EAAE,CAACC,IAAI,EAAEC,GAAG,KAAI;AACxC,MAAA,IAAI,CAACpD,GAAG,CAACqD,WAAW,CAACD,GAAG,CAAC;AAC3B,IAAA,CAAC,CAAC;IAEF5D,OAAO,CAACC,MAAM,CAACyD,EAAE,CAAC,OAAO,EAAEE,GAAG,IAAG;AAC/B,MAAA,IAAIA,GAAG,CAACE,OAAO,IAAIF,GAAG,CAACE,OAAO,CAACC,QAAQ,CAAC,sBAAsB,CAAC,EAAE;AAC/D,QAAA,MAAMC,WAAW,GAAGhE,OAAO,CAACC,MAAa;AACzC,QAAA,IAAI,CAAC+D,WAAW,CAACC,OAAO,EAAE;UACxBjE,OAAO,CAACC,MAAM,CAACiE,GAAG,EAAE,CAACC,KAAK,CAACP,GAAG,IAAG;AAC/B,YAAA,IAAI,CAACpD,GAAG,CAACqD,WAAW,CAACD,GAAG,CAAC;AAC3B,UAAA,CAAC,CAAC;AACJ,QAAA;AACF,MAAA;AACF,IAAA,CAAC,CAAC;AAEF;AACA,IAAA,OAAO5D,OAAO;AAChB,EAAA;AAEAoE,EAAAA,YAAYA,CAAOxF,IAA4B,EAAEmB,QAAgB,EAAA;AAC/D,IAAA,MAAMS,GAAG,GAAG,IAAI,CAACA,GAAG;AACpB;IACA,MAAMJ,MAAM,GAAG,EAAiB;AAChC;AACA,IAAA,MAAMY,MAAM,GAAG,IAAI,CAACC,UAAU,EAAE;AAChC;AACA,IAAA,MAAMe,WAAW,GAAGxB,GAAG,CAACnB,IAAI,CAACC,KAAK,CAACC,KAAK,CAAC0C,eAAe,CAACrD,IAAI,CAACa,SAAS,CAAC;AACxE;AACA,IAAA,MAAM4E,YAAY,GAAGrC,WAAW,EAAEnD,OAAO,EAAEU,KAAK;AAEhD;IACA,MAAM+E,eAAe,GAAG9D,GAAG,CAACnB,IAAI,CAACwD,KAAK,CAAC/B,GAAG,CAAC,OAAO,CAAC;IACnD,MAAMyD,aAAa,GAAGxB,MAAM,CAACC,MAAM,CAAC,EAAE,EAAEqB,YAAY,EAAE;MAAErD,MAAM;AAAEkC,MAAAA,UAAU,EAAEoB;AAAe,KAAE,CAAC;IAC9FlE,MAAM,CAACoC,MAAM,GAAGR,WAAW;IAC3B5B,MAAM,CAACvB,OAAO,GAAG0F,aAAa;IAC9BnE,MAAM,CAACb,KAAK,GAAG,IAAI6D,IAAI,CAACoB,KAAK,CAACzE,QAAQ,EAAEwE,aAAa,CAAC;AAEtD;IACA,MAAME,gBAAgB,GAAGjE,GAAG,CAACnB,IAAI,CAACwD,KAAK,CAAC/B,GAAG,CAAC,OAAO,CAAC;AACpD,IAAA,MAAM4D,mBAAmB,GAAG;MAAE1D,MAAM;AAAEkC,MAAAA,UAAU,EAAEuB;KAA6C;IAC/FrE,MAAM,CAACuE,kBAAkB,GAAGD,mBAAmB;AAC/CtE,IAAAA,MAAM,CAACC,WAAW,GAAG8C,0BAA0B,CAC7C,MAAM,IAAIC,IAAI,CAACwB,WAAW,CAAC7E,QAAQ,EAAE2E,mBAAmB,CAAC,CAC1D;AACDtE,IAAAA,MAAM,CAACC,WAAW,CAACqD,EAAE,CAAC,WAAW,EAAE,CAAC;MAAE9C,KAAK;AAAEiE,MAAAA;AAAW,KAAE,KAAI;MAC5D,IAAI,CAACtD,iBAAiB,CAACP,MAAM,EAAEjB,QAAQ,EAAEa,KAAK,CAAC;MAC/C,IAAI,CAACkE,aAAa,CAAClE,KAAK,EAAEhB,SAAS,EAAEiF,WAAW,CAAC;AACnD,IAAA,CAAC,CAAC;AACF;AACAzE,IAAAA,MAAM,CAACC,WAAW,CAACqD,EAAE,CAAC,QAAQ,EAAE,CAAC;MAAE9C,KAAK;AAAEmE,MAAAA;AAAY,KAAE,KAAI;AAC1D,MAAA,IAAI,CAACxD,iBAAiB,CAACP,MAAM,EAAEjB,QAAQ,EAAEa,KAAK,EAAE,IAAI7B,KAAK,CAACgG,YAAY,CAAC,CAAC;MACxE,IAAI,CAACD,aAAa,CAAClE,KAAK,EAAEmE,YAAY,EAAEnF,SAAS,CAAC;AACpD,IAAA,CAAC,CAAC;AAEF;AACA,IAAA,OAAOQ,MAAM;AACf,EAAA;EAEAP,aAAaA,CAAOjB,IAA4B,EAAA;AAC9C;AACA,IAAA,MAAMmB,QAAQ,GAAG,IAAI,CAACiF,gBAAgB,CAACpG,IAAI,CAAC;IAC5C,MAAMqG,YAAY,GAAG,IAAI,CAAC9D,gBAAgB,CAACvC,IAAI,EAAEmB,QAAQ,CAAC;AAC1D;AACA,IAAA,IAAI,CAAC,IAAI,CAAC1B,QAAQ,CAAC0B,QAAQ,CAAC,EAAE;AAC5B,MAAA,IAAI,CAAC1B,QAAQ,CAAC0B,QAAQ,CAAC,GAAG,IAAI,CAACgC,aAAa,CAACnD,IAAI,EAAEmB,QAAQ,CAAC;MAC5D,IAAIkF,YAAY,EAAEA,YAAY,CAAChF,MAAM,GAAG,IAAI,CAAC5B,QAAQ,CAAC0B,QAAQ,CAAC;AACjE,IAAA;AACF,EAAA;EAEAmF,YAAYA,CAAOtG,IAA4B,EAAA;AAC7C;AACA,IAAA,IAAI,CAACiB,aAAa,CAACjB,IAAI,CAAC;AACxB;AACA,IAAA,MAAMmB,QAAQ,GAAG,IAAI,CAACiF,gBAAgB,CAACpG,IAAI,CAAC;IAC5C,MAAMqG,YAAY,GAAG,IAAI,CAAC9D,gBAAgB,CAACvC,IAAI,EAAEmB,QAAQ,CAAC;AAC1D;AACA,IAAA,IAAI,CAAC,IAAI,CAACzB,OAAO,CAACyB,QAAQ,CAAC,EAAE;AAC3B,MAAA,IAAI,CAACzB,OAAO,CAACyB,QAAQ,CAAC,GAAG,IAAI,CAACqE,YAAY,CAACxF,IAAI,EAAEmB,QAAQ,CAAC;MAC1D,IAAIkF,YAAY,EAAEA,YAAY,CAAC1F,KAAK,GAAG,IAAI,CAACjB,OAAO,CAACyB,QAAQ,CAAC;AAC/D,IAAA;AACA;AACA,IAAA,OAAO,IAAI,CAACzB,OAAO,CAACyB,QAAQ,CAAC;AAC/B,EAAA;AAEAoF,EAAAA,QAAQA,CAAC1F,SAA6B,EAAEN,YAAuD,EAAA;IAC7F,OAAO,IAAI,CAACiG,SAAS,CAAC;MACpB3F,SAAS;AACTE,MAAAA,IAAI,EAAEC,SAAgB;AACtBf,MAAAA,OAAO,EAAE;AACPM,QAAAA,YAAY,EAAEA,YAAY,IAAI,IAAI,CAACkG,GAAG,CAAClG;AACxC;AACF,KAAA,CAAC;AACJ,EAAA;EAEAiG,SAASA,CAAOxG,IAA4B,EAAA;AAC1C,IAAA,OAAO,IAAI,CAACsG,YAAY,CAACtG,IAAI,CAAC,CAACW,KAAK;AACtC,EAAA;AAEAuF,EAAAA,aAAaA,CACXlE,KAAsB,EACtBmE,YAAgC,EAChCpF,IAAsB,EAAA;AAEtB,IAAA,MAAM2F,SAAS,GAAG,IAAI,CAAC/G,eAAe,CAACqC,KAAK,CAAC;AAC7C,IAAA,IAAI0E,SAAS,EAAE;AACb,MAAA,OAAO,IAAI,CAAC/G,eAAe,CAACqC,KAAK,CAAC;AAClC0E,MAAAA,SAAS,CAACC,QAAQ,CAACR,YAAY,GAAG,IAAIhG,KAAK,CAACgG,YAAY,CAAC,GAAGnF,SAAS,EAAED,IAAI,CAAC;AAC9E,IAAA;AACF,EAAA;AAEA,EAAA,MAAMX,UAAUA,CAAeJ,IAA4B,EAAE4G,OAAgB,EAAA;IAC3E,MAAMjF,KAAK,GAAGiF,OAAO,GAAGpE,sBAAsB,CAAC,IAAI,CAACZ,GAAG,CAAC,GAAGZ,SAAS;AACpE,IAAA,IAAIgC,MAA6C;AACjD,IAAA,IAAI6D,QAAoC;IACxC,IAAIC,YAAY,GAAG,KAAK;AACxB,IAAA,IAAIC,aAAiC;IACrC,MAAMC,SAAS,GAAG,IAAI,CAACrD,MAAM,CAACqD,SAAS,CAAClD,OAAO,CAACkD,SAAS;AACzD,IAAA,MAAMC,IAAI,GAAGD,SAAS,CAACE,OAAO,GAC1BF,SAAS,CAACG,SAAS,CAAC,CAAA,WAAA,EAAcC,MAAM,CAACpH,IAAI,CAACa,SAAS,CAAC,EAAE,EAAE;AAC1DwG,MAAAA,IAAI,EAAE,CAAC;AACPC,MAAAA,UAAU,EAAE;AACV,QAAA,0BAA0B,EAAE,MAAM;AAClC,QAAA,4BAA4B,EAAEF,MAAM,CAACpH,IAAI,CAACa,SAAS;AACpD;KACF,CAAC,GACFG,SAAS;AACb,IAAA,IAAIiG,IAAI,EAAE;AACRjH,MAAAA,IAAI,GAAG;AACL,QAAA,GAAGA,IAAI;AACPC,QAAAA,OAAO,EAAE;UACP,GAAGD,IAAI,CAACC,OAAO;UACf+G,SAAS,EAAEA,SAAS,CAACO,aAAa,CAACP,SAAS,CAACQ,aAAa,CAACP,IAAI,CAAC;AACjE;OACF;AACH,IAAA;AACA,IAAA,IAAIN,QAAqC;IACzC,IAAI;AACF3D,MAAAA,MAAM,GAAGrB,KAAK,EAAE8F,OAAO,CAAC,CAAA,MAAA,EAASL,MAAM,CAACpH,IAAI,CAACa,SAAS,CAAC,EAAE,CAAC;AAC1D,MAAA,IAAIc,KAAK,EAAE;AACTA,QAAAA,KAAK,CAAC+F,WAAW,CAAC1H,IAAI,CAACC,OAAO,CAAC;AAC/B,QAAA,IAAID,IAAI,CAACa,SAAS,KAAK,oBAAoB,EAAE;UAC3C,MAAM,IAAIV,KAAK,CAAC,CAAA,yBAAA,EAA4BiH,MAAM,CAACpH,IAAI,CAACa,SAAS,CAAC,CAAA,CAAE,CAAC;AACvE,QAAA;AACA,QAAA,IAAI,CAAC8G,sBAAsB,CAAC3H,IAAI,EAAE2B,KAAK,CAAC;AAC1C,MAAA;AACA;AACA,MAAA,MAAMyB,WAAW,GAAG,IAAI,CAAC3C,IAAI,CAACC,KAAK,CAACC,KAAK,CAAC0C,eAAe,CAACrD,IAAI,CAACa,SAAS,CAAC;AACzE;AACA,MAAA,MAAM+G,cAAc,GAAGxE,WAAW,EAAEnD,OAAO,EAAE6B,GAAG;AAChD;AACA,MAAA,MAAME,KAAK,GAAGL,KAAK,GAAGkG,MAAM,EAAE,GAAG7H,IAAI,CAACC,OAAO,EAAE6H,UAAU,EAAE9F,KAAK,IAAI6F,MAAM,EAAE;MAC5E,MAAME,OAAO,GAAG/H,IAAI,CAACC,OAAO,EAAE8H,OAAO,IAAI/F,KAAK;MAC9C,MAAM8F,UAAU,GAAGE,UAAU,CAAC;AAAEhG,QAAAA;OAAO,EAAE4F,cAAc,EAAE5H,IAAI,CAACC,OAAO,EAAE6H,UAAU,CAAC;AAClF,MAAA,IAAInG,KAAK,EAAE;AACT,QAAA,IACEmG,UAAU,CAAC9F,KAAK,KAAKA,KAAK,IAC1B8F,UAAU,CAACG,MAAM,IACjBH,UAAU,CAACI,aAAa,IACxBJ,UAAU,CAACK,QAAQ,EACnB;AACA,UAAA,MAAM,IAAIhI,KAAK,CAAC,uEAAuE,CAAC;AAC1F,QAAA;AACA,QAAA,MAAMiC,MAAM,GAAG,IAAI,CAACC,UAAU,EAAE;AAChC,QAAA,MAAMlB,QAAQ,GAAG,IAAI,CAACiF,gBAAgB,CAACpG,IAAI,CAAC;AAC5C6G,QAAAA,QAAQ,GAAG;UAAElF,KAAK;AAAEqB,UAAAA,MAAM,EAAEA,MAAO;UAAEZ,MAAM;UAAEjB,QAAQ;UAAEa,KAAK;AAAEe,UAAAA,UAAU,EAAE;SAAG;AAC7E,QAAA,IAAI,CAACnD,UAAU,CAAC6C,GAAG,CAAC,IAAI,CAACC,YAAY,CAACN,MAAM,EAAEjB,QAAQ,EAAEa,KAAK,CAAC,EAAE6E,QAAQ,CAAC;AAC3E,MAAA;AACA;AACA,MAAA,MAAMuB,UAAU,GAAG,IAAI,CAAC9B,YAAY,CAACtG,IAAI,CAAC;AAC1C,MAAA,MAAMW,KAAK,GAAGyH,UAAU,CAACzH,KAAK;AAC9B;AACA;MACA,IAAI,CAACiG,OAAO,EAAE;QACZ,MAAMjG,KAAK,CAAC0H,GAAG,CAACN,OAAO,EAAE/H,IAAI,EAAE8H,UAAU,CAAC;AAC1C,QAAA,OAAO9G,SAAgB;AACzB,MAAA;AACA,MAAA,MAAM,IAAI,CAACsH,iBAAiB,CAACF,UAAU,CAAC;AACxCrB,MAAAA,aAAa,GAAG/E,KAAK;MACrB2E,QAAQ,GAAG,IAAI4B,OAAO,CAAC,CAACC,OAAO,EAAEC,MAAM,KAAI;AACzC,QAAA,IAAI,CAAC9I,eAAe,CAACqC,KAAK,CAAC,GAAG;UAC5BhC,IAAI;AACJ2G,UAAAA,QAAQ,EAAEA,CAAC3B,GAAG,EAAEjE,IAAI,KAAI;AACtB,YAAA,IAAIiE,GAAG,EAAE,OAAOyD,MAAM,CAACzD,GAAG,CAAC;YAC3BwD,OAAO,CAACzH,IAAyB,CAAC;AACpC,UAAA;SACD;AACH,MAAA,CAAC,CAAC;AACF;MACA,IAAIY,KAAK,EAAE,KAAKgF,QAAQ,CAACpB,KAAK,CAAC,MAAK,CAAE,CAAC,CAAC;AACxCuB,MAAAA,YAAY,GAAG,IAAI;MACnB,MAAMnG,KAAK,CAAC0H,GAAG,CAACN,OAAO,EAAE/H,IAAI,EAAE8H,UAAU,CAAC;IAC5C,CAAC,CAAC,OAAO9C,GAAG,EAAE;MACZ,IAAIrD,KAAK,IAAIqB,MAAM,EAAE;AACnB,QAAA,IAAI8D,YAAY,EAAE;AAChB;AACA;UACA,KAAKnF,KAAK,CACP+G,KAAK,CAAC,CAAA,UAAA,EAAa7B,QAAQ,EAAE7E,KAAK,CAAA,CAAE,EAAE,YAAW;AAChD,YAAA,MAAMgD,GAAG;AACX,UAAA,CAAC,CAAC,CACDO,KAAK,CAAC,MAAK,CAAE,CAAC,CAAC;AACpB,QAAA,CAAC,MAAM;AACL,UAAA,IAAIsB,QAAQ,EAAE;YACZ,IAAI,CAACjH,UAAU,CAAC0C,MAAM,CACpB,IAAI,CAACI,YAAY,CAACmE,QAAQ,CAACzE,MAAM,EAAEyE,QAAQ,CAAC1F,QAAQ,EAAE0F,QAAQ,CAAC7E,KAAK,CAAC,CACtE;AACH,UAAA;UACA,IAAI+E,aAAa,EAAE,OAAO,IAAI,CAACpH,eAAe,CAACoH,aAAa,CAAC;AAC7D/D,UAAAA,MAAM,CAACC,QAAQ,CAAC+B,GAAG,CAAC;AACtB,QAAA;AACF,MAAA;MACA,IAAIiC,IAAI,EAAED,SAAS,CAAC2B,eAAe,CAAC1B,IAAI,EAAEjC,GAAG,CAAC;AAC9C,MAAA,MAAMA,GAAG;AACX,IAAA,CAAC,SAAS;MACRiC,IAAI,EAAE2B,GAAG,EAAE;AACb,IAAA;AACA,IAAA,OAAO,MAAMjC,QAAS;AACxB,EAAA;EAEA,MAAM2B,iBAAiBA,CAACF,UAAuB,EAAA;IAC7C,IAAIA,UAAU,CAACS,gBAAgB,EAAE;AACjC,IAAA,MAAMT,UAAU,CAAC3G,WAAW,CAACqH,cAAc,EAAE;IAC7CV,UAAU,CAACS,gBAAgB,GAAG,IAAI;AACpC,EAAA;EAEAzC,gBAAgBA,CAAOpG,IAA4B,EAAA;IACjD,MAAMO,YAAY,GAAGwI,YAAY,CAAC/I,IAAI,CAACC,OAAO,EAAEM,YAAY,CAAC;IAC7D,OAAO,CAAA,EAAGA,YAAY,CAAA,EAAA,EAAKyI,yBAAyB,CAAChJ,IAAI,CAACa,SAAS,EAAE,OAAO,CAAC,CAAA,CAAE;AACjF,EAAA;EAEA,MAAM6D,YAAYA,CAAe5C,GAA+B,EAAA;AAC9D,IAAA,MAAM+E,QAAQ,GAAG/E,GAAG,CAACmH,EAAE,GACnB,IAAI,CAACrJ,UAAU,CAACsC,GAAG,CAAC,IAAI,CAACQ,YAAY,CAACZ,GAAG,CAACM,MAAM,EAAEN,GAAG,CAACjB,SAAS,EAAEiB,GAAG,CAACmH,EAAE,CAAC,CAAC,GACzEjI,SAAS;AACb,IAAA,IAAI,CAAC6F,QAAQ,EAAE,OAAO,MAAMtC,0BAA0B,CAAC,MAAM,IAAI,CAAC2E,iBAAiB,CAACpH,GAAG,CAAC,CAAC;IACzF+E,QAAQ,CAAC9D,UAAU,EAAE;IACrB,IAAI;AACF,MAAA,OAAO,MAAM8D,QAAQ,CAAClF,KAAK,CAACwH,IAAI,CAAC,YAAW;QAC1CtC,QAAQ,CAAClF,KAAK,CAAC+F,WAAW,CAAC5F,GAAG,CAACf,IAAI,CAACd,OAAO,CAAC;AAC5C,QAAA,IAAI,IAAI,CAACmG,gBAAgB,CAACtE,GAAG,CAACf,IAAI,CAAC,KAAK8F,QAAQ,CAAC1F,QAAQ,EAAE;AACzD,UAAA,MAAM,IAAIhB,KAAK,CAAC,+DAA+D,CAAC;AAClF,QAAA;QACA,IAAI,CAACwH,sBAAsB,CAAC7F,GAAG,CAACf,IAAI,EAAE8F,QAAQ,CAAClF,KAAK,CAAC;AACrD,QAAA,OAAO,MAAM,IAAI,CAACuH,iBAAiB,CAACpH,GAAG,CAAC;AAC1C,MAAA,CAAC,CAAC;AACJ,IAAA,CAAC,SAAS;MACR+E,QAAQ,CAAC9D,UAAU,EAAE;MACrB,IAAI8D,QAAQ,CAAC/D,QAAQ,EAAE;QACrB,IAAI,CAACH,iBAAiB,CACpBkE,QAAQ,CAACzE,MAAM,EACfyE,QAAQ,CAAC1F,QAAQ,EACjB0F,QAAQ,CAAC7E,KAAK,EACd6E,QAAQ,CAAC/D,QAAQ,CAACF,KAAK,CACxB;AACH,MAAA;AACF,IAAA;AACF,EAAA;AAEQ+E,EAAAA,sBAAsBA,CAAO3H,IAA4B,EAAE2B,KAA0B,EAAA;AAC3F;AACA,IAAA,IACE3B,IAAI,CAACa,SAAS,KAAK,oBAAoB,IACtCb,IAAI,CAACe,IAAgC,EAAEqI,UAAU,KAAKzH,KAAK,CAACyH,UAAU,EACvE;AACA,MAAA,MAAM,IAAIjJ,KAAK,CAAC,4DAA4D,CAAC;AAC/E,IAAA;AACF,EAAA;EAEQ,MAAM+I,iBAAiBA,CAAepH,GAA+B,EAAA;AAC3E,IAAA,MAAM9B,IAAI,GAAG8B,GAAG,CAACf,IAAI;IACrB,MAAMiG,SAAS,GAAG,IAAI,CAACrD,MAAM,CAACqD,SAAS,CAAClD,OAAO,CAACkD,SAAS;IACzD,MAAMqC,MAAM,GAAGrC,SAAS,CAACsC,cAAc,CAACtJ,IAAI,CAACC,OAAO,EAAE+G,SAAS,CAAC;AAChE,IAAA,MAAMC,IAAI,GAAGD,SAAS,CAACE,OAAO,GAC1BF,SAAS,CAACG,SAAS,CACjB,CAAA,cAAA,EAAiBC,MAAM,CAACpH,IAAI,CAACa,SAAS,CAAC,EAAE,EACzC;AACEwG,MAAAA,IAAI,EAAE,CAAC;AACPC,MAAAA,UAAU,EAAE;AACV,QAAA,0BAA0B,EAAE,SAAS;AACrC,QAAA,4BAA4B,EAAEF,MAAM,CAACpH,IAAI,CAACa,SAAS,CAAC;QACpD,+BAA+B,EAAEiB,GAAG,CAACyH;AACtC;AACF,KAAA,EACDF,MAAM,CACP,GACDrI,SAAS;AACb;AACA,IAAA,MAAMR,SAAS,GAAG,IAAI,CAACC,IAAI,CAACC,KAAK,CAACC,KAAK,CAAC6I,aAAa,CAACxJ,IAAI,CAACa,SAAS,CAAC;AACrE,IAAA,MAAMuC,WAAW,GAAG,IAAI,CAAC3C,IAAI,CAACC,KAAK,CAACC,KAAK,CAAC0C,eAAe,CAACrD,IAAI,CAACa,SAAS,CAAC;AACzE,IAAA,MAAM4I,OAAO,GAAGA,MACd,IAAI,CAAChJ,IAAI,CAACiJ,QAAQ,CAACC,MAAM,CACvB,YAAW;AACT,MAAA,MAAMC,YAAY,GAAGpJ,SAAS,CAACqJ,WAAW,CAACD,YAAY;MACvD,MAAME,YAAY,GAAwB,IAAI,CAAClI,GAAG,CAACnB,IAAI,CAACsJ,QAAQ,CAACH,YAAmB,CAAC;AACrF,MAAA,OAAO,MAAME,YAAY,CAACL,OAAO,CAACzJ,IAAI,CAACe,IAAI,EAAEf,IAAI,CAACC,OAAO,EAAE6B,GAAG,CAAC;AACjE,IAAA,CAAC,EACD;AACE5B,MAAAA,MAAM,EAAEF,IAAI,CAACC,OAAO,EAAEC,MAAM;AAC5B8J,MAAAA,MAAM,EAAEhK,IAAI,CAACC,OAAO,EAAE+J,MAAM;AAC5BC,MAAAA,EAAE,EAAEjK,IAAI,CAACC,OAAO,EAAEgK,EAAE;AACpB1J,MAAAA,YAAY,EAAEP,IAAI,CAACC,OAAO,EAAEM,YAAY;AACxC2J,MAAAA,SAAS,EAAE;AACT,QAAA,GAAGlK,IAAI,CAACC,OAAO,EAAEiK,SAAS;AAC1BC,QAAAA,KAAK,EAAE;AACL,UAAA,GAAGnK,IAAI,CAACC,OAAO,EAAEiK,SAAS,EAAEC,KAAK;UACjCnD,SAAS,EAAEC,IAAI,GACX;YAAEmD,OAAO,EAAEpD,SAAS,CAACQ,aAAa,CAACP,IAAI,EAAEoC,MAAM,CAAC;AAAEpC,YAAAA;AAAI,WAAE,GACxDjG;AACL;OACF;MACDqJ,WAAW,EAAEjH,WAAW,EAAEiH;AAC3B,KAAA,CACF;IACH,IAAI;AACF,MAAA,OAAO,OAAOpD,IAAI,GAAGD,SAAS,CAACsD,QAAQ,CAACrD,IAAI,EAAEwC,OAAO,CAAC,GAAGA,OAAO,EAAE,CAAC;IACrE,CAAC,CAAC,OAAOzE,GAAG,EAAE;MACZ,IAAIiC,IAAI,EAAED,SAAS,CAAC2B,eAAe,CAAC1B,IAAI,EAAEjC,GAAG,CAAC;AAC9C,MAAA,MAAMA,GAAG;AACX,IAAA,CAAC,SAAS;MACRiC,IAAI,EAAE2B,GAAG,EAAE;AACb,IAAA;AACF,EAAA;AAEA2B,EAAAA,YAAYA,CAACxC,OAAe,EAAEE,MAA0B,EAAA;AACtD,IAAA,MAAMuC,OAAO,GAAGvC,MAAM,CAACuC,OAAO,GAAG,IAAIC,IAAI,CAACxC,MAAM,CAACuC,OAAO,CAAC,CAACE,OAAO,EAAE,GAAG,EAAE;AACxE,IAAA,MAAMT,EAAE,GAAGhC,MAAM,CAACgC,EAAE,IAAI,EAAE;AAC1B,IAAA,MAAMU,OAAO,GAAG1C,MAAM,CAAC0C,OAAO;IAC9B,MAAMC,MAAM,GAAGD,OAAO,IAAIvD,MAAM,CAACa,MAAM,CAAC4C,KAAK,CAAC,IAAI,EAAE;IACpD,OAAO,CAAA,EAAG9C,OAAO,CAAA,CAAA,EAAIyC,OAAO,IAAIP,EAAE,CAAA,CAAA,EAAIW,MAAM,CAAA,CAAE;AAChD,EAAA;AAEAE,EAAAA,cAAcA,CACZjK,SAA6B,EAC7BE,IAAU,EACVd,OAA2B,EAAA;AAE3BA,IAAAA,OAAO,GAAG,IAAI,CAAC0D,MAAM,CAAC+F,QAAQ,CAAC5F,OAAO,CAAC4F,QAAQ,CAACqB,kBAAkB,CAAC9K,OAAO,CAAC;AAC3E;IACA,OAAO;MAAEY,SAAS;MAAEE,IAAI;AAAEd,MAAAA;KAAS;AACrC,EAAA;;AApgBWZ,YAAY,GAAA2L,UAAA,CAAA,CADxBC,OAAO,EAAE,EAAAC,QAAA,CAAA;EAAAC,MAAA,EAAA;AAAA,CAAA,CAAA,CACG,EAAA9L,YAAY,CAqgBxB;;AC3iBM,IAAM+L,QAAQ,GAAd,MAAMA,QAAS,SAAQ9L,QAAQ,CAAA;EACpC,MAAM+L,MAAMA,CAACC,YAAuC,EAAA;IAClD,IAAI,CAAC1J,GAAG,CAACnB,IAAI,CAACY,MAAM,CAACgK,MAAM,EAAE;AAC/B,EAAA;;AAHWD,QAAQ,GAAAJ,UAAA,CAAA,CADpBO,GAAG,EAAE,EAAAL,QAAA,CAAA;EAAAC,MAAA,EAAA;AAAA,CAAA,CAAA,CACO,EAAAC,QAAQ,CAIpB;;ACLM,IAAMI,uBAAuB,GAA7B,MAAMA,uBAAwB,SAAQlM,QAAQ,CAAA;EACnD,MAAMmK,OAAOA,GAAA;AACX;IACA,IAAI,CAAC,IAAI,CAAC7H,GAAG,CAAC6J,IAAI,CAACC,MAAM,EAAE;AACzB,MAAA,IAAI,CAACrH,KAAK,CAACP,OAAO,CAACnD,KAAK,CAACL,gBAAgB,CAAC,IAAI,CAACmG,GAAG,CAAClG,YAAY,CAAC;AAClE,IAAA;AACF,EAAA;;AANWiL,uBAAuB,GAAAR,UAAA,CAAA,CADnCW,OAAO,CAAC;AAAEC,EAAAA,QAAQ,EAAE,IAAI;AAAEC,EAAAA,KAAK,EAAE;AAAI,CAAE,CAAC,EAAAX,QAAA,CAAA;EAAAC,MAAA,EAAA;AAAA,CAAA,CAAA,CAC5B,EAAAK,uBAAuB,CAOnC;;ACVK,SAAU5H,MAAMA,CAACkI,IAAqB,EAAA;EAC1C,OAAO;AACLzK,IAAAA,MAAM,EAAE;AACN0K,MAAAA,gBAAgB,EAAE;AAAEC,QAAAA,KAAK,EAAE;OAAG;AAC9BC,MAAAA,YAAY,EAAE;AAAED,QAAAA,KAAK,EAAE;OAAG;MAC1BE,YAAY,EAAE,EAAE,GAAG,IAAI;AACvBC,MAAAA,eAAe,EAAE,IAAI;MACrBC,eAAe,EAAE,EAAE,GAAG;AACD;GACxB;AACH;;ACbO,MAAMC,cAAc,GAAG,SAAS;;ACMjC,MAAOC,MAAO,SAAQC,UAAU,CAAA;EACpC,MAAMC,QAAQA,GAAA;IACZ,MAAMC,SAAS,GAAG,IAAI,CAAC7K,GAAG,CAACyC,KAAK,CAACgI,cAAc,CAAC;IAChD,MAAMI,SAAS,CAAC3I,OAAO,CAACnD,KAAK,CAACO,YAAY,EAAE;AAC9C,EAAA;EAEA,MAAMwL,SAASA,GAAA;IACb,MAAMD,SAAS,GAAG,IAAI,CAAC7K,GAAG,CAACyC,KAAK,CAACgI,cAAc,CAAC;IAChD,MAAMI,SAAS,CAAC3I,OAAO,CAACnD,KAAK,CAACY,WAAW,EAAE;AAC7C,EAAA;AACD;;ACwFM,IAAMoL,iBAAiB,GAAvB,MAAMA,iBAAkB,SAAQC,aAAa,CAAA;AAAvCD,iBAAiB,GAAA3B,UAAA,CAAA,CAD7B6B,KAAK,EAAE,EAAA3B,QAAA,CAAA;EAAAC,MAAA,EAAA;AAAA,CAAA,CAAA,CACK,EAAAwB,iBAAiB,CAAyB;AA0BvD;;AC5HM,MAAOG,aAAgD,SAAQxN,QAAQ,CAAA;AAC3E,EAAA,MAAMe,SAASA,CAACU,IAAU,EAAEd,OAA2B,EAAA;AACrD,IAAA,OAAO,MAAM,IAAI,CAAC0D,MAAM,CAAChD,KAAK,CAACmD,OAAO,CAACnD,KAAK,CAACN,SAAS,CACpD,IAAI,CAACsD,MAAM,CAAChD,KAAK,CAACmD,OAAO,CAACnD,KAAK,CAACmK,cAAc,CAAC,IAAI,CAACiC,UAAiB,EAAEhM,IAAI,EAAEd,OAAO,CAAC,CACtF;AACH,EAAA;AAEAF,EAAAA,IAAIA,CAACgB,IAAU,EAAEd,OAA2B,EAAA;AAC1C,IAAA,OAAO,IAAI,CAAC0D,MAAM,CAAChD,KAAK,CAACmD,OAAO,CAACnD,KAAK,CAACZ,IAAI,CACzC,IAAI,CAAC4D,MAAM,CAAChD,KAAK,CAACmD,OAAO,CAACnD,KAAK,CAACmK,cAAc,CAAC,IAAI,CAACiC,UAAiB,EAAEhM,IAAI,EAAEd,OAAO,CAAC,CACtF;AACH,EAAA;EAEAsG,QAAQA,CAAChG,YAAoC,EAAA;AAC3C,IAAA,OAAO,IAAI,CAACoD,MAAM,CAAChD,KAAK,CAACmD,OAAO,CAACnD,KAAK,CAAC4F,QAAQ,CAC7C,IAAI,CAACwG,UAAgC,EACrCxM,YAAY,CACb;AACH,EAAA;AACD;;ACrBK,SAAUqF,KAAKA,CAAC3F,OAAgC,EAAA;AACpD,EAAA,OAAO+M,mBAAmB,CAAC,OAAO,EAAE/M,OAAO,CAAC;AAC9C;;"}
|
package/dist/service/queue.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { IInstanceRecord } from 'vona';
|
|
1
|
+
import type { IInstanceRecord, TestAsyncCompletion } from 'vona';
|
|
2
2
|
import * as Bull from 'bullmq';
|
|
3
3
|
import { BeanBase } from 'vona';
|
|
4
4
|
import type { IQueueCallbacks, IQueueJobContext, IQueuePushOptions, IQueueQueue, IQueueQueues, IQueueRecord, IQueueWork, IQueueWorks, TypeQueueJob } from '../types/queue.ts';
|
|
@@ -6,11 +6,17 @@ export declare class ServiceQueue extends BeanBase {
|
|
|
6
6
|
_workers: IQueueWorks;
|
|
7
7
|
_queues: IQueueQueues;
|
|
8
8
|
_queueCallbacks: IQueueCallbacks;
|
|
9
|
+
private _ownedJobs;
|
|
10
|
+
private _ownedHandles;
|
|
9
11
|
push<DATA>(info: IQueueJobContext<DATA>): Promise<void>;
|
|
10
12
|
pushAsync<DATA, RESULT>(info: IQueueJobContext<DATA>): Promise<RESULT>;
|
|
11
13
|
loadQueueWorkers(instanceName?: keyof IInstanceRecord | undefined | null): void;
|
|
12
14
|
clearWorkers(): Promise<void>;
|
|
13
15
|
clearQueues(): Promise<void>;
|
|
16
|
+
releaseOwnedInstanceHandles(owner: TestAsyncCompletion): Promise<void>;
|
|
17
|
+
private _getOwnedHandles;
|
|
18
|
+
private _ownedJobKey;
|
|
19
|
+
private _completeOwnedJob;
|
|
14
20
|
_getPrefix(): string;
|
|
15
21
|
_createWorker<DATA>(info: IQueueJobContext<DATA>, queueKey: string): IQueueWork;
|
|
16
22
|
_createQueue<DATA>(info: IQueueJobContext<DATA>, queueKey: string): IQueueQueue;
|
|
@@ -23,6 +29,8 @@ export declare class ServiceQueue extends BeanBase {
|
|
|
23
29
|
_queueEventsReady(queueQueue: IQueueQueue): Promise<void>;
|
|
24
30
|
_combineQueueKey<DATA>(info: IQueueJobContext<DATA>): string;
|
|
25
31
|
_performTask<DATA, RESULT>(job: TypeQueueJob<DATA, RESULT>): Promise<unknown>;
|
|
32
|
+
private _assertOwnedDatasource;
|
|
33
|
+
private _performTaskInner;
|
|
26
34
|
getRepeatKey(jobName: string, repeat: Bull.RepeatOptions): string;
|
|
27
35
|
prepareJobInfo<DATA>(queueName: keyof IQueueRecord, data: DATA, options?: IQueuePushOptions): IQueueJobContext<DATA>;
|
|
28
36
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"queue.d.ts","sourceRoot":"","sources":["../../src/service/queue.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"queue.d.ts","sourceRoot":"","sources":["../../src/service/queue.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,mBAAmB,EAA6B,MAAM,MAAM,CAAC;AAE5F,OAAO,KAAK,IAAI,MAAM,QAAQ,CAAC;AAC/B,OAAO,EACL,QAAQ,EAOT,MAAM,MAAM,CAAC;AAId,OAAO,KAAK,EACV,eAAe,EAEf,gBAAgB,EAChB,iBAAiB,EACjB,WAAW,EACX,YAAY,EACZ,YAAY,EACZ,UAAU,EACV,WAAW,EACX,YAAY,EACb,MAAM,mBAAmB,CAAC;AAkB3B,qBACa,YAAa,SAAQ,QAAQ;IACxC,QAAQ,EAAE,WAAW,CAAM;IAC3B,OAAO,EAAE,YAAY,CAAM;IAC3B,eAAe,EAAE,eAAe,CAAM;IACtC,OAAO,CAAC,UAAU,CAAqC;IACvD,OAAO,CAAC,aAAa,CAAmE;IAExF,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,gBAAgB,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAMvD,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,gBAAgB,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;IAKtE,gBAAgB,CAAC,YAAY,CAAC,EAAE,MAAM,eAAe,GAAG,SAAS,GAAG,IAAI;IAalE,YAAY;IAQZ,WAAW;IASX,2BAA2B,CAAC,KAAK,EAAE,mBAAmB,GAAG,OAAO,CAAC,IAAI,CAAC;IAiC5E,OAAO,CAAC,gBAAgB;IAwBxB,OAAO,CAAC,YAAY;IAIpB,OAAO,CAAC,iBAAiB;IAUzB,UAAU;IAIV,aAAa,CAAC,IAAI,EAAE,IAAI,EAAE,gBAAgB,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,MAAM;IAyElE,YAAY,CAAC,IAAI,EAAE,IAAI,EAAE,gBAAgB,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,MAAM;IAuCjE,aAAa,CAAC,IAAI,EAAE,IAAI,EAAE,gBAAgB,CAAC,IAAI,CAAC;IAWhD,YAAY,CAAC,IAAI,EAAE,IAAI,EAAE,gBAAgB,CAAC,IAAI,CAAC;IAe/C,QAAQ,CAAC,SAAS,EAAE,MAAM,YAAY,EAAE,YAAY,CAAC,EAAE,MAAM,eAAe,GAAG,SAAS,GAAG,IAAI;IAU/F,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,gBAAgB,CAAC,IAAI,CAAC;IAI5C,aAAa,CAAC,IAAI,EAChB,KAAK,EAAE,MAAM,GAAG,MAAM,EACtB,YAAY,EAAE,MAAM,GAAG,SAAS,EAChC,IAAI,EAAE,IAAI,GAAG,SAAS;IASlB,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,gBAAgB,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC;IA6GzF,iBAAiB,CAAC,UAAU,EAAE,WAAW;IAM/C,gBAAgB,CAAC,IAAI,EAAE,IAAI,EAAE,gBAAgB,CAAC,IAAI,CAAC;IAK7C,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC;IA4BhE,OAAO,CAAC,sBAAsB;YAUhB,iBAAiB;IAuD/B,YAAY,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,CAAC,aAAa;IAQxD,cAAc,CAAC,IAAI,EACjB,SAAS,EAAE,MAAM,YAAY,EAC7B,IAAI,EAAE,IAAI,EACV,OAAO,CAAC,EAAE,iBAAiB,GAC1B,gBAAgB,CAAC,IAAI,CAAC;CAK1B"}
|
package/package.json
CHANGED
package/src/service/queue.ts
CHANGED
|
@@ -1,7 +1,15 @@
|
|
|
1
|
-
import type { IInstanceRecord } from 'vona';
|
|
1
|
+
import type { IInstanceRecord, TestAsyncCompletion, TestAsyncCompletionTicket } from 'vona';
|
|
2
2
|
|
|
3
3
|
import * as Bull from 'bullmq';
|
|
4
|
-
import {
|
|
4
|
+
import {
|
|
5
|
+
BeanBase,
|
|
6
|
+
beanFullNameFromOnionName,
|
|
7
|
+
deepExtend,
|
|
8
|
+
getTestAsyncCompletion,
|
|
9
|
+
instanceDesp,
|
|
10
|
+
uuidv4,
|
|
11
|
+
withoutTestAsyncCompletion,
|
|
12
|
+
} from 'vona';
|
|
5
13
|
import { Service } from 'vona-module-a-bean';
|
|
6
14
|
import { getRedisClientKeyPrefix } from 'vona-module-a-redis';
|
|
7
15
|
|
|
@@ -18,11 +26,29 @@ import type {
|
|
|
18
26
|
TypeQueueJob,
|
|
19
27
|
} from '../types/queue.ts';
|
|
20
28
|
|
|
29
|
+
interface IOwnedQueueJob {
|
|
30
|
+
owner: TestAsyncCompletion;
|
|
31
|
+
ticket: TestAsyncCompletionTicket;
|
|
32
|
+
prefix: string;
|
|
33
|
+
queueKey: string;
|
|
34
|
+
jobId: string;
|
|
35
|
+
executions: number;
|
|
36
|
+
terminal?: { error?: Error };
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
interface IOwnedQueueHandles {
|
|
40
|
+
prefix: string;
|
|
41
|
+
worker?: IQueueWork;
|
|
42
|
+
queue?: IQueueQueue;
|
|
43
|
+
}
|
|
44
|
+
|
|
21
45
|
@Service()
|
|
22
46
|
export class ServiceQueue extends BeanBase {
|
|
23
47
|
_workers: IQueueWorks = {};
|
|
24
48
|
_queues: IQueueQueues = {};
|
|
25
49
|
_queueCallbacks: IQueueCallbacks = {};
|
|
50
|
+
private _ownedJobs = new Map<string, IOwnedQueueJob>();
|
|
51
|
+
private _ownedHandles = new Map<TestAsyncCompletion, Map<string, IOwnedQueueHandles>>();
|
|
26
52
|
|
|
27
53
|
push<DATA>(info: IQueueJobContext<DATA>): Promise<void> {
|
|
28
54
|
if (!info.options?.dbInfo) throw new Error('should specify the options.dbInfo');
|
|
@@ -65,6 +91,77 @@ export class ServiceQueue extends BeanBase {
|
|
|
65
91
|
this._queues = {};
|
|
66
92
|
}
|
|
67
93
|
|
|
94
|
+
async releaseOwnedInstanceHandles(owner: TestAsyncCompletion): Promise<void> {
|
|
95
|
+
if (owner.app !== this.app) throw new Error('Queue completion owner belongs to another app');
|
|
96
|
+
owner.assertSettled();
|
|
97
|
+
for (const job of this._ownedJobs.values()) {
|
|
98
|
+
if (job.owner === owner) throw new Error(`Owned queue job is still pending: ${job.jobId}`);
|
|
99
|
+
}
|
|
100
|
+
const handles = this._ownedHandles.get(owner);
|
|
101
|
+
if (!handles) return;
|
|
102
|
+
for (const [queueKey, entry] of handles) {
|
|
103
|
+
if (
|
|
104
|
+
entry.prefix !== this._getPrefix() ||
|
|
105
|
+
(entry.worker && this._workers[queueKey] !== entry.worker) ||
|
|
106
|
+
(entry.queue && this._queues[queueKey] !== entry.queue)
|
|
107
|
+
) {
|
|
108
|
+
throw new Error(`Owned queue handles changed: ${queueKey}`);
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
for (const [queueKey, entry] of handles) {
|
|
112
|
+
if (entry.worker) {
|
|
113
|
+
await entry.worker.worker.close();
|
|
114
|
+
delete this._workers[queueKey];
|
|
115
|
+
entry.worker = undefined;
|
|
116
|
+
}
|
|
117
|
+
if (entry.queue) {
|
|
118
|
+
await entry.queue.queue.close();
|
|
119
|
+
await entry.queue.queueEvents.close();
|
|
120
|
+
delete this._queues[queueKey];
|
|
121
|
+
entry.queue = undefined;
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
this._ownedHandles.delete(owner);
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
private _getOwnedHandles<DATA>(info: IQueueJobContext<DATA>, queueKey: string) {
|
|
128
|
+
const owner = getTestAsyncCompletion(this.app);
|
|
129
|
+
// Global/default handles belong to the runner, even during an owned operation.
|
|
130
|
+
if (!owner || info.options?.instanceName !== owner.instanceName) return;
|
|
131
|
+
let handles = this._ownedHandles.get(owner);
|
|
132
|
+
let entry = handles?.get(queueKey);
|
|
133
|
+
if (
|
|
134
|
+
(this._workers[queueKey] && this._workers[queueKey] !== entry?.worker) ||
|
|
135
|
+
(this._queues[queueKey] && this._queues[queueKey] !== entry?.queue) ||
|
|
136
|
+
(entry && entry.prefix !== this._getPrefix())
|
|
137
|
+
) {
|
|
138
|
+
throw new Error(`Cannot share preexisting owned-instance queue handles: ${queueKey}`);
|
|
139
|
+
}
|
|
140
|
+
if (!handles) {
|
|
141
|
+
handles = new Map();
|
|
142
|
+
this._ownedHandles.set(owner, handles);
|
|
143
|
+
}
|
|
144
|
+
if (!entry) {
|
|
145
|
+
entry = { prefix: this._getPrefix() };
|
|
146
|
+
handles.set(queueKey, entry);
|
|
147
|
+
}
|
|
148
|
+
return entry;
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
private _ownedJobKey(prefix: string, queueKey: string, jobId: string) {
|
|
152
|
+
return `${prefix}||${queueKey}||${jobId}`;
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
private _completeOwnedJob(prefix: string, queueKey: string, jobId: string, error?: Error) {
|
|
156
|
+
const key = this._ownedJobKey(prefix, queueKey, jobId);
|
|
157
|
+
const job = this._ownedJobs.get(key);
|
|
158
|
+
if (!job) return;
|
|
159
|
+
job.terminal = { error };
|
|
160
|
+
if (job.executions) return;
|
|
161
|
+
this._ownedJobs.delete(key);
|
|
162
|
+
job.ticket.complete(error);
|
|
163
|
+
}
|
|
164
|
+
|
|
68
165
|
_getPrefix() {
|
|
69
166
|
return `${getRedisClientKeyPrefix('bull', this.app)}queue`;
|
|
70
167
|
}
|
|
@@ -94,30 +191,33 @@ export class ServiceQueue extends BeanBase {
|
|
|
94
191
|
connection: connectionWorker,
|
|
95
192
|
});
|
|
96
193
|
_worker.workerOptions = _workerOptions;
|
|
97
|
-
_worker.worker =
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
194
|
+
_worker.worker = withoutTestAsyncCompletion(
|
|
195
|
+
() =>
|
|
196
|
+
new Bull.Worker(
|
|
197
|
+
queueKey,
|
|
198
|
+
async job => {
|
|
199
|
+
// concurrency
|
|
200
|
+
if (queueConfig?.concurrency) {
|
|
201
|
+
return await this._performTask(job);
|
|
202
|
+
}
|
|
203
|
+
// redlock
|
|
204
|
+
const info = job.data as IQueueJobContext<DATA>;
|
|
205
|
+
const queueNameSub = info.options?.queueNameSub;
|
|
206
|
+
const _lockResource = `queue:${queueKey}${queueNameSub ? `#${queueNameSub}` : ''}`;
|
|
207
|
+
return await this.$scope.redlock.service.redlock.lock(
|
|
208
|
+
_lockResource,
|
|
209
|
+
async () => {
|
|
210
|
+
return await this._performTask(job);
|
|
211
|
+
},
|
|
212
|
+
{
|
|
213
|
+
// instanceName: job.data.instanceName, // need not
|
|
214
|
+
redlock: _worker.redlock,
|
|
215
|
+
lockTTL: _lockTTL,
|
|
216
|
+
},
|
|
217
|
+
);
|
|
112
218
|
},
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
redlock: _worker.redlock,
|
|
116
|
-
lockTTL: _lockTTL,
|
|
117
|
-
},
|
|
118
|
-
);
|
|
119
|
-
},
|
|
120
|
-
_workerOptions,
|
|
219
|
+
_workerOptions,
|
|
220
|
+
),
|
|
121
221
|
);
|
|
122
222
|
|
|
123
223
|
_worker.worker.on('failed', (_job, err) => {
|
|
@@ -161,11 +261,16 @@ export class ServiceQueue extends BeanBase {
|
|
|
161
261
|
const connectionEvents = app.bean.redis.get('queue');
|
|
162
262
|
const _queueEventsOptions = { prefix, connection: connectionEvents } as Bull.QueueEventsOptions;
|
|
163
263
|
_queue.queueEventsOptions = _queueEventsOptions;
|
|
164
|
-
_queue.queueEvents =
|
|
264
|
+
_queue.queueEvents = withoutTestAsyncCompletion(
|
|
265
|
+
() => new Bull.QueueEvents(queueKey, _queueEventsOptions),
|
|
266
|
+
);
|
|
165
267
|
_queue.queueEvents.on('completed', ({ jobId, returnvalue }) => {
|
|
268
|
+
this._completeOwnedJob(prefix, queueKey, jobId);
|
|
166
269
|
this._callCallback(jobId, undefined, returnvalue);
|
|
167
270
|
});
|
|
271
|
+
// QueueEvents reports terminal failure; Worker 'failed' also fires before retries.
|
|
168
272
|
_queue.queueEvents.on('failed', ({ jobId, failedReason }) => {
|
|
273
|
+
this._completeOwnedJob(prefix, queueKey, jobId, new Error(failedReason));
|
|
169
274
|
this._callCallback(jobId, failedReason, undefined);
|
|
170
275
|
});
|
|
171
276
|
|
|
@@ -176,9 +281,11 @@ export class ServiceQueue extends BeanBase {
|
|
|
176
281
|
_ensureWorker<DATA>(info: IQueueJobContext<DATA>) {
|
|
177
282
|
// queueKey
|
|
178
283
|
const queueKey = this._combineQueueKey(info);
|
|
284
|
+
const ownedHandles = this._getOwnedHandles(info, queueKey);
|
|
179
285
|
// worker
|
|
180
286
|
if (!this._workers[queueKey]) {
|
|
181
287
|
this._workers[queueKey] = this._createWorker(info, queueKey);
|
|
288
|
+
if (ownedHandles) ownedHandles.worker = this._workers[queueKey];
|
|
182
289
|
}
|
|
183
290
|
}
|
|
184
291
|
|
|
@@ -187,9 +294,11 @@ export class ServiceQueue extends BeanBase {
|
|
|
187
294
|
this._ensureWorker(info);
|
|
188
295
|
// queueKey
|
|
189
296
|
const queueKey = this._combineQueueKey(info);
|
|
297
|
+
const ownedHandles = this._getOwnedHandles(info, queueKey);
|
|
190
298
|
// queue
|
|
191
299
|
if (!this._queues[queueKey]) {
|
|
192
300
|
this._queues[queueKey] = this._createQueue(info, queueKey);
|
|
301
|
+
if (ownedHandles) ownedHandles.queue = this._queues[queueKey];
|
|
193
302
|
}
|
|
194
303
|
// ok
|
|
195
304
|
return this._queues[queueKey];
|
|
@@ -222,6 +331,11 @@ export class ServiceQueue extends BeanBase {
|
|
|
222
331
|
}
|
|
223
332
|
|
|
224
333
|
async _queuePush<DATA, RESULT>(info: IQueueJobContext<DATA>, isAsync: boolean): Promise<RESULT> {
|
|
334
|
+
const owner = isAsync ? getTestAsyncCompletion(this.app) : undefined;
|
|
335
|
+
let ticket: TestAsyncCompletionTicket | undefined;
|
|
336
|
+
let ownedJob: IOwnedQueueJob | undefined;
|
|
337
|
+
let addAttempted = false;
|
|
338
|
+
let callbackJobId: string | undefined;
|
|
225
339
|
const telemetry = this.$scope.telemetry.service.telemetry;
|
|
226
340
|
const span = telemetry.enabled
|
|
227
341
|
? telemetry.startSpan(`queue send ${String(info.queueName)}`, {
|
|
@@ -243,17 +357,39 @@ export class ServiceQueue extends BeanBase {
|
|
|
243
357
|
}
|
|
244
358
|
let callback: Promise<RESULT> | undefined;
|
|
245
359
|
try {
|
|
360
|
+
ticket = owner?.reserve(`queue ${String(info.queueName)}`);
|
|
361
|
+
if (owner) {
|
|
362
|
+
owner.assertScope(info.options);
|
|
363
|
+
if (info.queueName !== 'a-orm:doubleDelete') {
|
|
364
|
+
throw new Error(`Unsupported owned queue: ${String(info.queueName)}`);
|
|
365
|
+
}
|
|
366
|
+
this._assertOwnedDatasource(info, owner);
|
|
367
|
+
}
|
|
246
368
|
// queue config
|
|
247
369
|
const queueConfig = this.bean.onion.queue.getOnionOptions(info.queueName);
|
|
248
370
|
// queueConfig.options: queue/worker/job/limiter
|
|
249
371
|
const jobOptionsBase = queueConfig?.options?.job;
|
|
250
|
-
// queue
|
|
251
|
-
const queueQueue = this._ensureQueue(info);
|
|
252
|
-
const queue = queueQueue.queue;
|
|
253
372
|
// job
|
|
254
|
-
const jobId = info.options?.jobOptions?.jobId || uuidv4();
|
|
373
|
+
const jobId = owner ? uuidv4() : info.options?.jobOptions?.jobId || uuidv4();
|
|
255
374
|
const jobName = info.options?.jobName || jobId;
|
|
256
375
|
const jobOptions = deepExtend({ jobId }, jobOptionsBase, info.options?.jobOptions);
|
|
376
|
+
if (owner) {
|
|
377
|
+
if (
|
|
378
|
+
jobOptions.jobId !== jobId ||
|
|
379
|
+
jobOptions.repeat ||
|
|
380
|
+
jobOptions.deduplication ||
|
|
381
|
+
jobOptions.debounce
|
|
382
|
+
) {
|
|
383
|
+
throw new Error('Owned queue jobs require a local UUID without repeat or deduplication');
|
|
384
|
+
}
|
|
385
|
+
const prefix = this._getPrefix();
|
|
386
|
+
const queueKey = this._combineQueueKey(info);
|
|
387
|
+
ownedJob = { owner, ticket: ticket!, prefix, queueKey, jobId, executions: 0 };
|
|
388
|
+
this._ownedJobs.set(this._ownedJobKey(prefix, queueKey, jobId), ownedJob);
|
|
389
|
+
}
|
|
390
|
+
// Register ownership before creating handles or awaiting QueueEvents readiness.
|
|
391
|
+
const queueQueue = this._ensureQueue(info);
|
|
392
|
+
const queue = queueQueue.queue;
|
|
257
393
|
// should not change info, hold original info.options?.jobName, info.options?.jobOptions
|
|
258
394
|
// info = deepExtend({}, info, { options: { jobName, jobOptions } });
|
|
259
395
|
if (!isAsync) {
|
|
@@ -261,6 +397,7 @@ export class ServiceQueue extends BeanBase {
|
|
|
261
397
|
return undefined as any;
|
|
262
398
|
}
|
|
263
399
|
await this._queueEventsReady(queueQueue);
|
|
400
|
+
callbackJobId = jobId;
|
|
264
401
|
callback = new Promise((resolve, reject) => {
|
|
265
402
|
this._queueCallbacks[jobId] = {
|
|
266
403
|
info,
|
|
@@ -270,8 +407,30 @@ export class ServiceQueue extends BeanBase {
|
|
|
270
407
|
},
|
|
271
408
|
};
|
|
272
409
|
});
|
|
410
|
+
// Observe terminal rejection even if queue.add rejects before we can await callback.
|
|
411
|
+
if (owner) void callback.catch(() => {});
|
|
412
|
+
addAttempted = true;
|
|
273
413
|
await queue.add(jobName, info, jobOptions);
|
|
274
414
|
} catch (err) {
|
|
415
|
+
if (owner && ticket) {
|
|
416
|
+
if (addAttempted) {
|
|
417
|
+
// A transport error cannot prove the job was not enqueued. Keep ownership
|
|
418
|
+
// pending until its actual terminal event, but retain the send failure.
|
|
419
|
+
void owner
|
|
420
|
+
.track(`queue add ${ownedJob?.jobId}`, async () => {
|
|
421
|
+
throw err;
|
|
422
|
+
})
|
|
423
|
+
.catch(() => {});
|
|
424
|
+
} else {
|
|
425
|
+
if (ownedJob) {
|
|
426
|
+
this._ownedJobs.delete(
|
|
427
|
+
this._ownedJobKey(ownedJob.prefix, ownedJob.queueKey, ownedJob.jobId),
|
|
428
|
+
);
|
|
429
|
+
}
|
|
430
|
+
if (callbackJobId) delete this._queueCallbacks[callbackJobId];
|
|
431
|
+
ticket.complete(err);
|
|
432
|
+
}
|
|
433
|
+
}
|
|
275
434
|
if (span) telemetry.recordException(span, err);
|
|
276
435
|
throw err;
|
|
277
436
|
} finally {
|
|
@@ -292,6 +451,44 @@ export class ServiceQueue extends BeanBase {
|
|
|
292
451
|
}
|
|
293
452
|
|
|
294
453
|
async _performTask<DATA, RESULT>(job: TypeQueueJob<DATA, RESULT>) {
|
|
454
|
+
const ownedJob = job.id
|
|
455
|
+
? this._ownedJobs.get(this._ownedJobKey(job.prefix, job.queueName, job.id))
|
|
456
|
+
: undefined;
|
|
457
|
+
if (!ownedJob) return await withoutTestAsyncCompletion(() => this._performTaskInner(job));
|
|
458
|
+
ownedJob.executions++;
|
|
459
|
+
try {
|
|
460
|
+
return await ownedJob.owner.bind(async () => {
|
|
461
|
+
ownedJob.owner.assertScope(job.data.options);
|
|
462
|
+
if (this._combineQueueKey(job.data) !== ownedJob.queueKey) {
|
|
463
|
+
throw new Error('Owned queue execution key does not match its registered scope');
|
|
464
|
+
}
|
|
465
|
+
this._assertOwnedDatasource(job.data, ownedJob.owner);
|
|
466
|
+
return await this._performTaskInner(job);
|
|
467
|
+
});
|
|
468
|
+
} finally {
|
|
469
|
+
ownedJob.executions--;
|
|
470
|
+
if (ownedJob.terminal) {
|
|
471
|
+
this._completeOwnedJob(
|
|
472
|
+
ownedJob.prefix,
|
|
473
|
+
ownedJob.queueKey,
|
|
474
|
+
ownedJob.jobId,
|
|
475
|
+
ownedJob.terminal.error,
|
|
476
|
+
);
|
|
477
|
+
}
|
|
478
|
+
}
|
|
479
|
+
}
|
|
480
|
+
|
|
481
|
+
private _assertOwnedDatasource<DATA>(info: IQueueJobContext<DATA>, owner: TestAsyncCompletion) {
|
|
482
|
+
// This supported queue executes against data.clientName, not just newCtx's dbInfo.
|
|
483
|
+
if (
|
|
484
|
+
info.queueName !== 'a-orm:doubleDelete' ||
|
|
485
|
+
(info.data as { clientName?: string })?.clientName !== owner.clientName
|
|
486
|
+
) {
|
|
487
|
+
throw new Error('Owned queue datasource does not match its completion owner');
|
|
488
|
+
}
|
|
489
|
+
}
|
|
490
|
+
|
|
491
|
+
private async _performTaskInner<DATA, RESULT>(job: TypeQueueJob<DATA, RESULT>) {
|
|
295
492
|
const info = job.data;
|
|
296
493
|
const telemetry = this.$scope.telemetry.service.telemetry;
|
|
297
494
|
const parent = telemetry.extractCarrier(info.options?.telemetry);
|