vona-module-a-queue 5.0.10 → 5.0.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +334 -3
- package/dist/lib/beanQueueBase.d.ts +2 -1
- package/dist/service/queue.d.ts +3 -2
- package/package.json +2 -2
- package/dist/.metadata/index.js +0 -32
- package/dist/.metadata/this.js +0 -2
- package/dist/bean/startup.loadQueueWorkers.js +0 -20
- package/dist/config/config.js +0 -9
- package/dist/lib/beanQueueBase.js +0 -12
- package/dist/lib/index.js +0 -2
- package/dist/lib/queue.js +0 -4
- package/dist/monkey.js +0 -12
- package/dist/service/queue.js +0 -250
- package/dist/types/index.js +0 -1
- package/dist/types/queue.js +0 -1
package/dist/index.js
CHANGED
|
@@ -1,3 +1,334 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
1
|
+
import { BeanInfo, BeanBase, BeanSimple, uuidv4, deepExtend, instanceDesp, beanFullNameFromOnionName, BeanScopeBase, createBeanDecorator } from 'vona';
|
|
2
|
+
import { Scope } from 'vona-module-a-bean';
|
|
3
|
+
import { Startup } from 'vona-module-a-startup';
|
|
4
|
+
import * as Bull from 'bullmq';
|
|
5
|
+
import { Service } from 'vona-module-a-web';
|
|
6
|
+
|
|
7
|
+
var _dec$2, _dec2$2, _class$2;
|
|
8
|
+
let StartupLoadQueueWorkers = (_dec$2 = Startup({
|
|
9
|
+
instance: true,
|
|
10
|
+
after: true
|
|
11
|
+
}), _dec2$2 = BeanInfo({
|
|
12
|
+
module: "a-queue"
|
|
13
|
+
}), _dec$2(_class$2 = _dec2$2(_class$2 = class StartupLoadQueueWorkers extends BeanBase {
|
|
14
|
+
async execute() {
|
|
15
|
+
// load queue workers
|
|
16
|
+
if (!this.app.meta.isTest) {
|
|
17
|
+
this.scope.service.queue.loadQueueWorkers(this.ctx.instanceName);
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
}) || _class$2) || _class$2);
|
|
21
|
+
|
|
22
|
+
function config(_app) {
|
|
23
|
+
return {
|
|
24
|
+
worker: {
|
|
25
|
+
lockDuration: 30 * 1000,
|
|
26
|
+
maxStalledCount: 1000,
|
|
27
|
+
stalledInterval: 10 * 1000
|
|
28
|
+
}
|
|
29
|
+
};
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
const __ThisModule__ = 'a-queue';
|
|
33
|
+
|
|
34
|
+
class Monkey extends BeanSimple {
|
|
35
|
+
async appClose() {
|
|
36
|
+
const scopeSelf = this.bean.scope(__ThisModule__);
|
|
37
|
+
await scopeSelf.service.queue.clearWorkers();
|
|
38
|
+
}
|
|
39
|
+
async appClosed() {
|
|
40
|
+
const scopeSelf = this.bean.scope(__ThisModule__);
|
|
41
|
+
await scopeSelf.service.queue.clearQueues();
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
var _dec$1, _dec2$1, _class$1;
|
|
46
|
+
let ServiceQueue = (_dec$1 = Service(), _dec2$1 = BeanInfo({
|
|
47
|
+
module: "a-queue"
|
|
48
|
+
}), _dec$1(_class$1 = _dec2$1(_class$1 = class ServiceQueue extends BeanBase {
|
|
49
|
+
constructor(...args) {
|
|
50
|
+
super(...args);
|
|
51
|
+
this._workers = {};
|
|
52
|
+
this._queues = {};
|
|
53
|
+
this._queueCallbacks = {};
|
|
54
|
+
}
|
|
55
|
+
push(info) {
|
|
56
|
+
if (!info.options?.dbInfo) throw new Error('should specify the options.dbInfo');
|
|
57
|
+
this._queuePush(info, false);
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
// { locale, instanceName, module, queueName,queueNameSub,data }
|
|
61
|
+
pushAsync(info) {
|
|
62
|
+
if (!info.options?.dbInfo) throw new Error('should specify the options.dbInfo');
|
|
63
|
+
return this._queuePush(info, true);
|
|
64
|
+
}
|
|
65
|
+
loadQueueWorkers(instanceName) {
|
|
66
|
+
for (const queueItem of this.bean.onion.queue.getOnionsEnabled()) {
|
|
67
|
+
const info = {
|
|
68
|
+
queueName: queueItem.name,
|
|
69
|
+
data: undefined,
|
|
70
|
+
options: {
|
|
71
|
+
instanceName
|
|
72
|
+
}
|
|
73
|
+
};
|
|
74
|
+
this._ensureWorker(info);
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
async clearWorkers() {
|
|
78
|
+
for (const queueKey in this._workers) {
|
|
79
|
+
const _worker = this._workers[queueKey];
|
|
80
|
+
await _worker.worker.close();
|
|
81
|
+
}
|
|
82
|
+
this._workers = {};
|
|
83
|
+
}
|
|
84
|
+
async clearQueues() {
|
|
85
|
+
for (const queueKey in this._queues) {
|
|
86
|
+
const _queue = this._queues[queueKey];
|
|
87
|
+
await _queue.queue.close();
|
|
88
|
+
await _queue.queueEvents.close();
|
|
89
|
+
}
|
|
90
|
+
this._queues = {};
|
|
91
|
+
}
|
|
92
|
+
_createWorker(info, queueKey) {
|
|
93
|
+
const app = this.app;
|
|
94
|
+
// worker
|
|
95
|
+
const _worker = {};
|
|
96
|
+
// prefix
|
|
97
|
+
const prefix = `bull_${app.name}:queue`;
|
|
98
|
+
// queue config
|
|
99
|
+
const queueConfig = app.bean.onion.queue.getOnionOptions(info.queueName);
|
|
100
|
+
// queueConfig.options: queue/worker/job/redlock
|
|
101
|
+
const workerOptions = queueConfig?.options?.worker;
|
|
102
|
+
const redlockOptions = queueConfig?.options?.redlock;
|
|
103
|
+
const _redlockOptions = Object.assign({}, this.$scope.redlock.config.redlock.options, redlockOptions);
|
|
104
|
+
const _lockTTL = redlockOptions?.lockTTL ?? this.$scope.redlock.config.redlock.lockTTL;
|
|
105
|
+
|
|
106
|
+
// redlock
|
|
107
|
+
if (!queueConfig?.concurrency) {
|
|
108
|
+
_worker.redlock = this.$scope.redlock.service.redlock.create(_redlockOptions);
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
// create work
|
|
112
|
+
const connectionWorker = app.bean.redis.get('queue');
|
|
113
|
+
const _workerOptions = Object.assign({}, this.scope.config.worker, workerOptions, {
|
|
114
|
+
prefix,
|
|
115
|
+
connection: connectionWorker
|
|
116
|
+
});
|
|
117
|
+
_worker.workerOptions = _workerOptions;
|
|
118
|
+
_worker.worker = new Bull.Worker(queueKey, async job => {
|
|
119
|
+
// concurrency
|
|
120
|
+
if (queueConfig?.concurrency) {
|
|
121
|
+
return await this._performTask(job);
|
|
122
|
+
}
|
|
123
|
+
// redlock
|
|
124
|
+
const info = job.data;
|
|
125
|
+
const queueNameSub = info.options?.queueNameSub;
|
|
126
|
+
const _lockResource = `queue:${queueKey}${queueNameSub ? `#${queueNameSub}` : ''}`;
|
|
127
|
+
return await this.$scope.redlock.service.redlock.lock(_lockResource, async () => {
|
|
128
|
+
return await this._performTask(job);
|
|
129
|
+
}, {
|
|
130
|
+
// instanceName: job.data.instanceName, // need not
|
|
131
|
+
redlock: _worker.redlock,
|
|
132
|
+
lockTTL: _lockTTL
|
|
133
|
+
});
|
|
134
|
+
}, _workerOptions);
|
|
135
|
+
_worker.worker.on('failed', (_job, err) => {
|
|
136
|
+
this.$logger.error(err);
|
|
137
|
+
});
|
|
138
|
+
_worker.worker.on('error', err => {
|
|
139
|
+
if (err.message && err.message.includes('Missing lock for job')) {
|
|
140
|
+
const workerInner = _worker.worker;
|
|
141
|
+
if (!workerInner.running) {
|
|
142
|
+
_worker.worker.run().catch(err => {
|
|
143
|
+
this.$logger.error(err);
|
|
144
|
+
});
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
});
|
|
148
|
+
|
|
149
|
+
// ok
|
|
150
|
+
return _worker;
|
|
151
|
+
}
|
|
152
|
+
_createQueue(info, queueKey) {
|
|
153
|
+
const app = this.app;
|
|
154
|
+
// queue
|
|
155
|
+
const _queue = {};
|
|
156
|
+
// prefix
|
|
157
|
+
const prefix = `bull_${app.name}:queue`;
|
|
158
|
+
// queue config
|
|
159
|
+
const queueConfig = app.bean.onion.queue.getOnionOptions(info.queueName);
|
|
160
|
+
// queueConfig.options: queue/worker/job/limiter
|
|
161
|
+
const queueOptions = queueConfig?.options?.queue;
|
|
162
|
+
|
|
163
|
+
// create queue
|
|
164
|
+
const connectionQueue = app.bean.redis.get('queue');
|
|
165
|
+
const _queueOptions = Object.assign({}, queueOptions, {
|
|
166
|
+
prefix,
|
|
167
|
+
connection: connectionQueue
|
|
168
|
+
});
|
|
169
|
+
_queue.config = queueConfig;
|
|
170
|
+
_queue.options = _queueOptions;
|
|
171
|
+
_queue.queue = new Bull.Queue(queueKey, _queueOptions);
|
|
172
|
+
|
|
173
|
+
// create events
|
|
174
|
+
const connectionEvents = app.bean.redis.get('queue');
|
|
175
|
+
const _queueEventsOptions = {
|
|
176
|
+
prefix,
|
|
177
|
+
connection: connectionEvents
|
|
178
|
+
};
|
|
179
|
+
_queue.queueEventsOptions = _queueEventsOptions;
|
|
180
|
+
_queue.queueEvents = new Bull.QueueEvents(queueKey, _queueEventsOptions);
|
|
181
|
+
_queue.queueEvents.on('completed', ({
|
|
182
|
+
jobId,
|
|
183
|
+
returnvalue
|
|
184
|
+
}) => {
|
|
185
|
+
this._callCallback(jobId, undefined, returnvalue);
|
|
186
|
+
});
|
|
187
|
+
_queue.queueEvents.on('failed', ({
|
|
188
|
+
jobId,
|
|
189
|
+
failedReason
|
|
190
|
+
}) => {
|
|
191
|
+
this._callCallback(jobId, failedReason, undefined);
|
|
192
|
+
});
|
|
193
|
+
|
|
194
|
+
// ok
|
|
195
|
+
return _queue;
|
|
196
|
+
}
|
|
197
|
+
_ensureWorker(info) {
|
|
198
|
+
// queueKey
|
|
199
|
+
const queueKey = this._combineQueueKey(info);
|
|
200
|
+
// worker
|
|
201
|
+
if (!this._workers[queueKey]) {
|
|
202
|
+
this._workers[queueKey] = this._createWorker(info, queueKey);
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
_ensureQueue(info) {
|
|
206
|
+
// worker
|
|
207
|
+
this._ensureWorker(info);
|
|
208
|
+
// queueKey
|
|
209
|
+
const queueKey = this._combineQueueKey(info);
|
|
210
|
+
// queue
|
|
211
|
+
if (!this._queues[queueKey]) {
|
|
212
|
+
this._queues[queueKey] = this._createQueue(info, queueKey);
|
|
213
|
+
}
|
|
214
|
+
// ok
|
|
215
|
+
return this._queues[queueKey];
|
|
216
|
+
}
|
|
217
|
+
getQueue(queueName, instanceName) {
|
|
218
|
+
return this._getQueue({
|
|
219
|
+
queueName,
|
|
220
|
+
data: undefined,
|
|
221
|
+
options: {
|
|
222
|
+
instanceName: instanceName ?? this.ctx.instanceName
|
|
223
|
+
}
|
|
224
|
+
});
|
|
225
|
+
}
|
|
226
|
+
_getQueue(info) {
|
|
227
|
+
return this._ensureQueue(info).queue;
|
|
228
|
+
}
|
|
229
|
+
_callCallback(jobId, failedReason, data) {
|
|
230
|
+
const _callback = this._queueCallbacks[jobId];
|
|
231
|
+
if (_callback) {
|
|
232
|
+
delete this._queueCallbacks[jobId];
|
|
233
|
+
_callback.callback(failedReason ? new Error(failedReason) : undefined, data);
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
async _queuePush(info, isAsync) {
|
|
237
|
+
// queue config
|
|
238
|
+
const queueConfig = this.bean.onion.queue.getOnionOptions(info.queueName);
|
|
239
|
+
// queueConfig.options: queue/worker/job/limiter
|
|
240
|
+
const jobOptionsBase = queueConfig?.options?.job;
|
|
241
|
+
// queue
|
|
242
|
+
const queue = this._getQueue(info);
|
|
243
|
+
// job
|
|
244
|
+
const jobId = info.options?.jobOptions?.jobId || uuidv4();
|
|
245
|
+
const jobName = info.options?.jobName || jobId;
|
|
246
|
+
const jobOptions = deepExtend({
|
|
247
|
+
jobId
|
|
248
|
+
}, jobOptionsBase, info.options?.jobOptions);
|
|
249
|
+
// should not change info, hold original info.options?.jobName, info.options?.jobOptions
|
|
250
|
+
// info = deepExtend({}, info, { options: { jobName, jobOptions } });
|
|
251
|
+
// not async
|
|
252
|
+
if (!isAsync) {
|
|
253
|
+
// add job
|
|
254
|
+
queue.add(jobName, info, jobOptions);
|
|
255
|
+
return undefined;
|
|
256
|
+
}
|
|
257
|
+
// async
|
|
258
|
+
return new Promise((resolve, reject) => {
|
|
259
|
+
// callback
|
|
260
|
+
this._queueCallbacks[jobId] = {
|
|
261
|
+
info,
|
|
262
|
+
callback: (err, data) => {
|
|
263
|
+
if (err) return reject(err);
|
|
264
|
+
resolve(data);
|
|
265
|
+
}
|
|
266
|
+
};
|
|
267
|
+
// add job
|
|
268
|
+
return queue.add(jobName, info, jobOptions);
|
|
269
|
+
});
|
|
270
|
+
}
|
|
271
|
+
_combineQueueKey(info) {
|
|
272
|
+
const instanceName = instanceDesp(info.options?.instanceName);
|
|
273
|
+
return `${instanceName}||${beanFullNameFromOnionName(info.queueName, 'queue')}`;
|
|
274
|
+
}
|
|
275
|
+
async _performTask(job) {
|
|
276
|
+
const info = job.data;
|
|
277
|
+
// queue config
|
|
278
|
+
const queueItem = this.bean.onion.queue.getOnionSlice(info.queueName);
|
|
279
|
+
const queueConfig = this.bean.onion.queue.getOnionOptions(info.queueName);
|
|
280
|
+
// execute
|
|
281
|
+
return await this.bean.executor.newCtx(async () => {
|
|
282
|
+
const beanFullName = queueItem.beanOptions.beanFullName;
|
|
283
|
+
const beanInstance = this.app.bean._getBean(beanFullName);
|
|
284
|
+
return await beanInstance.execute(info.data, info.options, job);
|
|
285
|
+
}, {
|
|
286
|
+
dbInfo: info.options?.dbInfo,
|
|
287
|
+
locale: info.options?.locale,
|
|
288
|
+
instanceName: info.options?.instanceName,
|
|
289
|
+
extraData: info.options?.extraData,
|
|
290
|
+
transaction: queueConfig?.transaction
|
|
291
|
+
});
|
|
292
|
+
}
|
|
293
|
+
getRepeatKey(jobName, repeat) {
|
|
294
|
+
const endDate = repeat.endDate ? new Date(repeat.endDate).getTime() : '';
|
|
295
|
+
const tz = repeat.tz || '';
|
|
296
|
+
const pattern = repeat.pattern;
|
|
297
|
+
const suffix = pattern || String(repeat.every) || '';
|
|
298
|
+
return `${jobName}:${endDate}:${tz}:${suffix}`;
|
|
299
|
+
}
|
|
300
|
+
prepareJobInfo(queueName, data, options) {
|
|
301
|
+
options = this.$scope.executor.service.executor.prepareGeneralInfo(options);
|
|
302
|
+
// info
|
|
303
|
+
return {
|
|
304
|
+
queueName,
|
|
305
|
+
data,
|
|
306
|
+
options
|
|
307
|
+
};
|
|
308
|
+
}
|
|
309
|
+
}) || _class$1) || _class$1);
|
|
310
|
+
|
|
311
|
+
var _dec, _dec2, _class;
|
|
312
|
+
let ScopeModuleAQueue = (_dec = Scope(), _dec2 = BeanInfo({
|
|
313
|
+
module: "a-queue"
|
|
314
|
+
}), _dec(_class = _dec2(_class = class ScopeModuleAQueue extends BeanScopeBase {}) || _class) || _class);
|
|
315
|
+
|
|
316
|
+
/** scope: end */
|
|
317
|
+
|
|
318
|
+
class BeanQueueBase extends BeanBase {
|
|
319
|
+
async pushAsync(data, options) {
|
|
320
|
+
return await this.$scope.queue.service.queue.pushAsync(this.$scope.queue.service.queue.prepareJobInfo(this.$onionName, data, options));
|
|
321
|
+
}
|
|
322
|
+
push(data, options) {
|
|
323
|
+
return this.$scope.queue.service.queue.push(this.$scope.queue.service.queue.prepareJobInfo(this.$onionName, data, options));
|
|
324
|
+
}
|
|
325
|
+
getQueue(instanceName) {
|
|
326
|
+
return this.$scope.queue.service.queue.getQueue(this.$onionName, instanceName);
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
function Queue(options) {
|
|
331
|
+
return createBeanDecorator('queue', options);
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
export { BeanQueueBase, Monkey, Queue, ScopeModuleAQueue, ServiceQueue, StartupLoadQueueWorkers, config };
|
|
@@ -1,7 +1,8 @@
|
|
|
1
|
+
import type { IInstanceRecord } from 'vona';
|
|
1
2
|
import type { IQueuePushOptions } from '../types/queue.ts';
|
|
2
3
|
import { BeanBase } from 'vona';
|
|
3
4
|
export declare class BeanQueueBase<DATA = unknown, RESULT = unknown> extends BeanBase {
|
|
4
5
|
pushAsync(data: DATA, options?: IQueuePushOptions): Promise<RESULT>;
|
|
5
6
|
push(data: DATA, options?: IQueuePushOptions): void;
|
|
6
|
-
getQueue(instanceName?:
|
|
7
|
+
getQueue(instanceName?: keyof IInstanceRecord): import("bullmq").Queue<any, any, string, any, any, string>;
|
|
7
8
|
}
|
package/dist/service/queue.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { IInstanceRecord } from 'vona';
|
|
1
2
|
import type { IQueueCallbacks, IQueueJobContext, IQueuePushOptions, IQueueQueue, IQueueQueues, IQueueRecord, IQueueWork, IQueueWorks, TypeQueueJob } from '../types/queue.ts';
|
|
2
3
|
import * as Bull from 'bullmq';
|
|
3
4
|
import { BeanBase } from 'vona';
|
|
@@ -7,14 +8,14 @@ export declare class ServiceQueue extends BeanBase {
|
|
|
7
8
|
_queueCallbacks: IQueueCallbacks;
|
|
8
9
|
push<DATA>(info: IQueueJobContext<DATA>): void;
|
|
9
10
|
pushAsync<DATA, RESULT>(info: IQueueJobContext<DATA>): Promise<RESULT>;
|
|
10
|
-
loadQueueWorkers(instanceName?:
|
|
11
|
+
loadQueueWorkers(instanceName?: keyof IInstanceRecord | undefined | null): void;
|
|
11
12
|
clearWorkers(): Promise<void>;
|
|
12
13
|
clearQueues(): Promise<void>;
|
|
13
14
|
_createWorker<DATA>(info: IQueueJobContext<DATA>, queueKey: string): IQueueWork;
|
|
14
15
|
_createQueue<DATA>(info: IQueueJobContext<DATA>, queueKey: string): IQueueQueue;
|
|
15
16
|
_ensureWorker<DATA>(info: IQueueJobContext<DATA>): void;
|
|
16
17
|
_ensureQueue<DATA>(info: IQueueJobContext<DATA>): IQueueQueue;
|
|
17
|
-
getQueue(queueName: keyof IQueueRecord, instanceName?:
|
|
18
|
+
getQueue(queueName: keyof IQueueRecord, instanceName?: keyof IInstanceRecord | undefined | null): Bull.Queue<any, any, string, any, any, string>;
|
|
18
19
|
_getQueue<DATA>(info: IQueueJobContext<DATA>): Bull.Queue<any, any, string, any, any, string>;
|
|
19
20
|
_callCallback<DATA>(jobId: string | number, failedReason: string | undefined, data: DATA | undefined): void;
|
|
20
21
|
_queuePush<DATA, RESULT>(info: IQueueJobContext<DATA>, isAsync: boolean): Promise<RESULT>;
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "vona-module-a-queue",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "5.0.
|
|
4
|
+
"version": "5.0.12",
|
|
5
5
|
"title": "a-queue",
|
|
6
6
|
"vonaModule": {
|
|
7
7
|
"dependencies": {},
|
|
@@ -49,6 +49,6 @@
|
|
|
49
49
|
},
|
|
50
50
|
"scripts": {
|
|
51
51
|
"clean": "rimraf dist tsconfig.build.tsbuildinfo",
|
|
52
|
-
"tsc:publish": "npm run clean && tsc -p tsconfig.build.json"
|
|
52
|
+
"tsc:publish": "npm run clean && vona :bin:buildModule && tsc -p tsconfig.build.json"
|
|
53
53
|
}
|
|
54
54
|
}
|
package/dist/.metadata/index.js
DELETED
|
@@ -1,32 +0,0 @@
|
|
|
1
|
-
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
2
|
-
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
3
|
-
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
4
|
-
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
5
|
-
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
6
|
-
};
|
|
7
|
-
/** service: end */
|
|
8
|
-
/** service: begin */
|
|
9
|
-
/** monkey: end */
|
|
10
|
-
/** scope: begin */
|
|
11
|
-
import { BeanScopeBase } from 'vona';
|
|
12
|
-
import { Scope } from 'vona-module-a-bean';
|
|
13
|
-
import 'vona';
|
|
14
|
-
import 'vona';
|
|
15
|
-
import 'vona';
|
|
16
|
-
export * from "../bean/startup.loadQueueWorkers.js";
|
|
17
|
-
/** service: end */
|
|
18
|
-
/** config: begin */
|
|
19
|
-
export * from "../config/config.js";
|
|
20
|
-
/** config: end */
|
|
21
|
-
/** monkey: begin */
|
|
22
|
-
export * from "../monkey.js";
|
|
23
|
-
/** startup: end */
|
|
24
|
-
/** service: begin */
|
|
25
|
-
export * from "../service/queue.js";
|
|
26
|
-
let ScopeModuleAQueue = class ScopeModuleAQueue extends BeanScopeBase {
|
|
27
|
-
};
|
|
28
|
-
ScopeModuleAQueue = __decorate([
|
|
29
|
-
Scope()
|
|
30
|
-
], ScopeModuleAQueue);
|
|
31
|
-
export { ScopeModuleAQueue };
|
|
32
|
-
/** scope: end */
|
package/dist/.metadata/this.js
DELETED
|
@@ -1,20 +0,0 @@
|
|
|
1
|
-
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
2
|
-
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
3
|
-
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
4
|
-
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
5
|
-
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
6
|
-
};
|
|
7
|
-
import { BeanBase } from 'vona';
|
|
8
|
-
import { Startup } from 'vona-module-a-startup';
|
|
9
|
-
let StartupLoadQueueWorkers = class StartupLoadQueueWorkers extends BeanBase {
|
|
10
|
-
async execute() {
|
|
11
|
-
// load queue workers
|
|
12
|
-
if (!this.app.meta.isTest) {
|
|
13
|
-
this.scope.service.queue.loadQueueWorkers(this.ctx.instanceName);
|
|
14
|
-
}
|
|
15
|
-
}
|
|
16
|
-
};
|
|
17
|
-
StartupLoadQueueWorkers = __decorate([
|
|
18
|
-
Startup({ instance: true, after: true })
|
|
19
|
-
], StartupLoadQueueWorkers);
|
|
20
|
-
export { StartupLoadQueueWorkers };
|
package/dist/config/config.js
DELETED
|
@@ -1,12 +0,0 @@
|
|
|
1
|
-
import { BeanBase } from 'vona';
|
|
2
|
-
export class BeanQueueBase extends BeanBase {
|
|
3
|
-
async pushAsync(data, options) {
|
|
4
|
-
return await this.$scope.queue.service.queue.pushAsync(this.$scope.queue.service.queue.prepareJobInfo(this.$onionName, data, options));
|
|
5
|
-
}
|
|
6
|
-
push(data, options) {
|
|
7
|
-
return this.$scope.queue.service.queue.push(this.$scope.queue.service.queue.prepareJobInfo(this.$onionName, data, options));
|
|
8
|
-
}
|
|
9
|
-
getQueue(instanceName) {
|
|
10
|
-
return this.$scope.queue.service.queue.getQueue(this.$onionName, instanceName);
|
|
11
|
-
}
|
|
12
|
-
}
|
package/dist/lib/index.js
DELETED
package/dist/lib/queue.js
DELETED
package/dist/monkey.js
DELETED
|
@@ -1,12 +0,0 @@
|
|
|
1
|
-
import { BeanSimple } from 'vona';
|
|
2
|
-
import { __ThisModule__ } from "./.metadata/this.js";
|
|
3
|
-
export class Monkey extends BeanSimple {
|
|
4
|
-
async appClose() {
|
|
5
|
-
const scopeSelf = this.bean.scope(__ThisModule__);
|
|
6
|
-
await scopeSelf.service.queue.clearWorkers();
|
|
7
|
-
}
|
|
8
|
-
async appClosed() {
|
|
9
|
-
const scopeSelf = this.bean.scope(__ThisModule__);
|
|
10
|
-
await scopeSelf.service.queue.clearQueues();
|
|
11
|
-
}
|
|
12
|
-
}
|
package/dist/service/queue.js
DELETED
|
@@ -1,250 +0,0 @@
|
|
|
1
|
-
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
2
|
-
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
3
|
-
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
4
|
-
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
5
|
-
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
6
|
-
};
|
|
7
|
-
import * as Bull from 'bullmq';
|
|
8
|
-
import { BeanBase, beanFullNameFromOnionName, deepExtend, instanceDesp, uuidv4 } from 'vona';
|
|
9
|
-
import { Service } from 'vona-module-a-web';
|
|
10
|
-
let ServiceQueue = class ServiceQueue extends BeanBase {
|
|
11
|
-
_workers = {};
|
|
12
|
-
_queues = {};
|
|
13
|
-
_queueCallbacks = {};
|
|
14
|
-
push(info) {
|
|
15
|
-
if (!info.options?.dbInfo)
|
|
16
|
-
throw new Error('should specify the options.dbInfo');
|
|
17
|
-
this._queuePush(info, false);
|
|
18
|
-
}
|
|
19
|
-
// { locale, instanceName, module, queueName,queueNameSub,data }
|
|
20
|
-
pushAsync(info) {
|
|
21
|
-
if (!info.options?.dbInfo)
|
|
22
|
-
throw new Error('should specify the options.dbInfo');
|
|
23
|
-
return this._queuePush(info, true);
|
|
24
|
-
}
|
|
25
|
-
loadQueueWorkers(instanceName) {
|
|
26
|
-
for (const queueItem of this.bean.onion.queue.getOnionsEnabled()) {
|
|
27
|
-
const info = {
|
|
28
|
-
queueName: queueItem.name,
|
|
29
|
-
data: undefined,
|
|
30
|
-
options: {
|
|
31
|
-
instanceName,
|
|
32
|
-
},
|
|
33
|
-
};
|
|
34
|
-
this._ensureWorker(info);
|
|
35
|
-
}
|
|
36
|
-
}
|
|
37
|
-
async clearWorkers() {
|
|
38
|
-
for (const queueKey in this._workers) {
|
|
39
|
-
const _worker = this._workers[queueKey];
|
|
40
|
-
await _worker.worker.close();
|
|
41
|
-
}
|
|
42
|
-
this._workers = {};
|
|
43
|
-
}
|
|
44
|
-
async clearQueues() {
|
|
45
|
-
for (const queueKey in this._queues) {
|
|
46
|
-
const _queue = this._queues[queueKey];
|
|
47
|
-
await _queue.queue.close();
|
|
48
|
-
await _queue.queueEvents.close();
|
|
49
|
-
}
|
|
50
|
-
this._queues = {};
|
|
51
|
-
}
|
|
52
|
-
_createWorker(info, queueKey) {
|
|
53
|
-
const app = this.app;
|
|
54
|
-
// worker
|
|
55
|
-
const _worker = {};
|
|
56
|
-
// prefix
|
|
57
|
-
const prefix = `bull_${app.name}:queue`;
|
|
58
|
-
// queue config
|
|
59
|
-
const queueConfig = app.bean.onion.queue.getOnionOptions(info.queueName);
|
|
60
|
-
// queueConfig.options: queue/worker/job/redlock
|
|
61
|
-
const workerOptions = queueConfig?.options?.worker;
|
|
62
|
-
const redlockOptions = queueConfig?.options?.redlock;
|
|
63
|
-
const _redlockOptions = Object.assign({}, this.$scope.redlock.config.redlock.options, redlockOptions);
|
|
64
|
-
const _lockTTL = redlockOptions?.lockTTL ?? this.$scope.redlock.config.redlock.lockTTL;
|
|
65
|
-
// redlock
|
|
66
|
-
if (!queueConfig?.concurrency) {
|
|
67
|
-
_worker.redlock = this.$scope.redlock.service.redlock.create(_redlockOptions);
|
|
68
|
-
}
|
|
69
|
-
// create work
|
|
70
|
-
const connectionWorker = app.bean.redis.get('queue');
|
|
71
|
-
const _workerOptions = Object.assign({}, this.scope.config.worker, workerOptions, {
|
|
72
|
-
prefix,
|
|
73
|
-
connection: connectionWorker,
|
|
74
|
-
});
|
|
75
|
-
_worker.workerOptions = _workerOptions;
|
|
76
|
-
_worker.worker = new Bull.Worker(queueKey, async (job) => {
|
|
77
|
-
// concurrency
|
|
78
|
-
if (queueConfig?.concurrency) {
|
|
79
|
-
return await this._performTask(job);
|
|
80
|
-
}
|
|
81
|
-
// redlock
|
|
82
|
-
const info = job.data;
|
|
83
|
-
const queueNameSub = info.options?.queueNameSub;
|
|
84
|
-
const _lockResource = `queue:${queueKey}${queueNameSub ? `#${queueNameSub}` : ''}`;
|
|
85
|
-
return await this.$scope.redlock.service.redlock.lock(_lockResource, async () => {
|
|
86
|
-
return await this._performTask(job);
|
|
87
|
-
}, {
|
|
88
|
-
// instanceName: job.data.instanceName, // need not
|
|
89
|
-
redlock: _worker.redlock,
|
|
90
|
-
lockTTL: _lockTTL,
|
|
91
|
-
});
|
|
92
|
-
}, _workerOptions);
|
|
93
|
-
_worker.worker.on('failed', (_job, err) => {
|
|
94
|
-
this.$logger.error(err);
|
|
95
|
-
});
|
|
96
|
-
_worker.worker.on('error', err => {
|
|
97
|
-
if (err.message && err.message.includes('Missing lock for job')) {
|
|
98
|
-
const workerInner = _worker.worker;
|
|
99
|
-
if (!workerInner.running) {
|
|
100
|
-
_worker.worker.run().catch(err => {
|
|
101
|
-
this.$logger.error(err);
|
|
102
|
-
});
|
|
103
|
-
}
|
|
104
|
-
}
|
|
105
|
-
});
|
|
106
|
-
// ok
|
|
107
|
-
return _worker;
|
|
108
|
-
}
|
|
109
|
-
_createQueue(info, queueKey) {
|
|
110
|
-
const app = this.app;
|
|
111
|
-
// queue
|
|
112
|
-
const _queue = {};
|
|
113
|
-
// prefix
|
|
114
|
-
const prefix = `bull_${app.name}:queue`;
|
|
115
|
-
// queue config
|
|
116
|
-
const queueConfig = app.bean.onion.queue.getOnionOptions(info.queueName);
|
|
117
|
-
// queueConfig.options: queue/worker/job/limiter
|
|
118
|
-
const queueOptions = queueConfig?.options?.queue;
|
|
119
|
-
// create queue
|
|
120
|
-
const connectionQueue = app.bean.redis.get('queue');
|
|
121
|
-
const _queueOptions = Object.assign({}, queueOptions, { prefix, connection: connectionQueue });
|
|
122
|
-
_queue.config = queueConfig;
|
|
123
|
-
_queue.options = _queueOptions;
|
|
124
|
-
_queue.queue = new Bull.Queue(queueKey, _queueOptions);
|
|
125
|
-
// create events
|
|
126
|
-
const connectionEvents = app.bean.redis.get('queue');
|
|
127
|
-
const _queueEventsOptions = { prefix, connection: connectionEvents };
|
|
128
|
-
_queue.queueEventsOptions = _queueEventsOptions;
|
|
129
|
-
_queue.queueEvents = new Bull.QueueEvents(queueKey, _queueEventsOptions);
|
|
130
|
-
_queue.queueEvents.on('completed', ({ jobId, returnvalue }) => {
|
|
131
|
-
this._callCallback(jobId, undefined, returnvalue);
|
|
132
|
-
});
|
|
133
|
-
_queue.queueEvents.on('failed', ({ jobId, failedReason }) => {
|
|
134
|
-
this._callCallback(jobId, failedReason, undefined);
|
|
135
|
-
});
|
|
136
|
-
// ok
|
|
137
|
-
return _queue;
|
|
138
|
-
}
|
|
139
|
-
_ensureWorker(info) {
|
|
140
|
-
// queueKey
|
|
141
|
-
const queueKey = this._combineQueueKey(info);
|
|
142
|
-
// worker
|
|
143
|
-
if (!this._workers[queueKey]) {
|
|
144
|
-
this._workers[queueKey] = this._createWorker(info, queueKey);
|
|
145
|
-
}
|
|
146
|
-
}
|
|
147
|
-
_ensureQueue(info) {
|
|
148
|
-
// worker
|
|
149
|
-
this._ensureWorker(info);
|
|
150
|
-
// queueKey
|
|
151
|
-
const queueKey = this._combineQueueKey(info);
|
|
152
|
-
// queue
|
|
153
|
-
if (!this._queues[queueKey]) {
|
|
154
|
-
this._queues[queueKey] = this._createQueue(info, queueKey);
|
|
155
|
-
}
|
|
156
|
-
// ok
|
|
157
|
-
return this._queues[queueKey];
|
|
158
|
-
}
|
|
159
|
-
getQueue(queueName, instanceName) {
|
|
160
|
-
return this._getQueue({
|
|
161
|
-
queueName,
|
|
162
|
-
data: undefined,
|
|
163
|
-
options: {
|
|
164
|
-
instanceName: instanceName ?? this.ctx.instanceName,
|
|
165
|
-
},
|
|
166
|
-
});
|
|
167
|
-
}
|
|
168
|
-
_getQueue(info) {
|
|
169
|
-
return this._ensureQueue(info).queue;
|
|
170
|
-
}
|
|
171
|
-
_callCallback(jobId, failedReason, data) {
|
|
172
|
-
const _callback = this._queueCallbacks[jobId];
|
|
173
|
-
if (_callback) {
|
|
174
|
-
delete this._queueCallbacks[jobId];
|
|
175
|
-
_callback.callback(failedReason ? new Error(failedReason) : undefined, data);
|
|
176
|
-
}
|
|
177
|
-
}
|
|
178
|
-
async _queuePush(info, isAsync) {
|
|
179
|
-
// queue config
|
|
180
|
-
const queueConfig = this.bean.onion.queue.getOnionOptions(info.queueName);
|
|
181
|
-
// queueConfig.options: queue/worker/job/limiter
|
|
182
|
-
const jobOptionsBase = queueConfig?.options?.job;
|
|
183
|
-
// queue
|
|
184
|
-
const queue = this._getQueue(info);
|
|
185
|
-
// job
|
|
186
|
-
const jobId = info.options?.jobOptions?.jobId || uuidv4();
|
|
187
|
-
const jobName = info.options?.jobName || jobId;
|
|
188
|
-
const jobOptions = deepExtend({ jobId }, jobOptionsBase, info.options?.jobOptions);
|
|
189
|
-
// should not change info, hold original info.options?.jobName, info.options?.jobOptions
|
|
190
|
-
// info = deepExtend({}, info, { options: { jobName, jobOptions } });
|
|
191
|
-
// not async
|
|
192
|
-
if (!isAsync) {
|
|
193
|
-
// add job
|
|
194
|
-
queue.add(jobName, info, jobOptions);
|
|
195
|
-
return undefined;
|
|
196
|
-
}
|
|
197
|
-
// async
|
|
198
|
-
return new Promise((resolve, reject) => {
|
|
199
|
-
// callback
|
|
200
|
-
this._queueCallbacks[jobId] = {
|
|
201
|
-
info,
|
|
202
|
-
callback: (err, data) => {
|
|
203
|
-
if (err)
|
|
204
|
-
return reject(err);
|
|
205
|
-
resolve(data);
|
|
206
|
-
},
|
|
207
|
-
};
|
|
208
|
-
// add job
|
|
209
|
-
return queue.add(jobName, info, jobOptions);
|
|
210
|
-
});
|
|
211
|
-
}
|
|
212
|
-
_combineQueueKey(info) {
|
|
213
|
-
const instanceName = instanceDesp(info.options?.instanceName);
|
|
214
|
-
return `${instanceName}||${beanFullNameFromOnionName(info.queueName, 'queue')}`;
|
|
215
|
-
}
|
|
216
|
-
async _performTask(job) {
|
|
217
|
-
const info = job.data;
|
|
218
|
-
// queue config
|
|
219
|
-
const queueItem = this.bean.onion.queue.getOnionSlice(info.queueName);
|
|
220
|
-
const queueConfig = this.bean.onion.queue.getOnionOptions(info.queueName);
|
|
221
|
-
// execute
|
|
222
|
-
return await this.bean.executor.newCtx(async () => {
|
|
223
|
-
const beanFullName = queueItem.beanOptions.beanFullName;
|
|
224
|
-
const beanInstance = this.app.bean._getBean(beanFullName);
|
|
225
|
-
return await beanInstance.execute(info.data, info.options, job);
|
|
226
|
-
}, {
|
|
227
|
-
dbInfo: info.options?.dbInfo,
|
|
228
|
-
locale: info.options?.locale,
|
|
229
|
-
instanceName: info.options?.instanceName,
|
|
230
|
-
extraData: info.options?.extraData,
|
|
231
|
-
transaction: queueConfig?.transaction,
|
|
232
|
-
});
|
|
233
|
-
}
|
|
234
|
-
getRepeatKey(jobName, repeat) {
|
|
235
|
-
const endDate = repeat.endDate ? new Date(repeat.endDate).getTime() : '';
|
|
236
|
-
const tz = repeat.tz || '';
|
|
237
|
-
const pattern = repeat.pattern;
|
|
238
|
-
const suffix = (pattern || String(repeat.every)) || '';
|
|
239
|
-
return `${jobName}:${endDate}:${tz}:${suffix}`;
|
|
240
|
-
}
|
|
241
|
-
prepareJobInfo(queueName, data, options) {
|
|
242
|
-
options = this.$scope.executor.service.executor.prepareGeneralInfo(options);
|
|
243
|
-
// info
|
|
244
|
-
return { queueName, data, options };
|
|
245
|
-
}
|
|
246
|
-
};
|
|
247
|
-
ServiceQueue = __decorate([
|
|
248
|
-
Service()
|
|
249
|
-
], ServiceQueue);
|
|
250
|
-
export { ServiceQueue };
|
package/dist/types/index.js
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export * from "./queue.js";
|
package/dist/types/queue.js
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export {};
|