tspace-mysql 1.9.3 → 1.9.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +134 -17
- package/dist/lib/constants/index.d.ts +1 -1
- package/dist/lib/constants/index.js +8 -1
- package/dist/lib/constants/index.js.map +1 -1
- package/dist/lib/core/Abstracts/AbstractBuilder.d.ts +1 -1
- package/dist/lib/core/Abstracts/AbstractBuilder.js +1 -0
- package/dist/lib/core/Abstracts/AbstractBuilder.js.map +1 -1
- package/dist/lib/core/Blueprint.d.ts +43 -10
- package/dist/lib/core/Blueprint.js +43 -2
- package/dist/lib/core/Blueprint.js.map +1 -1
- package/dist/lib/core/Builder.d.ts +6 -6
- package/dist/lib/core/Builder.js +25 -28
- package/dist/lib/core/Builder.js.map +1 -1
- package/dist/lib/core/Contracts/Audit.d.ts +2 -2
- package/dist/lib/core/Contracts/Logger.d.ts +2 -2
- package/dist/lib/core/DB.d.ts +137 -6
- package/dist/lib/core/DB.js +153 -30
- package/dist/lib/core/DB.js.map +1 -1
- package/dist/lib/core/Driver/index.d.ts +1 -0
- package/dist/lib/core/Driver/index.js +11 -0
- package/dist/lib/core/Driver/index.js.map +1 -1
- package/dist/lib/core/Driver/mariadb/MariadbDriver.d.ts +3 -0
- package/dist/lib/core/Driver/mariadb/MariadbDriver.js +17 -2
- package/dist/lib/core/Driver/mariadb/MariadbDriver.js.map +1 -1
- package/dist/lib/core/Driver/mongodb/MongodbDriver.d.ts +3 -0
- package/dist/lib/core/Driver/mongodb/MongodbDriver.js +4 -1
- package/dist/lib/core/Driver/mongodb/MongodbDriver.js.map +1 -1
- package/dist/lib/core/Driver/mysql/MysqlDriver.d.ts +3 -0
- package/dist/lib/core/Driver/mysql/MysqlDriver.js +32 -2
- package/dist/lib/core/Driver/mysql/MysqlDriver.js.map +1 -1
- package/dist/lib/core/Driver/postgres/PostgresDriver.d.ts +3 -0
- package/dist/lib/core/Driver/postgres/PostgresDriver.js +21 -4
- package/dist/lib/core/Driver/postgres/PostgresDriver.js.map +1 -1
- package/dist/lib/core/Driver/sqlite/SqliteDriver.d.ts +3 -0
- package/dist/lib/core/Driver/sqlite/SqliteDriver.js +10 -1
- package/dist/lib/core/Driver/sqlite/SqliteDriver.js.map +1 -1
- package/dist/lib/core/Model.d.ts +331 -29
- package/dist/lib/core/Model.js +329 -19
- package/dist/lib/core/Model.js.map +1 -1
- package/dist/lib/core/Pool.js +6 -0
- package/dist/lib/core/Pool.js.map +1 -1
- package/dist/lib/core/{Queue.d.ts → Queue/index.d.ts} +115 -111
- package/dist/lib/core/Queue/index.js +283 -0
- package/dist/lib/core/Queue/index.js.map +1 -0
- package/dist/lib/core/Queue/types.d.ts +40 -0
- package/dist/lib/core/Queue/types.js +3 -0
- package/dist/lib/core/Queue/types.js.map +1 -0
- package/dist/lib/core/Queue/worker.d.ts +103 -0
- package/dist/lib/core/{Queue.js → Queue/worker.js} +251 -310
- package/dist/lib/core/Queue/worker.js.map +1 -0
- package/dist/lib/core/Repository.d.ts +40 -20
- package/dist/lib/core/Repository.js +35 -14
- package/dist/lib/core/Repository.js.map +1 -1
- package/dist/lib/core/UtilityTypes.d.ts +7 -3
- package/dist/lib/core/UtilityTypes.js.map +1 -1
- package/dist/lib/types/decorator/index.d.ts +3 -0
- package/dist/lib/types/index.d.ts +17 -16
- package/dist/lib/types/repository/index.d.ts +3 -3
- package/dist/lib/utils/index.d.ts +10 -5
- package/dist/lib/utils/index.js +42 -9
- package/dist/lib/utils/index.js.map +1 -1
- package/package.json +3 -5
- package/dist/lib/core/Queue.js.map +0 -1
|
@@ -1,11 +1,10 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.
|
|
4
|
-
const Blueprint_1 = require("
|
|
5
|
-
const DB_1 = require("
|
|
3
|
+
exports.Worker = void 0;
|
|
4
|
+
const Blueprint_1 = require("../Blueprint");
|
|
5
|
+
const DB_1 = require("../DB");
|
|
6
6
|
;
|
|
7
|
-
const Model_1 = require("
|
|
8
|
-
const Meta_1 = require("./Meta");
|
|
7
|
+
const Model_1 = require("../Model");
|
|
9
8
|
const QUEUE_STATUS = {
|
|
10
9
|
dispatch: 'Dispatch',
|
|
11
10
|
receive: 'Receive',
|
|
@@ -49,27 +48,31 @@ const schema = {
|
|
|
49
48
|
max_attempts: Blueprint_1.Blueprint.int().default(3),
|
|
50
49
|
delay_ms: Blueprint_1.Blueprint.int().default(0),
|
|
51
50
|
locked_by: Blueprint_1.Blueprint.text().null(),
|
|
52
|
-
locked_at: Blueprint_1.Blueprint.
|
|
53
|
-
available_at: Blueprint_1.Blueprint.
|
|
54
|
-
completed_at: Blueprint_1.Blueprint.
|
|
55
|
-
created_at: Blueprint_1.Blueprint.
|
|
56
|
-
updated_at: Blueprint_1.Blueprint.
|
|
51
|
+
locked_at: Blueprint_1.Blueprint.datetime().null(),
|
|
52
|
+
available_at: Blueprint_1.Blueprint.datetime().notNull(),
|
|
53
|
+
completed_at: Blueprint_1.Blueprint.datetime().null(),
|
|
54
|
+
created_at: Blueprint_1.Blueprint.datetime().null(),
|
|
55
|
+
updated_at: Blueprint_1.Blueprint.datetime().null()
|
|
57
56
|
};
|
|
58
57
|
class Worker extends Model_1.Model {
|
|
59
58
|
HOSTNAME = String(process.env?.hostname ?? 'unknown');
|
|
60
59
|
INSPECT_EXEC = false;
|
|
61
60
|
STOPPING = false;
|
|
62
61
|
IS_FLUSHING = false;
|
|
63
|
-
LIMIT_CONNECTIONS = 41;
|
|
64
|
-
MAX_IDLE_RETRIES = 15;
|
|
65
62
|
ACTIVE_JOBS = 0;
|
|
63
|
+
MAX_IDLE_RETRIES = 5;
|
|
66
64
|
BATCH_SIZE = 1000;
|
|
67
65
|
MAX_WAIT_MS = 50;
|
|
66
|
+
POLL = {
|
|
67
|
+
timeout: null,
|
|
68
|
+
interval: 1000 * 60
|
|
69
|
+
};
|
|
68
70
|
BUFFER = {
|
|
69
71
|
jobs: [],
|
|
70
72
|
timeout: null
|
|
71
73
|
};
|
|
72
74
|
WORKER_STATE = new Map();
|
|
75
|
+
EVENT_SUBSCRIBERS = new Map();
|
|
73
76
|
boot() {
|
|
74
77
|
this.useUUID();
|
|
75
78
|
this.useTimestamp();
|
|
@@ -82,6 +85,7 @@ class Worker extends Model_1.Model {
|
|
|
82
85
|
throw new Error('Queue is not supported for MongoDB. Use a different driver or disable queue features.');
|
|
83
86
|
}
|
|
84
87
|
await this.sync({ force: true, index: true }).catch(() => null);
|
|
88
|
+
await this._initializeWorkerJobs();
|
|
85
89
|
if (opts.inspect) {
|
|
86
90
|
this.INSPECT_EXEC = true;
|
|
87
91
|
console.log(`\x1b[34mQueue:\x1b[0m \x1b[32mJob processing started\x1b[0m`);
|
|
@@ -95,31 +99,41 @@ class Worker extends Model_1.Model {
|
|
|
95
99
|
if (opts.maxIdleRetries) {
|
|
96
100
|
this.MAX_IDLE_RETRIES = opts.maxIdleRetries;
|
|
97
101
|
}
|
|
98
|
-
if (opts.
|
|
99
|
-
this.
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
this.LIMIT_CONNECTIONS = maxConnections
|
|
104
|
-
? Math.max(10, Math.floor(maxConnections / 3))
|
|
105
|
-
: this.LIMIT_CONNECTIONS;
|
|
102
|
+
if (opts.poll?.enabled) {
|
|
103
|
+
this.POLL.interval = opts.poll?.interval ?? this.POLL.interval;
|
|
104
|
+
this.POLL.timeout = setInterval(() => {
|
|
105
|
+
this._pollWorkerJobs();
|
|
106
|
+
}, this.POLL.interval);
|
|
106
107
|
}
|
|
107
108
|
return this;
|
|
108
109
|
}
|
|
109
110
|
async shutdown() {
|
|
110
|
-
this.STOPPING = true;
|
|
111
111
|
while (this.ACTIVE_JOBS > 0) {
|
|
112
112
|
if (this.INSPECT_EXEC) {
|
|
113
|
-
console.log(
|
|
113
|
+
console.log([
|
|
114
|
+
`\x1b[34mQueue:\x1b[0m`,
|
|
115
|
+
`\x1b[38;2;77;215;240mCurrently processing\x1b[0m`,
|
|
116
|
+
`${this.ACTIVE_JOBS} job(s)`
|
|
117
|
+
].join(' '));
|
|
114
118
|
}
|
|
115
|
-
await
|
|
119
|
+
await this._sleep(200);
|
|
116
120
|
}
|
|
121
|
+
await this._sleep(2000);
|
|
122
|
+
if (this.POLL.timeout) {
|
|
123
|
+
clearInterval(this.POLL.interval);
|
|
124
|
+
this.POLL.timeout = null;
|
|
125
|
+
}
|
|
126
|
+
this.STOPPING = true;
|
|
117
127
|
if (this.INSPECT_EXEC) {
|
|
118
|
-
console.log("\x1b[34mQueue:\x1b[0m \x1b[
|
|
128
|
+
console.log("\x1b[34mQueue:\x1b[0m \x1b[31mJob processing stopped\x1b[0m");
|
|
119
129
|
}
|
|
120
130
|
}
|
|
121
131
|
async flush() {
|
|
132
|
+
const jobs = await new Worker().count('id');
|
|
122
133
|
await this.truncate({ force: true });
|
|
134
|
+
if (this.INSPECT_EXEC) {
|
|
135
|
+
console.log(`\x1b[34mQueue:\x1b[0m \x1b[31mFlush all jobs (${jobs} jobs)\x1b[0m`);
|
|
136
|
+
}
|
|
123
137
|
return;
|
|
124
138
|
}
|
|
125
139
|
async getJobOverallStats(name) {
|
|
@@ -148,7 +162,7 @@ class Worker extends Model_1.Model {
|
|
|
148
162
|
.when(name, (q) => q.where('name', 'LIKE', `%${DB_1.DB.escape(name)}%`))
|
|
149
163
|
.groupBy('name', 'status')
|
|
150
164
|
.orderBy('name')
|
|
151
|
-
.
|
|
165
|
+
.findMany();
|
|
152
166
|
const map = new Map();
|
|
153
167
|
for (const row of rows) {
|
|
154
168
|
const name = row.name;
|
|
@@ -178,7 +192,7 @@ class Worker extends Model_1.Model {
|
|
|
178
192
|
async getJobs(name) {
|
|
179
193
|
const jobs = await new Worker()
|
|
180
194
|
.when(name != null, (q) => q.where('name', 'LIKE', `%${DB_1.DB.escape(name)}%`))
|
|
181
|
-
.
|
|
195
|
+
.findMany();
|
|
182
196
|
return jobs;
|
|
183
197
|
}
|
|
184
198
|
async getNames() {
|
|
@@ -188,21 +202,23 @@ class Worker extends Model_1.Model {
|
|
|
188
202
|
return new Promise((resolve, reject) => {
|
|
189
203
|
const jobData = {
|
|
190
204
|
name,
|
|
191
|
-
payload: payload == null ? null : this.
|
|
205
|
+
payload: payload == null ? null : this._safeJsonStringify(payload),
|
|
192
206
|
status: 'pending',
|
|
193
|
-
delay_ms: opts.delayMs ?? 0,
|
|
194
|
-
available_at: opts.delayMs ? new Date(Date.now() + opts.delayMs) : new Date(),
|
|
195
207
|
priority: opts.priority ?? 0,
|
|
196
208
|
attempts: 0,
|
|
197
209
|
max_attempts: opts.maxAttempts ?? 3,
|
|
198
|
-
metadata: opts.metadata ? this.
|
|
210
|
+
metadata: opts.metadata ? this._safeJsonStringify(opts.metadata) : null,
|
|
211
|
+
delay_ms: opts.delayMs ?? 0,
|
|
212
|
+
available_at: opts.delayMs ? new Date(Date.now() + opts.delayMs) : new Date(),
|
|
213
|
+
created_at: new Date(),
|
|
214
|
+
updated_at: new Date()
|
|
199
215
|
};
|
|
200
216
|
this.BUFFER.jobs.push({ jobData, resolve, reject });
|
|
201
217
|
if (this.BUFFER.jobs.length >= this.BATCH_SIZE) {
|
|
202
|
-
this._flushBuffer(
|
|
218
|
+
this._flushBuffer();
|
|
203
219
|
}
|
|
204
220
|
else if (!this.BUFFER.timeout) {
|
|
205
|
-
this.BUFFER.timeout = setTimeout(() => this._flushBuffer(
|
|
221
|
+
this.BUFFER.timeout = setTimeout(() => this._flushBuffer(), this.MAX_WAIT_MS);
|
|
206
222
|
}
|
|
207
223
|
});
|
|
208
224
|
}
|
|
@@ -221,7 +237,11 @@ class Worker extends Model_1.Model {
|
|
|
221
237
|
}
|
|
222
238
|
});
|
|
223
239
|
if (this.INSPECT_EXEC) {
|
|
224
|
-
console.log(
|
|
240
|
+
console.log([
|
|
241
|
+
'\x1b[34mQueue:\x1b[0m',
|
|
242
|
+
`\x1b[35m'${name}'\x1b[0m`,
|
|
243
|
+
`\x1b[32m${QUEUE_STATUS.dispatch}\x1b[0m`,
|
|
244
|
+
].join(' '));
|
|
225
245
|
}
|
|
226
246
|
const dispatch = async () => {
|
|
227
247
|
if (this.STOPPING)
|
|
@@ -230,9 +250,9 @@ class Worker extends Model_1.Model {
|
|
|
230
250
|
if (!state)
|
|
231
251
|
return;
|
|
232
252
|
if (state.running >= state.opts.concurrency) {
|
|
233
|
-
const
|
|
253
|
+
const delayMs = opts.interval ?? 1000;
|
|
234
254
|
state.running--;
|
|
235
|
-
setTimeout(dispatch,
|
|
255
|
+
setTimeout(dispatch, delayMs);
|
|
236
256
|
return;
|
|
237
257
|
}
|
|
238
258
|
const capacity = state.opts.concurrency - state.running;
|
|
@@ -242,13 +262,16 @@ class Worker extends Model_1.Model {
|
|
|
242
262
|
if (state.idle >= this.MAX_IDLE_RETRIES) {
|
|
243
263
|
state.sleeping = true;
|
|
244
264
|
if (this.INSPECT_EXEC) {
|
|
245
|
-
console.log(
|
|
265
|
+
console.log([
|
|
266
|
+
'\x1b[34mQueue:\x1b[0m',
|
|
267
|
+
`\x1b[35m'${name}'\x1b[0m`,
|
|
268
|
+
`\x1b[90m${QUEUE_STATUS.idle} (no jobs available)\x1b[0m`,
|
|
269
|
+
].join(' '));
|
|
246
270
|
}
|
|
247
271
|
return;
|
|
248
272
|
}
|
|
249
|
-
const
|
|
250
|
-
|
|
251
|
-
setTimeout(dispatch, opts.interval + backoff + jitter);
|
|
273
|
+
const delayMs = opts.interval ?? 1000;
|
|
274
|
+
setTimeout(dispatch, delayMs);
|
|
252
275
|
return;
|
|
253
276
|
}
|
|
254
277
|
state.idle = 0;
|
|
@@ -258,42 +281,95 @@ class Worker extends Model_1.Model {
|
|
|
258
281
|
};
|
|
259
282
|
return dispatch();
|
|
260
283
|
}
|
|
284
|
+
async publish(event, payload, opts = {}) {
|
|
285
|
+
const subscribers = this.EVENT_SUBSCRIBERS.get(event);
|
|
286
|
+
if (!subscribers || subscribers.size === 0) {
|
|
287
|
+
return;
|
|
288
|
+
}
|
|
289
|
+
await Promise.all([...subscribers].map(name => this.add(name, payload, opts)));
|
|
290
|
+
}
|
|
291
|
+
async subscribe(event, name, handler, opts = {
|
|
292
|
+
interval: 1_000,
|
|
293
|
+
concurrency: 1
|
|
294
|
+
}) {
|
|
295
|
+
const jobName = `${event}.${name}`;
|
|
296
|
+
if (!this.EVENT_SUBSCRIBERS.has(event)) {
|
|
297
|
+
this.EVENT_SUBSCRIBERS.set(event, new Set());
|
|
298
|
+
}
|
|
299
|
+
this.EVENT_SUBSCRIBERS
|
|
300
|
+
.get(event)
|
|
301
|
+
.add(jobName);
|
|
302
|
+
await this.process(jobName, handler, opts);
|
|
303
|
+
}
|
|
261
304
|
async _runJob(name, job, state) {
|
|
262
305
|
state.running++;
|
|
263
306
|
this.ACTIVE_JOBS++;
|
|
264
|
-
const
|
|
307
|
+
const startTime = +new Date();
|
|
308
|
+
if (job.__job.delay_ms) {
|
|
309
|
+
setTimeout(async () => {
|
|
310
|
+
await this._wakeWorker(name);
|
|
311
|
+
await this._executeJob({ name, job, state, startTime });
|
|
312
|
+
state.running--;
|
|
313
|
+
this.ACTIVE_JOBS--;
|
|
314
|
+
}, job.__job.delay_ms);
|
|
315
|
+
return;
|
|
316
|
+
}
|
|
317
|
+
await this._executeJob({ name, job, state, startTime });
|
|
318
|
+
state.running--;
|
|
319
|
+
this.ACTIVE_JOBS--;
|
|
320
|
+
return;
|
|
321
|
+
}
|
|
322
|
+
async _executeJob({ name, job, state, startTime }) {
|
|
323
|
+
if (this.INSPECT_EXEC) {
|
|
324
|
+
console.log([
|
|
325
|
+
'\x1b[34mQueue:\x1b[0m',
|
|
326
|
+
`\x1b[35m'${name}'\x1b[0m`,
|
|
327
|
+
`\x1b[38;2;77;215;240m${QUEUE_STATUS.processing}\x1b[0m`,
|
|
328
|
+
`job \x1b[38;5;208m${job.id}\x1b[0m`,
|
|
329
|
+
].join(' '));
|
|
330
|
+
}
|
|
265
331
|
try {
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
}
|
|
273
|
-
const result = await handler(job);
|
|
332
|
+
const result = await state.handler({
|
|
333
|
+
id: job.id,
|
|
334
|
+
name: job.name,
|
|
335
|
+
status: job.status,
|
|
336
|
+
payload: job.payload
|
|
337
|
+
});
|
|
274
338
|
await new Worker()
|
|
275
339
|
.where('id', job.id)
|
|
276
340
|
.update({
|
|
277
341
|
status: 'completed',
|
|
278
|
-
result: this.
|
|
279
|
-
completed_at: this.$utils.timestamp()
|
|
342
|
+
result: this._safeJsonStringify(result),
|
|
343
|
+
completed_at: this.$utils.timestamp(),
|
|
344
|
+
attempts: 0
|
|
280
345
|
})
|
|
281
346
|
.void()
|
|
282
347
|
.save();
|
|
283
348
|
const endTime = +new Date();
|
|
284
349
|
if (this.INSPECT_EXEC) {
|
|
285
|
-
console.log(
|
|
350
|
+
console.log([
|
|
351
|
+
'\x1b[34mQueue:\x1b[0m',
|
|
352
|
+
`\x1b[35m'${name}'\x1b[0m`,
|
|
353
|
+
`\x1b[32m${QUEUE_STATUS.completed}\x1b[0m`,
|
|
354
|
+
`job \x1b[38;5;208m${job.id}\x1b[0m`,
|
|
355
|
+
`(${endTime - startTime}ms)`,
|
|
356
|
+
].join(' '));
|
|
286
357
|
}
|
|
287
358
|
}
|
|
288
359
|
catch (err) {
|
|
289
360
|
if (this.INSPECT_EXEC) {
|
|
290
|
-
console.log(
|
|
361
|
+
console.log([
|
|
362
|
+
'\x1b[34mQueue:\x1b[0m',
|
|
363
|
+
`\x1b[35m'${name}'\x1b[0m`,
|
|
364
|
+
`\x1b[31m${QUEUE_STATUS.failed}\x1b[0m`,
|
|
365
|
+
`job \x1b[38;5;208m${job.id}\x1b[0m`,
|
|
366
|
+
].join(' '));
|
|
291
367
|
}
|
|
292
368
|
await new Worker()
|
|
293
369
|
.where('id', job.id)
|
|
294
370
|
.update({
|
|
295
371
|
status: 'failed',
|
|
296
|
-
error: this.
|
|
372
|
+
error: this._safeJsonStringify({
|
|
297
373
|
message: err.message,
|
|
298
374
|
name: err.name,
|
|
299
375
|
stack: err.stack,
|
|
@@ -308,26 +384,40 @@ class Worker extends Model_1.Model {
|
|
|
308
384
|
attempts++;
|
|
309
385
|
try {
|
|
310
386
|
const startTime = +new Date();
|
|
311
|
-
const result = await handler(job);
|
|
387
|
+
const result = await state.handler(job);
|
|
312
388
|
const endTime = +new Date();
|
|
313
389
|
await new Worker()
|
|
314
390
|
.where('id', job.id)
|
|
315
391
|
.update({
|
|
316
392
|
status: 'completed',
|
|
317
393
|
attempts,
|
|
318
|
-
result: this.
|
|
319
|
-
completed_at: this.$utils.timestamp()
|
|
394
|
+
result: this._safeJsonStringify(result),
|
|
395
|
+
completed_at: this.$utils.timestamp(),
|
|
320
396
|
})
|
|
321
397
|
.void()
|
|
322
398
|
.save();
|
|
323
399
|
if (this.INSPECT_EXEC) {
|
|
324
|
-
console.log(
|
|
400
|
+
console.log([
|
|
401
|
+
'\x1b[34mQueue:\x1b[0m',
|
|
402
|
+
`\x1b[35m'${name}'\x1b[0m`,
|
|
403
|
+
`\x1b[32m${QUEUE_STATUS.retry.completed}\x1b[0m`,
|
|
404
|
+
'job',
|
|
405
|
+
`\x1b[38;5;208m${job.id}\x1b[0m`,
|
|
406
|
+
`(${endTime - startTime}ms ${attempts}/${maxAttempts})`,
|
|
407
|
+
].join(' '));
|
|
325
408
|
}
|
|
326
409
|
break;
|
|
327
410
|
}
|
|
328
411
|
catch (err) {
|
|
329
412
|
if (this.INSPECT_EXEC) {
|
|
330
|
-
console.log(
|
|
413
|
+
console.log([
|
|
414
|
+
'\x1b[34mQueue:\x1b[0m',
|
|
415
|
+
`\x1b[35m'${name}'\x1b[0m`,
|
|
416
|
+
`\x1b[31m${QUEUE_STATUS.retry.attempts}\x1b[0m`,
|
|
417
|
+
'job',
|
|
418
|
+
`\x1b[38;5;208m${job.id}\x1b[0m`,
|
|
419
|
+
`(${attempts}/${maxAttempts})`,
|
|
420
|
+
].join(' '));
|
|
331
421
|
}
|
|
332
422
|
if (attempts >= maxAttempts) {
|
|
333
423
|
await new Worker()
|
|
@@ -335,7 +425,7 @@ class Worker extends Model_1.Model {
|
|
|
335
425
|
.update({
|
|
336
426
|
status: 'failed',
|
|
337
427
|
attempts,
|
|
338
|
-
error: this.
|
|
428
|
+
error: this._safeJsonStringify({
|
|
339
429
|
retry: true,
|
|
340
430
|
message: err?.message,
|
|
341
431
|
name: err?.name,
|
|
@@ -346,7 +436,14 @@ class Worker extends Model_1.Model {
|
|
|
346
436
|
.void()
|
|
347
437
|
.save();
|
|
348
438
|
if (this.INSPECT_EXEC) {
|
|
349
|
-
console.log(
|
|
439
|
+
console.log([
|
|
440
|
+
'\x1b[34mQueue:\x1b[0m',
|
|
441
|
+
`\x1b[35m'${name}'\x1b[0m`,
|
|
442
|
+
`\x1b[31m${QUEUE_STATUS.retry.failed}\x1b[0m`,
|
|
443
|
+
'job',
|
|
444
|
+
`\x1b[38;5;208m${job.id}\x1b[0m`,
|
|
445
|
+
'(\x1b[33mmax attempts reached\x1b[0m)',
|
|
446
|
+
].join(' '));
|
|
350
447
|
}
|
|
351
448
|
break;
|
|
352
449
|
}
|
|
@@ -354,48 +451,25 @@ class Worker extends Model_1.Model {
|
|
|
354
451
|
}
|
|
355
452
|
}
|
|
356
453
|
}
|
|
357
|
-
finally {
|
|
358
|
-
state.running--;
|
|
359
|
-
this.ACTIVE_JOBS--;
|
|
360
|
-
}
|
|
361
454
|
}
|
|
362
455
|
async _dequeueMany(name, limit) {
|
|
363
456
|
if (this.STOPPING)
|
|
364
457
|
return [];
|
|
458
|
+
const now = this.$utils.timestamp();
|
|
365
459
|
const findJobs = await new Worker()
|
|
366
460
|
.select('id')
|
|
367
461
|
.where('name', name)
|
|
368
462
|
.whereQuery(query => {
|
|
369
463
|
return query
|
|
370
|
-
.where('status', '
|
|
371
|
-
.
|
|
372
|
-
|
|
373
|
-
${(0, Meta_1.Meta)(Worker).columnRef('available_at')},
|
|
374
|
-
INTERVAL (${(0, Meta_1.Meta)(Worker).columnRef('delay_ms')} * 1000) MICROSECOND
|
|
375
|
-
) <= ?`, [`'${this.$utils.timestamp()}'`]);
|
|
376
|
-
})
|
|
377
|
-
.when(DB_1.DB.driver() === 'postgres', (q) => {
|
|
378
|
-
return q.whereRaw(`${(0, Meta_1.Meta)(Worker).columnRef('available_at')}
|
|
379
|
-
-
|
|
380
|
-
(${(0, Meta_1.Meta)(Worker).columnRef('delay_ms')} * INTERVAL '1 millisecond')
|
|
381
|
-
<= ?`, [`'${this.$utils.timestamp()}'`]);
|
|
382
|
-
})
|
|
383
|
-
.when(DB_1.DB.driver() === 'sqlite', (q) => {
|
|
384
|
-
return q.whereRaw(`datetime(
|
|
385
|
-
${(0, Meta_1.Meta)(Worker).columnRef('available_at')},
|
|
386
|
-
'-' || (${(0, Meta_1.Meta)(Worker).columnRef('delay_ms')} / 1000.0) || ' seconds'
|
|
387
|
-
) <= ?`, [`'${this.$utils.timestamp()}'`]);
|
|
388
|
-
})
|
|
389
|
-
.orWhereQuery((q) => {
|
|
390
|
-
return q
|
|
391
|
-
.where('status', '=', 'active')
|
|
392
|
-
.where('locked_at', '<', this.$utils.timestamp(new Date(Date.now() - 60 * 1000)));
|
|
393
|
-
});
|
|
464
|
+
.where('status', 'pending')
|
|
465
|
+
.where('created_at', '<', now)
|
|
466
|
+
.whereNull('locked_at');
|
|
394
467
|
})
|
|
395
468
|
.latest('priority')
|
|
469
|
+
.oldest('delay_ms')
|
|
396
470
|
.oldest('id')
|
|
397
471
|
.limit(limit)
|
|
398
|
-
.
|
|
472
|
+
.findMany();
|
|
399
473
|
if (!findJobs.length) {
|
|
400
474
|
return [];
|
|
401
475
|
}
|
|
@@ -403,11 +477,12 @@ class Worker extends Model_1.Model {
|
|
|
403
477
|
const jobs = await new Worker()
|
|
404
478
|
.whereIn('id', findJobs.map(v => v.id))
|
|
405
479
|
.latest('priority')
|
|
480
|
+
.oldest('delay_ms')
|
|
406
481
|
.oldest('id')
|
|
407
482
|
.limit(limit)
|
|
408
483
|
.forUpdate({ skipLocked: true })
|
|
409
484
|
.bind(trx)
|
|
410
|
-
.
|
|
485
|
+
.findMany();
|
|
411
486
|
if (!jobs.length) {
|
|
412
487
|
return [];
|
|
413
488
|
}
|
|
@@ -426,12 +501,12 @@ class Worker extends Model_1.Model {
|
|
|
426
501
|
id: job.id,
|
|
427
502
|
name: job.name,
|
|
428
503
|
status: job.status,
|
|
429
|
-
payload: this.
|
|
504
|
+
payload: this._safeJsonParse(job.payload),
|
|
430
505
|
__job: job
|
|
431
506
|
}));
|
|
432
507
|
});
|
|
433
508
|
}
|
|
434
|
-
async _flushBuffer(
|
|
509
|
+
async _flushBuffer() {
|
|
435
510
|
if (this.IS_FLUSHING || this.BUFFER.jobs.length === 0)
|
|
436
511
|
return;
|
|
437
512
|
if (this.BUFFER.timeout) {
|
|
@@ -445,21 +520,36 @@ class Worker extends Model_1.Model {
|
|
|
445
520
|
try {
|
|
446
521
|
const jobsToInsert = currentBatch.map(b => b.jobData);
|
|
447
522
|
const insertedJobds = await new Worker()
|
|
448
|
-
.select('id')
|
|
523
|
+
.select('id', 'name')
|
|
449
524
|
.insertMany(jobsToInsert)
|
|
450
525
|
.save();
|
|
451
526
|
if (this.INSPECT_EXEC) {
|
|
452
|
-
const
|
|
453
|
-
const
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
527
|
+
const names = [...new Set(insertedJobds.map(job => job.name))];
|
|
528
|
+
for (const name of names) {
|
|
529
|
+
const ids = insertedJobds.filter(v => v.name === name).map(v => v.id);
|
|
530
|
+
const preview = [
|
|
531
|
+
...ids.slice(0, 3),
|
|
532
|
+
...(ids.length > 3 ? ['...', ...ids.slice(Math.max(3, ids.length - 2))] : [])
|
|
533
|
+
].join(', ');
|
|
534
|
+
if (ids.length === 1) {
|
|
535
|
+
console.log([
|
|
536
|
+
'\x1b[34mQueue:\x1b[0m',
|
|
537
|
+
`\x1b[35m'${name}'\x1b[0m`,
|
|
538
|
+
`\x1b[32m${QUEUE_STATUS.receive}\x1b[0m`,
|
|
539
|
+
'job',
|
|
540
|
+
`\x1b[38;5;208m${ids}\x1b[0m`,
|
|
541
|
+
].join(' '));
|
|
542
|
+
}
|
|
543
|
+
else {
|
|
544
|
+
console.log([
|
|
545
|
+
'\x1b[34mQueue:\x1b[0m',
|
|
546
|
+
`\x1b[35m'${name}'\x1b[0m`,
|
|
547
|
+
`\x1b[32m${QUEUE_STATUS.receive}\x1b[0m`,
|
|
548
|
+
'jobs',
|
|
549
|
+
`[\x1b[38;5;208m${preview}\x1b[0m]`,
|
|
550
|
+
`total=(\x1b[38;5;208m${ids.length}\x1b[0m)`,
|
|
551
|
+
].join(' '));
|
|
552
|
+
}
|
|
463
553
|
}
|
|
464
554
|
}
|
|
465
555
|
for (let i = 0; i < currentBatch.length; i++) {
|
|
@@ -475,11 +565,11 @@ class Worker extends Model_1.Model {
|
|
|
475
565
|
}
|
|
476
566
|
finally {
|
|
477
567
|
if (this.BUFFER.jobs.length) {
|
|
478
|
-
this._flushBuffer(
|
|
568
|
+
this._flushBuffer();
|
|
479
569
|
}
|
|
480
570
|
}
|
|
481
571
|
}
|
|
482
|
-
_wakeWorker(name) {
|
|
572
|
+
async _wakeWorker(name) {
|
|
483
573
|
const state = this.WORKER_STATE.get(name);
|
|
484
574
|
if (!state || !state.sleeping || !state.handler)
|
|
485
575
|
return;
|
|
@@ -487,15 +577,60 @@ class Worker extends Model_1.Model {
|
|
|
487
577
|
state.sleeping = false;
|
|
488
578
|
state.idle = 0;
|
|
489
579
|
if (this.INSPECT_EXEC) {
|
|
490
|
-
console.log(
|
|
580
|
+
console.log([
|
|
581
|
+
'\x1b[34mQueue:\x1b[0m',
|
|
582
|
+
`\x1b[35m'${name}'\x1b[0m`,
|
|
583
|
+
`\x1b[36m${QUEUE_STATUS.wokeUp}\x1b[0m`,
|
|
584
|
+
].join(' '));
|
|
491
585
|
}
|
|
492
586
|
if (isSleeping) {
|
|
493
|
-
this.process(name, state.handler,
|
|
494
|
-
|
|
495
|
-
|
|
587
|
+
await this.process(name, state.handler, state.opts);
|
|
588
|
+
}
|
|
589
|
+
return;
|
|
590
|
+
}
|
|
591
|
+
async _pollWorkerJobs() {
|
|
592
|
+
const jobs = await this._getPendingOrFailedJobs();
|
|
593
|
+
for (const job of jobs) {
|
|
594
|
+
this._wakeWorker(job);
|
|
595
|
+
}
|
|
596
|
+
return;
|
|
597
|
+
}
|
|
598
|
+
async _initializeWorkerJobs() {
|
|
599
|
+
const jobs = await this._getPendingOrFailedJobs();
|
|
600
|
+
await Promise.all(jobs.map(job => this._wakeWorker(job)));
|
|
601
|
+
return;
|
|
602
|
+
}
|
|
603
|
+
async _getPendingOrFailedJobs() {
|
|
604
|
+
const jobsToProcess = await new Worker()
|
|
605
|
+
.select('id')
|
|
606
|
+
.whereIn('status', ['pending', 'failed'])
|
|
607
|
+
.where('available_at', '<=', this.$utils.timestamp())
|
|
608
|
+
.findMany();
|
|
609
|
+
await new Worker()
|
|
610
|
+
.updateMany({
|
|
611
|
+
status: 'pending',
|
|
612
|
+
attempts: 0
|
|
613
|
+
})
|
|
614
|
+
.whereIn('id', jobsToProcess.map(f => f.id))
|
|
615
|
+
.void()
|
|
616
|
+
.save();
|
|
617
|
+
const jobs = await new Worker()
|
|
618
|
+
.select('name')
|
|
619
|
+
.where('status', 'pending')
|
|
620
|
+
.where('available_at', '<=', this.$utils.timestamp())
|
|
621
|
+
.groupBy('name')
|
|
622
|
+
.toArray('name');
|
|
623
|
+
if (this.INSPECT_EXEC) {
|
|
624
|
+
console.log([
|
|
625
|
+
'\x1b[34mQueue:\x1b[0m',
|
|
626
|
+
'\x1b[32mPoll checked\x1b[0m',
|
|
627
|
+
`(${jobs.length} jobs)`,
|
|
628
|
+
].join(' '));
|
|
496
629
|
}
|
|
630
|
+
;
|
|
631
|
+
return jobs;
|
|
497
632
|
}
|
|
498
|
-
|
|
633
|
+
_safeJsonParse(payload) {
|
|
499
634
|
try {
|
|
500
635
|
return JSON.parse(payload);
|
|
501
636
|
}
|
|
@@ -503,7 +638,7 @@ class Worker extends Model_1.Model {
|
|
|
503
638
|
return payload;
|
|
504
639
|
}
|
|
505
640
|
}
|
|
506
|
-
|
|
641
|
+
_safeJsonStringify(payload) {
|
|
507
642
|
if (payload == null)
|
|
508
643
|
return null;
|
|
509
644
|
try {
|
|
@@ -524,203 +659,9 @@ class Worker extends Model_1.Model {
|
|
|
524
659
|
return payload;
|
|
525
660
|
}
|
|
526
661
|
}
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
* Queue facade class (static API wrapper)
|
|
530
|
-
*
|
|
531
|
-
* This class provides a singleton-style interface over the underlying Worker instance.
|
|
532
|
-
* It must be initialized before use via `Queue.start()`.
|
|
533
|
-
*
|
|
534
|
-
* @example
|
|
535
|
-
* ```ts
|
|
536
|
-
* const sendEmail = (job) => console.log('send mail :' + job.payload.email)
|
|
537
|
-
*
|
|
538
|
-
* await Queue.start({ inspect : true, flush : true // **remove all jobs });
|
|
539
|
-
*
|
|
540
|
-
* // register
|
|
541
|
-
* Queue.progress("send-email", async (job) => {
|
|
542
|
-
* return await sendEmail(job);
|
|
543
|
-
* }, { concurrency : 3 });
|
|
544
|
-
*
|
|
545
|
-
* // add
|
|
546
|
-
* Queue.add("send-email", { email: "test@gmail.com" });
|
|
547
|
-
*
|
|
548
|
-
* ```
|
|
549
|
-
*/
|
|
550
|
-
class Queue {
|
|
551
|
-
/**
|
|
552
|
-
* Internal Worker instance used for all queue operations.
|
|
553
|
-
* @type {Worker | null}
|
|
554
|
-
*/
|
|
555
|
-
static WORKER;
|
|
556
|
-
static MESSAGE = {
|
|
557
|
-
INIT_ERROR: `Queue is not initialized. Please call 'await Queue.start()' before using it.`
|
|
558
|
-
};
|
|
559
|
-
/**
|
|
560
|
-
* The 'start' method is used to initialize the Queue system.
|
|
561
|
-
* Creates and prepares the underlying Worker instance.
|
|
562
|
-
* @param {Object} [opts] - options (inspect, flush)
|
|
563
|
-
* @property {boolean} opts.inspect queue work flow
|
|
564
|
-
* @property {boolean} opts.flush remove all queue
|
|
565
|
-
* @property {number} opts.maxIdleRetries - Maximum idle time () when no jobs are available
|
|
566
|
-
* @property {number} opts.limitConnections - Allowed DB connections limit before pausing
|
|
567
|
-
* @returns {Promise<void>}
|
|
568
|
-
*/
|
|
569
|
-
static async start(opts = {}) {
|
|
570
|
-
this.WORKER = await new Worker().initialize(opts);
|
|
571
|
-
return;
|
|
572
|
-
}
|
|
573
|
-
/**
|
|
574
|
-
* The 'end' method is used to shutdown the Queue system.
|
|
575
|
-
*
|
|
576
|
-
* @returns {Promise<void>}
|
|
577
|
-
*/
|
|
578
|
-
static async end() {
|
|
579
|
-
if (this.WORKER == null) {
|
|
580
|
-
throw new Error(this.MESSAGE.INIT_ERROR);
|
|
581
|
-
}
|
|
582
|
-
await this.WORKER.shutdown();
|
|
583
|
-
this.WORKER = null;
|
|
584
|
-
return;
|
|
585
|
-
}
|
|
586
|
-
/**
|
|
587
|
-
* The 'flush' method is used to flush all jobs in the queue (dangerous operation).
|
|
588
|
-
*
|
|
589
|
-
* @throws {Error} If Queue is not initialized.
|
|
590
|
-
* @returns {Promise<void>}
|
|
591
|
-
*/
|
|
592
|
-
static async flush() {
|
|
593
|
-
if (this.WORKER == null) {
|
|
594
|
-
throw new Error(this.MESSAGE.INIT_ERROR);
|
|
595
|
-
}
|
|
596
|
-
await this.WORKER.flush();
|
|
597
|
-
}
|
|
598
|
-
/**
|
|
599
|
-
* The 'getJobOverallStats' method is used to get aggregated queue statistics.
|
|
600
|
-
*
|
|
601
|
-
* @param {string} [name] - Optional queue name filter.
|
|
602
|
-
* @throws {Error} If Queue is not initialized.
|
|
603
|
-
* @returns {Promise<any>}
|
|
604
|
-
*/
|
|
605
|
-
static async getJobOverallStats(name) {
|
|
606
|
-
if (this.WORKER == null) {
|
|
607
|
-
throw new Error(this.MESSAGE.INIT_ERROR);
|
|
608
|
-
}
|
|
609
|
-
return await this.WORKER.getJobOverallStats(name);
|
|
610
|
-
}
|
|
611
|
-
/**
|
|
612
|
-
* The 'getJobStats' method is used to Get jobs statistics grouped by name.
|
|
613
|
-
*
|
|
614
|
-
* @param {string} [name] - Optional queue name filter.
|
|
615
|
-
* @throws {Error} If Queue is not initialized.
|
|
616
|
-
* @returns {Promise<any>}
|
|
617
|
-
*/
|
|
618
|
-
static async getJobStats(name) {
|
|
619
|
-
if (this.WORKER == null) {
|
|
620
|
-
throw new Error(this.MESSAGE.INIT_ERROR);
|
|
621
|
-
}
|
|
622
|
-
return await this.WORKER.getJobStats(name);
|
|
623
|
-
}
|
|
624
|
-
/**
|
|
625
|
-
* The 'getJobs' method is used to Get jobs.
|
|
626
|
-
*
|
|
627
|
-
* @param {string} [name] - Optional queue name filter.
|
|
628
|
-
* @throws {Error} If Queue is not initialized.
|
|
629
|
-
* @returns {Promise<T.Result<Worker>[]>}
|
|
630
|
-
*/
|
|
631
|
-
static async getJobs(name) {
|
|
632
|
-
if (this.WORKER == null) {
|
|
633
|
-
throw new Error(this.MESSAGE.INIT_ERROR);
|
|
634
|
-
}
|
|
635
|
-
return await this.WORKER.getJobs(name);
|
|
636
|
-
}
|
|
637
|
-
/**
|
|
638
|
-
* Get all unique queue names.
|
|
639
|
-
*
|
|
640
|
-
* @throws {Error} If Queue is not initialized.
|
|
641
|
-
* @returns {Promise<string[]>}
|
|
642
|
-
*/
|
|
643
|
-
static async getNames() {
|
|
644
|
-
if (this.WORKER == null) {
|
|
645
|
-
throw new Error(this.MESSAGE.INIT_ERROR);
|
|
646
|
-
}
|
|
647
|
-
return await this.WORKER.getNames();
|
|
648
|
-
}
|
|
649
|
-
/**
|
|
650
|
-
* Access raw Worker instance safely.
|
|
651
|
-
*
|
|
652
|
-
* @param {(worker: Worker) => any} cb - Callback with Worker instance.
|
|
653
|
-
* @throws {Error} If Queue is not initialized.
|
|
654
|
-
* @returns {Promise<Work>}
|
|
655
|
-
*/
|
|
656
|
-
static async worker(cb) {
|
|
657
|
-
if (this.WORKER == null) {
|
|
658
|
-
throw new Error(this.MESSAGE.INIT_ERROR);
|
|
659
|
-
}
|
|
660
|
-
return await cb(this.WORKER);
|
|
661
|
-
}
|
|
662
|
-
/**
|
|
663
|
-
* Start a worker for processing jobs of a specific name.
|
|
664
|
-
*
|
|
665
|
-
* @param {string} name - Queue name to process.
|
|
666
|
-
* @param {Handler} handler - Job handler function.
|
|
667
|
-
* @param {QueueProcessOptions} [opts] - Job options (interval, concurrency)
|
|
668
|
-
* @throws {Error} If Queue is not initialized.
|
|
669
|
-
* @returns {Promise<void>}
|
|
670
|
-
*
|
|
671
|
-
* @example
|
|
672
|
-
* const helloWorld = (job) => console.log('hello world :' + job.id);
|
|
673
|
-
*
|
|
674
|
-
* Queue.progress("hello", async (job) => {
|
|
675
|
-
* return await helloWorld(job)
|
|
676
|
-
* }, { concurrency : 3 });
|
|
677
|
-
*/
|
|
678
|
-
static async process(name, handler, opts = { interval: 1_000, concurrency: 1 }) {
|
|
679
|
-
if (this.WORKER == null) {
|
|
680
|
-
throw new Error(this.MESSAGE.INIT_ERROR);
|
|
681
|
-
}
|
|
682
|
-
return await this.WORKER.process(name, handler, opts);
|
|
683
|
-
}
|
|
684
|
-
/**
|
|
685
|
-
* Start a worker for processing jobs of a specific name.
|
|
686
|
-
*
|
|
687
|
-
* @param {string} name - Queue name to process.
|
|
688
|
-
* @param {Handler} handler - Job handler function.
|
|
689
|
-
* @param {QueueProcessOptions} [opts] - Job options (interval, concurrency)
|
|
690
|
-
* @throws {Error} If Queue is not initialized.
|
|
691
|
-
* @returns {Promise<void>}
|
|
692
|
-
*
|
|
693
|
-
* @example
|
|
694
|
-
* const helloWorld = (job) => console.log('hello world :' + job.id);
|
|
695
|
-
*
|
|
696
|
-
* Queue.on("hello", async (job) => {
|
|
697
|
-
* return await helloWorld(job)
|
|
698
|
-
* }, { concurrency : 3 });
|
|
699
|
-
*/
|
|
700
|
-
static async on(name, handler, opts = { interval: 1_000, concurrency: 1 }) {
|
|
701
|
-
return await this.process(name, handler, opts);
|
|
702
|
-
}
|
|
703
|
-
/**
|
|
704
|
-
* Add a new job into the queue.
|
|
705
|
-
*
|
|
706
|
-
* @param {string} name - Queue name / job type.
|
|
707
|
-
* @param {any} payload - Job payload data.
|
|
708
|
-
* @param {QueueAddOptions} [opts] - Job options (delay, priority, retry, etc.)
|
|
709
|
-
* @throws {Error} If Queue is not initialized.
|
|
710
|
-
* @returns {Promise<T.Result<Worker>>}
|
|
711
|
-
*
|
|
712
|
-
* @example
|
|
713
|
-
* ```ts
|
|
714
|
-
* Queue.add("send-email", { email: "test@gmail.com" });
|
|
715
|
-
* ```
|
|
716
|
-
*/
|
|
717
|
-
static async add(name, payload, opts = {}) {
|
|
718
|
-
if (this.WORKER == null) {
|
|
719
|
-
throw new Error(this.MESSAGE.INIT_ERROR);
|
|
720
|
-
}
|
|
721
|
-
return await this.WORKER.add(name, payload, opts);
|
|
662
|
+
async _sleep(ms) {
|
|
663
|
+
return await new Promise(r => setTimeout(r, ms));
|
|
722
664
|
}
|
|
723
665
|
}
|
|
724
|
-
exports.
|
|
725
|
-
|
|
726
|
-
//# sourceMappingURL=Queue.js.map
|
|
666
|
+
exports.Worker = Worker;
|
|
667
|
+
//# sourceMappingURL=worker.js.map
|