tspace-mysql 1.9.2 → 1.9.3-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +142 -17
- package/dist/lib/core/Abstracts/AbstractBuilder.d.ts +0 -1
- package/dist/lib/core/Abstracts/AbstractBuilder.js.map +1 -1
- package/dist/lib/core/Blueprint.d.ts +43 -10
- package/dist/lib/core/Blueprint.js +43 -2
- package/dist/lib/core/Blueprint.js.map +1 -1
- package/dist/lib/core/Builder.d.ts +5 -5
- package/dist/lib/core/Builder.js +15 -12
- package/dist/lib/core/Builder.js.map +1 -1
- package/dist/lib/core/Cache/MemoryCache.js +9 -4
- package/dist/lib/core/Cache/MemoryCache.js.map +1 -1
- package/dist/lib/core/Contracts/Audit.d.ts +2 -2
- package/dist/lib/core/Contracts/Logger.d.ts +2 -2
- package/dist/lib/core/Model.d.ts +169 -29
- package/dist/lib/core/Model.js +153 -4
- package/dist/lib/core/Model.js.map +1 -1
- package/dist/lib/core/{Queue.d.ts → Queue/index.d.ts} +115 -91
- package/dist/lib/core/Queue/index.js +282 -0
- package/dist/lib/core/Queue/index.js.map +1 -0
- package/dist/lib/core/Queue/types.d.ts +39 -0
- package/dist/lib/core/Queue/types.js +3 -0
- package/dist/lib/core/Queue/types.js.map +1 -0
- package/dist/lib/core/Queue/worker.d.ts +102 -0
- package/dist/lib/core/{Queue.js → Queue/worker.js} +133 -268
- package/dist/lib/core/Queue/worker.js.map +1 -0
- package/dist/lib/core/Repository.d.ts +40 -20
- package/dist/lib/core/Repository.js +35 -14
- package/dist/lib/core/Repository.js.map +1 -1
- package/dist/lib/core/UtilityTypes.d.ts +9 -7
- package/dist/lib/core/UtilityTypes.js.map +1 -1
- package/dist/lib/types/decorator/index.d.ts +3 -0
- package/dist/lib/types/index.d.ts +17 -16
- package/dist/lib/types/repository/index.d.ts +3 -3
- package/dist/lib/utils/index.d.ts +4 -2
- package/dist/lib/utils/index.js +17 -1
- package/dist/lib/utils/index.js.map +1 -1
- package/package.json +1 -1
- package/dist/lib/core/Queue.js.map +0 -1
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
import type { T } from "../UtilityTypes";
|
|
2
|
+
import { Blueprint } from "../Blueprint";
|
|
3
|
+
import { Model } from "../Model";
|
|
4
|
+
import type { Handler, QueueAddOptions, QueueProcessOptions } from "./types";
|
|
5
|
+
declare const schema: {
|
|
6
|
+
id: Blueprint<number>;
|
|
7
|
+
uuid: Blueprint<string | null>;
|
|
8
|
+
name: Blueprint<string>;
|
|
9
|
+
status: Blueprint<T.Default<NonNullable<"pending" | "active" | "completed" | "failed">>>;
|
|
10
|
+
priority: Blueprint<T.Default<number>>;
|
|
11
|
+
payload: Blueprint<string | null>;
|
|
12
|
+
result: Blueprint<string | null>;
|
|
13
|
+
error: Blueprint<string | null>;
|
|
14
|
+
metadata: Blueprint<string | null>;
|
|
15
|
+
attempts: Blueprint<T.Default<number>>;
|
|
16
|
+
max_attempts: Blueprint<T.Default<number>>;
|
|
17
|
+
delay_ms: Blueprint<T.Default<number>>;
|
|
18
|
+
locked_by: Blueprint<string | null>;
|
|
19
|
+
locked_at: Blueprint<T.DateTime | null>;
|
|
20
|
+
available_at: Blueprint<NonNullable<T.DateTime>>;
|
|
21
|
+
completed_at: Blueprint<T.DateTime | null>;
|
|
22
|
+
created_at: Blueprint<T.DateTime | null>;
|
|
23
|
+
updated_at: Blueprint<T.DateTime | null>;
|
|
24
|
+
};
|
|
25
|
+
export declare class Worker extends Model<T.Schema<typeof schema>> {
|
|
26
|
+
private HOSTNAME;
|
|
27
|
+
private INSPECT_EXEC;
|
|
28
|
+
private STOPPING;
|
|
29
|
+
private IS_FLUSHING;
|
|
30
|
+
private ACTIVE_JOBS;
|
|
31
|
+
private MAX_IDLE_RETRIES;
|
|
32
|
+
private BATCH_SIZE;
|
|
33
|
+
private MAX_WAIT_MS;
|
|
34
|
+
private POLL;
|
|
35
|
+
private BUFFER;
|
|
36
|
+
private WORKER_STATE;
|
|
37
|
+
private EVENT_SUBSCRIBERS;
|
|
38
|
+
protected boot(): void;
|
|
39
|
+
initialize(opts?: {
|
|
40
|
+
inspect?: boolean;
|
|
41
|
+
flush?: boolean;
|
|
42
|
+
hostname?: string;
|
|
43
|
+
maxIdleRetries?: number;
|
|
44
|
+
poll?: {
|
|
45
|
+
enabled?: boolean;
|
|
46
|
+
timeout?: number;
|
|
47
|
+
};
|
|
48
|
+
}): Promise<this>;
|
|
49
|
+
shutdown(): Promise<void>;
|
|
50
|
+
flush(): Promise<void>;
|
|
51
|
+
getJobOverallStats(name?: string): Promise<{
|
|
52
|
+
total: number;
|
|
53
|
+
completed: number;
|
|
54
|
+
active: number;
|
|
55
|
+
pending: number;
|
|
56
|
+
failed: number;
|
|
57
|
+
}>;
|
|
58
|
+
getJobStats(name?: string): Promise<{
|
|
59
|
+
name: string;
|
|
60
|
+
completed: number;
|
|
61
|
+
active: number;
|
|
62
|
+
pending: number;
|
|
63
|
+
failed: number;
|
|
64
|
+
}[]>;
|
|
65
|
+
getJobs(name?: string): Promise<{
|
|
66
|
+
[x: string]: unknown;
|
|
67
|
+
id: number;
|
|
68
|
+
uuid: string | null;
|
|
69
|
+
name: string;
|
|
70
|
+
status: "pending" | "active" | "completed" | "failed";
|
|
71
|
+
priority: number;
|
|
72
|
+
payload: string | null;
|
|
73
|
+
result: string | null;
|
|
74
|
+
error: string | null;
|
|
75
|
+
metadata: string | null;
|
|
76
|
+
attempts: number;
|
|
77
|
+
max_attempts: number;
|
|
78
|
+
delay_ms: number;
|
|
79
|
+
locked_by: string | null;
|
|
80
|
+
locked_at: T.DateTime | null;
|
|
81
|
+
available_at: NonNullable<T.DateTime>;
|
|
82
|
+
completed_at: T.DateTime | null;
|
|
83
|
+
created_at: T.DateTime | null;
|
|
84
|
+
updated_at: T.DateTime | null;
|
|
85
|
+
}[]>;
|
|
86
|
+
getNames(): Promise<any[]>;
|
|
87
|
+
add(name: string, payload: any, opts?: QueueAddOptions): Promise<void>;
|
|
88
|
+
process(name: string, handler: Handler, opts?: QueueProcessOptions): Promise<void>;
|
|
89
|
+
publish(event: string, payload: any, opts?: QueueAddOptions): Promise<void>;
|
|
90
|
+
subscribe(event: string, name: string, handler: Handler, opts?: QueueProcessOptions): Promise<void>;
|
|
91
|
+
private _runJob;
|
|
92
|
+
private _dequeueMany;
|
|
93
|
+
private _flushBuffer;
|
|
94
|
+
private _wakeWorker;
|
|
95
|
+
private _pollWorkerJobs;
|
|
96
|
+
private _initializeWorkerJobs;
|
|
97
|
+
private _getPendingOrFailedJobs;
|
|
98
|
+
private _safeJsonParse;
|
|
99
|
+
private _safeJsonStringify;
|
|
100
|
+
private _sleep;
|
|
101
|
+
}
|
|
102
|
+
export {};
|
|
@@ -1,11 +1,10 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.
|
|
4
|
-
const Blueprint_1 = require("
|
|
5
|
-
const DB_1 = require("
|
|
3
|
+
exports.Worker = void 0;
|
|
4
|
+
const Blueprint_1 = require("../Blueprint");
|
|
5
|
+
const DB_1 = require("../DB");
|
|
6
6
|
;
|
|
7
|
-
const Model_1 = require("
|
|
8
|
-
const Meta_1 = require("./Meta");
|
|
7
|
+
const Model_1 = require("../Model");
|
|
9
8
|
const QUEUE_STATUS = {
|
|
10
9
|
dispatch: 'Dispatch',
|
|
11
10
|
receive: 'Receive',
|
|
@@ -49,27 +48,31 @@ const schema = {
|
|
|
49
48
|
max_attempts: Blueprint_1.Blueprint.int().default(3),
|
|
50
49
|
delay_ms: Blueprint_1.Blueprint.int().default(0),
|
|
51
50
|
locked_by: Blueprint_1.Blueprint.text().null(),
|
|
52
|
-
locked_at: Blueprint_1.Blueprint.
|
|
53
|
-
available_at: Blueprint_1.Blueprint.
|
|
54
|
-
completed_at: Blueprint_1.Blueprint.
|
|
55
|
-
created_at: Blueprint_1.Blueprint.
|
|
56
|
-
updated_at: Blueprint_1.Blueprint.
|
|
51
|
+
locked_at: Blueprint_1.Blueprint.datetime().null(),
|
|
52
|
+
available_at: Blueprint_1.Blueprint.datetime().notNull(),
|
|
53
|
+
completed_at: Blueprint_1.Blueprint.datetime().null(),
|
|
54
|
+
created_at: Blueprint_1.Blueprint.datetime().null(),
|
|
55
|
+
updated_at: Blueprint_1.Blueprint.datetime().null()
|
|
57
56
|
};
|
|
58
57
|
class Worker extends Model_1.Model {
|
|
59
58
|
HOSTNAME = String(process.env?.hostname ?? 'unknown');
|
|
60
59
|
INSPECT_EXEC = false;
|
|
61
60
|
STOPPING = false;
|
|
62
61
|
IS_FLUSHING = false;
|
|
63
|
-
LIMIT_CONNECTIONS = 41;
|
|
64
|
-
MAX_IDLE_RETRIES = 15;
|
|
65
62
|
ACTIVE_JOBS = 0;
|
|
63
|
+
MAX_IDLE_RETRIES = 3;
|
|
66
64
|
BATCH_SIZE = 1000;
|
|
67
65
|
MAX_WAIT_MS = 50;
|
|
66
|
+
POLL = {
|
|
67
|
+
interval: null,
|
|
68
|
+
timeout: 1000 * 60
|
|
69
|
+
};
|
|
68
70
|
BUFFER = {
|
|
69
71
|
jobs: [],
|
|
70
72
|
timeout: null
|
|
71
73
|
};
|
|
72
74
|
WORKER_STATE = new Map();
|
|
75
|
+
EVENT_SUBSCRIBERS = new Map();
|
|
73
76
|
boot() {
|
|
74
77
|
this.useUUID();
|
|
75
78
|
this.useTimestamp();
|
|
@@ -82,6 +85,7 @@ class Worker extends Model_1.Model {
|
|
|
82
85
|
throw new Error('Queue is not supported for MongoDB. Use a different driver or disable queue features.');
|
|
83
86
|
}
|
|
84
87
|
await this.sync({ force: true, index: true }).catch(() => null);
|
|
88
|
+
await this._initializeWorkerJobs();
|
|
85
89
|
if (opts.inspect) {
|
|
86
90
|
this.INSPECT_EXEC = true;
|
|
87
91
|
console.log(`\x1b[34mQueue:\x1b[0m \x1b[32mJob processing started\x1b[0m`);
|
|
@@ -95,31 +99,37 @@ class Worker extends Model_1.Model {
|
|
|
95
99
|
if (opts.maxIdleRetries) {
|
|
96
100
|
this.MAX_IDLE_RETRIES = opts.maxIdleRetries;
|
|
97
101
|
}
|
|
98
|
-
if (opts.
|
|
99
|
-
this.
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
this.LIMIT_CONNECTIONS = maxConnections
|
|
104
|
-
? Math.max(10, Math.floor(maxConnections / 3))
|
|
105
|
-
: this.LIMIT_CONNECTIONS;
|
|
102
|
+
if (opts.poll?.enabled) {
|
|
103
|
+
this.POLL.timeout = opts.poll?.timeout ?? this.POLL.timeout;
|
|
104
|
+
this.POLL.interval = setInterval(() => {
|
|
105
|
+
this._pollWorkerJobs();
|
|
106
|
+
}, this.POLL.timeout);
|
|
106
107
|
}
|
|
107
108
|
return this;
|
|
108
109
|
}
|
|
109
110
|
async shutdown() {
|
|
110
|
-
this.STOPPING = true;
|
|
111
111
|
while (this.ACTIVE_JOBS > 0) {
|
|
112
112
|
if (this.INSPECT_EXEC) {
|
|
113
113
|
console.log(`\x1b[34mQueue:\x1b[0m Currently processing ${this.ACTIVE_JOBS} job(s)`);
|
|
114
114
|
}
|
|
115
|
-
await
|
|
115
|
+
await this._sleep(200);
|
|
116
|
+
}
|
|
117
|
+
await this._sleep(2000);
|
|
118
|
+
if (this.POLL.interval) {
|
|
119
|
+
clearInterval(this.POLL.timeout);
|
|
120
|
+
this.POLL.interval = null;
|
|
116
121
|
}
|
|
122
|
+
this.STOPPING = true;
|
|
117
123
|
if (this.INSPECT_EXEC) {
|
|
118
|
-
console.log("\x1b[34mQueue:\x1b[0m \x1b[
|
|
124
|
+
console.log("\x1b[34mQueue:\x1b[0m \x1b[31mJob processing stopped\x1b[0m");
|
|
119
125
|
}
|
|
120
126
|
}
|
|
121
127
|
async flush() {
|
|
128
|
+
const jobs = await new Worker().count('id');
|
|
122
129
|
await this.truncate({ force: true });
|
|
130
|
+
if (this.INSPECT_EXEC) {
|
|
131
|
+
console.log(`\x1b[34mQueue:\x1b[0m \x1b[31mFlush all jobs (${jobs} jobs)\x1b[0m`);
|
|
132
|
+
}
|
|
123
133
|
return;
|
|
124
134
|
}
|
|
125
135
|
async getJobOverallStats(name) {
|
|
@@ -148,7 +158,7 @@ class Worker extends Model_1.Model {
|
|
|
148
158
|
.when(name, (q) => q.where('name', 'LIKE', `%${DB_1.DB.escape(name)}%`))
|
|
149
159
|
.groupBy('name', 'status')
|
|
150
160
|
.orderBy('name')
|
|
151
|
-
.
|
|
161
|
+
.findMany();
|
|
152
162
|
const map = new Map();
|
|
153
163
|
for (const row of rows) {
|
|
154
164
|
const name = row.name;
|
|
@@ -178,7 +188,7 @@ class Worker extends Model_1.Model {
|
|
|
178
188
|
async getJobs(name) {
|
|
179
189
|
const jobs = await new Worker()
|
|
180
190
|
.when(name != null, (q) => q.where('name', 'LIKE', `%${DB_1.DB.escape(name)}%`))
|
|
181
|
-
.
|
|
191
|
+
.findMany();
|
|
182
192
|
return jobs;
|
|
183
193
|
}
|
|
184
194
|
async getNames() {
|
|
@@ -188,14 +198,16 @@ class Worker extends Model_1.Model {
|
|
|
188
198
|
return new Promise((resolve, reject) => {
|
|
189
199
|
const jobData = {
|
|
190
200
|
name,
|
|
191
|
-
payload: payload == null ? null : this.
|
|
201
|
+
payload: payload == null ? null : this._safeJsonStringify(payload),
|
|
192
202
|
status: 'pending',
|
|
193
|
-
delay_ms: opts.delayMs ?? 0,
|
|
194
|
-
available_at: opts.delayMs ? new Date(Date.now() + opts.delayMs) : new Date(),
|
|
195
203
|
priority: opts.priority ?? 0,
|
|
196
204
|
attempts: 0,
|
|
197
205
|
max_attempts: opts.maxAttempts ?? 3,
|
|
198
|
-
metadata: opts.metadata ? this.
|
|
206
|
+
metadata: opts.metadata ? this._safeJsonStringify(opts.metadata) : null,
|
|
207
|
+
delay_ms: opts.delayMs ?? 0,
|
|
208
|
+
available_at: opts.delayMs ? new Date(Date.now() + opts.delayMs) : new Date(),
|
|
209
|
+
created_at: new Date(),
|
|
210
|
+
updated_at: new Date()
|
|
199
211
|
};
|
|
200
212
|
this.BUFFER.jobs.push({ jobData, resolve, reject });
|
|
201
213
|
if (this.BUFFER.jobs.length >= this.BATCH_SIZE) {
|
|
@@ -230,9 +242,10 @@ class Worker extends Model_1.Model {
|
|
|
230
242
|
if (!state)
|
|
231
243
|
return;
|
|
232
244
|
if (state.running >= state.opts.concurrency) {
|
|
233
|
-
const jitter = Math.random() *
|
|
245
|
+
const jitter = Math.floor(Math.random() * 2000) + 500;
|
|
246
|
+
const delayMs = (state.opts.interval ?? 0) + jitter;
|
|
234
247
|
state.running--;
|
|
235
|
-
setTimeout(dispatch,
|
|
248
|
+
setTimeout(dispatch, delayMs);
|
|
236
249
|
return;
|
|
237
250
|
}
|
|
238
251
|
const capacity = state.opts.concurrency - state.running;
|
|
@@ -246,9 +259,9 @@ class Worker extends Model_1.Model {
|
|
|
246
259
|
}
|
|
247
260
|
return;
|
|
248
261
|
}
|
|
249
|
-
const
|
|
250
|
-
const
|
|
251
|
-
setTimeout(dispatch,
|
|
262
|
+
const jitter = Math.floor(Math.random() * 4000) + 1000;
|
|
263
|
+
const delayMs = (opts.interval ?? 0) + jitter;
|
|
264
|
+
setTimeout(dispatch, delayMs);
|
|
252
265
|
return;
|
|
253
266
|
}
|
|
254
267
|
state.idle = 0;
|
|
@@ -258,6 +271,26 @@ class Worker extends Model_1.Model {
|
|
|
258
271
|
};
|
|
259
272
|
return dispatch();
|
|
260
273
|
}
|
|
274
|
+
async publish(event, payload, opts = {}) {
|
|
275
|
+
const subscribers = this.EVENT_SUBSCRIBERS.get(event);
|
|
276
|
+
if (!subscribers || subscribers.size === 0) {
|
|
277
|
+
return;
|
|
278
|
+
}
|
|
279
|
+
await Promise.all([...subscribers].map(name => this.add(name, payload, opts)));
|
|
280
|
+
}
|
|
281
|
+
async subscribe(event, name, handler, opts = {
|
|
282
|
+
interval: 1_000,
|
|
283
|
+
concurrency: 1
|
|
284
|
+
}) {
|
|
285
|
+
const jobName = `${event}.${name}`;
|
|
286
|
+
if (!this.EVENT_SUBSCRIBERS.has(event)) {
|
|
287
|
+
this.EVENT_SUBSCRIBERS.set(event, new Set());
|
|
288
|
+
}
|
|
289
|
+
this.EVENT_SUBSCRIBERS
|
|
290
|
+
.get(event)
|
|
291
|
+
.add(jobName);
|
|
292
|
+
await this.process(jobName, handler, opts);
|
|
293
|
+
}
|
|
261
294
|
async _runJob(name, job, state) {
|
|
262
295
|
state.running++;
|
|
263
296
|
this.ACTIVE_JOBS++;
|
|
@@ -268,15 +301,21 @@ class Worker extends Model_1.Model {
|
|
|
268
301
|
}
|
|
269
302
|
const startTime = +new Date();
|
|
270
303
|
if (job.__job.delay_ms) {
|
|
271
|
-
await
|
|
304
|
+
await this._sleep(job.__job.delay_ms);
|
|
272
305
|
}
|
|
273
|
-
const result = await handler(
|
|
306
|
+
const result = await handler({
|
|
307
|
+
id: job.id,
|
|
308
|
+
name: job.name,
|
|
309
|
+
status: job.status,
|
|
310
|
+
payload: job.payload
|
|
311
|
+
});
|
|
274
312
|
await new Worker()
|
|
275
313
|
.where('id', job.id)
|
|
276
314
|
.update({
|
|
277
315
|
status: 'completed',
|
|
278
|
-
result: this.
|
|
279
|
-
completed_at: this.$utils.timestamp()
|
|
316
|
+
result: this._safeJsonStringify(result),
|
|
317
|
+
completed_at: this.$utils.timestamp(),
|
|
318
|
+
attempts: 0
|
|
280
319
|
})
|
|
281
320
|
.void()
|
|
282
321
|
.save();
|
|
@@ -293,7 +332,7 @@ class Worker extends Model_1.Model {
|
|
|
293
332
|
.where('id', job.id)
|
|
294
333
|
.update({
|
|
295
334
|
status: 'failed',
|
|
296
|
-
error: this.
|
|
335
|
+
error: this._safeJsonStringify({
|
|
297
336
|
message: err.message,
|
|
298
337
|
name: err.name,
|
|
299
338
|
stack: err.stack,
|
|
@@ -315,8 +354,8 @@ class Worker extends Model_1.Model {
|
|
|
315
354
|
.update({
|
|
316
355
|
status: 'completed',
|
|
317
356
|
attempts,
|
|
318
|
-
result: this.
|
|
319
|
-
completed_at: this.$utils.timestamp()
|
|
357
|
+
result: this._safeJsonStringify(result),
|
|
358
|
+
completed_at: this.$utils.timestamp(),
|
|
320
359
|
})
|
|
321
360
|
.void()
|
|
322
361
|
.save();
|
|
@@ -335,7 +374,7 @@ class Worker extends Model_1.Model {
|
|
|
335
374
|
.update({
|
|
336
375
|
status: 'failed',
|
|
337
376
|
attempts,
|
|
338
|
-
error: this.
|
|
377
|
+
error: this._safeJsonStringify({
|
|
339
378
|
retry: true,
|
|
340
379
|
message: err?.message,
|
|
341
380
|
name: err?.name,
|
|
@@ -367,25 +406,8 @@ class Worker extends Model_1.Model {
|
|
|
367
406
|
.where('name', name)
|
|
368
407
|
.whereQuery(query => {
|
|
369
408
|
return query
|
|
370
|
-
.
|
|
371
|
-
.
|
|
372
|
-
return q.whereRaw(`DATE_SUB(
|
|
373
|
-
${(0, Meta_1.Meta)(Worker).columnRef('available_at')},
|
|
374
|
-
INTERVAL (${(0, Meta_1.Meta)(Worker).columnRef('delay_ms')} * 1000) MICROSECOND
|
|
375
|
-
) <= ?`, [`'${this.$utils.timestamp()}'`]);
|
|
376
|
-
})
|
|
377
|
-
.when(DB_1.DB.driver() === 'postgres', (q) => {
|
|
378
|
-
return q.whereRaw(`${(0, Meta_1.Meta)(Worker).columnRef('available_at')}
|
|
379
|
-
-
|
|
380
|
-
(${(0, Meta_1.Meta)(Worker).columnRef('delay_ms')} * INTERVAL '1 millisecond')
|
|
381
|
-
<= ?`, [`'${this.$utils.timestamp()}'`]);
|
|
382
|
-
})
|
|
383
|
-
.when(DB_1.DB.driver() === 'sqlite', (q) => {
|
|
384
|
-
return q.whereRaw(`datetime(
|
|
385
|
-
${(0, Meta_1.Meta)(Worker).columnRef('available_at')},
|
|
386
|
-
'-' || (${(0, Meta_1.Meta)(Worker).columnRef('delay_ms')} / 1000.0) || ' seconds'
|
|
387
|
-
) <= ?`, [`'${this.$utils.timestamp()}'`]);
|
|
388
|
-
})
|
|
409
|
+
.whereIn('status', ['pending'])
|
|
410
|
+
.where('created_at', '<=', this.$utils.timestamp())
|
|
389
411
|
.orWhereQuery((q) => {
|
|
390
412
|
return q
|
|
391
413
|
.where('status', '=', 'active')
|
|
@@ -395,7 +417,7 @@ class Worker extends Model_1.Model {
|
|
|
395
417
|
.latest('priority')
|
|
396
418
|
.oldest('id')
|
|
397
419
|
.limit(limit)
|
|
398
|
-
.
|
|
420
|
+
.findMany();
|
|
399
421
|
if (!findJobs.length) {
|
|
400
422
|
return [];
|
|
401
423
|
}
|
|
@@ -407,7 +429,7 @@ class Worker extends Model_1.Model {
|
|
|
407
429
|
.limit(limit)
|
|
408
430
|
.forUpdate({ skipLocked: true })
|
|
409
431
|
.bind(trx)
|
|
410
|
-
.
|
|
432
|
+
.findMany();
|
|
411
433
|
if (!jobs.length) {
|
|
412
434
|
return [];
|
|
413
435
|
}
|
|
@@ -426,7 +448,7 @@ class Worker extends Model_1.Model {
|
|
|
426
448
|
id: job.id,
|
|
427
449
|
name: job.name,
|
|
428
450
|
status: job.status,
|
|
429
|
-
payload: this.
|
|
451
|
+
payload: this._safeJsonParse(job.payload),
|
|
430
452
|
__job: job
|
|
431
453
|
}));
|
|
432
454
|
});
|
|
@@ -479,7 +501,7 @@ class Worker extends Model_1.Model {
|
|
|
479
501
|
}
|
|
480
502
|
}
|
|
481
503
|
}
|
|
482
|
-
_wakeWorker(name) {
|
|
504
|
+
async _wakeWorker(name) {
|
|
483
505
|
const state = this.WORKER_STATE.get(name);
|
|
484
506
|
if (!state || !state.sleeping || !state.handler)
|
|
485
507
|
return;
|
|
@@ -490,12 +512,49 @@ class Worker extends Model_1.Model {
|
|
|
490
512
|
console.log(`\x1b[34mQueue:\x1b[0m \x1b[35m'${name}'\x1b[0m \x1b[36m${QUEUE_STATUS.wokeUp}\x1b[0m`);
|
|
491
513
|
}
|
|
492
514
|
if (isSleeping) {
|
|
493
|
-
this.process(name, state.handler,
|
|
494
|
-
|
|
495
|
-
|
|
515
|
+
await this.process(name, state.handler, state.opts);
|
|
516
|
+
}
|
|
517
|
+
return;
|
|
518
|
+
}
|
|
519
|
+
async _pollWorkerJobs() {
|
|
520
|
+
const jobs = await this._getPendingOrFailedJobs();
|
|
521
|
+
for (const job of jobs) {
|
|
522
|
+
this._wakeWorker(job);
|
|
523
|
+
}
|
|
524
|
+
return;
|
|
525
|
+
}
|
|
526
|
+
async _initializeWorkerJobs() {
|
|
527
|
+
const jobs = await this._getPendingOrFailedJobs();
|
|
528
|
+
await Promise.all(jobs.map(job => this._wakeWorker(job)));
|
|
529
|
+
return;
|
|
530
|
+
}
|
|
531
|
+
async _getPendingOrFailedJobs() {
|
|
532
|
+
const jobsToProcess = await new Worker()
|
|
533
|
+
.select('id')
|
|
534
|
+
.whereIn('status', ['pending', 'failed'])
|
|
535
|
+
.where('available_at', '<=', this.$utils.timestamp())
|
|
536
|
+
.findMany();
|
|
537
|
+
await new Worker()
|
|
538
|
+
.updateMany({
|
|
539
|
+
status: 'pending',
|
|
540
|
+
attempts: 0
|
|
541
|
+
})
|
|
542
|
+
.whereIn('id', jobsToProcess.map(f => f.id))
|
|
543
|
+
.void()
|
|
544
|
+
.save();
|
|
545
|
+
const jobs = await new Worker()
|
|
546
|
+
.select('name')
|
|
547
|
+
.where('status', 'pending')
|
|
548
|
+
.where('available_at', '<=', this.$utils.timestamp())
|
|
549
|
+
.groupBy('name')
|
|
550
|
+
.toArray('name');
|
|
551
|
+
if (this.INSPECT_EXEC) {
|
|
552
|
+
console.log(`\x1b[34mQueue:\x1b[0m \x1b[32mPoll checked\x1b[0m (${jobs.length} jobs)`);
|
|
496
553
|
}
|
|
554
|
+
;
|
|
555
|
+
return jobs;
|
|
497
556
|
}
|
|
498
|
-
|
|
557
|
+
_safeJsonParse(payload) {
|
|
499
558
|
try {
|
|
500
559
|
return JSON.parse(payload);
|
|
501
560
|
}
|
|
@@ -503,7 +562,7 @@ class Worker extends Model_1.Model {
|
|
|
503
562
|
return payload;
|
|
504
563
|
}
|
|
505
564
|
}
|
|
506
|
-
|
|
565
|
+
_safeJsonStringify(payload) {
|
|
507
566
|
if (payload == null)
|
|
508
567
|
return null;
|
|
509
568
|
try {
|
|
@@ -524,203 +583,9 @@ class Worker extends Model_1.Model {
|
|
|
524
583
|
return payload;
|
|
525
584
|
}
|
|
526
585
|
}
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
* Queue facade class (static API wrapper)
|
|
530
|
-
*
|
|
531
|
-
* This class provides a singleton-style interface over the underlying Worker instance.
|
|
532
|
-
* It must be initialized before use via `Queue.start()`.
|
|
533
|
-
*
|
|
534
|
-
* @example
|
|
535
|
-
* ```ts
|
|
536
|
-
* const sendEmail = (job) => console.log('send mail :' + job.payload.email)
|
|
537
|
-
*
|
|
538
|
-
* await Queue.start({ inspect : true, flush : true // **remove all jobs });
|
|
539
|
-
*
|
|
540
|
-
* // register
|
|
541
|
-
* Queue.progress("send-email", async (job) => {
|
|
542
|
-
* return await sendEmail(job);
|
|
543
|
-
* }, { concurrency : 3 });
|
|
544
|
-
*
|
|
545
|
-
* // add
|
|
546
|
-
* Queue.add("send-email", { email: "test@gmail.com" });
|
|
547
|
-
*
|
|
548
|
-
* ```
|
|
549
|
-
*/
|
|
550
|
-
class Queue {
|
|
551
|
-
/**
|
|
552
|
-
* Internal Worker instance used for all queue operations.
|
|
553
|
-
* @type {Worker | null}
|
|
554
|
-
*/
|
|
555
|
-
static WORKER;
|
|
556
|
-
static MESSAGE = {
|
|
557
|
-
INIT_ERROR: `Queue is not initialized. Please call 'await Queue.start()' before using it.`
|
|
558
|
-
};
|
|
559
|
-
/**
|
|
560
|
-
* The 'start' method is used to initialize the Queue system.
|
|
561
|
-
* Creates and prepares the underlying Worker instance.
|
|
562
|
-
* @param {Object} [opts] - options (inspect, flush)
|
|
563
|
-
* @property {boolean} opts.inspect queue work flow
|
|
564
|
-
* @property {boolean} opts.flush remove all queue
|
|
565
|
-
* @property {number} opts.maxIdleRetries - Maximum idle time () when no jobs are available
|
|
566
|
-
* @property {number} opts.limitConnections - Allowed DB connections limit before pausing
|
|
567
|
-
* @returns {Promise<void>}
|
|
568
|
-
*/
|
|
569
|
-
static async start(opts = {}) {
|
|
570
|
-
this.WORKER = await new Worker().initialize(opts);
|
|
571
|
-
return;
|
|
572
|
-
}
|
|
573
|
-
/**
|
|
574
|
-
* The 'end' method is used to shutdown the Queue system.
|
|
575
|
-
*
|
|
576
|
-
* @returns {Promise<void>}
|
|
577
|
-
*/
|
|
578
|
-
static async end() {
|
|
579
|
-
if (this.WORKER == null) {
|
|
580
|
-
throw new Error(this.MESSAGE.INIT_ERROR);
|
|
581
|
-
}
|
|
582
|
-
await this.WORKER.shutdown();
|
|
583
|
-
this.WORKER = null;
|
|
584
|
-
return;
|
|
585
|
-
}
|
|
586
|
-
/**
|
|
587
|
-
* The 'flush' method is used to flush all jobs in the queue (dangerous operation).
|
|
588
|
-
*
|
|
589
|
-
* @throws {Error} If Queue is not initialized.
|
|
590
|
-
* @returns {Promise<void>}
|
|
591
|
-
*/
|
|
592
|
-
static async flush() {
|
|
593
|
-
if (this.WORKER == null) {
|
|
594
|
-
throw new Error(this.MESSAGE.INIT_ERROR);
|
|
595
|
-
}
|
|
596
|
-
await this.WORKER.flush();
|
|
597
|
-
}
|
|
598
|
-
/**
|
|
599
|
-
* The 'getJobOverallStats' method is used to get aggregated queue statistics.
|
|
600
|
-
*
|
|
601
|
-
* @param {string} [name] - Optional queue name filter.
|
|
602
|
-
* @throws {Error} If Queue is not initialized.
|
|
603
|
-
* @returns {Promise<any>}
|
|
604
|
-
*/
|
|
605
|
-
static async getJobOverallStats(name) {
|
|
606
|
-
if (this.WORKER == null) {
|
|
607
|
-
throw new Error(this.MESSAGE.INIT_ERROR);
|
|
608
|
-
}
|
|
609
|
-
return await this.WORKER.getJobOverallStats(name);
|
|
610
|
-
}
|
|
611
|
-
/**
|
|
612
|
-
* The 'getJobStats' method is used to Get jobs statistics grouped by name.
|
|
613
|
-
*
|
|
614
|
-
* @param {string} [name] - Optional queue name filter.
|
|
615
|
-
* @throws {Error} If Queue is not initialized.
|
|
616
|
-
* @returns {Promise<any>}
|
|
617
|
-
*/
|
|
618
|
-
static async getJobStats(name) {
|
|
619
|
-
if (this.WORKER == null) {
|
|
620
|
-
throw new Error(this.MESSAGE.INIT_ERROR);
|
|
621
|
-
}
|
|
622
|
-
return await this.WORKER.getJobStats(name);
|
|
623
|
-
}
|
|
624
|
-
/**
|
|
625
|
-
* The 'getJobs' method is used to Get jobs.
|
|
626
|
-
*
|
|
627
|
-
* @param {string} [name] - Optional queue name filter.
|
|
628
|
-
* @throws {Error} If Queue is not initialized.
|
|
629
|
-
* @returns {Promise<T.Result<Worker>[]>}
|
|
630
|
-
*/
|
|
631
|
-
static async getJobs(name) {
|
|
632
|
-
if (this.WORKER == null) {
|
|
633
|
-
throw new Error(this.MESSAGE.INIT_ERROR);
|
|
634
|
-
}
|
|
635
|
-
return await this.WORKER.getJobs(name);
|
|
636
|
-
}
|
|
637
|
-
/**
|
|
638
|
-
* Get all unique queue names.
|
|
639
|
-
*
|
|
640
|
-
* @throws {Error} If Queue is not initialized.
|
|
641
|
-
* @returns {Promise<string[]>}
|
|
642
|
-
*/
|
|
643
|
-
static async getNames() {
|
|
644
|
-
if (this.WORKER == null) {
|
|
645
|
-
throw new Error(this.MESSAGE.INIT_ERROR);
|
|
646
|
-
}
|
|
647
|
-
return await this.WORKER.getNames();
|
|
648
|
-
}
|
|
649
|
-
/**
|
|
650
|
-
* Access raw Worker instance safely.
|
|
651
|
-
*
|
|
652
|
-
* @param {(worker: Worker) => any} cb - Callback with Worker instance.
|
|
653
|
-
* @throws {Error} If Queue is not initialized.
|
|
654
|
-
* @returns {Promise<Work>}
|
|
655
|
-
*/
|
|
656
|
-
static async worker(cb) {
|
|
657
|
-
if (this.WORKER == null) {
|
|
658
|
-
throw new Error(this.MESSAGE.INIT_ERROR);
|
|
659
|
-
}
|
|
660
|
-
return await cb(this.WORKER);
|
|
661
|
-
}
|
|
662
|
-
/**
|
|
663
|
-
* Start a worker for processing jobs of a specific name.
|
|
664
|
-
*
|
|
665
|
-
* @param {string} name - Queue name to process.
|
|
666
|
-
* @param {Handler} handler - Job handler function.
|
|
667
|
-
* @param {QueueProcessOptions} [opts] - Job options (interval, concurrency)
|
|
668
|
-
* @throws {Error} If Queue is not initialized.
|
|
669
|
-
* @returns {Promise<void>}
|
|
670
|
-
*
|
|
671
|
-
* @example
|
|
672
|
-
* const helloWorld = (job) => console.log('hello world :' + job.id);
|
|
673
|
-
*
|
|
674
|
-
* Queue.progress("hello", async (job) => {
|
|
675
|
-
* return await helloWorld(job)
|
|
676
|
-
* }, { concurrency : 3 });
|
|
677
|
-
*/
|
|
678
|
-
static async process(name, handler, opts = { interval: 1_000, concurrency: 1 }) {
|
|
679
|
-
if (this.WORKER == null) {
|
|
680
|
-
throw new Error(this.MESSAGE.INIT_ERROR);
|
|
681
|
-
}
|
|
682
|
-
return await this.WORKER.process(name, handler, opts);
|
|
683
|
-
}
|
|
684
|
-
/**
|
|
685
|
-
* Start a worker for processing jobs of a specific name.
|
|
686
|
-
*
|
|
687
|
-
* @param {string} name - Queue name to process.
|
|
688
|
-
* @param {Handler} handler - Job handler function.
|
|
689
|
-
* @param {QueueProcessOptions} [opts] - Job options (interval, concurrency)
|
|
690
|
-
* @throws {Error} If Queue is not initialized.
|
|
691
|
-
* @returns {Promise<void>}
|
|
692
|
-
*
|
|
693
|
-
* @example
|
|
694
|
-
* const helloWorld = (job) => console.log('hello world :' + job.id);
|
|
695
|
-
*
|
|
696
|
-
* Queue.on("hello", async (job) => {
|
|
697
|
-
* return await helloWorld(job)
|
|
698
|
-
* }, { concurrency : 3 });
|
|
699
|
-
*/
|
|
700
|
-
static async on(name, handler, opts = { interval: 1_000, concurrency: 1 }) {
|
|
701
|
-
return await this.process(name, handler, opts);
|
|
702
|
-
}
|
|
703
|
-
/**
|
|
704
|
-
* Add a new job into the queue.
|
|
705
|
-
*
|
|
706
|
-
* @param {string} name - Queue name / job type.
|
|
707
|
-
* @param {any} payload - Job payload data.
|
|
708
|
-
* @param {QueueAddOptions} [opts] - Job options (delay, priority, retry, etc.)
|
|
709
|
-
* @throws {Error} If Queue is not initialized.
|
|
710
|
-
* @returns {Promise<T.Result<Worker>>}
|
|
711
|
-
*
|
|
712
|
-
* @example
|
|
713
|
-
* ```ts
|
|
714
|
-
* Queue.add("send-email", { email: "test@gmail.com" });
|
|
715
|
-
* ```
|
|
716
|
-
*/
|
|
717
|
-
static async add(name, payload, opts = {}) {
|
|
718
|
-
if (this.WORKER == null) {
|
|
719
|
-
throw new Error(this.MESSAGE.INIT_ERROR);
|
|
720
|
-
}
|
|
721
|
-
return await this.WORKER.add(name, payload, opts);
|
|
586
|
+
async _sleep(ms) {
|
|
587
|
+
return await new Promise(r => setTimeout(r, ms));
|
|
722
588
|
}
|
|
723
589
|
}
|
|
724
|
-
exports.
|
|
725
|
-
|
|
726
|
-
//# sourceMappingURL=Queue.js.map
|
|
590
|
+
exports.Worker = Worker;
|
|
591
|
+
//# sourceMappingURL=worker.js.map
|