tspace-mysql 1.9.0 → 1.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4357 -133
- package/dist/lib/config/index.d.ts +2 -2
- package/dist/lib/config/index.js +24 -16
- package/dist/lib/config/index.js.map +1 -1
- package/dist/lib/constants/index.d.ts +10 -1
- package/dist/lib/constants/index.js +11 -2
- package/dist/lib/constants/index.js.map +1 -1
- package/dist/lib/core/Abstracts/AbstractBuilder.d.ts +11 -7
- package/dist/lib/core/Abstracts/AbstractBuilder.js +11 -13
- package/dist/lib/core/Abstracts/AbstractBuilder.js.map +1 -1
- package/dist/lib/core/Abstracts/AbstractDB.d.ts +4 -1
- package/dist/lib/core/Abstracts/AbstractDB.js.map +1 -1
- package/dist/lib/core/Abstracts/AbstractView.js.map +1 -1
- package/dist/lib/core/Blueprint.d.ts +30 -10
- package/dist/lib/core/Blueprint.js +39 -14
- package/dist/lib/core/Blueprint.js.map +1 -1
- package/dist/lib/core/Builder.d.ts +263 -9
- package/dist/lib/core/Builder.js +610 -347
- package/dist/lib/core/Builder.js.map +1 -1
- package/dist/lib/core/Cache/DBCache.js +19 -20
- package/dist/lib/core/Cache/DBCache.js.map +1 -1
- package/dist/lib/core/Cache/MemoryCache.js +3 -3
- package/dist/lib/core/Cache/MemoryCache.js.map +1 -1
- package/dist/lib/core/Cache/RedisCache.js +27 -17
- package/dist/lib/core/Cache/RedisCache.js.map +1 -1
- package/dist/lib/core/Cache/index.js +5 -4
- package/dist/lib/core/Cache/index.js.map +1 -1
- package/dist/lib/core/Contracts/AlterTable.d.ts +152 -0
- package/dist/lib/core/Contracts/AlterTable.js +243 -0
- package/dist/lib/core/Contracts/AlterTable.js.map +1 -0
- package/dist/lib/core/Contracts/Audit.js +2 -5
- package/dist/lib/core/Contracts/Audit.js.map +1 -1
- package/dist/lib/core/DB.d.ts +101 -9
- package/dist/lib/core/DB.js +168 -29
- package/dist/lib/core/DB.js.map +1 -1
- package/dist/lib/core/Decorator.d.ts +144 -16
- package/dist/lib/core/Decorator.js +156 -14
- package/dist/lib/core/Decorator.js.map +1 -1
- package/dist/lib/core/Driver/index.d.ts +43 -9
- package/dist/lib/core/Driver/index.js +9 -7
- package/dist/lib/core/Driver/index.js.map +1 -1
- package/dist/lib/core/Driver/mariadb/MariadbDriver.js +74 -27
- package/dist/lib/core/Driver/mariadb/MariadbDriver.js.map +1 -1
- package/dist/lib/core/Driver/mariadb/MariadbQueryBuilder.d.ts +45 -9
- package/dist/lib/core/Driver/mariadb/MariadbQueryBuilder.js +165 -21
- package/dist/lib/core/Driver/mariadb/MariadbQueryBuilder.js.map +1 -1
- package/dist/lib/core/Driver/mongodb/MongodbDriver.d.ts +24 -0
- package/dist/lib/core/Driver/mongodb/MongodbDriver.js +255 -0
- package/dist/lib/core/Driver/mongodb/MongodbDriver.js.map +1 -0
- package/dist/lib/core/Driver/mongodb/MongodbQueryBuilder.d.ts +141 -0
- package/dist/lib/core/Driver/mongodb/MongodbQueryBuilder.js +563 -0
- package/dist/lib/core/Driver/mongodb/MongodbQueryBuilder.js.map +1 -0
- package/dist/lib/core/Driver/mysql/MysqlDriver.js +105 -95
- package/dist/lib/core/Driver/mysql/MysqlDriver.js.map +1 -1
- package/dist/lib/core/Driver/mysql/MysqlQueryBuilder.d.ts +45 -9
- package/dist/lib/core/Driver/mysql/MysqlQueryBuilder.js +165 -21
- package/dist/lib/core/Driver/mysql/MysqlQueryBuilder.js.map +1 -1
- package/dist/lib/core/Driver/postgres/PostgresDriver.js +97 -72
- package/dist/lib/core/Driver/postgres/PostgresDriver.js.map +1 -1
- package/dist/lib/core/Driver/postgres/PostgresQueryBuilder.d.ts +50 -9
- package/dist/lib/core/Driver/postgres/PostgresQueryBuilder.js +296 -41
- package/dist/lib/core/Driver/postgres/PostgresQueryBuilder.js.map +1 -1
- package/dist/lib/core/Driver/sqlite/SqliteDriver.d.ts +20 -0
- package/dist/lib/core/Driver/sqlite/SqliteDriver.js +192 -0
- package/dist/lib/core/Driver/sqlite/SqliteDriver.js.map +1 -0
- package/dist/lib/core/Driver/sqlite/SqliteQueryBuilder.d.ts +144 -0
- package/dist/lib/core/Driver/sqlite/SqliteQueryBuilder.js +689 -0
- package/dist/lib/core/Driver/sqlite/SqliteQueryBuilder.js.map +1 -0
- package/dist/lib/core/JoinModel.js +2 -2
- package/dist/lib/core/JoinModel.js.map +1 -1
- package/dist/lib/core/Meta.d.ts +17 -11
- package/dist/lib/core/Meta.js +25 -19
- package/dist/lib/core/Meta.js.map +1 -1
- package/dist/lib/core/Model.d.ts +450 -108
- package/dist/lib/core/Model.js +1031 -587
- package/dist/lib/core/Model.js.map +1 -1
- package/dist/lib/{tools/index.d.ts → core/Package.d.ts} +11 -3
- package/dist/lib/{tools/index.js → core/Package.js} +20 -7
- package/dist/lib/core/Package.js.map +1 -0
- package/dist/lib/core/Pool.js +42 -30
- package/dist/lib/core/Pool.js.map +1 -1
- package/dist/lib/core/Queue.d.ts +240 -0
- package/dist/lib/core/Queue.js +686 -0
- package/dist/lib/core/Queue.js.map +1 -0
- package/dist/lib/core/RelationManager.js +27 -25
- package/dist/lib/core/RelationManager.js.map +1 -1
- package/dist/lib/core/Repository.d.ts +73 -207
- package/dist/lib/core/Repository.js +120 -235
- package/dist/lib/core/Repository.js.map +1 -1
- package/dist/lib/core/Schema.d.ts +218 -0
- package/dist/lib/core/Schema.js +370 -65
- package/dist/lib/core/Schema.js.map +1 -1
- package/dist/lib/core/StateManager.d.ts +25 -12
- package/dist/lib/core/StateManager.js +10 -5
- package/dist/lib/core/StateManager.js.map +1 -1
- package/dist/lib/core/UtilityTypes.d.ts +85 -56
- package/dist/lib/core/UtilityTypes.js.map +1 -1
- package/dist/lib/core/index.d.ts +4 -0
- package/dist/lib/core/index.js +6 -2
- package/dist/lib/core/index.js.map +1 -1
- package/dist/lib/types/decorator/index.d.ts +1 -1
- package/dist/lib/types/index.d.ts +69 -22
- package/dist/lib/types/repository/index.d.ts +68 -37
- package/dist/lib/utils/index.d.ts +11 -1
- package/dist/lib/utils/index.js +58 -2
- package/dist/lib/utils/index.js.map +1 -1
- package/package.json +14 -6
- package/dist/lib/tools/index.js.map +0 -1
|
@@ -0,0 +1,686 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.Queue = void 0;
|
|
4
|
+
const Blueprint_1 = require("./Blueprint");
|
|
5
|
+
const DB_1 = require("./DB");
|
|
6
|
+
const Model_1 = require("./Model");
|
|
7
|
+
const QUEUE_STATUS = {
|
|
8
|
+
dispatch: 'Dispatch',
|
|
9
|
+
receive: 'Receive',
|
|
10
|
+
processing: 'Processing',
|
|
11
|
+
completed: 'Completed',
|
|
12
|
+
idle: 'Idle',
|
|
13
|
+
wokeUp: 'Woke up',
|
|
14
|
+
failed: 'Failed',
|
|
15
|
+
waiting: 'Waiting',
|
|
16
|
+
retry: {
|
|
17
|
+
attempts: 'Attempts',
|
|
18
|
+
failed: 'Retry Failed',
|
|
19
|
+
completed: 'Retry Completed',
|
|
20
|
+
}
|
|
21
|
+
};
|
|
22
|
+
const schema = {
|
|
23
|
+
id: Blueprint_1.Blueprint.int().primary().autoIncrement(),
|
|
24
|
+
uuid: Blueprint_1.Blueprint.varchar(36).null(),
|
|
25
|
+
name: Blueprint_1.Blueprint.varchar(255).notNull()
|
|
26
|
+
.index()
|
|
27
|
+
.compositeIndex([
|
|
28
|
+
"status", "available_at", "priority", "id"
|
|
29
|
+
]),
|
|
30
|
+
status: Blueprint_1.Blueprint.enum('pending', 'active', 'completed', 'failed').notNull().default('pending').index(),
|
|
31
|
+
priority: Blueprint_1.Blueprint.int().notNull().default(0),
|
|
32
|
+
payload: Blueprint_1.Blueprint.mediumtext().null(),
|
|
33
|
+
result: Blueprint_1.Blueprint.text().null(),
|
|
34
|
+
error: Blueprint_1.Blueprint.text().null(),
|
|
35
|
+
metadata: Blueprint_1.Blueprint.text().null(),
|
|
36
|
+
attempts: Blueprint_1.Blueprint.int().notNull().default(0),
|
|
37
|
+
max_attempts: Blueprint_1.Blueprint.int().notNull().default(3),
|
|
38
|
+
locked_by: Blueprint_1.Blueprint.text().null(),
|
|
39
|
+
locked_at: Blueprint_1.Blueprint.timestamp().null(),
|
|
40
|
+
available_at: Blueprint_1.Blueprint.timestamp().notNull(),
|
|
41
|
+
completed_at: Blueprint_1.Blueprint.timestamp().null(),
|
|
42
|
+
created_at: Blueprint_1.Blueprint.timestamp().null(),
|
|
43
|
+
updated_at: Blueprint_1.Blueprint.timestamp().null()
|
|
44
|
+
};
|
|
45
|
+
class Worker extends Model_1.Model {
|
|
46
|
+
HOSTNAME = String(process.env?.hostname ?? 'unknown');
|
|
47
|
+
INSPECT_EXEC = false;
|
|
48
|
+
STOPPING = false;
|
|
49
|
+
IS_FLUSHING = false;
|
|
50
|
+
LIMIT_CONNECTIONS = 41;
|
|
51
|
+
MAX_IDLE_RETRIES = 15;
|
|
52
|
+
ACTIVE_JOBS = 0;
|
|
53
|
+
BATCH_SIZE = 1000;
|
|
54
|
+
MAX_WAIT_MS = 50;
|
|
55
|
+
BUFFER = {
|
|
56
|
+
jobs: [],
|
|
57
|
+
timeout: null
|
|
58
|
+
};
|
|
59
|
+
WORKER_STATE = new Map();
|
|
60
|
+
boot() {
|
|
61
|
+
this.useUUID();
|
|
62
|
+
this.useTimestamp();
|
|
63
|
+
this.useSchema(schema);
|
|
64
|
+
this.useTable(this.$state.get("TABLE_JOB"));
|
|
65
|
+
}
|
|
66
|
+
async initialize(opts = {}) {
|
|
67
|
+
const driver = this.driver();
|
|
68
|
+
if (driver === 'mongodb') {
|
|
69
|
+
throw new Error('Queue is not supported for MongoDB. Use a different driver or disable queue features.');
|
|
70
|
+
}
|
|
71
|
+
await this.sync({ force: true, index: true }).catch(() => null);
|
|
72
|
+
if (opts.inspect) {
|
|
73
|
+
this.INSPECT_EXEC = true;
|
|
74
|
+
console.log(`\x1b[34mQueue:\x1b[0m \x1b[32mJob processing started\x1b[0m`);
|
|
75
|
+
}
|
|
76
|
+
if (opts.flush) {
|
|
77
|
+
await this.flush();
|
|
78
|
+
}
|
|
79
|
+
if (opts.hostname) {
|
|
80
|
+
this.HOSTNAME = opts.hostname;
|
|
81
|
+
}
|
|
82
|
+
if (opts.maxIdleRetries) {
|
|
83
|
+
this.MAX_IDLE_RETRIES = opts.maxIdleRetries;
|
|
84
|
+
}
|
|
85
|
+
if (opts.limitConnections) {
|
|
86
|
+
this.LIMIT_CONNECTIONS = opts.limitConnections;
|
|
87
|
+
}
|
|
88
|
+
else {
|
|
89
|
+
const maxConnections = await DB_1.DB.getMaxConnections().catch(() => null);
|
|
90
|
+
this.LIMIT_CONNECTIONS = maxConnections
|
|
91
|
+
? Math.max(10, Math.floor(maxConnections / 3))
|
|
92
|
+
: this.LIMIT_CONNECTIONS;
|
|
93
|
+
}
|
|
94
|
+
return this;
|
|
95
|
+
}
|
|
96
|
+
async shutdown() {
|
|
97
|
+
this.STOPPING = true;
|
|
98
|
+
while (this.ACTIVE_JOBS > 0) {
|
|
99
|
+
if (this.INSPECT_EXEC) {
|
|
100
|
+
console.log(`\x1b[34mQueue:\x1b[0m waiting active jobs total '${this.ACTIVE_JOBS}'`);
|
|
101
|
+
}
|
|
102
|
+
await new Promise(r => setTimeout(r, 200));
|
|
103
|
+
}
|
|
104
|
+
if (this.INSPECT_EXEC) {
|
|
105
|
+
console.log("\x1b[34mQueue:\x1b[0m \x1b[32mJob processing stopped\x1b[0m");
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
async flush() {
|
|
109
|
+
await this.truncate({ force: true });
|
|
110
|
+
return;
|
|
111
|
+
}
|
|
112
|
+
async getJobOverallStats(name) {
|
|
113
|
+
const where = (q) => {
|
|
114
|
+
if (name)
|
|
115
|
+
q.where('name', 'LIKE', `%${name}%`);
|
|
116
|
+
return q;
|
|
117
|
+
};
|
|
118
|
+
const completed = await where(new Worker()).where('status', 'completed').count();
|
|
119
|
+
const active = await where(new Worker()).where('status', 'active').count();
|
|
120
|
+
const pending = await where(new Worker()).where('status', 'pending').count();
|
|
121
|
+
const failed = await where(new Worker()).where('status', 'failed').count();
|
|
122
|
+
const total = await where(new Worker()).count();
|
|
123
|
+
return {
|
|
124
|
+
total,
|
|
125
|
+
completed,
|
|
126
|
+
active,
|
|
127
|
+
pending,
|
|
128
|
+
failed,
|
|
129
|
+
};
|
|
130
|
+
}
|
|
131
|
+
async getJobStats(name) {
|
|
132
|
+
const rows = await new Worker()
|
|
133
|
+
.select('name', 'status')
|
|
134
|
+
.selectRaw('COUNT(1) AS total')
|
|
135
|
+
.when(name, (q) => q.where('name', 'LIKE', `%${name}%`))
|
|
136
|
+
.groupBy('name', 'status')
|
|
137
|
+
.orderBy('name')
|
|
138
|
+
.get();
|
|
139
|
+
const map = new Map();
|
|
140
|
+
for (const row of rows) {
|
|
141
|
+
const name = row.name;
|
|
142
|
+
const status = row.status;
|
|
143
|
+
const total = Number(row.total);
|
|
144
|
+
if (!map.has(name)) {
|
|
145
|
+
map.set(name, {
|
|
146
|
+
name,
|
|
147
|
+
completed: 0,
|
|
148
|
+
active: 0,
|
|
149
|
+
pending: 0,
|
|
150
|
+
failed: 0,
|
|
151
|
+
});
|
|
152
|
+
}
|
|
153
|
+
const stats = map.get(name);
|
|
154
|
+
if (status === 'completed')
|
|
155
|
+
stats.completed = total;
|
|
156
|
+
else if (status === 'active')
|
|
157
|
+
stats.active = total;
|
|
158
|
+
else if (status === 'pending')
|
|
159
|
+
stats.pending = total;
|
|
160
|
+
else if (status === 'failed')
|
|
161
|
+
stats.failed = total;
|
|
162
|
+
}
|
|
163
|
+
return Array.from(map.values());
|
|
164
|
+
}
|
|
165
|
+
async getNames() {
|
|
166
|
+
return await new Worker().select('name').toArray('name');
|
|
167
|
+
}
|
|
168
|
+
async add(name, payload, opts = {}) {
|
|
169
|
+
return new Promise((resolve, reject) => {
|
|
170
|
+
const jobData = {
|
|
171
|
+
name,
|
|
172
|
+
payload: payload == null ? null : this.safeJsonStringify(payload),
|
|
173
|
+
status: 'pending',
|
|
174
|
+
available_at: opts.delayMs ? new Date(Date.now() + opts.delayMs) : new Date(),
|
|
175
|
+
priority: opts.priority ?? 0,
|
|
176
|
+
attempts: 0,
|
|
177
|
+
max_attempts: opts.maxAttempts ?? 3,
|
|
178
|
+
metadata: opts.metadata ? this.safeJsonStringify(opts.metadata) : null
|
|
179
|
+
};
|
|
180
|
+
this.BUFFER.jobs.push({ jobData, resolve, reject });
|
|
181
|
+
if (this.BUFFER.jobs.length >= this.BATCH_SIZE) {
|
|
182
|
+
this._flushBuffer(name);
|
|
183
|
+
}
|
|
184
|
+
else if (!this.BUFFER.timeout) {
|
|
185
|
+
this.BUFFER.timeout = setTimeout(() => this._flushBuffer(name), this.MAX_WAIT_MS);
|
|
186
|
+
}
|
|
187
|
+
});
|
|
188
|
+
}
|
|
189
|
+
async process(name, handler, opts = {
|
|
190
|
+
interval: 1_000,
|
|
191
|
+
concurrency: 1
|
|
192
|
+
}) {
|
|
193
|
+
this.WORKER_STATE.set(name, {
|
|
194
|
+
handler: handler,
|
|
195
|
+
idle: 0,
|
|
196
|
+
sleeping: false,
|
|
197
|
+
running: 0,
|
|
198
|
+
opts: {
|
|
199
|
+
concurrency: opts.concurrency,
|
|
200
|
+
interval: opts.interval
|
|
201
|
+
}
|
|
202
|
+
});
|
|
203
|
+
if (this.INSPECT_EXEC) {
|
|
204
|
+
console.log(`\x1b[34mQueue:\x1b[0m \x1b[35m'${name}'\x1b[0m \x1b[32m${QUEUE_STATUS.dispatch}\x1b[0m`);
|
|
205
|
+
}
|
|
206
|
+
const dispatch = async () => {
|
|
207
|
+
if (this.STOPPING)
|
|
208
|
+
return;
|
|
209
|
+
const state = this.WORKER_STATE.get(name);
|
|
210
|
+
if (!state)
|
|
211
|
+
return;
|
|
212
|
+
if (state.running >= state.opts.concurrency) {
|
|
213
|
+
const jitter = Math.random() * 200;
|
|
214
|
+
state.running--;
|
|
215
|
+
setTimeout(dispatch, state.opts.interval + jitter);
|
|
216
|
+
return;
|
|
217
|
+
}
|
|
218
|
+
const capacity = state.opts.concurrency - state.running;
|
|
219
|
+
const jobs = await this._dequeueMany(name, capacity);
|
|
220
|
+
if (!jobs || jobs.length === 0) {
|
|
221
|
+
state.idle++;
|
|
222
|
+
if (state.idle >= this.MAX_IDLE_RETRIES) {
|
|
223
|
+
state.sleeping = true;
|
|
224
|
+
if (this.INSPECT_EXEC) {
|
|
225
|
+
console.log(`\x1b[34mQueue:\x1b[0m \x1b[35m'${name}'\x1b[0m \x1b[90m${QUEUE_STATUS.idle} (no jobs available)\x1b[0m`);
|
|
226
|
+
}
|
|
227
|
+
return;
|
|
228
|
+
}
|
|
229
|
+
const backoff = Math.min(1000, 50 * state.idle);
|
|
230
|
+
const jitter = Math.random() * 200;
|
|
231
|
+
setTimeout(dispatch, opts.interval + backoff + jitter);
|
|
232
|
+
return;
|
|
233
|
+
}
|
|
234
|
+
state.idle = 0;
|
|
235
|
+
await Promise.all(jobs.map(job => this._runJob(name, job, state)));
|
|
236
|
+
setImmediate(dispatch);
|
|
237
|
+
return;
|
|
238
|
+
};
|
|
239
|
+
return dispatch();
|
|
240
|
+
}
|
|
241
|
+
async _runJob(name, job, state) {
|
|
242
|
+
state.running++;
|
|
243
|
+
this.ACTIVE_JOBS++;
|
|
244
|
+
const handler = state.handler;
|
|
245
|
+
try {
|
|
246
|
+
if (this.INSPECT_EXEC) {
|
|
247
|
+
console.log(`\x1b[34mQueue:\x1b[0m \x1b[35m'${name}'\x1b[0m \x1b[38;2;77;215;240m${QUEUE_STATUS.processing}\x1b[0m job \x1b[38;5;208m${job.id}\x1b[0m`);
|
|
248
|
+
}
|
|
249
|
+
const startTime = +new Date();
|
|
250
|
+
const result = await handler(job);
|
|
251
|
+
await new Worker()
|
|
252
|
+
.where('id', job.id)
|
|
253
|
+
.update({
|
|
254
|
+
status: 'completed',
|
|
255
|
+
result: this.safeJsonStringify(result),
|
|
256
|
+
completed_at: this.$utils.timestamp()
|
|
257
|
+
})
|
|
258
|
+
.void()
|
|
259
|
+
.save();
|
|
260
|
+
const endTime = +new Date();
|
|
261
|
+
if (this.INSPECT_EXEC) {
|
|
262
|
+
console.log(`\x1b[34mQueue:\x1b[0m \x1b[35m'${name}'\x1b[0m \x1b[32m${QUEUE_STATUS.completed}\x1b[0m job \x1b[38;5;208m${job.id}\x1b[0m (${endTime - startTime}ms)`);
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
catch (err) {
|
|
266
|
+
if (this.INSPECT_EXEC) {
|
|
267
|
+
console.log(`\x1b[34mQueue:\x1b[0m \x1b[35m'${name}'\x1b[0m \x1b[31m${QUEUE_STATUS.failed}\x1b[0m job \x1b[38;5;208m${job.id}\x1b[0m`);
|
|
268
|
+
}
|
|
269
|
+
await new Worker()
|
|
270
|
+
.where('id', job.id)
|
|
271
|
+
.update({
|
|
272
|
+
status: 'failed',
|
|
273
|
+
error: this.safeJsonStringify({
|
|
274
|
+
message: err.message,
|
|
275
|
+
name: err.name,
|
|
276
|
+
stack: err.stack,
|
|
277
|
+
code: err.code,
|
|
278
|
+
})
|
|
279
|
+
})
|
|
280
|
+
.void()
|
|
281
|
+
.save();
|
|
282
|
+
const maxAttempts = job.__job.max_attempts;
|
|
283
|
+
let attempts = job.__job.attempts;
|
|
284
|
+
while (attempts < maxAttempts) {
|
|
285
|
+
attempts++;
|
|
286
|
+
try {
|
|
287
|
+
const startTime = +new Date();
|
|
288
|
+
const result = await handler(job);
|
|
289
|
+
const endTime = +new Date();
|
|
290
|
+
await new Worker()
|
|
291
|
+
.where('id', job.id)
|
|
292
|
+
.update({
|
|
293
|
+
status: 'completed',
|
|
294
|
+
attempts,
|
|
295
|
+
result: this.safeJsonStringify(result),
|
|
296
|
+
completed_at: this.$utils.timestamp()
|
|
297
|
+
})
|
|
298
|
+
.void()
|
|
299
|
+
.save();
|
|
300
|
+
if (this.INSPECT_EXEC) {
|
|
301
|
+
console.log(`\x1b[34mQueue:\x1b[0m \x1b[35m'${name}'\x1b[0m \x1b[32m${QUEUE_STATUS.retry.completed}\x1b[0m job \x1b[38;5;208m${job.id}\x1b[0m (${endTime - startTime}ms ${attempts}/${maxAttempts})`);
|
|
302
|
+
}
|
|
303
|
+
break;
|
|
304
|
+
}
|
|
305
|
+
catch (err) {
|
|
306
|
+
if (this.INSPECT_EXEC) {
|
|
307
|
+
console.log(`\x1b[34mQueue:\x1b[0m \x1b[35m'${name}'\x1b[0m \x1b[31m${QUEUE_STATUS.retry.attempts}\x1b[0m job \x1b[38;5;208m${job.id}\x1b[0m (${attempts}/${maxAttempts})`);
|
|
308
|
+
}
|
|
309
|
+
if (attempts >= maxAttempts) {
|
|
310
|
+
await new Worker()
|
|
311
|
+
.where('id', job.id)
|
|
312
|
+
.update({
|
|
313
|
+
status: 'failed',
|
|
314
|
+
attempts,
|
|
315
|
+
error: this.safeJsonStringify({
|
|
316
|
+
retry: true,
|
|
317
|
+
message: err?.message,
|
|
318
|
+
name: err?.name,
|
|
319
|
+
stack: err?.stack,
|
|
320
|
+
code: err?.code,
|
|
321
|
+
}),
|
|
322
|
+
})
|
|
323
|
+
.void()
|
|
324
|
+
.save();
|
|
325
|
+
if (this.INSPECT_EXEC) {
|
|
326
|
+
console.log(`\x1b[34mQueue:\x1b[0m \x1b[35m'${name}'\x1b[0m \x1b[31m${QUEUE_STATUS.retry.failed}\x1b[0m job \x1b[38;5;208m${job.id}\x1b[0m (\x1b[33mmax attempts reached\x1b[0m)`);
|
|
327
|
+
}
|
|
328
|
+
break;
|
|
329
|
+
}
|
|
330
|
+
await new Promise((r) => setTimeout(r, 1_000 * 2));
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
finally {
|
|
335
|
+
state.running--;
|
|
336
|
+
this.ACTIVE_JOBS--;
|
|
337
|
+
}
|
|
338
|
+
}
|
|
339
|
+
async _waitForSafeConnections(name) {
|
|
340
|
+
let activeConnections = 0;
|
|
341
|
+
// while (true) {
|
|
342
|
+
// activeConnections = await DB.getActiveConnections();
|
|
343
|
+
// if (activeConnections <= this.LIMIT_CONNECTIONS) {
|
|
344
|
+
// break;
|
|
345
|
+
// }
|
|
346
|
+
// if (this.INSPECT_EXEC) {
|
|
347
|
+
// console.log(`\x1b[34mQueue:\x1b[0m \x1b[35m'${name}'\x1b[0m \x1b[31m${QUEUE_STATUS.waiting}\x1b[0m DB connections high \x1b[33m (${activeConnections}/${this.LIMIT_CONNECTIONS})\x1b[0m`);
|
|
348
|
+
// }
|
|
349
|
+
// await new Promise(resolve => setTimeout(resolve, 1000 * 5));
|
|
350
|
+
// }
|
|
351
|
+
return;
|
|
352
|
+
}
|
|
353
|
+
async _dequeueMany(name, limit) {
|
|
354
|
+
if (this.STOPPING)
|
|
355
|
+
return [];
|
|
356
|
+
await this._waitForSafeConnections(name);
|
|
357
|
+
const findJobs = await new Worker()
|
|
358
|
+
.select('id')
|
|
359
|
+
.where('name', name)
|
|
360
|
+
.whereQuery(q => {
|
|
361
|
+
return q
|
|
362
|
+
.where('status', 'pending')
|
|
363
|
+
.where('available_at', '<=', this.$utils.timestamp())
|
|
364
|
+
.orWhereQuery((q) => {
|
|
365
|
+
return q
|
|
366
|
+
.where('status', 'active')
|
|
367
|
+
.where('locked_at', '<', this.$utils.timestamp(new Date(Date.now() - 60 * 1000)));
|
|
368
|
+
});
|
|
369
|
+
})
|
|
370
|
+
.limit(limit)
|
|
371
|
+
.get();
|
|
372
|
+
if (!findJobs.length) {
|
|
373
|
+
return [];
|
|
374
|
+
}
|
|
375
|
+
return await DB_1.DB.transaction(async (trx) => {
|
|
376
|
+
const jobs = await new Worker()
|
|
377
|
+
.whereIn('id', findJobs.map(v => v.id))
|
|
378
|
+
.latest('priority')
|
|
379
|
+
.oldest('id')
|
|
380
|
+
.limit(limit)
|
|
381
|
+
.forUpdate({ skipLocked: true })
|
|
382
|
+
.bind(trx)
|
|
383
|
+
.get();
|
|
384
|
+
if (!jobs.length) {
|
|
385
|
+
return [];
|
|
386
|
+
}
|
|
387
|
+
await new Worker()
|
|
388
|
+
.whereIn('id', jobs.map(v => v.id))
|
|
389
|
+
.updateMany({
|
|
390
|
+
status: 'active',
|
|
391
|
+
locked_at: this.$utils.timestamp(),
|
|
392
|
+
locked_by: this.HOSTNAME
|
|
393
|
+
})
|
|
394
|
+
.void()
|
|
395
|
+
.bind(trx)
|
|
396
|
+
.limit(limit)
|
|
397
|
+
.save();
|
|
398
|
+
return (jobs ?? []).map((job) => ({
|
|
399
|
+
id: job.id,
|
|
400
|
+
name: job.name,
|
|
401
|
+
status: job.status,
|
|
402
|
+
payload: this.safeJsonParse(job.payload),
|
|
403
|
+
__job: job
|
|
404
|
+
}));
|
|
405
|
+
});
|
|
406
|
+
}
|
|
407
|
+
async _flushBuffer(name) {
|
|
408
|
+
if (this.IS_FLUSHING || this.BUFFER.jobs.length === 0)
|
|
409
|
+
return;
|
|
410
|
+
if (this.BUFFER.timeout) {
|
|
411
|
+
clearTimeout(this.BUFFER.timeout);
|
|
412
|
+
this.BUFFER.timeout = null;
|
|
413
|
+
}
|
|
414
|
+
const currentBatch = this.BUFFER.jobs;
|
|
415
|
+
this.IS_FLUSHING = true;
|
|
416
|
+
this.BUFFER.jobs = [];
|
|
417
|
+
this.IS_FLUSHING = false;
|
|
418
|
+
try {
|
|
419
|
+
const jobsToInsert = currentBatch.map(b => b.jobData);
|
|
420
|
+
const insertedJobds = await new Worker()
|
|
421
|
+
.select('id')
|
|
422
|
+
.insertMany(jobsToInsert)
|
|
423
|
+
.save();
|
|
424
|
+
if (this.INSPECT_EXEC) {
|
|
425
|
+
const ids = insertedJobds.map(v => v.id);
|
|
426
|
+
const preview = [
|
|
427
|
+
...ids.slice(0, 3),
|
|
428
|
+
"...",
|
|
429
|
+
...ids.slice(-2),
|
|
430
|
+
].join(', ');
|
|
431
|
+
if (ids.length === 1) {
|
|
432
|
+
console.log(`\x1b[34mQueue:\x1b[0m \x1b[35m'${name}'\x1b[0m \x1b[32m${QUEUE_STATUS.receive}\x1b[0m job \x1b[38;5;208m${ids}\x1b[0m`);
|
|
433
|
+
}
|
|
434
|
+
else {
|
|
435
|
+
console.log(`\x1b[34mQueue:\x1b[0m \x1b[35m'${name}'\x1b[0m \x1b[32m${QUEUE_STATUS.receive}\x1b[0m jobs [\x1b[38;5;208m${preview}\x1b[0m] total=(\x1b[38;5;208m${ids.length}\x1b[0m)`);
|
|
436
|
+
}
|
|
437
|
+
}
|
|
438
|
+
for (let i = 0; i < currentBatch.length; i++) {
|
|
439
|
+
currentBatch[i].resolve(undefined);
|
|
440
|
+
}
|
|
441
|
+
const uniqueNames = [...new Set(currentBatch.map(b => b.jobData.name))];
|
|
442
|
+
for (const name of uniqueNames) {
|
|
443
|
+
this._wakeWorker(name);
|
|
444
|
+
}
|
|
445
|
+
}
|
|
446
|
+
catch (error) {
|
|
447
|
+
currentBatch.forEach(b => b.reject(error));
|
|
448
|
+
}
|
|
449
|
+
finally {
|
|
450
|
+
if (this.BUFFER.jobs.length) {
|
|
451
|
+
this._flushBuffer(name);
|
|
452
|
+
}
|
|
453
|
+
}
|
|
454
|
+
}
|
|
455
|
+
_wakeWorker(name) {
|
|
456
|
+
const state = this.WORKER_STATE.get(name);
|
|
457
|
+
if (!state || !state.sleeping || !state.handler)
|
|
458
|
+
return;
|
|
459
|
+
const isSleeping = state.sleeping;
|
|
460
|
+
state.sleeping = false;
|
|
461
|
+
state.idle = 0;
|
|
462
|
+
if (this.INSPECT_EXEC) {
|
|
463
|
+
console.log(`\x1b[34mQueue:\x1b[0m \x1b[35m'${name}'\x1b[0m \x1b[36m${QUEUE_STATUS.wokeUp}\x1b[0m`);
|
|
464
|
+
}
|
|
465
|
+
if (isSleeping) {
|
|
466
|
+
this.process(name, state.handler, {
|
|
467
|
+
concurrency: state.opts.concurrency
|
|
468
|
+
});
|
|
469
|
+
}
|
|
470
|
+
}
|
|
471
|
+
safeJsonParse(payload) {
|
|
472
|
+
try {
|
|
473
|
+
return JSON.parse(payload);
|
|
474
|
+
}
|
|
475
|
+
catch (err) {
|
|
476
|
+
return payload;
|
|
477
|
+
}
|
|
478
|
+
}
|
|
479
|
+
safeJsonStringify(payload) {
|
|
480
|
+
if (payload == null)
|
|
481
|
+
return null;
|
|
482
|
+
try {
|
|
483
|
+
return JSON.stringify(payload, (_, value) => {
|
|
484
|
+
if (typeof value === 'bigint') {
|
|
485
|
+
return value.toString();
|
|
486
|
+
}
|
|
487
|
+
if (value instanceof Map) {
|
|
488
|
+
return Object.fromEntries(value);
|
|
489
|
+
}
|
|
490
|
+
if (value instanceof Set) {
|
|
491
|
+
return Array.from(value);
|
|
492
|
+
}
|
|
493
|
+
return value;
|
|
494
|
+
});
|
|
495
|
+
}
|
|
496
|
+
catch (err) {
|
|
497
|
+
return payload;
|
|
498
|
+
}
|
|
499
|
+
}
|
|
500
|
+
}
|
|
501
|
+
/**
|
|
502
|
+
* Queue facade class (static API wrapper)
|
|
503
|
+
*
|
|
504
|
+
* This class provides a singleton-style interface over the underlying Worker instance.
|
|
505
|
+
* It must be initialized before use via `Queue.start()`.
|
|
506
|
+
*
|
|
507
|
+
* @example
|
|
508
|
+
* ```ts
|
|
509
|
+
* const sendEmail = (job) => console.log('send mail :' + job.payload.email)
|
|
510
|
+
*
|
|
511
|
+
* await Queue.start({ inspect : true, flush : true // **remove all jobs });
|
|
512
|
+
*
|
|
513
|
+
* // register
|
|
514
|
+
* Queue.progress("send-email", async (job) => {
|
|
515
|
+
* return await sendEmail(job);
|
|
516
|
+
* }, { concurrency : 3 });
|
|
517
|
+
*
|
|
518
|
+
* // add
|
|
519
|
+
* Queue.add("send-email", { email: "test@gmail.com" });
|
|
520
|
+
*
|
|
521
|
+
* ```
|
|
522
|
+
*/
|
|
523
|
+
class Queue {
|
|
524
|
+
/**
|
|
525
|
+
* Internal Worker instance used for all queue operations.
|
|
526
|
+
* @type {Worker | null}
|
|
527
|
+
*/
|
|
528
|
+
static WORKER;
|
|
529
|
+
static MESSAGE = {
|
|
530
|
+
INIT_ERROR: `Queue is not initialized. Please call 'await Queue.start()' before using it.`
|
|
531
|
+
};
|
|
532
|
+
/**
|
|
533
|
+
* The 'start' method is used to initialize the Queue system.
|
|
534
|
+
* Creates and prepares the underlying Worker instance.
|
|
535
|
+
* @param {Object} [opts] - options (inspect, flush)
|
|
536
|
+
* @property {boolean} opts.inspect queue work flow
|
|
537
|
+
* @property {boolean} opts.flush remove all queue
|
|
538
|
+
* @property {number} opts.maxIdleRetries - Maximum idle time () when no jobs are available
|
|
539
|
+
* @property {number} opts.limitConnections - Allowed DB connections limit before pausing
|
|
540
|
+
* @returns {Promise<void>}
|
|
541
|
+
*/
|
|
542
|
+
static async start(opts = {}) {
|
|
543
|
+
this.WORKER = await new Worker().initialize(opts);
|
|
544
|
+
return;
|
|
545
|
+
}
|
|
546
|
+
/**
|
|
547
|
+
* The 'end' method is used to shutdown the Queue system.
|
|
548
|
+
*
|
|
549
|
+
* @returns {Promise<void>}
|
|
550
|
+
*/
|
|
551
|
+
static async end() {
|
|
552
|
+
if (this.WORKER == null) {
|
|
553
|
+
throw new Error(this.MESSAGE.INIT_ERROR);
|
|
554
|
+
}
|
|
555
|
+
await this.WORKER.shutdown();
|
|
556
|
+
this.WORKER = null;
|
|
557
|
+
return;
|
|
558
|
+
}
|
|
559
|
+
/**
|
|
560
|
+
* The 'flush' method is used to flush all jobs in the queue (dangerous operation).
|
|
561
|
+
*
|
|
562
|
+
* @throws {Error} If Queue is not initialized.
|
|
563
|
+
* @returns {Promise<void>}
|
|
564
|
+
*/
|
|
565
|
+
static async flush() {
|
|
566
|
+
if (this.WORKER == null) {
|
|
567
|
+
throw new Error(this.MESSAGE.INIT_ERROR);
|
|
568
|
+
}
|
|
569
|
+
await this.WORKER.flush();
|
|
570
|
+
}
|
|
571
|
+
/**
|
|
572
|
+
* The 'getJobOverallStats' method is used to get aggregated queue statistics.
|
|
573
|
+
*
|
|
574
|
+
* @param {string} [name] - Optional queue name filter.
|
|
575
|
+
* @throws {Error} If Queue is not initialized.
|
|
576
|
+
* @returns {Promise<any>}
|
|
577
|
+
*/
|
|
578
|
+
static async getJobOverallStats(name) {
|
|
579
|
+
if (this.WORKER == null) {
|
|
580
|
+
throw new Error(this.MESSAGE.INIT_ERROR);
|
|
581
|
+
}
|
|
582
|
+
return await this.WORKER.getJobOverallStats(name);
|
|
583
|
+
}
|
|
584
|
+
/**
|
|
585
|
+
* The 'getJobStats' method is used to Get jobs statistics grouped by name.
|
|
586
|
+
*
|
|
587
|
+
* @param {string} [name] - Optional queue name filter.
|
|
588
|
+
* @throws {Error} If Queue is not initialized.
|
|
589
|
+
* @returns {Promise<Record<string,any>>}
|
|
590
|
+
*/
|
|
591
|
+
static async getJobStats(name) {
|
|
592
|
+
if (this.WORKER == null) {
|
|
593
|
+
throw new Error(this.MESSAGE.INIT_ERROR);
|
|
594
|
+
}
|
|
595
|
+
return await this.WORKER.getJobStats(name);
|
|
596
|
+
}
|
|
597
|
+
/**
|
|
598
|
+
* Get all unique queue names.
|
|
599
|
+
*
|
|
600
|
+
* @throws {Error} If Queue is not initialized.
|
|
601
|
+
* @returns {Promise<string[]>}
|
|
602
|
+
*/
|
|
603
|
+
static async getNames() {
|
|
604
|
+
if (this.WORKER == null) {
|
|
605
|
+
throw new Error(this.MESSAGE.INIT_ERROR);
|
|
606
|
+
}
|
|
607
|
+
return await this.WORKER.getNames();
|
|
608
|
+
}
|
|
609
|
+
/**
|
|
610
|
+
* Access raw Worker instance safely.
|
|
611
|
+
*
|
|
612
|
+
* @param {(worker: Worker) => any} cb - Callback with Worker instance.
|
|
613
|
+
* @throws {Error} If Queue is not initialized.
|
|
614
|
+
* @returns {Promise<Work>}
|
|
615
|
+
*/
|
|
616
|
+
static async worker(cb) {
|
|
617
|
+
if (this.WORKER == null) {
|
|
618
|
+
throw new Error(this.MESSAGE.INIT_ERROR);
|
|
619
|
+
}
|
|
620
|
+
return await cb(this.WORKER);
|
|
621
|
+
}
|
|
622
|
+
/**
|
|
623
|
+
* Start a worker for processing jobs of a specific name.
|
|
624
|
+
*
|
|
625
|
+
* @param {string} name - Queue name to process.
|
|
626
|
+
* @param {Handler} handler - Job handler function.
|
|
627
|
+
* @param {QueueProcessOptions} [opts] - Job options (interval, concurrency)
|
|
628
|
+
* @throws {Error} If Queue is not initialized.
|
|
629
|
+
* @returns {Promise<void>}
|
|
630
|
+
*
|
|
631
|
+
* @example
|
|
632
|
+
* const helloWorld = (job) => console.log('hello world :' + job.id);
|
|
633
|
+
*
|
|
634
|
+
* Queue.progress("hello", async (job) => {
|
|
635
|
+
* return await helloWorld(job)
|
|
636
|
+
* }, { concurrency : 3 });
|
|
637
|
+
*/
|
|
638
|
+
static async process(name, handler, opts = { interval: 1_000, concurrency: 1 }) {
|
|
639
|
+
if (this.WORKER == null) {
|
|
640
|
+
throw new Error(this.MESSAGE.INIT_ERROR);
|
|
641
|
+
}
|
|
642
|
+
return await this.WORKER.process(name, handler, opts);
|
|
643
|
+
}
|
|
644
|
+
/**
|
|
645
|
+
* Start a worker for processing jobs of a specific name.
|
|
646
|
+
*
|
|
647
|
+
* @param {string} name - Queue name to process.
|
|
648
|
+
* @param {Handler} handler - Job handler function.
|
|
649
|
+
* @param {QueueProcessOptions} [opts] - Job options (interval, concurrency)
|
|
650
|
+
* @throws {Error} If Queue is not initialized.
|
|
651
|
+
* @returns {Promise<void>}
|
|
652
|
+
*
|
|
653
|
+
* @example
|
|
654
|
+
* const helloWorld = (job) => console.log('hello world :' + job.id);
|
|
655
|
+
*
|
|
656
|
+
* Queue.on("hello", async (job) => {
|
|
657
|
+
* return await helloWorld(job)
|
|
658
|
+
* }, { concurrency : 3 });
|
|
659
|
+
*/
|
|
660
|
+
static async on(name, handler, opts = { interval: 1_000, concurrency: 1 }) {
|
|
661
|
+
return await this.process(name, handler, opts);
|
|
662
|
+
}
|
|
663
|
+
/**
|
|
664
|
+
* Add a new job into the queue.
|
|
665
|
+
*
|
|
666
|
+
* @param {string} name - Queue name / job type.
|
|
667
|
+
* @param {any} payload - Job payload data.
|
|
668
|
+
* @param {QueueAddOptions} [opts] - Job options (delay, priority, retry, etc.)
|
|
669
|
+
* @throws {Error} If Queue is not initialized.
|
|
670
|
+
* @returns {Promise<T.Result<Worker>>}
|
|
671
|
+
*
|
|
672
|
+
* @example
|
|
673
|
+
* ```ts
|
|
674
|
+
* Queue.add("send-email", { email: "test@gmail.com" });
|
|
675
|
+
* ```
|
|
676
|
+
*/
|
|
677
|
+
static async add(name, payload, opts = {}) {
|
|
678
|
+
if (this.WORKER == null) {
|
|
679
|
+
throw new Error(this.MESSAGE.INIT_ERROR);
|
|
680
|
+
}
|
|
681
|
+
return await this.WORKER.add(name, payload, opts);
|
|
682
|
+
}
|
|
683
|
+
}
|
|
684
|
+
exports.Queue = Queue;
|
|
685
|
+
exports.default = Queue;
|
|
686
|
+
//# sourceMappingURL=Queue.js.map
|