pg-boss 10.3.3 → 10.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/settings.local.json +25 -0
- package/dist/attorney.d.ts +19 -0
- package/dist/attorney.d.ts.map +1 -0
- package/dist/attorney.js +227 -0
- package/dist/bam.d.ts +14 -0
- package/dist/bam.d.ts.map +1 -0
- package/dist/bam.js +114 -0
- package/dist/boss.d.ts +16 -0
- package/dist/boss.d.ts.map +1 -0
- package/dist/boss.js +163 -0
- package/dist/cli.d.ts +3 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +333 -0
- package/dist/contractor.d.ts +22 -0
- package/dist/contractor.d.ts.map +1 -0
- package/dist/contractor.js +81 -0
- package/dist/db.d.ts +16 -0
- package/dist/db.d.ts.map +1 -0
- package/dist/db.js +46 -0
- package/dist/index.d.ts +72 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +266 -0
- package/dist/manager.d.ts +93 -0
- package/dist/manager.d.ts.map +1 -0
- package/dist/manager.js +746 -0
- package/dist/migrationStore.d.ts +7 -0
- package/dist/migrationStore.d.ts.map +1 -0
- package/dist/migrationStore.js +425 -0
- package/dist/plans.d.ts +102 -0
- package/dist/plans.d.ts.map +1 -0
- package/dist/plans.js +1220 -0
- package/dist/spy.d.ts +23 -0
- package/dist/spy.d.ts.map +1 -0
- package/dist/spy.js +73 -0
- package/dist/timekeeper.d.ts +34 -0
- package/dist/timekeeper.d.ts.map +1 -0
- package/dist/timekeeper.js +158 -0
- package/dist/tools.d.ts +18 -0
- package/dist/tools.d.ts.map +1 -0
- package/dist/tools.js +45 -0
- package/dist/types.d.ts +301 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +1 -0
- package/dist/worker.d.ts +43 -0
- package/dist/worker.d.ts.map +1 -0
- package/dist/worker.js +113 -0
- package/package.json +1 -1
- package/src/plans.js +22 -7
package/dist/manager.js
ADDED
|
@@ -0,0 +1,746 @@
|
|
|
1
|
+
import assert, { notStrictEqual } from 'node:assert';
|
|
2
|
+
import { randomUUID } from 'node:crypto';
|
|
3
|
+
import EventEmitter from 'node:events';
|
|
4
|
+
import { serializeError as stringify } from 'serialize-error';
|
|
5
|
+
import * as Attorney from "./attorney.js";
|
|
6
|
+
import * as plans from "./plans.js";
|
|
7
|
+
import * as timekeeper from "./timekeeper.js";
|
|
8
|
+
import { resolveWithinSeconds } from "./tools.js";
|
|
9
|
+
import * as types from "./types.js";
|
|
10
|
+
import Worker from "./worker.js";
|
|
11
|
+
import { JobSpy } from "./spy.js";
|
|
12
|
+
const INTERNAL_QUEUES = Object.values(timekeeper.QUEUES).reduce((acc, i) => ({ ...acc, [i]: i }), {});
|
|
13
|
+
const events = {
|
|
14
|
+
error: 'error',
|
|
15
|
+
wip: 'wip'
|
|
16
|
+
};
|
|
17
|
+
class Manager extends EventEmitter {
|
|
18
|
+
events = events;
|
|
19
|
+
db;
|
|
20
|
+
config;
|
|
21
|
+
wipTs;
|
|
22
|
+
workers;
|
|
23
|
+
stopped;
|
|
24
|
+
queueCacheInterval;
|
|
25
|
+
timekeeper;
|
|
26
|
+
queues;
|
|
27
|
+
pendingOffWorkCleanups;
|
|
28
|
+
#spies;
|
|
29
|
+
#localGroupActive;
|
|
30
|
+
#localGroupConfig;
|
|
31
|
+
constructor(db, config) {
|
|
32
|
+
super();
|
|
33
|
+
this.config = config;
|
|
34
|
+
this.db = db;
|
|
35
|
+
this.wipTs = Date.now();
|
|
36
|
+
this.workers = new Map();
|
|
37
|
+
this.queues = null;
|
|
38
|
+
this.pendingOffWorkCleanups = new Set();
|
|
39
|
+
this.#spies = new Map();
|
|
40
|
+
this.#localGroupActive = new Map();
|
|
41
|
+
this.#localGroupConfig = new Map();
|
|
42
|
+
}
|
|
43
|
+
getSpy(name) {
|
|
44
|
+
if (!this.config.__test__enableSpies) {
|
|
45
|
+
throw new Error('Spy is not enabled. Set __test__enableSpies: true in constructor options to use spies.');
|
|
46
|
+
}
|
|
47
|
+
let spy = this.#spies.get(name);
|
|
48
|
+
if (!spy) {
|
|
49
|
+
spy = new JobSpy();
|
|
50
|
+
this.#spies.set(name, spy);
|
|
51
|
+
}
|
|
52
|
+
return spy;
|
|
53
|
+
}
|
|
54
|
+
clearSpies() {
|
|
55
|
+
for (const spy of this.#spies.values()) {
|
|
56
|
+
spy.clear();
|
|
57
|
+
}
|
|
58
|
+
this.#spies.clear();
|
|
59
|
+
}
|
|
60
|
+
#getLocalGroupLimit(queueName, groupTier) {
|
|
61
|
+
const config = this.#localGroupConfig.get(queueName);
|
|
62
|
+
if (!config)
|
|
63
|
+
return Infinity;
|
|
64
|
+
if (groupTier && config.tiers && groupTier in config.tiers) {
|
|
65
|
+
return config.tiers[groupTier];
|
|
66
|
+
}
|
|
67
|
+
return config.default;
|
|
68
|
+
}
|
|
69
|
+
#getGroupsAtLocalCapacity(queueName) {
|
|
70
|
+
const config = this.#localGroupConfig.get(queueName);
|
|
71
|
+
if (!config)
|
|
72
|
+
return [];
|
|
73
|
+
const queueGroups = this.#localGroupActive.get(queueName);
|
|
74
|
+
if (!queueGroups)
|
|
75
|
+
return [];
|
|
76
|
+
const atCapacity = [];
|
|
77
|
+
for (const [groupId, activeCount] of queueGroups.entries()) {
|
|
78
|
+
// We don't have tier info here, so use default limit
|
|
79
|
+
// Jobs with tiers will be checked individually after fetch
|
|
80
|
+
const limit = config.default;
|
|
81
|
+
if (activeCount >= limit) {
|
|
82
|
+
atCapacity.push(groupId);
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
return atCapacity;
|
|
86
|
+
}
|
|
87
|
+
#incrementLocalGroupCount(queueName, groupId) {
|
|
88
|
+
let queueGroups = this.#localGroupActive.get(queueName);
|
|
89
|
+
if (!queueGroups) {
|
|
90
|
+
queueGroups = new Map();
|
|
91
|
+
this.#localGroupActive.set(queueName, queueGroups);
|
|
92
|
+
}
|
|
93
|
+
const current = queueGroups.get(groupId) || 0;
|
|
94
|
+
queueGroups.set(groupId, current + 1);
|
|
95
|
+
}
|
|
96
|
+
#decrementLocalGroupCount(queueName, groupId) {
|
|
97
|
+
const queueGroups = this.#localGroupActive.get(queueName);
|
|
98
|
+
if (!queueGroups)
|
|
99
|
+
return;
|
|
100
|
+
const current = queueGroups.get(groupId) || 0;
|
|
101
|
+
if (current <= 1) {
|
|
102
|
+
queueGroups.delete(groupId);
|
|
103
|
+
}
|
|
104
|
+
else {
|
|
105
|
+
queueGroups.set(groupId, current - 1);
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
#trackJobsActive(name, jobs) {
|
|
109
|
+
const spy = this.config.__test__enableSpies ? this.#spies.get(name) : undefined;
|
|
110
|
+
if (spy) {
|
|
111
|
+
for (const job of jobs) {
|
|
112
|
+
spy.addJob(job.id, name, job.data, 'active');
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
#trackJobsCompleted(name, jobs, result) {
|
|
117
|
+
const spy = this.config.__test__enableSpies ? this.#spies.get(name) : undefined;
|
|
118
|
+
if (spy) {
|
|
119
|
+
const output = jobs.length === 1 ? result : undefined;
|
|
120
|
+
for (const job of jobs) {
|
|
121
|
+
spy.addJob(job.id, name, job.data, 'completed', output);
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
#trackJobsFailed(name, jobs, err) {
|
|
126
|
+
const spy = this.config.__test__enableSpies ? this.#spies.get(name) : undefined;
|
|
127
|
+
if (spy) {
|
|
128
|
+
for (const job of jobs) {
|
|
129
|
+
spy.addJob(job.id, name, job.data, 'failed', { message: err?.message, stack: err?.stack });
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
#storeLocalGroupConfig(name, localGroupConcurrency) {
|
|
134
|
+
const config = typeof localGroupConcurrency === 'number'
|
|
135
|
+
? { default: localGroupConcurrency }
|
|
136
|
+
: localGroupConcurrency;
|
|
137
|
+
this.#localGroupConfig.set(name, config);
|
|
138
|
+
}
|
|
139
|
+
#cleanupLocalGroupTracking(name) {
|
|
140
|
+
// Only cleanup if no more workers exist for this queue
|
|
141
|
+
const hasWorkersForQueue = this.getWorkers().some(w => w.name === name && !w.stopping && !w.stopped);
|
|
142
|
+
if (!hasWorkersForQueue) {
|
|
143
|
+
this.#localGroupConfig.delete(name);
|
|
144
|
+
this.#localGroupActive.delete(name);
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
#trackLocalGroupStart(name, jobs) {
|
|
148
|
+
const allowed = [];
|
|
149
|
+
const excess = [];
|
|
150
|
+
const groupedJobs = [];
|
|
151
|
+
for (const job of jobs) {
|
|
152
|
+
if (!job.groupId) {
|
|
153
|
+
// Jobs without group bypass local group limits
|
|
154
|
+
allowed.push(job);
|
|
155
|
+
continue;
|
|
156
|
+
}
|
|
157
|
+
const currentCount = this.#localGroupActive.get(name)?.get(job.groupId) || 0;
|
|
158
|
+
const limit = this.#getLocalGroupLimit(name, job.groupTier);
|
|
159
|
+
if (currentCount < limit) {
|
|
160
|
+
this.#incrementLocalGroupCount(name, job.groupId);
|
|
161
|
+
allowed.push(job);
|
|
162
|
+
groupedJobs.push(job);
|
|
163
|
+
}
|
|
164
|
+
else {
|
|
165
|
+
excess.push(job);
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
return { allowed, excess, groupedJobs };
|
|
169
|
+
}
|
|
170
|
+
#trackLocalGroupEnd(name, groupedJobs) {
|
|
171
|
+
for (const job of groupedJobs) {
|
|
172
|
+
if (job.groupId) {
|
|
173
|
+
this.#decrementLocalGroupCount(name, job.groupId);
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
async #processJobs(name, jobs, callback, worker) {
|
|
178
|
+
const jobIds = jobs.map(job => job.id);
|
|
179
|
+
const maxExpiration = jobs.reduce((acc, i) => Math.max(acc, i.expireInSeconds), 0);
|
|
180
|
+
const ac = new AbortController();
|
|
181
|
+
jobs.forEach(job => { job.signal = ac.signal; });
|
|
182
|
+
// Store AbortController on worker so it can be aborted after graceful shutdown
|
|
183
|
+
if (worker) {
|
|
184
|
+
worker.abortController = ac;
|
|
185
|
+
}
|
|
186
|
+
try {
|
|
187
|
+
const result = await resolveWithinSeconds(callback(jobs), maxExpiration, `handler execution exceeded ${maxExpiration}s`, ac);
|
|
188
|
+
await this.complete(name, jobIds, jobIds.length === 1 ? result : undefined);
|
|
189
|
+
this.#trackJobsCompleted(name, jobs, result);
|
|
190
|
+
}
|
|
191
|
+
catch (err) {
|
|
192
|
+
await this.fail(name, jobIds, err);
|
|
193
|
+
this.#trackJobsFailed(name, jobs, err);
|
|
194
|
+
}
|
|
195
|
+
finally {
|
|
196
|
+
if (worker) {
|
|
197
|
+
// Clear between jobs
|
|
198
|
+
worker.abortController = null;
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
async start() {
|
|
203
|
+
this.stopped = false;
|
|
204
|
+
this.queueCacheInterval = setInterval(() => this.onCacheQueues({ emit: true }), this.config.queueCacheIntervalSeconds * 1000);
|
|
205
|
+
await this.onCacheQueues();
|
|
206
|
+
}
|
|
207
|
+
async onCacheQueues({ emit = false } = {}) {
|
|
208
|
+
try {
|
|
209
|
+
assert(!this.config.__test__throw_queueCache, 'test error');
|
|
210
|
+
const queues = await this.getQueues();
|
|
211
|
+
this.queues = queues.reduce((acc, i) => { acc[i.name] = i; return acc; }, {});
|
|
212
|
+
}
|
|
213
|
+
catch (error) {
|
|
214
|
+
emit && this.emit(events.error, { ...error, message: error.message, stack: error.stack });
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
async getQueueCache(name) {
|
|
218
|
+
assert(this.queues, 'Queue cache is not initialized');
|
|
219
|
+
let queue = this.queues[name];
|
|
220
|
+
if (queue) {
|
|
221
|
+
return queue;
|
|
222
|
+
}
|
|
223
|
+
queue = await this.getQueue(name);
|
|
224
|
+
if (!queue) {
|
|
225
|
+
throw new Error(`Queue ${name} does not exist`);
|
|
226
|
+
}
|
|
227
|
+
this.queues[name] = queue;
|
|
228
|
+
return queue;
|
|
229
|
+
}
|
|
230
|
+
async stop() {
|
|
231
|
+
this.stopped = true;
|
|
232
|
+
clearInterval(this.queueCacheInterval);
|
|
233
|
+
await Promise.allSettled([...this.workers.values()]
|
|
234
|
+
.filter(worker => !INTERNAL_QUEUES[worker.name])
|
|
235
|
+
.map(async (worker) => await this.offWork(worker.name, { wait: false })));
|
|
236
|
+
// Clean up all local group tracking on full stop
|
|
237
|
+
this.#localGroupConfig.clear();
|
|
238
|
+
this.#localGroupActive.clear();
|
|
239
|
+
}
|
|
240
|
+
async failWip() {
|
|
241
|
+
for (const worker of this.workers.values()) {
|
|
242
|
+
const jobIds = worker.jobs.map(j => j.id);
|
|
243
|
+
if (jobIds.length) {
|
|
244
|
+
await this.fail(worker.name, jobIds, 'pg-boss shut down while active');
|
|
245
|
+
}
|
|
246
|
+
worker.abort();
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
async work(name, ...args) {
|
|
250
|
+
const { options, callback } = Attorney.checkWorkArgs(name, args);
|
|
251
|
+
if (this.stopped) {
|
|
252
|
+
throw new Error('Workers are disabled. pg-boss is stopped');
|
|
253
|
+
}
|
|
254
|
+
const { pollingInterval: interval, batchSize = 1, includeMetadata = false, priority = true, localConcurrency = 1, localGroupConcurrency, groupConcurrency, orderByCreatedOn = true } = options;
|
|
255
|
+
if (localGroupConcurrency != null) {
|
|
256
|
+
this.#storeLocalGroupConfig(name, localGroupConcurrency);
|
|
257
|
+
}
|
|
258
|
+
const firstWorkerId = randomUUID({ disableEntropyCache: true });
|
|
259
|
+
const createWorker = (workerId) => {
|
|
260
|
+
const fetch = () => {
|
|
261
|
+
const ignoreGroups = localGroupConcurrency != null
|
|
262
|
+
? this.#getGroupsAtLocalCapacity(name)
|
|
263
|
+
: undefined;
|
|
264
|
+
return this.fetch(name, { batchSize, includeMetadata, priority, orderByCreatedOn, groupConcurrency, ignoreGroups });
|
|
265
|
+
};
|
|
266
|
+
const onFetch = async (jobs) => {
|
|
267
|
+
if (!jobs.length)
|
|
268
|
+
return;
|
|
269
|
+
if (this.config.__test__throw_worker)
|
|
270
|
+
throw new Error('__test__throw_worker');
|
|
271
|
+
this.emitWip(name);
|
|
272
|
+
this.#trackJobsActive(name, jobs);
|
|
273
|
+
// Get the worker instance for abort controller tracking
|
|
274
|
+
const worker = this.workers.get(workerId);
|
|
275
|
+
// Skip all in-memory group tracking when localGroupConcurrency is not enabled
|
|
276
|
+
if (localGroupConcurrency == null) {
|
|
277
|
+
await this.#processJobs(name, jobs, callback, worker);
|
|
278
|
+
}
|
|
279
|
+
else {
|
|
280
|
+
const { allowed, excess, groupedJobs } = this.#trackLocalGroupStart(name, jobs);
|
|
281
|
+
if (excess.length > 0) {
|
|
282
|
+
const excessIds = excess.map(job => job.id);
|
|
283
|
+
await this.restore(name, excessIds);
|
|
284
|
+
}
|
|
285
|
+
if (allowed.length > 0) {
|
|
286
|
+
try {
|
|
287
|
+
await this.#processJobs(name, allowed, callback, worker);
|
|
288
|
+
}
|
|
289
|
+
finally {
|
|
290
|
+
this.#trackLocalGroupEnd(name, groupedJobs);
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
this.emitWip(name);
|
|
295
|
+
};
|
|
296
|
+
const onError = (error) => {
|
|
297
|
+
this.emit(events.error, { ...error, message: error.message, stack: error.stack, queue: name, worker: workerId });
|
|
298
|
+
};
|
|
299
|
+
return new Worker({ id: workerId, name, options, interval, fetch, onFetch, onError });
|
|
300
|
+
};
|
|
301
|
+
// Spawn workers based on localConcurrency setting
|
|
302
|
+
for (let i = 0; i < localConcurrency; i++) {
|
|
303
|
+
const workerId = i === 0 ? firstWorkerId : randomUUID({ disableEntropyCache: true });
|
|
304
|
+
const worker = createWorker(workerId);
|
|
305
|
+
this.addWorker(worker);
|
|
306
|
+
worker.start();
|
|
307
|
+
}
|
|
308
|
+
return firstWorkerId;
|
|
309
|
+
}
|
|
310
|
+
addWorker(worker) {
|
|
311
|
+
this.workers.set(worker.id, worker);
|
|
312
|
+
}
|
|
313
|
+
removeWorker(worker) {
|
|
314
|
+
this.workers.delete(worker.id);
|
|
315
|
+
}
|
|
316
|
+
getWorkers() {
|
|
317
|
+
return Array.from(this.workers.values());
|
|
318
|
+
}
|
|
319
|
+
emitWip(name) {
|
|
320
|
+
if (!INTERNAL_QUEUES[name]) {
|
|
321
|
+
const now = Date.now();
|
|
322
|
+
if (now - this.wipTs > 2000) {
|
|
323
|
+
this.emit(events.wip, this.getWipData());
|
|
324
|
+
this.wipTs = now;
|
|
325
|
+
}
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
getWipData(options = {}) {
|
|
329
|
+
const { includeInternal = false } = options;
|
|
330
|
+
const data = this.getWorkers()
|
|
331
|
+
.map(i => i.toWipData())
|
|
332
|
+
.filter(i => i.state !== 'stopped' && (!INTERNAL_QUEUES[i.name] || includeInternal));
|
|
333
|
+
return data;
|
|
334
|
+
}
|
|
335
|
+
hasPendingCleanups() {
|
|
336
|
+
return this.pendingOffWorkCleanups.size > 0;
|
|
337
|
+
}
|
|
338
|
+
async offWork(name, options = { wait: true }) {
|
|
339
|
+
assert(name, 'queue name is required');
|
|
340
|
+
assert(typeof name === 'string', 'queue name must be a string');
|
|
341
|
+
const query = (i) => options?.id ? i.id === options.id : i.name === name;
|
|
342
|
+
const workers = this.getWorkers().filter(i => query(i) && !i.stopping && !i.stopped);
|
|
343
|
+
if (workers.length === 0) {
|
|
344
|
+
return;
|
|
345
|
+
}
|
|
346
|
+
const cleanupPromise = Promise.allSettled(workers.map(async (worker) => {
|
|
347
|
+
await worker.stop();
|
|
348
|
+
this.removeWorker(worker);
|
|
349
|
+
}));
|
|
350
|
+
if (options.wait) {
|
|
351
|
+
await cleanupPromise;
|
|
352
|
+
this.#cleanupLocalGroupTracking(name);
|
|
353
|
+
}
|
|
354
|
+
else {
|
|
355
|
+
this.pendingOffWorkCleanups.add(cleanupPromise);
|
|
356
|
+
cleanupPromise.finally(() => {
|
|
357
|
+
this.pendingOffWorkCleanups.delete(cleanupPromise);
|
|
358
|
+
this.#cleanupLocalGroupTracking(name);
|
|
359
|
+
});
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
notifyWorker(workerId) {
|
|
363
|
+
this.workers.get(workerId)?.notify();
|
|
364
|
+
}
|
|
365
|
+
async subscribe(event, name) {
|
|
366
|
+
assert(event, 'Missing required argument');
|
|
367
|
+
assert(name, 'Missing required argument');
|
|
368
|
+
const sql = plans.subscribe(this.config.schema);
|
|
369
|
+
await this.db.executeSql(sql, [event, name]);
|
|
370
|
+
}
|
|
371
|
+
async unsubscribe(event, name) {
|
|
372
|
+
assert(event, 'Missing required argument');
|
|
373
|
+
assert(name, 'Missing required argument');
|
|
374
|
+
const sql = plans.unsubscribe(this.config.schema);
|
|
375
|
+
await this.db.executeSql(sql, [event, name]);
|
|
376
|
+
}
|
|
377
|
+
async publish(event, data, options) {
|
|
378
|
+
assert(event, 'Missing required argument');
|
|
379
|
+
const sql = plans.getQueuesForEvent(this.config.schema);
|
|
380
|
+
const { rows } = await this.db.executeSql(sql, [event]);
|
|
381
|
+
await Promise.allSettled(rows.map(({ name }) => this.send(name, data, options)));
|
|
382
|
+
}
|
|
383
|
+
async send(...args) {
|
|
384
|
+
const result = Attorney.checkSendArgs(args);
|
|
385
|
+
return await this.createJob(result);
|
|
386
|
+
}
|
|
387
|
+
async sendAfter(name, data, options, after) {
|
|
388
|
+
options = options ? { ...options } : {};
|
|
389
|
+
options.startAfter = after;
|
|
390
|
+
const result = Attorney.checkSendArgs([name, data, options]);
|
|
391
|
+
return await this.createJob(result);
|
|
392
|
+
}
|
|
393
|
+
async sendThrottled(name, data, options, seconds, key) {
|
|
394
|
+
options = options ? { ...options } : {};
|
|
395
|
+
options.singletonSeconds = seconds;
|
|
396
|
+
options.singletonNextSlot = false;
|
|
397
|
+
options.singletonKey = key;
|
|
398
|
+
const result = Attorney.checkSendArgs([name, data, options]);
|
|
399
|
+
return await this.createJob(result);
|
|
400
|
+
}
|
|
401
|
+
async sendDebounced(name, data, options, seconds, key) {
|
|
402
|
+
options = options ? { ...options } : {};
|
|
403
|
+
options.singletonSeconds = seconds;
|
|
404
|
+
options.singletonNextSlot = true;
|
|
405
|
+
options.singletonKey = key;
|
|
406
|
+
const result = Attorney.checkSendArgs([name, data, options]);
|
|
407
|
+
return await this.createJob(result);
|
|
408
|
+
}
|
|
409
|
+
async createJob(request) {
|
|
410
|
+
const { name, data = null, options = {} } = request;
|
|
411
|
+
const { id = null, db: wrapper, priority, startAfter, singletonKey = null, singletonSeconds, singletonNextSlot, expireInSeconds, deleteAfterSeconds, retentionSeconds, keepUntil, retryLimit, retryDelay, retryBackoff, retryDelayMax, group } = options;
|
|
412
|
+
const job = {
|
|
413
|
+
id,
|
|
414
|
+
name,
|
|
415
|
+
data,
|
|
416
|
+
priority,
|
|
417
|
+
startAfter,
|
|
418
|
+
singletonKey,
|
|
419
|
+
singletonSeconds,
|
|
420
|
+
singletonOffset: 0,
|
|
421
|
+
groupId: group?.id ?? null,
|
|
422
|
+
groupTier: group?.tier ?? null,
|
|
423
|
+
expireInSeconds,
|
|
424
|
+
deleteAfterSeconds,
|
|
425
|
+
retentionSeconds,
|
|
426
|
+
keepUntil,
|
|
427
|
+
retryLimit,
|
|
428
|
+
retryDelay,
|
|
429
|
+
retryBackoff,
|
|
430
|
+
retryDelayMax
|
|
431
|
+
};
|
|
432
|
+
const db = wrapper || this.db;
|
|
433
|
+
const { table } = await this.getQueueCache(name);
|
|
434
|
+
const sql = plans.insertJobs(this.config.schema, { table, name, returnId: true });
|
|
435
|
+
const { rows: try1 } = await db.executeSql(sql, [JSON.stringify([job])]);
|
|
436
|
+
if (try1.length === 1) {
|
|
437
|
+
const jobId = try1[0].id;
|
|
438
|
+
if (this.config.__test__enableSpies) {
|
|
439
|
+
const spy = this.#spies.get(name);
|
|
440
|
+
if (spy) {
|
|
441
|
+
spy.addJob(jobId, name, data || {}, 'created');
|
|
442
|
+
}
|
|
443
|
+
}
|
|
444
|
+
return jobId;
|
|
445
|
+
}
|
|
446
|
+
if (singletonNextSlot) {
|
|
447
|
+
// delay starting by the offset to honor throttling config
|
|
448
|
+
job.startAfter = this.getDebounceStartAfter(singletonSeconds, this.timekeeper.clockSkew);
|
|
449
|
+
job.singletonOffset = singletonSeconds;
|
|
450
|
+
const { rows: try2 } = await db.executeSql(sql, [JSON.stringify([job])]);
|
|
451
|
+
if (try2.length === 1) {
|
|
452
|
+
const jobId = try2[0].id;
|
|
453
|
+
if (this.config.__test__enableSpies) {
|
|
454
|
+
const spy = this.#spies.get(name);
|
|
455
|
+
if (spy) {
|
|
456
|
+
spy.addJob(jobId, name, data || {}, 'created');
|
|
457
|
+
}
|
|
458
|
+
}
|
|
459
|
+
return jobId;
|
|
460
|
+
}
|
|
461
|
+
}
|
|
462
|
+
return null;
|
|
463
|
+
}
|
|
464
|
+
async insert(name, jobs, options = {}) {
|
|
465
|
+
assert(Array.isArray(jobs), 'jobs argument should be an array');
|
|
466
|
+
const { table } = await this.getQueueCache(name);
|
|
467
|
+
const db = this.assertDb(options);
|
|
468
|
+
const spy = this.config.__test__enableSpies ? this.#spies.get(name) : undefined;
|
|
469
|
+
// Return IDs if spy is active for this queue (needed for job tracking)
|
|
470
|
+
const returnId = !!spy || !!options.returnId;
|
|
471
|
+
const sql = plans.insertJobs(this.config.schema, { table, name, returnId });
|
|
472
|
+
const { rows } = await db.executeSql(sql, [JSON.stringify(jobs)]);
|
|
473
|
+
if (rows.length) {
|
|
474
|
+
if (spy) {
|
|
475
|
+
for (let i = 0; i < rows.length; i++) {
|
|
476
|
+
spy.addJob(rows[i].id, name, jobs[i].data || {}, 'created');
|
|
477
|
+
}
|
|
478
|
+
}
|
|
479
|
+
return rows.map((i) => i.id);
|
|
480
|
+
}
|
|
481
|
+
return null;
|
|
482
|
+
}
|
|
483
|
+
getDebounceStartAfter(singletonSeconds, clockOffset) {
|
|
484
|
+
const debounceInterval = singletonSeconds * 1000;
|
|
485
|
+
const now = Date.now() + clockOffset;
|
|
486
|
+
const slot = Math.floor(now / debounceInterval) * debounceInterval;
|
|
487
|
+
// prevent startAfter=0 during debouncing
|
|
488
|
+
let startAfter = (singletonSeconds - Math.floor((now - slot) / 1000)) || 1;
|
|
489
|
+
if (singletonSeconds > 1) {
|
|
490
|
+
startAfter++;
|
|
491
|
+
}
|
|
492
|
+
return startAfter;
|
|
493
|
+
}
|
|
494
|
+
async fetch(name, options = {}) {
|
|
495
|
+
Attorney.checkFetchArgs(name, options);
|
|
496
|
+
const db = this.assertDb(options);
|
|
497
|
+
const { table, policy, singletonsActive } = await this.getQueueCache(name);
|
|
498
|
+
const fetchOptions = {
|
|
499
|
+
...options,
|
|
500
|
+
schema: this.config.schema,
|
|
501
|
+
table,
|
|
502
|
+
name,
|
|
503
|
+
policy,
|
|
504
|
+
limit: options.batchSize || 1,
|
|
505
|
+
ignoreSingletons: singletonsActive
|
|
506
|
+
};
|
|
507
|
+
const query = plans.fetchNextJob(fetchOptions);
|
|
508
|
+
let result;
|
|
509
|
+
try {
|
|
510
|
+
result = await db.executeSql(query.text, query.values);
|
|
511
|
+
}
|
|
512
|
+
catch (err) {
|
|
513
|
+
// errors from fetchquery should only be unique constraint violations
|
|
514
|
+
}
|
|
515
|
+
return result?.rows || [];
|
|
516
|
+
}
|
|
517
|
+
mapCompletionIdArg(id, funcName) {
|
|
518
|
+
const errorMessage = `${funcName}() requires an id`;
|
|
519
|
+
assert(id, errorMessage);
|
|
520
|
+
const ids = Array.isArray(id) ? id : [id];
|
|
521
|
+
assert(ids.length, errorMessage);
|
|
522
|
+
return ids;
|
|
523
|
+
}
|
|
524
|
+
mapCompletionDataArg(data) {
|
|
525
|
+
if (data === null || typeof data === 'undefined' || typeof data === 'function') {
|
|
526
|
+
return null;
|
|
527
|
+
}
|
|
528
|
+
const result = (typeof data === 'object' && !Array.isArray(data))
|
|
529
|
+
? data
|
|
530
|
+
: { value: data };
|
|
531
|
+
return stringify(result);
|
|
532
|
+
}
|
|
533
|
+
mapCommandResponse(ids, result) {
|
|
534
|
+
return {
|
|
535
|
+
jobs: ids,
|
|
536
|
+
requested: ids.length,
|
|
537
|
+
affected: result && result.rows ? parseInt(result.rows[0].count) : 0
|
|
538
|
+
};
|
|
539
|
+
}
|
|
540
|
+
async complete(name, id, data, options = {}) {
|
|
541
|
+
Attorney.assertQueueName(name);
|
|
542
|
+
const db = this.assertDb(options);
|
|
543
|
+
const ids = this.mapCompletionIdArg(id, 'complete');
|
|
544
|
+
const { table } = await this.getQueueCache(name);
|
|
545
|
+
const sql = plans.completeJobs(this.config.schema, table);
|
|
546
|
+
const result = await db.executeSql(sql, [name, ids, this.mapCompletionDataArg(data)]);
|
|
547
|
+
return this.mapCommandResponse(ids, result);
|
|
548
|
+
}
|
|
549
|
+
async fail(name, id, data, options = {}) {
|
|
550
|
+
Attorney.assertQueueName(name);
|
|
551
|
+
const db = this.assertDb(options);
|
|
552
|
+
const ids = this.mapCompletionIdArg(id, 'fail');
|
|
553
|
+
const { table } = await this.getQueueCache(name);
|
|
554
|
+
const sql = plans.failJobsById(this.config.schema, table);
|
|
555
|
+
const result = await db.executeSql(sql, [name, ids, this.mapCompletionDataArg(data)]);
|
|
556
|
+
return this.mapCommandResponse(ids, result);
|
|
557
|
+
}
|
|
558
|
+
async deleteJob(name, id, options = {}) {
|
|
559
|
+
Attorney.assertQueueName(name);
|
|
560
|
+
const db = this.assertDb(options);
|
|
561
|
+
const ids = this.mapCompletionIdArg(id, 'deleteJob');
|
|
562
|
+
const { table } = await this.getQueueCache(name);
|
|
563
|
+
const sql = plans.deleteJobsById(this.config.schema, table);
|
|
564
|
+
const result = await db.executeSql(sql, [name, ids]);
|
|
565
|
+
return this.mapCommandResponse(ids, result);
|
|
566
|
+
}
|
|
567
|
+
async cancel(name, id, options = {}) {
|
|
568
|
+
Attorney.assertQueueName(name);
|
|
569
|
+
const db = this.assertDb(options);
|
|
570
|
+
const ids = this.mapCompletionIdArg(id, 'cancel');
|
|
571
|
+
const { table } = await this.getQueueCache(name);
|
|
572
|
+
const sql = plans.cancelJobs(this.config.schema, table);
|
|
573
|
+
const result = await db.executeSql(sql, [name, ids]);
|
|
574
|
+
return this.mapCommandResponse(ids, result);
|
|
575
|
+
}
|
|
576
|
+
async resume(name, id, options = {}) {
|
|
577
|
+
Attorney.assertQueueName(name);
|
|
578
|
+
const db = this.assertDb(options);
|
|
579
|
+
const ids = this.mapCompletionIdArg(id, 'resume');
|
|
580
|
+
const { table } = await this.getQueueCache(name);
|
|
581
|
+
const sql = plans.resumeJobs(this.config.schema, table);
|
|
582
|
+
const result = await db.executeSql(sql, [name, ids]);
|
|
583
|
+
return this.mapCommandResponse(ids, result);
|
|
584
|
+
}
|
|
585
|
+
async restore(name, id, options = {}) {
|
|
586
|
+
Attorney.assertQueueName(name);
|
|
587
|
+
const db = this.assertDb(options);
|
|
588
|
+
const ids = this.mapCompletionIdArg(id, 'restore');
|
|
589
|
+
const { table } = await this.getQueueCache(name);
|
|
590
|
+
const sql = plans.restoreJobs(this.config.schema, table);
|
|
591
|
+
await db.executeSql(sql, [name, ids]);
|
|
592
|
+
}
|
|
593
|
+
async retry(name, id, options = {}) {
|
|
594
|
+
Attorney.assertQueueName(name);
|
|
595
|
+
const db = options.db || this.db;
|
|
596
|
+
const ids = this.mapCompletionIdArg(id, 'retry');
|
|
597
|
+
const { table } = await this.getQueueCache(name);
|
|
598
|
+
const sql = plans.retryJobs(this.config.schema, table);
|
|
599
|
+
const result = await db.executeSql(sql, [name, ids]);
|
|
600
|
+
return this.mapCommandResponse(ids, result);
|
|
601
|
+
}
|
|
602
|
+
async createQueue(name, options = {}) {
|
|
603
|
+
name = name || options.name;
|
|
604
|
+
Attorney.assertQueueName(name);
|
|
605
|
+
const policy = options.policy || plans.QUEUE_POLICIES.standard;
|
|
606
|
+
assert(policy in plans.QUEUE_POLICIES, `${policy} is not a valid queue policy`);
|
|
607
|
+
Attorney.validateQueueArgs(options);
|
|
608
|
+
if (options.deadLetter) {
|
|
609
|
+
Attorney.assertQueueName(options.deadLetter);
|
|
610
|
+
notStrictEqual(name, options.deadLetter, 'deadLetter cannot be itself');
|
|
611
|
+
await this.getQueueCache(options.deadLetter);
|
|
612
|
+
}
|
|
613
|
+
const sql = plans.createQueue(this.config.schema, name, { ...options, policy });
|
|
614
|
+
await this.db.executeSql(sql);
|
|
615
|
+
}
|
|
616
|
+
async getQueues(names) {
|
|
617
|
+
names = Array.isArray(names) ? names : typeof names === 'string' ? [names] : undefined;
|
|
618
|
+
if (names) {
|
|
619
|
+
for (const name of names) {
|
|
620
|
+
Attorney.assertQueueName(name);
|
|
621
|
+
}
|
|
622
|
+
}
|
|
623
|
+
const query = plans.getQueues(this.config.schema, names);
|
|
624
|
+
const { rows } = await this.db.executeSql(query.text, query.values);
|
|
625
|
+
return rows;
|
|
626
|
+
}
|
|
627
|
+
async updateQueue(name, options = {}) {
|
|
628
|
+
Attorney.assertQueueName(name);
|
|
629
|
+
assert(Object.keys(options).length > 0, 'no properties found to update');
|
|
630
|
+
if ('policy' in options) {
|
|
631
|
+
throw new Error('queue policy cannot be changed after creation');
|
|
632
|
+
}
|
|
633
|
+
if ('partition' in options) {
|
|
634
|
+
throw new Error('queue partitioning cannot be changed after creation');
|
|
635
|
+
}
|
|
636
|
+
Attorney.validateQueueArgs(options);
|
|
637
|
+
const { deadLetter } = options;
|
|
638
|
+
if (deadLetter) {
|
|
639
|
+
Attorney.assertQueueName(deadLetter);
|
|
640
|
+
notStrictEqual(name, deadLetter, 'deadLetter cannot be itself');
|
|
641
|
+
}
|
|
642
|
+
const sql = plans.updateQueue(this.config.schema, { deadLetter });
|
|
643
|
+
await this.db.executeSql(sql, [name, options]);
|
|
644
|
+
}
|
|
645
|
+
async getQueue(name) {
|
|
646
|
+
Attorney.assertQueueName(name);
|
|
647
|
+
const query = plans.getQueues(this.config.schema, [name]);
|
|
648
|
+
const { rows } = await this.db.executeSql(query.text, query.values);
|
|
649
|
+
return rows[0] || null;
|
|
650
|
+
}
|
|
651
|
+
async deleteQueue(name) {
|
|
652
|
+
Attorney.assertQueueName(name);
|
|
653
|
+
try {
|
|
654
|
+
await this.getQueueCache(name);
|
|
655
|
+
const sql = plans.deleteQueue(this.config.schema, name);
|
|
656
|
+
await this.db.executeSql(sql);
|
|
657
|
+
}
|
|
658
|
+
catch { }
|
|
659
|
+
}
|
|
660
|
+
async deleteQueuedJobs(name) {
|
|
661
|
+
Attorney.assertQueueName(name);
|
|
662
|
+
const { table } = await this.getQueueCache(name);
|
|
663
|
+
const sql = plans.deleteQueuedJobs(this.config.schema, table);
|
|
664
|
+
await this.db.executeSql(sql, [name]);
|
|
665
|
+
}
|
|
666
|
+
async deleteStoredJobs(name) {
|
|
667
|
+
Attorney.assertQueueName(name);
|
|
668
|
+
const { table } = await this.getQueueCache(name);
|
|
669
|
+
const sql = plans.deleteStoredJobs(this.config.schema, table);
|
|
670
|
+
await this.db.executeSql(sql, [name]);
|
|
671
|
+
}
|
|
672
|
+
async deleteAllJobs(name) {
|
|
673
|
+
if (!name) {
|
|
674
|
+
const sql = plans.truncateTable(this.config.schema, 'job');
|
|
675
|
+
await this.db.executeSql(sql);
|
|
676
|
+
return;
|
|
677
|
+
}
|
|
678
|
+
Attorney.assertQueueName(name);
|
|
679
|
+
const { table, partition } = await this.getQueueCache(name);
|
|
680
|
+
if (partition) {
|
|
681
|
+
const sql = plans.truncateTable(this.config.schema, table);
|
|
682
|
+
await this.db.executeSql(sql);
|
|
683
|
+
}
|
|
684
|
+
else {
|
|
685
|
+
const sql = plans.deleteAllJobs(this.config.schema, table);
|
|
686
|
+
await this.db.executeSql(sql, [name]);
|
|
687
|
+
}
|
|
688
|
+
}
|
|
689
|
+
async getQueueStats(name) {
|
|
690
|
+
Attorney.assertQueueName(name);
|
|
691
|
+
const queue = await this.getQueueCache(name);
|
|
692
|
+
const query = plans.getQueueStats(this.config.schema, queue.table, [name]);
|
|
693
|
+
const { rows } = await this.db.executeSql(query.text, query.values);
|
|
694
|
+
return Object.assign(queue, rows.at(0) ||
|
|
695
|
+
{
|
|
696
|
+
deferredCount: 0,
|
|
697
|
+
queuedCount: 0,
|
|
698
|
+
activeCount: 0,
|
|
699
|
+
totalCount: 0
|
|
700
|
+
});
|
|
701
|
+
}
|
|
702
|
+
async getJobById(name, id, options = {}) {
|
|
703
|
+
Attorney.assertQueueName(name);
|
|
704
|
+
const db = this.assertDb(options);
|
|
705
|
+
const { table } = await this.getQueueCache(name);
|
|
706
|
+
const sql = plans.getJobById(this.config.schema, table);
|
|
707
|
+
const result1 = await db.executeSql(sql, [name, id]);
|
|
708
|
+
if (result1?.rows?.length === 1) {
|
|
709
|
+
return result1.rows[0];
|
|
710
|
+
}
|
|
711
|
+
else {
|
|
712
|
+
return null;
|
|
713
|
+
}
|
|
714
|
+
}
|
|
715
|
+
async findJobs(name, options = {}) {
|
|
716
|
+
Attorney.assertQueueName(name);
|
|
717
|
+
const db = this.assertDb(options);
|
|
718
|
+
const { table } = await this.getQueueCache(name);
|
|
719
|
+
const { id, key, data, queued = false } = options;
|
|
720
|
+
const sql = plans.findJobs(this.config.schema, table, {
|
|
721
|
+
byId: id !== undefined,
|
|
722
|
+
byKey: key !== undefined,
|
|
723
|
+
byData: data !== undefined,
|
|
724
|
+
queued
|
|
725
|
+
});
|
|
726
|
+
const values = [name];
|
|
727
|
+
if (id !== undefined)
|
|
728
|
+
values.push(id);
|
|
729
|
+
if (key !== undefined)
|
|
730
|
+
values.push(key);
|
|
731
|
+
if (data !== undefined)
|
|
732
|
+
values.push(JSON.stringify(data));
|
|
733
|
+
const result = await db.executeSql(sql, values);
|
|
734
|
+
return result?.rows || [];
|
|
735
|
+
}
|
|
736
|
+
assertDb(options) {
|
|
737
|
+
if (options.db) {
|
|
738
|
+
return options.db;
|
|
739
|
+
}
|
|
740
|
+
if (this.db._pgbdb) {
|
|
741
|
+
assert(this.db.opened, 'Database connection is not opened');
|
|
742
|
+
}
|
|
743
|
+
return this.db;
|
|
744
|
+
}
|
|
745
|
+
}
|
|
746
|
+
export default Manager;
|