pg-boss 6.2.2-beta → 7.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -9
- package/package.json +4 -3
- package/src/attorney.js +6 -7
- package/src/boss.js +8 -8
- package/src/manager.js +120 -57
- package/src/migrationStore.js +17 -0
- package/src/plans.js +41 -0
- package/src/timekeeper.js +8 -8
- package/types.d.ts +36 -30
- package/version.json +1 -1
package/README.md
CHANGED
|
@@ -16,11 +16,11 @@ async function readme() {
|
|
|
16
16
|
|
|
17
17
|
const queue = 'some-queue';
|
|
18
18
|
|
|
19
|
-
let jobId = await boss.
|
|
19
|
+
let jobId = await boss.send(queue, { param1: 'foo' })
|
|
20
20
|
|
|
21
21
|
console.log(`created job in queue ${queue}: ${jobId}`);
|
|
22
22
|
|
|
23
|
-
await boss.
|
|
23
|
+
await boss.work(queue, someAsyncJobHandler);
|
|
24
24
|
}
|
|
25
25
|
|
|
26
26
|
async function someAsyncJobHandler(job) {
|
|
@@ -38,15 +38,15 @@ pg-boss relies on [SKIP LOCKED](http://blog.2ndquadrant.com/what-is-select-skip-
|
|
|
38
38
|
This will likely cater the most to teams already familiar with the simplicity of relational database semantics and operations (SQL, querying, and backups). It will be especially useful to those already relying on PostgreSQL that want to limit how many systems are required to monitor and support in their architecture.
|
|
39
39
|
|
|
40
40
|
## Features
|
|
41
|
-
* Backpressure-compatible
|
|
41
|
+
* Backpressure-compatible workers for polling queues
|
|
42
42
|
* Distributed cron-based job scheduling with database clock synchronization
|
|
43
|
+
* Pub/sub API for fan-out queue relationships
|
|
43
44
|
* Job deferral, retries (with exponential backoff), throttling, rate limiting, debouncing
|
|
44
|
-
* Job completion
|
|
45
|
-
* Direct
|
|
46
|
-
* Batching API for chunked job fetching
|
|
45
|
+
* Job completion hooks for orchestrations/sagas
|
|
46
|
+
* Direct send, fetch and completion APIs for custom integrations
|
|
47
47
|
* Direct table access for bulk loads via COPY or INSERT
|
|
48
48
|
* Multi-master compatible when running multiple instances (for example, in a Kubernetes ReplicaSet)
|
|
49
|
-
* Automatic provisioning of required storage
|
|
49
|
+
* Automatic provisioning of required storage
|
|
50
50
|
* Automatic maintenance operations to manage table growth
|
|
51
51
|
|
|
52
52
|
## Requirements
|
|
@@ -64,8 +64,7 @@ yarn add pg-boss
|
|
|
64
64
|
```
|
|
65
65
|
|
|
66
66
|
## Documentation
|
|
67
|
-
* [
|
|
68
|
-
* [Configuration](docs/configuration.md)
|
|
67
|
+
* [Docs](docs/readme.md)
|
|
69
68
|
|
|
70
69
|
## Contributing
|
|
71
70
|
|
package/package.json
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pg-boss",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "7.0.1",
|
|
4
4
|
"description": "Queueing jobs in Node.js using PostgreSQL like a boss",
|
|
5
5
|
"main": "./src/index.js",
|
|
6
6
|
"engines": {
|
|
7
7
|
"node": ">=12.0.0"
|
|
8
8
|
},
|
|
9
9
|
"dependencies": {
|
|
10
|
-
"cron-parser": "^
|
|
10
|
+
"cron-parser": "^4.0.0",
|
|
11
11
|
"delay": "^5.0.0",
|
|
12
12
|
"lodash.debounce": "^4.0.8",
|
|
13
13
|
"p-map": "^4.0.0",
|
|
@@ -27,7 +27,8 @@
|
|
|
27
27
|
"forcover": "npm run cover && nyc report --reporter=text-lcov | coveralls",
|
|
28
28
|
"export-schema": "node ./scripts/construct.js",
|
|
29
29
|
"export-migration": "node ./scripts/migrate.js",
|
|
30
|
-
"export-rollback": "node ./scripts/rollback.js"
|
|
30
|
+
"export-rollback": "node ./scripts/rollback.js",
|
|
31
|
+
"tsc": "tsc --noEmit types.d.ts"
|
|
31
32
|
},
|
|
32
33
|
"mocha": {
|
|
33
34
|
"timeout": 10000,
|
package/src/attorney.js
CHANGED
|
@@ -3,8 +3,8 @@ const { DEFAULT_SCHEMA } = require('./plans')
|
|
|
3
3
|
|
|
4
4
|
module.exports = {
|
|
5
5
|
getConfig,
|
|
6
|
-
|
|
7
|
-
|
|
6
|
+
checkSendArgs,
|
|
7
|
+
checkWorkArgs,
|
|
8
8
|
checkFetchArgs,
|
|
9
9
|
warnClockSkew
|
|
10
10
|
}
|
|
@@ -24,18 +24,18 @@ const WARNINGS = {
|
|
|
24
24
|
}
|
|
25
25
|
}
|
|
26
26
|
|
|
27
|
-
function
|
|
27
|
+
function checkSendArgs (args, defaults) {
|
|
28
28
|
let name, data, options
|
|
29
29
|
|
|
30
30
|
if (typeof args[0] === 'string') {
|
|
31
31
|
name = args[0]
|
|
32
32
|
data = args[1]
|
|
33
33
|
|
|
34
|
-
assert(typeof data !== 'function', '
|
|
34
|
+
assert(typeof data !== 'function', 'send() cannot accept a function as the payload. Did you intend to use work()?')
|
|
35
35
|
|
|
36
36
|
options = args[2]
|
|
37
37
|
} else if (typeof args[0] === 'object') {
|
|
38
|
-
assert(args.length === 1, '
|
|
38
|
+
assert(args.length === 1, 'send object API only accepts 1 argument')
|
|
39
39
|
|
|
40
40
|
const job = args[0]
|
|
41
41
|
|
|
@@ -84,7 +84,7 @@ function checkPublishArgs (args, defaults) {
|
|
|
84
84
|
return { name, data, options }
|
|
85
85
|
}
|
|
86
86
|
|
|
87
|
-
function
|
|
87
|
+
function checkWorkArgs (name, args, defaults) {
|
|
88
88
|
let options, callback
|
|
89
89
|
|
|
90
90
|
assert(name, 'missing job name')
|
|
@@ -145,7 +145,6 @@ function getConfig (value) {
|
|
|
145
145
|
applyMonitoringConfig(config)
|
|
146
146
|
applyUuidConfig(config)
|
|
147
147
|
|
|
148
|
-
// defaults for publish and subscribe
|
|
149
148
|
applyNewJobCheckInterval(config)
|
|
150
149
|
applyExpirationConfig(config)
|
|
151
150
|
applyRetentionConfig(config)
|
package/src/boss.js
CHANGED
|
@@ -56,11 +56,11 @@ class Boss extends EventEmitter {
|
|
|
56
56
|
|
|
57
57
|
await this.maintenanceAsync()
|
|
58
58
|
|
|
59
|
-
const
|
|
59
|
+
const maintenanceWorkOptions = {
|
|
60
60
|
newJobCheckIntervalSeconds: Math.max(1, this.maintenanceIntervalSeconds / 2)
|
|
61
61
|
}
|
|
62
62
|
|
|
63
|
-
await this.manager.
|
|
63
|
+
await this.manager.work(queues.MAINTENANCE, maintenanceWorkOptions, (job) => this.onMaintenance(job))
|
|
64
64
|
|
|
65
65
|
if (this.monitorStates) {
|
|
66
66
|
await this.manager.deleteQueue(COMPLETION_JOB_PREFIX + queues.MONITOR_STATES)
|
|
@@ -68,11 +68,11 @@ class Boss extends EventEmitter {
|
|
|
68
68
|
|
|
69
69
|
await this.monitorStatesAsync()
|
|
70
70
|
|
|
71
|
-
const
|
|
71
|
+
const monitorStatesWorkOptions = {
|
|
72
72
|
newJobCheckIntervalSeconds: Math.max(1, this.monitorIntervalSeconds / 2)
|
|
73
73
|
}
|
|
74
74
|
|
|
75
|
-
await this.manager.
|
|
75
|
+
await this.manager.work(queues.MONITOR_STATES, monitorStatesWorkOptions, (job) => this.onMonitorStates(job))
|
|
76
76
|
}
|
|
77
77
|
}
|
|
78
78
|
|
|
@@ -97,7 +97,7 @@ class Boss extends EventEmitter {
|
|
|
97
97
|
onComplete: false
|
|
98
98
|
}
|
|
99
99
|
|
|
100
|
-
await this.manager.
|
|
100
|
+
await this.manager.send(queues.MAINTENANCE, null, options)
|
|
101
101
|
}
|
|
102
102
|
|
|
103
103
|
async monitorStatesAsync (options = {}) {
|
|
@@ -110,7 +110,7 @@ class Boss extends EventEmitter {
|
|
|
110
110
|
onComplete: false
|
|
111
111
|
}
|
|
112
112
|
|
|
113
|
-
await this.manager.
|
|
113
|
+
await this.manager.send(queues.MONITOR_STATES, null, options)
|
|
114
114
|
}
|
|
115
115
|
|
|
116
116
|
async onMaintenance (job) {
|
|
@@ -169,10 +169,10 @@ class Boss extends EventEmitter {
|
|
|
169
169
|
clearInterval(this.metaMonitorInterval)
|
|
170
170
|
}
|
|
171
171
|
|
|
172
|
-
await this.manager.
|
|
172
|
+
await this.manager.offWork(queues.MAINTENANCE)
|
|
173
173
|
|
|
174
174
|
if (this.monitorStates) {
|
|
175
|
-
await this.manager.
|
|
175
|
+
await this.manager.offWork(queues.MONITOR_STATES)
|
|
176
176
|
}
|
|
177
177
|
|
|
178
178
|
this.stopped = true
|
package/src/manager.js
CHANGED
|
@@ -24,6 +24,23 @@ const events = {
|
|
|
24
24
|
wip: 'wip'
|
|
25
25
|
}
|
|
26
26
|
|
|
27
|
+
const resolveWithinSeconds = async (promise, seconds) => {
|
|
28
|
+
const timeout = Math.max(1, seconds) * 1000
|
|
29
|
+
const reject = delay.reject(timeout, { value: new Error(`handler execution exceeded ${timeout}ms`) })
|
|
30
|
+
|
|
31
|
+
let result
|
|
32
|
+
|
|
33
|
+
try {
|
|
34
|
+
result = await Promise.race([promise, reject])
|
|
35
|
+
} finally {
|
|
36
|
+
try {
|
|
37
|
+
reject.clear()
|
|
38
|
+
} catch {}
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
return result
|
|
42
|
+
}
|
|
43
|
+
|
|
27
44
|
class Manager extends EventEmitter {
|
|
28
45
|
constructor (db, config) {
|
|
29
46
|
super()
|
|
@@ -32,7 +49,7 @@ class Manager extends EventEmitter {
|
|
|
32
49
|
this.db = db
|
|
33
50
|
|
|
34
51
|
this.events = events
|
|
35
|
-
this.
|
|
52
|
+
this.workers = new Map()
|
|
36
53
|
|
|
37
54
|
this.nextJobCommand = plans.fetchNextJob(config.schema)
|
|
38
55
|
this.insertJobCommand = plans.insertJob(config.schema)
|
|
@@ -42,25 +59,31 @@ class Manager extends EventEmitter {
|
|
|
42
59
|
this.failJobsCommand = plans.failJobs(config.schema)
|
|
43
60
|
this.getJobByIdCommand = plans.getJobById(config.schema)
|
|
44
61
|
this.getArchivedJobByIdCommand = plans.getArchivedJobById(config.schema)
|
|
62
|
+
this.subscribeCommand = plans.subscribe(config.schema)
|
|
63
|
+
this.unsubscribeCommand = plans.unsubscribe(config.schema)
|
|
64
|
+
this.getQueuesForEventCommand = plans.getQueuesForEvent(config.schema)
|
|
45
65
|
|
|
46
66
|
// exported api to index
|
|
47
67
|
this.functions = [
|
|
48
|
-
this.fetch,
|
|
49
68
|
this.complete,
|
|
50
69
|
this.cancel,
|
|
51
70
|
this.fail,
|
|
71
|
+
this.fetch,
|
|
72
|
+
this.fetchCompleted,
|
|
73
|
+
this.work,
|
|
74
|
+
this.offWork,
|
|
75
|
+
this.onComplete,
|
|
76
|
+
this.offComplete,
|
|
52
77
|
this.publish,
|
|
53
|
-
this.insert,
|
|
54
78
|
this.subscribe,
|
|
55
79
|
this.unsubscribe,
|
|
56
|
-
this.
|
|
57
|
-
this.
|
|
58
|
-
this.
|
|
59
|
-
this.
|
|
60
|
-
this.
|
|
61
|
-
this.
|
|
62
|
-
this.
|
|
63
|
-
this.publishSingleton,
|
|
80
|
+
this.insert,
|
|
81
|
+
this.send,
|
|
82
|
+
this.sendDebounced,
|
|
83
|
+
this.sendThrottled,
|
|
84
|
+
this.sendOnce,
|
|
85
|
+
this.sendAfter,
|
|
86
|
+
this.sendSingleton,
|
|
64
87
|
this.deleteQueue,
|
|
65
88
|
this.deleteAllQueues,
|
|
66
89
|
this.clearStorage,
|
|
@@ -78,33 +101,33 @@ class Manager extends EventEmitter {
|
|
|
78
101
|
async stop () {
|
|
79
102
|
this.stopping = true
|
|
80
103
|
|
|
81
|
-
for (const sub of this.
|
|
104
|
+
for (const sub of this.workers.values()) {
|
|
82
105
|
if (!INTERNAL_QUEUES[sub.name]) {
|
|
83
|
-
await this.
|
|
106
|
+
await this.offWork(sub.name)
|
|
84
107
|
}
|
|
85
108
|
}
|
|
86
109
|
}
|
|
87
110
|
|
|
88
|
-
async
|
|
89
|
-
const { options, callback } = Attorney.
|
|
111
|
+
async work (name, ...args) {
|
|
112
|
+
const { options, callback } = Attorney.checkWorkArgs(name, args, this.config)
|
|
90
113
|
return await this.watch(name, options, callback)
|
|
91
114
|
}
|
|
92
115
|
|
|
93
116
|
async onComplete (name, ...args) {
|
|
94
|
-
const { options, callback } = Attorney.
|
|
117
|
+
const { options, callback } = Attorney.checkWorkArgs(name, args, this.config)
|
|
95
118
|
return await this.watch(COMPLETION_JOB_PREFIX + name, options, callback)
|
|
96
119
|
}
|
|
97
120
|
|
|
98
121
|
addWorker (worker) {
|
|
99
|
-
this.
|
|
122
|
+
this.workers.set(worker.id, worker)
|
|
100
123
|
}
|
|
101
124
|
|
|
102
125
|
removeWorker (worker) {
|
|
103
|
-
this.
|
|
126
|
+
this.workers.delete(worker.id)
|
|
104
127
|
}
|
|
105
128
|
|
|
106
129
|
getWorkers () {
|
|
107
|
-
return Array.from(this.
|
|
130
|
+
return Array.from(this.workers.values())
|
|
108
131
|
}
|
|
109
132
|
|
|
110
133
|
emitWip (name) {
|
|
@@ -149,7 +172,7 @@ class Manager extends EventEmitter {
|
|
|
149
172
|
|
|
150
173
|
async watch (name, options, callback) {
|
|
151
174
|
if (this.stopping) {
|
|
152
|
-
throw new Error('
|
|
175
|
+
throw new Error('Workers are disabled. pg-boss is stopping.')
|
|
153
176
|
}
|
|
154
177
|
|
|
155
178
|
const {
|
|
@@ -157,57 +180,71 @@ class Manager extends EventEmitter {
|
|
|
157
180
|
batchSize,
|
|
158
181
|
teamSize = 1,
|
|
159
182
|
teamConcurrency = 1,
|
|
183
|
+
teamRefill: refill = false,
|
|
160
184
|
includeMetadata = false
|
|
161
185
|
} = options
|
|
162
186
|
|
|
163
187
|
const id = uuid.v4()
|
|
164
188
|
|
|
165
|
-
|
|
189
|
+
let queueSize = 0
|
|
166
190
|
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
throw new Error('__test__throw_subscription')
|
|
170
|
-
}
|
|
191
|
+
let refillTeamPromise
|
|
192
|
+
let resolveRefillTeam
|
|
171
193
|
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
194
|
+
// Setup a promise that onFetch can await for when at least one
|
|
195
|
+
// job is finished and so the team is ready to be topped up
|
|
196
|
+
const createTeamRefillPromise = () => {
|
|
197
|
+
refillTeamPromise = new Promise((resolve) => { resolveRefillTeam = resolve })
|
|
198
|
+
}
|
|
175
199
|
|
|
176
|
-
|
|
200
|
+
createTeamRefillPromise()
|
|
177
201
|
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
} catch {}
|
|
184
|
-
}
|
|
202
|
+
const onRefill = () => {
|
|
203
|
+
queueSize--
|
|
204
|
+
resolveRefillTeam()
|
|
205
|
+
createTeamRefillPromise()
|
|
206
|
+
}
|
|
185
207
|
|
|
186
|
-
|
|
208
|
+
const fetch = () => this.fetch(name, batchSize || (teamSize - queueSize), { includeMetadata })
|
|
209
|
+
|
|
210
|
+
const onFetch = async (jobs) => {
|
|
211
|
+
if (this.config.__test__throw_worker) {
|
|
212
|
+
throw new Error('__test__throw_worker')
|
|
187
213
|
}
|
|
188
214
|
|
|
189
215
|
this.emitWip(name)
|
|
190
216
|
|
|
191
|
-
let result
|
|
192
|
-
|
|
193
217
|
if (batchSize) {
|
|
194
218
|
const maxExpiration = jobs.reduce((acc, i) => Math.max(acc, i.expire_in_seconds), 0)
|
|
195
219
|
|
|
196
220
|
// Failing will fail all fetched jobs
|
|
197
|
-
|
|
221
|
+
await resolveWithinSeconds(Promise.all([callback(jobs)]), maxExpiration)
|
|
198
222
|
.catch(err => this.fail(jobs.map(job => job.id), err))
|
|
199
223
|
} else {
|
|
200
|
-
|
|
224
|
+
if (refill) {
|
|
225
|
+
queueSize += jobs.length || 1
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
const allTeamPromise = pMap(jobs, job =>
|
|
201
229
|
resolveWithinSeconds(callback(job), job.expire_in_seconds)
|
|
202
230
|
.then(result => this.complete(job.id, result))
|
|
203
231
|
.catch(err => this.fail(job.id, err))
|
|
232
|
+
.then(() => refill ? onRefill() : null)
|
|
204
233
|
, { concurrency: teamConcurrency }
|
|
205
234
|
).catch(() => {}) // allow promises & non-promises to live together in harmony
|
|
235
|
+
|
|
236
|
+
if (refill) {
|
|
237
|
+
if (queueSize < teamSize) {
|
|
238
|
+
return
|
|
239
|
+
} else {
|
|
240
|
+
await refillTeamPromise
|
|
241
|
+
}
|
|
242
|
+
} else {
|
|
243
|
+
await allTeamPromise
|
|
244
|
+
}
|
|
206
245
|
}
|
|
207
246
|
|
|
208
247
|
this.emitWip(name)
|
|
209
|
-
|
|
210
|
-
return result
|
|
211
248
|
}
|
|
212
249
|
|
|
213
250
|
const onError = error => {
|
|
@@ -223,7 +260,7 @@ class Manager extends EventEmitter {
|
|
|
223
260
|
return id
|
|
224
261
|
}
|
|
225
262
|
|
|
226
|
-
async
|
|
263
|
+
async offWork (value) {
|
|
227
264
|
assert(value, 'Missing required argument')
|
|
228
265
|
|
|
229
266
|
const query = (typeof value === 'string')
|
|
@@ -255,66 +292,92 @@ class Manager extends EventEmitter {
|
|
|
255
292
|
})
|
|
256
293
|
}
|
|
257
294
|
|
|
295
|
+
async subscribe (event, name) {
|
|
296
|
+
assert(event, 'Missing required argument')
|
|
297
|
+
assert(name, 'Missing required argument')
|
|
298
|
+
|
|
299
|
+
return await this.db.executeSql(this.subscribeCommand, [event, name])
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
async unsubscribe (event, name) {
|
|
303
|
+
assert(event, 'Missing required argument')
|
|
304
|
+
assert(name, 'Missing required argument')
|
|
305
|
+
|
|
306
|
+
return await this.db.executeSql(this.unsubscribeCommand, [event, name])
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
async publish (event, ...args) {
|
|
310
|
+
assert(event, 'Missing required argument')
|
|
311
|
+
|
|
312
|
+
const result = await this.db.executeSql(this.getQueuesForEventCommand, [event])
|
|
313
|
+
|
|
314
|
+
if (!result || result.rowCount === 0) {
|
|
315
|
+
return []
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
return await Promise.all(result.rows.map(({ name }) => this.send(name, ...args)))
|
|
319
|
+
}
|
|
320
|
+
|
|
258
321
|
async offComplete (value) {
|
|
259
322
|
if (typeof value === 'string') {
|
|
260
323
|
value = COMPLETION_JOB_PREFIX + value
|
|
261
324
|
}
|
|
262
325
|
|
|
263
|
-
return await this.
|
|
326
|
+
return await this.offWork(value)
|
|
264
327
|
}
|
|
265
328
|
|
|
266
|
-
async
|
|
267
|
-
const { name, data, options } = Attorney.
|
|
329
|
+
async send (...args) {
|
|
330
|
+
const { name, data, options } = Attorney.checkSendArgs(args, this.config)
|
|
268
331
|
return await this.createJob(name, data, options)
|
|
269
332
|
}
|
|
270
333
|
|
|
271
|
-
async
|
|
334
|
+
async sendOnce (name, data, options, key) {
|
|
272
335
|
options = options || {}
|
|
273
336
|
|
|
274
337
|
options.singletonKey = key || name
|
|
275
338
|
|
|
276
|
-
const result = Attorney.
|
|
339
|
+
const result = Attorney.checkSendArgs([name, data, options], this.config)
|
|
277
340
|
|
|
278
341
|
return await this.createJob(result.name, result.data, result.options)
|
|
279
342
|
}
|
|
280
343
|
|
|
281
|
-
async
|
|
344
|
+
async sendSingleton (name, data, options) {
|
|
282
345
|
options = options || {}
|
|
283
346
|
|
|
284
347
|
options.singletonKey = SINGLETON_QUEUE_KEY
|
|
285
348
|
|
|
286
|
-
const result = Attorney.
|
|
349
|
+
const result = Attorney.checkSendArgs([name, data, options], this.config)
|
|
287
350
|
|
|
288
351
|
return await this.createJob(result.name, result.data, result.options)
|
|
289
352
|
}
|
|
290
353
|
|
|
291
|
-
async
|
|
354
|
+
async sendAfter (name, data, options, after) {
|
|
292
355
|
options = options || {}
|
|
293
356
|
options.startAfter = after
|
|
294
357
|
|
|
295
|
-
const result = Attorney.
|
|
358
|
+
const result = Attorney.checkSendArgs([name, data, options], this.config)
|
|
296
359
|
|
|
297
360
|
return await this.createJob(result.name, result.data, result.options)
|
|
298
361
|
}
|
|
299
362
|
|
|
300
|
-
async
|
|
363
|
+
async sendThrottled (name, data, options, seconds, key) {
|
|
301
364
|
options = options || {}
|
|
302
365
|
options.singletonSeconds = seconds
|
|
303
366
|
options.singletonNextSlot = false
|
|
304
367
|
options.singletonKey = key
|
|
305
368
|
|
|
306
|
-
const result = Attorney.
|
|
369
|
+
const result = Attorney.checkSendArgs([name, data, options], this.config)
|
|
307
370
|
|
|
308
371
|
return await this.createJob(result.name, result.data, result.options)
|
|
309
372
|
}
|
|
310
373
|
|
|
311
|
-
async
|
|
374
|
+
async sendDebounced (name, data, options, seconds, key) {
|
|
312
375
|
options = options || {}
|
|
313
376
|
options.singletonSeconds = seconds
|
|
314
377
|
options.singletonNextSlot = true
|
|
315
378
|
options.singletonKey = key
|
|
316
379
|
|
|
317
|
-
const result = Attorney.
|
|
380
|
+
const result = Attorney.checkSendArgs([name, data, options], this.config)
|
|
318
381
|
|
|
319
382
|
return await this.createJob(result.name, result.data, result.options)
|
|
320
383
|
}
|
package/src/migrationStore.js
CHANGED
|
@@ -67,6 +67,23 @@ function getAll (schema, config) {
|
|
|
67
67
|
const keepUntil = config ? config.keepUntil : DEFAULT_RETENTION
|
|
68
68
|
|
|
69
69
|
return [
|
|
70
|
+
{
|
|
71
|
+
release: '7.0.0',
|
|
72
|
+
version: 19,
|
|
73
|
+
previous: 18,
|
|
74
|
+
install: [
|
|
75
|
+
`CREATE TABLE ${schema}.subscription (
|
|
76
|
+
event text not null,
|
|
77
|
+
name text not null,
|
|
78
|
+
created_on timestamp with time zone not null default now(),
|
|
79
|
+
updated_on timestamp with time zone not null default now(),
|
|
80
|
+
PRIMARY KEY(event, name)
|
|
81
|
+
)`
|
|
82
|
+
],
|
|
83
|
+
uninstall: [
|
|
84
|
+
`DROP TABLE ${schema}.subscription`
|
|
85
|
+
]
|
|
86
|
+
},
|
|
70
87
|
{
|
|
71
88
|
release: '6.1.1',
|
|
72
89
|
version: 18,
|
package/src/plans.js
CHANGED
|
@@ -33,6 +33,9 @@ module.exports = {
|
|
|
33
33
|
getSchedules,
|
|
34
34
|
schedule,
|
|
35
35
|
unschedule,
|
|
36
|
+
subscribe,
|
|
37
|
+
unsubscribe,
|
|
38
|
+
getQueuesForEvent,
|
|
36
39
|
expire,
|
|
37
40
|
archive,
|
|
38
41
|
purge,
|
|
@@ -78,6 +81,7 @@ function create (schema, version) {
|
|
|
78
81
|
createJobTable(schema),
|
|
79
82
|
cloneJobTableForArchive(schema),
|
|
80
83
|
createScheduleTable(schema),
|
|
84
|
+
createSubscriptionTable(schema),
|
|
81
85
|
addIdIndexToArchive(schema),
|
|
82
86
|
addArchivedOnToArchive(schema),
|
|
83
87
|
addArchivedOnIndexToArchive(schema),
|
|
@@ -260,6 +264,18 @@ function createScheduleTable (schema) {
|
|
|
260
264
|
`
|
|
261
265
|
}
|
|
262
266
|
|
|
267
|
+
function createSubscriptionTable (schema) {
|
|
268
|
+
return `
|
|
269
|
+
CREATE TABLE ${schema}.subscription (
|
|
270
|
+
event text not null,
|
|
271
|
+
name text not null,
|
|
272
|
+
created_on timestamp with time zone not null default now(),
|
|
273
|
+
updated_on timestamp with time zone not null default now(),
|
|
274
|
+
PRIMARY KEY(event, name)
|
|
275
|
+
)
|
|
276
|
+
`
|
|
277
|
+
}
|
|
278
|
+
|
|
263
279
|
function getSchedules (schema) {
|
|
264
280
|
return `
|
|
265
281
|
SELECT * FROM ${schema}.schedule
|
|
@@ -286,6 +302,31 @@ function unschedule (schema) {
|
|
|
286
302
|
`
|
|
287
303
|
}
|
|
288
304
|
|
|
305
|
+
function subscribe (schema) {
|
|
306
|
+
return `
|
|
307
|
+
INSERT INTO ${schema}.subscription (event, name)
|
|
308
|
+
VALUES ($1, $2)
|
|
309
|
+
ON CONFLICT (event, name) DO UPDATE SET
|
|
310
|
+
event = EXCLUDED.event,
|
|
311
|
+
name = EXCLUDED.name,
|
|
312
|
+
updated_on = now()
|
|
313
|
+
`
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
function unsubscribe (schema) {
|
|
317
|
+
return `
|
|
318
|
+
DELETE FROM ${schema}.subscription
|
|
319
|
+
WHERE event = $1 and name = $2
|
|
320
|
+
`
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
function getQueuesForEvent (schema) {
|
|
324
|
+
return `
|
|
325
|
+
SELECT name FROM ${schema}.subscription
|
|
326
|
+
WHERE event = $1
|
|
327
|
+
`
|
|
328
|
+
}
|
|
329
|
+
|
|
289
330
|
function getTime () {
|
|
290
331
|
return "SELECT round(date_part('epoch', now()) * 1000) as time"
|
|
291
332
|
}
|
package/src/timekeeper.js
CHANGED
|
@@ -50,8 +50,8 @@ class Timekeeper extends EventEmitter {
|
|
|
50
50
|
|
|
51
51
|
await this.cacheClockSkew()
|
|
52
52
|
|
|
53
|
-
await this.manager.
|
|
54
|
-
await this.manager.
|
|
53
|
+
await this.manager.work(queues.CRON, { newJobCheckIntervalSeconds: 4 }, (job) => this.onCron(job))
|
|
54
|
+
await this.manager.work(queues.SEND_IT, { newJobCheckIntervalSeconds: 4, teamSize: 50, teamConcurrency: 5 }, (job) => this.onSendIt(job))
|
|
55
55
|
|
|
56
56
|
await this.cronMonitorAsync()
|
|
57
57
|
|
|
@@ -68,8 +68,8 @@ class Timekeeper extends EventEmitter {
|
|
|
68
68
|
|
|
69
69
|
this.stopped = true
|
|
70
70
|
|
|
71
|
-
await this.manager.
|
|
72
|
-
await this.manager.
|
|
71
|
+
await this.manager.offWork(queues.CRON)
|
|
72
|
+
await this.manager.offWork(queues.SEND_IT)
|
|
73
73
|
|
|
74
74
|
if (this.skewMonitorInterval) {
|
|
75
75
|
clearInterval(this.skewMonitorInterval)
|
|
@@ -115,7 +115,7 @@ class Timekeeper extends EventEmitter {
|
|
|
115
115
|
onComplete: false
|
|
116
116
|
}
|
|
117
117
|
|
|
118
|
-
await this.manager.
|
|
118
|
+
await this.manager.sendDebounced(queues.CRON, null, opts, 60)
|
|
119
119
|
}
|
|
120
120
|
|
|
121
121
|
async onCron () {
|
|
@@ -165,13 +165,13 @@ class Timekeeper extends EventEmitter {
|
|
|
165
165
|
onComplete: false
|
|
166
166
|
}
|
|
167
167
|
|
|
168
|
-
await this.manager.
|
|
168
|
+
await this.manager.send(queues.SEND_IT, job, options)
|
|
169
169
|
}
|
|
170
170
|
|
|
171
171
|
async onSendIt (job) {
|
|
172
172
|
if (this.stopped) return
|
|
173
173
|
const { name, data, options } = job.data
|
|
174
|
-
await this.manager.
|
|
174
|
+
await this.manager.send(name, data, options)
|
|
175
175
|
}
|
|
176
176
|
|
|
177
177
|
async getSchedules () {
|
|
@@ -185,7 +185,7 @@ class Timekeeper extends EventEmitter {
|
|
|
185
185
|
cronParser.parseExpression(cron, { tz })
|
|
186
186
|
|
|
187
187
|
// validation pre-check
|
|
188
|
-
Attorney.
|
|
188
|
+
Attorney.checkSendArgs([name, data, options], this.config)
|
|
189
189
|
|
|
190
190
|
const values = [name, cron, tz, data, options]
|
|
191
191
|
|
package/types.d.ts
CHANGED
|
@@ -88,9 +88,9 @@ declare namespace PgBoss {
|
|
|
88
88
|
singletonNextSlot?: boolean;
|
|
89
89
|
}
|
|
90
90
|
|
|
91
|
-
type
|
|
91
|
+
type SendOptions = JobOptions & ExpirationOptions & RetentionOptions & RetryOptions & CompletionOptions
|
|
92
92
|
|
|
93
|
-
type ScheduleOptions =
|
|
93
|
+
type ScheduleOptions = SendOptions & { tz?: string }
|
|
94
94
|
|
|
95
95
|
interface JobPollingOptions {
|
|
96
96
|
newJobCheckInterval?: number;
|
|
@@ -104,24 +104,24 @@ declare namespace PgBoss {
|
|
|
104
104
|
includeMetadata?: boolean;
|
|
105
105
|
}
|
|
106
106
|
|
|
107
|
-
type
|
|
107
|
+
type WorkOptions = JobFetchOptions & JobPollingOptions
|
|
108
108
|
|
|
109
109
|
type FetchOptions = {
|
|
110
110
|
includeMetadata?: boolean;
|
|
111
111
|
}
|
|
112
112
|
|
|
113
|
-
interface
|
|
113
|
+
interface WorkHandler<ReqData, ResData> {
|
|
114
114
|
(job: PgBoss.JobWithDoneCallback<ReqData, ResData>): Promise<ResData> | void;
|
|
115
115
|
}
|
|
116
116
|
|
|
117
|
-
interface
|
|
117
|
+
interface WorkWithMetadataHandler<ReqData, ResData> {
|
|
118
118
|
(job: PgBoss.JobWithMetadataDoneCallback<ReqData, ResData>): Promise<ResData> | void;
|
|
119
119
|
}
|
|
120
120
|
|
|
121
121
|
interface Request {
|
|
122
122
|
name: string;
|
|
123
123
|
data?: object;
|
|
124
|
-
options?:
|
|
124
|
+
options?: SendOptions;
|
|
125
125
|
}
|
|
126
126
|
|
|
127
127
|
interface Schedule {
|
|
@@ -213,10 +213,10 @@ declare namespace PgBoss {
|
|
|
213
213
|
queues: object;
|
|
214
214
|
}
|
|
215
215
|
|
|
216
|
-
interface
|
|
216
|
+
interface Worker {
|
|
217
217
|
id: string,
|
|
218
218
|
name: string,
|
|
219
|
-
options:
|
|
219
|
+
options: WorkOptions,
|
|
220
220
|
state: 'created' | 'retry' | 'active' | 'completed' | 'expired' | 'cancelled' | 'failed',
|
|
221
221
|
count: number,
|
|
222
222
|
createdOn: Date,
|
|
@@ -233,7 +233,7 @@ declare namespace PgBoss {
|
|
|
233
233
|
timeout?: number
|
|
234
234
|
}
|
|
235
235
|
|
|
236
|
-
interface
|
|
236
|
+
interface OffWorkOptions {
|
|
237
237
|
id: string
|
|
238
238
|
}
|
|
239
239
|
|
|
@@ -263,8 +263,8 @@ declare class PgBoss {
|
|
|
263
263
|
on(event: "monitor-states", handler: (monitorStates: PgBoss.MonitorStates) => void): void;
|
|
264
264
|
off(event: "monitor-states", handler: (monitorStates: PgBoss.MonitorStates) => void): void;
|
|
265
265
|
|
|
266
|
-
on(event: "wip", handler: (data: PgBoss.
|
|
267
|
-
off(event: "wip", handler: (data: PgBoss.
|
|
266
|
+
on(event: "wip", handler: (data: PgBoss.Worker[]) => void): void;
|
|
267
|
+
off(event: "wip", handler: (data: PgBoss.Worker[]) => void): void;
|
|
268
268
|
|
|
269
269
|
on(event: "stopped", handler: () => void): void;
|
|
270
270
|
off(event: "stopped", handler: () => void): void;
|
|
@@ -272,38 +272,44 @@ declare class PgBoss {
|
|
|
272
272
|
start(): Promise<PgBoss>;
|
|
273
273
|
stop(options?: PgBoss.StopOptions): Promise<void>;
|
|
274
274
|
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
275
|
+
send(request: PgBoss.Request): Promise<string | null>;
|
|
276
|
+
send(name: string, data: object): Promise<string | null>;
|
|
277
|
+
send(name: string, data: object, options: PgBoss.SendOptions): Promise<string | null>;
|
|
278
278
|
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
279
|
+
sendAfter(name: string, data: object, options: PgBoss.SendOptions, date: Date): Promise<string | null>;
|
|
280
|
+
sendAfter(name: string, data: object, options: PgBoss.SendOptions, dateString: string): Promise<string | null>;
|
|
281
|
+
sendAfter(name: string, data: object, options: PgBoss.SendOptions, seconds: number): Promise<string | null>;
|
|
282
282
|
|
|
283
|
-
|
|
283
|
+
sendOnce(name: string, data: object, options: PgBoss.SendOptions, key: string): Promise<string | null>;
|
|
284
284
|
|
|
285
|
-
|
|
285
|
+
sendSingleton(name: string, data: object, options: PgBoss.SendOptions): Promise<string | null>;
|
|
286
286
|
|
|
287
|
-
|
|
288
|
-
|
|
287
|
+
sendThrottled(name: string, data: object, options: PgBoss.SendOptions, seconds: number): Promise<string | null>;
|
|
288
|
+
sendThrottled(name: string, data: object, options: PgBoss.SendOptions, seconds: number, key: string): Promise<string | null>;
|
|
289
289
|
|
|
290
|
-
|
|
291
|
-
|
|
290
|
+
sendDebounced(name: string, data: object, options: PgBoss.SendOptions, seconds: number): Promise<string | null>;
|
|
291
|
+
sendDebounced(name: string, data: object, options: PgBoss.SendOptions, seconds: number, key: string): Promise<string | null>;
|
|
292
292
|
|
|
293
293
|
insert(jobs: PgBoss.JobInsert[]): Promise<void>;
|
|
294
294
|
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
295
|
+
work<ReqData, ResData>(name: string, handler: PgBoss.WorkHandler<ReqData, ResData>): Promise<string>;
|
|
296
|
+
work<ReqData, ResData>(name: string, options: PgBoss.WorkOptions & { includeMetadata: true }, handler: PgBoss.WorkWithMetadataHandler<ReqData, ResData>): Promise<string>;
|
|
297
|
+
work<ReqData, ResData>(name: string, options: PgBoss.WorkOptions, handler: PgBoss.WorkHandler<ReqData, ResData>): Promise<string>;
|
|
298
298
|
|
|
299
299
|
onComplete(name: string, handler: Function): Promise<string>;
|
|
300
|
-
onComplete(name: string, options: PgBoss.
|
|
300
|
+
onComplete(name: string, options: PgBoss.WorkOptions, handler: Function): Promise<string>;
|
|
301
301
|
|
|
302
|
-
|
|
303
|
-
|
|
302
|
+
offWork(name: string): Promise<void>;
|
|
303
|
+
offWork(options: PgBoss.OffWorkOptions): Promise<void>;
|
|
304
|
+
|
|
305
|
+
subscribe(event: string, name: string): Promise<void>;
|
|
306
|
+
unsubscribe(event: string, name: string): Promise<void>;
|
|
307
|
+
publish(event: string): Promise<string[]>;
|
|
308
|
+
publish(event: string, data: object): Promise<string[]>;
|
|
309
|
+
publish(event: string, data: object, options: PgBoss.SendOptions): Promise<string[]>;
|
|
304
310
|
|
|
305
311
|
offComplete(name: string): Promise<void>;
|
|
306
|
-
offComplete(options: PgBoss.
|
|
312
|
+
offComplete(options: PgBoss.OffWorkOptions): Promise<void>;
|
|
307
313
|
|
|
308
314
|
fetch<T>(name: string): Promise<PgBoss.Job<T> | null>;
|
|
309
315
|
fetch<T>(name: string, batchSize: number): Promise<PgBoss.Job<T>[] | null>;
|
package/version.json
CHANGED