pg-boss 10.0.0-beta2 → 10.0.0-beta4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -12
- package/package.json +1 -1
- package/src/attorney.js +16 -11
- package/src/boss.js +2 -8
- package/src/contractor.js +6 -0
- package/src/db.js +6 -6
- package/src/index.js +88 -65
- package/src/manager.js +5 -3
- package/src/plans.js +56 -11
- package/src/timekeeper.js +40 -57
- package/types.d.ts +1 -1
package/README.md
CHANGED
|
@@ -9,24 +9,23 @@ async function readme() {
|
|
|
9
9
|
const PgBoss = require('pg-boss');
|
|
10
10
|
const boss = new PgBoss('postgres://user:pass@host/database');
|
|
11
11
|
|
|
12
|
-
boss.on('error',
|
|
12
|
+
boss.on('error', console.error)
|
|
13
13
|
|
|
14
|
-
await boss.start()
|
|
14
|
+
await boss.start()
|
|
15
15
|
|
|
16
|
-
const queue = '
|
|
16
|
+
const queue = 'readme-queue'
|
|
17
17
|
|
|
18
|
-
|
|
18
|
+
try {
|
|
19
|
+
await boss.createQueue(queue)
|
|
20
|
+
} catch {}
|
|
19
21
|
|
|
20
|
-
|
|
22
|
+
const id = await boss.send(queue, { arg1: 'read me' })
|
|
21
23
|
|
|
22
|
-
|
|
23
|
-
}
|
|
24
|
-
|
|
25
|
-
async function someAsyncJobHandler(job) {
|
|
26
|
-
console.log(`job ${job.id} received with data:`);
|
|
27
|
-
console.log(JSON.stringify(job.data));
|
|
24
|
+
console.log(`created job ${id} in queue ${queue}`)
|
|
28
25
|
|
|
29
|
-
await
|
|
26
|
+
await boss.work(queue, async job => {
|
|
27
|
+
console.log(`received job ${job.id} with data ${JSON.stringify(job.data)}`)
|
|
28
|
+
})
|
|
30
29
|
}
|
|
31
30
|
```
|
|
32
31
|
|
package/package.json
CHANGED
package/src/attorney.js
CHANGED
|
@@ -8,7 +8,8 @@ module.exports = {
|
|
|
8
8
|
checkWorkArgs,
|
|
9
9
|
checkFetchArgs,
|
|
10
10
|
warnClockSkew,
|
|
11
|
-
assertPostgresObjectName
|
|
11
|
+
assertPostgresObjectName,
|
|
12
|
+
assertQueueName
|
|
12
13
|
}
|
|
13
14
|
|
|
14
15
|
const MAX_INTERVAL_HOURS = 24
|
|
@@ -29,8 +30,6 @@ const WARNINGS = {
|
|
|
29
30
|
}
|
|
30
31
|
|
|
31
32
|
function checkQueueArgs (name, options = {}) {
|
|
32
|
-
assertPostgresObjectName(name)
|
|
33
|
-
|
|
34
33
|
assert(!('deadLetter' in options) || (typeof options.deadLetter === 'string'), 'deadLetter must be a string')
|
|
35
34
|
|
|
36
35
|
applyRetryConfig(options)
|
|
@@ -178,8 +177,14 @@ function applySchemaConfig (config) {
|
|
|
178
177
|
function assertPostgresObjectName (name) {
|
|
179
178
|
assert(typeof name === 'string', 'Name must be a string')
|
|
180
179
|
assert(name.length <= 50, 'Name cannot exceed 50 characters')
|
|
181
|
-
assert(!/\W/.test(name), 'Name can only contain alphanumeric characters
|
|
182
|
-
assert(
|
|
180
|
+
assert(!/\W/.test(name), 'Name can only contain alphanumeric characters or underscores')
|
|
181
|
+
assert(!/^\d/.test(name), 'Name cannot start with a number')
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
function assertQueueName (name) {
|
|
185
|
+
assert(typeof name === 'string', 'Name must be a string')
|
|
186
|
+
assert(name.length <= 50, 'Name cannot exceed 50 characters')
|
|
187
|
+
assert(/[\w-]/.test(name), 'Name can only contain alphanumeric characters, underscores, or hyphens')
|
|
183
188
|
}
|
|
184
189
|
|
|
185
190
|
function applyArchiveConfig (config) {
|
|
@@ -363,21 +368,21 @@ function applyMonitoringConfig (config) {
|
|
|
363
368
|
? config.clockMonitorIntervalSeconds
|
|
364
369
|
: TEN_MINUTES_IN_SECONDS
|
|
365
370
|
|
|
366
|
-
assert(!('cronMonitorIntervalSeconds' in config) || (config.cronMonitorIntervalSeconds >= 1 && config.cronMonitorIntervalSeconds <=
|
|
367
|
-
'configuration assert: cronMonitorIntervalSeconds must be between 1 and
|
|
371
|
+
assert(!('cronMonitorIntervalSeconds' in config) || (config.cronMonitorIntervalSeconds >= 1 && config.cronMonitorIntervalSeconds <= 45),
|
|
372
|
+
'configuration assert: cronMonitorIntervalSeconds must be between 1 and 45 seconds')
|
|
368
373
|
|
|
369
374
|
config.cronMonitorIntervalSeconds =
|
|
370
375
|
('cronMonitorIntervalSeconds' in config)
|
|
371
376
|
? config.cronMonitorIntervalSeconds
|
|
372
|
-
:
|
|
377
|
+
: 30
|
|
373
378
|
|
|
374
|
-
assert(!('cronWorkerIntervalSeconds' in config) || (config.cronWorkerIntervalSeconds >= 1 && config.cronWorkerIntervalSeconds <=
|
|
375
|
-
'configuration assert: cronWorkerIntervalSeconds must be between 1 and
|
|
379
|
+
assert(!('cronWorkerIntervalSeconds' in config) || (config.cronWorkerIntervalSeconds >= 1 && config.cronWorkerIntervalSeconds <= 45),
|
|
380
|
+
'configuration assert: cronWorkerIntervalSeconds must be between 1 and 45 seconds')
|
|
376
381
|
|
|
377
382
|
config.cronWorkerIntervalSeconds =
|
|
378
383
|
('cronWorkerIntervalSeconds' in config)
|
|
379
384
|
? config.cronWorkerIntervalSeconds
|
|
380
|
-
:
|
|
385
|
+
: 5
|
|
381
386
|
}
|
|
382
387
|
|
|
383
388
|
function warnClockSkew (message) {
|
package/src/boss.js
CHANGED
|
@@ -77,10 +77,7 @@ class Boss extends EventEmitter {
|
|
|
77
77
|
} catch (err) {
|
|
78
78
|
this.emit(events.error, err)
|
|
79
79
|
} finally {
|
|
80
|
-
|
|
81
|
-
await locker.unlock()
|
|
82
|
-
}
|
|
83
|
-
|
|
80
|
+
await locker?.unlock()
|
|
84
81
|
this.monitoring = false
|
|
85
82
|
}
|
|
86
83
|
}
|
|
@@ -119,10 +116,7 @@ class Boss extends EventEmitter {
|
|
|
119
116
|
} catch (err) {
|
|
120
117
|
this.emit(events.error, err)
|
|
121
118
|
} finally {
|
|
122
|
-
|
|
123
|
-
await locker.unlock()
|
|
124
|
-
}
|
|
125
|
-
|
|
119
|
+
await locker?.unlock()
|
|
126
120
|
this.maintaining = false
|
|
127
121
|
}
|
|
128
122
|
}
|
package/src/contractor.js
CHANGED
|
@@ -21,6 +21,12 @@ class Contractor {
|
|
|
21
21
|
this.config = config
|
|
22
22
|
this.db = db
|
|
23
23
|
this.migrations = this.config.migrations || migrationStore.getAll(this.config.schema)
|
|
24
|
+
|
|
25
|
+
// exported api to index
|
|
26
|
+
this.functions = [
|
|
27
|
+
this.version,
|
|
28
|
+
this.isInstalled
|
|
29
|
+
]
|
|
24
30
|
}
|
|
25
31
|
|
|
26
32
|
async version () {
|
package/src/db.js
CHANGED
|
@@ -11,6 +11,10 @@ class Db extends EventEmitter {
|
|
|
11
11
|
this.config = config
|
|
12
12
|
}
|
|
13
13
|
|
|
14
|
+
events = {
|
|
15
|
+
error: 'error'
|
|
16
|
+
}
|
|
17
|
+
|
|
14
18
|
async open () {
|
|
15
19
|
this.pool = new pg.Pool(this.config)
|
|
16
20
|
this.pool.on('error', error => this.emit('error', error))
|
|
@@ -41,27 +45,23 @@ class Db extends EventEmitter {
|
|
|
41
45
|
}
|
|
42
46
|
|
|
43
47
|
async lock ({ timeout = 30, key } = {}) {
|
|
44
|
-
// const lockedClient = new pg.Client(this.config)
|
|
45
|
-
// await lockedClient.connect()
|
|
46
48
|
const lockedClient = await this.pool.connect()
|
|
47
49
|
|
|
48
50
|
const query = `
|
|
49
51
|
BEGIN;
|
|
50
52
|
SET LOCAL lock_timeout = '${timeout}s';
|
|
51
|
-
SET LOCAL idle_in_transaction_session_timeout = '
|
|
53
|
+
SET LOCAL idle_in_transaction_session_timeout = '${timeout}s';
|
|
52
54
|
${advisoryLock(this.config.schema, key)};
|
|
53
55
|
`
|
|
54
56
|
|
|
55
57
|
await lockedClient.query(query)
|
|
56
58
|
|
|
57
59
|
const locker = {
|
|
58
|
-
locked: true,
|
|
59
60
|
unlock: async function () {
|
|
60
61
|
try {
|
|
61
62
|
await lockedClient.query('COMMIT')
|
|
62
|
-
await lockedClient.end()
|
|
63
63
|
} finally {
|
|
64
|
-
|
|
64
|
+
lockedClient.release()
|
|
65
65
|
}
|
|
66
66
|
}
|
|
67
67
|
}
|
package/src/index.js
CHANGED
|
@@ -13,6 +13,17 @@ const events = {
|
|
|
13
13
|
stopped: 'stopped'
|
|
14
14
|
}
|
|
15
15
|
class PgBoss extends EventEmitter {
|
|
16
|
+
#stoppingOn
|
|
17
|
+
#stopped
|
|
18
|
+
#starting
|
|
19
|
+
#started
|
|
20
|
+
#config
|
|
21
|
+
#db
|
|
22
|
+
#boss
|
|
23
|
+
#contractor
|
|
24
|
+
#manager
|
|
25
|
+
#timekeeper
|
|
26
|
+
|
|
16
27
|
static getConstructionPlans (schema) {
|
|
17
28
|
return Contractor.constructionPlans(schema)
|
|
18
29
|
}
|
|
@@ -26,100 +37,112 @@ class PgBoss extends EventEmitter {
|
|
|
26
37
|
}
|
|
27
38
|
|
|
28
39
|
constructor (value) {
|
|
29
|
-
const config = Attorney.getConfig(value)
|
|
30
|
-
|
|
31
40
|
super()
|
|
32
41
|
|
|
33
|
-
|
|
42
|
+
this.#stoppingOn = null
|
|
43
|
+
this.#stopped = true
|
|
44
|
+
|
|
45
|
+
const config = Attorney.getConfig(value)
|
|
46
|
+
this.#config = config
|
|
47
|
+
|
|
48
|
+
const db = this.getDb()
|
|
49
|
+
this.#db = db
|
|
34
50
|
|
|
35
51
|
if (db.isOurs) {
|
|
36
|
-
|
|
52
|
+
this.#promoteEvents(db)
|
|
37
53
|
}
|
|
38
54
|
|
|
39
|
-
const
|
|
40
|
-
Object.keys(manager.events).forEach(event => promoteEvent.call(this, manager, manager.events[event]))
|
|
41
|
-
manager.functions.forEach(func => promoteFunction.call(this, manager, func))
|
|
55
|
+
const contractor = new Contractor(db, config)
|
|
42
56
|
|
|
57
|
+
const manager = new Manager(db, config)
|
|
43
58
|
const bossConfig = { ...config, manager }
|
|
44
59
|
|
|
45
60
|
const boss = new Boss(db, bossConfig)
|
|
46
|
-
Object.keys(boss.events).forEach(event => promoteEvent.call(this, boss, boss.events[event]))
|
|
47
|
-
boss.functions.forEach(func => promoteFunction.call(this, boss, func))
|
|
48
61
|
|
|
49
62
|
const timekeeper = new Timekeeper(db, bossConfig)
|
|
50
|
-
Object.keys(timekeeper.events).forEach(event => promoteEvent.call(this, timekeeper, timekeeper.events[event]))
|
|
51
|
-
timekeeper.functions.forEach(func => promoteFunction.call(this, timekeeper, func))
|
|
52
|
-
|
|
53
63
|
manager.timekeeper = timekeeper
|
|
54
64
|
|
|
55
|
-
this
|
|
56
|
-
this
|
|
57
|
-
this
|
|
58
|
-
this.db = db
|
|
59
|
-
this.boss = boss
|
|
60
|
-
this.contractor = new Contractor(db, config)
|
|
61
|
-
this.manager = manager
|
|
62
|
-
this.timekeeper = timekeeper
|
|
63
|
-
|
|
64
|
-
function getDb (config) {
|
|
65
|
-
if (config.db) {
|
|
66
|
-
return config.db
|
|
67
|
-
}
|
|
65
|
+
this.#promoteEvents(manager)
|
|
66
|
+
this.#promoteEvents(boss)
|
|
67
|
+
this.#promoteEvents(timekeeper)
|
|
68
68
|
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
69
|
+
this.#promoteFunctions(boss)
|
|
70
|
+
this.#promoteFunctions(contractor)
|
|
71
|
+
this.#promoteFunctions(manager)
|
|
72
|
+
this.#promoteFunctions(timekeeper)
|
|
73
|
+
|
|
74
|
+
this.#boss = boss
|
|
75
|
+
this.#contractor = contractor
|
|
76
|
+
this.#manager = manager
|
|
77
|
+
this.#timekeeper = timekeeper
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
getDb () {
|
|
81
|
+
if (this.#db) {
|
|
82
|
+
return this.#db
|
|
72
83
|
}
|
|
73
84
|
|
|
74
|
-
|
|
75
|
-
this
|
|
85
|
+
if (this.#config.db) {
|
|
86
|
+
return this.#config.db
|
|
76
87
|
}
|
|
77
88
|
|
|
78
|
-
|
|
89
|
+
const db = new Db(this.#config)
|
|
90
|
+
db.isOurs = true
|
|
91
|
+
return db
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
#promoteEvents (emitter) {
|
|
95
|
+
for (const event of Object.values(emitter?.events)) {
|
|
79
96
|
emitter.on(event, arg => this.emit(event, arg))
|
|
80
97
|
}
|
|
81
98
|
}
|
|
82
99
|
|
|
100
|
+
#promoteFunctions (obj) {
|
|
101
|
+
for (const func of obj?.functions) {
|
|
102
|
+
this[func.name] = (...args) => func.apply(obj, args)
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
|
|
83
106
|
async start () {
|
|
84
|
-
if (this
|
|
107
|
+
if (this.#starting || this.#started) {
|
|
85
108
|
return
|
|
86
109
|
}
|
|
87
110
|
|
|
88
|
-
this
|
|
111
|
+
this.#starting = true
|
|
89
112
|
|
|
90
|
-
if (this
|
|
91
|
-
await this
|
|
113
|
+
if (this.#db.isOurs && !this.#db.opened) {
|
|
114
|
+
await this.#db.open()
|
|
92
115
|
}
|
|
93
116
|
|
|
94
|
-
if (this
|
|
95
|
-
await this
|
|
117
|
+
if (this.#config.migrate) {
|
|
118
|
+
await this.#contractor.start()
|
|
96
119
|
} else {
|
|
97
|
-
await this
|
|
120
|
+
await this.#contractor.check()
|
|
98
121
|
}
|
|
99
122
|
|
|
100
|
-
this
|
|
123
|
+
this.#manager.start()
|
|
101
124
|
|
|
102
|
-
if (this
|
|
103
|
-
await this
|
|
125
|
+
if (this.#config.supervise) {
|
|
126
|
+
await this.#boss.supervise()
|
|
104
127
|
}
|
|
105
128
|
|
|
106
|
-
if (this
|
|
107
|
-
await this
|
|
129
|
+
if (this.#config.monitorStateIntervalSeconds) {
|
|
130
|
+
await this.#boss.monitor()
|
|
108
131
|
}
|
|
109
132
|
|
|
110
|
-
if (this
|
|
111
|
-
await this
|
|
133
|
+
if (this.#config.schedule) {
|
|
134
|
+
await this.#timekeeper.start()
|
|
112
135
|
}
|
|
113
136
|
|
|
114
|
-
this
|
|
115
|
-
this
|
|
116
|
-
this
|
|
137
|
+
this.#starting = false
|
|
138
|
+
this.#started = true
|
|
139
|
+
this.#stopped = false
|
|
117
140
|
|
|
118
141
|
return this
|
|
119
142
|
}
|
|
120
143
|
|
|
121
144
|
async stop (options = {}) {
|
|
122
|
-
if (this
|
|
145
|
+
if (this.#stoppingOn || this.#stopped) {
|
|
123
146
|
return
|
|
124
147
|
}
|
|
125
148
|
|
|
@@ -127,28 +150,28 @@ class PgBoss extends EventEmitter {
|
|
|
127
150
|
|
|
128
151
|
timeout = Math.max(timeout, 1000)
|
|
129
152
|
|
|
130
|
-
this
|
|
153
|
+
this.#stoppingOn = Date.now()
|
|
131
154
|
|
|
132
|
-
await this
|
|
133
|
-
await this
|
|
134
|
-
await this
|
|
155
|
+
await this.#manager.stop()
|
|
156
|
+
await this.#timekeeper.stop()
|
|
157
|
+
await this.#boss.stop()
|
|
135
158
|
|
|
136
159
|
await new Promise((resolve, reject) => {
|
|
137
160
|
const shutdown = async () => {
|
|
138
161
|
try {
|
|
139
|
-
if (this
|
|
140
|
-
throw new Error(this
|
|
162
|
+
if (this.#config.__test__throw_shutdown) {
|
|
163
|
+
throw new Error(this.#config.__test__throw_shutdown)
|
|
141
164
|
}
|
|
142
165
|
|
|
143
|
-
await this
|
|
166
|
+
await this.#manager.failWip()
|
|
144
167
|
|
|
145
|
-
if (this
|
|
146
|
-
await this
|
|
168
|
+
if (this.#db.isOurs && this.#db.opened && destroy) {
|
|
169
|
+
await this.#db.close()
|
|
147
170
|
}
|
|
148
171
|
|
|
149
|
-
this
|
|
150
|
-
this
|
|
151
|
-
this
|
|
172
|
+
this.#stopped = true
|
|
173
|
+
this.#stoppingOn = null
|
|
174
|
+
this.#started = false
|
|
152
175
|
|
|
153
176
|
this.emit(events.stopped)
|
|
154
177
|
resolve()
|
|
@@ -168,13 +191,13 @@ class PgBoss extends EventEmitter {
|
|
|
168
191
|
|
|
169
192
|
setImmediate(async () => {
|
|
170
193
|
try {
|
|
171
|
-
if (this
|
|
172
|
-
throw new Error(this
|
|
194
|
+
if (this.#config.__test__throw_stop_monitor) {
|
|
195
|
+
throw new Error(this.#config.__test__throw_stop_monitor)
|
|
173
196
|
}
|
|
174
197
|
|
|
175
|
-
const isWip = () => this
|
|
198
|
+
const isWip = () => this.#manager.getWipData({ includeInternal: false }).length > 0
|
|
176
199
|
|
|
177
|
-
while ((Date.now() - this
|
|
200
|
+
while ((Date.now() - this.#stoppingOn) < timeout && isWip()) {
|
|
178
201
|
await delay(500)
|
|
179
202
|
}
|
|
180
203
|
|
package/src/manager.js
CHANGED
|
@@ -547,6 +547,8 @@ class Manager extends EventEmitter {
|
|
|
547
547
|
async createQueue (name, options = {}) {
|
|
548
548
|
assert(name, 'Missing queue name argument')
|
|
549
549
|
|
|
550
|
+
Attorney.assertQueueName(name)
|
|
551
|
+
|
|
550
552
|
const { policy = QUEUE_POLICY.standard } = options
|
|
551
553
|
|
|
552
554
|
assert(policy in QUEUE_POLICY, `${policy} is not a valid queue policy`)
|
|
@@ -560,7 +562,7 @@ class Manager extends EventEmitter {
|
|
|
560
562
|
deadLetter
|
|
561
563
|
} = Attorney.checkQueueArgs(name, options)
|
|
562
564
|
|
|
563
|
-
const paritionSql = plans.
|
|
565
|
+
const paritionSql = plans.createPartition(this.config.schema, name)
|
|
564
566
|
|
|
565
567
|
await this.db.executeSql(paritionSql)
|
|
566
568
|
|
|
@@ -646,8 +648,8 @@ class Manager extends EventEmitter {
|
|
|
646
648
|
const result = await this.db.executeSql(queueSql, [name])
|
|
647
649
|
|
|
648
650
|
if (result?.rows?.length) {
|
|
649
|
-
Attorney.
|
|
650
|
-
const sql = plans.
|
|
651
|
+
Attorney.assertQueueName(name)
|
|
652
|
+
const sql = plans.dropPartition(this.config.schema, name)
|
|
651
653
|
await this.db.executeSql(sql)
|
|
652
654
|
}
|
|
653
655
|
|
package/src/plans.js
CHANGED
|
@@ -46,8 +46,8 @@ module.exports = {
|
|
|
46
46
|
countStates,
|
|
47
47
|
createQueue,
|
|
48
48
|
updateQueue,
|
|
49
|
-
|
|
50
|
-
|
|
49
|
+
createPartition,
|
|
50
|
+
dropPartition,
|
|
51
51
|
deleteQueueRecords,
|
|
52
52
|
getQueueByName,
|
|
53
53
|
getQueueSize,
|
|
@@ -96,6 +96,10 @@ function create (schema, version) {
|
|
|
96
96
|
createTableSchedule(schema),
|
|
97
97
|
createTableSubscription(schema),
|
|
98
98
|
|
|
99
|
+
getPartitionFunction(schema),
|
|
100
|
+
createPartitionFunction(schema),
|
|
101
|
+
dropPartitionFunction(schema),
|
|
102
|
+
|
|
99
103
|
insertVersion(schema, version)
|
|
100
104
|
]
|
|
101
105
|
|
|
@@ -162,16 +166,53 @@ function createTableJob (schema) {
|
|
|
162
166
|
`
|
|
163
167
|
}
|
|
164
168
|
|
|
165
|
-
function
|
|
169
|
+
function createPartition (schema, name) {
|
|
170
|
+
return `SELECT ${schema}.create_partition('${name}');`
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
function getPartitionFunction (schema) {
|
|
174
|
+
return `
|
|
175
|
+
CREATE FUNCTION ${schema}.get_partition(queue_name text, out name text) AS
|
|
176
|
+
$$
|
|
177
|
+
SELECT '${schema}.job_' || encode(digest(queue_name, 'sha1'), 'hex');
|
|
178
|
+
$$
|
|
179
|
+
LANGUAGE SQL
|
|
180
|
+
IMMUTABLE
|
|
181
|
+
`
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
function createPartitionFunction (schema) {
|
|
185
|
+
return `
|
|
186
|
+
CREATE FUNCTION ${schema}.create_partition(queue_name text)
|
|
187
|
+
RETURNS VOID AS
|
|
188
|
+
$$
|
|
189
|
+
DECLARE
|
|
190
|
+
table_name varchar := ${schema}.get_partition(queue_name);
|
|
191
|
+
BEGIN
|
|
192
|
+
EXECUTE format('CREATE TABLE %I (LIKE ${schema}.job INCLUDING DEFAULTS INCLUDING CONSTRAINTS)', table_name);
|
|
193
|
+
EXECUTE format('ALTER TABLE %I ADD CHECK (name=%L)', table_name, queue_name);
|
|
194
|
+
EXECUTE format('ALTER TABLE ${schema}.job ATTACH PARTITION %I FOR VALUES IN (%L)', table_name, queue_name);
|
|
195
|
+
END;
|
|
196
|
+
$$
|
|
197
|
+
LANGUAGE plpgsql;
|
|
198
|
+
`
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
function dropPartitionFunction (schema) {
|
|
166
202
|
return `
|
|
167
|
-
CREATE
|
|
168
|
-
|
|
169
|
-
|
|
203
|
+
CREATE FUNCTION ${schema}.drop_partition(queue_name text)
|
|
204
|
+
RETURNS VOID AS
|
|
205
|
+
$$
|
|
206
|
+
BEGIN
|
|
207
|
+
EXECUTE format('DROP TABLE IF EXISTS %I', ${schema}.get_partition(queue_name));
|
|
208
|
+
END;
|
|
209
|
+
$$
|
|
210
|
+
LANGUAGE plpgsql;
|
|
170
211
|
`
|
|
171
212
|
}
|
|
172
213
|
|
|
173
|
-
function
|
|
174
|
-
return `
|
|
214
|
+
function dropPartition (schema, name) {
|
|
215
|
+
return `SELECT ${schema}.drop_partition('${name}');`
|
|
175
216
|
}
|
|
176
217
|
|
|
177
218
|
function createPrimaryKeyArchive (schema) {
|
|
@@ -274,6 +315,8 @@ function getQueueByName (schema) {
|
|
|
274
315
|
function deleteQueueRecords (schema) {
|
|
275
316
|
return `WITH dq AS (
|
|
276
317
|
DELETE FROM ${schema}.queue WHERE name = $1
|
|
318
|
+
), ds AS (
|
|
319
|
+
DELETE FROM ${schema}.schedule WHERE name = $1
|
|
277
320
|
)
|
|
278
321
|
DELETE FROM ${schema}.job WHERE name = $1
|
|
279
322
|
`
|
|
@@ -337,7 +380,9 @@ function createTableSubscription (schema) {
|
|
|
337
380
|
|
|
338
381
|
function getSchedules (schema) {
|
|
339
382
|
return `
|
|
340
|
-
SELECT
|
|
383
|
+
SELECT s.*
|
|
384
|
+
FROM ${schema}.schedule s
|
|
385
|
+
JOIN ${schema}.queue q on s.name = q.name
|
|
341
386
|
`
|
|
342
387
|
}
|
|
343
388
|
|
|
@@ -723,8 +768,8 @@ function locked (schema, query) {
|
|
|
723
768
|
}
|
|
724
769
|
|
|
725
770
|
function advisoryLock (schema, key) {
|
|
726
|
-
return `SELECT pg_advisory_xact_lock(
|
|
727
|
-
('x' ||
|
|
771
|
+
return `SELECT pg_advisory_xact_lock(
|
|
772
|
+
('x' || encode(digest(current_database() || '.pgboss.${schema}${key || ''}', 'sha256'), 'hex'))::bit(64)::bigint
|
|
728
773
|
)`
|
|
729
774
|
}
|
|
730
775
|
|
package/src/timekeeper.js
CHANGED
|
@@ -5,8 +5,7 @@ const Attorney = require('./attorney')
|
|
|
5
5
|
const pMap = require('p-map')
|
|
6
6
|
|
|
7
7
|
const queues = {
|
|
8
|
-
|
|
9
|
-
SEND_IT: '__pgboss__send_it'
|
|
8
|
+
SEND_IT: '__pgboss__send-it'
|
|
10
9
|
}
|
|
11
10
|
|
|
12
11
|
const events = {
|
|
@@ -28,6 +27,7 @@ class Timekeeper extends EventEmitter {
|
|
|
28
27
|
this.events = events
|
|
29
28
|
|
|
30
29
|
this.getTimeCommand = plans.getTime(config.schema)
|
|
30
|
+
this.getQueueCommand = plans.getQueueByName(config.schema)
|
|
31
31
|
this.getSchedulesCommand = plans.getSchedules(config.schema)
|
|
32
32
|
this.scheduleCommand = plans.schedule(config.schema)
|
|
33
33
|
this.unscheduleCommand = plans.unschedule(config.schema)
|
|
@@ -44,6 +44,8 @@ class Timekeeper extends EventEmitter {
|
|
|
44
44
|
}
|
|
45
45
|
|
|
46
46
|
async start () {
|
|
47
|
+
this.stopped = false
|
|
48
|
+
|
|
47
49
|
// setting the archive config too low breaks the cron 60s debounce interval so don't even try
|
|
48
50
|
if (this.config.archiveSeconds < 60 || this.config.archiveFailedSeconds < 60) {
|
|
49
51
|
return
|
|
@@ -52,21 +54,18 @@ class Timekeeper extends EventEmitter {
|
|
|
52
54
|
// cache the clock skew from the db server
|
|
53
55
|
await this.cacheClockSkew()
|
|
54
56
|
|
|
55
|
-
|
|
56
|
-
|
|
57
|
+
try {
|
|
58
|
+
await this.manager.createQueue(queues.SEND_IT)
|
|
59
|
+
} catch {}
|
|
57
60
|
|
|
58
|
-
await this.manager.work(queues.CRON, { newJobCheckIntervalSeconds: this.config.cronWorkerIntervalSeconds }, (job) => this.onCron(job))
|
|
59
61
|
await this.manager.work(queues.SEND_IT, { newJobCheckIntervalSeconds: this.config.cronWorkerIntervalSeconds, teamSize: 50, teamConcurrency: 5 }, (job) => this.onSendIt(job))
|
|
60
62
|
|
|
61
|
-
|
|
62
|
-
await this.checkSchedulesAsync()
|
|
63
|
+
setImmediate(() => this.onCron())
|
|
63
64
|
|
|
64
65
|
// create monitoring interval to make sure cron hasn't crashed
|
|
65
|
-
this.cronMonitorInterval = setInterval(async () => await this.
|
|
66
|
+
this.cronMonitorInterval = setInterval(async () => await this.onCron(), this.cronMonitorIntervalMs)
|
|
66
67
|
// create monitoring interval to measure and adjust for drift in clock skew
|
|
67
68
|
this.skewMonitorInterval = setInterval(async () => await this.cacheClockSkew(), this.skewMonitorIntervalMs)
|
|
68
|
-
|
|
69
|
-
this.stopped = false
|
|
70
69
|
}
|
|
71
70
|
|
|
72
71
|
async stop () {
|
|
@@ -76,7 +75,6 @@ class Timekeeper extends EventEmitter {
|
|
|
76
75
|
|
|
77
76
|
this.stopped = true
|
|
78
77
|
|
|
79
|
-
await this.manager.offWork(queues.CRON)
|
|
80
78
|
await this.manager.offWork(queues.SEND_IT)
|
|
81
79
|
|
|
82
80
|
if (this.skewMonitorInterval) {
|
|
@@ -90,22 +88,6 @@ class Timekeeper extends EventEmitter {
|
|
|
90
88
|
}
|
|
91
89
|
}
|
|
92
90
|
|
|
93
|
-
async monitorCron () {
|
|
94
|
-
try {
|
|
95
|
-
if (this.config.__test__force_cron_monitoring_error) {
|
|
96
|
-
throw new Error(this.config.__test__force_cron_monitoring_error)
|
|
97
|
-
}
|
|
98
|
-
|
|
99
|
-
const { secondsAgo } = await this.getCronTime()
|
|
100
|
-
|
|
101
|
-
if (secondsAgo > 60) {
|
|
102
|
-
await this.checkSchedulesAsync()
|
|
103
|
-
}
|
|
104
|
-
} catch (err) {
|
|
105
|
-
this.emit(this.events.error, err)
|
|
106
|
-
}
|
|
107
|
-
}
|
|
108
|
-
|
|
109
91
|
async cacheClockSkew () {
|
|
110
92
|
let skew = 0
|
|
111
93
|
|
|
@@ -134,43 +116,42 @@ class Timekeeper extends EventEmitter {
|
|
|
134
116
|
}
|
|
135
117
|
}
|
|
136
118
|
|
|
137
|
-
async checkSchedulesAsync () {
|
|
138
|
-
const opts = {
|
|
139
|
-
retryLimit: 2,
|
|
140
|
-
retentionSeconds: 60
|
|
141
|
-
}
|
|
142
|
-
|
|
143
|
-
await this.manager.sendDebounced(queues.CRON, null, opts, 60)
|
|
144
|
-
}
|
|
145
|
-
|
|
146
119
|
async onCron () {
|
|
147
|
-
|
|
120
|
+
let locker
|
|
148
121
|
|
|
149
122
|
try {
|
|
150
|
-
if (this.
|
|
151
|
-
throw new Error(this.config.__test__throw_cron_processing)
|
|
152
|
-
}
|
|
123
|
+
if (this.stopped || this.timekeeping) return
|
|
153
124
|
|
|
154
|
-
|
|
125
|
+
if (this.config.__test__force_cron_monitoring_error) {
|
|
126
|
+
throw new Error(this.config.__test__force_cron_monitoring_error)
|
|
127
|
+
}
|
|
155
128
|
|
|
156
|
-
|
|
129
|
+
this.timekeeping = true
|
|
157
130
|
|
|
158
|
-
|
|
159
|
-
await pMap(sending, it => this.send(it), { concurrency: 5 })
|
|
160
|
-
}
|
|
131
|
+
locker = await this.db.lock({ key: 'timekeeper' })
|
|
161
132
|
|
|
162
|
-
|
|
133
|
+
const { secondsAgo } = await this.getCronTime()
|
|
163
134
|
|
|
164
|
-
|
|
165
|
-
|
|
135
|
+
if (secondsAgo > this.config.cronMonitorIntervalSeconds) {
|
|
136
|
+
await this.cron()
|
|
137
|
+
await this.setCronTime()
|
|
138
|
+
}
|
|
166
139
|
} catch (err) {
|
|
167
140
|
this.emit(this.events.error, err)
|
|
141
|
+
} finally {
|
|
142
|
+
this.timekeeping = false
|
|
143
|
+
await locker?.unlock()
|
|
168
144
|
}
|
|
145
|
+
}
|
|
169
146
|
|
|
170
|
-
|
|
147
|
+
async cron () {
|
|
148
|
+
const items = await this.getSchedules()
|
|
171
149
|
|
|
172
|
-
|
|
173
|
-
|
|
150
|
+
const sending = items.filter(i => this.shouldSendIt(i.cron, i.timezone))
|
|
151
|
+
|
|
152
|
+
if (sending.length && !this.stopped) {
|
|
153
|
+
await pMap(sending, it => this.send(it), { concurrency: 5 })
|
|
154
|
+
}
|
|
174
155
|
}
|
|
175
156
|
|
|
176
157
|
shouldSendIt (cron, tz) {
|
|
@@ -186,12 +167,7 @@ class Timekeeper extends EventEmitter {
|
|
|
186
167
|
}
|
|
187
168
|
|
|
188
169
|
async send (job) {
|
|
189
|
-
|
|
190
|
-
singletonKey: job.name,
|
|
191
|
-
singletonSeconds: 60
|
|
192
|
-
}
|
|
193
|
-
|
|
194
|
-
await this.manager.send(queues.SEND_IT, job, options)
|
|
170
|
+
await this.manager.send(queues.SEND_IT, job, { singletonKey: job.name, singletonSeconds: 60 })
|
|
195
171
|
}
|
|
196
172
|
|
|
197
173
|
async onSendIt (job) {
|
|
@@ -213,6 +189,13 @@ class Timekeeper extends EventEmitter {
|
|
|
213
189
|
// validation pre-check
|
|
214
190
|
Attorney.checkSendArgs([name, data, options], this.config)
|
|
215
191
|
|
|
192
|
+
// make sure queue exists before scheduling
|
|
193
|
+
const queue = await this.db.executeSql(this.getQueueCommand, [name])
|
|
194
|
+
|
|
195
|
+
if (!queue.rows.length === 0) {
|
|
196
|
+
throw new Error(`Queue '${name}' not found`)
|
|
197
|
+
}
|
|
198
|
+
|
|
216
199
|
const values = [name, cron, tz, data, options]
|
|
217
200
|
|
|
218
201
|
const result = await this.db.executeSql(this.scheduleCommand, values)
|
package/types.d.ts
CHANGED
|
@@ -348,7 +348,7 @@ declare class PgBoss extends EventEmitter {
|
|
|
348
348
|
getQueueSize(name: string, options?: object): Promise<number>;
|
|
349
349
|
getJobById(name: string, id: string, options?: PgBoss.ConnectionOptions): Promise<PgBoss.JobWithMetadata | null>;
|
|
350
350
|
|
|
351
|
-
createQueue(name: string, policy: 'standard' | 'short' | 'singleton' | 'stately'): Promise<void>;
|
|
351
|
+
createQueue(name: string, options?: { policy: 'standard' | 'short' | 'singleton' | 'stately' }): Promise<void>;
|
|
352
352
|
deleteQueue(name: string): Promise<void>;
|
|
353
353
|
purgeQueue(name: string): Promise<void>;
|
|
354
354
|
clearStorage(): Promise<void>;
|