pg-boss 10.0.0-beta6 → 10.0.0-beta8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/boss.js +13 -23
- package/src/contractor.js +5 -5
- package/src/db.js +0 -26
- package/src/index.js +3 -1
- package/src/manager.js +11 -16
- package/src/plans.js +58 -76
- package/src/timekeeper.js +14 -35
- package/types.d.ts +33 -10
package/package.json
CHANGED
package/src/boss.js
CHANGED
|
@@ -24,10 +24,8 @@ class Boss extends EventEmitter {
|
|
|
24
24
|
this.failJobsByTimeoutCommand = plans.locked(config.schema, plans.failJobsByTimeout(config.schema))
|
|
25
25
|
this.archiveCommand = plans.locked(config.schema, plans.archive(config.schema, config.archiveInterval, config.archiveFailedInterval))
|
|
26
26
|
this.dropCommand = plans.locked(config.schema, plans.drop(config.schema, config.deleteAfter))
|
|
27
|
-
this.
|
|
28
|
-
this.
|
|
29
|
-
this.getMonitorTimeCommand = plans.getMonitorTime(config.schema)
|
|
30
|
-
this.setMonitorTimeCommand = plans.setMonitorTime(config.schema)
|
|
27
|
+
this.trySetMaintenanceTimeCommand = plans.trySetMaintenanceTime(config.schema)
|
|
28
|
+
this.trySetMonitorTimeCommand = plans.trySetMonitorTime(config.schema)
|
|
31
29
|
this.countStatesCommand = plans.countStates(config.schema)
|
|
32
30
|
|
|
33
31
|
this.functions = [
|
|
@@ -48,8 +46,6 @@ class Boss extends EventEmitter {
|
|
|
48
46
|
}
|
|
49
47
|
|
|
50
48
|
async onMonitor () {
|
|
51
|
-
let locker
|
|
52
|
-
|
|
53
49
|
try {
|
|
54
50
|
if (this.monitoring) {
|
|
55
51
|
return
|
|
@@ -65,26 +61,24 @@ class Boss extends EventEmitter {
|
|
|
65
61
|
throw new Error(this.config.__test__throw_monitor)
|
|
66
62
|
}
|
|
67
63
|
|
|
68
|
-
|
|
64
|
+
if (this.stopped) {
|
|
65
|
+
return
|
|
66
|
+
}
|
|
69
67
|
|
|
70
|
-
const {
|
|
68
|
+
const { rows } = await this.db.executeSql(this.trySetMonitorTimeCommand, [this.config.monitorStateIntervalSeconds])
|
|
71
69
|
|
|
72
|
-
if (
|
|
70
|
+
if (rows.length === 1 && !this.stopped) {
|
|
73
71
|
const states = await this.countStates()
|
|
74
|
-
this.setMonitorTime()
|
|
75
72
|
this.emit(events.monitorStates, states)
|
|
76
73
|
}
|
|
77
74
|
} catch (err) {
|
|
78
75
|
this.emit(events.error, err)
|
|
79
76
|
} finally {
|
|
80
|
-
await locker?.unlock()
|
|
81
77
|
this.monitoring = false
|
|
82
78
|
}
|
|
83
79
|
}
|
|
84
80
|
|
|
85
81
|
async onSupervise () {
|
|
86
|
-
let locker
|
|
87
|
-
|
|
88
82
|
try {
|
|
89
83
|
if (this.maintaining) {
|
|
90
84
|
return
|
|
@@ -105,18 +99,15 @@ class Boss extends EventEmitter {
|
|
|
105
99
|
return
|
|
106
100
|
}
|
|
107
101
|
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
const { secondsAgo } = await this.getMaintenanceTime()
|
|
102
|
+
const { rows } = await this.db.executeSql(this.trySetMaintenanceTimeCommand, [this.config.maintenanceIntervalSeconds])
|
|
111
103
|
|
|
112
|
-
if (
|
|
104
|
+
if (rows.length === 1 && !this.stopped) {
|
|
113
105
|
const result = await this.maintain()
|
|
114
106
|
this.emit(events.maintenance, result)
|
|
115
107
|
}
|
|
116
108
|
} catch (err) {
|
|
117
109
|
this.emit(events.error, err)
|
|
118
110
|
} finally {
|
|
119
|
-
await locker?.unlock()
|
|
120
111
|
this.maintaining = false
|
|
121
112
|
}
|
|
122
113
|
}
|
|
@@ -130,8 +121,6 @@ class Boss extends EventEmitter {
|
|
|
130
121
|
|
|
131
122
|
const ended = Date.now()
|
|
132
123
|
|
|
133
|
-
await this.setMaintenanceTime()
|
|
134
|
-
|
|
135
124
|
return { ms: ended - started }
|
|
136
125
|
}
|
|
137
126
|
|
|
@@ -146,10 +135,11 @@ class Boss extends EventEmitter {
|
|
|
146
135
|
}
|
|
147
136
|
|
|
148
137
|
async countStates () {
|
|
149
|
-
const stateCountDefault = { ...plans.
|
|
138
|
+
const stateCountDefault = { ...plans.JOB_STATES }
|
|
150
139
|
|
|
151
|
-
Object.keys(stateCountDefault)
|
|
152
|
-
|
|
140
|
+
for (const key of Object.keys(stateCountDefault)) {
|
|
141
|
+
stateCountDefault[key] = 0
|
|
142
|
+
}
|
|
153
143
|
|
|
154
144
|
const counts = await this.db.executeSql(this.countStatesCommand)
|
|
155
145
|
|
package/src/contractor.js
CHANGED
|
@@ -24,26 +24,26 @@ class Contractor {
|
|
|
24
24
|
|
|
25
25
|
// exported api to index
|
|
26
26
|
this.functions = [
|
|
27
|
-
this.
|
|
27
|
+
this.schemaVersion,
|
|
28
28
|
this.isInstalled
|
|
29
29
|
]
|
|
30
30
|
}
|
|
31
31
|
|
|
32
|
-
async
|
|
32
|
+
async schemaVersion () {
|
|
33
33
|
const result = await this.db.executeSql(plans.getVersion(this.config.schema))
|
|
34
34
|
return result.rows.length ? parseInt(result.rows[0].version) : null
|
|
35
35
|
}
|
|
36
36
|
|
|
37
37
|
async isInstalled () {
|
|
38
38
|
const result = await this.db.executeSql(plans.versionTableExists(this.config.schema))
|
|
39
|
-
return result.rows
|
|
39
|
+
return !!result.rows[0].name
|
|
40
40
|
}
|
|
41
41
|
|
|
42
42
|
async start () {
|
|
43
43
|
const installed = await this.isInstalled()
|
|
44
44
|
|
|
45
45
|
if (installed) {
|
|
46
|
-
const version = await this.
|
|
46
|
+
const version = await this.schemaVersion()
|
|
47
47
|
|
|
48
48
|
if (schemaVersion > version) {
|
|
49
49
|
throw new Error('Migrations are not supported to v10')
|
|
@@ -61,7 +61,7 @@ class Contractor {
|
|
|
61
61
|
throw new Error('pg-boss is not installed')
|
|
62
62
|
}
|
|
63
63
|
|
|
64
|
-
const version = await this.
|
|
64
|
+
const version = await this.schemaVersion()
|
|
65
65
|
|
|
66
66
|
if (schemaVersion !== version) {
|
|
67
67
|
throw new Error('pg-boss database requires migrations')
|
package/src/db.js
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
const EventEmitter = require('events')
|
|
2
2
|
const pg = require('pg')
|
|
3
|
-
const { advisoryLock } = require('./plans')
|
|
4
3
|
|
|
5
4
|
class Db extends EventEmitter {
|
|
6
5
|
constructor (config) {
|
|
@@ -43,31 +42,6 @@ class Db extends EventEmitter {
|
|
|
43
42
|
return await this.pool.query(text, values)
|
|
44
43
|
}
|
|
45
44
|
}
|
|
46
|
-
|
|
47
|
-
async lock ({ timeout = 30, key } = {}) {
|
|
48
|
-
const lockedClient = await this.pool.connect()
|
|
49
|
-
|
|
50
|
-
const query = `
|
|
51
|
-
BEGIN;
|
|
52
|
-
SET LOCAL lock_timeout = '${timeout}s';
|
|
53
|
-
SET LOCAL idle_in_transaction_session_timeout = '${timeout}s';
|
|
54
|
-
${advisoryLock(this.config.schema, key)};
|
|
55
|
-
`
|
|
56
|
-
|
|
57
|
-
await lockedClient.query(query)
|
|
58
|
-
|
|
59
|
-
const locker = {
|
|
60
|
-
unlock: async function () {
|
|
61
|
-
try {
|
|
62
|
-
await lockedClient.query('COMMIT')
|
|
63
|
-
} finally {
|
|
64
|
-
lockedClient.release()
|
|
65
|
-
}
|
|
66
|
-
}
|
|
67
|
-
}
|
|
68
|
-
|
|
69
|
-
return locker
|
|
70
|
-
}
|
|
71
45
|
}
|
|
72
46
|
|
|
73
47
|
module.exports = Db
|
package/src/index.js
CHANGED
|
@@ -36,6 +36,9 @@ class PgBoss extends EventEmitter {
|
|
|
36
36
|
return Contractor.rollbackPlans(schema, version)
|
|
37
37
|
}
|
|
38
38
|
|
|
39
|
+
static states = plans.JOB_STATES
|
|
40
|
+
static policies = plans.QUEUE_POLICIES
|
|
41
|
+
|
|
39
42
|
constructor (value) {
|
|
40
43
|
super()
|
|
41
44
|
|
|
@@ -212,4 +215,3 @@ class PgBoss extends EventEmitter {
|
|
|
212
215
|
}
|
|
213
216
|
|
|
214
217
|
module.exports = PgBoss
|
|
215
|
-
module.exports.states = plans.states
|
package/src/manager.js
CHANGED
|
@@ -10,7 +10,7 @@ const Worker = require('./worker')
|
|
|
10
10
|
const plans = require('./plans')
|
|
11
11
|
|
|
12
12
|
const { QUEUES: TIMEKEEPER_QUEUES } = require('./timekeeper')
|
|
13
|
-
const {
|
|
13
|
+
const { QUEUE_POLICIES } = plans
|
|
14
14
|
|
|
15
15
|
const INTERNAL_QUEUES = Object.values(TIMEKEEPER_QUEUES).reduce((acc, i) => ({ ...acc, [i]: i }), {})
|
|
16
16
|
|
|
@@ -317,13 +317,9 @@ class Manager extends EventEmitter {
|
|
|
317
317
|
async publish (event, ...args) {
|
|
318
318
|
assert(event, 'Missing required argument')
|
|
319
319
|
|
|
320
|
-
const
|
|
320
|
+
const { rows } = await this.db.executeSql(this.getQueuesForEventCommand, [event])
|
|
321
321
|
|
|
322
|
-
|
|
323
|
-
return []
|
|
324
|
-
}
|
|
325
|
-
|
|
326
|
-
return await Promise.all(result.rows.map(({ name }) => this.send(name, ...args)))
|
|
322
|
+
return await Promise.all(rows.map(({ name }) => this.send(name, ...args)))
|
|
327
323
|
}
|
|
328
324
|
|
|
329
325
|
async send (...args) {
|
|
@@ -406,10 +402,10 @@ class Manager extends EventEmitter {
|
|
|
406
402
|
]
|
|
407
403
|
|
|
408
404
|
const db = wrapper || this.db
|
|
409
|
-
const
|
|
405
|
+
const { rows } = await db.executeSql(this.insertJobCommand, values)
|
|
410
406
|
|
|
411
|
-
if (
|
|
412
|
-
return
|
|
407
|
+
if (rows.length === 1) {
|
|
408
|
+
return rows[0].id
|
|
413
409
|
}
|
|
414
410
|
|
|
415
411
|
if (!options.singletonNextSlot) {
|
|
@@ -549,9 +545,9 @@ class Manager extends EventEmitter {
|
|
|
549
545
|
|
|
550
546
|
Attorney.assertQueueName(name)
|
|
551
547
|
|
|
552
|
-
const { policy =
|
|
548
|
+
const { policy = QUEUE_POLICIES.standard } = options
|
|
553
549
|
|
|
554
|
-
assert(policy in
|
|
550
|
+
assert(policy in QUEUE_POLICIES, `${policy} is not a valid queue policy`)
|
|
555
551
|
|
|
556
552
|
const {
|
|
557
553
|
retryLimit,
|
|
@@ -645,17 +641,16 @@ class Manager extends EventEmitter {
|
|
|
645
641
|
assert(name, 'Missing queue name argument')
|
|
646
642
|
|
|
647
643
|
const queueSql = plans.getQueueByName(this.config.schema)
|
|
648
|
-
const
|
|
644
|
+
const { rows } = await this.db.executeSql(queueSql, [name])
|
|
649
645
|
|
|
650
|
-
if (
|
|
646
|
+
if (rows.length) {
|
|
651
647
|
Attorney.assertQueueName(name)
|
|
652
648
|
const sql = plans.dropPartition(this.config.schema, name)
|
|
653
649
|
await this.db.executeSql(sql)
|
|
654
650
|
}
|
|
655
651
|
|
|
656
652
|
const sql = plans.deleteQueueRecords(this.config.schema)
|
|
657
|
-
|
|
658
|
-
return result2?.rowCount || null
|
|
653
|
+
await this.db.executeSql(sql, [name])
|
|
659
654
|
}
|
|
660
655
|
|
|
661
656
|
async purgeQueue (queue) {
|
package/src/plans.js
CHANGED
|
@@ -1,24 +1,22 @@
|
|
|
1
|
-
const
|
|
1
|
+
const DEFAULT_SCHEMA = 'pgboss'
|
|
2
|
+
const MIGRATE_RACE_MESSAGE = 'division by zero'
|
|
3
|
+
const CREATE_RACE_MESSAGE = 'already exists'
|
|
2
4
|
|
|
3
|
-
const
|
|
5
|
+
const JOB_STATES = Object.freeze({
|
|
4
6
|
created: 'created',
|
|
5
7
|
retry: 'retry',
|
|
6
8
|
active: 'active',
|
|
7
9
|
completed: 'completed',
|
|
8
10
|
cancelled: 'cancelled',
|
|
9
11
|
failed: 'failed'
|
|
10
|
-
}
|
|
12
|
+
})
|
|
11
13
|
|
|
12
|
-
const
|
|
13
|
-
const MIGRATE_RACE_MESSAGE = 'division by zero'
|
|
14
|
-
const CREATE_RACE_MESSAGE = 'already exists'
|
|
15
|
-
|
|
16
|
-
const QUEUE_POLICY = {
|
|
14
|
+
const QUEUE_POLICIES = Object.freeze({
|
|
17
15
|
standard: 'standard',
|
|
18
16
|
short: 'short',
|
|
19
17
|
singleton: 'singleton',
|
|
20
18
|
stately: 'stately'
|
|
21
|
-
}
|
|
19
|
+
})
|
|
22
20
|
|
|
23
21
|
module.exports = {
|
|
24
22
|
create,
|
|
@@ -53,31 +51,28 @@ module.exports = {
|
|
|
53
51
|
getQueueSize,
|
|
54
52
|
purgeQueue,
|
|
55
53
|
clearStorage,
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
setMonitorTime,
|
|
60
|
-
getCronTime,
|
|
61
|
-
setCronTime,
|
|
54
|
+
trySetMaintenanceTime,
|
|
55
|
+
trySetMonitorTime,
|
|
56
|
+
trySetCronTime,
|
|
62
57
|
locked,
|
|
63
|
-
advisoryLock,
|
|
64
58
|
assertMigration,
|
|
65
59
|
getArchivedJobById,
|
|
66
60
|
getJobById,
|
|
67
|
-
|
|
68
|
-
|
|
61
|
+
QUEUE_POLICIES,
|
|
62
|
+
JOB_STATES,
|
|
69
63
|
MIGRATE_RACE_MESSAGE,
|
|
70
64
|
CREATE_RACE_MESSAGE,
|
|
71
65
|
DEFAULT_SCHEMA
|
|
72
66
|
}
|
|
73
67
|
|
|
68
|
+
const assert = require('assert')
|
|
69
|
+
|
|
74
70
|
function create (schema, version) {
|
|
75
71
|
const commands = [
|
|
76
72
|
createSchema(schema),
|
|
77
73
|
createEnumJobState(schema),
|
|
78
74
|
|
|
79
75
|
createTableJob(schema),
|
|
80
|
-
createIndexJobName(schema),
|
|
81
76
|
createIndexJobFetch(schema),
|
|
82
77
|
createIndexJobPolicyStately(schema),
|
|
83
78
|
createIndexJobPolicyShort(schema),
|
|
@@ -89,7 +84,6 @@ function create (schema, version) {
|
|
|
89
84
|
createPrimaryKeyArchive(schema),
|
|
90
85
|
createColumnArchiveArchivedOn(schema),
|
|
91
86
|
createIndexArchiveArchivedOn(schema),
|
|
92
|
-
createIndexArchiveName(schema),
|
|
93
87
|
|
|
94
88
|
createTableVersion(schema),
|
|
95
89
|
createTableQueue(schema),
|
|
@@ -128,12 +122,12 @@ function createEnumJobState (schema) {
|
|
|
128
122
|
// base type is numeric and first values are less than last values
|
|
129
123
|
return `
|
|
130
124
|
CREATE TYPE ${schema}.job_state AS ENUM (
|
|
131
|
-
'${
|
|
132
|
-
'${
|
|
133
|
-
'${
|
|
134
|
-
'${
|
|
135
|
-
'${
|
|
136
|
-
'${
|
|
125
|
+
'${JOB_STATES.created}',
|
|
126
|
+
'${JOB_STATES.retry}',
|
|
127
|
+
'${JOB_STATES.active}',
|
|
128
|
+
'${JOB_STATES.completed}',
|
|
129
|
+
'${JOB_STATES.cancelled}',
|
|
130
|
+
'${JOB_STATES.failed}'
|
|
137
131
|
)
|
|
138
132
|
`
|
|
139
133
|
}
|
|
@@ -145,7 +139,7 @@ function createTableJob (schema) {
|
|
|
145
139
|
name text not null,
|
|
146
140
|
priority integer not null default(0),
|
|
147
141
|
data jsonb,
|
|
148
|
-
state ${schema}.job_state not null default('${
|
|
142
|
+
state ${schema}.job_state not null default('${JOB_STATES.created}'),
|
|
149
143
|
retry_limit integer not null default(0),
|
|
150
144
|
retry_count integer not null default(0),
|
|
151
145
|
retry_delay integer not null default(0),
|
|
@@ -238,31 +232,27 @@ function createPrimaryKeyArchive (schema) {
|
|
|
238
232
|
}
|
|
239
233
|
|
|
240
234
|
function createIndexJobPolicyShort (schema) {
|
|
241
|
-
return `CREATE UNIQUE INDEX job_policy_short ON ${schema}.job (name) WHERE state = '${
|
|
235
|
+
return `CREATE UNIQUE INDEX job_policy_short ON ${schema}.job (name) WHERE state = '${JOB_STATES.created}' AND policy = '${QUEUE_POLICIES.short}'`
|
|
242
236
|
}
|
|
243
237
|
|
|
244
238
|
function createIndexJobPolicySingleton (schema) {
|
|
245
|
-
return `CREATE UNIQUE INDEX job_policy_singleton ON ${schema}.job (name) WHERE state = '${
|
|
239
|
+
return `CREATE UNIQUE INDEX job_policy_singleton ON ${schema}.job (name) WHERE state = '${JOB_STATES.active}' AND policy = '${QUEUE_POLICIES.singleton}'`
|
|
246
240
|
}
|
|
247
241
|
|
|
248
242
|
function createIndexJobPolicyStately (schema) {
|
|
249
|
-
return `CREATE UNIQUE INDEX job_policy_stately ON ${schema}.job (name, state) WHERE state <= '${
|
|
243
|
+
return `CREATE UNIQUE INDEX job_policy_stately ON ${schema}.job (name, state) WHERE state <= '${JOB_STATES.active}' AND policy = '${QUEUE_POLICIES.stately}'`
|
|
250
244
|
}
|
|
251
245
|
|
|
252
246
|
function createIndexJobThrottleOn (schema) {
|
|
253
|
-
return `CREATE UNIQUE INDEX job_throttle_on ON ${schema}.job (name, singleton_on, COALESCE(singleton_key, '')) WHERE state <= '${
|
|
247
|
+
return `CREATE UNIQUE INDEX job_throttle_on ON ${schema}.job (name, singleton_on, COALESCE(singleton_key, '')) WHERE state <= '${JOB_STATES.completed}' AND singleton_on IS NOT NULL`
|
|
254
248
|
}
|
|
255
249
|
|
|
256
250
|
function createIndexJobThrottleKey (schema) {
|
|
257
|
-
return `CREATE UNIQUE INDEX job_throttle_key ON ${schema}.job (name, singleton_key) WHERE state <= '${
|
|
258
|
-
}
|
|
259
|
-
|
|
260
|
-
function createIndexJobName (schema) {
|
|
261
|
-
return `CREATE INDEX job_name ON ${schema}.job (name)`
|
|
251
|
+
return `CREATE UNIQUE INDEX job_throttle_key ON ${schema}.job (name, singleton_key) WHERE state <= '${JOB_STATES.completed}' AND singleton_on IS NULL`
|
|
262
252
|
}
|
|
263
253
|
|
|
264
254
|
function createIndexJobFetch (schema) {
|
|
265
|
-
return `CREATE INDEX job_fetch ON ${schema}.job (name, start_after) INCLUDE (priority, created_on, id) WHERE state < '${
|
|
255
|
+
return `CREATE INDEX job_fetch ON ${schema}.job (name, start_after) INCLUDE (priority, created_on, id) WHERE state < '${JOB_STATES.active}'`
|
|
266
256
|
}
|
|
267
257
|
|
|
268
258
|
function createTableArchive (schema) {
|
|
@@ -277,33 +267,24 @@ function createIndexArchiveArchivedOn (schema) {
|
|
|
277
267
|
return `CREATE INDEX archive_archived_on_idx ON ${schema}.archive(archived_on)`
|
|
278
268
|
}
|
|
279
269
|
|
|
280
|
-
function
|
|
281
|
-
return
|
|
270
|
+
function trySetMaintenanceTime (schema) {
|
|
271
|
+
return trySetTimestamp(schema, 'maintained_on')
|
|
282
272
|
}
|
|
283
273
|
|
|
284
|
-
function
|
|
285
|
-
return
|
|
274
|
+
function trySetMonitorTime (schema) {
|
|
275
|
+
return trySetTimestamp(schema, 'monitored_on')
|
|
286
276
|
}
|
|
287
277
|
|
|
288
|
-
function
|
|
289
|
-
return
|
|
278
|
+
function trySetCronTime (schema) {
|
|
279
|
+
return trySetTimestamp(schema, 'cron_on')
|
|
290
280
|
}
|
|
291
281
|
|
|
292
|
-
function
|
|
293
|
-
return `
|
|
294
|
-
}
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
}
|
|
299
|
-
|
|
300
|
-
function setCronTime (schema, time) {
|
|
301
|
-
time = time || 'now()'
|
|
302
|
-
return `UPDATE ${schema}.version SET cron_on = ${time}`
|
|
303
|
-
}
|
|
304
|
-
|
|
305
|
-
function getCronTime (schema) {
|
|
306
|
-
return `SELECT cron_on, EXTRACT( EPOCH FROM (now() - cron_on) ) seconds_ago FROM ${schema}.version`
|
|
282
|
+
function trySetTimestamp (schema, column) {
|
|
283
|
+
return `
|
|
284
|
+
UPDATE ${schema}.version SET ${column} = now()
|
|
285
|
+
WHERE EXTRACT( EPOCH FROM (now() - COALESCE(${column}, now() - interval '1 week') ) ) > $1
|
|
286
|
+
RETURNING true
|
|
287
|
+
`
|
|
307
288
|
}
|
|
308
289
|
|
|
309
290
|
function createQueue (schema) {
|
|
@@ -341,7 +322,7 @@ function deleteQueueRecords (schema) {
|
|
|
341
322
|
}
|
|
342
323
|
|
|
343
324
|
function purgeQueue (schema) {
|
|
344
|
-
return `DELETE from ${schema}.job WHERE name = $1 and state < '${
|
|
325
|
+
return `DELETE from ${schema}.job WHERE name = $1 and state < '${JOB_STATES.active}'`
|
|
345
326
|
}
|
|
346
327
|
|
|
347
328
|
function clearStorage (schema) {
|
|
@@ -349,8 +330,8 @@ function clearStorage (schema) {
|
|
|
349
330
|
}
|
|
350
331
|
|
|
351
332
|
function getQueueSize (schema, options = {}) {
|
|
352
|
-
options.before = options.before ||
|
|
353
|
-
assert(options.before in
|
|
333
|
+
options.before = options.before || JOB_STATES.active
|
|
334
|
+
assert(options.before in JOB_STATES, `${options.before} is not a valid state`)
|
|
354
335
|
return `SELECT count(*) as count FROM ${schema}.job WHERE name = $1 AND state < '${options.before}'`
|
|
355
336
|
}
|
|
356
337
|
|
|
@@ -475,14 +456,14 @@ function fetchNextJob (schema) {
|
|
|
475
456
|
SELECT id
|
|
476
457
|
FROM ${schema}.job
|
|
477
458
|
WHERE name = $1
|
|
478
|
-
AND state < '${
|
|
459
|
+
AND state < '${JOB_STATES.active}'
|
|
479
460
|
AND start_after < now()
|
|
480
461
|
ORDER BY ${priority && 'priority desc, '} created_on, id
|
|
481
462
|
LIMIT $2
|
|
482
463
|
FOR UPDATE SKIP LOCKED
|
|
483
464
|
)
|
|
484
465
|
UPDATE ${schema}.job j SET
|
|
485
|
-
state = '${
|
|
466
|
+
state = '${JOB_STATES.active}',
|
|
486
467
|
started_on = now(),
|
|
487
468
|
retry_count = CASE WHEN started_on IS NOT NULL THEN retry_count + 1 ELSE retry_count END
|
|
488
469
|
FROM next
|
|
@@ -496,11 +477,11 @@ function completeJobs (schema) {
|
|
|
496
477
|
WITH results AS (
|
|
497
478
|
UPDATE ${schema}.job
|
|
498
479
|
SET completed_on = now(),
|
|
499
|
-
state = '${
|
|
480
|
+
state = '${JOB_STATES.completed}',
|
|
500
481
|
output = $3::jsonb
|
|
501
482
|
WHERE name = $1
|
|
502
483
|
AND id IN (SELECT UNNEST($2::uuid[]))
|
|
503
|
-
AND state = '${
|
|
484
|
+
AND state = '${JOB_STATES.active}'
|
|
504
485
|
RETURNING *
|
|
505
486
|
)
|
|
506
487
|
SELECT COUNT(*) FROM results
|
|
@@ -508,14 +489,14 @@ function completeJobs (schema) {
|
|
|
508
489
|
}
|
|
509
490
|
|
|
510
491
|
function failJobsById (schema) {
|
|
511
|
-
const where = `name = $1 AND id IN (SELECT UNNEST($2::uuid[])) AND state < '${
|
|
492
|
+
const where = `name = $1 AND id IN (SELECT UNNEST($2::uuid[])) AND state < '${JOB_STATES.completed}'`
|
|
512
493
|
const output = '$3::jsonb'
|
|
513
494
|
|
|
514
495
|
return failJobs(schema, where, output)
|
|
515
496
|
}
|
|
516
497
|
|
|
517
498
|
function failJobsByTimeout (schema) {
|
|
518
|
-
const where = `state = '${
|
|
499
|
+
const where = `state = '${JOB_STATES.active}' AND (started_on + expire_in) < now()`
|
|
519
500
|
const output = '\'{ "value": { "message": "job failed by timeout in active state" } }\'::jsonb'
|
|
520
501
|
return failJobs(schema, where, output)
|
|
521
502
|
}
|
|
@@ -525,8 +506,8 @@ function failJobs (schema, where, output) {
|
|
|
525
506
|
WITH results AS (
|
|
526
507
|
UPDATE ${schema}.job SET
|
|
527
508
|
state = CASE
|
|
528
|
-
WHEN retry_count < retry_limit THEN '${
|
|
529
|
-
ELSE '${
|
|
509
|
+
WHEN retry_count < retry_limit THEN '${JOB_STATES.retry}'::${schema}.job_state
|
|
510
|
+
ELSE '${JOB_STATES.failed}'::${schema}.job_state
|
|
530
511
|
END,
|
|
531
512
|
completed_on = CASE
|
|
532
513
|
WHEN retry_count < retry_limit THEN NULL
|
|
@@ -552,7 +533,7 @@ function failJobs (schema, where, output) {
|
|
|
552
533
|
retry_limit,
|
|
553
534
|
keep_until + (keep_until - start_after)
|
|
554
535
|
FROM results
|
|
555
|
-
WHERE state = '${
|
|
536
|
+
WHERE state = '${JOB_STATES.failed}'
|
|
556
537
|
AND dead_letter IS NOT NULL
|
|
557
538
|
AND NOT name = dead_letter
|
|
558
539
|
)
|
|
@@ -565,10 +546,10 @@ function cancelJobs (schema) {
|
|
|
565
546
|
with results as (
|
|
566
547
|
UPDATE ${schema}.job
|
|
567
548
|
SET completed_on = now(),
|
|
568
|
-
state = '${
|
|
549
|
+
state = '${JOB_STATES.cancelled}'
|
|
569
550
|
WHERE name = $1
|
|
570
551
|
AND id IN (SELECT UNNEST($2::uuid[]))
|
|
571
|
-
AND state < '${
|
|
552
|
+
AND state < '${JOB_STATES.completed}'
|
|
572
553
|
RETURNING 1
|
|
573
554
|
)
|
|
574
555
|
SELECT COUNT(*) from results
|
|
@@ -580,7 +561,7 @@ function resumeJobs (schema) {
|
|
|
580
561
|
with results as (
|
|
581
562
|
UPDATE ${schema}.job
|
|
582
563
|
SET completed_on = NULL,
|
|
583
|
-
state = '${
|
|
564
|
+
state = '${JOB_STATES.created}'
|
|
584
565
|
WHERE name = $1
|
|
585
566
|
AND id IN (SELECT UNNEST($2::uuid[]))
|
|
586
567
|
RETURNING 1
|
|
@@ -750,9 +731,9 @@ function archive (schema, completedInterval, failedInterval = completedInterval)
|
|
|
750
731
|
return `
|
|
751
732
|
WITH archived_rows AS (
|
|
752
733
|
DELETE FROM ${schema}.job
|
|
753
|
-
WHERE (state <> '${
|
|
754
|
-
OR (state = '${
|
|
755
|
-
OR (state < '${
|
|
734
|
+
WHERE (state <> '${JOB_STATES.failed}' AND completed_on < (now() - interval '${completedInterval}'))
|
|
735
|
+
OR (state = '${JOB_STATES.failed}' AND completed_on < (now() - interval '${failedInterval}'))
|
|
736
|
+
OR (state < '${JOB_STATES.active}' AND keep_until < now())
|
|
756
737
|
RETURNING *
|
|
757
738
|
)
|
|
758
739
|
INSERT INTO ${schema}.archive (${columns})
|
|
@@ -777,6 +758,7 @@ function locked (schema, query) {
|
|
|
777
758
|
return `
|
|
778
759
|
BEGIN;
|
|
779
760
|
SET LOCAL lock_timeout = '30s';
|
|
761
|
+
SET LOCAL idle_in_transaction_session_timeout = '30s';
|
|
780
762
|
${advisoryLock(schema)};
|
|
781
763
|
${query};
|
|
782
764
|
COMMIT;
|
package/src/timekeeper.js
CHANGED
|
@@ -31,8 +31,7 @@ class Timekeeper extends EventEmitter {
|
|
|
31
31
|
this.getSchedulesCommand = plans.getSchedules(config.schema)
|
|
32
32
|
this.scheduleCommand = plans.schedule(config.schema)
|
|
33
33
|
this.unscheduleCommand = plans.unschedule(config.schema)
|
|
34
|
-
this.
|
|
35
|
-
this.setCronTimeCommand = plans.setCronTime(config.schema)
|
|
34
|
+
this.trySetCronTimeCommand = plans.trySetCronTime(config.schema)
|
|
36
35
|
|
|
37
36
|
this.functions = [
|
|
38
37
|
this.schedule,
|
|
@@ -44,27 +43,30 @@ class Timekeeper extends EventEmitter {
|
|
|
44
43
|
}
|
|
45
44
|
|
|
46
45
|
async start () {
|
|
47
|
-
this.stopped = false
|
|
48
|
-
|
|
49
46
|
// setting the archive config too low breaks the cron 60s debounce interval so don't even try
|
|
50
47
|
if (this.config.archiveSeconds < 60 || this.config.archiveFailedSeconds < 60) {
|
|
51
48
|
return
|
|
52
49
|
}
|
|
53
50
|
|
|
54
|
-
|
|
51
|
+
this.stopped = false
|
|
52
|
+
|
|
55
53
|
await this.cacheClockSkew()
|
|
56
54
|
|
|
57
55
|
try {
|
|
58
56
|
await this.manager.createQueue(queues.SEND_IT)
|
|
59
57
|
} catch {}
|
|
60
58
|
|
|
61
|
-
|
|
59
|
+
const options = {
|
|
60
|
+
newJobCheckIntervalSeconds: this.config.cronWorkerIntervalSeconds,
|
|
61
|
+
teamSize: 50,
|
|
62
|
+
teamConcurrency: 5
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
await this.manager.work(queues.SEND_IT, options, (job) => this.onSendIt(job))
|
|
62
66
|
|
|
63
67
|
setImmediate(() => this.onCron())
|
|
64
68
|
|
|
65
|
-
// create monitoring interval to make sure cron hasn't crashed
|
|
66
69
|
this.cronMonitorInterval = setInterval(async () => await this.onCron(), this.cronMonitorIntervalMs)
|
|
67
|
-
// create monitoring interval to measure and adjust for drift in clock skew
|
|
68
70
|
this.skewMonitorInterval = setInterval(async () => await this.cacheClockSkew(), this.skewMonitorIntervalMs)
|
|
69
71
|
}
|
|
70
72
|
|
|
@@ -117,8 +119,6 @@ class Timekeeper extends EventEmitter {
|
|
|
117
119
|
}
|
|
118
120
|
|
|
119
121
|
async onCron () {
|
|
120
|
-
let locker
|
|
121
|
-
|
|
122
122
|
try {
|
|
123
123
|
if (this.stopped || this.timekeeping) return
|
|
124
124
|
|
|
@@ -128,19 +128,15 @@ class Timekeeper extends EventEmitter {
|
|
|
128
128
|
|
|
129
129
|
this.timekeeping = true
|
|
130
130
|
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
const { secondsAgo } = await this.getCronTime()
|
|
131
|
+
const { rows } = await this.db.executeSql(this.trySetCronTimeCommand, [this.config.cronMonitorIntervalSeconds])
|
|
134
132
|
|
|
135
|
-
if (
|
|
133
|
+
if (rows.length === 1 && !this.stopped) {
|
|
136
134
|
await this.cron()
|
|
137
|
-
await this.setCronTime()
|
|
138
135
|
}
|
|
139
136
|
} catch (err) {
|
|
140
137
|
this.emit(this.events.error, err)
|
|
141
138
|
} finally {
|
|
142
139
|
this.timekeeping = false
|
|
143
|
-
await locker?.unlock()
|
|
144
140
|
}
|
|
145
141
|
}
|
|
146
142
|
|
|
@@ -198,28 +194,11 @@ class Timekeeper extends EventEmitter {
|
|
|
198
194
|
|
|
199
195
|
const values = [name, cron, tz, data, options]
|
|
200
196
|
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
return result ? result.rowCount : null
|
|
197
|
+
await this.db.executeSql(this.scheduleCommand, values)
|
|
204
198
|
}
|
|
205
199
|
|
|
206
200
|
async unschedule (name) {
|
|
207
|
-
|
|
208
|
-
return result ? result.rowCount : null
|
|
209
|
-
}
|
|
210
|
-
|
|
211
|
-
async setCronTime () {
|
|
212
|
-
await this.db.executeSql(this.setCronTimeCommand)
|
|
213
|
-
}
|
|
214
|
-
|
|
215
|
-
async getCronTime () {
|
|
216
|
-
const { rows } = await this.db.executeSql(this.getCronTimeCommand)
|
|
217
|
-
|
|
218
|
-
let { cron_on: cronOn, seconds_ago: secondsAgo } = rows[0]
|
|
219
|
-
|
|
220
|
-
secondsAgo = secondsAgo !== null ? parseFloat(secondsAgo) : 61
|
|
221
|
-
|
|
222
|
-
return { cronOn, secondsAgo }
|
|
201
|
+
await this.db.executeSql(this.unscheduleCommand, [name])
|
|
223
202
|
}
|
|
224
203
|
}
|
|
225
204
|
|
package/types.d.ts
CHANGED
|
@@ -1,8 +1,24 @@
|
|
|
1
1
|
import { EventEmitter } from 'events'
|
|
2
2
|
|
|
3
3
|
declare namespace PgBoss {
|
|
4
|
+
|
|
5
|
+
type JobStates = {
|
|
6
|
+
created : 'created',
|
|
7
|
+
retry: 'retry',
|
|
8
|
+
active: 'active',
|
|
9
|
+
completed: 'completed',
|
|
10
|
+
cancelled: 'cancelled',
|
|
11
|
+
failed: 'failed'
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
type QueuePolicies = {
|
|
15
|
+
standard: 'standard'
|
|
16
|
+
short: 'short',
|
|
17
|
+
singleton: 'singleton',
|
|
18
|
+
stately: 'stately'
|
|
19
|
+
}
|
|
4
20
|
interface Db {
|
|
5
|
-
executeSql(text: string, values: any[]): Promise<{ rows: any[]
|
|
21
|
+
executeSql(text: string, values: any[]): Promise<{ rows: any[] }>;
|
|
6
22
|
}
|
|
7
23
|
|
|
8
24
|
interface DatabaseOptions {
|
|
@@ -91,10 +107,14 @@ declare namespace PgBoss {
|
|
|
91
107
|
interface ConnectionOptions {
|
|
92
108
|
db?: Db;
|
|
93
109
|
}
|
|
94
|
-
|
|
110
|
+
|
|
95
111
|
type InsertOptions = ConnectionOptions;
|
|
96
|
-
|
|
112
|
+
|
|
97
113
|
type SendOptions = JobOptions & ExpirationOptions & RetentionOptions & RetryOptions & ConnectionOptions;
|
|
114
|
+
|
|
115
|
+
type QueuePolicy = 'standard' | 'short' | 'singleton' | 'stately'
|
|
116
|
+
type Queue = ExpirationOptions & RetentionOptions & RetryOptions & { policy: QueuePolicy }
|
|
117
|
+
type QueueUpdateOptions = ExpirationOptions & RetentionOptions & RetryOptions
|
|
98
118
|
|
|
99
119
|
type ScheduleOptions = SendOptions & { tz?: string }
|
|
100
120
|
|
|
@@ -195,7 +215,7 @@ declare namespace PgBoss {
|
|
|
195
215
|
completedOn: Date | null;
|
|
196
216
|
keepUntil: Date;
|
|
197
217
|
deadLetter: string,
|
|
198
|
-
policy:
|
|
218
|
+
policy: QueuePolicy,
|
|
199
219
|
output: object
|
|
200
220
|
}
|
|
201
221
|
|
|
@@ -271,6 +291,9 @@ declare class PgBoss extends EventEmitter {
|
|
|
271
291
|
static getRollbackPlans(schema: string): string;
|
|
272
292
|
static getRollbackPlans(): string;
|
|
273
293
|
|
|
294
|
+
static states: PgBoss.JobStates
|
|
295
|
+
static policies: PgBoss.QueuePolicies
|
|
296
|
+
|
|
274
297
|
on(event: "error", handler: (error: Error) => void): this;
|
|
275
298
|
off(event: "error", handler: (error: Error) => void): this;
|
|
276
299
|
|
|
@@ -316,10 +339,6 @@ declare class PgBoss extends EventEmitter {
|
|
|
316
339
|
offWork(name: string): Promise<void>;
|
|
317
340
|
offWork(options: PgBoss.OffWorkOptions): Promise<void>;
|
|
318
341
|
|
|
319
|
-
/**
|
|
320
|
-
* Notify worker that something has changed
|
|
321
|
-
* @param workerId
|
|
322
|
-
*/
|
|
323
342
|
notifyWorker(workerId: string): void;
|
|
324
343
|
|
|
325
344
|
subscribe(event: string, name: string): Promise<void>;
|
|
@@ -350,15 +369,19 @@ declare class PgBoss extends EventEmitter {
|
|
|
350
369
|
getQueueSize(name: string, options?: object): Promise<number>;
|
|
351
370
|
getJobById(name: string, id: string, options?: PgBoss.ConnectionOptions): Promise<PgBoss.JobWithMetadata | null>;
|
|
352
371
|
|
|
353
|
-
createQueue(name: string, options?:
|
|
372
|
+
createQueue(name: string, options?: PgBoss.Queue): Promise<void>;
|
|
373
|
+
getQueue(name: string): Promise<PgBoss.Queue | null>;
|
|
374
|
+
updateQueue(name: string, options?: PgBoss.QueueUpdateOptions): Promise<void>;
|
|
354
375
|
deleteQueue(name: string): Promise<void>;
|
|
355
376
|
purgeQueue(name: string): Promise<void>;
|
|
356
|
-
clearStorage(): Promise<void>;
|
|
357
377
|
|
|
378
|
+
clearStorage(): Promise<void>;
|
|
358
379
|
archive(): Promise<void>;
|
|
359
380
|
purge(): Promise<void>;
|
|
360
381
|
expire(): Promise<void>;
|
|
361
382
|
maintain(): Promise<void>;
|
|
383
|
+
isInstalled(): Promise<Boolean>;
|
|
384
|
+
schemaVersion(): Promise<Number>;
|
|
362
385
|
|
|
363
386
|
schedule(name: string, cron: string, data?: object, options?: PgBoss.ScheduleOptions): Promise<void>;
|
|
364
387
|
unschedule(name: string): Promise<void>;
|