pg-boss 9.0.3 → 10.0.0-beta2

package/src/manager.js

@@ -1,21 +1,18 @@
 const assert = require('assert')
 const EventEmitter = require('events')
-const delay = require('delay')
-const uuid = require('uuid')
+const { randomUUID } = require('crypto')
 const debounce = require('lodash.debounce')
 const { serializeError: stringify } = require('serialize-error')
+const pMap = require('p-map')
+const { delay } = require('./tools')
 const Attorney = require('./attorney')
 const Worker = require('./worker')
-const Db = require('./db')
-const pMap = require('p-map')
+const plans = require('./plans')
 
-const { QUEUES: BOSS_QUEUES } = require('./boss')
 const { QUEUES: TIMEKEEPER_QUEUES } = require('./timekeeper')
+const { QUEUE_POLICY } = plans
 
-const INTERNAL_QUEUES = Object.values(BOSS_QUEUES).concat(Object.values(TIMEKEEPER_QUEUES)).reduce((acc, i) => ({ ...acc, [i]: i }), {})
-
-const plans = require('./plans')
-const { COMPLETION_JOB_PREFIX, SINGLETON_QUEUE_KEY } = plans
+const INTERNAL_QUEUES = Object.values(TIMEKEEPER_QUEUES).reduce((acc, i) => ({ ...acc, [i]: i }), {})
 
 const WIP_EVENT_INTERVAL = 2000
 const WIP_DEBOUNCE_OPTIONS = { leading: true, trailing: true, maxWait: WIP_EVENT_INTERVAL }
@@ -27,16 +24,14 @@ const events = {
 
 const resolveWithinSeconds = async (promise, seconds) => {
   const timeout = Math.max(1, seconds) * 1000
-  const reject = delay.reject(timeout, { value: new Error(`handler execution exceeded ${timeout}ms`) })
+  const reject = delay(timeout, `handler execution exceeded ${timeout}ms`)
 
   let result
 
   try {
     result = await Promise.race([promise, reject])
   } finally {
-    try {
-      reject.clear()
-    } catch {}
+    reject.abort()
   }
 
   return result
@@ -58,7 +53,7 @@ class Manager extends EventEmitter {
     this.completeJobsCommand = plans.completeJobs(config.schema)
     this.cancelJobsCommand = plans.cancelJobs(config.schema)
     this.resumeJobsCommand = plans.resumeJobs(config.schema)
-    this.failJobsCommand = plans.failJobs(config.schema)
+    this.failJobsByIdCommand = plans.failJobsById(config.schema)
     this.getJobByIdCommand = plans.getJobById(config.schema)
     this.getArchivedJobByIdCommand = plans.getArchivedJobById(config.schema)
     this.subscribeCommand = plans.subscribe(config.schema)
@@ -72,12 +67,9 @@ class Manager extends EventEmitter {
       this.resume,
       this.fail,
       this.fetch,
-      this.fetchCompleted,
       this.work,
       this.offWork,
       this.notifyWorker,
-      this.onComplete,
-      this.offComplete,
       this.publish,
       this.subscribe,
       this.unsubscribe,
@@ -85,13 +77,14 @@ class Manager extends EventEmitter {
       this.send,
       this.sendDebounced,
       this.sendThrottled,
-      this.sendOnce,
       this.sendAfter,
-      this.sendSingleton,
+      this.createQueue,
+      this.updateQueue,
+      this.getQueue,
       this.deleteQueue,
-      this.deleteAllQueues,
-      this.clearStorage,
+      this.purgeQueue,
       this.getQueueSize,
+      this.clearStorage,
       this.getJobById
     ]
 
@@ -99,27 +92,31 @@ class Manager extends EventEmitter {
   }
 
   start () {
-    this.stopping = false
+    this.stopped = false
   }
 
   async stop () {
-    this.stopping = true
+    this.stopped = true
 
-    for (const sub of this.workers.values()) {
-      if (!INTERNAL_QUEUES[sub.name]) {
-        await this.offWork(sub.name)
+    for (const worker of this.workers.values()) {
+      if (!INTERNAL_QUEUES[worker.name]) {
+        await this.offWork(worker.name)
       }
     }
   }
 
-  async work (name, ...args) {
-    const { options, callback } = Attorney.checkWorkArgs(name, args, this.config)
-    return await this.watch(name, options, callback)
+  async failWip () {
+    for (const worker of this.workers.values()) {
+      const jobIds = worker.jobs.map(j => j.id)
+      if (jobIds.length) {
+        await this.fail(worker.name, jobIds, 'pg-boss shut down while active')
+      }
+    }
   }
 
-  async onComplete (name, ...args) {
+  async work (name, ...args) {
     const { options, callback } = Attorney.checkWorkArgs(name, args, this.config)
-    return await this.watch(COMPLETION_JOB_PREFIX + name, options, callback)
+    return await this.watch(name, options, callback)
   }
 
   addWorker (worker) {
@@ -175,8 +172,8 @@ class Manager extends EventEmitter {
   }
 
   async watch (name, options, callback) {
-    if (this.stopping) {
-      throw new Error('Workers are disabled. pg-boss is stopping.')
+    if (this.stopped) {
+      throw new Error('Workers are disabled. pg-boss is stopped')
     }
 
     const {
@@ -186,10 +183,10 @@ class Manager extends EventEmitter {
       teamConcurrency = 1,
       teamRefill: refill = false,
       includeMetadata = false,
-      enforceSingletonQueueActiveLimit = false
+      priority = true
     } = options
 
-    const id = uuid.v4()
+    const id = randomUUID({ disableEntropyCache: true })
 
     let queueSize = 0
 
@@ -210,7 +207,7 @@ class Manager extends EventEmitter {
       createTeamRefillPromise()
     }
 
-    const fetch = () => this.fetch(name, batchSize || (teamSize - queueSize), { includeMetadata, enforceSingletonQueueActiveLimit })
+    const fetch = () => this.fetch(name, batchSize || (teamSize - queueSize), { includeMetadata, priority })
 
     const onFetch = async (jobs) => {
       if (this.config.__test__throw_worker) {
@@ -223,8 +220,8 @@ class Manager extends EventEmitter {
         const maxExpiration = jobs.reduce((acc, i) => Math.max(acc, i.expire_in_seconds), 0)
 
         await resolveWithinSeconds(Promise.all([callback(jobs)]), maxExpiration)
-          .then(() => this.complete(jobs.map(job => job.id)))
-          .catch(err => this.fail(jobs.map(job => job.id), err))
+          .then(() => this.complete(name, jobs.map(job => job.id)))
+          .catch(err => this.fail(name, jobs.map(job => job.id), err))
       } else {
         if (refill) {
           queueSize += jobs.length || 1
@@ -232,8 +229,8 @@ class Manager extends EventEmitter {
 
         const allTeamPromise = pMap(jobs, job =>
           resolveWithinSeconds(callback(job), job.expire_in_seconds)
-            .then(result => this.complete(job.id, result))
-            .catch(err => this.fail(job.id, err))
+            .then(result => this.complete(name, job.id, result))
+            .catch(err => this.fail(name, job.id, err))
             .then(() => refill ? onRefill() : null)
         , { concurrency: teamConcurrency }
         ).catch(() => {}) // allow promises & non-promises to live together in harmony
@@ -329,39 +326,11 @@ class Manager extends EventEmitter {
     return await Promise.all(result.rows.map(({ name }) => this.send(name, ...args)))
   }
 
-  async offComplete (value) {
-    if (typeof value === 'string') {
-      value = COMPLETION_JOB_PREFIX + value
-    }
-
-    return await this.offWork(value)
-  }
-
   async send (...args) {
     const { name, data, options } = Attorney.checkSendArgs(args, this.config)
     return await this.createJob(name, data, options)
   }
 
-  async sendOnce (name, data, options, key) {
-    options = options ? { ...options } : {}
-
-    options.singletonKey = key || name
-
-    const result = Attorney.checkSendArgs([name, data, options], this.config)
-
-    return await this.createJob(result.name, result.data, result.options)
-  }
-
-  async sendSingleton (name, data, options) {
-    options = options ? { ...options } : {}
-
-    options.singletonKey = SINGLETON_QUEUE_KEY
-
-    const result = Attorney.checkSendArgs([name, data, options], this.config)
-
-    return await this.createJob(result.name, result.data, result.options)
-  }
-
   async sendAfter (name, data, options, after) {
     options = options ? { ...options } : {}
     options.startAfter = after
@@ -395,37 +364,47 @@ class Manager extends EventEmitter {
 
   async createJob (name, data, options, singletonOffset = 0) {
     const {
+      id = null,
       db: wrapper,
-      expireIn,
       priority,
       startAfter,
-      keepUntil,
       singletonKey = null,
       singletonSeconds,
-      retryBackoff,
+      deadLetter = null,
+      expireIn,
+      expireInDefault,
+      keepUntil,
+      keepUntilDefault,
       retryLimit,
+      retryLimitDefault,
       retryDelay,
-      onComplete
+      retryDelayDefault,
+      retryBackoff,
+      retryBackoffDefault
     } = options
 
-    const id = uuid[this.config.uuid]()
-
     const values = [
       id, // 1
       name, // 2
-      priority, // 3
-      retryLimit, // 4
+      data, // 3
+      priority, // 4
       startAfter, // 5
-      expireIn, // 6
-      data, // 7
-      singletonKey, // 8
-      singletonSeconds, // 9
-      singletonOffset, // 10
-      retryDelay, // 11
-      retryBackoff, // 12
-      keepUntil, // 13
-      onComplete // 14
+      singletonKey, // 6
+      singletonSeconds, // 7
+      singletonOffset, // 8
+      deadLetter, // 9
+      expireIn, // 10
+      expireInDefault, // 11
+      keepUntil, // 12
+      keepUntilDefault, // 13
+      retryLimit, // 14
+      retryLimitDefault, // 15
+      retryDelay, // 16
+      retryDelayDefault, // 17
+      retryBackoff, // 18
+      retryBackoffDefault // 19
     ]
+
     const db = wrapper || this.db
     const result = await db.executeSql(this.insertJobCommand, values)
 
@@ -449,11 +428,20 @@ class Manager extends EventEmitter {
   }
 
   async insert (jobs, options = {}) {
-    const { db: wrapper } = options
-    const db = wrapper || this.db
-    const checkedJobs = Attorney.checkInsertArgs(jobs)
-    const data = JSON.stringify(checkedJobs)
-    return await db.executeSql(this.insertJobsCommand, [data])
+    assert(Array.isArray(jobs), 'jobs argument should be an array')
+
+    const db = options.db || this.db
+
+    const params = [
+      JSON.stringify(jobs), // 1
+      this.config.expireIn, // 2
+      this.config.keepUntil, // 3
+      this.config.retryLimit, // 4
+      this.config.retryDelay, // 5
+      this.config.retryBackoff // 6
+    ]
+
+    return await db.executeSql(this.insertJobsCommand, params)
   }
 
   getDebounceStartAfter (singletonSeconds, clockOffset) {
@@ -476,25 +464,15 @@ class Manager extends EventEmitter {
   async fetch (name, batchSize, options = {}) {
     const values = Attorney.checkFetchArgs(name, batchSize, options)
     const db = options.db || this.db
-    const preparedStatement = this.nextJobCommand(options.includeMetadata || false, options.enforceSingletonQueueActiveLimit || false)
+    const nextJobSql = this.nextJobCommand({ ...options })
     const statementValues = [values.name, batchSize || 1]
 
     let result
-    if (options.enforceSingletonQueueActiveLimit && !options.db) {
-      // Prepare/format now and send multi-statement transaction
-      const fetchQuery = preparedStatement
-        .replace('$1', Db.quotePostgresStr(statementValues[0]))
-        .replace('$2', statementValues[1].toString())
-      // eslint-disable-next-line no-unused-vars
-      const [_begin, _setLocal, fetchResult, _commit] = await db.executeSql([
-        'BEGIN',
-        'SET LOCAL jit = OFF', // JIT can slow things down significantly
-        fetchQuery,
-        'COMMIT'
-      ].join(';\n'))
-      result = fetchResult
-    } else {
-      result = await db.executeSql(preparedStatement, statementValues)
+
+    try {
+      result = await db.executeSql(nextJobSql, statementValues)
+    } catch (err) {
+      // errors from fetchquery should only be unique constraint violations
     }
 
     if (!result || result.rows.length === 0) {
@@ -504,10 +482,6 @@ class Manager extends EventEmitter {
     return result.rows.length === 1 && !batchSize ? result.rows[0] : result.rows
   }
 
-  async fetchCompleted (name, batchSize, options = {}) {
-    return await this.fetch(COMPLETION_JOB_PREFIX + name, batchSize, options)
-  }
-
   mapCompletionIdArg (id, funcName) {
     const errorMessage = `${funcName}() requires an id`
 
@@ -538,45 +512,154 @@ class Manager extends EventEmitter {
     }
   }
 
-  async complete (id, data, options = {}) {
+  async complete (name, id, data, options = {}) {
+    assert(name, 'Missing queue name argument')
     const db = options.db || this.db
     const ids = this.mapCompletionIdArg(id, 'complete')
-    const result = await db.executeSql(this.completeJobsCommand, [ids, this.mapCompletionDataArg(data)])
+    const result = await db.executeSql(this.completeJobsCommand, [name, ids, this.mapCompletionDataArg(data)])
     return this.mapCompletionResponse(ids, result)
   }
 
-  async fail (id, data, options = {}) {
+  async fail (name, id, data, options = {}) {
+    assert(name, 'Missing queue name argument')
     const db = options.db || this.db
     const ids = this.mapCompletionIdArg(id, 'fail')
-    const result = await db.executeSql(this.failJobsCommand, [ids, this.mapCompletionDataArg(data)])
+    const result = await db.executeSql(this.failJobsByIdCommand, [name, ids, this.mapCompletionDataArg(data)])
     return this.mapCompletionResponse(ids, result)
   }
 
-  async cancel (id, options = {}) {
+  async cancel (name, id, options = {}) {
+    assert(name, 'Missing queue name argument')
     const db = options.db || this.db
     const ids = this.mapCompletionIdArg(id, 'cancel')
-    const result = await db.executeSql(this.cancelJobsCommand, [ids])
+    const result = await db.executeSql(this.cancelJobsCommand, [name, ids])
     return this.mapCompletionResponse(ids, result)
   }
 
-  async resume (id, options = {}) {
+  async resume (name, id, options = {}) {
+    assert(name, 'Missing queue name argument')
     const db = options.db || this.db
     const ids = this.mapCompletionIdArg(id, 'resume')
-    const result = await db.executeSql(this.resumeJobsCommand, [ids])
+    const result = await db.executeSql(this.resumeJobsCommand, [name, ids])
     return this.mapCompletionResponse(ids, result)
   }
 
-  async deleteQueue (queue, options) {
-    assert(queue, 'Missing queue name argument')
-    const sql = plans.deleteQueue(this.config.schema, options)
-    const result = await this.db.executeSql(sql, [queue])
-    return result ? result.rowCount : null
+  async createQueue (name, options = {}) {
+    assert(name, 'Missing queue name argument')
+
+    const { policy = QUEUE_POLICY.standard } = options
+
+    assert(policy in QUEUE_POLICY, `${policy} is not a valid queue policy`)
+
+    const {
+      retryLimit,
+      retryDelay,
+      retryBackoff,
+      expireInSeconds,
+      retentionMinutes,
+      deadLetter
+    } = Attorney.checkQueueArgs(name, options)
+
+    const paritionSql = plans.partitionCreateJobName(this.config.schema, name)
+
+    await this.db.executeSql(paritionSql)
+
+    const sql = plans.createQueue(this.config.schema, name)
+
+    const params = [
+      name,
+      policy,
+      retryLimit,
+      retryDelay,
+      retryBackoff,
+      expireInSeconds,
+      retentionMinutes,
+      deadLetter
+    ]
+
+    await this.db.executeSql(sql, params)
   }
 
-  async deleteAllQueues (options) {
-    const sql = plans.deleteAllQueues(this.config.schema, options)
-    const result = await this.db.executeSql(sql)
-    return result ? result.rowCount : null
+  async updateQueue (name, options = {}) {
+    assert(name, 'Missing queue name argument')
+
+    const {
+      retryLimit,
+      retryDelay,
+      retryBackoff,
+      expireInSeconds,
+      retentionMinutes,
+      deadLetter
+    } = Attorney.checkQueueArgs(name, options)
+
+    const sql = plans.updateQueue(this.config.schema)
+
+    const params = [
+      name,
+      retryLimit,
+      retryDelay,
+      retryBackoff,
+      expireInSeconds,
+      retentionMinutes,
+      deadLetter
+    ]
+
+    await this.db.executeSql(sql, params)
+  }
+
+  async getQueue (name) {
+    assert(name, 'Missing queue name argument')
+
+    const sql = plans.getQueueByName(this.config.schema)
+    const result = await this.db.executeSql(sql, [name])
+
+    if (result.rows.length === 0) {
+      return null
+    }
+
+    const {
+      policy,
+      retry_limit: retryLimit,
+      retry_delay: retryDelay,
+      retry_backoff: retryBackoff,
+      expire_seconds: expireInSeconds,
+      retention_minutes: retentionMinutes,
+      dead_letter: deadLetter
+    } = result.rows[0]
+
+    return {
+      name,
+      policy,
+      retryLimit,
+      retryDelay,
+      retryBackoff,
+      expireInSeconds,
+      retentionMinutes,
+      deadLetter
+    }
+  }
+
+  async deleteQueue (name) {
+    assert(name, 'Missing queue name argument')
+
+    const queueSql = plans.getQueueByName(this.config.schema)
+    const result = await this.db.executeSql(queueSql, [name])
+
+    if (result?.rows?.length) {
+      Attorney.assertPostgresObjectName(name)
+      const sql = plans.dropJobTablePartition(this.config.schema, name)
+      await this.db.executeSql(sql)
+    }
+
+    const sql = plans.deleteQueueRecords(this.config.schema)
+    const result2 = await this.db.executeSql(sql, [name])
+    return result2?.rowCount || null
+  }
+
+  async purgeQueue (queue) {
+    assert(queue, 'Missing queue name argument')
+    const sql = plans.purgeQueue(this.config.schema)
+    await this.db.executeSql(sql, [queue])
   }
 
   async clearStorage () {
@@ -594,15 +677,15 @@ class Manager extends EventEmitter {
     return result ? parseFloat(result.rows[0].count) : null
   }
 
-  async getJobById (id, options = {}) {
+  async getJobById (queue, id, options = {}) {
     const db = options.db || this.db
-    const result1 = await db.executeSql(this.getJobByIdCommand, [id])
+    const result1 = await db.executeSql(this.getJobByIdCommand, [queue, id])
 
     if (result1 && result1.rows && result1.rows.length === 1) {
       return result1.rows[0]
     }
 
-    const result2 = await db.executeSql(this.getArchivedJobByIdCommand, [id])
+    const result2 = await db.executeSql(this.getArchivedJobByIdCommand, [queue, id])
 
     if (result2 && result2.rows && result2.rows.length === 1) {
       return result2.rows[0]

package/src/migrationStore.js

@@ -64,110 +64,5 @@ function migrate (value, version, migrations) {
 
 function getAll (schema) {
   return [
-    {
-      release: '7.4.0',
-      version: 20,
-      previous: 19,
-      install: [
-        `DROP INDEX ${schema}.job_singletonKey`,
-        `DROP INDEX ${schema}.job_singleton_queue`,
-        `CREATE UNIQUE INDEX job_singletonKey ON ${schema}.job (name, singletonKey) WHERE state < 'completed' AND singletonOn IS NULL AND NOT singletonKey LIKE '\\_\\_pgboss\\_\\_singleton\\_queue%'`,
-        `CREATE UNIQUE INDEX job_singleton_queue ON ${schema}.job (name, singletonKey) WHERE state < 'active' AND singletonOn IS NULL AND singletonKey LIKE '\\_\\_pgboss\\_\\_singleton\\_queue%'`
-      ],
-      uninstall: [
-        `DROP INDEX ${schema}.job_singletonKey`,
-        `DROP INDEX ${schema}.job_singleton_queue`,
-        `CREATE UNIQUE INDEX job_singletonKey ON ${schema}.job (name, singletonKey) WHERE state < 'completed' AND singletonOn IS NULL AND NOT singletonKey = '__pgboss__singleton_queue'`,
-        `CREATE UNIQUE INDEX job_singleton_queue ON ${schema}.job (name, singletonKey) WHERE state < 'active' AND singletonOn IS NULL AND singletonKey = '__pgboss__singleton_queue'`
-      ]
-    },
-    {
-      release: '7.0.0',
-      version: 19,
-      previous: 18,
-      install: [
-        `CREATE TABLE ${schema}.subscription (
-          event text not null,
-          name text not null,
-          created_on timestamp with time zone not null default now(),
-          updated_on timestamp with time zone not null default now(),
-          PRIMARY KEY(event, name)
-        )`
-      ],
-      uninstall: [
-        `DROP TABLE ${schema}.subscription`
-      ]
-    },
-    {
-      release: '6.1.1',
-      version: 18,
-      previous: 17,
-      install: [
-        `ALTER TABLE ${schema}.job ALTER COLUMN on_complete SET DEFAULT false`
-      ]
-    },
-    {
-      release: '6.0.0',
-      version: 17,
-      previous: 16,
-      install: [
-        `DROP INDEX ${schema}.job_singletonKey`,
-        `CREATE UNIQUE INDEX job_singletonKey ON ${schema}.job (name, singletonKey) WHERE state < 'completed' AND singletonOn IS NULL AND NOT singletonKey = '__pgboss__singleton_queue'`,
-        `CREATE UNIQUE INDEX job_singleton_queue ON ${schema}.job (name, singletonKey) WHERE state < 'active' AND singletonOn IS NULL AND singletonKey = '__pgboss__singleton_queue'`,
-        `CREATE INDEX IF NOT EXISTS job_fetch ON ${schema}.job (name text_pattern_ops, startAfter) WHERE state < 'active'`,
-        `ALTER TABLE ${schema}.job ADD output jsonb`,
-        `ALTER TABLE ${schema}.archive ADD output jsonb`,
-        `ALTER TABLE ${schema}.job ALTER COLUMN on_complete SET DEFAULT false`,
-        `ALTER TABLE ${schema}.job ALTER COLUMN keepuntil SET DEFAULT now() + interval '14 days'`
-      ],
-      uninstall: [
-        `DROP INDEX ${schema}.job_fetch`,
-        `DROP INDEX ${schema}.job_singleton_queue`,
-        `DROP INDEX ${schema}.job_singletonKey`,
-        `CREATE UNIQUE INDEX job_singletonKey ON ${schema}.job (name, singletonKey) WHERE state < 'completed' AND singletonOn IS NULL`,
-        `ALTER TABLE ${schema}.job DROP COLUMN output`,
-        `ALTER TABLE ${schema}.archive DROP COLUMN output`,
-        `ALTER TABLE ${schema}.job ALTER COLUMN on_complete SET DEFAULT true`,
-        `ALTER TABLE ${schema}.job ALTER COLUMN keepuntil SET DEFAULT now() + interval '30 days'`
-      ]
-    },
-    {
-      release: '5.2.0',
-      version: 16,
-      previous: 15,
-      install: [
-          `ALTER TABLE ${schema}.job ADD on_complete boolean`,
-          `UPDATE ${schema}.job SET on_complete = true`,
-          `ALTER TABLE ${schema}.job ALTER COLUMN on_complete SET DEFAULT true`,
-          `ALTER TABLE ${schema}.job ALTER COLUMN on_complete SET NOT NULL`,
-          `ALTER TABLE ${schema}.archive ADD on_complete boolean`
-      ],
-      uninstall: [
-          `ALTER TABLE ${schema}.job DROP COLUMN on_complete`,
-          `ALTER TABLE ${schema}.archive DROP COLUMN on_complete`
-      ]
-    },
-    {
-      release: '5.0.6',
-      version: 15,
-      previous: 14,
-      install: [
-          `ALTER TABLE ${schema}.version ADD cron_on timestamp with time zone`
-      ],
-      uninstall: [
-          `ALTER TABLE ${schema}.version DROP COLUMN cron_on`
-      ]
-    },
-    {
-      release: '5.0.0',
-      version: 14,
-      previous: 13,
-      install: [
-          `ALTER TABLE ${schema}.version ADD maintained_on timestamp with time zone`
-      ],
-      uninstall: [
-          `ALTER TABLE ${schema}.version DROP COLUMN maintained_on`
-      ]
-    }
   ]
 }