pg-boss 10.0.0-beta2 → 10.0.0-beta21

package/src/manager.js

@@ -1,22 +1,17 @@
 const assert = require('assert')
 const EventEmitter = require('events')
 const { randomUUID } = require('crypto')
-const debounce = require('lodash.debounce')
 const { serializeError: stringify } = require('serialize-error')
-const pMap = require('p-map')
 const { delay } = require('./tools')
 const Attorney = require('./attorney')
 const Worker = require('./worker')
 const plans = require('./plans')
 
 const { QUEUES: TIMEKEEPER_QUEUES } = require('./timekeeper')
-const { QUEUE_POLICY } = plans
+const { QUEUE_POLICIES } = plans
 
 const INTERNAL_QUEUES = Object.values(TIMEKEEPER_QUEUES).reduce((acc, i) => ({ ...acc, [i]: i }), {})
 
-const WIP_EVENT_INTERVAL = 2000
-const WIP_DEBOUNCE_OPTIONS = { leading: true, trailing: true, maxWait: WIP_EVENT_INTERVAL }
-
 const events = {
   error: 'error',
   wip: 'wip'
@@ -45,6 +40,7 @@ class Manager extends EventEmitter {
     this.db = db
 
     this.events = events
+    this.wipTs = Date.now()
     this.workers = new Map()
 
     this.nextJobCommand = plans.fetchNextJob(config.schema)
@@ -53,18 +49,27 @@ class Manager extends EventEmitter {
     this.completeJobsCommand = plans.completeJobs(config.schema)
     this.cancelJobsCommand = plans.cancelJobs(config.schema)
     this.resumeJobsCommand = plans.resumeJobs(config.schema)
+    this.deleteJobsCommand = plans.deleteJobs(config.schema)
     this.failJobsByIdCommand = plans.failJobsById(config.schema)
     this.getJobByIdCommand = plans.getJobById(config.schema)
     this.getArchivedJobByIdCommand = plans.getArchivedJobById(config.schema)
     this.subscribeCommand = plans.subscribe(config.schema)
     this.unsubscribeCommand = plans.unsubscribe(config.schema)
+    this.getQueuesCommand = plans.getQueues(config.schema)
+    this.getQueueByNameCommand = plans.getQueueByName(config.schema)
     this.getQueuesForEventCommand = plans.getQueuesForEvent(config.schema)
+    this.createQueueCommand = plans.createQueue(config.schema)
+    this.updateQueueCommand = plans.updateQueue(config.schema)
+    this.purgeQueueCommand = plans.purgeQueue(config.schema)
+    this.deleteQueueCommand = plans.deleteQueue(config.schema)
+    this.clearStorageCommand = plans.clearStorage(config.schema)
 
     // exported api to index
     this.functions = [
       this.complete,
       this.cancel,
       this.resume,
+      this.deleteJob,
       this.fail,
       this.fetch,
       this.work,
@@ -80,15 +85,14 @@ class Manager extends EventEmitter {
       this.sendAfter,
       this.createQueue,
       this.updateQueue,
-      this.getQueue,
       this.deleteQueue,
       this.purgeQueue,
       this.getQueueSize,
+      this.getQueue,
+      this.getQueues,
       this.clearStorage,
       this.getJobById
     ]
-
-    this.emitWipThrottled = debounce(() => this.emit(events.wip, this.getWipData()), WIP_EVENT_INTERVAL, WIP_DEBOUNCE_OPTIONS)
   }
 
   start () {
@@ -133,7 +137,12 @@ class Manager extends EventEmitter {
 
   emitWip (name) {
     if (!INTERNAL_QUEUES[name]) {
-      this.emitWipThrottled()
+      const now = Date.now()
+
+      if (now - this.wipTs > 2000) {
+        this.emit(events.wip, this.getWipData())
+        this.wipTs = now
+      }
     }
   }
 
@@ -177,73 +186,35 @@ class Manager extends EventEmitter {
     }
 
     const {
-      newJobCheckInterval: interval = this.config.newJobCheckInterval,
+      pollingInterval: interval = this.config.pollingInterval,
       batchSize,
-      teamSize = 1,
-      teamConcurrency = 1,
-      teamRefill: refill = false,
       includeMetadata = false,
       priority = true
     } = options
 
     const id = randomUUID({ disableEntropyCache: true })
 
-    let queueSize = 0
-
-    let refillTeamPromise
-    let resolveRefillTeam
-
-    // Setup a promise that onFetch can await for when at least one
-    // job is finished and so the team is ready to be topped up
-    const createTeamRefillPromise = () => {
-      refillTeamPromise = new Promise((resolve) => { resolveRefillTeam = resolve })
-    }
-
-    createTeamRefillPromise()
-
-    const onRefill = () => {
-      queueSize--
-      resolveRefillTeam()
-      createTeamRefillPromise()
-    }
-
-    const fetch = () => this.fetch(name, batchSize || (teamSize - queueSize), { includeMetadata, priority })
+    const fetch = () => this.fetch(name, { batchSize, includeMetadata, priority })
 
     const onFetch = async (jobs) => {
+      if (!jobs.length) {
+        return
+      }
+
       if (this.config.__test__throw_worker) {
         throw new Error('__test__throw_worker')
       }
 
       this.emitWip(name)
 
-      if (batchSize) {
-        const maxExpiration = jobs.reduce((acc, i) => Math.max(acc, i.expire_in_seconds), 0)
-
-        await resolveWithinSeconds(Promise.all([callback(jobs)]), maxExpiration)
-          .then(() => this.complete(name, jobs.map(job => job.id)))
-          .catch(err => this.fail(name, jobs.map(job => job.id), err))
-      } else {
-        if (refill) {
-          queueSize += jobs.length || 1
-        }
-
-        const allTeamPromise = pMap(jobs, job =>
-          resolveWithinSeconds(callback(job), job.expire_in_seconds)
-            .then(result => this.complete(name, job.id, result))
-            .catch(err => this.fail(name, job.id, err))
-            .then(() => refill ? onRefill() : null)
-        , { concurrency: teamConcurrency }
-        ).catch(() => {}) // allow promises & non-promises to live together in harmony
-
-        if (refill) {
-          if (queueSize < teamSize) {
-            return
-          } else {
-            await refillTeamPromise
-          }
-        } else {
-          await allTeamPromise
-        }
+      const maxExpiration = jobs.reduce((acc, i) => Math.max(acc, i.expireInSeconds), 0)
+      const jobIds = jobs.map(job => job.id)
+
+      try {
+        const result = await resolveWithinSeconds(callback(jobs), maxExpiration)
+        this.complete(name, jobIds, jobIds.length === 1 ? result : undefined)
+      } catch (err) {
+        this.fail(name, jobIds, err)
       }
 
       this.emitWip(name)
@@ -317,13 +288,9 @@ class Manager extends EventEmitter {
   async publish (event, ...args) {
     assert(event, 'Missing required argument')
 
-    const result = await this.db.executeSql(this.getQueuesForEventCommand, [event])
+    const { rows } = await this.db.executeSql(this.getQueuesForEventCommand, [event])
 
-    if (!result || result.rowCount === 0) {
-      return []
-    }
-
-    return await Promise.all(result.rows.map(({ name }) => this.send(name, ...args)))
+    await Promise.allSettled(rows.map(({ name }) => this.send(name, ...args)))
   }
 
   async send (...args) {
@@ -406,10 +373,10 @@ class Manager extends EventEmitter {
     ]
 
     const db = wrapper || this.db
-    const result = await db.executeSql(this.insertJobCommand, values)
+    const { rows } = await db.executeSql(this.insertJobCommand, values)
 
-    if (result && result.rowCount === 1) {
-      return result.rows[0].id
+    if (rows.length === 1) {
+      return rows[0].id
     }
 
     if (!options.singletonNextSlot) {
@@ -461,25 +428,20 @@ class Manager extends EventEmitter {
     return startAfter
   }
 
-  async fetch (name, batchSize, options = {}) {
-    const values = Attorney.checkFetchArgs(name, batchSize, options)
+  async fetch (name, options = {}) {
+    Attorney.checkFetchArgs(name, options)
     const db = options.db || this.db
     const nextJobSql = this.nextJobCommand({ ...options })
-    const statementValues = [values.name, batchSize || 1]
 
     let result
 
     try {
-      result = await db.executeSql(nextJobSql, statementValues)
+      result = await db.executeSql(nextJobSql, [name, options.batchSize])
     } catch (err) {
       // errors from fetchquery should only be unique constraint violations
     }
 
-    if (!result || result.rows.length === 0) {
-      return null
-    }
-
-    return result.rows.length === 1 && !batchSize ? result.rows[0] : result.rows
+    return result?.rows || []
   }
 
   mapCompletionIdArg (id, funcName) {
@@ -504,52 +466,62 @@ class Manager extends EventEmitter {
     return stringify(result)
   }
 
-  mapCompletionResponse (ids, result) {
+  mapCommandResponse (ids, result) {
     return {
       jobs: ids,
       requested: ids.length,
-      updated: result && result.rows ? parseInt(result.rows[0].count) : 0
+      affected: result && result.rows ? parseInt(result.rows[0].count) : 0
     }
   }
 
   async complete (name, id, data, options = {}) {
-    assert(name, 'Missing queue name argument')
+    Attorney.assertQueueName(name)
     const db = options.db || this.db
     const ids = this.mapCompletionIdArg(id, 'complete')
     const result = await db.executeSql(this.completeJobsCommand, [name, ids, this.mapCompletionDataArg(data)])
-    return this.mapCompletionResponse(ids, result)
+    return this.mapCommandResponse(ids, result)
   }
 
   async fail (name, id, data, options = {}) {
-    assert(name, 'Missing queue name argument')
+    Attorney.assertQueueName(name)
     const db = options.db || this.db
     const ids = this.mapCompletionIdArg(id, 'fail')
     const result = await db.executeSql(this.failJobsByIdCommand, [name, ids, this.mapCompletionDataArg(data)])
-    return this.mapCompletionResponse(ids, result)
+    return this.mapCommandResponse(ids, result)
   }
 
   async cancel (name, id, options = {}) {
-    assert(name, 'Missing queue name argument')
+    Attorney.assertQueueName(name)
     const db = options.db || this.db
     const ids = this.mapCompletionIdArg(id, 'cancel')
     const result = await db.executeSql(this.cancelJobsCommand, [name, ids])
-    return this.mapCompletionResponse(ids, result)
+    return this.mapCommandResponse(ids, result)
+  }
+
+  async deleteJob (name, id, options = {}) {
+    Attorney.assertQueueName(name)
+    const db = options.db || this.db
+    const ids = this.mapCompletionIdArg(id, 'deleteJob')
+    const result = await db.executeSql(this.deleteJobsCommand, [name, ids])
+    return this.mapCommandResponse(ids, result)
   }
 
   async resume (name, id, options = {}) {
-    assert(name, 'Missing queue name argument')
+    Attorney.assertQueueName(name)
     const db = options.db || this.db
     const ids = this.mapCompletionIdArg(id, 'resume')
     const result = await db.executeSql(this.resumeJobsCommand, [name, ids])
-    return this.mapCompletionResponse(ids, result)
+    return this.mapCommandResponse(ids, result)
   }
 
   async createQueue (name, options = {}) {
-    assert(name, 'Missing queue name argument')
+    name = name || options.name
 
-    const { policy = QUEUE_POLICY.standard } = options
+    Attorney.assertQueueName(name)
 
-    assert(policy in QUEUE_POLICY, `${policy} is not a valid queue policy`)
+    const { policy = QUEUE_POLICIES.standard } = options
+
+    assert(policy in QUEUE_POLICIES, `${policy} is not a valid queue policy`)
 
     const {
       retryLimit,
@@ -560,14 +532,12 @@ class Manager extends EventEmitter {
       deadLetter
     } = Attorney.checkQueueArgs(name, options)
 
-    const paritionSql = plans.partitionCreateJobName(this.config.schema, name)
-
-    await this.db.executeSql(paritionSql)
-
-    const sql = plans.createQueue(this.config.schema, name)
+    if (deadLetter) {
+      Attorney.assertQueueName(deadLetter)
+    }
 
-    const params = [
-      name,
+    // todo: pull in defaults from constructor config
+    const data = {
       policy,
       retryLimit,
       retryDelay,
@@ -575,13 +545,22 @@ class Manager extends EventEmitter {
       expireInSeconds,
       retentionMinutes,
       deadLetter
-    ]
+    }
 
-    await this.db.executeSql(sql, params)
+    await this.db.executeSql(this.createQueueCommand, [name, data])
+  }
+
+  async getQueues () {
+    const { rows } = await this.db.executeSql(this.getQueuesCommand)
+    return rows
   }
 
   async updateQueue (name, options = {}) {
-    assert(name, 'Missing queue name argument')
+    Attorney.assertQueueName(name)
+
+    const { policy = QUEUE_POLICIES.standard } = options
+
+    assert(policy in QUEUE_POLICIES, `${policy} is not a valid queue policy`)
 
     const {
       retryLimit,
@@ -592,10 +571,9 @@ class Manager extends EventEmitter {
       deadLetter
     } = Attorney.checkQueueArgs(name, options)
 
-    const sql = plans.updateQueue(this.config.schema)
-
     const params = [
       name,
+      policy,
       retryLimit,
       retryDelay,
       retryBackoff,
@@ -604,94 +582,61 @@ class Manager extends EventEmitter {
       deadLetter
     ]
 
-    await this.db.executeSql(sql, params)
+    await this.db.executeSql(this.updateQueueCommand, params)
   }
 
   async getQueue (name) {
-    assert(name, 'Missing queue name argument')
-
-    const sql = plans.getQueueByName(this.config.schema)
-    const result = await this.db.executeSql(sql, [name])
+    Attorney.assertQueueName(name)
 
-    if (result.rows.length === 0) {
-      return null
-    }
-
-    const {
-      policy,
-      retry_limit: retryLimit,
-      retry_delay: retryDelay,
-      retry_backoff: retryBackoff,
-      expire_seconds: expireInSeconds,
-      retention_minutes: retentionMinutes,
-      dead_letter: deadLetter
-    } = result.rows[0]
+    const { rows } = await this.db.executeSql(this.getQueueByNameCommand, [name])
 
-    return {
-      name,
-      policy,
-      retryLimit,
-      retryDelay,
-      retryBackoff,
-      expireInSeconds,
-      retentionMinutes,
-      deadLetter
-    }
+    return rows[0] || null
   }
 
   async deleteQueue (name) {
-    assert(name, 'Missing queue name argument')
+    Attorney.assertQueueName(name)
 
-    const queueSql = plans.getQueueByName(this.config.schema)
-    const result = await this.db.executeSql(queueSql, [name])
+    const { rows } = await this.db.executeSql(this.getQueueByNameCommand, [name])
 
-    if (result?.rows?.length) {
-      Attorney.assertPostgresObjectName(name)
-      const sql = plans.dropJobTablePartition(this.config.schema, name)
-      await this.db.executeSql(sql)
+    if (rows.length === 1) {
+      await this.db.executeSql(this.deleteQueueCommand, [name])
     }
-
-    const sql = plans.deleteQueueRecords(this.config.schema)
-    const result2 = await this.db.executeSql(sql, [name])
-    return result2?.rowCount || null
   }
 
-  async purgeQueue (queue) {
-    assert(queue, 'Missing queue name argument')
-    const sql = plans.purgeQueue(this.config.schema)
-    await this.db.executeSql(sql, [queue])
+  async purgeQueue (name) {
+    Attorney.assertQueueName(name)
+    await this.db.executeSql(this.purgeQueueCommand, [name])
   }
 
   async clearStorage () {
-    const sql = plans.clearStorage(this.config.schema)
-    await this.db.executeSql(sql)
+    await this.db.executeSql(this.clearStorageCommand)
   }
 
-  async getQueueSize (queue, options) {
-    assert(queue, 'Missing queue name argument')
+  async getQueueSize (name, options) {
+    Attorney.assertQueueName(name)
 
     const sql = plans.getQueueSize(this.config.schema, options)
 
-    const result = await this.db.executeSql(sql, [queue])
+    const result = await this.db.executeSql(sql, [name])
 
     return result ? parseFloat(result.rows[0].count) : null
   }
 
-  async getJobById (queue, id, options = {}) {
-    const db = options.db || this.db
-    const result1 = await db.executeSql(this.getJobByIdCommand, [queue, id])
+  async getJobById (name, id, options = {}) {
+    Attorney.assertQueueName(name)
 
-    if (result1 && result1.rows && result1.rows.length === 1) {
-      return result1.rows[0]
-    }
+    const db = options.db || this.db
 
-    const result2 = await db.executeSql(this.getArchivedJobByIdCommand, [queue, id])
+    const result1 = await db.executeSql(this.getJobByIdCommand, [name, id])
 
-    if (result2 && result2.rows && result2.rows.length === 1) {
-      return result2.rows[0]
+    if (result1?.rows?.length === 1) {
+      return result1.rows[0]
+    } else if (options.includeArchive) {
+      const result2 = await db.executeSql(this.getArchivedJobByIdCommand, [name, id])
+      return result2?.rows[0] || null
+    } else {
+      return null
     }
-
-    return null
   }
 }