pg-boss 10.0.0-beta16 → 10.0.0-beta18

package/README.md

@@ -19,7 +19,7 @@ async function readme() {
 
   console.log(`created job ${id} in queue ${queue}`)
 
-  await boss.work(queue, async job => {
+  await boss.work(queue, async ([ job ]) => {
     console.log(`received job ${job.id} with data ${JSON.stringify(job.data)}`)
   })
 }
@@ -47,7 +47,7 @@ This will likely cater the most to teams already familiar with the simplicity of
 
 ## Installation
 
-``` bash
+```bash
 # npm
 npm install pg-boss
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pg-boss",
-  "version": "10.0.0-beta16",
+  "version": "10.0.0-beta18",
   "description": "Queueing jobs in Postgres from Node.js like a boss",
   "main": "./src/index.js",
   "engines": {
@@ -8,7 +8,6 @@
   },
   "dependencies": {
     "cron-parser": "^4.0.0",
-    "p-map": "^4.0.0",
     "pg": "^8.5.1",
     "serialize-error": "^8.1.0"
   },

package/src/attorney.js

@@ -124,24 +124,23 @@ function checkWorkArgs (name, args, defaults) {
 
   applyPollingInterval(options, defaults)
 
-  assert(!('teamConcurrency' in options) ||
-    (Number.isInteger(options.teamConcurrency) && options.teamConcurrency >= 1 && options.teamConcurrency <= 1000),
-  'teamConcurrency must be an integer between 1 and 1000')
-
-  assert(!('teamSize' in options) || (Number.isInteger(options.teamSize) && options.teamSize >= 1), 'teamSize must be an integer > 0')
   assert(!('batchSize' in options) || (Number.isInteger(options.batchSize) && options.batchSize >= 1), 'batchSize must be an integer > 0')
   assert(!('includeMetadata' in options) || typeof options.includeMetadata === 'boolean', 'includeMetadata must be a boolean')
+  assert(!('priority' in options) || typeof options.priority === 'boolean', 'priority must be a boolean')
+
+  options.batchSize = options.batchSize || 1
 
   return { options, callback }
 }
 
-function checkFetchArgs (name, batchSize, options) {
+function checkFetchArgs (name, options) {
   assert(name, 'missing queue name')
 
-  assert(!batchSize || (Number.isInteger(batchSize) && batchSize >= 1), 'batchSize must be an integer > 0')
+  assert(!('batchSize' in options) || (Number.isInteger(options.batchSize) && options.batchSize >= 1), 'batchSize must be an integer > 0')
   assert(!('includeMetadata' in options) || typeof options.includeMetadata === 'boolean', 'includeMetadata must be a boolean')
+  assert(!('priority' in options) || typeof options.priority === 'boolean', 'priority must be a boolean')
 
-  return { name }
+  options.batchSize = options.batchSize || 1
 }
 
 function getConfig (value) {

package/src/manager.js

@@ -2,7 +2,6 @@ const assert = require('assert')
 const EventEmitter = require('events')
 const { randomUUID } = require('crypto')
 const { serializeError: stringify } = require('serialize-error')
-const pMap = require('p-map')
 const { delay } = require('./tools')
 const Attorney = require('./attorney')
 const Worker = require('./worker')
@@ -50,27 +49,27 @@ class Manager extends EventEmitter {
     this.completeJobsCommand = plans.completeJobs(config.schema)
     this.cancelJobsCommand = plans.cancelJobs(config.schema)
     this.resumeJobsCommand = plans.resumeJobs(config.schema)
+    this.deleteJobsCommand = plans.deleteJobs(config.schema)
     this.failJobsByIdCommand = plans.failJobsById(config.schema)
     this.getJobByIdCommand = plans.getJobById(config.schema)
     this.getArchivedJobByIdCommand = plans.getArchivedJobById(config.schema)
     this.subscribeCommand = plans.subscribe(config.schema)
     this.unsubscribeCommand = plans.unsubscribe(config.schema)
     this.getQueuesCommand = plans.getQueues(config.schema)
-    this.getQueuesForEventCommand = plans.getQueuesForEvent(config.schema)
     this.getQueueByNameCommand = plans.getQueueByName(config.schema)
-    this.deleteQueueRecordsCommand = plans.deleteQueueRecords(config.schema)
-    this.insertQueueCommand = plans.insertQueue(config.schema)
+    this.getQueuesForEventCommand = plans.getQueuesForEvent(config.schema)
+    this.createQueueCommand = plans.createQueue(config.schema)
     this.updateQueueCommand = plans.updateQueue(config.schema)
-    this.createPartitionCommand = plans.createPartition(config.schema)
-    this.dropPartitionCommand = plans.dropPartition(config.schema)
-    this.clearStorageCommand = plans.clearStorage(config.schema)
     this.purgeQueueCommand = plans.purgeQueue(config.schema)
+    this.deleteQueueCommand = plans.deleteQueue(config.schema)
+    this.clearStorageCommand = plans.clearStorage(config.schema)
 
     // exported api to index
     this.functions = [
       this.complete,
       this.cancel,
       this.resume,
+      this.delete,
       this.fail,
       this.fetch,
       this.work,
@@ -189,71 +188,33 @@ class Manager extends EventEmitter {
     const {
       pollingInterval: interval = this.config.pollingInterval,
       batchSize,
-      teamSize = 1,
-      teamConcurrency = 1,
-      teamRefill: refill = false,
       includeMetadata = false,
       priority = true
     } = options
 
     const id = randomUUID({ disableEntropyCache: true })
 
-    let queueSize = 0
-
-    let refillTeamPromise
-    let resolveRefillTeam
-
-    // Setup a promise that onFetch can await for when at least one
-    // job is finished and so the team is ready to be topped up
-    const createTeamRefillPromise = () => {
-      refillTeamPromise = new Promise((resolve) => { resolveRefillTeam = resolve })
-    }
-
-    createTeamRefillPromise()
-
-    const onRefill = () => {
-      queueSize--
-      resolveRefillTeam()
-      createTeamRefillPromise()
-    }
-
-    const fetch = () => this.fetch(name, batchSize || (teamSize - queueSize), { includeMetadata, priority })
+    const fetch = () => this.fetch(name, { batchSize, includeMetadata, priority })
 
     const onFetch = async (jobs) => {
+      if (!jobs.length) {
+        return
+      }
+
       if (this.config.__test__throw_worker) {
         throw new Error('__test__throw_worker')
       }
 
       this.emitWip(name)
 
-      if (batchSize) {
-        const maxExpiration = jobs.reduce((acc, i) => Math.max(acc, i.expireInSeconds), 0)
-
-        await resolveWithinSeconds(Promise.all([callback(jobs)]), maxExpiration)
-          .then(() => this.complete(name, jobs.map(job => job.id)))
-          .catch(err => this.fail(name, jobs.map(job => job.id), err))
-      } else {
-        if (refill) {
-          queueSize += jobs.length || 1
-        }
-
-        const allTeamPromise = pMap(jobs, job =>
-          resolveWithinSeconds(callback(job), job.expireInSeconds)
-            .then(result => this.complete(name, job.id, result))
-            .catch(err => this.fail(name, job.id, err))
-            .then(() => refill ? onRefill() : null)
-        , { concurrency: teamConcurrency }
-        ).catch(() => {}) // allow promises & non-promises to live together in harmony
-
-        if (refill) {
-          if (queueSize < teamSize) {
-            return
-          } else {
-            await refillTeamPromise
-          }
-        } else {
-          await allTeamPromise
-        }
+      const maxExpiration = jobs.reduce((acc, i) => Math.max(acc, i.expireInSeconds), 0)
+      const jobIds = jobs.map(job => job.id)
+
+      try {
+        const result = await resolveWithinSeconds(callback(jobs), maxExpiration)
+        this.complete(name, jobIds, jobIds.length === 1 ? result : undefined)
+      } catch (err) {
+        this.fail(name, jobIds, err)
       }
 
       this.emitWip(name)
@@ -329,7 +290,7 @@ class Manager extends EventEmitter {
 
     const { rows } = await this.db.executeSql(this.getQueuesForEventCommand, [event])
 
-    return await Promise.all(rows.map(({ name }) => this.send(name, ...args)))
+    await Promise.allSettled(rows.map(({ name }) => this.send(name, ...args)))
   }
 
   async send (...args) {
@@ -467,25 +428,20 @@ class Manager extends EventEmitter {
     return startAfter
   }
 
-  async fetch (name, batchSize, options = {}) {
-    const values = Attorney.checkFetchArgs(name, batchSize, options)
+  async fetch (name, options = {}) {
+    Attorney.checkFetchArgs(name, options)
     const db = options.db || this.db
     const nextJobSql = this.nextJobCommand({ ...options })
-    const statementValues = [values.name, batchSize || 1]
 
     let result
 
     try {
-      result = await db.executeSql(nextJobSql, statementValues)
+      result = await db.executeSql(nextJobSql, [name, options.batchSize])
     } catch (err) {
       // errors from fetchquery should only be unique constraint violations
     }
 
-    if (!result || result.rows.length === 0) {
-      return null
-    }
-
-    return result.rows.length === 1 && !batchSize ? result.rows[0] : result.rows
+    return result?.rows || []
   }
 
   mapCompletionIdArg (id, funcName) {
@@ -510,11 +466,11 @@ class Manager extends EventEmitter {
     return stringify(result)
   }
 
-  mapCompletionResponse (ids, result) {
+  mapCommandResponse (ids, result) {
     return {
       jobs: ids,
       requested: ids.length,
-      updated: result && result.rows ? parseInt(result.rows[0].count) : 0
+      affected: result && result.rows ? parseInt(result.rows[0].count) : 0
     }
   }
 
@@ -523,7 +479,7 @@ class Manager extends EventEmitter {
     const db = options.db || this.db
     const ids = this.mapCompletionIdArg(id, 'complete')
     const result = await db.executeSql(this.completeJobsCommand, [name, ids, this.mapCompletionDataArg(data)])
-    return this.mapCompletionResponse(ids, result)
+    return this.mapCommandResponse(ids, result)
   }
 
   async fail (name, id, data, options = {}) {
@@ -531,7 +487,7 @@ class Manager extends EventEmitter {
     const db = options.db || this.db
     const ids = this.mapCompletionIdArg(id, 'fail')
     const result = await db.executeSql(this.failJobsByIdCommand, [name, ids, this.mapCompletionDataArg(data)])
-    return this.mapCompletionResponse(ids, result)
+    return this.mapCommandResponse(ids, result)
   }
 
   async cancel (name, id, options = {}) {
@@ -539,7 +495,15 @@ class Manager extends EventEmitter {
     const db = options.db || this.db
     const ids = this.mapCompletionIdArg(id, 'cancel')
     const result = await db.executeSql(this.cancelJobsCommand, [name, ids])
-    return this.mapCompletionResponse(ids, result)
+    return this.mapCommandResponse(ids, result)
+  }
+
+  async delete (name, id, options = {}) {
+    Attorney.assertQueueName(name)
+    const db = options.db || this.db
+    const ids = this.mapCompletionIdArg(id, 'delete')
+    const result = await db.executeSql(this.deleteJobsCommand, [name, ids])
+    return this.mapCommandResponse(ids, result)
   }
 
   async resume (name, id, options = {}) {
@@ -547,7 +511,7 @@ class Manager extends EventEmitter {
     const db = options.db || this.db
     const ids = this.mapCompletionIdArg(id, 'resume')
     const result = await db.executeSql(this.resumeJobsCommand, [name, ids])
-    return this.mapCompletionResponse(ids, result)
+    return this.mapCommandResponse(ids, result)
   }
 
   async createQueue (name, options = {}) {
@@ -572,10 +536,8 @@ class Manager extends EventEmitter {
       Attorney.assertQueueName(deadLetter)
     }
 
-    await this.db.executeSql(this.createPartitionCommand, [name])
-
-    const params = [
-      name,
+    // todo: pull in defaults from constructor config
+    const data = {
       policy,
       retryLimit,
       retryDelay,
@@ -583,9 +545,9 @@ class Manager extends EventEmitter {
       expireInSeconds,
       retentionMinutes,
       deadLetter
-    ]
+    }
 
-    await this.db.executeSql(this.insertQueueCommand, params)
+    await this.db.executeSql(this.createQueueCommand, [name, data])
   }
 
   async getQueues () {
@@ -626,32 +588,9 @@ class Manager extends EventEmitter {
   async getQueue (name) {
     Attorney.assertQueueName(name)
 
-    const result = await this.db.executeSql(this.getQueueByNameCommand, [name])
-
-    if (result.rows.length === 0) {
-      return null
-    }
-
-    const {
-      policy,
-      retry_limit: retryLimit,
-      retry_delay: retryDelay,
-      retry_backoff: retryBackoff,
-      expire_seconds: expireInSeconds,
-      retention_minutes: retentionMinutes,
-      dead_letter: deadLetter
-    } = result.rows[0]
+    const { rows } = await this.db.executeSql(this.getQueueByNameCommand, [name])
 
-    return {
-      name,
-      policy,
-      retryLimit,
-      retryDelay,
-      retryBackoff,
-      expireInSeconds,
-      retentionMinutes,
-      deadLetter
-    }
+    return rows[0] || null
   }
 
   async deleteQueue (name) {
@@ -659,9 +598,8 @@ class Manager extends EventEmitter {
 
     const { rows } = await this.db.executeSql(this.getQueueByNameCommand, [name])
 
-    if (rows.length) {
-      await this.db.executeSql(this.dropPartitionCommand, [name])
-      await this.db.executeSql(this.deleteQueueRecordsCommand, [name])
+    if (rows.length === 1) {
+      await this.db.executeSql(this.deleteQueueCommand, [name])
     }
   }
 
@@ -688,19 +626,17 @@ class Manager extends EventEmitter {
     Attorney.assertQueueName(name)
 
     const db = options.db || this.db
+
     const result1 = await db.executeSql(this.getJobByIdCommand, [name, id])
 
-    if (result1 && result1.rows && result1.rows.length === 1) {
+    if (result1?.rows?.length === 1) {
       return result1.rows[0]
+    } else if (options.includeArchive) {
+      const result2 = await db.executeSql(this.getArchivedJobByIdCommand, [name, id])
+      return result2?.rows[0] || null
+    } else {
+      return null
     }
-
-    const result2 = await db.executeSql(this.getArchivedJobByIdCommand, [name, id])
-
-    if (result2 && result2.rows && result2.rows.length === 1) {
-      return result2.rows[0]
-    }
-
-    return null
   }
 }
 

package/src/plans.js

@@ -28,6 +28,7 @@ module.exports = {
   completeJobs,
   cancelJobs,
   resumeJobs,
+  deleteJobs,
   failJobsById,
   failJobsByTimeout,
   insertJob,
@@ -42,11 +43,9 @@ module.exports = {
   archive,
   drop,
   countStates,
-  insertQueue,
   updateQueue,
-  createPartition,
-  dropPartition,
-  deleteQueueRecords,
+  createQueue,
+  deleteQueue,
   getQueues,
   getQueueByName,
   getQueueSize,
@@ -85,9 +84,8 @@ function create (schema, version) {
     createColumnArchiveArchivedOn(schema),
     createIndexArchiveArchivedOn(schema),
 
-    getPartitionFunction(schema),
-    createPartitionFunction(schema),
-    dropPartitionFunction(schema),
+    createQueueFunction(schema),
+    deleteQueueFunction(schema),
 
     insertVersion(schema, version)
   ]
@@ -137,8 +135,10 @@ function createTableQueue (schema) {
       retry_backoff bool,
       expire_seconds int,
       retention_minutes int,
-      dead_letter text,
+      dead_letter text REFERENCES ${schema}.queue (name),
+      partition_name text,
       created_on timestamp with time zone not null default now(),
+      updated_on timestamp with time zone not null default now(),
       PRIMARY KEY (name) 
     )
   `
@@ -199,7 +199,10 @@ function createTableJob (schema) {
 }
 
 const baseJobColumns = 'id, name, data, EXTRACT(epoch FROM expire_in) as "expireInSeconds"'
-const allJobColumns = `${baseJobColumns}, policy, state, priority,
+const allJobColumns = `${baseJobColumns},
+  policy,
+  state,
+  priority,
   retry_limit as "retryLimit",
   retry_count as "retryCount",
   retry_delay as "retryDelay",
@@ -216,29 +219,38 @@ const allJobColumns = `${baseJobColumns}, policy, state, priority,
   output
 `
 
-function createPartition (schema) {
-  return `SELECT ${schema}.create_partition($1)`
-}
-
-function getPartitionFunction (schema) {
-  return `
-    CREATE FUNCTION ${schema}.get_partition(queue_name text, out name text) AS
-    $$
-    SELECT 'j' || lower(left(regexp_replace(queue_name, '\\W', '', 'g'),10)) || left(encode(sha224(queue_name::bytea), 'hex'),10);
-    $$
-    LANGUAGE SQL
-    IMMUTABLE
-  `
-}
-
-function createPartitionFunction (schema) {
+function createQueueFunction (schema) {
   return `
-    CREATE FUNCTION ${schema}.create_partition(queue_name text)
+    CREATE FUNCTION ${schema}.create_queue(queue_name text, options json)
     RETURNS VOID AS
     $$
     DECLARE
-      table_name varchar := ${schema}.get_partition(queue_name);
+      table_name varchar := 'j' || encode(sha224(queue_name::bytea), 'hex');
     BEGIN
+
+      INSERT INTO ${schema}.queue (
+        name,
+        policy,
+        retry_limit,
+        retry_delay,
+        retry_backoff,
+        expire_seconds,
+        retention_minutes,
+        dead_letter,
+        partition_name
+      )
+      VALUES (
+        queue_name,
+        options->>'policy',
+        (options->>'retryLimit')::int,
+        (options->>'retryDelay')::int,
+        (options->>'retryBackoff')::bool,
+        (options->>'expireInSeconds')::int,
+        (options->>'retentionMinutes')::int,
+        options->>'deadLetter',
+        table_name
+      );
+
       EXECUTE format('CREATE TABLE ${schema}.%I (LIKE ${schema}.job INCLUDING DEFAULTS)', table_name);
       
       EXECUTE format('${formatPartitionCommand(createPrimaryKeyJob(schema))}', table_name);
@@ -259,24 +271,37 @@ function createPartitionFunction (schema) {
 }
 
 function formatPartitionCommand (command) {
-  return command.replace('.job', '.%1$I').replace('job_idx', '%1$s_idx').replaceAll('\'', '\'\'')
+  return command.replace('.job', '.%1$I').replace('job_i', '%1$s_i').replaceAll('\'', '\'\'')
 }
 
-function dropPartitionFunction (schema) {
+function deleteQueueFunction (schema) {
   return `
-    CREATE FUNCTION ${schema}.drop_partition(queue_name text)
+    CREATE FUNCTION ${schema}.delete_queue(queue_name text)
     RETURNS VOID AS
     $$
+    DECLARE
+      table_name varchar;
     BEGIN  
-      EXECUTE format('DROP TABLE IF EXISTS ${schema}.%I', ${schema}.get_partition(queue_name));
+      WITH deleted as (
+        DELETE FROM ${schema}.queue
+        WHERE name = queue_name
+        RETURNING partition_name
+      )
+      SELECT partition_name from deleted INTO table_name;
+
+      EXECUTE format('DROP TABLE IF EXISTS ${schema}.%I', table_name);
     END;
     $$
     LANGUAGE plpgsql;
   `
 }
 
-function dropPartition (schema) {
-  return `SELECT ${schema}.drop_partition($1)`
+function createQueue (schema) {
+  return `SELECT ${schema}.create_queue($1, $2)`
+}
+
+function deleteQueue (schema) {
+  return `SELECT ${schema}.delete_queue($1)`
 }
 
 function createPrimaryKeyJob (schema) {
@@ -284,11 +309,11 @@ function createPrimaryKeyJob (schema) {
 }
 
 function createQueueForeignKeyJob (schema) {
-  return `ALTER TABLE ${schema}.job ADD CONSTRAINT q_fkey FOREIGN KEY (name) REFERENCES ${schema}.queue (name) ON DELETE RESTRICT`
+  return `ALTER TABLE ${schema}.job ADD CONSTRAINT q_fkey FOREIGN KEY (name) REFERENCES ${schema}.queue (name) ON DELETE RESTRICT DEFERRABLE INITIALLY DEFERRED`
 }
 
 function createQueueForeignKeyJobDeadLetter (schema) {
-  return `ALTER TABLE ${schema}.job ADD CONSTRAINT dlq_fkey FOREIGN KEY (dead_letter) REFERENCES ${schema}.queue (name) ON DELETE RESTRICT`
+  return `ALTER TABLE ${schema}.job ADD CONSTRAINT dlq_fkey FOREIGN KEY (dead_letter) REFERENCES ${schema}.queue (name) ON DELETE RESTRICT DEFERRABLE INITIALLY DEFERRED`
 }
 
 function createPrimaryKeyArchive (schema) {
@@ -296,23 +321,23 @@ function createPrimaryKeyArchive (schema) {
 }
 
 function createIndexJobPolicyShort (schema) {
-  return `CREATE UNIQUE INDEX job_idx_psh ON ${schema}.job (name, COALESCE(singleton_key, '')) WHERE state = '${JOB_STATES.created}' AND policy = '${QUEUE_POLICIES.short}';`
+  return `CREATE UNIQUE INDEX job_i1 ON ${schema}.job (name, COALESCE(singleton_key, '')) WHERE state = '${JOB_STATES.created}' AND policy = '${QUEUE_POLICIES.short}';`
 }
 
 function createIndexJobPolicySingleton (schema) {
-  return `CREATE UNIQUE INDEX job_idx_psi ON ${schema}.job (name, COALESCE(singleton_key, '')) WHERE state = '${JOB_STATES.active}' AND policy = '${QUEUE_POLICIES.singleton}'`
+  return `CREATE UNIQUE INDEX job_i2 ON ${schema}.job (name, COALESCE(singleton_key, '')) WHERE state = '${JOB_STATES.active}' AND policy = '${QUEUE_POLICIES.singleton}'`
 }
 
 function createIndexJobPolicyStately (schema) {
-  return `CREATE UNIQUE INDEX job_idx_pst ON ${schema}.job (name, state, COALESCE(singleton_key, '')) WHERE state <= '${JOB_STATES.active}' AND policy = '${QUEUE_POLICIES.stately}'`
+  return `CREATE UNIQUE INDEX job_i3 ON ${schema}.job (name, state, COALESCE(singleton_key, '')) WHERE state <= '${JOB_STATES.active}' AND policy = '${QUEUE_POLICIES.stately}'`
 }
 
 function createIndexJobThrottle (schema) {
-  return `CREATE UNIQUE INDEX job_idx_to ON ${schema}.job (name, singleton_on, COALESCE(singleton_key, '')) WHERE state <> '${JOB_STATES.cancelled}' AND singleton_on IS NOT NULL`
+  return `CREATE UNIQUE INDEX job_i4 ON ${schema}.job (name, singleton_on, COALESCE(singleton_key, '')) WHERE state <> '${JOB_STATES.cancelled}' AND singleton_on IS NOT NULL`
 }
 
 function createIndexJobFetch (schema) {
-  return `CREATE INDEX job_idx_f ON ${schema}.job (name, start_after) INCLUDE (priority, created_on, id) WHERE state < '${JOB_STATES.active}'`
+  return `CREATE INDEX job_i5 ON ${schema}.job (name, start_after) INCLUDE (priority, created_on, id) WHERE state < '${JOB_STATES.active}'`
 }
 
 function createTableArchive (schema) {
@@ -324,7 +349,7 @@ function createColumnArchiveArchivedOn (schema) {
 }
 
 function createIndexArchiveArchivedOn (schema) {
-  return `CREATE INDEX archive_idx_ao ON ${schema}.archive(archived_on)`
+  return `CREATE INDEX archive_i1 ON ${schema}.archive(archived_on)`
 }
 
 function trySetMaintenanceTime (schema) {
@@ -347,13 +372,6 @@ function trySetTimestamp (schema, column) {
   `
 }
 
-function insertQueue (schema) {
-  return `
-    INSERT INTO ${schema}.queue (name, policy, retry_limit, retry_delay, retry_backoff, expire_seconds, retention_minutes, dead_letter)
-    VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
-  `
-}
-
 function updateQueue (schema) {
   return `
     UPDATE ${schema}.queue SET
@@ -363,21 +381,28 @@ function updateQueue (schema) {
       retry_backoff = COALESCE($5, retry_backoff),
       expire_seconds = COALESCE($6, expire_seconds),
       retention_minutes = COALESCE($7, retention_minutes),
-      dead_letter = COALESCE($8, dead_letter)
+      dead_letter = COALESCE($8, dead_letter),
+      updated_on = now()
     WHERE name = $1
   `
 }
 
 function getQueues (schema) {
-  return `SELECT * FROM ${schema}.queue`
+  return `
+    SELECT 
+      policy,
+      retry_limit as "retryLimit",
+      retry_delay as "retryDelay",
+      retry_backoff as "retryBackoff",
+      expire_seconds as "expireInSeconds",
+      retention_minutes as "retentionMinutes",
+      dead_letter as "deadLetter"
+    FROM ${schema}.queue    
+   `
 }
 
 function getQueueByName (schema) {
-  return `SELECT * FROM ${schema}.queue WHERE name = $1`
-}
-
-function deleteQueueRecords (schema) {
-  return `DELETE FROM ${schema}.queue WHERE name = $1`
+  return `${getQueues(schema)} WHERE name = $1`
 }
 
 function purgeQueue (schema) {
@@ -577,6 +602,19 @@ function resumeJobs (schema) {
         state = '${JOB_STATES.created}'
       WHERE name = $1
         AND id IN (SELECT UNNEST($2::uuid[]))
+        AND state = '${JOB_STATES.cancelled}'
+      RETURNING 1
+    )
+    SELECT COUNT(*) from results
+  `
+}
+
+function deleteJobs (schema) {
+  return `
+    with results as (
+      DELETE FROM ${schema}.job
+      WHERE name = $1
+        AND id IN (SELECT UNNEST($2::uuid[]))        
       RETURNING 1
     )
     SELECT COUNT(*) from results
@@ -654,7 +692,7 @@ function insertJob (schema) {
           $17::int as retry_delay_default,
           $18::bool as retry_backoff,
           $19::bool as retry_backoff_default
-      ) j LEFT JOIN ${schema}.queue q ON j.name = q.name
+      ) j JOIN ${schema}.queue q ON j.name = q.name
     ON CONFLICT DO NOTHING
     RETURNING id
   `
@@ -689,10 +727,10 @@ function insertJobs (schema) {
       COALESCE(id, gen_random_uuid()) as id,
       j.name,
       data,
-      COALESCE(priority, 0),
-      COALESCE("startAfter", now()),
-      "singletonKey",
-      COALESCE("deadLetter", q.dead_letter),
+      COALESCE(priority, 0) as priority,
+      j.start_after,
+      "singletonKey" as singleton_key,
+      COALESCE("deadLetter", q.dead_letter) as dead_letter,
       CASE
         WHEN "expireInSeconds" IS NOT NULL THEN "expireInSeconds" *  interval '1s'
         WHEN q.expire_seconds IS NOT NULL THEN q.expire_seconds * interval '1s'
@@ -701,7 +739,7 @@ function insertJobs (schema) {
         END as expire_in,
       CASE
         WHEN "keepUntil" IS NOT NULL THEN "keepUntil"
-        ELSE COALESCE("startAfter", now()) + CAST(COALESCE((q.retention_minutes * 60)::text, defaults.keep_until, '14 days') as interval)
+        ELSE COALESCE(j.start_after, now()) + CAST(COALESCE((q.retention_minutes * 60)::text, defaults.keep_until, '14 days') as interval)
         END as keep_until,
       COALESCE("retryLimit", q.retry_limit, defaults.retry_limit, 2),
       CASE
@@ -711,21 +749,29 @@ function insertJobs (schema) {
         END as retry_delay,      
       COALESCE("retryBackoff", q.retry_backoff, defaults.retry_backoff, false) as retry_backoff,
       q.policy
-    FROM json_to_recordset($1) as j (
-      id uuid,
-      name text,
-      priority integer,
-      data jsonb,
-      "startAfter" timestamp with time zone,
-      "retryLimit" integer,
-      "retryDelay" integer,
-      "retryBackoff" boolean,
-      "singletonKey" text,
-      "expireInSeconds" integer,
-      "keepUntil" timestamp with time zone,
-      "deadLetter" text
-    )
-    LEFT JOIN ${schema}.queue q ON j.name = q.name,
+    FROM (
+      SELECT *,
+        CASE
+          WHEN right("startAfter", 1) = 'Z' THEN CAST("startAfter" as timestamp with time zone)
+          ELSE now() + CAST(COALESCE("startAfter",'0') as interval)
+          END as start_after
+      FROM json_to_recordset($1) as x (
+        id uuid,
+        name text,
+        priority integer,
+        data jsonb,
+        "startAfter" text,
+        "retryLimit" integer,
+        "retryDelay" integer,
+        "retryBackoff" boolean,
+        "singletonKey" text,
+        "singletonOn" text,
+        "expireInSeconds" integer,
+        "keepUntil" timestamp with time zone,
+        "deadLetter" text
+      ) 
+    ) j
+    JOIN ${schema}.queue q ON j.name = q.name,
       defaults
     ON CONFLICT DO NOTHING
   `
@@ -752,6 +798,7 @@ function archive (schema, completedInterval, failedInterval = completedInterval)
     INSERT INTO ${schema}.archive (${columns})
     SELECT ${columns}
     FROM archived_rows
+    ON CONFLICT DO NOTHING
   `
 }
 
@@ -779,7 +826,7 @@ function locked (schema, query) {
 }
 
 function advisoryLock (schema, key) {
-  return `SELECT pg_advisory_xact_lock(      
+  return `SELECT pg_advisory_xact_lock(
       ('x' || encode(sha224((current_database() || '.pgboss.${schema}${key || ''}')::bytea), 'hex'))::bit(64)::bigint
   )`
 }

package/src/timekeeper.js

@@ -2,13 +2,12 @@ const EventEmitter = require('events')
 const plans = require('./plans')
 const cronParser = require('cron-parser')
 const Attorney = require('./attorney')
-const pMap = require('p-map')
 
-const queues = {
+const QUEUES = {
   SEND_IT: '__pgboss__send-it'
 }
 
-const events = {
+const EVENTS = {
   error: 'error',
   schedule: 'schedule'
 }
@@ -24,7 +23,7 @@ class Timekeeper extends EventEmitter {
     this.cronMonitorIntervalMs = config.cronMonitorIntervalSeconds * 1000
     this.clockSkew = 0
 
-    this.events = events
+    this.events = EVENTS
 
     this.getTimeCommand = plans.getTime(config.schema)
     this.getQueueCommand = plans.getQueueByName(config.schema)
@@ -53,16 +52,15 @@ class Timekeeper extends EventEmitter {
     await this.cacheClockSkew()
 
     try {
-      await this.manager.createQueue(queues.SEND_IT)
+      await this.manager.createQueue(QUEUES.SEND_IT)
     } catch {}
 
     const options = {
       pollingIntervalSeconds: this.config.cronWorkerIntervalSeconds,
-      teamSize: 50,
-      teamConcurrency: 5
+      batchSize: 50
     }
 
-    await this.manager.work(queues.SEND_IT, options, (job) => this.onSendIt(job))
+    await this.manager.work(QUEUES.SEND_IT, options, (jobs) => this.manager.insert(jobs.map(i => i.data)))
 
     setImmediate(() => this.onCron())
 
@@ -77,7 +75,7 @@ class Timekeeper extends EventEmitter {
 
     this.stopped = true
 
-    await this.manager.offWork(queues.SEND_IT)
+    await this.manager.offWork(QUEUES.SEND_IT)
 
     if (this.skewMonitorInterval) {
       clearInterval(this.skewMonitorInterval)
@@ -141,12 +139,15 @@ class Timekeeper extends EventEmitter {
   }
 
   async cron () {
-    const items = await this.getSchedules()
+    const schedules = await this.getSchedules()
 
-    const sending = items.filter(i => this.shouldSendIt(i.cron, i.timezone))
+    const scheduled = schedules
+      .filter(i => this.shouldSendIt(i.cron, i.timezone))
+      .map(({ name, data, options }) =>
+        ({ name: QUEUES.SEND_IT, data: { name, data, options }, options: { singletonKey: name, singletonSeconds: 60 } }))
 
-    if (sending.length && !this.stopped) {
-      await pMap(sending, it => this.send(it), { concurrency: 5 })
+    if (scheduled.length > 0 && !this.stopped) {
+      await this.manager.insert(scheduled)
     }
   }
 
@@ -162,16 +163,6 @@ class Timekeeper extends EventEmitter {
     return prevDiff < 60
   }
 
-  async send (job) {
-    await this.manager.send(queues.SEND_IT, job, { singletonKey: job.name, singletonSeconds: 60 })
-  }
-
-  async onSendIt (job) {
-    if (this.stopped) return
-    const { name, data, options } = job.data
-    await this.manager.send(name, data, options)
-  }
-
   async getSchedules () {
     const { rows } = await this.db.executeSql(this.getSchedulesCommand)
     return rows
@@ -203,4 +194,4 @@ class Timekeeper extends EventEmitter {
 }
 
 module.exports = Timekeeper
-module.exports.QUEUES = queues
+module.exports.QUEUES = QUEUES

package/types.d.ts

@@ -114,7 +114,7 @@ declare namespace PgBoss {
   
   type QueuePolicy = 'standard' | 'short' | 'singleton' | 'stately'
   
-  type Queue = RetryOptions & ExpirationOptions & RetentionOptions & { name: string, policy: QueuePolicy, deadLetter?: string }
+  type Queue = RetryOptions & ExpirationOptions & RetentionOptions & { name: string, policy?: QueuePolicy, deadLetter?: string }
   
   type ScheduleOptions = SendOptions & { tz?: string }
 
@@ -122,41 +122,20 @@ declare namespace PgBoss {
     pollingIntervalSeconds?: number;
   }
 
-  interface CommonJobFetchOptions {
+  interface JobFetchOptions {
     includeMetadata?: boolean;
     priority?: boolean;
-  }
-
-  type JobFetchOptions  = CommonJobFetchOptions & {
-    teamSize?: number;
-    teamConcurrency?: number;
-    teamRefill?: boolean;
-  }
-
-  type BatchJobFetchOptions = CommonJobFetchOptions & {
-    batchSize: number;
+    batchSize?: number;
   }
 
   type WorkOptions = JobFetchOptions & JobPollingOptions
-  type BatchWorkOptions = BatchJobFetchOptions & JobPollingOptions
-
-  type FetchOptions = {
-    includeMetadata?: boolean;
-  } & ConnectionOptions;
+  type FetchOptions = JobFetchOptions & ConnectionOptions;
 
   interface WorkHandler<ReqData> {
-    (job: PgBoss.Job<ReqData>): Promise<any>;
-  }
-
-  interface BatchWorkHandler<ReqData> {
     (job: PgBoss.Job<ReqData>[]): Promise<any>;
   }
 
   interface WorkWithMetadataHandler<ReqData> {
-    (job: PgBoss.JobWithMetadata<ReqData>): Promise<any>;
-  }
-
-  interface BatchWorkWithMetadataHandler<ReqData> {
     (job: PgBoss.JobWithMetadata<ReqData>[]): Promise<any>;
   }
 
@@ -328,13 +307,14 @@ declare class PgBoss extends EventEmitter {
   insert(jobs: PgBoss.JobInsert[]): Promise<void>;
   insert(jobs: PgBoss.JobInsert[], options: PgBoss.InsertOptions): Promise<void>;
 
+  fetch<T>(name: string): Promise<PgBoss.Job<T>[]>;
+  fetch<T>(name: string, options: PgBoss.FetchOptions & { includeMetadata: true }): Promise<PgBoss.JobWithMetadata<T>[]>;
+  fetch<T>(name: string, options: PgBoss.FetchOptions): Promise<PgBoss.Job<T>[]>;
+
   work<ReqData>(name: string, handler: PgBoss.WorkHandler<ReqData>): Promise<string>;
   work<ReqData>(name: string, options: PgBoss.WorkOptions & { includeMetadata: true }, handler: PgBoss.WorkWithMetadataHandler<ReqData>): Promise<string>;
   work<ReqData>(name: string, options: PgBoss.WorkOptions, handler: PgBoss.WorkHandler<ReqData>): Promise<string>;
 
-  work<ReqData>(name: string, options: PgBoss.BatchWorkOptions & { includeMetadata: true }, handler: PgBoss.BatchWorkWithMetadataHandler<ReqData>): Promise<string>;
-  work<ReqData>(name: string, options: PgBoss.BatchWorkOptions, handler: PgBoss.BatchWorkHandler<ReqData>): Promise<string>;
-
   offWork(name: string): Promise<void>;
   offWork(options: PgBoss.OffWorkOptions): Promise<void>;
 
@@ -342,14 +322,9 @@ declare class PgBoss extends EventEmitter {
 
   subscribe(event: string, name: string): Promise<void>;
   unsubscribe(event: string, name: string): Promise<void>;
-  publish(event: string): Promise<string[]>;
-  publish(event: string, data: object): Promise<string[]>;
-  publish(event: string, data: object, options: PgBoss.SendOptions): Promise<string[]>;
-
-  fetch<T>(name: string): Promise<PgBoss.Job<T> | null>;
-  fetch<T>(name: string, batchSize: number): Promise<PgBoss.Job<T>[] | null>;
-  fetch<T>(name: string, batchSize: number, options: PgBoss.FetchOptions & { includeMetadata: true }): Promise<PgBoss.JobWithMetadata<T>[] | null>;
-  fetch<T>(name: string, batchSize: number, options: PgBoss.FetchOptions): Promise<PgBoss.Job<T>[] | null>;
+  publish(event: string): Promise<void>;
+  publish(event: string, data: object): Promise<void>;
+  publish(event: string, data: object, options: PgBoss.SendOptions): Promise<void>;
 
   cancel(name: string, id: string, options?: PgBoss.ConnectionOptions): Promise<void>;
   cancel(name: string, ids: string[], options?: PgBoss.ConnectionOptions): Promise<void>;
@@ -357,6 +332,9 @@ declare class PgBoss extends EventEmitter {
   resume(name: string, id: string, options?: PgBoss.ConnectionOptions): Promise<void>;
   resume(name: string, ids: string[], options?: PgBoss.ConnectionOptions): Promise<void>;
 
+  delete(name: string, id: string, options?: PgBoss.ConnectionOptions): Promise<void>;
+  delete(name: string, ids: string[], options?: PgBoss.ConnectionOptions): Promise<void>;
+  
   complete(name: string, id: string, options?: PgBoss.ConnectionOptions): Promise<void>;
   complete(name: string, id: string, data: object, options?: PgBoss.ConnectionOptions): Promise<void>;
   complete(name: string, ids: string[], options?: PgBoss.ConnectionOptions): Promise<void>;
@@ -366,11 +344,11 @@ declare class PgBoss extends EventEmitter {
   fail(name: string, ids: string[], options?: PgBoss.ConnectionOptions): Promise<void>;
 
   getQueueSize(name: string, options?: object): Promise<number>;
-  getJobById(name: string, id: string, options?: PgBoss.ConnectionOptions): Promise<PgBoss.JobWithMetadata | null>;
+  getJobById(name: string, id: string, options?: PgBoss.ConnectionOptions & { includeArchive: bool }): Promise<PgBoss.JobWithMetadata | null>;
 
   createQueue(name: string, options?: PgBoss.Queue): Promise<void>;
   getQueue(name: string): Promise<PgBoss.Queue | null>;
-  getQueues(): Promise<[PgBoss.Queue]>;
+  getQueues(): Promise<PgBoss.Queue[]>;
   updateQueue(name: string, options?: PgBoss.Queue): Promise<void>;
   deleteQueue(name: string): Promise<void>;
   purgeQueue(name: string): Promise<void>;