pg-boss 7.2.1 → 8.0.0

package/README.md

@@ -50,7 +50,7 @@ This will likely cater the most to teams already familiar with the simplicity of
 * Automatic maintenance operations to manage table growth
 
 ## Requirements
-* Node 12 or higher
+* Node 14 or higher
 * PostgreSQL 9.5 or higher
 
 ## Installation

package/package.json

@@ -1,31 +1,34 @@
 {
   "name": "pg-boss",
-  "version": "7.2.1",
+  "version": "8.0.0",
   "description": "Queueing jobs in Node.js using PostgreSQL like a boss",
   "main": "./src/index.js",
   "engines": {
-    "node": ">=12.0.0"
+    "node": ">=14"
   },
   "dependencies": {
     "cron-parser": "^4.0.0",
     "delay": "^5.0.0",
     "lodash.debounce": "^4.0.8",
-    "p-map": "^4.0.0",
+    "p-map": "^5.3.0",
     "pg": "^8.5.1",
+    "serialize-error": "^11.0.0",
     "uuid": "^8.3.2"
   },
   "devDependencies": {
-    "@types/node": "^17.0.2",
+    "@types/node": "^18.0.0",
     "coveralls": "^3.1.0",
-    "luxon": "^2.0.1",
-    "mocha": "^9.0.1",
+    "luxon": "^3.0.1",
+    "mocha": "^10.0.0",
     "nyc": "^15.1.0",
-    "standard": "^16.0.3"
+    "standard": "^17.0.0"
   },
   "scripts": {
     "test": "standard && mocha",
     "cover": "nyc --reporter=text npm test",
-    "forcover": "npm run cover && nyc report --reporter=text-lcov | coveralls",
+    "travis-cover": "nyc --reporter=text npm run travis-test",
+    "travis-test": "standard && mocha --jobs 0 --exit",
+    "forcover": "npm run travis-cover && nyc report --reporter=text-lcov | coveralls",
     "export-schema": "node ./scripts/construct.js",
     "export-migration": "node ./scripts/migrate.js",
     "export-rollback": "node ./scripts/rollback.js",

package/src/attorney.js

@@ -1,9 +1,10 @@
 const assert = require('assert')
-const { DEFAULT_SCHEMA } = require('./plans')
+const { DEFAULT_SCHEMA, SINGLETON_QUEUE_KEY } = require('./plans')
 
 module.exports = {
   getConfig,
   checkSendArgs,
+  checkInsertArgs,
   checkWorkArgs,
   checkFetchArgs,
   warnClockSkew
@@ -60,6 +61,7 @@ function checkSendArgs (args, defaults) {
   applyExpirationConfig(options, defaults)
   applyRetentionConfig(options, defaults)
   applyCompletionConfig(options, defaults)
+  applySingletonKeyConfig(options)
 
   const { startAfter, singletonSeconds, singletonMinutes, singletonHours } = options
 
@@ -84,6 +86,22 @@ function checkSendArgs (args, defaults) {
   return { name, data, options }
 }
 
+function checkInsertArgs (jobs) {
+  assert(Array.isArray(jobs), `jobs argument should be an array.  Received '${typeof jobs}'`)
+  return jobs.map(job => {
+    job = { ...job }
+    applySingletonKeyConfig(job)
+    return job
+  })
+}
+
+function applySingletonKeyConfig (options) {
+  if (options.singletonKey && options.useSingletonQueue && options.singletonKey !== SINGLETON_QUEUE_KEY) {
+    options.singletonKey = SINGLETON_QUEUE_KEY + options.singletonKey
+  }
+  delete options.useSingletonQueue
+}
+
 function checkWorkArgs (name, args, defaults) {
   let options, callback
 

package/src/index.js

@@ -90,6 +90,8 @@ class PgBoss extends EventEmitter {
   }
 
   async start () {
+    const { serializeError } = await import('serialize-error')
+
     if (!this.stopped) {
       return this
     }
@@ -104,7 +106,7 @@ class PgBoss extends EventEmitter {
 
     this.started = true
 
-    this.manager.start()
+    this.manager.start({ stringify: serializeError })
 
     if (!this.config.noSupervisor) {
       await this.boss.supervise()

package/src/manager.js

@@ -1,6 +1,5 @@
 const assert = require('assert')
 const EventEmitter = require('events')
-const pMap = require('p-map')
 const delay = require('delay')
 const uuid = require('uuid')
 const debounce = require('lodash.debounce')
@@ -45,6 +44,8 @@ class Manager extends EventEmitter {
   constructor (db, config) {
     super()
 
+    this.stringify = null
+
     this.config = config
     this.db = db
 
@@ -74,6 +75,7 @@ class Manager extends EventEmitter {
       this.fetchCompleted,
       this.work,
       this.offWork,
+      this.notifyWorker,
       this.onComplete,
       this.offComplete,
       this.publish,
@@ -96,7 +98,8 @@ class Manager extends EventEmitter {
     this.emitWipThrottled = debounce(() => this.emit(events.wip, this.getWipData()), WIP_EVENT_INTERVAL, WIP_DEBOUNCE_OPTIONS)
   }
 
-  start () {
+  start ({ stringify }) {
+    this.stringify = stringify
     this.stopping = false
   }
 
@@ -210,6 +213,8 @@ class Manager extends EventEmitter {
     const fetch = () => this.fetch(name, batchSize || (teamSize - queueSize), { includeMetadata })
 
     const onFetch = async (jobs) => {
+      const { default: pMap } = await import('p-map')
+
       if (this.config.__test__throw_worker) {
         throw new Error('__test__throw_worker')
       }
@@ -294,6 +299,12 @@ class Manager extends EventEmitter {
     })
   }
 
+  notifyWorker (workerId) {
+    if (this.workers.has(workerId)) {
+      this.workers.get(workerId).notify()
+    }
+  }
+
   async subscribe (event, name) {
     assert(event, 'Missing required argument')
     assert(name, 'Missing required argument')
@@ -386,6 +397,7 @@ class Manager extends EventEmitter {
 
   async createJob (name, data, options, singletonOffset = 0) {
     const {
+      db: wrapper,
       expireIn,
       priority,
       startAfter,
@@ -416,8 +428,8 @@ class Manager extends EventEmitter {
       keepUntil, // 13
       onComplete // 14
     ]
-
-    const result = await this.db.executeSql(this.insertJobCommand, values)
+    const db = wrapper || this.db
+    const result = await db.executeSql(this.insertJobCommand, values)
 
     if (result && result.rowCount === 1) {
       return result.rows[0].id
@@ -438,10 +450,12 @@ class Manager extends EventEmitter {
     return await this.createJob(name, data, options, singletonOffset)
   }
 
-  async insert (jobs) {
-    assert(Array.isArray(jobs), `jobs argument should be an array.  Received '${typeof jobs}'`)
-    const data = JSON.stringify(jobs)
-    return await this.db.executeSql(this.insertJobsCommand, [data])
+  async insert (jobs, options = {}) {
+    const { db: wrapper } = options
+    const db = wrapper || this.db
+    const checkedJobs = Attorney.checkInsertArgs(jobs)
+    const data = JSON.stringify(checkedJobs)
+    return await db.executeSql(this.insertJobsCommand, [data])
   }
 
   getDebounceStartAfter (singletonSeconds, clockOffset) {
@@ -463,8 +477,8 @@ class Manager extends EventEmitter {
 
   async fetch (name, batchSize, options = {}) {
     const values = Attorney.checkFetchArgs(name, batchSize, options)
-
-    const result = await this.db.executeSql(
+    const db = options.db || this.db
+    const result = await db.executeSql(
       this.nextJobCommand(options.includeMetadata || false),
       [values.name, batchSize || 1]
     )
@@ -506,15 +520,11 @@ class Manager extends EventEmitter {
   mapCompletionDataArg (data) {
     if (data === null || typeof data === 'undefined' || typeof data === 'function') { return null }
 
-    if (data instanceof Error) {
-      const newData = {}
-      Object.getOwnPropertyNames(data).forEach(key => { newData[key] = data[key] })
-      data = newData
-    }
-
-    return (typeof data === 'object' && !Array.isArray(data))
+    const result = (typeof data === 'object' && !Array.isArray(data))
       ? data
       : { value: data }
+
+    return this.stringify(result)
   }
 
   mapCompletionResponse (ids, result) {

package/src/migrationStore.js

@@ -62,11 +62,25 @@ function migrate (value, version, migrations) {
   return flatten(schema, result.install, result.version)
 }
 
-function getAll (schema, config) {
-  const DEFAULT_RETENTION = '30 days'
-  const keepUntil = config ? config.keepUntil : DEFAULT_RETENTION
-
+function getAll (schema) {
   return [
+    {
+      release: '7.4.0',
+      version: 20,
+      previous: 19,
+      install: [
+        `DROP INDEX ${schema}.job_singletonKey`,
+        `DROP INDEX ${schema}.job_singleton_queue`,
+        `CREATE UNIQUE INDEX job_singletonKey ON ${schema}.job (name, singletonKey) WHERE state < 'completed' AND singletonOn IS NULL AND NOT singletonKey LIKE '\\_\\_pgboss\\_\\_singleton\\_queue%'`,
+        `CREATE UNIQUE INDEX job_singleton_queue ON ${schema}.job (name, singletonKey) WHERE state < 'active' AND singletonOn IS NULL AND singletonKey LIKE '\\_\\_pgboss\\_\\_singleton\\_queue%'`
+      ],
+      uninstall: [
+        `DROP INDEX ${schema}.job_singletonKey`,
+        `DROP INDEX ${schema}.job_singleton_queue`,
+        `CREATE UNIQUE INDEX job_singletonKey ON ${schema}.job (name, singletonKey) WHERE state < 'completed' AND singletonOn IS NULL AND NOT singletonKey = '__pgboss__singleton_queue'`,
+        `CREATE UNIQUE INDEX job_singleton_queue ON ${schema}.job (name, singletonKey) WHERE state < 'active' AND singletonOn IS NULL AND singletonKey = '__pgboss__singleton_queue'`
+      ]
+    },
     {
       release: '7.0.0',
       version: 19,
@@ -154,218 +168,6 @@ function getAll (schema, config) {
       uninstall: [
           `ALTER TABLE ${schema}.version DROP COLUMN maintained_on`
       ]
-    },
-    {
-      release: '5.0.0-beta1',
-      version: 13,
-      previous: 12,
-      install: [
-          `CREATE TABLE ${schema}.schedule (
-            name text primary key,
-            cron text not null,
-            timezone text,
-            data jsonb,
-            options jsonb,
-            created_on timestamp with time zone not null default now(),
-            updated_on timestamp with time zone not null default now()
-          )`
-      ],
-      uninstall: [
-          `DROP TABLE ${schema}.schedule`
-      ]
-    },
-    {
-      release: '4.0.0',
-      version: 12,
-      previous: 11,
-      install: [
-          `ALTER TABLE ${schema}.version ALTER COLUMN version TYPE int USING version::int`,
-          `ALTER TABLE ${schema}.job ADD COLUMN keepUntil timestamptz`,
-          `ALTER TABLE ${schema}.archive ADD COLUMN keepUntil timestamptz`,
-          `ALTER TABLE ${schema}.job ALTER COLUMN keepUntil SET DEFAULT now() + interval '${DEFAULT_RETENTION}'`,
-          `UPDATE ${schema}.job SET keepUntil = startAfter + interval '${keepUntil}'`,
-          `ALTER TABLE ${schema}.job ALTER COLUMN keepUntil SET NOT NULL`
-      ],
-      uninstall: [
-          `ALTER TABLE ${schema}.version ALTER COLUMN version TYPE text USING version::text`,
-          `ALTER TABLE ${schema}.job DROP COLUMN keepUntil`,
-          `ALTER TABLE ${schema}.archive DROP COLUMN keepUntil`
-      ]
-    },
-    {
-      release: '3.2.0',
-      version: 11,
-      previous: 10,
-      install: [
-          `CREATE INDEX archive_archivedon_idx ON ${schema}.archive(archivedon)`
-      ],
-      uninstall: [
-          `DROP INDEX ${schema}.archive_archivedon_idx`
-      ]
-    },
-    {
-      release: '3.1.3',
-      version: 10,
-      previous: 9,
-      install: [
-        `CREATE INDEX archive_id_idx ON ${schema}.archive(id)`
-      ],
-      uninstall: [
-        `DROP INDEX ${schema}.archive_id_idx`
-      ]
-    },
-    {
-      release: '3.1.0',
-      version: 9,
-      previous: 8,
-      install: [
-        `DROP INDEX ${schema}.job_fetch`,
-        `DROP INDEX ${schema}.job_name`,
-        `CREATE INDEX job_name ON ${schema}.job (name text_pattern_ops)`,
-        `UPDATE ${schema}.job set name = '__state__completed__' || substr(name, 1, position('__state__completed' in name) - 1) WHERE name LIKE '%__state__completed'`
-      ],
-      uninstall: [
-        `UPDATE ${schema}.job set name = substr(name, 21) || '__state__completed' WHERE name LIKE '__state__completed__%'`,
-        `CREATE INDEX job_fetch ON ${schema}.job (name, priority desc, createdOn, id) WHERE state < 'active'`,
-        `DROP INDEX ${schema}.job_name`,
-        `CREATE INDEX job_name ON ${schema}.job (name) WHERE state < 'active'`
-      ]
-    },
-    {
-      release: '3.0.0',
-      version: 8,
-      previous: 7,
-      install: [
-        'CREATE EXTENSION IF NOT EXISTS pgcrypto',
-        `ALTER TABLE ${schema}.job ALTER COLUMN id SET DEFAULT gen_random_uuid()`,
-        `ALTER TABLE ${schema}.job ADD retryDelay integer not null DEFAULT (0)`,
-        `ALTER TABLE ${schema}.job ADD retryBackoff boolean not null DEFAULT false`,
-        `ALTER TABLE ${schema}.job ADD startAfter timestamp with time zone not null default now()`,
-        `UPDATE ${schema}.job SET startAfter = createdOn + startIn`,
-        `ALTER TABLE ${schema}.job DROP COLUMN startIn`,
-        `UPDATE ${schema}.job SET expireIn = interval '15 minutes' WHERE expireIn IS NULL`,
-        `ALTER TABLE ${schema}.job ALTER COLUMN expireIn SET NOT NULL`,
-        `ALTER TABLE ${schema}.job ALTER COLUMN expireIn SET DEFAULT interval '15 minutes'`,
-        // archive table schema changes
-        `ALTER TABLE ${schema}.archive ADD retryDelay integer not null DEFAULT (0)`,
-        `ALTER TABLE ${schema}.archive ADD retryBackoff boolean not null DEFAULT false`,
-        `ALTER TABLE ${schema}.archive ADD startAfter timestamp with time zone`,
-        `UPDATE ${schema}.archive SET startAfter = createdOn + startIn`,
-        `ALTER TABLE ${schema}.archive DROP COLUMN startIn`,
-        // rename complete to completed for state enum - can't use ALTER TYPE :(
-        `DROP INDEX ${schema}.job_fetch`,
-        `DROP INDEX ${schema}.job_singletonOn`,
-        `DROP INDEX ${schema}.job_singletonKeyOn`,
-        `DROP INDEX ${schema}.job_singletonKey`,
-        `ALTER TABLE ${schema}.job ALTER COLUMN state DROP DEFAULT`,
-        `ALTER TABLE ${schema}.job ALTER COLUMN state SET DATA TYPE text USING state::text`,
-        `ALTER TABLE ${schema}.archive ALTER COLUMN state SET DATA TYPE text USING state::text`,
-        `DROP TYPE ${schema}.job_state`,
-        `CREATE TYPE ${schema}.job_state AS ENUM ('created', 'retry', 'active', 'completed', 'expired', 'cancelled', 'failed')`,
-        `UPDATE ${schema}.job SET state = 'completed' WHERE state = 'complete'`,
-        `UPDATE ${schema}.archive SET state = 'completed' WHERE state = 'complete'`,
-        `ALTER TABLE ${schema}.job ALTER COLUMN state SET DATA TYPE ${schema}.job_state USING state::${schema}.job_state`,
-        `ALTER TABLE ${schema}.job ALTER COLUMN state SET DEFAULT 'created'`,
-        `ALTER TABLE ${schema}.archive ALTER COLUMN state SET DATA TYPE ${schema}.job_state USING state::${schema}.job_state`,
-        `CREATE INDEX job_fetch ON ${schema}.job (name, priority desc, createdOn, id) WHERE state < 'active'`,
-        `CREATE UNIQUE INDEX job_singletonOn ON ${schema}.job (name, singletonOn) WHERE state < 'expired' AND singletonKey IS NULL`,
-        `CREATE UNIQUE INDEX job_singletonKeyOn ON ${schema}.job (name, singletonOn, singletonKey) WHERE state < 'expired'`,
-        `CREATE UNIQUE INDEX job_singletonKey ON ${schema}.job (name, singletonKey) WHERE state < 'completed' AND singletonOn IS NULL`,
-        // add new job name index
-        `CREATE INDEX job_name ON ${schema}.job (name) WHERE state < 'active'`
-      ],
-      uninstall: [
-        `ALTER TABLE ${schema}.job ALTER COLUMN id DROP DEFAULT`,
-        // won't know if we should drop pgcrypto extension so it stays
-        `ALTER TABLE ${schema}.job DROP COLUMN retryDelay`,
-        `ALTER TABLE ${schema}.job DROP COLUMN retryBackoff`,
-        `ALTER TABLE ${schema}.job DROP COLUMN startAfter`,
-        `ALTER TABLE ${schema}.job ADD COLUMN startIn interval not null default(interval '0')`,
-        // leaving migrated default data for expireIn
-        `ALTER TABLE ${schema}.job ALTER COLUMN expireIn DROP NOT NULL`,
-        `ALTER TABLE ${schema}.job ALTER COLUMN expireIn DROP DEFAULT`,
-        // archive table restore
-        `ALTER TABLE ${schema}.archive DROP COLUMN retryDelay`,
-        `ALTER TABLE ${schema}.archive DROP COLUMN retryBackoff`,
-        `ALTER TABLE ${schema}.archive DROP COLUMN startAfter`,
-        `ALTER TABLE ${schema}.archive ADD COLUMN startIn interval`,
-        // drop new job name index
-        `DROP INDEX ${schema}.job_name`,
-        // roll back to old enum def
-        `DROP INDEX ${schema}.job_fetch`,
-        `DROP INDEX ${schema}.job_singletonOn`,
-        `DROP INDEX ${schema}.job_singletonKeyOn`,
-        `DROP INDEX ${schema}.job_singletonKey`,
-        `ALTER TABLE ${schema}.job ALTER COLUMN state DROP DEFAULT`,
-        `ALTER TABLE ${schema}.job ALTER COLUMN state SET DATA TYPE text USING state::text`,
-        `ALTER TABLE ${schema}.archive ALTER COLUMN state SET DATA TYPE text USING state::text`,
-        `DROP TYPE ${schema}.job_state`,
-        `CREATE TYPE ${schema}.job_state AS ENUM ('created', 'retry', 'active', 'complete', 'expired', 'cancelled', 'failed')`,
-        `UPDATE ${schema}.job SET state = 'completed' WHERE state = 'complete'`,
-        `UPDATE ${schema}.archive SET state = 'complete' WHERE state = 'completed'`,
-        `ALTER TABLE ${schema}.job ALTER COLUMN state SET DATA TYPE ${schema}.job_state USING state::${schema}.job_state`,
-        `ALTER TABLE ${schema}.job ALTER COLUMN state SET DEFAULT 'created'`,
-        `ALTER TABLE ${schema}.archive ALTER COLUMN state SET DATA TYPE ${schema}.job_state USING state::${schema}.job_state`,
-        `CREATE INDEX job_fetch ON ${schema}.job (name, priority desc, createdOn, id) WHERE state < 'active'`,
-        `CREATE UNIQUE INDEX job_singletonOn ON ${schema}.job (name, singletonOn) WHERE state < 'expired' AND singletonKey IS NULL`,
-        `CREATE UNIQUE INDEX job_singletonKeyOn ON ${schema}.job (name, singletonOn, singletonKey) WHERE state < 'expired'`,
-        `CREATE UNIQUE INDEX job_singletonKey ON ${schema}.job (name, singletonKey) WHERE state < 'complete' AND singletonOn IS NULL`
-      ]
-    },
-    {
-      release: '2.5.0',
-      version: 7,
-      previous: 6,
-      install: [
-        `CREATE TABLE IF NOT EXISTS ${schema}.archive (LIKE ${schema}.job)`,
-        `ALTER TABLE ${schema}.archive ADD archivedOn timestamptz NOT NULL DEFAULT now()`
-      ],
-      uninstall: [
-        `DROP TABLE ${schema}.archive`
-      ]
-    },
-    {
-      release: '2.0.0',
-      version: 6,
-      previous: 5,
-      install: [
-        `CREATE INDEX job_fetch ON ${schema}.job (priority desc, createdOn, id) WHERE state < 'active'`
-      ],
-      uninstall: [
-        `DROP INDEX ${schema}.job_fetch`
-      ]
-    },
-    {
-      release: '2.0.0',
-      version: 5,
-      previous: 4,
-      install: [
-        `ALTER TABLE ${schema}.job ALTER COLUMN startIn SET DEFAULT (interval '0')`,
-        `ALTER TABLE ${schema}.job ALTER COLUMN state SET DEFAULT ('created')`,
-        `UPDATE ${schema}.job SET name = left(name, -9) || '__state__expired' WHERE name LIKE '%__expired'`
-      ],
-      uninstall: [
-        `ALTER TABLE ${schema}.job ALTER COLUMN startIn DROP DEFAULT`,
-        `ALTER TABLE ${schema}.job ALTER COLUMN state DROP DEFAULT`,
-        `UPDATE ${schema}.job SET name = left(name, -16) || '__expired' WHERE name LIKE '%__state__expired'`
-      ]
-    },
-    {
-      release: '1.1.0',
-      version: 4,
-      previous: 3,
-      install: [
-        `ALTER TABLE ${schema}.job ADD COLUMN priority integer not null default(0)`,
-        `ALTER TABLE ${schema}.job ALTER COLUMN createdOn SET DATA TYPE timestamptz`,
-        `ALTER TABLE ${schema}.job ALTER COLUMN startedOn SET DATA TYPE timestamptz`,
-        `ALTER TABLE ${schema}.job ALTER COLUMN completedOn SET DATA TYPE timestamptz`
-      ],
-      uninstall: [
-        `ALTER TABLE ${schema}.job DROP COLUMN priority`,
-        `ALTER TABLE ${schema}.job ALTER COLUMN createdOn SET DATA TYPE timestamp`,
-        `ALTER TABLE ${schema}.job ALTER COLUMN startedOn SET DATA TYPE timestamp`,
-        `ALTER TABLE ${schema}.job ALTER COLUMN completedOn SET DATA TYPE timestamp`
-      ]
     }
   ]
 }

package/src/plans.js

@@ -13,6 +13,7 @@ const states = {
 const DEFAULT_SCHEMA = 'pgboss'
 const COMPLETION_JOB_PREFIX = `__state__${states.completed}__`
 const SINGLETON_QUEUE_KEY = '__pgboss__singleton_queue'
+const SINGLETON_QUEUE_KEY_ESCAPED = SINGLETON_QUEUE_KEY.replace(/_/g, '\\_')
 
 const MIGRATE_RACE_MESSAGE = 'division by zero'
 const CREATE_RACE_MESSAGE = 'already exists'
@@ -68,6 +69,7 @@ function locked (schema, query) {
 
   return `
     BEGIN;
+    SET LOCAL statement_timeout = '30s';
     ${advisoryLock(schema)};
     ${query};
     COMMIT;
@@ -214,14 +216,14 @@ function getQueueSize (schema, options = {}) {
 function createIndexSingletonKey (schema) {
   // anything with singletonKey means "only 1 job can be queued or active at a time"
   return `
-    CREATE UNIQUE INDEX job_singletonKey ON ${schema}.job (name, singletonKey) WHERE state < '${states.completed}' AND singletonOn IS NULL AND NOT singletonKey = '${SINGLETON_QUEUE_KEY}'
+    CREATE UNIQUE INDEX job_singletonKey ON ${schema}.job (name, singletonKey) WHERE state < '${states.completed}' AND singletonOn IS NULL AND NOT singletonKey LIKE '${SINGLETON_QUEUE_KEY_ESCAPED}%'
   `
 }
 
 function createIndexSingletonQueue (schema) {
   // "singleton queue" means "only 1 job can be queued at a time"
   return `
-    CREATE UNIQUE INDEX job_singleton_queue ON ${schema}.job (name, singletonKey) WHERE state < '${states.active}' AND singletonOn IS NULL AND singletonKey = '${SINGLETON_QUEUE_KEY}'
+    CREATE UNIQUE INDEX job_singleton_queue ON ${schema}.job (name, singletonKey) WHERE state < '${states.active}' AND singletonOn IS NULL AND singletonKey LIKE '${SINGLETON_QUEUE_KEY_ESCAPED}%'
   `
 }
 

package/src/timekeeper.js

@@ -1,4 +1,3 @@
-const pMap = require('p-map')
 const EventEmitter = require('events')
 const plans = require('./plans')
 const cronParser = require('cron-parser')
@@ -44,18 +43,23 @@ class Timekeeper extends EventEmitter {
   }
 
   async start () {
+    // setting the archive config too low breaks the cron 60s debounce interval so don't even try
     if (this.config.archiveSeconds < 60) {
       return
     }
 
+    // cache the clock skew from the db server
     await this.cacheClockSkew()
 
     await this.manager.work(queues.CRON, { newJobCheckIntervalSeconds: 4 }, (job) => this.onCron(job))
     await this.manager.work(queues.SEND_IT, { newJobCheckIntervalSeconds: 4, teamSize: 50, teamConcurrency: 5 }, (job) => this.onSendIt(job))
 
-    await this.cronMonitorAsync()
+    // uses sendDebounced() to enqueue a cron check
+    await this.checkSchedulesAsync()
 
+    // create monitoring interval to make sure cron hasn't crashed
     this.cronMonitorInterval = setInterval(async () => await this.monitorCron(), this.cronMonitorIntervalMs)
+    // create monitoring interval to measure and adjust for drift in clock skew
     this.skewMonitorInterval = setInterval(async () => await this.cacheClockSkew(), this.skewMonitorIntervalMs)
 
     this.stopped = false
@@ -86,7 +90,7 @@ class Timekeeper extends EventEmitter {
     const { secondsAgo } = await this.getCronTime()
 
     if (secondsAgo > 60) {
-      await this.cronMonitorAsync()
+      await this.checkSchedulesAsync()
     }
   }
 
@@ -108,7 +112,7 @@ class Timekeeper extends EventEmitter {
     this.clockSkew = skew
   }
 
-  async cronMonitorAsync () {
+  async checkSchedulesAsync () {
     const opts = {
       retryLimit: 2,
       retentionSeconds: 60,
@@ -121,6 +125,8 @@ class Timekeeper extends EventEmitter {
   async onCron () {
     if (this.stopped) return
 
+    const { default: pMap } = await import('p-map')
+
     try {
       if (this.config.__test__throw_clock_monitoring) {
         throw new Error('clock monitoring error')
@@ -136,6 +142,7 @@ class Timekeeper extends EventEmitter {
 
       if (this.stopped) return
 
+      // set last time cron was evaluated for downstream usage in cron monitoring
       await this.setCronTime()
     } catch (err) {
       this.emit(this.events.error, err)
@@ -143,7 +150,8 @@ class Timekeeper extends EventEmitter {
 
     if (this.stopped) return
 
-    await this.cronMonitorAsync()
+    // uses sendDebounced() to enqueue a cron check
+    await this.checkSchedulesAsync()
   }
 
   shouldSendIt (cron, tz) {

package/src/worker.js

@@ -27,6 +27,15 @@ class Worker {
     this.stopping = false
     this.stopped = false
     this.loopDelayPromise = null
+    this.beenNotified = false
+  }
+
+  notify () {
+    this.beenNotified = true
+
+    if (this.loopDelayPromise) {
+      this.loopDelayPromise.clear()
+    }
   }
 
   async start () {
@@ -36,6 +45,7 @@ class Worker {
       const started = Date.now()
 
       try {
+        this.beenNotified = false
         const jobs = await this.fetch()
 
         this.lastFetchedOn = Date.now()
@@ -64,7 +74,7 @@ class Worker {
 
       this.lastJobDuration = duration
 
-      if (!this.stopping && duration < this.interval) {
+      if (!this.stopping && !this.beenNotified && duration < this.interval) {
         this.loopDelayPromise = delay(this.interval - duration)
         await this.loopDelayPromise
         this.loopDelayPromise = null

package/types.d.ts

@@ -84,13 +84,20 @@ declare namespace PgBoss {
     priority?: number;
     startAfter?: number | string | Date;
     singletonKey?: string;
+    useSingletonQueue?: boolean;
     singletonSeconds?: number;
     singletonMinutes?: number;
     singletonHours?: number;
     singletonNextSlot?: boolean;
   }
 
-  type SendOptions = JobOptions & ExpirationOptions & RetentionOptions & RetryOptions & CompletionOptions
+  interface ConnectionOptions {
+    db?: Db;
+  }
+
+  type InsertOptions = ConnectionOptions;
+
+  type SendOptions = JobOptions & ExpirationOptions & RetentionOptions & RetryOptions & CompletionOptions & ConnectionOptions;
 
   type ScheduleOptions = SendOptions & { tz?: string }
 
@@ -111,7 +118,7 @@ declare namespace PgBoss {
 
   type FetchOptions = {
     includeMetadata?: boolean;
-  }
+  } & ConnectionOptions;
 
   interface WorkHandler<ReqData, ResData> {
     (job: PgBoss.JobWithDoneCallback<ReqData, ResData>): Promise<ResData> | void;
@@ -189,7 +196,7 @@ declare namespace PgBoss {
     retryLimit?: number;
     retryDelay?: number;
     retryBackoff?: boolean;
-    startAfter?: Date | string;    
+    startAfter?: Date | string;
     singletonKey?: string;
     expireInSeconds?: number;
     keepUntil?: Date | string;
@@ -297,6 +304,7 @@ declare class PgBoss extends EventEmitter {
   sendDebounced(name: string, data: object, options: PgBoss.SendOptions, seconds: number, key: string): Promise<string | null>;
 
   insert(jobs: PgBoss.JobInsert[]): Promise<void>;
+  insert(jobs: PgBoss.JobInsert[], options: PgBoss.InsertOptions): Promise<void>;
 
   work<ReqData, ResData>(name: string, handler: PgBoss.WorkHandler<ReqData, ResData>): Promise<string>;
   work<ReqData, ResData>(name: string, options: PgBoss.WorkOptions & { includeMetadata: true }, handler: PgBoss.WorkWithMetadataHandler<ReqData, ResData>): Promise<string>;
@@ -308,6 +316,12 @@ declare class PgBoss extends EventEmitter {
   offWork(name: string): Promise<void>;
   offWork(options: PgBoss.OffWorkOptions): Promise<void>;
 
+  /**
+   * Notify worker that something has changed
+   * @param workerId
+   */
+  notifyWorker(workerId: string): void;
+
   subscribe(event: string, name: string): Promise<void>;
   unsubscribe(event: string, name: string): Promise<void>;
   publish(event: string): Promise<string[]>;

package/version.json

@@ -1,3 +1,3 @@
 {
-  "schema": 19
+  "schema": 20
 }