pg-boss 7.2.0 → 7.4.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pg-boss",
-  "version": "7.2.0",
+  "version": "7.4.0",
   "description": "Queueing jobs in Node.js using PostgreSQL like a boss",
   "main": "./src/index.js",
   "engines": {

package/src/attorney.js

@@ -1,9 +1,10 @@
 const assert = require('assert')
-const { DEFAULT_SCHEMA } = require('./plans')
+const { DEFAULT_SCHEMA, SINGLETON_QUEUE_KEY } = require('./plans')
 
 module.exports = {
   getConfig,
   checkSendArgs,
+  checkInsertArgs,
   checkWorkArgs,
   checkFetchArgs,
   warnClockSkew
@@ -60,6 +61,7 @@ function checkSendArgs (args, defaults) {
   applyExpirationConfig(options, defaults)
   applyRetentionConfig(options, defaults)
   applyCompletionConfig(options, defaults)
+  applySingletonKeyConfig(options)
 
   const { startAfter, singletonSeconds, singletonMinutes, singletonHours } = options
 
@@ -84,6 +86,22 @@ function checkSendArgs (args, defaults) {
   return { name, data, options }
 }
 
+function checkInsertArgs (jobs) {
+  assert(Array.isArray(jobs), `jobs argument should be an array.  Received '${typeof jobs}'`)
+  return jobs.map(job => {
+    job = { ...job }
+    applySingletonKeyConfig(job)
+    return job
+  })
+}
+
+function applySingletonKeyConfig (options) {
+  if (options.singletonKey && options.useSingletonQueue && options.singletonKey !== SINGLETON_QUEUE_KEY) {
+    options.singletonKey = SINGLETON_QUEUE_KEY + options.singletonKey
+  }
+  delete options.useSingletonQueue
+}
+
 function checkWorkArgs (name, args, defaults) {
   let options, callback
 

package/src/manager.js

@@ -74,6 +74,7 @@ class Manager extends EventEmitter {
       this.fetchCompleted,
       this.work,
       this.offWork,
+      this.notifyWorker,
       this.onComplete,
       this.offComplete,
       this.publish,
@@ -294,6 +295,12 @@ class Manager extends EventEmitter {
     })
   }
 
+  notifyWorker (workerId) {
+    if (this.workers.has(workerId)) {
+      this.workers.get(workerId).notify()
+    }
+  }
+
   async subscribe (event, name) {
     assert(event, 'Missing required argument')
     assert(name, 'Missing required argument')
@@ -439,8 +446,8 @@ class Manager extends EventEmitter {
   }
 
   async insert (jobs) {
-    assert(Array.isArray(jobs), `jobs argument should be an array.  Received '${typeof jobs}'`)
-    const data = JSON.stringify(jobs)
+    const checkedJobs = Attorney.checkInsertArgs(jobs)
+    const data = JSON.stringify(checkedJobs)
     return await this.db.executeSql(this.insertJobsCommand, [data])
   }
 

package/src/migrationStore.js

@@ -62,11 +62,25 @@ function migrate (value, version, migrations) {
   return flatten(schema, result.install, result.version)
 }
 
-function getAll (schema, config) {
-  const DEFAULT_RETENTION = '30 days'
-  const keepUntil = config ? config.keepUntil : DEFAULT_RETENTION
-
+function getAll (schema) {
   return [
+    {
+      release: '7.4.0',
+      version: 20,
+      previous: 19,
+      install: [
+        `DROP INDEX ${schema}.job_singletonKey`,
+        `DROP INDEX ${schema}.job_singleton_queue`,
+        `CREATE UNIQUE INDEX job_singletonKey ON ${schema}.job (name, singletonKey) WHERE state < 'completed' AND singletonOn IS NULL AND NOT singletonKey LIKE '\\_\\_pgboss\\_\\_singleton\\_queue%'`,
+        `CREATE UNIQUE INDEX job_singleton_queue ON ${schema}.job (name, singletonKey) WHERE state < 'active' AND singletonOn IS NULL AND singletonKey LIKE '\\_\\_pgboss\\_\\_singleton\\_queue%'`
+      ],
+      uninstall: [
+        `DROP INDEX ${schema}.job_singletonKey`,
+        `DROP INDEX ${schema}.job_singleton_queue`,
+        `CREATE UNIQUE INDEX job_singletonKey ON ${schema}.job (name, singletonKey) WHERE state < 'completed' AND singletonOn IS NULL AND NOT singletonKey = '__pgboss__singleton_queue'`,
+        `CREATE UNIQUE INDEX job_singleton_queue ON ${schema}.job (name, singletonKey) WHERE state < 'active' AND singletonOn IS NULL AND singletonKey = '__pgboss__singleton_queue'`
+      ]
+    },
     {
       release: '7.0.0',
       version: 19,
@@ -154,218 +168,6 @@ function getAll (schema, config) {
       uninstall: [
           `ALTER TABLE ${schema}.version DROP COLUMN maintained_on`
       ]
-    },
-    {
-      release: '5.0.0-beta1',
-      version: 13,
-      previous: 12,
-      install: [
-          `CREATE TABLE ${schema}.schedule (
-            name text primary key,
-            cron text not null,
-            timezone text,
-            data jsonb,
-            options jsonb,
-            created_on timestamp with time zone not null default now(),
-            updated_on timestamp with time zone not null default now()
-          )`
-      ],
-      uninstall: [
-          `DROP TABLE ${schema}.schedule`
-      ]
-    },
-    {
-      release: '4.0.0',
-      version: 12,
-      previous: 11,
-      install: [
-          `ALTER TABLE ${schema}.version ALTER COLUMN version TYPE int USING version::int`,
-          `ALTER TABLE ${schema}.job ADD COLUMN keepUntil timestamptz`,
-          `ALTER TABLE ${schema}.archive ADD COLUMN keepUntil timestamptz`,
-          `ALTER TABLE ${schema}.job ALTER COLUMN keepUntil SET DEFAULT now() + interval '${DEFAULT_RETENTION}'`,
-          `UPDATE ${schema}.job SET keepUntil = startAfter + interval '${keepUntil}'`,
-          `ALTER TABLE ${schema}.job ALTER COLUMN keepUntil SET NOT NULL`
-      ],
-      uninstall: [
-          `ALTER TABLE ${schema}.version ALTER COLUMN version TYPE text USING version::text`,
-          `ALTER TABLE ${schema}.job DROP COLUMN keepUntil`,
-          `ALTER TABLE ${schema}.archive DROP COLUMN keepUntil`
-      ]
-    },
-    {
-      release: '3.2.0',
-      version: 11,
-      previous: 10,
-      install: [
-          `CREATE INDEX archive_archivedon_idx ON ${schema}.archive(archivedon)`
-      ],
-      uninstall: [
-          `DROP INDEX ${schema}.archive_archivedon_idx`
-      ]
-    },
-    {
-      release: '3.1.3',
-      version: 10,
-      previous: 9,
-      install: [
-        `CREATE INDEX archive_id_idx ON ${schema}.archive(id)`
-      ],
-      uninstall: [
-        `DROP INDEX ${schema}.archive_id_idx`
-      ]
-    },
-    {
-      release: '3.1.0',
-      version: 9,
-      previous: 8,
-      install: [
-        `DROP INDEX ${schema}.job_fetch`,
-        `DROP INDEX ${schema}.job_name`,
-        `CREATE INDEX job_name ON ${schema}.job (name text_pattern_ops)`,
-        `UPDATE ${schema}.job set name = '__state__completed__' || substr(name, 1, position('__state__completed' in name) - 1) WHERE name LIKE '%__state__completed'`
-      ],
-      uninstall: [
-        `UPDATE ${schema}.job set name = substr(name, 21) || '__state__completed' WHERE name LIKE '__state__completed__%'`,
-        `CREATE INDEX job_fetch ON ${schema}.job (name, priority desc, createdOn, id) WHERE state < 'active'`,
-        `DROP INDEX ${schema}.job_name`,
-        `CREATE INDEX job_name ON ${schema}.job (name) WHERE state < 'active'`
-      ]
-    },
-    {
-      release: '3.0.0',
-      version: 8,
-      previous: 7,
-      install: [
-        'CREATE EXTENSION IF NOT EXISTS pgcrypto',
-        `ALTER TABLE ${schema}.job ALTER COLUMN id SET DEFAULT gen_random_uuid()`,
-        `ALTER TABLE ${schema}.job ADD retryDelay integer not null DEFAULT (0)`,
-        `ALTER TABLE ${schema}.job ADD retryBackoff boolean not null DEFAULT false`,
-        `ALTER TABLE ${schema}.job ADD startAfter timestamp with time zone not null default now()`,
-        `UPDATE ${schema}.job SET startAfter = createdOn + startIn`,
-        `ALTER TABLE ${schema}.job DROP COLUMN startIn`,
-        `UPDATE ${schema}.job SET expireIn = interval '15 minutes' WHERE expireIn IS NULL`,
-        `ALTER TABLE ${schema}.job ALTER COLUMN expireIn SET NOT NULL`,
-        `ALTER TABLE ${schema}.job ALTER COLUMN expireIn SET DEFAULT interval '15 minutes'`,
-        // archive table schema changes
-        `ALTER TABLE ${schema}.archive ADD retryDelay integer not null DEFAULT (0)`,
-        `ALTER TABLE ${schema}.archive ADD retryBackoff boolean not null DEFAULT false`,
-        `ALTER TABLE ${schema}.archive ADD startAfter timestamp with time zone`,
-        `UPDATE ${schema}.archive SET startAfter = createdOn + startIn`,
-        `ALTER TABLE ${schema}.archive DROP COLUMN startIn`,
-        // rename complete to completed for state enum - can't use ALTER TYPE :(
-        `DROP INDEX ${schema}.job_fetch`,
-        `DROP INDEX ${schema}.job_singletonOn`,
-        `DROP INDEX ${schema}.job_singletonKeyOn`,
-        `DROP INDEX ${schema}.job_singletonKey`,
-        `ALTER TABLE ${schema}.job ALTER COLUMN state DROP DEFAULT`,
-        `ALTER TABLE ${schema}.job ALTER COLUMN state SET DATA TYPE text USING state::text`,
-        `ALTER TABLE ${schema}.archive ALTER COLUMN state SET DATA TYPE text USING state::text`,
-        `DROP TYPE ${schema}.job_state`,
-        `CREATE TYPE ${schema}.job_state AS ENUM ('created', 'retry', 'active', 'completed', 'expired', 'cancelled', 'failed')`,
-        `UPDATE ${schema}.job SET state = 'completed' WHERE state = 'complete'`,
-        `UPDATE ${schema}.archive SET state = 'completed' WHERE state = 'complete'`,
-        `ALTER TABLE ${schema}.job ALTER COLUMN state SET DATA TYPE ${schema}.job_state USING state::${schema}.job_state`,
-        `ALTER TABLE ${schema}.job ALTER COLUMN state SET DEFAULT 'created'`,
-        `ALTER TABLE ${schema}.archive ALTER COLUMN state SET DATA TYPE ${schema}.job_state USING state::${schema}.job_state`,
-        `CREATE INDEX job_fetch ON ${schema}.job (name, priority desc, createdOn, id) WHERE state < 'active'`,
-        `CREATE UNIQUE INDEX job_singletonOn ON ${schema}.job (name, singletonOn) WHERE state < 'expired' AND singletonKey IS NULL`,
-        `CREATE UNIQUE INDEX job_singletonKeyOn ON ${schema}.job (name, singletonOn, singletonKey) WHERE state < 'expired'`,
-        `CREATE UNIQUE INDEX job_singletonKey ON ${schema}.job (name, singletonKey) WHERE state < 'completed' AND singletonOn IS NULL`,
-        // add new job name index
-        `CREATE INDEX job_name ON ${schema}.job (name) WHERE state < 'active'`
-      ],
-      uninstall: [
-        `ALTER TABLE ${schema}.job ALTER COLUMN id DROP DEFAULT`,
-        // won't know if we should drop pgcrypto extension so it stays
-        `ALTER TABLE ${schema}.job DROP COLUMN retryDelay`,
-        `ALTER TABLE ${schema}.job DROP COLUMN retryBackoff`,
-        `ALTER TABLE ${schema}.job DROP COLUMN startAfter`,
-        `ALTER TABLE ${schema}.job ADD COLUMN startIn interval not null default(interval '0')`,
-        // leaving migrated default data for expireIn
-        `ALTER TABLE ${schema}.job ALTER COLUMN expireIn DROP NOT NULL`,
-        `ALTER TABLE ${schema}.job ALTER COLUMN expireIn DROP DEFAULT`,
-        // archive table restore
-        `ALTER TABLE ${schema}.archive DROP COLUMN retryDelay`,
-        `ALTER TABLE ${schema}.archive DROP COLUMN retryBackoff`,
-        `ALTER TABLE ${schema}.archive DROP COLUMN startAfter`,
-        `ALTER TABLE ${schema}.archive ADD COLUMN startIn interval`,
-        // drop new job name index
-        `DROP INDEX ${schema}.job_name`,
-        // roll back to old enum def
-        `DROP INDEX ${schema}.job_fetch`,
-        `DROP INDEX ${schema}.job_singletonOn`,
-        `DROP INDEX ${schema}.job_singletonKeyOn`,
-        `DROP INDEX ${schema}.job_singletonKey`,
-        `ALTER TABLE ${schema}.job ALTER COLUMN state DROP DEFAULT`,
-        `ALTER TABLE ${schema}.job ALTER COLUMN state SET DATA TYPE text USING state::text`,
-        `ALTER TABLE ${schema}.archive ALTER COLUMN state SET DATA TYPE text USING state::text`,
-        `DROP TYPE ${schema}.job_state`,
-        `CREATE TYPE ${schema}.job_state AS ENUM ('created', 'retry', 'active', 'complete', 'expired', 'cancelled', 'failed')`,
-        `UPDATE ${schema}.job SET state = 'completed' WHERE state = 'complete'`,
-        `UPDATE ${schema}.archive SET state = 'complete' WHERE state = 'completed'`,
-        `ALTER TABLE ${schema}.job ALTER COLUMN state SET DATA TYPE ${schema}.job_state USING state::${schema}.job_state`,
-        `ALTER TABLE ${schema}.job ALTER COLUMN state SET DEFAULT 'created'`,
-        `ALTER TABLE ${schema}.archive ALTER COLUMN state SET DATA TYPE ${schema}.job_state USING state::${schema}.job_state`,
-        `CREATE INDEX job_fetch ON ${schema}.job (name, priority desc, createdOn, id) WHERE state < 'active'`,
-        `CREATE UNIQUE INDEX job_singletonOn ON ${schema}.job (name, singletonOn) WHERE state < 'expired' AND singletonKey IS NULL`,
-        `CREATE UNIQUE INDEX job_singletonKeyOn ON ${schema}.job (name, singletonOn, singletonKey) WHERE state < 'expired'`,
-        `CREATE UNIQUE INDEX job_singletonKey ON ${schema}.job (name, singletonKey) WHERE state < 'complete' AND singletonOn IS NULL`
-      ]
-    },
-    {
-      release: '2.5.0',
-      version: 7,
-      previous: 6,
-      install: [
-        `CREATE TABLE IF NOT EXISTS ${schema}.archive (LIKE ${schema}.job)`,
-        `ALTER TABLE ${schema}.archive ADD archivedOn timestamptz NOT NULL DEFAULT now()`
-      ],
-      uninstall: [
-        `DROP TABLE ${schema}.archive`
-      ]
-    },
-    {
-      release: '2.0.0',
-      version: 6,
-      previous: 5,
-      install: [
-        `CREATE INDEX job_fetch ON ${schema}.job (priority desc, createdOn, id) WHERE state < 'active'`
-      ],
-      uninstall: [
-        `DROP INDEX ${schema}.job_fetch`
-      ]
-    },
-    {
-      release: '2.0.0',
-      version: 5,
-      previous: 4,
-      install: [
-        `ALTER TABLE ${schema}.job ALTER COLUMN startIn SET DEFAULT (interval '0')`,
-        `ALTER TABLE ${schema}.job ALTER COLUMN state SET DEFAULT ('created')`,
-        `UPDATE ${schema}.job SET name = left(name, -9) || '__state__expired' WHERE name LIKE '%__expired'`
-      ],
-      uninstall: [
-        `ALTER TABLE ${schema}.job ALTER COLUMN startIn DROP DEFAULT`,
-        `ALTER TABLE ${schema}.job ALTER COLUMN state DROP DEFAULT`,
-        `UPDATE ${schema}.job SET name = left(name, -16) || '__expired' WHERE name LIKE '%__state__expired'`
-      ]
-    },
-    {
-      release: '1.1.0',
-      version: 4,
-      previous: 3,
-      install: [
-        `ALTER TABLE ${schema}.job ADD COLUMN priority integer not null default(0)`,
-        `ALTER TABLE ${schema}.job ALTER COLUMN createdOn SET DATA TYPE timestamptz`,
-        `ALTER TABLE ${schema}.job ALTER COLUMN startedOn SET DATA TYPE timestamptz`,
-        `ALTER TABLE ${schema}.job ALTER COLUMN completedOn SET DATA TYPE timestamptz`
-      ],
-      uninstall: [
-        `ALTER TABLE ${schema}.job DROP COLUMN priority`,
-        `ALTER TABLE ${schema}.job ALTER COLUMN createdOn SET DATA TYPE timestamp`,
-        `ALTER TABLE ${schema}.job ALTER COLUMN startedOn SET DATA TYPE timestamp`,
-        `ALTER TABLE ${schema}.job ALTER COLUMN completedOn SET DATA TYPE timestamp`
-      ]
     }
   ]
 }

package/src/plans.js

@@ -13,6 +13,7 @@ const states = {
 const DEFAULT_SCHEMA = 'pgboss'
 const COMPLETION_JOB_PREFIX = `__state__${states.completed}__`
 const SINGLETON_QUEUE_KEY = '__pgboss__singleton_queue'
+const SINGLETON_QUEUE_KEY_ESCAPED = SINGLETON_QUEUE_KEY.replace(/_/g, '\\_')
 
 const MIGRATE_RACE_MESSAGE = 'division by zero'
 const CREATE_RACE_MESSAGE = 'already exists'
@@ -214,14 +215,14 @@ function getQueueSize (schema, options = {}) {
 function createIndexSingletonKey (schema) {
   // anything with singletonKey means "only 1 job can be queued or active at a time"
   return `
-    CREATE UNIQUE INDEX job_singletonKey ON ${schema}.job (name, singletonKey) WHERE state < '${states.completed}' AND singletonOn IS NULL AND NOT singletonKey = '${SINGLETON_QUEUE_KEY}'
+    CREATE UNIQUE INDEX job_singletonKey ON ${schema}.job (name, singletonKey) WHERE state < '${states.completed}' AND singletonOn IS NULL AND NOT singletonKey LIKE '${SINGLETON_QUEUE_KEY_ESCAPED}%'
   `
 }
 
 function createIndexSingletonQueue (schema) {
   // "singleton queue" means "only 1 job can be queued at a time"
   return `
-    CREATE UNIQUE INDEX job_singleton_queue ON ${schema}.job (name, singletonKey) WHERE state < '${states.active}' AND singletonOn IS NULL AND singletonKey = '${SINGLETON_QUEUE_KEY}'
+    CREATE UNIQUE INDEX job_singleton_queue ON ${schema}.job (name, singletonKey) WHERE state < '${states.active}' AND singletonOn IS NULL AND singletonKey LIKE '${SINGLETON_QUEUE_KEY_ESCAPED}%'
   `
 }
 

package/src/timekeeper.js

@@ -44,18 +44,23 @@ class Timekeeper extends EventEmitter {
   }
 
   async start () {
+    // setting the archive config too low breaks the cron 60s debounce interval so don't even try
     if (this.config.archiveSeconds < 60) {
       return
     }
 
+    // cache the clock skew from the db server
     await this.cacheClockSkew()
 
     await this.manager.work(queues.CRON, { newJobCheckIntervalSeconds: 4 }, (job) => this.onCron(job))
     await this.manager.work(queues.SEND_IT, { newJobCheckIntervalSeconds: 4, teamSize: 50, teamConcurrency: 5 }, (job) => this.onSendIt(job))
 
-    await this.cronMonitorAsync()
+    // uses sendDebounced() to enqueue a cron check
+    await this.checkSchedulesAsync()
 
+    // create monitoring interval to make sure cron hasn't crashed
     this.cronMonitorInterval = setInterval(async () => await this.monitorCron(), this.cronMonitorIntervalMs)
+    // create monitoring interval to measure and adjust for drift in clock skew
     this.skewMonitorInterval = setInterval(async () => await this.cacheClockSkew(), this.skewMonitorIntervalMs)
 
     this.stopped = false
@@ -86,7 +91,7 @@ class Timekeeper extends EventEmitter {
     const { secondsAgo } = await this.getCronTime()
 
     if (secondsAgo > 60) {
-      await this.cronMonitorAsync()
+      await this.checkSchedulesAsync()
     }
   }
 
@@ -108,7 +113,7 @@ class Timekeeper extends EventEmitter {
     this.clockSkew = skew
   }
 
-  async cronMonitorAsync () {
+  async checkSchedulesAsync () {
     const opts = {
       retryLimit: 2,
       retentionSeconds: 60,
@@ -136,6 +141,7 @@ class Timekeeper extends EventEmitter {
 
       if (this.stopped) return
 
+      // set last time cron was evaluated for downstream usage in cron monitoring
       await this.setCronTime()
     } catch (err) {
       this.emit(this.events.error, err)
@@ -143,7 +149,8 @@ class Timekeeper extends EventEmitter {
 
     if (this.stopped) return
 
-    await this.cronMonitorAsync()
+    // uses sendDebounced() to enqueue a cron check
+    await this.checkSchedulesAsync()
   }
 
   shouldSendIt (cron, tz) {

package/src/worker.js

@@ -27,6 +27,15 @@ class Worker {
     this.stopping = false
     this.stopped = false
     this.loopDelayPromise = null
+    this.beenNotified = false
+  }
+
+  notify () {
+    this.beenNotified = true
+
+    if (this.loopDelayPromise) {
+      this.loopDelayPromise.clear()
+    }
   }
 
   async start () {
@@ -36,6 +45,7 @@ class Worker {
       const started = Date.now()
 
       try {
+        this.beenNotified = false
         const jobs = await this.fetch()
 
         this.lastFetchedOn = Date.now()
@@ -64,7 +74,7 @@ class Worker {
 
       this.lastJobDuration = duration
 
-      if (!this.stopping && duration < this.interval) {
+      if (!this.stopping && !this.beenNotified && duration < this.interval) {
         this.loopDelayPromise = delay(this.interval - duration)
         await this.loopDelayPromise
         this.loopDelayPromise = null

package/types.d.ts

@@ -84,6 +84,7 @@ declare namespace PgBoss {
     priority?: number;
     startAfter?: number | string | Date;
     singletonKey?: string;
+    useSingletonQueue?: boolean;
     singletonSeconds?: number;
     singletonMinutes?: number;
     singletonHours?: number;
@@ -102,6 +103,7 @@ declare namespace PgBoss {
   interface JobFetchOptions {
     teamSize?: number;
     teamConcurrency?: number;
+    teamRefill?: boolean;
     batchSize?: number;
     includeMetadata?: boolean;
   }
@@ -307,6 +309,12 @@ declare class PgBoss extends EventEmitter {
   offWork(name: string): Promise<void>;
   offWork(options: PgBoss.OffWorkOptions): Promise<void>;
 
+  /**
+   * Notify worker that something has changed
+   * @param workerId 
+   */
+  notifyWorker(workerId: string): void;
+
   subscribe(event: string, name: string): Promise<void>;
   unsubscribe(event: string, name: string): Promise<void>;
   publish(event: string): Promise<string[]>;

package/version.json

@@ -1,3 +1,3 @@
 {
-  "schema": 19
+  "schema": 20
 }