pg-boss 10.0.0-beta2 → 10.0.0-beta21

package/README.md

@@ -9,50 +9,47 @@ async function readme() {
   const PgBoss = require('pg-boss');
   const boss = new PgBoss('postgres://user:pass@host/database');
 
-  boss.on('error', error => console.error(error));
+  boss.on('error', console.error)
 
-  await boss.start();
+  await boss.start()
 
-  const queue = 'some-queue';
+  const queue = 'readme-queue'
 
-  let jobId = await boss.send(queue, { param1: 'foo' })
+  const id = await boss.send(queue, { arg1: 'read me' })
 
-  console.log(`created job in queue ${queue}: ${jobId}`);
+  console.log(`created job ${id} in queue ${queue}`)
 
-  await boss.work(queue, someAsyncJobHandler);
-}
-
-async function someAsyncJobHandler(job) {
-  console.log(`job ${job.id} received with data:`);
-  console.log(JSON.stringify(job.data));
-
-  await doSomethingAsyncWithThis(job.data);
+  await boss.work(queue, async ([ job ]) => {
+    console.log(`received job ${job.id} with data ${JSON.stringify(job.data)}`)
+  })
 }
 ```
 
 pg-boss is a job queue built in Node.js on top of PostgreSQL in order to provide background processing and reliable asynchronous execution to Node.js applications.
 
-pg-boss relies on [SKIP LOCKED](https://www.2ndquadrant.com/en/blog/what-is-select-skip-locked-for-in-postgresql-9-5/), a feature added to postgres specifically for message queues, in order to resolve record locking challenges inherent with relational databases. This brings the safety of guaranteed atomic commits of a relational database to your asynchronous job processing.
+pg-boss relies on [SKIP LOCKED](https://www.2ndquadrant.com/en/blog/what-is-select-skip-locked-for-in-postgresql-9-5/), a feature built specifically for message queues to resolve record locking challenges inherent with relational databases. This provides exactly-once delivery and the safety of guaranteed atomic commits to asynchronous job processing.
 
 This will likely cater the most to teams already familiar with the simplicity of relational database semantics and operations (SQL, querying, and backups). It will be especially useful to those already relying on PostgreSQL that want to limit how many systems are required to monitor and support in their architecture.
 
-## Features
+
+## Summary
 * Exactly-once job delivery
 * Backpressure-compatible polling workers
 * Cron scheduling
+* Queue storage policies to support a variety of rate limiting, debouncing, and concurrency use cases
+* Priority queues, dead letter queues, job deferral, automatic retries with exponential backoff
 * Pub/sub API for fan-out queue relationships
-* Priority queues, deferral, retries (with exponential backoff), rate limiting, debouncing
-* Table operations via SQL for bulk loads via COPY or INSERT
+* Raw SQL support for non-Node.js runtimes via INSERT or COPY
+* Serverless function compatible
 * Multi-master compatible (for example, in a Kubernetes ReplicaSet)
-* Dead letter queues
 
 ## Requirements
 * Node 20 or higher
-* PostgreSQL 12 or higher
+* PostgreSQL 13 or higher
 
 ## Installation
 
-``` bash
+```bash
 # npm
 npm install pg-boss
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pg-boss",
-  "version": "10.0.0-beta2",
+  "version": "10.0.0-beta21",
   "description": "Queueing jobs in Postgres from Node.js like a boss",
   "main": "./src/index.js",
   "engines": {
@@ -8,8 +8,6 @@
   },
   "dependencies": {
     "cron-parser": "^4.0.0",
-    "lodash.debounce": "^4.0.8",
-    "p-map": "^4.0.0",
     "pg": "^8.5.1",
     "serialize-error": "^8.1.0"
   },

package/src/attorney.js

@@ -8,7 +8,8 @@ module.exports = {
   checkWorkArgs,
   checkFetchArgs,
   warnClockSkew,
-  assertPostgresObjectName
+  assertPostgresObjectName,
+  assertQueueName
 }
 
 const MAX_INTERVAL_HOURS = 24
@@ -29,8 +30,6 @@ const WARNINGS = {
 }
 
 function checkQueueArgs (name, options = {}) {
-  assertPostgresObjectName(name)
-
   assert(!('deadLetter' in options) || (typeof options.deadLetter === 'string'), 'deadLetter must be a string')
 
   applyRetryConfig(options)
@@ -123,26 +122,25 @@ function checkWorkArgs (name, args, defaults) {
 
   options = { ...options }
 
-  applyNewJobCheckInterval(options, defaults)
-
-  assert(!('teamConcurrency' in options) ||
-    (Number.isInteger(options.teamConcurrency) && options.teamConcurrency >= 1 && options.teamConcurrency <= 1000),
-  'teamConcurrency must be an integer between 1 and 1000')
+  applyPollingInterval(options, defaults)
 
-  assert(!('teamSize' in options) || (Number.isInteger(options.teamSize) && options.teamSize >= 1), 'teamSize must be an integer > 0')
   assert(!('batchSize' in options) || (Number.isInteger(options.batchSize) && options.batchSize >= 1), 'batchSize must be an integer > 0')
   assert(!('includeMetadata' in options) || typeof options.includeMetadata === 'boolean', 'includeMetadata must be a boolean')
+  assert(!('priority' in options) || typeof options.priority === 'boolean', 'priority must be a boolean')
+
+  options.batchSize = options.batchSize || 1
 
   return { options, callback }
 }
 
-function checkFetchArgs (name, batchSize, options) {
+function checkFetchArgs (name, options) {
   assert(name, 'missing queue name')
 
-  assert(!batchSize || (Number.isInteger(batchSize) && batchSize >= 1), 'batchSize must be an integer > 0')
+  assert(!('batchSize' in options) || (Number.isInteger(options.batchSize) && options.batchSize >= 1), 'batchSize must be an integer > 0')
   assert(!('includeMetadata' in options) || typeof options.includeMetadata === 'boolean', 'includeMetadata must be a boolean')
+  assert(!('priority' in options) || typeof options.priority === 'boolean', 'priority must be a boolean')
 
-  return { name }
+  options.batchSize = options.batchSize || 1
 }
 
 function getConfig (value) {
@@ -153,6 +151,10 @@ function getConfig (value) {
     ? { connectionString: value }
     : { ...value }
 
+  config.schedule = ('schedule' in config) ? config.schedule : true
+  config.supervise = ('supervise' in config) ? config.supervise : true
+  config.migrate = ('migrate' in config) ? config.migrate : true
+
   applySchemaConfig(config)
   applyMaintenanceConfig(config)
   applyArchiveConfig(config)
@@ -160,7 +162,7 @@ function getConfig (value) {
   applyDeleteConfig(config)
   applyMonitoringConfig(config)
 
-  applyNewJobCheckInterval(config)
+  applyPollingInterval(config)
   applyExpirationConfig(config)
   applyRetentionConfig(config)
 
@@ -178,8 +180,14 @@ function applySchemaConfig (config) {
 function assertPostgresObjectName (name) {
   assert(typeof name === 'string', 'Name must be a string')
   assert(name.length <= 50, 'Name cannot exceed 50 characters')
-  assert(!/\W/.test(name), 'Name can only contain alphanumeric characters and underscores')
-  assert(!/^d/.test(name), 'Name cannot start with a number')
+  assert(!/\W/.test(name), 'Name can only contain alphanumeric characters or underscores')
+  assert(!/^\d/.test(name), 'Name cannot start with a number')
+}
+
+function assertQueueName (name) {
+  assert(name, 'Name is required')
+  assert(typeof name === 'string', 'Name must be a string')
+  assert(/[\w-]/.test(name), 'Name can only contain alphanumeric characters, underscores, or hyphens')
 }
 
 function applyArchiveConfig (config) {
@@ -270,18 +278,13 @@ function applyRetryConfig (config, defaults) {
   config.retryBackoffDefault = defaults?.retryBackoff
 }
 
-function applyNewJobCheckInterval (config, defaults) {
-  assert(!('newJobCheckInterval' in config) || config.newJobCheckInterval >= 500,
-    'configuration assert: newJobCheckInterval must be at least every 500ms')
-
-  assert(!('newJobCheckIntervalSeconds' in config) || config.newJobCheckIntervalSeconds >= 1,
-    'configuration assert: newJobCheckIntervalSeconds must be at least every second')
+function applyPollingInterval (config, defaults) {
+  assert(!('pollingIntervalSeconds' in config) || config.pollingIntervalSeconds >= 0.5,
+    'configuration assert: pollingIntervalSeconds must be at least every 500ms')
 
-  config.newJobCheckInterval = ('newJobCheckIntervalSeconds' in config)
-    ? config.newJobCheckIntervalSeconds * 1000
-    : ('newJobCheckInterval' in config)
-        ? config.newJobCheckInterval
-        : defaults?.newJobCheckInterval || 2000
+  config.pollingInterval = ('pollingIntervalSeconds' in config)
+    ? config.pollingIntervalSeconds * 1000
+    : defaults?.pollingInterval || 2000
 }
 
 function applyMaintenanceConfig (config) {
@@ -298,10 +301,6 @@ function applyMaintenanceConfig (config) {
         : 120
 
   assert(config.maintenanceIntervalSeconds / 60 / 60 < MAX_INTERVAL_HOURS, `configuration assert: maintenance interval cannot exceed ${MAX_INTERVAL_HOURS} hours`)
-
-  config.schedule = ('schedule' in config) ? config.schedule : true
-  config.supervise = ('supervise' in config) ? config.supervise : true
-  config.migrate = ('migrate' in config) ? config.migrate : true
 }
 
 function applyDeleteConfig (config) {
@@ -363,21 +362,21 @@ function applyMonitoringConfig (config) {
           ? config.clockMonitorIntervalSeconds
           : TEN_MINUTES_IN_SECONDS
 
-  assert(!('cronMonitorIntervalSeconds' in config) || (config.cronMonitorIntervalSeconds >= 1 && config.cronMonitorIntervalSeconds <= 60),
-    'configuration assert: cronMonitorIntervalSeconds must be between 1 and 60 seconds')
+  assert(!('cronMonitorIntervalSeconds' in config) || (config.cronMonitorIntervalSeconds >= 1 && config.cronMonitorIntervalSeconds <= 45),
+    'configuration assert: cronMonitorIntervalSeconds must be between 1 and 45 seconds')
 
   config.cronMonitorIntervalSeconds =
     ('cronMonitorIntervalSeconds' in config)
       ? config.cronMonitorIntervalSeconds
-      : 60
+      : 30
 
-  assert(!('cronWorkerIntervalSeconds' in config) || (config.cronWorkerIntervalSeconds >= 1 && config.cronWorkerIntervalSeconds <= 60),
-    'configuration assert: cronWorkerIntervalSeconds must be between 1 and 60 seconds')
+  assert(!('cronWorkerIntervalSeconds' in config) || (config.cronWorkerIntervalSeconds >= 1 && config.cronWorkerIntervalSeconds <= 45),
+    'configuration assert: cronWorkerIntervalSeconds must be between 1 and 45 seconds')
 
   config.cronWorkerIntervalSeconds =
     ('cronWorkerIntervalSeconds' in config)
       ? config.cronWorkerIntervalSeconds
-      : 4
+      : 5
 }
 
 function warnClockSkew (message) {

package/src/boss.js

@@ -24,10 +24,8 @@ class Boss extends EventEmitter {
     this.failJobsByTimeoutCommand = plans.locked(config.schema, plans.failJobsByTimeout(config.schema))
     this.archiveCommand = plans.locked(config.schema, plans.archive(config.schema, config.archiveInterval, config.archiveFailedInterval))
     this.dropCommand = plans.locked(config.schema, plans.drop(config.schema, config.deleteAfter))
-    this.getMaintenanceTimeCommand = plans.getMaintenanceTime(config.schema)
-    this.setMaintenanceTimeCommand = plans.setMaintenanceTime(config.schema)
-    this.getMonitorTimeCommand = plans.getMonitorTime(config.schema)
-    this.setMonitorTimeCommand = plans.setMonitorTime(config.schema)
+    this.trySetMaintenanceTimeCommand = plans.trySetMaintenanceTime(config.schema)
+    this.trySetMonitorTimeCommand = plans.trySetMonitorTime(config.schema)
     this.countStatesCommand = plans.countStates(config.schema)
 
     this.functions = [
@@ -48,8 +46,6 @@ class Boss extends EventEmitter {
   }
 
   async onMonitor () {
-    let locker
-
     try {
       if (this.monitoring) {
         return
@@ -65,29 +61,24 @@ class Boss extends EventEmitter {
         throw new Error(this.config.__test__throw_monitor)
       }
 
-      locker = await this.db.lock({ key: 'monitor' })
+      if (this.stopped) {
+        return
+      }
 
-      const { secondsAgo } = await this.getMonitorTime()
+      const { rows } = await this.db.executeSql(this.trySetMonitorTimeCommand, [this.config.monitorStateIntervalSeconds])
 
-      if (secondsAgo > this.monitorStateIntervalSeconds && !this.stopped) {
+      if (rows.length === 1 && !this.stopped) {
         const states = await this.countStates()
-        this.setMonitorTime()
         this.emit(events.monitorStates, states)
       }
     } catch (err) {
       this.emit(events.error, err)
     } finally {
-      if (locker?.locked) {
-        await locker.unlock()
-      }
-
       this.monitoring = false
     }
   }
 
   async onSupervise () {
-    let locker
-
     try {
       if (this.maintaining) {
         return
@@ -108,21 +99,15 @@ class Boss extends EventEmitter {
         return
       }
 
-      locker = await this.db.lock({ key: 'maintenance' })
+      const { rows } = await this.db.executeSql(this.trySetMaintenanceTimeCommand, [this.config.maintenanceIntervalSeconds])
 
-      const { secondsAgo } = await this.getMaintenanceTime()
-
-      if (secondsAgo > this.maintenanceIntervalSeconds) {
+      if (rows.length === 1 && !this.stopped) {
         const result = await this.maintain()
         this.emit(events.maintenance, result)
       }
     } catch (err) {
       this.emit(events.error, err)
     } finally {
-      if (locker?.locked) {
-        await locker.unlock()
-      }
-
       this.maintaining = false
     }
   }
@@ -136,8 +121,6 @@ class Boss extends EventEmitter {
 
     const ended = Date.now()
 
-    await this.setMaintenanceTime()
-
     return { ms: ended - started }
   }
 
@@ -152,10 +135,11 @@ class Boss extends EventEmitter {
   }
 
   async countStates () {
-    const stateCountDefault = { ...plans.states }
+    const stateCountDefault = { ...plans.JOB_STATES }
 
-    Object.keys(stateCountDefault)
-      .forEach(key => { stateCountDefault[key] = 0 })
+    for (const key of Object.keys(stateCountDefault)) {
+      stateCountDefault[key] = 0
+    }
 
     const counts = await this.db.executeSql(this.countStatesCommand)
 
@@ -187,34 +171,6 @@ class Boss extends EventEmitter {
   async drop () {
     await this.db.executeSql(this.dropCommand)
   }
-
-  async setMaintenanceTime () {
-    await this.db.executeSql(this.setMaintenanceTimeCommand)
-  }
-
-  async getMaintenanceTime () {
-    const { rows } = await this.db.executeSql(this.getMaintenanceTimeCommand)
-
-    let { maintained_on: maintainedOn, seconds_ago: secondsAgo } = rows[0]
-
-    secondsAgo = secondsAgo !== null ? parseFloat(secondsAgo) : 999_999_999
-
-    return { maintainedOn, secondsAgo }
-  }
-
-  async setMonitorTime () {
-    await this.db.executeSql(this.setMonitorTimeCommand)
-  }
-
-  async getMonitorTime () {
-    const { rows } = await this.db.executeSql(this.getMonitorTimeCommand)
-
-    let { monitored_on: monitoredOn, seconds_ago: secondsAgo } = rows[0]
-
-    secondsAgo = secondsAgo !== null ? parseFloat(secondsAgo) : 999_999_999
-
-    return { monitoredOn, secondsAgo }
-  }
 }
 
 module.exports = Boss

package/src/contractor.js

@@ -21,23 +21,29 @@ class Contractor {
     this.config = config
     this.db = db
     this.migrations = this.config.migrations || migrationStore.getAll(this.config.schema)
+
+    // exported api to index
+    this.functions = [
+      this.schemaVersion,
+      this.isInstalled
+    ]
   }
 
-  async version () {
+  async schemaVersion () {
     const result = await this.db.executeSql(plans.getVersion(this.config.schema))
     return result.rows.length ? parseInt(result.rows[0].version) : null
   }
 
   async isInstalled () {
     const result = await this.db.executeSql(plans.versionTableExists(this.config.schema))
-    return result.rows.length ? result.rows[0].name : null
+    return !!result.rows[0].name
   }
 
   async start () {
     const installed = await this.isInstalled()
 
     if (installed) {
-      const version = await this.version()
+      const version = await this.schemaVersion()
 
       if (schemaVersion > version) {
         throw new Error('Migrations are not supported to v10')
@@ -55,7 +61,7 @@ class Contractor {
       throw new Error('pg-boss is not installed')
     }
 
-    const version = await this.version()
+    const version = await this.schemaVersion()
 
     if (schemaVersion !== version) {
       throw new Error('pg-boss database requires migrations')
@@ -80,15 +86,15 @@ class Contractor {
     }
   }
 
-  async next (version) {
-    const commands = migrationStore.next(this.config.schema, version, this.migrations)
-    await this.db.executeSql(commands)
-  }
+  // async next (version) {
+  //   const commands = migrationStore.next(this.config.schema, version, this.migrations)
+  //   await this.db.executeSql(commands)
+  // }
 
-  async rollback (version) {
-    const commands = migrationStore.rollback(this.config.schema, version, this.migrations)
-    await this.db.executeSql(commands)
-  }
+  // async rollback (version) {
+  //   const commands = migrationStore.rollback(this.config.schema, version, this.migrations)
+  //   await this.db.executeSql(commands)
+  // }
 }
 
 module.exports = Contractor

package/src/db.js

@@ -1,6 +1,5 @@
 const EventEmitter = require('events')
 const pg = require('pg')
-const { advisoryLock } = require('./plans')
 
 class Db extends EventEmitter {
   constructor (config) {
@@ -11,6 +10,10 @@ class Db extends EventEmitter {
     this.config = config
   }
 
+  events = {
+    error: 'error'
+  }
+
   async open () {
     this.pool = new pg.Pool(this.config)
     this.pool.on('error', error => this.emit('error', error))
@@ -26,48 +29,19 @@ class Db extends EventEmitter {
 
   async executeSql (text, values) {
     if (this.opened) {
-      if (this.config.debug === true) {
-        console.log(`${new Date().toISOString()}: DEBUG SQL`)
-        console.log(text)
+      // if (this.config.debug === true) {
+      //   console.log(`${new Date().toISOString()}: DEBUG SQL`)
+      //   console.log(text)
 
-        if (values) {
-          console.log(`${new Date().toISOString()}: DEBUG VALUES`)
-          console.log(values)
-        }
-      }
+      //   if (values) {
+      //     console.log(`${new Date().toISOString()}: DEBUG VALUES`)
+      //     console.log(values)
+      //   }
+      // }
 
       return await this.pool.query(text, values)
     }
   }
-
-  async lock ({ timeout = 30, key } = {}) {
-    // const lockedClient = new pg.Client(this.config)
-    // await lockedClient.connect()
-    const lockedClient = await this.pool.connect()
-
-    const query = `
-        BEGIN;
-        SET LOCAL lock_timeout = '${timeout}s';
-        SET LOCAL idle_in_transaction_session_timeout = '3600s';
-        ${advisoryLock(this.config.schema, key)};
-    `
-
-    await lockedClient.query(query)
-
-    const locker = {
-      locked: true,
-      unlock: async function () {
-        try {
-          await lockedClient.query('COMMIT')
-          await lockedClient.end()
-        } finally {
-          this.locked = false
-        }
-      }
-    }
-
-    return locker
-  }
 }
 
 module.exports = Db