pg-boss 10.0.0-beta1 → 10.0.0-beta11

package/README.md

@@ -1,4 +1,4 @@
-Queueing jobs in Node.js using PostgreSQL like a boss.
+Queueing jobs in Postgres from Node.js like a boss.
 
 [![npm version](https://badge.fury.io/js/pg-boss.svg)](https://badge.fury.io/js/pg-boss)
 [![Build](https://github.com/timgit/pg-boss/actions/workflows/ci.yml/badge.svg?branch=master)](https://github.com/timgit/pg-boss/actions/workflows/ci.yml)
@@ -9,24 +9,19 @@ async function readme() {
   const PgBoss = require('pg-boss');
   const boss = new PgBoss('postgres://user:pass@host/database');
 
-  boss.on('error', error => console.error(error));
+  boss.on('error', console.error)
 
-  await boss.start();
+  await boss.start()
 
-  const queue = 'some-queue';
+  const queue = 'readme-queue'
 
-  let jobId = await boss.send(queue, { param1: 'foo' })
+  const id = await boss.send(queue, { arg1: 'read me' })
 
-  console.log(`created job in queue ${queue}: ${jobId}`);
+  console.log(`created job ${id} in queue ${queue}`)
 
-  await boss.work(queue, someAsyncJobHandler);
-}
-
-async function someAsyncJobHandler(job) {
-  console.log(`job ${job.id} received with data:`);
-  console.log(JSON.stringify(job.data));
-
-  await doSomethingAsyncWithThis(job.data);
+  await boss.work(queue, async job => {
+    console.log(`received job ${job.id} with data ${JSON.stringify(job.data)}`)
+  })
 }
 ```
 
@@ -41,16 +36,14 @@ This will likely cater the most to teams already familiar with the simplicity of
 * Backpressure-compatible polling workers
 * Cron scheduling
 * Pub/sub API for fan-out queue relationships
-* Priority, deferral, retries (with exponential backoff), rate limiting, debouncing
-* Direct table access for bulk loads via COPY or INSERT
+* Priority queues, deferral, retries (with exponential backoff), rate limiting, debouncing
+* Table operations via SQL for bulk loads via COPY or INSERT
 * Multi-master compatible (for example, in a Kubernetes ReplicaSet)
 * Dead letter queues
-* Automatic creation and migration of storage tables
-* Automatic maintenance operations to manage table growth
 
 ## Requirements
-* Node 18 or higher
-* PostgreSQL 12 or higher
+* Node 20 or higher
+* PostgreSQL 13 or higher
 
 ## Installation
 
@@ -66,7 +59,6 @@ yarn add pg-boss
 * [Docs](docs/readme.md)
 
 ## Contributing
-
 To setup a development environment for this library:
 
 ```bash

package/package.json

@@ -1,10 +1,10 @@
 {
   "name": "pg-boss",
-  "version": "10.0.0-beta1",
-  "description": "Queueing jobs in Node.js using PostgreSQL like a boss",
+  "version": "10.0.0-beta11",
+  "description": "Queueing jobs in Postgres from Node.js like a boss",
   "main": "./src/index.js",
   "engines": {
-    "node": ">=18"
+    "node": ">=20"
   },
   "dependencies": {
     "cron-parser": "^4.0.0",
@@ -17,14 +17,15 @@
     "@types/node": "^20.3.3",
     "luxon": "^3.0.1",
     "mocha": "^10.0.0",
-    "nyc": "^15.1.0",
+    "nyc": "^17.0.0",
     "standard": "^17.0.0"
   },
   "scripts": {
     "test": "standard && mocha",
     "cover": "nyc npm test",
     "tsc": "tsc --noEmit types.d.ts",
-    "readme": "node ./test/readme.js"
+    "readme": "node ./test/readme.js",
+    "migrate": "node -e 'console.log(require(\"./src\").getMigrationPlans())'"
   },
   "mocha": {
     "timeout": 10000,

package/src/attorney.js

@@ -8,8 +8,8 @@ module.exports = {
   checkWorkArgs,
   checkFetchArgs,
   warnClockSkew,
-  queueNameHasPatternMatch,
-  assertPostgresObjectName
+  assertPostgresObjectName,
+  assertQueueName
 }
 
 const MAX_INTERVAL_HOURS = 24
@@ -30,8 +30,6 @@ const WARNINGS = {
 }
 
 function checkQueueArgs (name, options = {}) {
-  assertPostgresObjectName(name)
-
   assert(!('deadLetter' in options) || (typeof options.deadLetter === 'string'), 'deadLetter must be a string')
 
   applyRetryConfig(options)
@@ -124,7 +122,7 @@ function checkWorkArgs (name, args, defaults) {
 
   options = { ...options }
 
-  applyNewJobCheckInterval(options, defaults)
+  applyPollingInterval(options, defaults)
 
   assert(!('teamConcurrency' in options) ||
     (Number.isInteger(options.teamConcurrency) && options.teamConcurrency >= 1 && options.teamConcurrency <= 1000),
@@ -140,24 +138,12 @@ function checkWorkArgs (name, args, defaults) {
 function checkFetchArgs (name, batchSize, options) {
   assert(name, 'missing queue name')
 
-  if (queueNameHasPatternMatch(name)) {
-    name = sanitizeQueueNameForFetch(name)
-  }
-
   assert(!batchSize || (Number.isInteger(batchSize) && batchSize >= 1), 'batchSize must be an integer > 0')
   assert(!('includeMetadata' in options) || typeof options.includeMetadata === 'boolean', 'includeMetadata must be a boolean')
 
   return { name }
 }
 
-function sanitizeQueueNameForFetch (name) {
-  return name.replace(/[%_*]/g, match => match === '*' ? '%' : '\\' + match)
-}
-
-function queueNameHasPatternMatch (name) {
-  return name.includes('*')
-}
-
 function getConfig (value) {
   assert(value && (typeof value === 'object' || typeof value === 'string'),
     'configuration assert: string or config object is required to connect to postgres')
@@ -166,6 +152,10 @@ function getConfig (value) {
     ? { connectionString: value }
     : { ...value }
 
+  config.schedule = ('schedule' in config) ? config.schedule : true
+  config.supervise = ('supervise' in config) ? config.supervise : true
+  config.migrate = ('migrate' in config) ? config.migrate : true
+
   applySchemaConfig(config)
   applyMaintenanceConfig(config)
   applyArchiveConfig(config)
@@ -173,7 +163,7 @@ function getConfig (value) {
   applyDeleteConfig(config)
   applyMonitoringConfig(config)
 
-  applyNewJobCheckInterval(config)
+  applyPollingInterval(config)
   applyExpirationConfig(config)
   applyRetentionConfig(config)
 
@@ -191,8 +181,13 @@ function applySchemaConfig (config) {
 function assertPostgresObjectName (name) {
   assert(typeof name === 'string', 'Name must be a string')
   assert(name.length <= 50, 'Name cannot exceed 50 characters')
-  assert(!/\W/.test(name), 'Name can only contain alphanumeric characters and underscores')
-  assert(!/^d/.test(name), 'Name cannot start with a number')
+  assert(!/\W/.test(name), 'Name can only contain alphanumeric characters or underscores')
+  assert(!/^\d/.test(name), 'Name cannot start with a number')
+}
+
+function assertQueueName (name) {
+  assert(typeof name === 'string', 'Name must be a string')
+  assert(/[\w-]/.test(name), 'Name can only contain alphanumeric characters, underscores, or hyphens')
 }
 
 function applyArchiveConfig (config) {
@@ -283,18 +278,13 @@ function applyRetryConfig (config, defaults) {
   config.retryBackoffDefault = defaults?.retryBackoff
 }
 
-function applyNewJobCheckInterval (config, defaults) {
-  assert(!('newJobCheckInterval' in config) || config.newJobCheckInterval >= 500,
-    'configuration assert: newJobCheckInterval must be at least every 500ms')
-
-  assert(!('newJobCheckIntervalSeconds' in config) || config.newJobCheckIntervalSeconds >= 1,
-    'configuration assert: newJobCheckIntervalSeconds must be at least every second')
+function applyPollingInterval (config, defaults) {
+  assert(!('pollingIntervalSeconds' in config) || config.pollingIntervalSeconds >= 0.5,
+    'configuration assert: pollingIntervalSeconds must be at least every 500ms')
 
-  config.newJobCheckInterval = ('newJobCheckIntervalSeconds' in config)
-    ? config.newJobCheckIntervalSeconds * 1000
-    : ('newJobCheckInterval' in config)
-        ? config.newJobCheckInterval
-        : defaults?.newJobCheckInterval || 2000
+  config.pollingInterval = ('pollingIntervalSeconds' in config)
+    ? config.pollingIntervalSeconds * 1000
+    : defaults?.pollingInterval || 2000
 }
 
 function applyMaintenanceConfig (config) {
@@ -311,10 +301,6 @@ function applyMaintenanceConfig (config) {
         : 120
 
   assert(config.maintenanceIntervalSeconds / 60 / 60 < MAX_INTERVAL_HOURS, `configuration assert: maintenance interval cannot exceed ${MAX_INTERVAL_HOURS} hours`)
-
-  config.schedule = ('schedule' in config) ? config.schedule : true
-  config.supervise = ('supervise' in config) ? config.supervise : true
-  config.migrate = ('migrate' in config) ? config.migrate : true
 }
 
 function applyDeleteConfig (config) {
@@ -376,21 +362,21 @@ function applyMonitoringConfig (config) {
           ? config.clockMonitorIntervalSeconds
           : TEN_MINUTES_IN_SECONDS
 
-  assert(!('cronMonitorIntervalSeconds' in config) || (config.cronMonitorIntervalSeconds >= 1 && config.cronMonitorIntervalSeconds <= 60),
-    'configuration assert: cronMonitorIntervalSeconds must be between 1 and 60 seconds')
+  assert(!('cronMonitorIntervalSeconds' in config) || (config.cronMonitorIntervalSeconds >= 1 && config.cronMonitorIntervalSeconds <= 45),
+    'configuration assert: cronMonitorIntervalSeconds must be between 1 and 45 seconds')
 
   config.cronMonitorIntervalSeconds =
     ('cronMonitorIntervalSeconds' in config)
       ? config.cronMonitorIntervalSeconds
-      : 60
+      : 30
 
-  assert(!('cronWorkerIntervalSeconds' in config) || (config.cronWorkerIntervalSeconds >= 1 && config.cronWorkerIntervalSeconds <= 60),
-    'configuration assert: cronWorkerIntervalSeconds must be between 1 and 60 seconds')
+  assert(!('cronWorkerIntervalSeconds' in config) || (config.cronWorkerIntervalSeconds >= 1 && config.cronWorkerIntervalSeconds <= 45),
+    'configuration assert: cronWorkerIntervalSeconds must be between 1 and 45 seconds')
 
   config.cronWorkerIntervalSeconds =
     ('cronWorkerIntervalSeconds' in config)
       ? config.cronWorkerIntervalSeconds
-      : 4
+      : 5
 }
 
 function warnClockSkew (message) {

package/src/boss.js

@@ -24,10 +24,8 @@ class Boss extends EventEmitter {
     this.failJobsByTimeoutCommand = plans.locked(config.schema, plans.failJobsByTimeout(config.schema))
     this.archiveCommand = plans.locked(config.schema, plans.archive(config.schema, config.archiveInterval, config.archiveFailedInterval))
     this.dropCommand = plans.locked(config.schema, plans.drop(config.schema, config.deleteAfter))
-    this.getMaintenanceTimeCommand = plans.getMaintenanceTime(config.schema)
-    this.setMaintenanceTimeCommand = plans.setMaintenanceTime(config.schema)
-    this.getMonitorTimeCommand = plans.getMonitorTime(config.schema)
-    this.setMonitorTimeCommand = plans.setMonitorTime(config.schema)
+    this.trySetMaintenanceTimeCommand = plans.trySetMaintenanceTime(config.schema)
+    this.trySetMonitorTimeCommand = plans.trySetMonitorTime(config.schema)
     this.countStatesCommand = plans.countStates(config.schema)
 
     this.functions = [
@@ -48,8 +46,6 @@ class Boss extends EventEmitter {
   }
 
   async onMonitor () {
-    let locker
-
     try {
       if (this.monitoring) {
         return
@@ -65,29 +61,24 @@ class Boss extends EventEmitter {
         throw new Error(this.config.__test__throw_monitor)
       }
 
-      locker = await this.db.lock({ key: 'monitor' })
+      if (this.stopped) {
+        return
+      }
 
-      const { secondsAgo } = await this.getMonitorTime()
+      const { rows } = await this.db.executeSql(this.trySetMonitorTimeCommand, [this.config.monitorStateIntervalSeconds])
 
-      if (secondsAgo > this.monitorStateIntervalSeconds && !this.stopped) {
+      if (rows.length === 1 && !this.stopped) {
         const states = await this.countStates()
-        this.setMonitorTime()
         this.emit(events.monitorStates, states)
       }
     } catch (err) {
       this.emit(events.error, err)
     } finally {
-      if (locker?.locked) {
-        await locker.unlock()
-      }
-
       this.monitoring = false
     }
   }
 
   async onSupervise () {
-    let locker
-
     try {
       if (this.maintaining) {
         return
@@ -108,21 +99,15 @@ class Boss extends EventEmitter {
         return
       }
 
-      locker = await this.db.lock({ key: 'maintenance' })
-
-      const { secondsAgo } = await this.getMaintenanceTime()
+      const { rows } = await this.db.executeSql(this.trySetMaintenanceTimeCommand, [this.config.maintenanceIntervalSeconds])
 
-      if (secondsAgo > this.maintenanceIntervalSeconds) {
+      if (rows.length === 1 && !this.stopped) {
         const result = await this.maintain()
         this.emit(events.maintenance, result)
       }
     } catch (err) {
       this.emit(events.error, err)
     } finally {
-      if (locker?.locked) {
-        await locker.unlock()
-      }
-
       this.maintaining = false
     }
   }
@@ -136,8 +121,6 @@ class Boss extends EventEmitter {
 
     const ended = Date.now()
 
-    await this.setMaintenanceTime()
-
     return { ms: ended - started }
   }
 
@@ -152,10 +135,11 @@ class Boss extends EventEmitter {
   }
 
   async countStates () {
-    const stateCountDefault = { ...plans.states }
+    const stateCountDefault = { ...plans.JOB_STATES }
 
-    Object.keys(stateCountDefault)
-      .forEach(key => { stateCountDefault[key] = 0 })
+    for (const key of Object.keys(stateCountDefault)) {
+      stateCountDefault[key] = 0
+    }
 
     const counts = await this.db.executeSql(this.countStatesCommand)
 

package/src/contractor.js

@@ -21,26 +21,33 @@ class Contractor {
     this.config = config
     this.db = db
     this.migrations = this.config.migrations || migrationStore.getAll(this.config.schema)
+
+    // exported api to index
+    this.functions = [
+      this.schemaVersion,
+      this.isInstalled
+    ]
   }
 
-  async version () {
+  async schemaVersion () {
     const result = await this.db.executeSql(plans.getVersion(this.config.schema))
     return result.rows.length ? parseInt(result.rows[0].version) : null
   }
 
   async isInstalled () {
     const result = await this.db.executeSql(plans.versionTableExists(this.config.schema))
-    return result.rows.length ? result.rows[0].name : null
+    return !!result.rows[0].name
   }
 
   async start () {
     const installed = await this.isInstalled()
 
     if (installed) {
-      const version = await this.version()
+      const version = await this.schemaVersion()
 
       if (schemaVersion > version) {
-        await this.migrate(version)
+        throw new Error('Migrations are not supported to v10')
+        // await this.migrate(version)
       }
     } else {
       await this.create()
@@ -54,7 +61,7 @@ class Contractor {
       throw new Error('pg-boss is not installed')
     }
 
-    const version = await this.version()
+    const version = await this.schemaVersion()
 
     if (schemaVersion !== version) {
       throw new Error('pg-boss database requires migrations')

package/src/db.js

@@ -1,6 +1,5 @@
 const EventEmitter = require('events')
 const pg = require('pg')
-const { advisoryLock } = require('./plans')
 
 class Db extends EventEmitter {
   constructor (config) {
@@ -11,6 +10,10 @@ class Db extends EventEmitter {
     this.config = config
   }
 
+  events = {
+    error: 'error'
+  }
+
   async open () {
     this.pool = new pg.Pool(this.config)
     this.pool.on('error', error => this.emit('error', error))
@@ -26,55 +29,18 @@ class Db extends EventEmitter {
 
   async executeSql (text, values) {
     if (this.opened) {
-      // if (this.config.debug === true) {
-      //   console.log(`${new Date().toISOString()}: DEBUG SQL`)
-      //   console.log(text)
-
-      //   if (values) {
-      //     console.log(`${new Date().toISOString()}: DEBUG VALUES`)
-      //     console.log(values)
-      //   }
-      // }
-
-      return await this.pool.query(text, values)
-    }
-  }
+      if (this.config.debug === true) {
+        console.log(`${new Date().toISOString()}: DEBUG SQL`)
+        console.log(text)
 
-  async lock ({ timeout = 30, key } = {}) {
-    // const lockedClient = new pg.Client(this.config)
-    // await lockedClient.connect()
-    const lockedClient = await this.pool.connect()
-
-    const query = `
-        BEGIN;
-        SET LOCAL lock_timeout = '${timeout}s';
-        SET LOCAL idle_in_transaction_session_timeout = '3600s';
-        ${advisoryLock(this.config.schema, key)};
-    `
-
-    await lockedClient.query(query)
-
-    const locker = {
-      locked: true,
-      unlock: async function () {
-        try {
-          await lockedClient.query('COMMIT')
-          await lockedClient.end()
-        } finally {
-          this.locked = false
+        if (values) {
+          console.log(`${new Date().toISOString()}: DEBUG VALUES`)
+          console.log(values)
         }
       }
-    }
 
-    return locker
-  }
-
-  static quotePostgresStr (str) {
-    const delimeter = '$sanitize$'
-    if (str.includes(delimeter)) {
-      throw new Error(`Attempted to quote string that contains reserved Postgres delimeter: ${str}`)
+      return await this.pool.query(text, values)
     }
-    return `${delimeter}${str}${delimeter}`
   }
 }