pg-boss 9.0.2 → 10.0.0-beta1

package/README.md

@@ -1,8 +1,7 @@
 Queueing jobs in Node.js using PostgreSQL like a boss.
 
-[![PostgreSql Version](https://img.shields.io/badge/PostgreSQL-11+-blue.svg?maxAge=2592000)](http://www.postgresql.org)
 [![npm version](https://badge.fury.io/js/pg-boss.svg)](https://badge.fury.io/js/pg-boss)
-![Build](https://github.com/timgit/pg-boss/actions/workflows/ci.yml/badge.svg?branch=master)
+[![Build](https://github.com/timgit/pg-boss/actions/workflows/ci.yml/badge.svg?branch=master)](https://github.com/timgit/pg-boss/actions/workflows/ci.yml)
 [![Coverage Status](https://coveralls.io/repos/github/timgit/pg-boss/badge.svg?branch=master)](https://coveralls.io/github/timgit/pg-boss?branch=master)
 
 ```js
@@ -42,16 +41,16 @@ This will likely cater the most to teams already familiar with the simplicity of
 * Backpressure-compatible polling workers
 * Cron scheduling
 * Pub/sub API for fan-out queue relationships
-* Deferral, retries (with exponential backoff), rate limiting, debouncing
-* Completion jobs for orchestrations/sagas
+* Priority, deferral, retries (with exponential backoff), rate limiting, debouncing
 * Direct table access for bulk loads via COPY or INSERT
 * Multi-master compatible (for example, in a Kubernetes ReplicaSet)
+* Dead letter queues
 * Automatic creation and migration of storage tables
 * Automatic maintenance operations to manage table growth
 
 ## Requirements
-* Node 16 or higher
-* PostgreSQL 11 or higher
+* Node 18 or higher
+* PostgreSQL 12 or higher
 
 ## Installation
 
@@ -73,23 +72,11 @@ To setup a development environment for this library:
 ```bash
 git clone https://github.com/timgit/pg-boss.git
 npm install
-
-```
-
-To run the test suite you will need to pgboss access to an empty postgres database. You can set one up using the following commands on a local postgres instance:
-
-```sql
-CREATE DATABASE pgboss;
-CREATE user postgres WITH PASSWORD 'postgres';
-GRANT ALL PRIVILEGES ON DATABASE pgboss to postgres;
--- run the following command in the context of the pgboss database
-CREATE EXTENSION pgcrypto;
 ```
 
-If you use a different database name, username or password, or want to run the test suite against a database that is running on a remote machine then you will need to edit the `test/config.json` file with the appropriate connection values.
-
-You can then run the linter and test suite using
-
+To run the test suite, linter and code coverage:
 ```bash
-npm test
+npm run cover
 ```
+
+The test suite will try and create a new database named pgboss. The [config.json](test/config.json) file has the default credentials to connect to postgres. 

package/package.json

@@ -1,19 +1,17 @@
 {
   "name": "pg-boss",
-  "version": "9.0.2",
+  "version": "10.0.0-beta1",
   "description": "Queueing jobs in Node.js using PostgreSQL like a boss",
   "main": "./src/index.js",
   "engines": {
-    "node": ">=16"
+    "node": ">=18"
   },
   "dependencies": {
     "cron-parser": "^4.0.0",
-    "delay": "^5.0.0",
     "lodash.debounce": "^4.0.8",
     "p-map": "^4.0.0",
     "pg": "^8.5.1",
-    "serialize-error": "^8.1.0",
-    "uuid": "^9.0.0"
+    "serialize-error": "^8.1.0"
   },
   "devDependencies": {
     "@types/node": "^20.3.3",
@@ -25,9 +23,6 @@
   "scripts": {
     "test": "standard && mocha",
     "cover": "nyc npm test",
-    "export-schema": "node ./scripts/construct.js",
-    "export-migration": "node ./scripts/migrate.js",
-    "export-rollback": "node ./scripts/rollback.js",
     "tsc": "tsc --noEmit types.d.ts",
     "readme": "node ./test/readme.js"
   },

package/src/attorney.js

@@ -1,20 +1,20 @@
 const assert = require('assert')
-const { DEFAULT_SCHEMA, SINGLETON_QUEUE_KEY } = require('./plans')
+const { DEFAULT_SCHEMA } = require('./plans')
 
 module.exports = {
   getConfig,
   checkSendArgs,
-  checkInsertArgs,
+  checkQueueArgs,
   checkWorkArgs,
   checkFetchArgs,
-  warnClockSkew
+  warnClockSkew,
+  queueNameHasPatternMatch,
+  assertPostgresObjectName
 }
 
+const MAX_INTERVAL_HOURS = 24
+
 const WARNINGS = {
-  EXPIRE_IN_REMOVED: {
-    message: '\'expireIn\' option detected. This option has been removed. Use expireInSeconds, expireInMinutes or expireInHours.',
-    code: 'pg-boss-w01'
-  },
   CLOCK_SKEW: {
     message: 'Timekeeper detected clock skew between this instance and the database server. This will not affect scheduling operations, but this warning is shown any time the skew exceeds 60 seconds.',
     code: 'pg-boss-w02'
@@ -22,9 +22,25 @@ const WARNINGS = {
   CRON_DISABLED: {
     message: 'Archive interval is set less than 60s.  Cron processing is disabled.',
     code: 'pg-boss-w03'
+  },
+  ON_COMPLETE_REMOVED: {
+    message: '\'onComplete\' option detected. This option has been removed. Consider deadLetter if needed.',
+    code: 'pg-boss-w04'
   }
 }
 
+function checkQueueArgs (name, options = {}) {
+  assertPostgresObjectName(name)
+
+  assert(!('deadLetter' in options) || (typeof options.deadLetter === 'string'), 'deadLetter must be a string')
+
+  applyRetryConfig(options)
+  applyExpirationConfig(options)
+  applyRetentionConfig(options)
+
+  return options
+}
+
 function checkSendArgs (args, defaults) {
   let name, data, options
 
@@ -57,11 +73,11 @@ function checkSendArgs (args, defaults) {
   assert(!('priority' in options) || (Number.isInteger(options.priority)), 'priority must be an integer')
   options.priority = options.priority || 0
 
+  assert(!('deadLetter' in options) || (typeof options.deadLetter === 'string'), 'deadLetter must be a string')
+
   applyRetryConfig(options, defaults)
   applyExpirationConfig(options, defaults)
   applyRetentionConfig(options, defaults)
-  applyCompletionConfig(options, defaults)
-  applySingletonKeyConfig(options)
 
   const { startAfter, singletonSeconds, singletonMinutes, singletonHours } = options
 
@@ -83,23 +99,11 @@ function checkSendArgs (args, defaults) {
 
   assert(!singletonSeconds || singletonSeconds <= defaults.archiveSeconds, `throttling interval ${singletonSeconds}s cannot exceed archive interval ${defaults.archiveSeconds}s`)
 
-  return { name, data, options }
-}
-
-function checkInsertArgs (jobs) {
-  assert(Array.isArray(jobs), `jobs argument should be an array.  Received '${typeof jobs}'`)
-  return jobs.map(job => {
-    job = { ...job }
-    applySingletonKeyConfig(job)
-    return job
-  })
-}
-
-function applySingletonKeyConfig (options) {
-  if (options.singletonKey && options.useSingletonQueue && options.singletonKey !== SINGLETON_QUEUE_KEY) {
-    options.singletonKey = SINGLETON_QUEUE_KEY + options.singletonKey
+  if (options.onComplete) {
+    emitWarning(WARNINGS.ON_COMPLETE_REMOVED)
   }
-  delete options.useSingletonQueue
+
+  return { name, data, options }
 }
 
 function checkWorkArgs (name, args, defaults) {
@@ -129,7 +133,6 @@ function checkWorkArgs (name, args, defaults) {
   assert(!('teamSize' in options) || (Number.isInteger(options.teamSize) && options.teamSize >= 1), 'teamSize must be an integer > 0')
   assert(!('batchSize' in options) || (Number.isInteger(options.batchSize) && options.batchSize >= 1), 'batchSize must be an integer > 0')
   assert(!('includeMetadata' in options) || typeof options.includeMetadata === 'boolean', 'includeMetadata must be a boolean')
-  assert(!('enforceSingletonQueueActiveLimit' in options) || typeof options.enforceSingletonQueueActiveLimit === 'boolean', 'enforceSingletonQueueActiveLimit must be a boolean')
 
   return { options, callback }
 }
@@ -137,11 +140,12 @@ function checkWorkArgs (name, args, defaults) {
 function checkFetchArgs (name, batchSize, options) {
   assert(name, 'missing queue name')
 
-  name = sanitizeQueueNameForFetch(name)
+  if (queueNameHasPatternMatch(name)) {
+    name = sanitizeQueueNameForFetch(name)
+  }
 
   assert(!batchSize || (Number.isInteger(batchSize) && batchSize >= 1), 'batchSize must be an integer > 0')
   assert(!('includeMetadata' in options) || typeof options.includeMetadata === 'boolean', 'includeMetadata must be a boolean')
-  assert(!('enforceSingletonQueueActiveLimit' in options) || typeof options.enforceSingletonQueueActiveLimit === 'boolean', 'enforceSingletonQueueActiveLimit must be a boolean')
 
   return { name }
 }
@@ -150,6 +154,10 @@ function sanitizeQueueNameForFetch (name) {
   return name.replace(/[%_*]/g, match => match === '*' ? '%' : '\\' + match)
 }
 
+function queueNameHasPatternMatch (name) {
+  return name.includes('*')
+}
+
 function getConfig (value) {
   assert(value && (typeof value === 'object' || typeof value === 'string'),
     'configuration assert: string or config object is required to connect to postgres')
@@ -158,32 +166,35 @@ function getConfig (value) {
     ? { connectionString: value }
     : { ...value }
 
-  applyDatabaseConfig(config)
+  applySchemaConfig(config)
   applyMaintenanceConfig(config)
   applyArchiveConfig(config)
   applyArchiveFailedConfig(config)
   applyDeleteConfig(config)
   applyMonitoringConfig(config)
-  applyUuidConfig(config)
 
   applyNewJobCheckInterval(config)
   applyExpirationConfig(config)
   applyRetentionConfig(config)
-  applyCompletionConfig(config)
 
   return config
 }
 
-function applyDatabaseConfig (config) {
+function applySchemaConfig (config) {
   if (config.schema) {
-    assert(typeof config.schema === 'string', 'configuration assert: schema must be a string')
-    assert(config.schema.length <= 50, 'configuration assert: schema name cannot exceed 50 characters')
-    assert(!/\W/.test(config.schema), `configuration assert: ${config.schema} cannot be used as a schema. Only alphanumeric characters and underscores are allowed`)
+    assertPostgresObjectName(config.schema)
   }
 
   config.schema = config.schema || DEFAULT_SCHEMA
 }
 
+function assertPostgresObjectName (name) {
+  assert(typeof name === 'string', 'Name must be a string')
+  assert(name.length <= 50, 'Name cannot exceed 50 characters')
+  assert(!/\W/.test(name), 'Name can only contain alphanumeric characters and underscores')
+  assert(!/^d/.test(name), 'Name cannot start with a number')
+}
+
 function applyArchiveConfig (config) {
   const ARCHIVE_DEFAULT = 60 * 60 * 12
 
@@ -211,18 +222,7 @@ function applyArchiveFailedConfig (config) {
   }
 }
 
-function applyCompletionConfig (config, defaults) {
-  assert(!('onComplete' in config) || config.onComplete === true || config.onComplete === false,
-    'configuration assert: onComplete must be either true or false')
-
-  if (!('onComplete' in config)) {
-    config.onComplete = defaults
-      ? defaults.onComplete
-      : false
-  }
-}
-
-function applyRetentionConfig (config, defaults) {
+function applyRetentionConfig (config, defaults = {}) {
   assert(!('retentionSeconds' in config) || config.retentionSeconds >= 1,
     'configuration assert: retentionSeconds must be at least every second')
 
@@ -243,18 +243,13 @@ function applyRetentionConfig (config, defaults) {
             ? `${config.retentionMinutes} minutes`
             : ('retentionSeconds' in config)
                 ? `${config.retentionSeconds} seconds`
-                : defaults
-                  ? defaults.keepUntil
-                  : '14 days'
+                : null
 
   config.keepUntil = keepUntil
+  config.keepUntilDefault = defaults?.keepUntil
 }
 
-function applyExpirationConfig (config, defaults) {
-  if ('expireIn' in config) {
-    emitWarning(WARNINGS.EXPIRE_IN_REMOVED)
-  }
-
+function applyExpirationConfig (config, defaults = {}) {
   assert(!('expireInSeconds' in config) || config.expireInSeconds >= 1,
     'configuration assert: expireInSeconds must be at least every second')
 
@@ -265,16 +260,17 @@ function applyExpirationConfig (config, defaults) {
     'configuration assert: expireInHours must be at least every hour')
 
   const expireIn = ('expireInHours' in config)
-    ? `${config.expireInHours} hours`
+    ? config.expireInHours * 60 * 60
     : ('expireInMinutes' in config)
-        ? `${config.expireInMinutes} minutes`
+        ? config.expireInMinutes * 60
         : ('expireInSeconds' in config)
-            ? `${config.expireInSeconds} seconds`
-            : defaults
-              ? defaults.expireIn
-              : '15 minutes'
+            ? config.expireInSeconds
+            : null
+
+  assert(!expireIn || expireIn / 60 / 60 < MAX_INTERVAL_HOURS, `configuration assert: expiration cannot exceed ${MAX_INTERVAL_HOURS} hours`)
 
   config.expireIn = expireIn
+  config.expireInDefault = defaults?.expireIn
 }
 
 function applyRetryConfig (config, defaults) {
@@ -282,35 +278,23 @@ function applyRetryConfig (config, defaults) {
   assert(!('retryLimit' in config) || (Number.isInteger(config.retryLimit) && config.retryLimit >= 0), 'retryLimit must be an integer >= 0')
   assert(!('retryBackoff' in config) || (config.retryBackoff === true || config.retryBackoff === false), 'retryBackoff must be either true or false')
 
-  if (defaults) {
-    config.retryDelay = config.retryDelay || defaults.retryDelay
-    config.retryLimit = config.retryLimit || defaults.retryLimit
-    config.retryBackoff = config.retryBackoff || defaults.retryBackoff
-  }
-
-  config.retryDelay = config.retryDelay || 0
-  config.retryLimit = config.retryLimit || 0
-  config.retryBackoff = !!config.retryBackoff
-  config.retryDelay = (config.retryBackoff && !config.retryDelay) ? 1 : config.retryDelay
-  config.retryLimit = (config.retryDelay && !config.retryLimit) ? 1 : config.retryLimit
+  config.retryDelayDefault = defaults?.retryDelay
+  config.retryLimitDefault = defaults?.retryLimit
+  config.retryBackoffDefault = defaults?.retryBackoff
 }
 
 function applyNewJobCheckInterval (config, defaults) {
-  const second = 1000
-
-  assert(!('newJobCheckInterval' in config) || config.newJobCheckInterval >= 100,
-    'configuration assert: newJobCheckInterval must be at least every 100ms')
+  assert(!('newJobCheckInterval' in config) || config.newJobCheckInterval >= 500,
+    'configuration assert: newJobCheckInterval must be at least every 500ms')
 
   assert(!('newJobCheckIntervalSeconds' in config) || config.newJobCheckIntervalSeconds >= 1,
     'configuration assert: newJobCheckIntervalSeconds must be at least every second')
 
   config.newJobCheckInterval = ('newJobCheckIntervalSeconds' in config)
-    ? config.newJobCheckIntervalSeconds * second
+    ? config.newJobCheckIntervalSeconds * 1000
     : ('newJobCheckInterval' in config)
         ? config.newJobCheckInterval
-        : defaults
-          ? defaults.newJobCheckInterval
-          : second * 2
+        : defaults?.newJobCheckInterval || 2000
 }
 
 function applyMaintenanceConfig (config) {
@@ -325,6 +309,12 @@ function applyMaintenanceConfig (config) {
     : ('maintenanceIntervalSeconds' in config)
         ? config.maintenanceIntervalSeconds
         : 120
+
+  assert(config.maintenanceIntervalSeconds / 60 / 60 < MAX_INTERVAL_HOURS, `configuration assert: maintenance interval cannot exceed ${MAX_INTERVAL_HOURS} hours`)
+
+  config.schedule = ('schedule' in config) ? config.schedule : true
+  config.supervise = ('supervise' in config) ? config.supervise : true
+  config.migrate = ('migrate' in config) ? config.migrate : true
 }
 
 function applyDeleteConfig (config) {
@@ -367,6 +357,10 @@ function applyMonitoringConfig (config) {
           ? config.monitorStateIntervalSeconds
           : null
 
+  if (config.monitorStateIntervalSeconds) {
+    assert(config.monitorStateIntervalSeconds / 60 / 60 < MAX_INTERVAL_HOURS, `configuration assert: state monitoring interval cannot exceed ${MAX_INTERVAL_HOURS} hours`)
+  }
+
   const TEN_MINUTES_IN_SECONDS = 600
 
   assert(!('clockMonitorIntervalSeconds' in config) || (config.clockMonitorIntervalSeconds >= 1 && config.clockMonitorIntervalSeconds <= TEN_MINUTES_IN_SECONDS),
@@ -399,11 +393,6 @@ function applyMonitoringConfig (config) {
       : 4
 }
 
-function applyUuidConfig (config) {
-  assert(!('uuid' in config) || config.uuid === 'v1' || config.uuid === 'v4', 'configuration assert: uuid option only supports v1 or v4')
-  config.uuid = config.uuid || 'v4'
-}
-
 function warnClockSkew (message) {
   emitWarning(WARNINGS.CLOCK_SKEW, message, { force: true })
 }