pg-boss 8.4.2 → 9.0.1

package/README.md

@@ -1,6 +1,6 @@
 Queueing jobs in Node.js using PostgreSQL like a boss.
 
-[![PostgreSql Version](https://img.shields.io/badge/PostgreSQL-9.5+-blue.svg?maxAge=2592000)](http://www.postgresql.org)
+[![PostgreSql Version](https://img.shields.io/badge/PostgreSQL-11+-blue.svg?maxAge=2592000)](http://www.postgresql.org)
 [![npm version](https://badge.fury.io/js/pg-boss.svg)](https://badge.fury.io/js/pg-boss)
 [![Build Status](https://app.travis-ci.com/timgit/pg-boss.svg?branch=master)](https://app.travis-ci.com/github/timgit/pg-boss)
 [![Coverage Status](https://coveralls.io/repos/github/timgit/pg-boss/badge.svg?branch=master)](https://coveralls.io/github/timgit/pg-boss?branch=master)
@@ -50,8 +50,8 @@ This will likely cater the most to teams already familiar with the simplicity of
 * Automatic maintenance operations to manage table growth
 
 ## Requirements
-* Node 14 or higher
-* PostgreSQL 9.5 or higher
+* Node 16 or higher
+* PostgreSQL 11 or higher
 
 ## Installation
 

package/package.json

@@ -1,10 +1,10 @@
 {
   "name": "pg-boss",
-  "version": "8.4.2",
+  "version": "9.0.1",
   "description": "Queueing jobs in Node.js using PostgreSQL like a boss",
   "main": "./src/index.js",
   "engines": {
-    "node": ">=14"
+    "node": ">=16"
   },
   "dependencies": {
     "cron-parser": "^4.0.0",

package/src/attorney.js

@@ -129,6 +129,7 @@ function checkWorkArgs (name, args, defaults) {
   assert(!('teamSize' in options) || (Number.isInteger(options.teamSize) && options.teamSize >= 1), 'teamSize must be an integer > 0')
   assert(!('batchSize' in options) || (Number.isInteger(options.batchSize) && options.batchSize >= 1), 'batchSize must be an integer > 0')
   assert(!('includeMetadata' in options) || typeof options.includeMetadata === 'boolean', 'includeMetadata must be a boolean')
+  assert(!('enforceSingletonQueueActiveLimit' in options) || typeof options.enforceSingletonQueueActiveLimit === 'boolean', 'enforceSingletonQueueActiveLimit must be a boolean')
 
   return { options, callback }
 }
@@ -140,6 +141,7 @@ function checkFetchArgs (name, batchSize, options) {
 
   assert(!batchSize || (Number.isInteger(batchSize) && batchSize >= 1), 'batchSize must be an integer > 0')
   assert(!('includeMetadata' in options) || typeof options.includeMetadata === 'boolean', 'includeMetadata must be a boolean')
+  assert(!('enforceSingletonQueueActiveLimit' in options) || typeof options.enforceSingletonQueueActiveLimit === 'boolean', 'enforceSingletonQueueActiveLimit must be a boolean')
 
   return { name }
 }

package/src/boss.js

@@ -140,7 +140,7 @@ class Boss extends EventEmitter {
       this.emit('maintenance', { ms: ended - started })
 
       if (!this.stopped) {
-        await job.done() // pre-complete to bypass throttling
+        await this.manager.complete(job.id) // pre-complete to bypass throttling
         await this.maintenanceAsync({ startAfter: this.maintenanceIntervalSeconds })
       }
     } catch (err) {
@@ -159,7 +159,7 @@ class Boss extends EventEmitter {
       this.emit(events.monitorStates, states)
 
       if (!this.stopped && this.monitorStates) {
-        await job.done() // pre-complete to bypass throttling
+        await this.manager.complete(job.id) // pre-complete to bypass throttling
         await this.monitorStatesAsync({ startAfter: this.monitorIntervalSeconds })
       }
     } catch (err) {

package/src/db.js

@@ -28,6 +28,14 @@ class Db extends EventEmitter {
       return await this.pool.query(text, values)
     }
   }
+
+  static quotePostgresStr (str) {
+    const delimeter = '$sanitize$'
+    if (str.includes(delimeter)) {
+      throw new Error(`Attempted to quote string that contains reserved Postgres delimeter: ${str}`)
+    }
+    return `${delimeter}${str}${delimeter}`
+  }
 }
 
 module.exports = Db

package/src/manager.js

@@ -6,6 +6,7 @@ const debounce = require('lodash.debounce')
 const { serializeError: stringify } = require('serialize-error')
 const Attorney = require('./attorney')
 const Worker = require('./worker')
+const Db = require('./db')
 const pMap = require('p-map')
 
 const { QUEUES: BOSS_QUEUES } = require('./boss')
@@ -184,7 +185,8 @@ class Manager extends EventEmitter {
       teamSize = 1,
       teamConcurrency = 1,
       teamRefill: refill = false,
-      includeMetadata = false
+      includeMetadata = false,
+      enforceSingletonQueueActiveLimit = false
     } = options
 
     const id = uuid.v4()
@@ -208,7 +210,7 @@ class Manager extends EventEmitter {
       createTeamRefillPromise()
     }
 
-    const fetch = () => this.fetch(name, batchSize || (teamSize - queueSize), { includeMetadata })
+    const fetch = () => this.fetch(name, batchSize || (teamSize - queueSize), { includeMetadata, enforceSingletonQueueActiveLimit })
 
     const onFetch = async (jobs) => {
       if (this.config.__test__throw_worker) {
@@ -220,8 +222,8 @@ class Manager extends EventEmitter {
       if (batchSize) {
         const maxExpiration = jobs.reduce((acc, i) => Math.max(acc, i.expire_in_seconds), 0)
 
-        // Failing will fail all fetched jobs
         await resolveWithinSeconds(Promise.all([callback(jobs)]), maxExpiration)
+          .then(() => this.complete(jobs.map(job => job.id)))
           .catch(err => this.fail(jobs.map(job => job.id), err))
       } else {
         if (refill) {
@@ -474,27 +476,32 @@ class Manager extends EventEmitter {
   async fetch (name, batchSize, options = {}) {
     const values = Attorney.checkFetchArgs(name, batchSize, options)
     const db = options.db || this.db
-    const result = await db.executeSql(
-      this.nextJobCommand(options.includeMetadata || false),
-      [values.name, batchSize || 1]
-    )
+    const preparedStatement = this.nextJobCommand(options.includeMetadata || false, options.enforceSingletonQueueActiveLimit || false)
+    const statementValues = [values.name, batchSize || 1]
+
+    let result
+    if (options.enforceSingletonQueueActiveLimit && !options.db) {
+      // Prepare/format now and send multi-statement transaction
+      const fetchQuery = preparedStatement
+        .replace('$1', Db.quotePostgresStr(statementValues[0]))
+        .replace('$2', statementValues[1].toString())
+      // eslint-disable-next-line no-unused-vars
+      const [_begin, _setLocal, fetchResult, _commit] = await db.executeSql([
+        'BEGIN',
+        'SET LOCAL jit = OFF', // JIT can slow things down significantly
+        fetchQuery,
+        'COMMIT'
+      ].join(';\n'))
+      result = fetchResult
+    } else {
+      result = await db.executeSql(preparedStatement, statementValues)
+    }
 
     if (!result || result.rows.length === 0) {
       return null
     }
 
-    const jobs = result.rows.map(job => {
-      job.done = async (error, response) => {
-        if (error) {
-          await this.fail(job.id, error)
-        } else {
-          await this.complete(job.id, response)
-        }
-      }
-      return job
-    })
-
-    return jobs.length === 1 && !batchSize ? jobs[0] : jobs
+    return result.rows.length === 1 && !batchSize ? result.rows[0] : result.rows
   }
 
   async fetchCompleted (name, batchSize, options = {}) {

package/src/plans.js

@@ -351,13 +351,31 @@ function insertVersion (schema, version) {
 }
 
 function fetchNextJob (schema) {
-  return (includeMetadata) => `
+  return (includeMetadata, enforceSingletonQueueActiveLimit) => `
     WITH nextJob as (
       SELECT id
-      FROM ${schema}.job
+      FROM ${schema}.job j
       WHERE state < '${states.active}'
         AND name LIKE $1
         AND startAfter < now()
+        ${enforceSingletonQueueActiveLimit
+        ? `AND (
+          CASE
+            WHEN singletonKey IS NOT NULL
+              AND singletonKey LIKE '${SINGLETON_QUEUE_KEY_ESCAPED}%'
+              THEN NOT EXISTS (
+                SELECT 1
+                FROM ${schema}.job active_job
+                WHERE active_job.state = '${states.active}'
+                  AND active_job.name = j.name
+                  AND active_job.singletonKey = j.singletonKey
+                  LIMIT 1
+              )
+            ELSE
+              true
+          END
+        )`
+        : ''}
       ORDER BY priority desc, createdOn, id
       LIMIT $2
       FOR UPDATE SKIP LOCKED

package/types.d.ts

@@ -113,20 +113,22 @@ declare namespace PgBoss {
     teamRefill?: boolean;
     batchSize?: number;
     includeMetadata?: boolean;
+    enforceSingletonQueueActiveLimit?: boolean;
   }
 
   type WorkOptions = JobFetchOptions & JobPollingOptions
 
   type FetchOptions = {
     includeMetadata?: boolean;
+    enforceSingletonQueueActiveLimit?: boolean;
   } & ConnectionOptions;
 
-  interface WorkHandler<ReqData, ResData> {
-    (job: PgBoss.JobWithDoneCallback<ReqData, ResData>): Promise<ResData> | void;
+  interface WorkHandler<ReqData> {
+    (job: PgBoss.Job<ReqData>): Promise<void>;
   }
 
-  interface WorkWithMetadataHandler<ReqData, ResData> {
-    (job: PgBoss.JobWithMetadataDoneCallback<ReqData, ResData>): Promise<ResData> | void;
+  interface WorkWithMetadataHandler<ReqData> {
+    (job: PgBoss.JobWithMetadata<ReqData>): Promise<void>;
   }
 
   interface Request {
@@ -142,10 +144,6 @@ declare namespace PgBoss {
     options?: ScheduleOptions;
   }
 
-  interface JobDoneCallback<T> {
-    (err?: Error | null, data?: T): void;
-  }
-
   // source (for now): https://github.com/bendrucker/postgres-interval/blob/master/index.d.ts
   interface PostgresInterval {
     years?: number;
@@ -204,14 +202,6 @@ declare namespace PgBoss {
     onComplete?: boolean
   }
 
-  interface JobWithDoneCallback<ReqData, ResData> extends Job<ReqData> {
-    done: JobDoneCallback<ResData>;
-  }
-
-  interface JobWithMetadataDoneCallback<ReqData, ResData> extends JobWithMetadata<ReqData> {
-    done: JobDoneCallback<ResData>;
-  }
-
   interface MonitorState {
     all: number;
     created: number;
@@ -308,9 +298,9 @@ declare class PgBoss extends EventEmitter {
   insert(jobs: PgBoss.JobInsert[]): Promise<void>;
   insert(jobs: PgBoss.JobInsert[], options: PgBoss.InsertOptions): Promise<void>;
 
-  work<ReqData, ResData>(name: string, handler: PgBoss.WorkHandler<ReqData, ResData>): Promise<string>;
-  work<ReqData, ResData>(name: string, options: PgBoss.WorkOptions & { includeMetadata: true }, handler: PgBoss.WorkWithMetadataHandler<ReqData, ResData>): Promise<string>;
-  work<ReqData, ResData>(name: string, options: PgBoss.WorkOptions, handler: PgBoss.WorkHandler<ReqData, ResData>): Promise<string>;
+  work<ReqData>(name: string, handler: PgBoss.WorkHandler<ReqData>): Promise<string>;
+  work<ReqData>(name: string, options: PgBoss.WorkOptions & { includeMetadata: true }, handler: PgBoss.WorkWithMetadataHandler<ReqData>): Promise<string>;
+  work<ReqData>(name: string, options: PgBoss.WorkOptions, handler: PgBoss.WorkHandler<ReqData>): Promise<string>;
 
   onComplete(name: string, handler: Function): Promise<string>;
   onComplete(name: string, options: PgBoss.WorkOptions, handler: Function): Promise<string>;