pg-boss 11.0.7 → 11.1.0

package/README.md

@@ -82,10 +82,10 @@ To run the test suite, linter and code coverage:
 npm run cover
 ```
 
-The test suite will try and create a new database named pgboss. The [config.json](test/config.json) file has the default credentials to connect to postgres.
+The test suite will try and create a new database named pgboss. The [config.json](https://github.com/timgit/pg-boss/test/config.json) file has the default credentials to connect to postgres.
 
-The [Docker Compose](docker-compose.yaml) file can be used to start a local postgres instance for testing:
+The [Docker Compose](https://github.com/timgit/pg-boss/docker-compose.yaml) file can be used to start a local postgres instance for testing:
 
 ```bash
-docker-compose up -d
+docker compose up
 ```

package/docker-compose.yaml

@@ -1,10 +1,10 @@
 services:
   db:
-    image: postgres:17
+    image: postgres:18
     ports:
       - 5432:5432
     volumes:
-      - db_volume:/var/lib/postgresql/data
+      - db_volume:/var/lib/postgresql/data18
     environment:
       - POSTGRES_DB=pgboss
       - POSTGRES_NAME=pgboss

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pg-boss",
-  "version": "11.0.7",
+  "version": "11.1.0",
   "description": "Queueing jobs in Postgres from Node.js like a boss",
   "main": "./src/index.js",
   "type": "commonjs",
@@ -62,6 +62,6 @@
   "bugs": {
     "url": "https://github.com/timgit/pg-boss/issues"
   },
-  "homepage": "https://github.com/timgit/pg-boss#readme",
+  "homepage": "https://timgit.github.io/pg-boss",
   "types": "./types.d.ts"
 }

package/src/manager.js

@@ -585,7 +585,11 @@ class Manager extends EventEmitter {
     assert(Object.keys(options).length > 0, 'no properties found to update')
 
     if ('policy' in options) {
-      assert(options.policy in QUEUE_POLICIES, `${options.policy} is not a valid queue policy`)
+      throw new Error('queue policy cannot be changed after creation')
+    }
+
+    if ('partition' in options) {
+      throw new Error('queue partitioning cannot be changed after creation')
     }
 
     Attorney.validateQueueArgs(options)

package/src/migrationStore.js

@@ -64,5 +64,92 @@ function migrate (value, version, migrations) {
 
 function getAll (schema) {
   return [
-  ]
+    {
+      release: '11.1.0',
+      version: 26,
+      previous: 25,
+      install: [
+        `
+        CREATE OR REPLACE FUNCTION ${schema}.create_queue(queue_name text, options jsonb)
+        RETURNS VOID AS
+        $$
+        DECLARE
+          tablename varchar := CASE WHEN options->>'partition' = 'true'
+                                THEN 'j' || encode(sha224(queue_name::bytea), 'hex')
+                                ELSE 'job_common'
+                                END;
+          queue_created_on timestamptz;
+        BEGIN
+
+          WITH q as (
+            INSERT INTO ${schema}.queue (
+              name,
+              policy,
+              retry_limit,
+              retry_delay,
+              retry_backoff,
+              retry_delay_max,
+              expire_seconds,
+              retention_seconds,
+              deletion_seconds,
+              warning_queued,
+              dead_letter,
+              partition,
+              table_name
+            )
+            VALUES (
+              queue_name,
+              options->>'policy',
+              COALESCE((options->>'retryLimit')::int, 2),
+              COALESCE((options->>'retryDelay')::int, 0),
+              COALESCE((options->>'retryBackoff')::bool, false),
+              (options->>'retryDelayMax')::int,
+              COALESCE((options->>'expireInSeconds')::int, 900),
+              COALESCE((options->>'retentionSeconds')::int, 1209600),
+              COALESCE((options->>'deleteAfterSeconds')::int, 604800),
+              COALESCE((options->>'warningQueueSize')::int, 0),
+              options->>'deadLetter',
+              COALESCE((options->>'partition')::bool, false),
+              tablename
+            )
+            ON CONFLICT DO NOTHING
+            RETURNING created_on
+          )
+          SELECT created_on into queue_created_on from q;
+
+          IF queue_created_on IS NULL OR options->>'partition' IS DISTINCT FROM 'true' THEN
+            RETURN;
+          END IF;
+
+          EXECUTE format('CREATE TABLE ${schema}.%I (LIKE ${schema}.job INCLUDING DEFAULTS)', tablename);
+          
+          EXECUTE format('ALTER TABLE ${schema}.%1$I ADD PRIMARY KEY (name, id)', tablename);
+          EXECUTE format('ALTER TABLE ${schema}.%1$I ADD CONSTRAINT q_fkey FOREIGN KEY (name) REFERENCES ${schema}.queue (name) ON DELETE RESTRICT DEFERRABLE INITIALLY DEFERRED', tablename);
+          EXECUTE format('ALTER TABLE ${schema}.%1$I ADD CONSTRAINT dlq_fkey FOREIGN KEY (dead_letter) REFERENCES ${schema}.queue (name) ON DELETE RESTRICT DEFERRABLE INITIALLY DEFERRED', tablename);
+
+          EXECUTE format('CREATE INDEX %1$s_i5 ON ${schema}.%1$I (name, start_after) INCLUDE (priority, created_on, id) WHERE state < ''active''', tablename);
+          EXECUTE format('CREATE UNIQUE INDEX %1$s_i4 ON ${schema}.%1$I (name, singleton_on, COALESCE(singleton_key, '''')) WHERE state <> ''cancelled'' AND singleton_on IS NOT NULL', tablename);
+          
+          IF options->>'policy' = 'short' THEN
+            EXECUTE format('CREATE UNIQUE INDEX %1$s_i1 ON ${schema}.%1$I (name, COALESCE(singleton_key, '''')) WHERE state = ''created'' AND policy = ''short''', tablename);
+          ELSIF options->>'policy' = 'singleton' THEN
+            EXECUTE format('CREATE UNIQUE INDEX %1$s_i2 ON ${schema}.%1$I (name, COALESCE(singleton_key, '''')) WHERE state = ''active'' AND policy = ''singleton''', tablename);
+          ELSIF options->>'policy' = 'stately' THEN
+            EXECUTE format('CREATE UNIQUE INDEX %1$s_i3 ON ${schema}.%1$I (name, state, COALESCE(singleton_key, '''')) WHERE state <= ''active'' AND policy = ''stately''', tablename);
+          ELSIF options->>'policy' = 'exclusive' THEN
+            EXECUTE format('CREATE UNIQUE INDEX %1$s_i6 ON ${schema}.%1$I (name, COALESCE(singleton_key, '''')) WHERE state <= ''active'' AND policy = ''exclusive''', tablename);
+          END IF;
+
+          EXECUTE format('ALTER TABLE ${schema}.%I ADD CONSTRAINT cjc CHECK (name=%L)', tablename, queue_name);
+          EXECUTE format('ALTER TABLE ${schema}.job ATTACH PARTITION ${schema}.%I FOR VALUES IN (%L)', tablename, queue_name);
+        END;
+        $$
+        LANGUAGE plpgsql;
+        `,
+        `CREATE UNIQUE INDEX job_i6 ON ${schema}.job_common (name, COALESCE(singleton_key, '')) WHERE state <= 'active' AND policy = 'exclusive'`
+      ],
+      uninstall: [
+        `DROP INDEX ${schema}.job_i6`
+      ]
+    },]
 }

package/src/plans.js

@@ -1,6 +1,9 @@
 const DEFAULT_SCHEMA = 'pgboss'
 const MIGRATE_RACE_MESSAGE = 'division by zero'
 const CREATE_RACE_MESSAGE = 'already exists'
+const FIFTEEN_MINUTES = 60 * 15
+const FORTEEN_DAYS = 60 * 60 * 24 * 14
+const SEVEN_DAYS = 60 * 60 * 24 * 7
 
 const JOB_STATES = Object.freeze({
   created: 'created',
@@ -15,9 +18,23 @@ const QUEUE_POLICIES = Object.freeze({
   standard: 'standard',
   short: 'short',
   singleton: 'singleton',
-  stately: 'stately'
+  stately: 'stately',
+  exclusive: 'exclusive'
 })
 
+const QUEUE_DEFAULTS = {
+  expire_seconds: FIFTEEN_MINUTES,
+  retention_seconds: FORTEEN_DAYS,
+  deletion_seconds: SEVEN_DAYS,
+  retry_limit: 2,
+  retry_delay: 0,
+  warning_queued: 0,
+  retry_backoff: false,
+  partition: false
+}
+
+const COMMON_JOB_TABLE = 'job_common'
+
 module.exports = {
   create,
   insertVersion,
@@ -62,24 +79,7 @@ module.exports = {
   JOB_STATES,
   MIGRATE_RACE_MESSAGE,
   CREATE_RACE_MESSAGE,
-  DEFAULT_SCHEMA
-}
-
-const COMMON_JOB_TABLE = 'job_common'
-
-const FIFTEEN_MINUTES = 60 * 15
-const FORTEEN_DAYS = 60 * 60 * 24 * 14
-const SEVEN_DAYS = 60 * 60 * 24 * 7
-
-const QUEUE_DEFAULTS = {
-  expire_seconds: FIFTEEN_MINUTES,
-  retention_seconds: FORTEEN_DAYS,
-  deletion_seconds: SEVEN_DAYS,
-  retry_limit: 2,
-  retry_delay: 0,
-  warning_queued: 0,
-  retry_backoff: false,
-  partition: false
+  DEFAULT_SCHEMA,
 }
 
 function create (schema, version) {
@@ -253,6 +253,7 @@ function createTableJobCommon (schema, table) {
     ${format(createIndexJobPolicyShort(schema))}
     ${format(createIndexJobPolicySingleton(schema))}
     ${format(createIndexJobPolicyStately(schema))}
+    ${format(createIndexJobPolicyExclusive(schema))}
     ${format(createIndexJobThrottle(schema))}
     ${format(createIndexJobFetch(schema))}
 
@@ -318,11 +319,19 @@ function createQueueFunction (schema) {
       EXECUTE format('${formatPartitionCommand(createPrimaryKeyJob(schema))}', tablename);
       EXECUTE format('${formatPartitionCommand(createQueueForeignKeyJob(schema))}', tablename);
       EXECUTE format('${formatPartitionCommand(createQueueForeignKeyJobDeadLetter(schema))}', tablename);
-      EXECUTE format('${formatPartitionCommand(createIndexJobPolicyShort(schema))}', tablename);
-      EXECUTE format('${formatPartitionCommand(createIndexJobPolicySingleton(schema))}', tablename);
-      EXECUTE format('${formatPartitionCommand(createIndexJobPolicyStately(schema))}', tablename);
-      EXECUTE format('${formatPartitionCommand(createIndexJobThrottle(schema))}', tablename);
+
       EXECUTE format('${formatPartitionCommand(createIndexJobFetch(schema))}', tablename);
+      EXECUTE format('${formatPartitionCommand(createIndexJobThrottle(schema))}', tablename);
+      
+      IF options->>'policy' = 'short' THEN
+        EXECUTE format('${formatPartitionCommand(createIndexJobPolicyShort(schema))}', tablename);
+      ELSIF options->>'policy' = 'singleton' THEN
+        EXECUTE format('${formatPartitionCommand(createIndexJobPolicySingleton(schema))}', tablename);
+      ELSIF options->>'policy' = 'stately' THEN
+        EXECUTE format('${formatPartitionCommand(createIndexJobPolicyStately(schema))}', tablename);
+      ELSIF options->>'policy' = 'exclusive' THEN
+        EXECUTE format('${formatPartitionCommand(createIndexJobPolicyExclusive(schema))}', tablename);
+      END IF;
 
       EXECUTE format('ALTER TABLE ${schema}.%I ADD CONSTRAINT cjc CHECK (name=%L)', tablename, queue_name);
       EXECUTE format('ALTER TABLE ${schema}.job ATTACH PARTITION ${schema}.%I FOR VALUES IN (%L)', tablename, queue_name);
@@ -408,6 +417,10 @@ function createIndexJobFetch (schema) {
   return `CREATE INDEX job_i5 ON ${schema}.job (name, start_after) INCLUDE (priority, created_on, id) WHERE state < '${JOB_STATES.active}'`
 }
 
+function createIndexJobPolicyExclusive (schema) {
+  return `CREATE UNIQUE INDEX job_i6 ON ${schema}.job (name, COALESCE(singleton_key, '')) WHERE state <= '${JOB_STATES.active}' AND policy = '${QUEUE_POLICIES.exclusive}'`
+}
+
 function trySetQueueMonitorTime (schema, queues, seconds) {
   return trySetQueueTimestamp(schema, queues, 'monitor_on', seconds)
 }
@@ -443,7 +456,6 @@ function updateQueue (schema, { deadLetter } = {}) {
   return `
     WITH options as (SELECT $2::jsonb as data)
     UPDATE ${schema}.queue SET
-      policy = COALESCE(o.data->>'policy', policy),
       retry_limit = COALESCE((o.data->>'retryLimit')::int, retry_limit),
       retry_delay = COALESCE((o.data->>'retryDelay')::int, retry_delay),
       retry_backoff = COALESCE((o.data->>'retryBackoff')::bool, retry_backoff),
@@ -943,7 +955,7 @@ function getQueueStats (schema, table, queues) {
         (count(*) FILTER (WHERE state < '${JOB_STATES.active}'))::int as "queuedCount",
         (count(*) FILTER (WHERE state = '${JOB_STATES.active}'))::int as "activeCount",
         count(*)::int as "totalCount",
-        array_agg(singleton_key) FILTER (WHERE policy IN ('${QUEUE_POLICIES.singleton}','${QUEUE_POLICIES.stately}') AND state IN ('${JOB_STATES.retry}','${JOB_STATES.active}')) as "singletonsActive"
+        array_agg(singleton_key) FILTER (WHERE policy IN ('${QUEUE_POLICIES.singleton}','${QUEUE_POLICIES.stately}') AND state = '${JOB_STATES.active}') as "singletonsActive"
       FROM ${schema}.${table}
       WHERE name IN (${getQueueInClause(queues)})
       GROUP BY 1

package/types.d.ts

@@ -83,7 +83,7 @@ declare namespace PgBoss {
 
   type SendOptions = JobOptions & QueueOptions & ConnectionOptions
 
-  type QueuePolicy = 'standard' | 'short' | 'singleton' | 'stately'
+  type QueuePolicy = 'standard' | 'short' | 'singleton' | 'stately' | 'exclusive'
 
   type Queue = {
     name: string;
@@ -302,8 +302,8 @@ declare class PgBoss extends EventEmitter {
 
   createQueue (name: string, options?: Omit<PgBoss.Queue, 'name'>): Promise<void>
   createQueue (options: PgBoss.Queue): Promise<void>
-  updateQueue (name: string, options?: Omit<PgBoss.Queue, 'name'>): Promise<void>
-  updateQueue (options: PgBoss.Queue): Promise<void>
+  updateQueue (name: string, options?: Omit<PgBoss.Queue, 'name', 'partition', 'policy'>): Promise<void>
+  updateQueue (options: Omit<PgBoss.Queue, 'partition', 'policy'>): Promise<void>
   deleteQueue (name: string): Promise<void>
   getQueues (): Promise<PgBoss.QueueResult[]>
   getQueue (name: string): Promise<PgBoss.QueueResult | null>

package/version.json

@@ -1,3 +1,3 @@
 {
-  "schema": 25
+  "schema": 26
 }