pg-boss 10.0.0-beta15 → 10.0.0-beta17

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pg-boss",
-  "version": "10.0.0-beta15",
+  "version": "10.0.0-beta17",
   "description": "Queueing jobs in Postgres from Node.js like a boss",
   "main": "./src/index.js",
   "engines": {

package/src/manager.js

@@ -56,15 +56,13 @@ class Manager extends EventEmitter {
     this.subscribeCommand = plans.subscribe(config.schema)
     this.unsubscribeCommand = plans.unsubscribe(config.schema)
     this.getQueuesCommand = plans.getQueues(config.schema)
-    this.getQueuesForEventCommand = plans.getQueuesForEvent(config.schema)
     this.getQueueByNameCommand = plans.getQueueByName(config.schema)
-    this.deleteQueueRecordsCommand = plans.deleteQueueRecords(config.schema)
-    this.insertQueueCommand = plans.insertQueue(config.schema)
+    this.getQueuesForEventCommand = plans.getQueuesForEvent(config.schema)
+    this.createQueueCommand = plans.createQueue(config.schema)
     this.updateQueueCommand = plans.updateQueue(config.schema)
-    this.createPartitionCommand = plans.createPartition(config.schema)
-    this.dropPartitionCommand = plans.dropPartition(config.schema)
-    this.clearStorageCommand = plans.clearStorage(config.schema)
     this.purgeQueueCommand = plans.purgeQueue(config.schema)
+    this.deleteQueueCommand = plans.deleteQueue(config.schema)
+    this.clearStorageCommand = plans.clearStorage(config.schema)
 
     // exported api to index
     this.functions = [
@@ -572,10 +570,7 @@ class Manager extends EventEmitter {
       Attorney.assertQueueName(deadLetter)
     }
 
-    await this.db.executeSql(this.createPartitionCommand, [name])
-
-    const params = [
-      name,
+    const data = {
       policy,
       retryLimit,
       retryDelay,
@@ -583,9 +578,9 @@ class Manager extends EventEmitter {
       expireInSeconds,
       retentionMinutes,
       deadLetter
-    ]
+    }
 
-    await this.db.executeSql(this.insertQueueCommand, params)
+    await this.db.executeSql(this.createQueueCommand, [name, data])
   }
 
   async getQueues () {
@@ -626,32 +621,9 @@ class Manager extends EventEmitter {
   async getQueue (name) {
     Attorney.assertQueueName(name)
 
-    const result = await this.db.executeSql(this.getQueueByNameCommand, [name])
-
-    if (result.rows.length === 0) {
-      return null
-    }
-
-    const {
-      policy,
-      retry_limit: retryLimit,
-      retry_delay: retryDelay,
-      retry_backoff: retryBackoff,
-      expire_seconds: expireInSeconds,
-      retention_minutes: retentionMinutes,
-      dead_letter: deadLetter
-    } = result.rows[0]
+    const { rows } = await this.db.executeSql(this.getQueueByNameCommand, [name])
 
-    return {
-      name,
-      policy,
-      retryLimit,
-      retryDelay,
-      retryBackoff,
-      expireInSeconds,
-      retentionMinutes,
-      deadLetter
-    }
+    return rows[0] || null
   }
 
   async deleteQueue (name) {
@@ -659,9 +631,8 @@ class Manager extends EventEmitter {
 
     const { rows } = await this.db.executeSql(this.getQueueByNameCommand, [name])
 
-    if (rows.length) {
-      await this.db.executeSql(this.dropPartitionCommand, [name])
-      await this.db.executeSql(this.deleteQueueRecordsCommand, [name])
+    if (rows.length === 1) {
+      await this.db.executeSql(this.deleteQueueCommand, [name])
     }
   }
 

package/src/plans.js

@@ -42,11 +42,9 @@ module.exports = {
   archive,
   drop,
   countStates,
-  insertQueue,
   updateQueue,
-  createPartition,
-  dropPartition,
-  deleteQueueRecords,
+  createQueue,
+  deleteQueue,
   getQueues,
   getQueueByName,
   getQueueSize,
@@ -85,9 +83,8 @@ function create (schema, version) {
     createColumnArchiveArchivedOn(schema),
     createIndexArchiveArchivedOn(schema),
 
-    getPartitionFunction(schema),
-    createPartitionFunction(schema),
-    dropPartitionFunction(schema),
+    createQueueFunction(schema),
+    deleteQueueFunction(schema),
 
     insertVersion(schema, version)
   ]
@@ -137,8 +134,10 @@ function createTableQueue (schema) {
       retry_backoff bool,
       expire_seconds int,
       retention_minutes int,
-      dead_letter text,
+      dead_letter text REFERENCES ${schema}.queue (name),
+      partition_name text,
       created_on timestamp with time zone not null default now(),
+      updated_on timestamp with time zone not null default now(),
       PRIMARY KEY (name) 
     )
   `
@@ -199,7 +198,10 @@ function createTableJob (schema) {
 }
 
 const baseJobColumns = 'id, name, data, EXTRACT(epoch FROM expire_in) as "expireInSeconds"'
-const allJobColumns = `${baseJobColumns}, policy, state, priority,
+const allJobColumns = `${baseJobColumns},
+  policy,
+  state,
+  priority,
   retry_limit as "retryLimit",
   retry_count as "retryCount",
   retry_delay as "retryDelay",
@@ -216,29 +218,38 @@ const allJobColumns = `${baseJobColumns}, policy, state, priority,
   output
 `
 
-function createPartition (schema) {
-  return `SELECT ${schema}.create_partition($1)`
-}
-
-function getPartitionFunction (schema) {
+function createQueueFunction (schema) {
   return `
-    CREATE FUNCTION ${schema}.get_partition(queue_name text, out name text) AS
-    $$
-    SELECT 'j' || left(regexp_replace(queue_name, '\\W', '', 'g'),10) || left(encode(sha224(queue_name::bytea), 'hex'),10);
-    $$
-    LANGUAGE SQL
-    IMMUTABLE
-  `
-}
-
-function createPartitionFunction (schema) {
-  return `
-    CREATE FUNCTION ${schema}.create_partition(queue_name text)
+    CREATE FUNCTION ${schema}.create_queue(queue_name text, options json)
     RETURNS VOID AS
     $$
     DECLARE
-      table_name varchar := ${schema}.get_partition(queue_name);
+      table_name varchar := 'j' || encode(sha224(queue_name::bytea), 'hex');
     BEGIN
+
+      INSERT INTO ${schema}.queue (
+        name,
+        policy,
+        retry_limit,
+        retry_delay,
+        retry_backoff,
+        expire_seconds,
+        retention_minutes,
+        dead_letter,
+        partition_name
+      )
+      VALUES (
+        queue_name,
+        options->>'policy',
+        (options->>'retryLimit')::int,
+        (options->>'retryDelay')::int,
+        (options->>'retryBackoff')::bool,
+        (options->>'expireInSeconds')::int,
+        (options->>'retentionMinutes')::int,
+        options->>'deadLetter',
+        table_name
+      );
+
       EXECUTE format('CREATE TABLE ${schema}.%I (LIKE ${schema}.job INCLUDING DEFAULTS)', table_name);
       
       EXECUTE format('${formatPartitionCommand(createPrimaryKeyJob(schema))}', table_name);
@@ -259,24 +270,37 @@ function createPartitionFunction (schema) {
 }
 
 function formatPartitionCommand (command) {
-  return command.replace('.job', '.%1$I').replace('job_idx', '%1$s_idx').replaceAll('\'', '\'\'')
+  return command.replace('.job', '.%1$I').replace('job_i', '%1$s_i').replaceAll('\'', '\'\'')
 }
 
-function dropPartitionFunction (schema) {
+function deleteQueueFunction (schema) {
   return `
-    CREATE FUNCTION ${schema}.drop_partition(queue_name text)
+    CREATE FUNCTION ${schema}.delete_queue(queue_name text)
     RETURNS VOID AS
     $$
+    DECLARE
+      table_name varchar;
     BEGIN  
-      EXECUTE format('DROP TABLE IF EXISTS ${schema}.%I', ${schema}.get_partition(queue_name));
+      WITH deleted as (
+        DELETE FROM ${schema}.queue
+        WHERE name = queue_name
+        RETURNING partition_name
+      )
+      SELECT partition_name from deleted INTO table_name;
+
+      EXECUTE format('DROP TABLE IF EXISTS ${schema}.%I', table_name);
     END;
     $$
     LANGUAGE plpgsql;
   `
 }
 
-function dropPartition (schema) {
-  return `SELECT ${schema}.drop_partition($1)`
+function createQueue (schema) {
+  return `SELECT ${schema}.create_queue($1, $2)`
+}
+
+function deleteQueue (schema) {
+  return `SELECT ${schema}.delete_queue($1)`
 }
 
 function createPrimaryKeyJob (schema) {
@@ -284,11 +308,11 @@ function createPrimaryKeyJob (schema) {
 }
 
 function createQueueForeignKeyJob (schema) {
-  return `ALTER TABLE ${schema}.job ADD CONSTRAINT q_fkey FOREIGN KEY (name) REFERENCES ${schema}.queue (name) ON DELETE RESTRICT`
+  return `ALTER TABLE ${schema}.job ADD CONSTRAINT q_fkey FOREIGN KEY (name) REFERENCES ${schema}.queue (name) ON DELETE RESTRICT DEFERRABLE INITIALLY DEFERRED`
 }
 
 function createQueueForeignKeyJobDeadLetter (schema) {
-  return `ALTER TABLE ${schema}.job ADD CONSTRAINT dlq_fkey FOREIGN KEY (dead_letter) REFERENCES ${schema}.queue (name) ON DELETE RESTRICT`
+  return `ALTER TABLE ${schema}.job ADD CONSTRAINT dlq_fkey FOREIGN KEY (dead_letter) REFERENCES ${schema}.queue (name) ON DELETE RESTRICT DEFERRABLE INITIALLY DEFERRED`
 }
 
 function createPrimaryKeyArchive (schema) {
@@ -296,23 +320,23 @@ function createPrimaryKeyArchive (schema) {
 }
 
 function createIndexJobPolicyShort (schema) {
-  return `CREATE UNIQUE INDEX job_idx_psh ON ${schema}.job (name, COALESCE(singleton_key, '')) WHERE state = '${JOB_STATES.created}' AND policy = '${QUEUE_POLICIES.short}';`
+  return `CREATE UNIQUE INDEX job_i1 ON ${schema}.job (name, COALESCE(singleton_key, '')) WHERE state = '${JOB_STATES.created}' AND policy = '${QUEUE_POLICIES.short}';`
 }
 
 function createIndexJobPolicySingleton (schema) {
-  return `CREATE UNIQUE INDEX job_idx_psi ON ${schema}.job (name, COALESCE(singleton_key, '')) WHERE state = '${JOB_STATES.active}' AND policy = '${QUEUE_POLICIES.singleton}'`
+  return `CREATE UNIQUE INDEX job_i2 ON ${schema}.job (name, COALESCE(singleton_key, '')) WHERE state = '${JOB_STATES.active}' AND policy = '${QUEUE_POLICIES.singleton}'`
 }
 
 function createIndexJobPolicyStately (schema) {
-  return `CREATE UNIQUE INDEX job_idx_pst ON ${schema}.job (name, state, COALESCE(singleton_key, '')) WHERE state <= '${JOB_STATES.active}' AND policy = '${QUEUE_POLICIES.stately}'`
+  return `CREATE UNIQUE INDEX job_i3 ON ${schema}.job (name, state, COALESCE(singleton_key, '')) WHERE state <= '${JOB_STATES.active}' AND policy = '${QUEUE_POLICIES.stately}'`
 }
 
 function createIndexJobThrottle (schema) {
-  return `CREATE UNIQUE INDEX job_idx_to ON ${schema}.job (name, singleton_on, COALESCE(singleton_key, '')) WHERE state <> '${JOB_STATES.cancelled}' AND singleton_on IS NOT NULL`
+  return `CREATE UNIQUE INDEX job_i4 ON ${schema}.job (name, singleton_on, COALESCE(singleton_key, '')) WHERE state <> '${JOB_STATES.cancelled}' AND singleton_on IS NOT NULL`
 }
 
 function createIndexJobFetch (schema) {
-  return `CREATE INDEX job_idx_f ON ${schema}.job (name, start_after) INCLUDE (priority, created_on, id) WHERE state < '${JOB_STATES.active}'`
+  return `CREATE INDEX job_i5 ON ${schema}.job (name, start_after) INCLUDE (priority, created_on, id) WHERE state < '${JOB_STATES.active}'`
 }
 
 function createTableArchive (schema) {
@@ -324,7 +348,7 @@ function createColumnArchiveArchivedOn (schema) {
 }
 
 function createIndexArchiveArchivedOn (schema) {
-  return `CREATE INDEX archive_idx_ao ON ${schema}.archive(archived_on)`
+  return `CREATE INDEX archive_i1 ON ${schema}.archive(archived_on)`
 }
 
 function trySetMaintenanceTime (schema) {
@@ -347,13 +371,6 @@ function trySetTimestamp (schema, column) {
   `
 }
 
-function insertQueue (schema) {
-  return `
-    INSERT INTO ${schema}.queue (name, policy, retry_limit, retry_delay, retry_backoff, expire_seconds, retention_minutes, dead_letter)
-    VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
-  `
-}
-
 function updateQueue (schema) {
   return `
     UPDATE ${schema}.queue SET
@@ -363,21 +380,28 @@ function updateQueue (schema) {
       retry_backoff = COALESCE($5, retry_backoff),
       expire_seconds = COALESCE($6, expire_seconds),
       retention_minutes = COALESCE($7, retention_minutes),
-      dead_letter = COALESCE($8, dead_letter)
+      dead_letter = COALESCE($8, dead_letter),
+      updated_on = now()
     WHERE name = $1
   `
 }
 
 function getQueues (schema) {
-  return `SELECT * FROM ${schema}.queue`
+  return `
+    SELECT 
+      policy,
+      retry_limit as "retryLimit",
+      retry_delay as "retryDelay",
+      retry_backoff as "retryBackoff",
+      expire_seconds as "expireInSeconds",
+      retention_minutes as "retentionMinutes",
+      dead_letter as "deadLetter"
+    FROM ${schema}.queue    
+   `
 }
 
 function getQueueByName (schema) {
-  return `SELECT * FROM ${schema}.queue WHERE name = $1`
-}
-
-function deleteQueueRecords (schema) {
-  return `DELETE FROM ${schema}.queue WHERE name = $1`
+  return `${getQueues(schema)} WHERE name = $1`
 }
 
 function purgeQueue (schema) {
@@ -779,7 +803,7 @@ function locked (schema, query) {
 }
 
 function advisoryLock (schema, key) {
-  return `SELECT pg_advisory_xact_lock(      
+  return `SELECT pg_advisory_xact_lock(
       ('x' || encode(sha224((current_database() || '.pgboss.${schema}${key || ''}')::bytea), 'hex'))::bit(64)::bigint
   )`
 }