pg-boss 10.0.0-beta11 → 10.0.0-beta13

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pg-boss",
-  "version": "10.0.0-beta11",
+  "version": "10.0.0-beta13",
   "description": "Queueing jobs in Postgres from Node.js like a boss",
   "main": "./src/index.js",
   "engines": {

package/src/attorney.js

@@ -186,6 +186,7 @@ function assertPostgresObjectName (name) {
 }
 
 function assertQueueName (name) {
+  assert(name, 'Name is required')
   assert(typeof name === 'string', 'Name must be a string')
   assert(/[\w-]/.test(name), 'Name can only contain alphanumeric characters, underscores, or hyphens')
 }

package/src/manager.js

@@ -58,6 +58,7 @@ class Manager extends EventEmitter {
     this.getArchivedJobByIdCommand = plans.getArchivedJobById(config.schema)
     this.subscribeCommand = plans.subscribe(config.schema)
     this.unsubscribeCommand = plans.unsubscribe(config.schema)
+    this.getQueuesCommand = plans.getQueues(config.schema)
     this.getQueuesForEventCommand = plans.getQueuesForEvent(config.schema)
 
     // exported api to index
@@ -80,10 +81,11 @@ class Manager extends EventEmitter {
       this.sendAfter,
       this.createQueue,
       this.updateQueue,
-      this.getQueue,
       this.deleteQueue,
       this.purgeQueue,
       this.getQueueSize,
+      this.getQueue,
+      this.getQueues,
       this.clearStorage,
       this.getJobById
     ]
@@ -509,7 +511,7 @@ class Manager extends EventEmitter {
   }
 
   async complete (name, id, data, options = {}) {
-    assert(name, 'Missing queue name argument')
+    Attorney.assertQueueName(name)
     const db = options.db || this.db
     const ids = this.mapCompletionIdArg(id, 'complete')
     const result = await db.executeSql(this.completeJobsCommand, [name, ids, this.mapCompletionDataArg(data)])
@@ -517,7 +519,7 @@ class Manager extends EventEmitter {
   }
 
   async fail (name, id, data, options = {}) {
-    assert(name, 'Missing queue name argument')
+    Attorney.assertQueueName(name)
     const db = options.db || this.db
     const ids = this.mapCompletionIdArg(id, 'fail')
     const result = await db.executeSql(this.failJobsByIdCommand, [name, ids, this.mapCompletionDataArg(data)])
@@ -525,7 +527,7 @@ class Manager extends EventEmitter {
   }
 
   async cancel (name, id, options = {}) {
-    assert(name, 'Missing queue name argument')
+    Attorney.assertQueueName(name)
     const db = options.db || this.db
     const ids = this.mapCompletionIdArg(id, 'cancel')
     const result = await db.executeSql(this.cancelJobsCommand, [name, ids])
@@ -533,7 +535,7 @@ class Manager extends EventEmitter {
   }
 
   async resume (name, id, options = {}) {
-    assert(name, 'Missing queue name argument')
+    Attorney.assertQueueName(name)
     const db = options.db || this.db
     const ids = this.mapCompletionIdArg(id, 'resume')
     const result = await db.executeSql(this.resumeJobsCommand, [name, ids])
@@ -541,7 +543,7 @@ class Manager extends EventEmitter {
   }
 
   async createQueue (name, options = {}) {
-    assert(name, 'Missing queue name argument')
+    name = name || options.name
 
     Attorney.assertQueueName(name)
 
@@ -558,11 +560,15 @@ class Manager extends EventEmitter {
       deadLetter
     } = Attorney.checkQueueArgs(name, options)
 
+    if (deadLetter) {
+      Attorney.assertQueueName(deadLetter)
+    }
+
     const paritionSql = plans.createPartition(this.config.schema, name)
 
     await this.db.executeSql(paritionSql)
 
-    const sql = plans.createQueue(this.config.schema, name)
+    const sql = plans.insertQueue(this.config.schema)
 
     const params = [
       name,
@@ -578,8 +584,17 @@ class Manager extends EventEmitter {
     await this.db.executeSql(sql, params)
   }
 
+  async getQueues () {
+    const { rows } = await this.db.executeSql(this.getQueuesCommand)
+    return rows
+  }
+
   async updateQueue (name, options = {}) {
-    assert(name, 'Missing queue name argument')
+    Attorney.assertQueueName(name)
+
+    const { policy = QUEUE_POLICIES.standard } = options
+
+    assert(policy in QUEUE_POLICIES, `${policy} is not a valid queue policy`)
 
     const {
       retryLimit,
@@ -594,6 +609,7 @@ class Manager extends EventEmitter {
 
     const params = [
       name,
+      policy,
       retryLimit,
       retryDelay,
       retryBackoff,
@@ -606,7 +622,7 @@ class Manager extends EventEmitter {
   }
 
   async getQueue (name) {
-    assert(name, 'Missing queue name argument')
+    Attorney.assertQueueName(name)
 
     const sql = plans.getQueueByName(this.config.schema)
     const result = await this.db.executeSql(sql, [name])
@@ -638,25 +654,21 @@ class Manager extends EventEmitter {
   }
 
   async deleteQueue (name) {
-    assert(name, 'Missing queue name argument')
+    Attorney.assertQueueName(name)
 
     const queueSql = plans.getQueueByName(this.config.schema)
     const { rows } = await this.db.executeSql(queueSql, [name])
 
     if (rows.length) {
-      Attorney.assertQueueName(name)
-      const sql = plans.dropPartition(this.config.schema, name)
-      await this.db.executeSql(sql)
+      await this.db.executeSql(plans.dropPartition(this.config.schema, name))
+      await this.db.executeSql(plans.deleteQueueRecords(this.config.schema), [name])
     }
-
-    const sql = plans.deleteQueueRecords(this.config.schema)
-    await this.db.executeSql(sql, [name])
   }
 
-  async purgeQueue (queue) {
-    assert(queue, 'Missing queue name argument')
+  async purgeQueue (name) {
+    Attorney.assertQueueName(name)
     const sql = plans.purgeQueue(this.config.schema)
-    await this.db.executeSql(sql, [queue])
+    await this.db.executeSql(sql, [name])
   }
 
   async clearStorage () {
@@ -664,25 +676,27 @@ class Manager extends EventEmitter {
     await this.db.executeSql(sql)
   }
 
-  async getQueueSize (queue, options) {
-    assert(queue, 'Missing queue name argument')
+  async getQueueSize (name, options) {
+    Attorney.assertQueueName(name)
 
     const sql = plans.getQueueSize(this.config.schema, options)
 
-    const result = await this.db.executeSql(sql, [queue])
+    const result = await this.db.executeSql(sql, [name])
 
     return result ? parseFloat(result.rows[0].count) : null
   }
 
-  async getJobById (queue, id, options = {}) {
+  async getJobById (name, id, options = {}) {
+    Attorney.assertQueueName(name)
+
     const db = options.db || this.db
-    const result1 = await db.executeSql(this.getJobByIdCommand, [queue, id])
+    const result1 = await db.executeSql(this.getJobByIdCommand, [name, id])
 
     if (result1 && result1.rows && result1.rows.length === 1) {
       return result1.rows[0]
     }
 
-    const result2 = await db.executeSql(this.getArchivedJobByIdCommand, [queue, id])
+    const result2 = await db.executeSql(this.getArchivedJobByIdCommand, [name, id])
 
     if (result2 && result2.rows && result2.rows.length === 1) {
       return result2.rows[0]

package/src/plans.js

@@ -42,11 +42,12 @@ module.exports = {
   archive,
   drop,
   countStates,
-  createQueue,
+  insertQueue,
   updateQueue,
   createPartition,
   dropPartition,
   deleteQueueRecords,
+  getQueues,
   getQueueByName,
   getQueueSize,
   purgeQueue,
@@ -72,24 +73,18 @@ function create (schema, version) {
     createSchema(schema),
     createEnumJobState(schema),
 
+    createTableVersion(schema),
+    createTableQueue(schema),
+    createTableSchedule(schema),
+    createTableSubscription(schema),
+
     createTableJob(schema),
-    createIndexJobFetch(schema),
-    createIndexJobPolicyStately(schema),
-    createIndexJobPolicyShort(schema),
-    createIndexJobPolicySingleton(schema),
-    createIndexJobThrottleOn(schema),
-    createIndexJobThrottleKey(schema),
 
     createTableArchive(schema),
     createPrimaryKeyArchive(schema),
     createColumnArchiveArchivedOn(schema),
     createIndexArchiveArchivedOn(schema),
 
-    createTableVersion(schema),
-    createTableQueue(schema),
-    createTableSchedule(schema),
-    createTableSubscription(schema),
-
     getPartitionFunction(schema),
     createPartitionFunction(schema),
     dropPartitionFunction(schema),
@@ -106,6 +101,21 @@ function createSchema (schema) {
   `
 }
 
+function createEnumJobState (schema) {
+  // ENUM definition order is important
+  // base type is numeric and first values are less than last values
+  return `
+    CREATE TYPE ${schema}.job_state AS ENUM (
+      '${JOB_STATES.created}',
+      '${JOB_STATES.retry}',
+      '${JOB_STATES.active}',
+      '${JOB_STATES.completed}',
+      '${JOB_STATES.cancelled}',
+      '${JOB_STATES.failed}'
+    )
+  `
+}
+
 function createTableVersion (schema) {
   return `
     CREATE TABLE ${schema}.version (
@@ -117,17 +127,46 @@ function createTableVersion (schema) {
   `
 }
 
-function createEnumJobState (schema) {
-  // ENUM definition order is important
-  // base type is numeric and first values are less than last values
+function createTableQueue (schema) {
   return `
-    CREATE TYPE ${schema}.job_state AS ENUM (
-      '${JOB_STATES.created}',
-      '${JOB_STATES.retry}',
-      '${JOB_STATES.active}',
-      '${JOB_STATES.completed}',
-      '${JOB_STATES.cancelled}',
-      '${JOB_STATES.failed}'
+    CREATE TABLE ${schema}.queue (
+      name text,
+      policy text,
+      retry_limit int,
+      retry_delay int,
+      retry_backoff bool,
+      expire_seconds int,
+      retention_minutes int,
+      dead_letter text,
+      created_on timestamp with time zone not null default now(),
+      PRIMARY KEY (name) 
+    )
+  `
+}
+
+function createTableSchedule (schema) {
+  return `
+    CREATE TABLE ${schema}.schedule (
+      name text REFERENCES ${schema}.queue ON DELETE CASCADE,
+      cron text not null,
+      timezone text,
+      data jsonb,
+      options jsonb,
+      created_on timestamp with time zone not null default now(),
+      updated_on timestamp with time zone not null default now(),
+      PRIMARY KEY (name)
+    )
+  `
+}
+
+function createTableSubscription (schema) {
+  return `
+    CREATE TABLE ${schema}.subscription (
+      event text not null,
+      name text not null REFERENCES ${schema}.queue ON DELETE CASCADE,
+      created_on timestamp with time zone not null default now(),
+      updated_on timestamp with time zone not null default now(),
+      PRIMARY KEY(event, name)
     )
   `
 }
@@ -154,8 +193,7 @@ function createTableJob (schema) {
       keep_until timestamp with time zone NOT NULL default now() + interval '14 days',
       output jsonb,
       dead_letter text,
-      policy text,
-      CONSTRAINT job_pkey PRIMARY KEY (name, id)
+      policy text      
     ) PARTITION BY LIST (name)
   `
 }
@@ -186,7 +224,7 @@ function getPartitionFunction (schema) {
   return `
     CREATE FUNCTION ${schema}.get_partition(queue_name text, out name text) AS
     $$
-    SELECT 'job_' || encode(sha224(queue_name::bytea), 'hex');
+    SELECT 'j' || left(regexp_replace(queue_name, '\\W', '', 'g'),10) || left(encode(sha224(queue_name::bytea), 'hex'),10);
     $$
     LANGUAGE SQL
     IMMUTABLE
@@ -201,8 +239,18 @@ function createPartitionFunction (schema) {
     DECLARE
       table_name varchar := ${schema}.get_partition(queue_name);
     BEGIN
-      EXECUTE format('CREATE TABLE ${schema}.%I (LIKE ${schema}.job INCLUDING DEFAULTS INCLUDING CONSTRAINTS)', table_name);
-      EXECUTE format('ALTER TABLE ${schema}.%I ADD CHECK (name=%L)', table_name, queue_name);
+      EXECUTE format('CREATE TABLE ${schema}.%I (LIKE ${schema}.job INCLUDING DEFAULTS)', table_name);
+      
+      EXECUTE format('${formatPartitionCommand(createPrimaryKeyJob(schema))}', table_name);
+      EXECUTE format('${formatPartitionCommand(createQueueForeignKeyJob(schema))}', table_name);
+      EXECUTE format('${formatPartitionCommand(createQueueForeignKeyJobDeadLetter(schema))}', table_name);
+      EXECUTE format('${formatPartitionCommand(createIndexJobPolicyShort(schema))}', table_name);
+      EXECUTE format('${formatPartitionCommand(createIndexJobPolicySingleton(schema))}', table_name);
+      EXECUTE format('${formatPartitionCommand(createIndexJobPolicyStately(schema))}', table_name);
+      EXECUTE format('${formatPartitionCommand(createIndexJobThrottle(schema))}', table_name);
+      EXECUTE format('${formatPartitionCommand(createIndexJobFetch(schema))}', table_name);
+
+      EXECUTE format('ALTER TABLE ${schema}.%I ADD CONSTRAINT cjc CHECK (name=%L)', table_name, queue_name);
       EXECUTE format('ALTER TABLE ${schema}.job ATTACH PARTITION ${schema}.%I FOR VALUES IN (%L)', table_name, queue_name);
     END;
     $$
@@ -210,13 +258,17 @@ function createPartitionFunction (schema) {
   `
 }
 
+function formatPartitionCommand (command) {
+  return command.replace('.job', '.%1$I').replace('job_idx', '%1$s_idx').replaceAll('\'', '\'\'')
+}
+
 function dropPartitionFunction (schema) {
   return `
     CREATE FUNCTION ${schema}.drop_partition(queue_name text)
     RETURNS VOID AS
     $$
     BEGIN  
-      EXECUTE format('DROP TABLE IF EXISTS %I', ${schema}.get_partition(queue_name));
+      EXECUTE format('DROP TABLE IF EXISTS ${schema}.%I', ${schema}.get_partition(queue_name));
     END;
     $$
     LANGUAGE plpgsql;
@@ -227,34 +279,40 @@ function dropPartition (schema, name) {
   return `SELECT ${schema}.drop_partition('${name}');`
 }
 
+function createPrimaryKeyJob (schema) {
+  return `ALTER TABLE ${schema}.job ADD PRIMARY KEY (name, id)`
+}
+
+function createQueueForeignKeyJob (schema) {
+  return `ALTER TABLE ${schema}.job ADD FOREIGN KEY (name) REFERENCES ${schema}.queue (name) ON DELETE RESTRICT`
+}
+
+function createQueueForeignKeyJobDeadLetter (schema) {
+  return `ALTER TABLE ${schema}.job ADD FOREIGN KEY (dead_letter) REFERENCES ${schema}.queue (name) ON DELETE RESTRICT`
+}
+
 function createPrimaryKeyArchive (schema) {
-  return `ALTER TABLE ${schema}.archive ADD CONSTRAINT archive_pkey PRIMARY KEY (name, id)`
+  return `ALTER TABLE ${schema}.archive ADD PRIMARY KEY (name, id)`
 }
 
 function createIndexJobPolicyShort (schema) {
-  // todo: how to combine these policies with singleton key?
-  return `CREATE UNIQUE INDEX job_policy_short ON ${schema}.job (name) WHERE state = '${JOB_STATES.created}' AND policy = '${QUEUE_POLICIES.short}'`
+  return `CREATE UNIQUE INDEX job_idx_psh ON ${schema}.job (name, COALESCE(singleton_key, '')) WHERE state = '${JOB_STATES.created}' AND policy = '${QUEUE_POLICIES.short}';`
 }
 
 function createIndexJobPolicySingleton (schema) {
-  return `CREATE UNIQUE INDEX job_policy_singleton ON ${schema}.job (name) WHERE state = '${JOB_STATES.active}' AND policy = '${QUEUE_POLICIES.singleton}'`
+  return `CREATE UNIQUE INDEX job_idx_psi ON ${schema}.job (name, COALESCE(singleton_key, '')) WHERE state = '${JOB_STATES.active}' AND policy = '${QUEUE_POLICIES.singleton}'`
 }
 
 function createIndexJobPolicyStately (schema) {
-  return `CREATE UNIQUE INDEX job_policy_stately ON ${schema}.job (name, state) WHERE state <= '${JOB_STATES.active}' AND policy = '${QUEUE_POLICIES.stately}'`
+  return `CREATE UNIQUE INDEX job_idx_pst ON ${schema}.job (name, state, COALESCE(singleton_key, '')) WHERE state <= '${JOB_STATES.active}' AND policy = '${QUEUE_POLICIES.stately}'`
 }
 
-function createIndexJobThrottleOn (schema) {
-  return `CREATE UNIQUE INDEX job_throttle_on ON ${schema}.job (name, singleton_on, COALESCE(singleton_key, '')) WHERE state <= '${JOB_STATES.completed}' AND singleton_on IS NOT NULL`
-}
-
-function createIndexJobThrottleKey (schema) {
-  // how useful is this really?
-  return `CREATE UNIQUE INDEX job_throttle_key ON ${schema}.job (name, singleton_key) WHERE state <= '${JOB_STATES.completed}' AND singleton_on IS NULL`
+function createIndexJobThrottle (schema) {
+  return `CREATE UNIQUE INDEX job_idx_to ON ${schema}.job (name, singleton_on, COALESCE(singleton_key, '')) WHERE state <> '${JOB_STATES.cancelled}' AND singleton_on IS NOT NULL`
 }
 
 function createIndexJobFetch (schema) {
-  return `CREATE INDEX job_fetch ON ${schema}.job (name, start_after) INCLUDE (priority, created_on, id) WHERE state < '${JOB_STATES.active}'`
+  return `CREATE INDEX job_idx_f ON ${schema}.job (name, start_after) INCLUDE (priority, created_on, id) WHERE state < '${JOB_STATES.active}'`
 }
 
 function createTableArchive (schema) {
@@ -266,7 +324,7 @@ function createColumnArchiveArchivedOn (schema) {
 }
 
 function createIndexArchiveArchivedOn (schema) {
-  return `CREATE INDEX archive_archived_on_idx ON ${schema}.archive(archived_on)`
+  return `CREATE INDEX archive_idx_ao ON ${schema}.archive(archived_on)`
 }
 
 function trySetMaintenanceTime (schema) {
@@ -289,7 +347,7 @@ function trySetTimestamp (schema, column) {
   `
 }
 
-function createQueue (schema) {
+function insertQueue (schema) {
   return `
     INSERT INTO ${schema}.queue (name, policy, retry_limit, retry_delay, retry_backoff, expire_seconds, retention_minutes, dead_letter)
     VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
@@ -299,28 +357,27 @@ function createQueue (schema) {
 function updateQueue (schema) {
   return `
     UPDATE ${schema}.queue SET
-      retry_limit = COALESCE($2, retry_limit),
-      retry_delay = COALESCE($3, retry_delay),
-      retry_backoff = COALESCE($4, retry_backoff),
-      expire_seconds = COALESCE($5, expire_seconds),
-      retention_minutes = COALESCE($6, retention_minutes),
-      dead_letter = COALESCE($7, dead_letter)
+      policy = COALESCE($2, policy),
+      retry_limit = COALESCE($3, retry_limit),
+      retry_delay = COALESCE($4, retry_delay),
+      retry_backoff = COALESCE($5, retry_backoff),
+      expire_seconds = COALESCE($6, expire_seconds),
+      retention_minutes = COALESCE($7, retention_minutes),
+      dead_letter = COALESCE($8, dead_letter)
     WHERE name = $1
   `
 }
 
+function getQueues (schema) {
+  return `SELECT * FROM ${schema}.queue`
+}
+
 function getQueueByName (schema) {
   return `SELECT * FROM ${schema}.queue WHERE name = $1`
 }
 
 function deleteQueueRecords (schema) {
-  return `WITH dq AS (
-      DELETE FROM ${schema}.queue WHERE name = $1
-    ), ds AS (
-      DELETE FROM ${schema}.schedule WHERE name = $1
-    )
-    DELETE FROM ${schema}.job WHERE name = $1
-  `
+  return `DELETE FROM ${schema}.queue WHERE name = $1`
 }
 
 function purgeQueue (schema) {
@@ -337,54 +394,8 @@ function getQueueSize (schema, options = {}) {
   return `SELECT count(*) as count FROM ${schema}.job WHERE name = $1 AND state < '${options.before}'`
 }
 
-function createTableQueue (schema) {
-  return `
-    CREATE TABLE ${schema}.queue (
-      name text primary key,
-      policy text,
-      retry_limit int,
-      retry_delay int,
-      retry_backoff bool,
-      expire_seconds int,
-      retention_minutes int,
-      dead_letter text,
-      created_on timestamp with time zone not null default now()
-    )
-  `
-}
-
-function createTableSchedule (schema) {
-  return `
-    CREATE TABLE ${schema}.schedule (
-      name text primary key,
-      cron text not null,
-      timezone text,
-      data jsonb,
-      options jsonb,
-      created_on timestamp with time zone not null default now(),
-      updated_on timestamp with time zone not null default now()
-    )
-  `
-}
-
-function createTableSubscription (schema) {
-  return `
-    CREATE TABLE ${schema}.subscription (
-      event text not null,
-      name text not null,
-      created_on timestamp with time zone not null default now(),
-      updated_on timestamp with time zone not null default now(),
-      PRIMARY KEY(event, name)
-    )
-  `
-}
-
 function getSchedules (schema) {
-  return `
-    SELECT s.*
-    FROM ${schema}.schedule s
-      JOIN ${schema}.queue q on s.name = q.name
-  `
+  return `SELECT * FROM ${schema}.schedule`
 }
 
 function schedule (schema) {

package/types.d.ts

@@ -113,9 +113,9 @@ declare namespace PgBoss {
   type SendOptions = JobOptions & ExpirationOptions & RetentionOptions & RetryOptions & ConnectionOptions;
   
   type QueuePolicy = 'standard' | 'short' | 'singleton' | 'stately'
-  type Queue = ExpirationOptions & RetentionOptions & RetryOptions & { policy: QueuePolicy }
-  type QueueUpdateOptions = ExpirationOptions & RetentionOptions & RetryOptions
-
+  
+  type Queue = RetryOptions & ExpirationOptions & RetentionOptions & { name: string, policy: QueuePolicy, deadLetter?: string }
+  
   type ScheduleOptions = SendOptions & { tz?: string }
 
   interface JobPollingOptions {
@@ -370,7 +370,8 @@ declare class PgBoss extends EventEmitter {
 
   createQueue(name: string, options?: PgBoss.Queue): Promise<void>;
   getQueue(name: string): Promise<PgBoss.Queue | null>;
-  updateQueue(name: string, options?: PgBoss.QueueUpdateOptions): Promise<void>;
+  getQueues(): Promise<[PgBoss.Queue]>;
+  updateQueue(name: string, options?: PgBoss.Queue): Promise<void>;
   deleteQueue(name: string): Promise<void>;
   purgeQueue(name: string): Promise<void>;