pg-boss 10.0.0-beta12 → 10.0.0-beta14

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pg-boss",
-  "version": "10.0.0-beta12",
+  "version": "10.0.0-beta14",
   "description": "Queueing jobs in Postgres from Node.js like a boss",
   "main": "./src/index.js",
   "engines": {

package/src/attorney.js

@@ -186,6 +186,7 @@ function assertPostgresObjectName (name) {
 }
 
 function assertQueueName (name) {
+  assert(name, 'Name is required')
   assert(typeof name === 'string', 'Name must be a string')
   assert(/[\w-]/.test(name), 'Name can only contain alphanumeric characters, underscores, or hyphens')
 }

package/src/manager.js

@@ -60,6 +60,14 @@ class Manager extends EventEmitter {
     this.unsubscribeCommand = plans.unsubscribe(config.schema)
     this.getQueuesCommand = plans.getQueues(config.schema)
     this.getQueuesForEventCommand = plans.getQueuesForEvent(config.schema)
+    this.getQueueByNameCommand = plans.getQueueByName(config.schema)
+    this.deleteQueueRecordsCommand = plans.deleteQueueRecords(config.schema)
+    this.insertQueueCommand = plans.insertQueue(config.schema)
+    this.updateQueueCommand = plans.updateQueue(config.schema)
+    this.createPartitionCommand = plans.createPartition(config.schema)
+    this.dropPartitionCommand = plans.dropPartition(config.schema)
+    this.clearStorageCommand = plans.clearStorage(config.schema)
+    this.purgeQueueCommand = plans.purgeQueue(config.schema)
 
     // exported api to index
     this.functions = [
@@ -511,7 +519,7 @@ class Manager extends EventEmitter {
   }
 
   async complete (name, id, data, options = {}) {
-    assert(name, 'Missing queue name argument')
+    Attorney.assertQueueName(name)
     const db = options.db || this.db
     const ids = this.mapCompletionIdArg(id, 'complete')
     const result = await db.executeSql(this.completeJobsCommand, [name, ids, this.mapCompletionDataArg(data)])
@@ -519,7 +527,7 @@ class Manager extends EventEmitter {
   }
 
   async fail (name, id, data, options = {}) {
-    assert(name, 'Missing queue name argument')
+    Attorney.assertQueueName(name)
     const db = options.db || this.db
     const ids = this.mapCompletionIdArg(id, 'fail')
     const result = await db.executeSql(this.failJobsByIdCommand, [name, ids, this.mapCompletionDataArg(data)])
@@ -527,7 +535,7 @@ class Manager extends EventEmitter {
   }
 
   async cancel (name, id, options = {}) {
-    assert(name, 'Missing queue name argument')
+    Attorney.assertQueueName(name)
     const db = options.db || this.db
     const ids = this.mapCompletionIdArg(id, 'cancel')
     const result = await db.executeSql(this.cancelJobsCommand, [name, ids])
@@ -535,7 +543,7 @@ class Manager extends EventEmitter {
   }
 
   async resume (name, id, options = {}) {
-    assert(name, 'Missing queue name argument')
+    Attorney.assertQueueName(name)
     const db = options.db || this.db
     const ids = this.mapCompletionIdArg(id, 'resume')
     const result = await db.executeSql(this.resumeJobsCommand, [name, ids])
@@ -543,7 +551,7 @@ class Manager extends EventEmitter {
   }
 
   async createQueue (name, options = {}) {
-    assert(name, 'Missing queue name argument')
+    name = name || options.name
 
     Attorney.assertQueueName(name)
 
@@ -560,11 +568,11 @@ class Manager extends EventEmitter {
       deadLetter
     } = Attorney.checkQueueArgs(name, options)
 
-    const paritionSql = plans.createPartition(this.config.schema, name)
-
-    await this.db.executeSql(paritionSql)
+    if (deadLetter) {
+      Attorney.assertQueueName(deadLetter)
+    }
 
-    const sql = plans.insertQueue(this.config.schema)
+    await this.db.executeSql(this.createPartitionCommand, [name])
 
     const params = [
       name,
@@ -577,7 +585,7 @@ class Manager extends EventEmitter {
       deadLetter
     ]
 
-    await this.db.executeSql(sql, params)
+    await this.db.executeSql(this.insertQueueCommand, params)
   }
 
   async getQueues () {
@@ -586,7 +594,11 @@ class Manager extends EventEmitter {
   }
 
   async updateQueue (name, options = {}) {
-    assert(name, 'Missing queue name argument')
+    Attorney.assertQueueName(name)
+
+    const { policy = QUEUE_POLICIES.standard } = options
+
+    assert(policy in QUEUE_POLICIES, `${policy} is not a valid queue policy`)
 
     const {
       retryLimit,
@@ -597,10 +609,9 @@ class Manager extends EventEmitter {
       deadLetter
     } = Attorney.checkQueueArgs(name, options)
 
-    const sql = plans.updateQueue(this.config.schema)
-
     const params = [
       name,
+      policy,
       retryLimit,
       retryDelay,
       retryBackoff,
@@ -609,14 +620,13 @@ class Manager extends EventEmitter {
       deadLetter
     ]
 
-    await this.db.executeSql(sql, params)
+    await this.db.executeSql(this.updateQueueCommand, params)
   }
 
   async getQueue (name) {
-    assert(name, 'Missing queue name argument')
+    Attorney.assertQueueName(name)
 
-    const sql = plans.getQueueByName(this.config.schema)
-    const result = await this.db.executeSql(sql, [name])
+    const result = await this.db.executeSql(this.getQueueByNameCommand, [name])
 
     if (result.rows.length === 0) {
       return null
@@ -645,51 +655,46 @@ class Manager extends EventEmitter {
   }
 
   async deleteQueue (name) {
-    assert(name, 'Missing queue name argument')
+    Attorney.assertQueueName(name)
 
-    const queueSql = plans.getQueueByName(this.config.schema)
-    const { rows } = await this.db.executeSql(queueSql, [name])
+    const { rows } = await this.db.executeSql(this.getQueueByNameCommand, [name])
 
     if (rows.length) {
-      Attorney.assertQueueName(name)
-      const sql = plans.dropPartition(this.config.schema, name)
-      await this.db.executeSql(sql)
+      await this.db.executeSql(this.dropPartitionCommand, [name])
+      await this.db.executeSql(this.deleteQueueRecordsCommand, [name])
     }
-
-    const sql = plans.deleteQueueRecords(this.config.schema)
-    await this.db.executeSql(sql, [name])
   }
 
-  async purgeQueue (queue) {
-    assert(queue, 'Missing queue name argument')
-    const sql = plans.purgeQueue(this.config.schema)
-    await this.db.executeSql(sql, [queue])
+  async purgeQueue (name) {
+    Attorney.assertQueueName(name)
+    await this.db.executeSql(this.purgeQueueCommand, [name])
   }
 
   async clearStorage () {
-    const sql = plans.clearStorage(this.config.schema)
-    await this.db.executeSql(sql)
+    await this.db.executeSql(this.clearStorageCommand)
   }
 
-  async getQueueSize (queue, options) {
-    assert(queue, 'Missing queue name argument')
+  async getQueueSize (name, options) {
+    Attorney.assertQueueName(name)
 
     const sql = plans.getQueueSize(this.config.schema, options)
 
-    const result = await this.db.executeSql(sql, [queue])
+    const result = await this.db.executeSql(sql, [name])
 
     return result ? parseFloat(result.rows[0].count) : null
   }
 
-  async getJobById (queue, id, options = {}) {
+  async getJobById (name, id, options = {}) {
+    Attorney.assertQueueName(name)
+
     const db = options.db || this.db
-    const result1 = await db.executeSql(this.getJobByIdCommand, [queue, id])
+    const result1 = await db.executeSql(this.getJobByIdCommand, [name, id])
 
     if (result1 && result1.rows && result1.rows.length === 1) {
       return result1.rows[0]
     }
 
-    const result2 = await db.executeSql(this.getArchivedJobByIdCommand, [queue, id])
+    const result2 = await db.executeSql(this.getArchivedJobByIdCommand, [name, id])
 
     if (result2 && result2.rows && result2.rows.length === 1) {
       return result2.rows[0]

package/src/plans.js

@@ -73,24 +73,18 @@ function create (schema, version) {
     createSchema(schema),
     createEnumJobState(schema),
 
+    createTableVersion(schema),
+    createTableQueue(schema),
+    createTableSchedule(schema),
+    createTableSubscription(schema),
+
     createTableJob(schema),
-    createIndexJobFetch(schema),
-    createIndexJobPolicyStately(schema),
-    createIndexJobPolicyShort(schema),
-    createIndexJobPolicySingleton(schema),
-    createIndexJobThrottleOn(schema),
-    createIndexJobThrottleKey(schema),
 
     createTableArchive(schema),
     createPrimaryKeyArchive(schema),
     createColumnArchiveArchivedOn(schema),
     createIndexArchiveArchivedOn(schema),
 
-    createTableVersion(schema),
-    createTableQueue(schema),
-    createTableSchedule(schema),
-    createTableSubscription(schema),
-
     getPartitionFunction(schema),
     createPartitionFunction(schema),
     dropPartitionFunction(schema),
@@ -107,6 +101,21 @@ function createSchema (schema) {
   `
 }
 
+function createEnumJobState (schema) {
+  // ENUM definition order is important
+  // base type is numeric and first values are less than last values
+  return `
+    CREATE TYPE ${schema}.job_state AS ENUM (
+      '${JOB_STATES.created}',
+      '${JOB_STATES.retry}',
+      '${JOB_STATES.active}',
+      '${JOB_STATES.completed}',
+      '${JOB_STATES.cancelled}',
+      '${JOB_STATES.failed}'
+    )
+  `
+}
+
 function createTableVersion (schema) {
   return `
     CREATE TABLE ${schema}.version (
@@ -118,17 +127,46 @@ function createTableVersion (schema) {
   `
 }
 
-function createEnumJobState (schema) {
-  // ENUM definition order is important
-  // base type is numeric and first values are less than last values
+function createTableQueue (schema) {
   return `
-    CREATE TYPE ${schema}.job_state AS ENUM (
-      '${JOB_STATES.created}',
-      '${JOB_STATES.retry}',
-      '${JOB_STATES.active}',
-      '${JOB_STATES.completed}',
-      '${JOB_STATES.cancelled}',
-      '${JOB_STATES.failed}'
+    CREATE TABLE ${schema}.queue (
+      name text,
+      policy text,
+      retry_limit int,
+      retry_delay int,
+      retry_backoff bool,
+      expire_seconds int,
+      retention_minutes int,
+      dead_letter text,
+      created_on timestamp with time zone not null default now(),
+      PRIMARY KEY (name) 
+    )
+  `
+}
+
+function createTableSchedule (schema) {
+  return `
+    CREATE TABLE ${schema}.schedule (
+      name text REFERENCES ${schema}.queue ON DELETE CASCADE,
+      cron text not null,
+      timezone text,
+      data jsonb,
+      options jsonb,
+      created_on timestamp with time zone not null default now(),
+      updated_on timestamp with time zone not null default now(),
+      PRIMARY KEY (name)
+    )
+  `
+}
+
+function createTableSubscription (schema) {
+  return `
+    CREATE TABLE ${schema}.subscription (
+      event text not null,
+      name text not null REFERENCES ${schema}.queue ON DELETE CASCADE,
+      created_on timestamp with time zone not null default now(),
+      updated_on timestamp with time zone not null default now(),
+      PRIMARY KEY(event, name)
     )
   `
 }
@@ -155,8 +193,7 @@ function createTableJob (schema) {
       keep_until timestamp with time zone NOT NULL default now() + interval '14 days',
       output jsonb,
       dead_letter text,
-      policy text,
-      CONSTRAINT job_pkey PRIMARY KEY (name, id)
+      policy text      
     ) PARTITION BY LIST (name)
   `
 }
@@ -179,15 +216,15 @@ const allJobColumns = `${baseJobColumns}, policy, state, priority,
   output
 `
 
-function createPartition (schema, name) {
-  return `SELECT ${schema}.create_partition('${name}');`
+function createPartition (schema) {
+  return `SELECT ${schema}.create_partition($1)`
 }
 
 function getPartitionFunction (schema) {
   return `
     CREATE FUNCTION ${schema}.get_partition(queue_name text, out name text) AS
     $$
-    SELECT 'job_' || encode(sha224(queue_name::bytea), 'hex');
+    SELECT 'j' || left(regexp_replace(queue_name, '\\W', '', 'g'),10) || left(encode(sha224(queue_name::bytea), 'hex'),10);
     $$
     LANGUAGE SQL
     IMMUTABLE
@@ -202,8 +239,18 @@ function createPartitionFunction (schema) {
     DECLARE
       table_name varchar := ${schema}.get_partition(queue_name);
     BEGIN
-      EXECUTE format('CREATE TABLE ${schema}.%I (LIKE ${schema}.job INCLUDING DEFAULTS INCLUDING CONSTRAINTS)', table_name);
-      EXECUTE format('ALTER TABLE ${schema}.%I ADD CHECK (name=%L)', table_name, queue_name);
+      EXECUTE format('CREATE TABLE ${schema}.%I (LIKE ${schema}.job INCLUDING DEFAULTS)', table_name);
+      
+      EXECUTE format('${formatPartitionCommand(createPrimaryKeyJob(schema))}', table_name);
+      EXECUTE format('${formatPartitionCommand(createQueueForeignKeyJob(schema))}', table_name);
+      EXECUTE format('${formatPartitionCommand(createQueueForeignKeyJobDeadLetter(schema))}', table_name);
+      EXECUTE format('${formatPartitionCommand(createIndexJobPolicyShort(schema))}', table_name);
+      EXECUTE format('${formatPartitionCommand(createIndexJobPolicySingleton(schema))}', table_name);
+      EXECUTE format('${formatPartitionCommand(createIndexJobPolicyStately(schema))}', table_name);
+      EXECUTE format('${formatPartitionCommand(createIndexJobThrottle(schema))}', table_name);
+      EXECUTE format('${formatPartitionCommand(createIndexJobFetch(schema))}', table_name);
+
+      EXECUTE format('ALTER TABLE ${schema}.%I ADD CONSTRAINT cjc CHECK (name=%L)', table_name, queue_name);
       EXECUTE format('ALTER TABLE ${schema}.job ATTACH PARTITION ${schema}.%I FOR VALUES IN (%L)', table_name, queue_name);
     END;
     $$
@@ -211,6 +258,10 @@ function createPartitionFunction (schema) {
   `
 }
 
+function formatPartitionCommand (command) {
+  return command.replace('.job', '.%1$I').replace('job_idx', '%1$s_idx').replaceAll('\'', '\'\'')
+}
+
 function dropPartitionFunction (schema) {
   return `
     CREATE FUNCTION ${schema}.drop_partition(queue_name text)
@@ -224,38 +275,44 @@ function dropPartitionFunction (schema) {
   `
 }
 
-function dropPartition (schema, name) {
-  return `SELECT ${schema}.drop_partition('${name}');`
+function dropPartition (schema) {
+  return `SELECT ${schema}.drop_partition($1)`
+}
+
+function createPrimaryKeyJob (schema) {
+  return `ALTER TABLE ${schema}.job ADD PRIMARY KEY (name, id)`
+}
+
+function createQueueForeignKeyJob (schema) {
+  return `ALTER TABLE ${schema}.job ADD CONSTRAINT q_fkey FOREIGN KEY (name) REFERENCES ${schema}.queue (name) ON DELETE RESTRICT`
+}
+
+function createQueueForeignKeyJobDeadLetter (schema) {
+  return `ALTER TABLE ${schema}.job ADD CONSTRAINT dlq_fkey FOREIGN KEY (dead_letter) REFERENCES ${schema}.queue (name) ON DELETE RESTRICT`
 }
 
 function createPrimaryKeyArchive (schema) {
-  return `ALTER TABLE ${schema}.archive ADD CONSTRAINT archive_pkey PRIMARY KEY (name, id)`
+  return `ALTER TABLE ${schema}.archive ADD PRIMARY KEY (name, id)`
 }
 
 function createIndexJobPolicyShort (schema) {
-  // todo: how to combine these policies with singleton key?
-  return `CREATE UNIQUE INDEX job_policy_short ON ${schema}.job (name) WHERE state = '${JOB_STATES.created}' AND policy = '${QUEUE_POLICIES.short}'`
+  return `CREATE UNIQUE INDEX job_idx_psh ON ${schema}.job (name, COALESCE(singleton_key, '')) WHERE state = '${JOB_STATES.created}' AND policy = '${QUEUE_POLICIES.short}';`
 }
 
 function createIndexJobPolicySingleton (schema) {
-  return `CREATE UNIQUE INDEX job_policy_singleton ON ${schema}.job (name) WHERE state = '${JOB_STATES.active}' AND policy = '${QUEUE_POLICIES.singleton}'`
+  return `CREATE UNIQUE INDEX job_idx_psi ON ${schema}.job (name, COALESCE(singleton_key, '')) WHERE state = '${JOB_STATES.active}' AND policy = '${QUEUE_POLICIES.singleton}'`
 }
 
 function createIndexJobPolicyStately (schema) {
-  return `CREATE UNIQUE INDEX job_policy_stately ON ${schema}.job (name, state) WHERE state <= '${JOB_STATES.active}' AND policy = '${QUEUE_POLICIES.stately}'`
-}
-
-function createIndexJobThrottleOn (schema) {
-  return `CREATE UNIQUE INDEX job_throttle_on ON ${schema}.job (name, singleton_on, COALESCE(singleton_key, '')) WHERE state <= '${JOB_STATES.completed}' AND singleton_on IS NOT NULL`
+  return `CREATE UNIQUE INDEX job_idx_pst ON ${schema}.job (name, state, COALESCE(singleton_key, '')) WHERE state <= '${JOB_STATES.active}' AND policy = '${QUEUE_POLICIES.stately}'`
 }
 
-function createIndexJobThrottleKey (schema) {
-  // how useful is this really?
-  return `CREATE UNIQUE INDEX job_throttle_key ON ${schema}.job (name, singleton_key) WHERE state <= '${JOB_STATES.completed}' AND singleton_on IS NULL`
+function createIndexJobThrottle (schema) {
+  return `CREATE UNIQUE INDEX job_idx_to ON ${schema}.job (name, singleton_on, COALESCE(singleton_key, '')) WHERE state <> '${JOB_STATES.cancelled}' AND singleton_on IS NOT NULL`
 }
 
 function createIndexJobFetch (schema) {
-  return `CREATE INDEX job_fetch ON ${schema}.job (name, start_after) INCLUDE (priority, created_on, id) WHERE state < '${JOB_STATES.active}'`
+  return `CREATE INDEX job_idx_f ON ${schema}.job (name, start_after) INCLUDE (priority, created_on, id) WHERE state < '${JOB_STATES.active}'`
 }
 
 function createTableArchive (schema) {
@@ -267,7 +324,7 @@ function createColumnArchiveArchivedOn (schema) {
 }
 
 function createIndexArchiveArchivedOn (schema) {
-  return `CREATE INDEX archive_archived_on_idx ON ${schema}.archive(archived_on)`
+  return `CREATE INDEX archive_idx_ao ON ${schema}.archive(archived_on)`
 }
 
 function trySetMaintenanceTime (schema) {
@@ -300,12 +357,13 @@ function insertQueue (schema) {
 function updateQueue (schema) {
   return `
     UPDATE ${schema}.queue SET
-      retry_limit = COALESCE($2, retry_limit),
-      retry_delay = COALESCE($3, retry_delay),
-      retry_backoff = COALESCE($4, retry_backoff),
-      expire_seconds = COALESCE($5, expire_seconds),
-      retention_minutes = COALESCE($6, retention_minutes),
-      dead_letter = COALESCE($7, dead_letter)
+      policy = COALESCE($2, policy),
+      retry_limit = COALESCE($3, retry_limit),
+      retry_delay = COALESCE($4, retry_delay),
+      retry_backoff = COALESCE($5, retry_backoff),
+      expire_seconds = COALESCE($6, expire_seconds),
+      retention_minutes = COALESCE($7, retention_minutes),
+      dead_letter = COALESCE($8, dead_letter)
     WHERE name = $1
   `
 }
@@ -319,13 +377,7 @@ function getQueueByName (schema) {
 }
 
 function deleteQueueRecords (schema) {
-  return `WITH dq AS (
-      DELETE FROM ${schema}.queue WHERE name = $1
-    ), ds AS (
-      DELETE FROM ${schema}.schedule WHERE name = $1
-    )
-    DELETE FROM ${schema}.job WHERE name = $1
-  `
+  return `DELETE FROM ${schema}.queue WHERE name = $1`
 }
 
 function purgeQueue (schema) {
@@ -342,54 +394,8 @@ function getQueueSize (schema, options = {}) {
   return `SELECT count(*) as count FROM ${schema}.job WHERE name = $1 AND state < '${options.before}'`
 }
 
-function createTableQueue (schema) {
-  return `
-    CREATE TABLE ${schema}.queue (
-      name text primary key,
-      policy text,
-      retry_limit int,
-      retry_delay int,
-      retry_backoff bool,
-      expire_seconds int,
-      retention_minutes int,
-      dead_letter text,
-      created_on timestamp with time zone not null default now()
-    )
-  `
-}
-
-function createTableSchedule (schema) {
-  return `
-    CREATE TABLE ${schema}.schedule (
-      name text primary key,
-      cron text not null,
-      timezone text,
-      data jsonb,
-      options jsonb,
-      created_on timestamp with time zone not null default now(),
-      updated_on timestamp with time zone not null default now()
-    )
-  `
-}
-
-function createTableSubscription (schema) {
-  return `
-    CREATE TABLE ${schema}.subscription (
-      event text not null,
-      name text not null,
-      created_on timestamp with time zone not null default now(),
-      updated_on timestamp with time zone not null default now(),
-      PRIMARY KEY(event, name)
-    )
-  `
-}
-
 function getSchedules (schema) {
-  return `
-    SELECT s.*
-    FROM ${schema}.schedule s
-      JOIN ${schema}.queue q on s.name = q.name
-  `
+  return `SELECT * FROM ${schema}.schedule`
 }
 
 function schedule (schema) {

package/types.d.ts

@@ -113,9 +113,9 @@ declare namespace PgBoss {
   type SendOptions = JobOptions & ExpirationOptions & RetentionOptions & RetryOptions & ConnectionOptions;
   
   type QueuePolicy = 'standard' | 'short' | 'singleton' | 'stately'
-  type Queue = ExpirationOptions & RetentionOptions & RetryOptions & { policy: QueuePolicy }
-  type QueueUpdateOptions = ExpirationOptions & RetentionOptions & RetryOptions
-
+  
+  type Queue = RetryOptions & ExpirationOptions & RetentionOptions & { name: string, policy: QueuePolicy, deadLetter?: string }
+  
   type ScheduleOptions = SendOptions & { tz?: string }
 
   interface JobPollingOptions {
@@ -371,7 +371,7 @@ declare class PgBoss extends EventEmitter {
   createQueue(name: string, options?: PgBoss.Queue): Promise<void>;
   getQueue(name: string): Promise<PgBoss.Queue | null>;
   getQueues(): Promise<[PgBoss.Queue]>;
-  updateQueue(name: string, options?: PgBoss.QueueUpdateOptions): Promise<void>;
+  updateQueue(name: string, options?: PgBoss.Queue): Promise<void>;
   deleteQueue(name: string): Promise<void>;
   purgeQueue(name: string): Promise<void>;