pg-boss 10.0.0-beta1 → 10.0.0-beta3

package/README.md

@@ -1,4 +1,4 @@
-Queueing jobs in Node.js using PostgreSQL like a boss.
+Queueing jobs in Postgres from Node.js like a boss.
 
 [![npm version](https://badge.fury.io/js/pg-boss.svg)](https://badge.fury.io/js/pg-boss)
 [![Build](https://github.com/timgit/pg-boss/actions/workflows/ci.yml/badge.svg?branch=master)](https://github.com/timgit/pg-boss/actions/workflows/ci.yml)
@@ -41,15 +41,13 @@ This will likely cater the most to teams already familiar with the simplicity of
 * Backpressure-compatible polling workers
 * Cron scheduling
 * Pub/sub API for fan-out queue relationships
-* Priority, deferral, retries (with exponential backoff), rate limiting, debouncing
-* Direct table access for bulk loads via COPY or INSERT
+* Priority queues, deferral, retries (with exponential backoff), rate limiting, debouncing
+* Table operations via SQL for bulk loads via COPY or INSERT
 * Multi-master compatible (for example, in a Kubernetes ReplicaSet)
 * Dead letter queues
-* Automatic creation and migration of storage tables
-* Automatic maintenance operations to manage table growth
 
 ## Requirements
-* Node 18 or higher
+* Node 20 or higher
 * PostgreSQL 12 or higher
 
 ## Installation
@@ -66,7 +64,6 @@ yarn add pg-boss
 * [Docs](docs/readme.md)
 
 ## Contributing
-
 To setup a development environment for this library:
 
 ```bash

package/package.json

@@ -1,10 +1,10 @@
 {
   "name": "pg-boss",
-  "version": "10.0.0-beta1",
-  "description": "Queueing jobs in Node.js using PostgreSQL like a boss",
+  "version": "10.0.0-beta3",
+  "description": "Queueing jobs in Postgres from Node.js like a boss",
   "main": "./src/index.js",
   "engines": {
-    "node": ">=18"
+    "node": ">=20"
   },
   "dependencies": {
     "cron-parser": "^4.0.0",
@@ -17,14 +17,15 @@
     "@types/node": "^20.3.3",
     "luxon": "^3.0.1",
     "mocha": "^10.0.0",
-    "nyc": "^15.1.0",
+    "nyc": "^17.0.0",
     "standard": "^17.0.0"
   },
   "scripts": {
     "test": "standard && mocha",
     "cover": "nyc npm test",
     "tsc": "tsc --noEmit types.d.ts",
-    "readme": "node ./test/readme.js"
+    "readme": "node ./test/readme.js",
+    "migrate": "node -e 'console.log(require(\"./src\").getMigrationPlans())'"
   },
   "mocha": {
     "timeout": 10000,

package/src/attorney.js

@@ -8,8 +8,8 @@ module.exports = {
   checkWorkArgs,
   checkFetchArgs,
   warnClockSkew,
-  queueNameHasPatternMatch,
-  assertPostgresObjectName
+  assertPostgresObjectName,
+  assertQueueName
 }
 
 const MAX_INTERVAL_HOURS = 24
@@ -30,8 +30,6 @@ const WARNINGS = {
 }
 
 function checkQueueArgs (name, options = {}) {
-  assertPostgresObjectName(name)
-
   assert(!('deadLetter' in options) || (typeof options.deadLetter === 'string'), 'deadLetter must be a string')
 
   applyRetryConfig(options)
@@ -140,24 +138,12 @@ function checkWorkArgs (name, args, defaults) {
 function checkFetchArgs (name, batchSize, options) {
   assert(name, 'missing queue name')
 
-  if (queueNameHasPatternMatch(name)) {
-    name = sanitizeQueueNameForFetch(name)
-  }
-
   assert(!batchSize || (Number.isInteger(batchSize) && batchSize >= 1), 'batchSize must be an integer > 0')
   assert(!('includeMetadata' in options) || typeof options.includeMetadata === 'boolean', 'includeMetadata must be a boolean')
 
   return { name }
 }
 
-function sanitizeQueueNameForFetch (name) {
-  return name.replace(/[%_*]/g, match => match === '*' ? '%' : '\\' + match)
-}
-
-function queueNameHasPatternMatch (name) {
-  return name.includes('*')
-}
-
 function getConfig (value) {
   assert(value && (typeof value === 'object' || typeof value === 'string'),
     'configuration assert: string or config object is required to connect to postgres')
@@ -191,8 +177,14 @@ function applySchemaConfig (config) {
 function assertPostgresObjectName (name) {
   assert(typeof name === 'string', 'Name must be a string')
   assert(name.length <= 50, 'Name cannot exceed 50 characters')
-  assert(!/\W/.test(name), 'Name can only contain alphanumeric characters and underscores')
-  assert(!/^d/.test(name), 'Name cannot start with a number')
+  assert(!/\W/.test(name), 'Name can only contain alphanumeric characters or underscores')
+  assert(!/^\d/.test(name), 'Name cannot start with a number')
+}
+
+function assertQueueName (name) {
+  assert(typeof name === 'string', 'Name must be a string')
+  assert(name.length <= 50, 'Name cannot exceed 50 characters')
+  assert(/[\w-]/.test(name), 'Name can only contain alphanumeric characters, underscores, or hyphens')
 }
 
 function applyArchiveConfig (config) {

package/src/contractor.js

@@ -21,6 +21,12 @@ class Contractor {
     this.config = config
     this.db = db
     this.migrations = this.config.migrations || migrationStore.getAll(this.config.schema)
+
+    // exported api to index
+    this.functions = [
+      this.version,
+      this.isInstalled
+    ]
   }
 
   async version () {
@@ -40,7 +46,8 @@ class Contractor {
       const version = await this.version()
 
       if (schemaVersion > version) {
-        await this.migrate(version)
+        throw new Error('Migrations are not supported to v10')
+        // await this.migrate(version)
       }
     } else {
       await this.create()

package/src/db.js

@@ -11,6 +11,10 @@ class Db extends EventEmitter {
     this.config = config
   }
 
+  events = {
+    error: 'error'
+  }
+
   async open () {
     this.pool = new pg.Pool(this.config)
     this.pool.on('error', error => this.emit('error', error))
@@ -26,15 +30,15 @@ class Db extends EventEmitter {
 
   async executeSql (text, values) {
     if (this.opened) {
-      // if (this.config.debug === true) {
-      //   console.log(`${new Date().toISOString()}: DEBUG SQL`)
-      //   console.log(text)
+      if (this.config.debug === true) {
+        console.log(`${new Date().toISOString()}: DEBUG SQL`)
+        console.log(text)
 
-      //   if (values) {
-      //     console.log(`${new Date().toISOString()}: DEBUG VALUES`)
-      //     console.log(values)
-      //   }
-      // }
+        if (values) {
+          console.log(`${new Date().toISOString()}: DEBUG VALUES`)
+          console.log(values)
+        }
+      }
 
       return await this.pool.query(text, values)
     }
@@ -68,14 +72,6 @@ class Db extends EventEmitter {
 
     return locker
   }
-
-  static quotePostgresStr (str) {
-    const delimeter = '$sanitize$'
-    if (str.includes(delimeter)) {
-      throw new Error(`Attempted to quote string that contains reserved Postgres delimeter: ${str}`)
-    }
-    return `${delimeter}${str}${delimeter}`
-  }
 }
 
 module.exports = Db

package/src/index.js

@@ -13,6 +13,15 @@ const events = {
   stopped: 'stopped'
 }
 class PgBoss extends EventEmitter {
+  #stoppingOn
+  #stopped
+  #config
+  #db
+  #boss
+  #contractor
+  #manager
+  #timekeeper
+
   static getConstructionPlans (schema) {
     return Contractor.constructionPlans(schema)
   }
@@ -26,60 +35,72 @@ class PgBoss extends EventEmitter {
   }
 
   constructor (value) {
-    const config = Attorney.getConfig(value)
-
     super()
 
-    const db = getDb(config)
+    this.#stoppingOn = null
+    this.#stopped = true
+
+    const config = Attorney.getConfig(value)
+    this.#config = config
+
+    const db = this.getDb()
+    this.#db = db
 
     if (db.isOurs) {
-      promoteEvent.call(this, db, 'error')
+      this.#promoteEvents(db)
     }
 
-    const manager = new Manager(db, config)
-    Object.keys(manager.events).forEach(event => promoteEvent.call(this, manager, manager.events[event]))
-    manager.functions.forEach(func => promoteFunction.call(this, manager, func))
+    const contractor = new Contractor(db, config)
 
+    const manager = new Manager(db, config)
     const bossConfig = { ...config, manager }
 
     const boss = new Boss(db, bossConfig)
-    Object.keys(boss.events).forEach(event => promoteEvent.call(this, boss, boss.events[event]))
-    boss.functions.forEach(func => promoteFunction.call(this, boss, func))
 
     const timekeeper = new Timekeeper(db, bossConfig)
-    Object.keys(timekeeper.events).forEach(event => promoteEvent.call(this, timekeeper, timekeeper.events[event]))
-    timekeeper.functions.forEach(func => promoteFunction.call(this, timekeeper, func))
-
     manager.timekeeper = timekeeper
 
-    this.stoppingOn = null
-    this.stopped = true
-    this.config = config
-    this.db = db
-    this.boss = boss
-    this.contractor = new Contractor(db, config)
-    this.manager = manager
-    this.timekeeper = timekeeper
-
-    function getDb (config) {
-      if (config.db) {
-        return config.db
-      }
+    this.#promoteEvents(manager)
+    this.#promoteEvents(boss)
+    this.#promoteEvents(timekeeper)
 
-      const db = new Db(config)
-      db.isOurs = true
-      return db
+    this.#promoteFunctions(boss)
+    this.#promoteFunctions(contractor)
+    this.#promoteFunctions(manager)
+    this.#promoteFunctions(timekeeper)
+
+    this.#boss = boss
+    this.#contractor = contractor
+    this.#manager = manager
+    this.#timekeeper = timekeeper
+  }
+
+  getDb () {
+    if (this.#db) {
+      return this.#db
     }
 
-    function promoteFunction (obj, func) {
-      this[func.name] = (...args) => func.apply(obj, args)
+    if (this.#config.db) {
+      return this.#config.db
     }
 
-    function promoteEvent (emitter, event) {
+    const db = new Db(this.#config)
+    db.isOurs = true
+    return db
+  }
+
+  #promoteEvents (emitter) {
+    for (const event of Object.values(emitter?.events)) {
       emitter.on(event, arg => this.emit(event, arg))
     }
   }
 
+  #promoteFunctions (obj) {
+    for (const func of obj?.functions) {
+      this[func.name] = (...args) => func.apply(obj, args)
+    }
+  }
+
   async start () {
     if (this.starting || this.started) {
       return
@@ -87,39 +108,39 @@ class PgBoss extends EventEmitter {
 
     this.starting = true
 
-    if (this.db.isOurs && !this.db.opened) {
-      await this.db.open()
+    if (this.#db.isOurs && !this.#db.opened) {
+      await this.#db.open()
     }
 
-    if (this.config.migrate) {
-      await this.contractor.start()
+    if (this.#config.migrate) {
+      await this.#contractor.start()
     } else {
-      await this.contractor.check()
+      await this.#contractor.check()
     }
 
-    this.manager.start()
+    this.#manager.start()
 
-    if (this.config.supervise) {
-      await this.boss.supervise()
+    if (this.#config.supervise) {
+      await this.#boss.supervise()
     }
 
-    if (this.config.monitorStateIntervalSeconds) {
-      await this.boss.monitor()
+    if (this.#config.monitorStateIntervalSeconds) {
+      await this.#boss.monitor()
     }
 
-    if (this.config.schedule) {
-      await this.timekeeper.start()
+    if (this.#config.schedule) {
+      await this.#timekeeper.start()
     }
 
     this.starting = false
     this.started = true
-    this.stopped = false
+    this.#stopped = false
 
     return this
   }
 
   async stop (options = {}) {
-    if (this.stoppingOn || this.stopped) {
+    if (this.#stoppingOn || this.#stopped) {
       return
     }
 
@@ -127,27 +148,27 @@ class PgBoss extends EventEmitter {
 
     timeout = Math.max(timeout, 1000)
 
-    this.stoppingOn = Date.now()
+    this.#stoppingOn = Date.now()
 
-    await this.manager.stop()
-    await this.timekeeper.stop()
-    await this.boss.stop()
+    await this.#manager.stop()
+    await this.#timekeeper.stop()
+    await this.#boss.stop()
 
     await new Promise((resolve, reject) => {
       const shutdown = async () => {
         try {
-          if (this.config.__test__throw_shutdown) {
-            throw new Error(this.config.__test__throw_shutdown)
+          if (this.#config.__test__throw_shutdown) {
+            throw new Error(this.#config.__test__throw_shutdown)
           }
 
-          await this.manager.failWip()
+          await this.#manager.failWip()
 
-          if (this.db.isOurs && this.db.opened && destroy) {
-            await this.db.close()
+          if (this.#db.isOurs && this.#db.opened && destroy) {
+            await this.#db.close()
           }
 
-          this.stopped = true
-          this.stoppingOn = null
+          this.#stopped = true
+          this.#stoppingOn = null
           this.started = false
 
           this.emit(events.stopped)
@@ -168,13 +189,13 @@ class PgBoss extends EventEmitter {
 
       setImmediate(async () => {
         try {
-          if (this.config.__test__throw_stop_monitor) {
-            throw new Error(this.config.__test__throw_stop_monitor)
+          if (this.#config.__test__throw_stop_monitor) {
+            throw new Error(this.#config.__test__throw_stop_monitor)
           }
 
-          const isWip = () => this.manager.getWipData({ includeInternal: false }).length > 0
+          const isWip = () => this.#manager.getWipData({ includeInternal: false }).length > 0
 
-          while ((Date.now() - this.stoppingOn) < timeout && isWip()) {
+          while ((Date.now() - this.#stoppingOn) < timeout && isWip()) {
             await delay(500)
           }
 

package/src/manager.js

@@ -8,7 +8,6 @@ const { delay } = require('./tools')
 const Attorney = require('./attorney')
 const Worker = require('./worker')
 const plans = require('./plans')
-const Db = require('./db')
 
 const { QUEUES: TIMEKEEPER_QUEUES } = require('./timekeeper')
 const { QUEUE_POLICY } = plans
@@ -107,10 +106,11 @@ class Manager extends EventEmitter {
   }
 
   async failWip () {
-    const jobIds = Array.from(this.workers.values()).flatMap(w => w.jobs.map(j => j.id))
-
-    if (jobIds.length) {
-      await this.fail(jobIds, 'pg-boss shut down while active')
+    for (const worker of this.workers.values()) {
+      const jobIds = worker.jobs.map(j => j.id)
+      if (jobIds.length) {
+        await this.fail(worker.name, jobIds, 'pg-boss shut down while active')
+      }
     }
   }
 
@@ -220,8 +220,8 @@ class Manager extends EventEmitter {
         const maxExpiration = jobs.reduce((acc, i) => Math.max(acc, i.expire_in_seconds), 0)
 
         await resolveWithinSeconds(Promise.all([callback(jobs)]), maxExpiration)
-          .then(() => this.complete(jobs.map(job => job.id)))
-          .catch(err => this.fail(jobs.map(job => job.id), err))
+          .then(() => this.complete(name, jobs.map(job => job.id)))
+          .catch(err => this.fail(name, jobs.map(job => job.id), err))
       } else {
         if (refill) {
           queueSize += jobs.length || 1
@@ -229,8 +229,8 @@ class Manager extends EventEmitter {
 
         const allTeamPromise = pMap(jobs, job =>
           resolveWithinSeconds(callback(job), job.expire_in_seconds)
-            .then(result => this.complete(job.id, result))
-            .catch(err => this.fail(job.id, err))
+            .then(result => this.complete(name, job.id, result))
+            .catch(err => this.fail(name, job.id, err))
             .then(() => refill ? onRefill() : null)
         , { concurrency: teamConcurrency }
         ).catch(() => {}) // allow promises & non-promises to live together in harmony
@@ -462,32 +462,15 @@ class Manager extends EventEmitter {
   }
 
   async fetch (name, batchSize, options = {}) {
-    const patternMatch = Attorney.queueNameHasPatternMatch(name)
     const values = Attorney.checkFetchArgs(name, batchSize, options)
     const db = options.db || this.db
-    const nextJobSql = this.nextJobCommand({ ...options, patternMatch })
+    const nextJobSql = this.nextJobCommand({ ...options })
     const statementValues = [values.name, batchSize || 1]
 
     let result
 
     try {
-      if (!options.db) {
-        // Prepare/format now and send multi-statement transaction
-        const fetchQuery = nextJobSql
-          .replace('$1', Db.quotePostgresStr(statementValues[0]))
-          .replace('$2', statementValues[1].toString())
-
-        // eslint-disable-next-line no-unused-vars
-        const [_begin, _setLocal, fetchResult, _commit] = await db.executeSql([
-          'BEGIN',
-          'SET LOCAL jit = OFF', // JIT can slow things down significantly
-          fetchQuery,
-          'COMMIT'
-        ].join(';\n'))
-        result = fetchResult
-      } else {
-        result = await db.executeSql(nextJobSql, statementValues)
-      }
+      result = await db.executeSql(nextJobSql, statementValues)
     } catch (err) {
       // errors from fetchquery should only be unique constraint violations
     }
@@ -529,37 +512,43 @@ class Manager extends EventEmitter {
     }
   }
 
-  async complete (id, data, options = {}) {
+  async complete (name, id, data, options = {}) {
+    assert(name, 'Missing queue name argument')
     const db = options.db || this.db
     const ids = this.mapCompletionIdArg(id, 'complete')
-    const result = await db.executeSql(this.completeJobsCommand, [ids, this.mapCompletionDataArg(data)])
+    const result = await db.executeSql(this.completeJobsCommand, [name, ids, this.mapCompletionDataArg(data)])
     return this.mapCompletionResponse(ids, result)
   }
 
-  async fail (id, data, options = {}) {
+  async fail (name, id, data, options = {}) {
+    assert(name, 'Missing queue name argument')
     const db = options.db || this.db
     const ids = this.mapCompletionIdArg(id, 'fail')
-    const result = await db.executeSql(this.failJobsByIdCommand, [ids, this.mapCompletionDataArg(data)])
+    const result = await db.executeSql(this.failJobsByIdCommand, [name, ids, this.mapCompletionDataArg(data)])
     return this.mapCompletionResponse(ids, result)
   }
 
-  async cancel (id, options = {}) {
+  async cancel (name, id, options = {}) {
+    assert(name, 'Missing queue name argument')
     const db = options.db || this.db
     const ids = this.mapCompletionIdArg(id, 'cancel')
-    const result = await db.executeSql(this.cancelJobsCommand, [ids])
+    const result = await db.executeSql(this.cancelJobsCommand, [name, ids])
     return this.mapCompletionResponse(ids, result)
   }
 
-  async resume (id, options = {}) {
+  async resume (name, id, options = {}) {
+    assert(name, 'Missing queue name argument')
     const db = options.db || this.db
     const ids = this.mapCompletionIdArg(id, 'resume')
-    const result = await db.executeSql(this.resumeJobsCommand, [ids])
+    const result = await db.executeSql(this.resumeJobsCommand, [name, ids])
     return this.mapCompletionResponse(ids, result)
   }
 
   async createQueue (name, options = {}) {
     assert(name, 'Missing queue name argument')
 
+    Attorney.assertQueueName(name)
+
     const { policy = QUEUE_POLICY.standard } = options
 
     assert(policy in QUEUE_POLICY, `${policy} is not a valid queue policy`)
@@ -573,7 +562,7 @@ class Manager extends EventEmitter {
       deadLetter
     } = Attorney.checkQueueArgs(name, options)
 
-    const paritionSql = plans.partitionCreateJobName(this.config.schema, name)
+    const paritionSql = plans.createPartition(this.config.schema, name)
 
     await this.db.executeSql(paritionSql)
 
@@ -659,8 +648,8 @@ class Manager extends EventEmitter {
     const result = await this.db.executeSql(queueSql, [name])
 
     if (result?.rows?.length) {
-      Attorney.assertPostgresObjectName(name)
-      const sql = plans.dropJobTablePartition(this.config.schema, name)
+      Attorney.assertQueueName(name)
+      const sql = plans.dropPartition(this.config.schema, name)
       await this.db.executeSql(sql)
     }
 
@@ -690,15 +679,15 @@ class Manager extends EventEmitter {
     return result ? parseFloat(result.rows[0].count) : null
   }
 
-  async getJobById (id, options = {}) {
+  async getJobById (queue, id, options = {}) {
     const db = options.db || this.db
-    const result1 = await db.executeSql(this.getJobByIdCommand, [id])
+    const result1 = await db.executeSql(this.getJobByIdCommand, [queue, id])
 
     if (result1 && result1.rows && result1.rows.length === 1) {
       return result1.rows[0]
     }
 
-    const result2 = await db.executeSql(this.getArchivedJobByIdCommand, [id])
+    const result2 = await db.executeSql(this.getArchivedJobByIdCommand, [queue, id])
 
     if (result2 && result2.rows && result2.rows.length === 1) {
       return result2.rows[0]

package/src/migrationStore.js

@@ -64,166 +64,5 @@ function migrate (value, version, migrations) {
 
 function getAll (schema) {
   return [
-    {
-      release: '10.0.0',
-      version: 21,
-      previous: 20,
-      install: [
-        `DROP INDEX ${schema}.job_singletonKey`,
-        `DROP INDEX ${schema}.job_singleton_queue`,
-        `DROP INDEX ${schema}.job_singletonOn`,
-        `DROP INDEX ${schema}.job_singletonKeyOn`,
-        `DROP INDEX ${schema}.job_fetch`,
-        `DROP INDEX ${schema}.job_name`,
-
-        `ALTER TABLE ${schema}.job ADD COLUMN deadletter text`,
-        `ALTER TABLE ${schema}.job ADD COLUMN policy text`,
-        `ALTER TABLE ${schema}.job DROP COLUMN on_complete`,
-
-        // update state enum
-        `ALTER TABLE ${schema}.job ALTER COLUMN state TYPE text`,
-        `ALTER TABLE ${schema}.job ALTER COLUMN state DROP DEFAULT`,
-        `ALTER TABLE ${schema}.archive ALTER COLUMN state TYPE text`,
-
-        `DROP TABLE IF EXISTS ${schema}.archive_backup`,
-        `ALTER TABLE ${schema}.archive RENAME to archive_backup`,
-        `ALTER INDEX ${schema}.archive_archivedon_idx RENAME to archive_backup_archivedon_idx`,
-
-        `DROP TYPE ${schema}.job_state`,
-        `CREATE TYPE ${schema}.job_state AS ENUM ('created','retry','active','completed','cancelled','failed')`,
-
-        `ALTER TABLE ${schema}.job ALTER COLUMN state TYPE ${schema}.job_state USING state::${schema}.job_state`,
-        `ALTER TABLE ${schema}.job ALTER COLUMN state SET DEFAULT 'created'::${schema}.job_state`,
-
-        `DELETE FROM ${schema}.job WHERE name LIKE '__pgboss__%'`,
-
-        // set up job partitioning
-        `ALTER TABLE ${schema}.job RENAME TO job_default`,
-        `ALTER TABLE ${schema}.job_default DROP CONSTRAINT IF EXISTS job_pkey`,
-
-        `CREATE TABLE ${schema}.job (
-          id uuid not null default gen_random_uuid(),
-          name text not null,
-          priority integer not null default(0),
-          data jsonb,
-          state ${schema}.job_state not null default('created'),
-          retryLimit integer not null default(0),
-          retryCount integer not null default(0),
-          retryDelay integer not null default(0),
-          retryBackoff boolean not null default false,
-          startAfter timestamp with time zone not null default now(),
-          startedOn timestamp with time zone,
-          singletonKey text,
-          singletonOn timestamp without time zone,
-          expireIn interval not null default interval '15 minutes',
-          createdOn timestamp with time zone not null default now(),
-          completedOn timestamp with time zone,
-          keepUntil timestamp with time zone NOT NULL default now() + interval '14 days',
-          output jsonb,
-          deadletter text,
-          policy text,
-          CONSTRAINT job_pkey PRIMARY KEY (name, id)
-        ) PARTITION BY LIST (name)`,
-
-        `ALTER TABLE ${schema}.job ATTACH PARTITION ${schema}.job_default DEFAULT`,
-
-        `CREATE TABLE ${schema}.archive (LIKE ${schema}.job)`,
-        `ALTER TABLE ${schema}.archive ADD CONSTRAINT archive_pkey PRIMARY KEY (name, id)`,
-        `ALTER TABLE ${schema}.archive ADD archivedOn timestamptz NOT NULL DEFAULT now()`,
-        `CREATE INDEX archive_archivedon_idx ON ${schema}.archive(archivedon)`,
-        `CREATE INDEX archive_name_idx ON ${schema}.archive(name)`,
-
-        `CREATE INDEX job_fetch ON ${schema}.job (name text_pattern_ops, startAfter) INCLUDE (priority, createdOn, id) WHERE state < 'active'`,
-        `CREATE INDEX job_name ON ${schema}.job (name text_pattern_ops)`,
-        `CREATE UNIQUE INDEX job_policy_short ON ${schema}.job (name) WHERE state = 'created' AND policy = 'short'`,
-        `CREATE UNIQUE INDEX job_policy_singleton ON ${schema}.job (name) WHERE state = 'active' AND policy = 'singleton'`,
-        `CREATE UNIQUE INDEX job_policy_stately ON ${schema}.job (name, state) WHERE state <= 'active' AND policy = 'stately'`,
-        `CREATE UNIQUE INDEX job_throttle_key ON ${schema}.job (name, singletonKey) WHERE state <= 'completed' AND singletonOn IS NULL`,
-        `CREATE UNIQUE INDEX job_throttle_on ON ${schema}.job (name, singletonOn, COALESCE(singletonKey, '')) WHERE state <= 'completed' AND singletonOn IS NOT NULL`,
-
-        `ALTER TABLE ${schema}.version ADD COLUMN monitored_on timestamp with time zone`,
-
-        `CREATE TABLE ${schema}.queue (
-          name text primary key,
-          policy text,
-          retry_limit int,
-          retry_delay int,
-          retry_backoff bool,
-          expire_seconds int,
-          retention_minutes int,
-          dead_letter text,
-          created_on timestamp with time zone not null default now()
-        )`
-      ],
-      uninstall: [
-        `DROP INDEX ${schema}.job_policy_stately`,
-        `DROP INDEX ${schema}.job_policy_short`,
-        `DROP INDEX ${schema}.job_policy_singleton`,
-        `DROP INDEX ${schema}.job_throttle_on`,
-        `DROP INDEX ${schema}.job_throttle_key`,
-        `DROP INDEX ${schema}.job_fetch`,
-        `DROP INDEX ${schema}.job_name`,
-        `ALTER TABLE ${schema}.job DETACH PARTITION ${schema}.job_default`,
-        `DROP TABLE ${schema}.job`,
-        `ALTER TABLE ${schema}.job_default RENAME TO job`,
-        `DROP TABLE IF EXISTS ${schema}.archive_backup`,
-        `DROP INDEX ${schema}.archive_name_idx`,
-        `ALTER TABLE ${schema}.job DROP COLUMN deadletter`,
-        `ALTER TABLE ${schema}.job DROP COLUMN policy`,
-        `ALTER TABLE ${schema}.job ALTER COLUMN state TYPE text`,
-        `ALTER TABLE ${schema}.job ALTER COLUMN state DROP DEFAULT`,
-        `ALTER TABLE ${schema}.archive ALTER COLUMN state TYPE text`,
-        `DROP TYPE ${schema}.job_state`,
-        `CREATE TYPE ${schema}.job_state AS ENUM ('created','retry','active','completed','expired','cancelled','failed')`,
-        `ALTER TABLE ${schema}.job ALTER COLUMN state TYPE ${schema}.job_state USING state::${schema}.job_state`,
-        `ALTER TABLE ${schema}.job ALTER COLUMN state SET DEFAULT 'created'::${schema}.job_state`,
-        `ALTER TABLE ${schema}.job ADD COLUMN on_complete bool NOT NULL DEFAULT false`,
-        `ALTER TABLE ${schema}.archive ALTER COLUMN state TYPE ${schema}.job_state USING state::${schema}.job_state`,
-        `ALTER TABLE ${schema}.archive DROP COLUMN policy`,
-        `ALTER TABLE ${schema}.archive DROP CONSTRAINT archive_pkey`,
-        `CREATE INDEX job_fetch ON ${schema}.job (name text_pattern_ops, startAfter) WHERE state < 'active'`,
-        `CREATE INDEX job_name ON ${schema}.job (name text_pattern_ops)`,
-        `CREATE UNIQUE INDEX job_singletonOn ON ${schema}.job (name, singletonOn) WHERE state < 'expired' AND singletonKey IS NULL`,
-        `CREATE UNIQUE INDEX job_singletonKeyOn ON ${schema}.job (name, singletonOn, singletonKey) WHERE state < 'expired'`,
-        `CREATE UNIQUE INDEX job_singletonKey ON ${schema}.job (name, singletonKey) WHERE state < 'completed' AND singletonOn IS NULL AND NOT singletonKey LIKE '\\_\\_pgboss\\_\\_singleton\\_queue%'`,
-        `CREATE UNIQUE INDEX job_singleton_queue ON ${schema}.job (name, singletonKey) WHERE state < 'active' AND singletonOn IS NULL AND singletonKey LIKE '\\_\\_pgboss\\_\\_singleton\\_queue%'`,
-        `DROP TABLE ${schema}.queue`,
-        `ALTER TABLE ${schema}.version DROP COLUMN monitored_on`
-      ]
-    },
-    {
-      release: '7.4.0',
-      version: 20,
-      previous: 19,
-      install: [
-        `DROP INDEX ${schema}.job_singletonKey`,
-        `DROP INDEX ${schema}.job_singleton_queue`,
-        `CREATE UNIQUE INDEX job_singletonKey ON ${schema}.job (name, singletonKey) WHERE state < 'completed' AND singletonOn IS NULL AND NOT singletonKey LIKE '\\_\\_pgboss\\_\\_singleton\\_queue%'`,
-        `CREATE UNIQUE INDEX job_singleton_queue ON ${schema}.job (name, singletonKey) WHERE state < 'active' AND singletonOn IS NULL AND singletonKey LIKE '\\_\\_pgboss\\_\\_singleton\\_queue%'`
-      ],
-      uninstall: [
-        `DROP INDEX ${schema}.job_singletonKey`,
-        `DROP INDEX ${schema}.job_singleton_queue`,
-        `CREATE UNIQUE INDEX job_singletonKey ON ${schema}.job (name, singletonKey) WHERE state < 'completed' AND singletonOn IS NULL AND NOT singletonKey = '__pgboss__singleton_queue'`,
-        `CREATE UNIQUE INDEX job_singleton_queue ON ${schema}.job (name, singletonKey) WHERE state < 'active' AND singletonOn IS NULL AND singletonKey = '__pgboss__singleton_queue'`
-      ]
-    },
-    {
-      release: '7.0.0',
-      version: 19,
-      previous: 18,
-      install: [
-        `CREATE TABLE ${schema}.subscription (
-          event text not null,
-          name text not null,
-          created_on timestamp with time zone not null default now(),
-          updated_on timestamp with time zone not null default now(),
-          PRIMARY KEY(event, name)
-        )`
-      ],
-      uninstall: [
-        `DROP TABLE ${schema}.subscription`
-      ]
-    }
   ]
 }

package/src/plans.js

@@ -46,8 +46,8 @@ module.exports = {
   countStates,
   createQueue,
   updateQueue,
-  partitionCreateJobName,
-  dropJobTablePartition,
+  createPartition,
+  dropPartition,
   deleteQueueRecords,
   getQueueByName,
   getQueueSize,
@@ -77,8 +77,6 @@ function create (schema, version) {
     createEnumJobState(schema),
 
     createTableJob(schema),
-    createTableJobDefault(schema),
-    attachPartitionJobDefault(schema),
     createIndexJobName(schema),
     createIndexJobFetch(schema),
     createIndexJobPolicyStately(schema),
@@ -92,13 +90,16 @@ function create (schema, version) {
     createColumnArchiveArchivedOn(schema),
     createIndexArchiveArchivedOn(schema),
     createIndexArchiveName(schema),
-    createArchiveBackupTable(schema),
 
     createTableVersion(schema),
     createTableQueue(schema),
     createTableSchedule(schema),
     createTableSubscription(schema),
 
+    getPartitionFunction(schema),
+    createPartitionFunction(schema),
+    dropPartitionFunction(schema),
+
     insertVersion(schema, version)
   ]
 
@@ -165,24 +166,53 @@ function createTableJob (schema) {
   `
 }
 
-function createTableJobDefault (schema) {
-  return `CREATE TABLE ${schema}.job_default (LIKE ${schema}.job INCLUDING DEFAULTS INCLUDING CONSTRAINTS)`
+function createPartition (schema, name) {
+  return `SELECT ${schema}.create_partition('${name}');`
 }
 
-function attachPartitionJobDefault (schema) {
-  return `ALTER TABLE ${schema}.job ATTACH PARTITION ${schema}.job_default DEFAULT`
+function getPartitionFunction (schema) {
+  return `
+    CREATE FUNCTION ${schema}.get_partition(queue_name text, out name text) AS
+    $$
+    SELECT '${schema}.j' || encode(digest(queue_name, 'sha1'), 'hex');
+    $$
+    LANGUAGE SQL
+    IMMUTABLE
+  `
 }
 
-function partitionCreateJobName (schema, name) {
+function createPartitionFunction (schema) {
   return `
-    CREATE TABLE ${schema}.job_${name} (LIKE ${schema}.job INCLUDING DEFAULTS INCLUDING CONSTRAINTS);
-    ALTER TABLE ${schema}.job_${name} ADD CONSTRAINT job_check_${name} CHECK (name='${name}');
-    ALTER TABLE ${schema}.job ATTACH PARTITION ${schema}.job_${name} FOR VALUES IN ('${name}');
+    CREATE FUNCTION ${schema}.create_partition(queue_name text)
+    RETURNS VOID AS
+    $$
+    DECLARE
+      table_name varchar := ${schema}.get_partition(queue_name);
+    BEGIN
+      EXECUTE format('CREATE TABLE %I (LIKE ${schema}.job INCLUDING DEFAULTS INCLUDING CONSTRAINTS)', table_name);
+      EXECUTE format('ALTER TABLE %I ADD CHECK (name=%L)', table_name, queue_name);
+      EXECUTE format('ALTER TABLE ${schema}.job ATTACH PARTITION %I FOR VALUES IN (%L)', table_name, queue_name);
+    END;
+    $$
+    LANGUAGE plpgsql;
   `
 }
 
-function dropJobTablePartition (schema, name) {
-  return `DROP TABLE IF EXISTS ${schema}.job_${name}`
+function dropPartitionFunction (schema) {
+  return `
+    CREATE FUNCTION ${schema}.drop_partition(queue_name text)
+    RETURNS VOID AS
+    $$
+    BEGIN  
+      EXECUTE format('DROP TABLE IF EXISTS %I', ${schema}.get_partition(queue_name));
+    END;
+    $$
+    LANGUAGE plpgsql;
+  `
+}
+
+function dropPartition (schema, name) {
+  return `SELECT ${schema}.drop_partition('${name}');`
 }
 
 function createPrimaryKeyArchive (schema) {
@@ -210,21 +240,17 @@ function createIndexJobThrottleKey (schema) {
 }
 
 function createIndexJobName (schema) {
-  return `CREATE INDEX job_name ON ${schema}.job (name text_pattern_ops)`
+  return `CREATE INDEX job_name ON ${schema}.job (name)`
 }
 
 function createIndexJobFetch (schema) {
-  return `CREATE INDEX job_fetch ON ${schema}.job (name text_pattern_ops, startAfter) INCLUDE (priority, createdOn, id) WHERE state < '${states.active}'`
+  return `CREATE INDEX job_fetch ON ${schema}.job (name, startAfter) INCLUDE (priority, createdOn, id) WHERE state < '${states.active}'`
 }
 
 function createTableArchive (schema) {
   return `CREATE TABLE ${schema}.archive (LIKE ${schema}.job)`
 }
 
-function createArchiveBackupTable (schema) {
-  return `CREATE TABLE ${schema}.archive_backup (LIKE ${schema}.job)`
-}
-
 function createColumnArchiveArchivedOn (schema) {
   return `ALTER TABLE ${schema}.archive ADD archivedOn timestamptz NOT NULL DEFAULT now()`
 }
@@ -422,12 +448,12 @@ function insertVersion (schema, version) {
 }
 
 function fetchNextJob (schema) {
-  return ({ includeMetadata, patternMatch, priority = true } = {}) => `
+  return ({ includeMetadata, priority = true } = {}) => `
     WITH next as (
       SELECT id
       FROM ${schema}.job
-      WHERE state < '${states.active}'
-        AND name ${patternMatch ? 'LIKE' : '='} $1
+      WHERE name = $1
+        AND state < '${states.active}'
         AND startAfter < now()
       ORDER BY ${priority && 'priority desc, '} createdOn, id
       LIMIT $2
@@ -438,7 +464,7 @@ function fetchNextJob (schema) {
       startedOn = now(),
       retryCount = CASE WHEN startedOn IS NOT NULL THEN retryCount + 1 ELSE retryCount END
     FROM next
-    WHERE j.id = next.id
+    WHERE name = $1 AND j.id = next.id
     RETURNING ${includeMetadata ? 'j.*' : 'j.id, name, data'}, 
       EXTRACT(epoch FROM expireIn) as expire_in_seconds
   `
@@ -450,8 +476,9 @@ function completeJobs (schema) {
       UPDATE ${schema}.job
       SET completedOn = now(),
         state = '${states.completed}',
-        output = $2::jsonb
-      WHERE id IN (SELECT UNNEST($1::uuid[]))
+        output = $3::jsonb
+      WHERE name = $1
+        AND id IN (SELECT UNNEST($2::uuid[]))
         AND state = '${states.active}'
       RETURNING *
     )
@@ -460,8 +487,8 @@ function completeJobs (schema) {
 }
 
 function failJobsById (schema) {
-  const where = `id IN (SELECT UNNEST($1::uuid[])) AND state < '${states.completed}'`
-  const output = '$2::jsonb'
+  const where = `name = $1 AND id IN (SELECT UNNEST($2::uuid[])) AND state < '${states.completed}'`
+  const output = '$3::jsonb'
 
   return failJobs(schema, where, output)
 }
@@ -518,7 +545,8 @@ function cancelJobs (schema) {
       UPDATE ${schema}.job
       SET completedOn = now(),
         state = '${states.cancelled}'
-      WHERE id IN (SELECT UNNEST($1::uuid[]))
+      WHERE name = $1
+        AND id IN (SELECT UNNEST($2::uuid[]))
         AND state < '${states.completed}'
       RETURNING 1
     )
@@ -532,7 +560,8 @@ function resumeJobs (schema) {
       UPDATE ${schema}.job
       SET completedOn = NULL,
         state = '${states.created}'
-      WHERE id IN (SELECT UNNEST($1::uuid[]))
+      WHERE name = $1
+        AND id IN (SELECT UNNEST($2::uuid[]))
       RETURNING 1
     )
     SELECT COUNT(*) from results
@@ -735,8 +764,8 @@ function locked (schema, query) {
 }
 
 function advisoryLock (schema, key) {
-  return `SELECT pg_advisory_xact_lock(
-      ('x' || md5(current_database() || '.pgboss.${schema}${key || ''}'))::bit(64)::bigint
+  return `SELECT pg_advisory_xact_lock(      
+      ('x' || encode(digest(current_database() || '.pgboss.${schema}${key || ''}', 'sha256'), 'hex'))::bit(64)::bigint
   )`
 }
 
@@ -754,5 +783,5 @@ function getArchivedJobById (schema) {
 }
 
 function getJobByTableAndId (schema, table) {
-  return `SELECT * FROM ${schema}.${table} WHERE id = $1`
+  return `SELECT * FROM ${schema}.${table} WHERE name = $1 AND id = $2`
 }

package/src/timekeeper.js

@@ -52,6 +52,9 @@ class Timekeeper extends EventEmitter {
     // cache the clock skew from the db server
     await this.cacheClockSkew()
 
+    await this.manager.createQueue(queues.CRON)
+    await this.manager.createQueue(queues.SEND_IT)
+
     await this.manager.work(queues.CRON, { newJobCheckIntervalSeconds: this.config.cronWorkerIntervalSeconds }, (job) => this.onCron(job))
     await this.manager.work(queues.SEND_IT, { newJobCheckIntervalSeconds: this.config.cronWorkerIntervalSeconds, teamSize: 50, teamConcurrency: 5 }, (job) => this.onSendIt(job))
 

package/src/worker.js

@@ -74,7 +74,7 @@ class Worker {
 
       this.lastJobDuration = duration
 
-      if (!this.stopping && !this.beenNotified && duration < this.interval) {
+      if (!this.stopping && !this.beenNotified && (this.interval - duration > 500)) {
         this.loopDelayPromise = delay(this.interval - duration)
         await this.loopDelayPromise
         this.loopDelayPromise = null

package/types.d.ts

@@ -331,24 +331,24 @@ declare class PgBoss extends EventEmitter {
   fetch<T>(name: string, batchSize: number, options: PgBoss.FetchOptions & { includeMetadata: true }): Promise<PgBoss.JobWithMetadata<T>[] | null>;
   fetch<T>(name: string, batchSize: number, options: PgBoss.FetchOptions): Promise<PgBoss.Job<T>[] | null>;
 
-  cancel(id: string, options?: PgBoss.ConnectionOptions): Promise<void>;
-  cancel(ids: string[], options?: PgBoss.ConnectionOptions): Promise<void>;
+  cancel(name: string, id: string, options?: PgBoss.ConnectionOptions): Promise<void>;
+  cancel(name: string, ids: string[], options?: PgBoss.ConnectionOptions): Promise<void>;
 
-  resume(id: string, options?: PgBoss.ConnectionOptions): Promise<void>;
-  resume(ids: string[], options?: PgBoss.ConnectionOptions): Promise<void>;
+  resume(name: string, id: string, options?: PgBoss.ConnectionOptions): Promise<void>;
+  resume(name: string, ids: string[], options?: PgBoss.ConnectionOptions): Promise<void>;
 
-  complete(id: string, options?: PgBoss.ConnectionOptions): Promise<void>;
-  complete(id: string, data: object, options?: PgBoss.ConnectionOptions): Promise<void>;
-  complete(ids: string[], options?: PgBoss.ConnectionOptions): Promise<void>;
+  complete(name: string, id: string, options?: PgBoss.ConnectionOptions): Promise<void>;
+  complete(name: string, id: string, data: object, options?: PgBoss.ConnectionOptions): Promise<void>;
+  complete(name: string, ids: string[], options?: PgBoss.ConnectionOptions): Promise<void>;
 
-  fail(id: string, options?: PgBoss.ConnectionOptions): Promise<void>;
-  fail(id: string, data: object, options?: PgBoss.ConnectionOptions): Promise<void>;
-  fail(ids: string[], options?: PgBoss.ConnectionOptions): Promise<void>;
+  fail(name: string, id: string, options?: PgBoss.ConnectionOptions): Promise<void>;
+  fail(name: string, id: string, data: object, options?: PgBoss.ConnectionOptions): Promise<void>;
+  fail(name: string, ids: string[], options?: PgBoss.ConnectionOptions): Promise<void>;
 
   getQueueSize(name: string, options?: object): Promise<number>;
-  getJobById(id: string, options?: PgBoss.ConnectionOptions): Promise<PgBoss.JobWithMetadata | null>;
+  getJobById(name: string, id: string, options?: PgBoss.ConnectionOptions): Promise<PgBoss.JobWithMetadata | null>;
 
-  createQueue(name: string, policy: 'standard' | 'short' | 'singleton' | 'stately'): Promise<void>;
+  createQueue(name: string, options?: { policy: 'standard' | 'short' | 'singleton' | 'stately' }): Promise<void>;
   deleteQueue(name: string): Promise<void>;
   purgeQueue(name: string): Promise<void>;
   clearStorage(): Promise<void>;