pg-boss 10.0.0-beta3 → 10.0.0-beta5

package/README.md

@@ -9,24 +9,19 @@ async function readme() {
   const PgBoss = require('pg-boss');
   const boss = new PgBoss('postgres://user:pass@host/database');
 
-  boss.on('error', error => console.error(error));
+  boss.on('error', console.error)
 
-  await boss.start();
+  await boss.start()
 
-  const queue = 'some-queue';
+  const queue = 'readme-queue'
 
-  let jobId = await boss.send(queue, { param1: 'foo' })
+  const id = await boss.send(queue, { arg1: 'read me' })
 
-  console.log(`created job in queue ${queue}: ${jobId}`);
+  console.log(`created job ${id} in queue ${queue}`)
 
-  await boss.work(queue, someAsyncJobHandler);
-}
-
-async function someAsyncJobHandler(job) {
-  console.log(`job ${job.id} received with data:`);
-  console.log(JSON.stringify(job.data));
-
-  await doSomethingAsyncWithThis(job.data);
+  await boss.work(queue, async job => {
+    console.log(`received job ${job.id} with data ${JSON.stringify(job.data)}`)
+  })
 }
 ```
 
@@ -48,7 +43,7 @@ This will likely cater the most to teams already familiar with the simplicity of
 
 ## Requirements
 * Node 20 or higher
-* PostgreSQL 12 or higher
+* PostgreSQL 13 or higher
 
 ## Installation
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pg-boss",
-  "version": "10.0.0-beta3",
+  "version": "10.0.0-beta5",
   "description": "Queueing jobs in Postgres from Node.js like a boss",
   "main": "./src/index.js",
   "engines": {

package/src/attorney.js

@@ -368,21 +368,21 @@ function applyMonitoringConfig (config) {
           ? config.clockMonitorIntervalSeconds
           : TEN_MINUTES_IN_SECONDS
 
-  assert(!('cronMonitorIntervalSeconds' in config) || (config.cronMonitorIntervalSeconds >= 1 && config.cronMonitorIntervalSeconds <= 60),
-    'configuration assert: cronMonitorIntervalSeconds must be between 1 and 60 seconds')
+  assert(!('cronMonitorIntervalSeconds' in config) || (config.cronMonitorIntervalSeconds >= 1 && config.cronMonitorIntervalSeconds <= 45),
+    'configuration assert: cronMonitorIntervalSeconds must be between 1 and 45 seconds')
 
   config.cronMonitorIntervalSeconds =
     ('cronMonitorIntervalSeconds' in config)
       ? config.cronMonitorIntervalSeconds
-      : 60
+      : 30
 
-  assert(!('cronWorkerIntervalSeconds' in config) || (config.cronWorkerIntervalSeconds >= 1 && config.cronWorkerIntervalSeconds <= 60),
-    'configuration assert: cronWorkerIntervalSeconds must be between 1 and 60 seconds')
+  assert(!('cronWorkerIntervalSeconds' in config) || (config.cronWorkerIntervalSeconds >= 1 && config.cronWorkerIntervalSeconds <= 45),
+    'configuration assert: cronWorkerIntervalSeconds must be between 1 and 45 seconds')
 
   config.cronWorkerIntervalSeconds =
     ('cronWorkerIntervalSeconds' in config)
       ? config.cronWorkerIntervalSeconds
-      : 4
+      : 5
 }
 
 function warnClockSkew (message) {

package/src/boss.js

@@ -77,10 +77,7 @@ class Boss extends EventEmitter {
     } catch (err) {
       this.emit(events.error, err)
     } finally {
-      if (locker?.locked) {
-        await locker.unlock()
-      }
-
+      await locker?.unlock()
       this.monitoring = false
     }
   }
@@ -119,10 +116,7 @@ class Boss extends EventEmitter {
     } catch (err) {
       this.emit(events.error, err)
     } finally {
-      if (locker?.locked) {
-        await locker.unlock()
-      }
-
+      await locker?.unlock()
       this.maintaining = false
     }
   }

package/src/db.js

@@ -45,27 +45,23 @@ class Db extends EventEmitter {
   }
 
   async lock ({ timeout = 30, key } = {}) {
-    // const lockedClient = new pg.Client(this.config)
-    // await lockedClient.connect()
     const lockedClient = await this.pool.connect()
 
     const query = `
         BEGIN;
         SET LOCAL lock_timeout = '${timeout}s';
-        SET LOCAL idle_in_transaction_session_timeout = '3600s';
+        SET LOCAL idle_in_transaction_session_timeout = '${timeout}s';
         ${advisoryLock(this.config.schema, key)};
     `
 
     await lockedClient.query(query)
 
     const locker = {
-      locked: true,
       unlock: async function () {
         try {
           await lockedClient.query('COMMIT')
-          await lockedClient.end()
         } finally {
-          this.locked = false
+          lockedClient.release()
         }
       }
     }

package/src/index.js

@@ -15,6 +15,8 @@ const events = {
 class PgBoss extends EventEmitter {
   #stoppingOn
   #stopped
+  #starting
+  #started
   #config
   #db
   #boss
@@ -102,11 +104,11 @@ class PgBoss extends EventEmitter {
   }
 
   async start () {
-    if (this.starting || this.started) {
+    if (this.#starting || this.#started) {
       return
     }
 
-    this.starting = true
+    this.#starting = true
 
     if (this.#db.isOurs && !this.#db.opened) {
       await this.#db.open()
@@ -132,8 +134,8 @@ class PgBoss extends EventEmitter {
       await this.#timekeeper.start()
     }
 
-    this.starting = false
-    this.started = true
+    this.#starting = false
+    this.#started = true
     this.#stopped = false
 
     return this
@@ -169,7 +171,7 @@ class PgBoss extends EventEmitter {
 
           this.#stopped = true
           this.#stoppingOn = null
-          this.started = false
+          this.#started = false
 
           this.emit(events.stopped)
           resolve()

package/src/plans.js

@@ -174,7 +174,7 @@ function getPartitionFunction (schema) {
   return `
     CREATE FUNCTION ${schema}.get_partition(queue_name text, out name text) AS
     $$
-    SELECT '${schema}.j' || encode(digest(queue_name, 'sha1'), 'hex');
+    SELECT '${schema}.job_' || encode(sha224(queue_name::bytea), 'hex');
     $$
     LANGUAGE SQL
     IMMUTABLE
@@ -315,6 +315,8 @@ function getQueueByName (schema) {
 function deleteQueueRecords (schema) {
   return `WITH dq AS (
       DELETE FROM ${schema}.queue WHERE name = $1
+    ), ds AS (
+      DELETE FROM ${schema}.schedule WHERE name = $1
     )
     DELETE FROM ${schema}.job WHERE name = $1
   `
@@ -378,7 +380,9 @@ function createTableSubscription (schema) {
 
 function getSchedules (schema) {
   return `
-    SELECT * FROM ${schema}.schedule
+    SELECT s.*
+    FROM ${schema}.schedule s
+      JOIN ${schema}.queue q on s.name = q.name
   `
 }
 
@@ -765,7 +769,7 @@ function locked (schema, query) {
 
 function advisoryLock (schema, key) {
   return `SELECT pg_advisory_xact_lock(      
-      ('x' || encode(digest(current_database() || '.pgboss.${schema}${key || ''}', 'sha256'), 'hex'))::bit(64)::bigint
+      ('x' || encode(sha224((current_database() || '.pgboss.${schema}${key || ''}')::bytea), 'hex'))::bit(64)::bigint
   )`
 }
 

package/src/timekeeper.js

@@ -5,7 +5,6 @@ const Attorney = require('./attorney')
 const pMap = require('p-map')
 
 const queues = {
-  CRON: '__pgboss__cron',
   SEND_IT: '__pgboss__send-it'
 }
 
@@ -28,6 +27,7 @@ class Timekeeper extends EventEmitter {
     this.events = events
 
     this.getTimeCommand = plans.getTime(config.schema)
+    this.getQueueCommand = plans.getQueueByName(config.schema)
     this.getSchedulesCommand = plans.getSchedules(config.schema)
     this.scheduleCommand = plans.schedule(config.schema)
     this.unscheduleCommand = plans.unschedule(config.schema)
@@ -44,6 +44,8 @@ class Timekeeper extends EventEmitter {
   }
 
   async start () {
+    this.stopped = false
+
     // setting the archive config too low breaks the cron 60s debounce interval so don't even try
     if (this.config.archiveSeconds < 60 || this.config.archiveFailedSeconds < 60) {
       return
@@ -52,21 +54,18 @@ class Timekeeper extends EventEmitter {
     // cache the clock skew from the db server
     await this.cacheClockSkew()
 
-    await this.manager.createQueue(queues.CRON)
-    await this.manager.createQueue(queues.SEND_IT)
+    try {
+      await this.manager.createQueue(queues.SEND_IT)
+    } catch {}
 
-    await this.manager.work(queues.CRON, { newJobCheckIntervalSeconds: this.config.cronWorkerIntervalSeconds }, (job) => this.onCron(job))
     await this.manager.work(queues.SEND_IT, { newJobCheckIntervalSeconds: this.config.cronWorkerIntervalSeconds, teamSize: 50, teamConcurrency: 5 }, (job) => this.onSendIt(job))
 
-    // uses sendDebounced() to enqueue a cron check
-    await this.checkSchedulesAsync()
+    setImmediate(() => this.onCron())
 
     // create monitoring interval to make sure cron hasn't crashed
-    this.cronMonitorInterval = setInterval(async () => await this.monitorCron(), this.cronMonitorIntervalMs)
+    this.cronMonitorInterval = setInterval(async () => await this.onCron(), this.cronMonitorIntervalMs)
     // create monitoring interval to measure and adjust for drift in clock skew
     this.skewMonitorInterval = setInterval(async () => await this.cacheClockSkew(), this.skewMonitorIntervalMs)
-
-    this.stopped = false
   }
 
   async stop () {
@@ -76,7 +75,6 @@ class Timekeeper extends EventEmitter {
 
     this.stopped = true
 
-    await this.manager.offWork(queues.CRON)
     await this.manager.offWork(queues.SEND_IT)
 
     if (this.skewMonitorInterval) {
@@ -90,22 +88,6 @@ class Timekeeper extends EventEmitter {
     }
   }
 
-  async monitorCron () {
-    try {
-      if (this.config.__test__force_cron_monitoring_error) {
-        throw new Error(this.config.__test__force_cron_monitoring_error)
-      }
-
-      const { secondsAgo } = await this.getCronTime()
-
-      if (secondsAgo > 60) {
-        await this.checkSchedulesAsync()
-      }
-    } catch (err) {
-      this.emit(this.events.error, err)
-    }
-  }
-
   async cacheClockSkew () {
     let skew = 0
 
@@ -134,43 +116,42 @@ class Timekeeper extends EventEmitter {
     }
   }
 
-  async checkSchedulesAsync () {
-    const opts = {
-      retryLimit: 2,
-      retentionSeconds: 60
-    }
-
-    await this.manager.sendDebounced(queues.CRON, null, opts, 60)
-  }
-
   async onCron () {
-    if (this.stopped) return
+    let locker
 
     try {
-      if (this.config.__test__throw_cron_processing) {
-        throw new Error(this.config.__test__throw_cron_processing)
-      }
+      if (this.stopped || this.timekeeping) return
 
-      const items = await this.getSchedules()
+      if (this.config.__test__force_cron_monitoring_error) {
+        throw new Error(this.config.__test__force_cron_monitoring_error)
+      }
 
-      const sending = items.filter(i => this.shouldSendIt(i.cron, i.timezone))
+      this.timekeeping = true
 
-      if (sending.length && !this.stopped) {
-        await pMap(sending, it => this.send(it), { concurrency: 5 })
-      }
+      locker = await this.db.lock({ key: 'timekeeper' })
 
-      if (this.stopped) return
+      const { secondsAgo } = await this.getCronTime()
 
-      // set last time cron was evaluated for downstream usage in cron monitoring
-      await this.setCronTime()
+      if (secondsAgo > this.config.cronMonitorIntervalSeconds) {
+        await this.cron()
+        await this.setCronTime()
+      }
     } catch (err) {
       this.emit(this.events.error, err)
+    } finally {
+      this.timekeeping = false
+      await locker?.unlock()
     }
+  }
 
-    if (this.stopped) return
+  async cron () {
+    const items = await this.getSchedules()
 
-    // uses sendDebounced() to enqueue a cron check
-    await this.checkSchedulesAsync()
+    const sending = items.filter(i => this.shouldSendIt(i.cron, i.timezone))
+
+    if (sending.length && !this.stopped) {
+      await pMap(sending, it => this.send(it), { concurrency: 5 })
+    }
   }
 
   shouldSendIt (cron, tz) {
@@ -186,12 +167,7 @@ class Timekeeper extends EventEmitter {
   }
 
   async send (job) {
-    const options = {
-      singletonKey: job.name,
-      singletonSeconds: 60
-    }
-
-    await this.manager.send(queues.SEND_IT, job, options)
+    await this.manager.send(queues.SEND_IT, job, { singletonKey: job.name, singletonSeconds: 60 })
   }
 
   async onSendIt (job) {
@@ -213,6 +189,13 @@ class Timekeeper extends EventEmitter {
     // validation pre-check
     Attorney.checkSendArgs([name, data, options], this.config)
 
+    // make sure queue exists before scheduling
+    const queue = await this.db.executeSql(this.getQueueCommand, [name])
+
+    if (!queue.rows.length === 0) {
+      throw new Error(`Queue '${name}' not found`)
+    }
+
     const values = [name, cron, tz, data, options]
 
     const result = await this.db.executeSql(this.scheduleCommand, values)