pg-boss 10.3.2 → 11.0.0

package/src/boss.js

@@ -1,175 +1,158 @@
 const EventEmitter = require('node:events')
 const plans = require('./plans')
-const { delay } = require('./tools')
 
 const events = {
   error: 'error',
-  monitorStates: 'monitor-states',
-  maintenance: 'maintenance'
+  warning: 'warning'
+}
+
+const WARNINGS = {
+  SLOW_QUERY: { seconds: 30, message: 'Warning: slow query. Your queues and/or database server should be reviewed' },
+  LARGE_QUEUE: { size: 10_000, mesasge: 'Warning: large queue backlog. Your queue should be reviewed' }
 }
 
 class Boss extends EventEmitter {
+  #stopped
+  #maintaining
+  #superviseInterval
+  #db
+  #config
+  #manager
+
   constructor (db, config) {
     super()
 
-    this.db = db
-    this.config = config
-    this.manager = config.manager
-
-    this.maintenanceIntervalSeconds = config.maintenanceIntervalSeconds
-    this.monitorStateIntervalSeconds = config.monitorStateIntervalSeconds
+    this.#db = db
+    this.#config = config
+    this.#manager = config.manager
+    this.#stopped = true
 
     this.events = events
-
-    this.failJobsByTimeoutCommand = plans.locked(config.schema, plans.failJobsByTimeout(config.schema))
-    this.archiveCommand = plans.locked(config.schema, plans.archive(config.schema, config.archiveInterval, config.archiveFailedInterval))
-    this.dropCommand = plans.locked(config.schema, plans.drop(config.schema, config.deleteAfter))
-    this.trySetMaintenanceTimeCommand = plans.trySetMaintenanceTime(config.schema)
-    this.trySetMonitorTimeCommand = plans.trySetMonitorTime(config.schema)
-    this.countStatesCommand = plans.countStates(config.schema)
-
     this.functions = [
-      this.expire,
-      this.archive,
-      this.drop,
-      this.countStates,
-      this.maintain
+      this.supervise
     ]
-  }
 
-  async supervise () {
-    this.maintenanceInterval = setInterval(() => this.onSupervise(), this.maintenanceIntervalSeconds * 1000)
-  }
+    if (config.warningSlowQuerySeconds) {
+      WARNINGS.SLOW_QUERY.seconds = config.warningSlowQuerySeconds
+    }
 
-  async monitor () {
-    this.monitorInterval = setInterval(() => this.onMonitor(), this.monitorStateIntervalSeconds * 1000)
+    if (config.warningQueueSize) {
+      WARNINGS.LARGE_QUEUE.size = config.warningQueueSize
+    }
   }
 
-  async onMonitor () {
-    try {
-      if (this.monitoring) {
-        return
-      }
+  async start () {
+    if (this.#stopped) {
+      this.#superviseInterval = setInterval(() => this.#onSupervise(), this.#config.superviseIntervalSeconds * 1000)
+      this.#stopped = false
+    }
+  }
 
-      this.monitoring = true
+  async stop () {
+    if (!this.#stopped) {
+      if (this.#superviseInterval) clearInterval(this.#superviseInterval)
+      this.#stopped = true
+    }
+  }
 
-      if (this.config.__test__delay_monitor) {
-        await delay(this.config.__test__delay_monitor)
-      }
+  async #executeSql (sql, values) {
+    const started = Date.now()
 
-      if (this.config.__test__throw_monitor) {
-        throw new Error(this.config.__test__throw_monitor)
-      }
+    const result = await this.#db.executeSql(sql, values)
 
-      if (this.stopped) {
-        return
-      }
+    const ended = Date.now()
 
-      const { rows } = await this.db.executeSql(this.trySetMonitorTimeCommand, [this.config.monitorStateIntervalSeconds])
+    const elapsed = (ended - started) / 1000
 
-      if (rows.length === 1 && !this.stopped) {
-        const states = await this.countStates()
-        this.emit(events.monitorStates, states)
-      }
-    } catch (err) {
-      this.emit(events.error, err)
-    } finally {
-      this.monitoring = false
+    if (elapsed > WARNINGS.SLOW_QUERY.seconds || this.#config.__test__warn_slow_query) {
+      this.emit(events.warning, { message: WARNINGS.SLOW_QUERY.message, data: { elapsed, sql, values } })
     }
+
+    return result
   }
 
-  async onSupervise () {
+  async #onSupervise () {
     try {
-      if (this.maintaining) {
-        return
-      }
+      if (this.#stopped) return
+      if (this.#maintaining) return
+      if (this.#config.__test__throw_maint) throw new Error(this.#config.__test__throw_maint)
 
-      this.maintaining = true
+      this.#maintaining = true
 
-      if (this.config.__test__delay_maintenance && !this.stopped) {
-        this.__testDelayPromise = delay(this.config.__test__delay_maintenance)
-        await this.__testDelayPromise
-      }
+      const queues = await this.#manager.getQueues()
 
-      if (this.config.__test__throw_maint) {
-        throw new Error(this.config.__test__throw_maint)
-      }
-
-      if (this.stopped) {
-        return
-      }
-
-      const { rows } = await this.db.executeSql(this.trySetMaintenanceTimeCommand, [this.config.maintenanceIntervalSeconds])
-
-      if (rows.length === 1 && !this.stopped) {
-        const result = await this.maintain()
-        this.emit(events.maintenance, result)
+      for (const queue of queues) {
+        !this.#stopped && await this.supervise(queue)
       }
     } catch (err) {
       this.emit(events.error, err)
     } finally {
-      this.maintaining = false
+      this.#maintaining = false
     }
   }
 
-  async maintain () {
-    const started = Date.now()
+  async supervise (value) {
+    let queues
 
-    !this.stopped && await this.expire()
-    !this.stopped && await this.archive()
-    !this.stopped && await this.drop()
+    if (!value) {
+      queues = await this.#manager.getQueues()
+    } if (typeof value === 'string') {
+      queues = await this.#manager.getQueues(value)
+    } else if (typeof value === 'object') {
+      queues = [value]
+    }
 
-    const ended = Date.now()
+    const queueGroups = queues.reduce((acc, q) => {
+      const { table } = q
+      acc[table] = acc[table] || { table, queues: [] }
+      acc[table].queues.push(q)
+      return acc
+    }, {})
 
-    return { ms: ended - started }
-  }
+    for (const queueGroup of Object.values(queueGroups)) {
+      const { table, queues } = queueGroup
+      const names = queues.map(i => i.name)
 
-  async stop () {
-    if (!this.stopped) {
-      if (this.__testDelayPromise) this.__testDelayPromise.abort()
-      if (this.maintenanceInterval) clearInterval(this.maintenanceInterval)
-      if (this.monitorInterval) clearInterval(this.monitorInterval)
+      while (names.length) {
+        const chunk = names.splice(0, 100)
 
-      this.stopped = true
+        await this.#monitor(table, chunk)
+        await this.#maintain(table, chunk)
+      }
     }
   }
 
-  async countStates () {
-    const stateCountDefault = { ...plans.JOB_STATES }
+  async #monitor (table, names) {
+    const command = plans.trySetQueueMonitorTime(this.#config.schema, names, this.#config.monitorIntervalSeconds)
+    const { rows } = await this.#executeSql(command)
 
-    for (const key of Object.keys(stateCountDefault)) {
-      stateCountDefault[key] = 0
-    }
+    if (rows.length) {
+      const queues = rows.map(q => q.name)
 
-    const counts = await this.db.executeSql(this.countStatesCommand)
+      const cacheStatsSql = plans.cacheQueueStats(this.#config.schema, table, queues)
+      const results = await this.#executeSql(cacheStatsSql)
 
-    const states = counts.rows.reduce((acc, item) => {
-      if (item.name) {
-        acc.queues[item.name] = acc.queues[item.name] || { ...stateCountDefault }
-      }
+      const inter = results.flatMap(i => i.rows)
+      const warnings = inter.filter(i => i.queuedCount > (i.warningQueueSize || WARNINGS.LARGE_QUEUE.size))
 
-      const queue = item.name ? acc.queues[item.name] : acc
-      const state = item.state || 'all'
-
-      // parsing int64 since pg returns it as string
-      queue[state] = parseFloat(item.size)
-
-      return acc
-    }, { ...stateCountDefault, queues: {} })
-
-    return states
-  }
+      for (const warning of warnings) {
+        this.emit(events.warning, { message: WARNINGS.LARGE_QUEUE.mesasge, data: warning })
+      }
 
-  async expire () {
-    await this.db.executeSql(this.failJobsByTimeoutCommand)
+      const sql = plans.failJobsByTimeout(this.#config.schema, table, queues)
+      await this.#executeSql(sql)
+    }
   }
 
-  async archive () {
-    await this.db.executeSql(this.archiveCommand)
-  }
+  async #maintain (table, names) {
+    const command = plans.trySetQueueDeletionTime(this.#config.schema, names, this.#config.maintenanceIntervalSeconds)
+    const { rows } = await this.#executeSql(command)
 
-  async drop () {
-    await this.db.executeSql(this.dropCommand)
+    if (rows.length) {
+      const queues = rows.map(q => q.name)
+      const sql = plans.deletion(this.#config.schema, table, queues)
+      await this.#executeSql(sql)
+    }
   }
 }
 

package/src/db.js

@@ -6,8 +6,11 @@ class Db extends EventEmitter {
     super()
 
     config.application_name = config.application_name || 'pgboss'
+    // config.maxUses = config.maxUses || 1000
 
     this.config = config
+    this._pgbdb = true
+    this.opened = false
   }
 
   events = {

package/src/index.js

@@ -51,7 +51,7 @@ class PgBoss extends EventEmitter {
     const db = this.getDb()
     this.#db = db
 
-    if (db.isOurs) {
+    if (db._pgbdb) {
       this.#promoteEvents(db)
     }
 
@@ -89,9 +89,7 @@ class PgBoss extends EventEmitter {
       return this.#config.db
     }
 
-    const db = new Db(this.#config)
-    db.isOurs = true
-    return db
+    return new Db(this.#config)
   }
 
   #promoteEvents (emitter) {
@@ -113,7 +111,7 @@ class PgBoss extends EventEmitter {
 
     this.#starting = true
 
-    if (this.#db.isOurs && !this.#db.opened) {
+    if (this.#db._pgbdb && !this.#db.opened) {
       await this.#db.open()
     }
 
@@ -123,14 +121,10 @@ class PgBoss extends EventEmitter {
       await this.#contractor.check()
     }
 
-    this.#manager.start()
+    await this.#manager.start()
 
     if (this.#config.supervise) {
-      await this.#boss.supervise()
-    }
-
-    if (this.#config.monitorStateIntervalSeconds) {
-      await this.#boss.monitor()
+      await this.#boss.start()
     }
 
     if (this.#config.schedule) {
@@ -168,7 +162,7 @@ class PgBoss extends EventEmitter {
 
           await this.#manager.failWip()
 
-          if (this.#db.isOurs && this.#db.opened && close) {
+          if (this.#db._pgbdb && this.#db.opened && close) {
             await this.#db.close()
           }