pg-boss 10.3.3 → 11.0.0

package/src/timekeeper.js

@@ -9,7 +9,14 @@ const QUEUES = {
 
 const EVENTS = {
   error: 'error',
-  schedule: 'schedule'
+  schedule: 'schedule',
+  warning: 'warning'
+}
+
+const WARNINGS = {
+  CLOCK_SKEW: {
+    message: 'Warning: Clock skew between this instance and the database server. This will not break scheduling, but is emitted any time the skew exceeds 60 seconds.'
+  }
 }
 
 class Timekeeper extends EventEmitter {
@@ -19,19 +26,9 @@ class Timekeeper extends EventEmitter {
     this.db = db
     this.config = config
     this.manager = config.manager
-    this.skewMonitorIntervalMs = config.clockMonitorIntervalSeconds * 1000
-    this.cronMonitorIntervalMs = config.cronMonitorIntervalSeconds * 1000
     this.clockSkew = 0
-
     this.events = EVENTS
 
-    this.getTimeCommand = plans.getTime(config.schema)
-    this.getQueueCommand = plans.getQueueByName(config.schema)
-    this.getSchedulesCommand = plans.getSchedules(config.schema)
-    this.scheduleCommand = plans.schedule(config.schema)
-    this.unscheduleCommand = plans.unschedule(config.schema)
-    this.trySetCronTimeCommand = plans.trySetCronTime(config.schema)
-
     this.functions = [
       this.schedule,
       this.unschedule,
@@ -42,30 +39,22 @@ class Timekeeper extends EventEmitter {
   }
 
   async start () {
-    // setting the archive config too low breaks the cron 60s debounce interval so don't even try
-    if (this.config.archiveSeconds < 60 || this.config.archiveFailedSeconds < 60) {
-      return
-    }
-
     this.stopped = false
 
     await this.cacheClockSkew()
-
-    try {
-      await this.manager.createQueue(QUEUES.SEND_IT)
-    } catch {}
+    await this.manager.createQueue(QUEUES.SEND_IT)
 
     const options = {
       pollingIntervalSeconds: this.config.cronWorkerIntervalSeconds,
       batchSize: 50
     }
 
-    await this.manager.work(QUEUES.SEND_IT, options, async (jobs) => { await this.manager.insert(jobs.map(i => i.data)) })
+    await this.manager.work(QUEUES.SEND_IT, options, (jobs) => this.onSendIt(jobs))
 
     setImmediate(() => this.onCron())
 
-    this.cronMonitorInterval = setInterval(async () => await this.onCron(), this.cronMonitorIntervalMs)
-    this.skewMonitorInterval = setInterval(async () => await this.cacheClockSkew(), this.skewMonitorIntervalMs)
+    this.cronMonitorInterval = setInterval(async () => await this.onCron(), this.config.cronMonitorIntervalSeconds * 1000)
+    this.skewMonitorInterval = setInterval(async () => await this.cacheClockSkew(), this.config.clockMonitorIntervalSeconds * 1000)
   }
 
   async stop () {
@@ -96,7 +85,7 @@ class Timekeeper extends EventEmitter {
         throw new Error(this.config.__test__force_clock_monitoring_error)
       }
 
-      const { rows } = await this.db.executeSql(this.getTimeCommand)
+      const { rows } = await this.db.executeSql(plans.getTime())
 
       const local = Date.now()
 
@@ -107,7 +96,7 @@ class Timekeeper extends EventEmitter {
       const skewSeconds = Math.abs(skew) / 1000
 
       if (skewSeconds >= 60 || this.config.__test__force_clock_skew_warning) {
-        Attorney.warnClockSkew(`Instance clock is ${skewSeconds}s ${skew > 0 ? 'slower' : 'faster'} than database.`)
+        this.emit(this.events.warning, { message: WARNINGS.CLOCK_SKEW.message, data: { seconds: skewSeconds, direction: skew > 0 ? 'slower' : 'faster' } })
       }
     } catch (err) {
       this.emit(this.events.error, err)
@@ -126,10 +115,14 @@ class Timekeeper extends EventEmitter {
 
       this.timekeeping = true
 
-      const { rows } = await this.db.executeSql(this.trySetCronTimeCommand, [this.config.cronMonitorIntervalSeconds])
+      const sql = plans.trySetCronTime(this.config.schema, this.config.cronMonitorIntervalSeconds)
+
+      if (!this.stopped) {
+        const { rows } = await this.db.executeSql(sql)
 
-      if (rows.length === 1 && !this.stopped) {
-        await this.cron()
+        if (!this.stopped && rows.length === 1) {
+          await this.cron()
+        }
       }
     } catch (err) {
       this.emit(this.events.error, err)
@@ -143,11 +136,10 @@ class Timekeeper extends EventEmitter {
 
     const scheduled = schedules
       .filter(i => this.shouldSendIt(i.cron, i.timezone))
-      .map(({ name, data, options }) =>
-        ({ name: QUEUES.SEND_IT, data: { name, data, ...options }, singletonKey: name, singletonSeconds: 60 }))
+      .map(({ name, data, options }) => ({ data: { name, data, options }, singletonKey: name, singletonSeconds: 60 }))
 
     if (scheduled.length > 0 && !this.stopped) {
-      await this.manager.insert(scheduled)
+      await this.manager.insert(QUEUES.SEND_IT, scheduled)
     }
   }
 
@@ -163,22 +155,35 @@ class Timekeeper extends EventEmitter {
     return prevDiff < 60
   }
 
-  async getSchedules () {
-    const { rows } = await this.db.executeSql(this.getSchedulesCommand)
+  async onSendIt (jobs) {
+    await Promise.allSettled(jobs.map(({ data }) => this.manager.send(data)))
+  }
+
+  async getSchedules (name, key = '') {
+    let sql = plans.getSchedules(this.config.schema)
+    let params = []
+
+    if (name) {
+      sql = plans.getSchedulesByQueue(this.config.schema)
+      params = [name, key]
+    }
+
+    const { rows } = await this.db.executeSql(sql, params)
+
     return rows
   }
 
   async schedule (name, cron, data, options = {}) {
-    const { tz = 'UTC' } = options
+    const { tz = 'UTC', key = '', ...rest } = options
 
     cronParser.parseExpression(cron, { tz })
 
-    Attorney.checkSendArgs([name, data, options], this.config)
-
-    const values = [name, cron, tz, data, options]
+    Attorney.checkSendArgs([name, data, { ...rest }])
+    Attorney.assertKey(key)
 
     try {
-      await this.db.executeSql(this.scheduleCommand, values)
+      const sql = plans.schedule(this.config.schema)
+      await this.db.executeSql(sql, [name, key, cron, tz, data, options])
     } catch (err) {
       if (err.message.includes('foreign key')) {
         err.message = `Queue ${name} not found`
@@ -188,8 +193,9 @@ class Timekeeper extends EventEmitter {
     }
   }
 
-  async unschedule (name) {
-    await this.db.executeSql(this.unscheduleCommand, [name])
+  async unschedule (name, key = '') {
+    const sql = plans.unschedule(this.config.schema)
+    await this.db.executeSql(sql, [name, key])
   }
 }
 

package/src/tools.js

@@ -1,9 +1,11 @@
+const { setTimeout } = require('node:timers/promises')
+
 module.exports = {
-  delay
+  delay,
+  resolveWithinSeconds
 }
 
 function delay (ms, error) {
-  const { setTimeout } = require('node:timers/promises')
   const ac = new AbortController()
 
   const promise = new Promise((resolve, reject) => {
@@ -26,3 +28,18 @@ function delay (ms, error) {
 
   return promise
 }
+
+async function resolveWithinSeconds (promise, seconds, message) {
+  const timeout = Math.max(1, seconds) * 1000
+  const reject = delay(timeout, message)
+
+  let result
+
+  try {
+    result = await Promise.race([promise, reject])
+  } finally {
+    reject.abort()
+  }
+
+  return result
+}

package/types.d.ts

@@ -17,6 +17,7 @@ declare namespace PgBoss {
     singleton: 'singleton',
     stately: 'stately'
   }
+  
   interface Db {
     executeSql(text: string, values: any[]): Promise<{ rows: any[] }>;
   }
@@ -35,76 +36,43 @@ declare namespace PgBoss {
     db?: Db;
   }
 
-  interface QueueOptions {
-    monitorStateIntervalSeconds?: number;
-    monitorStateIntervalMinutes?: number;
-  }
-
   interface SchedulingOptions {
     schedule?: boolean;
-
     clockMonitorIntervalSeconds?: number;
-    clockMonitorIntervalMinutes?: number;
-
-    cronMonitorIntervalSeconds?: number;
     cronWorkerIntervalSeconds?: number;
+    cronMonitorIntervalSeconds?: number;
   }
 
   interface MaintenanceOptions {
     supervise?: boolean;
     migrate?: boolean;
-
-    deleteAfterSeconds?: number;
-    deleteAfterMinutes?: number;
-    deleteAfterHours?: number;
-    deleteAfterDays?: number;
-
+    warningSlowQuerySeconds?: number;
+    warningQueueSize?: number;
+    superviseIntervalSeconds?: number;
     maintenanceIntervalSeconds?: number;
-    maintenanceIntervalMinutes?: number;
-
-    archiveCompletedAfterSeconds?: number;
-    archiveFailedAfterSeconds?: number;
+    queueCacheIntervalSeconds?: number;
+    monitorIntervalSeconds?: number;
   }
 
-  type ConstructorOptions =
-    DatabaseOptions
-    & QueueOptions
-    & SchedulingOptions
-    & MaintenanceOptions
-    & ExpirationOptions
-    & RetentionOptions
-    & RetryOptions
-    & JobPollingOptions
-
-  interface ExpirationOptions {
-    expireInSeconds?: number;
-    expireInMinutes?: number;
-    expireInHours?: number;
-  }
+  type ConstructorOptions = DatabaseOptions & SchedulingOptions & MaintenanceOptions
 
-  interface RetentionOptions {
+  interface QueueOptions {
+    expireInSeconds?: number;
     retentionSeconds?: number;
-    retentionMinutes?: number;
-    retentionHours?: number;
-    retentionDays?: number;
-  }
-
-  interface RetryOptions {
+    deleteAfterSeconds?: number;    
     retryLimit?: number;
     retryDelay?: number;
     retryBackoff?: boolean;
+    retryDelayMax?: number;
   }
 
   interface JobOptions {
-    id?: string,
+    id?: string;
     priority?: number;
     startAfter?: number | string | Date;
     singletonKey?: string;
     singletonSeconds?: number;
-    singletonMinutes?: number;
-    singletonHours?: number;
     singletonNextSlot?: boolean;
-    deadLetter?: string;
   }
 
   interface ConnectionOptions {
@@ -113,13 +81,29 @@ declare namespace PgBoss {
 
   type InsertOptions = ConnectionOptions;
 
-  type SendOptions = JobOptions & ExpirationOptions & RetentionOptions & RetryOptions & ConnectionOptions;
+  type SendOptions = JobOptions & QueueOptions & ConnectionOptions;
 
   type QueuePolicy = 'standard' | 'short' | 'singleton' | 'stately'
 
-  type Queue = RetryOptions & ExpirationOptions & RetentionOptions & { name: string, policy?: QueuePolicy, deadLetter?: string }
-  type QueueResult = Queue & { createdOn: Date, updatedOn: Date }
-  type ScheduleOptions = SendOptions & { tz?: string }
+  type Queue = {
+    name: string;
+    policy?: QueuePolicy;
+    partition?: boolean;
+    deadLetter?: string;
+    warningQueueSize?: number;
+  } & QueueOptions
+
+  type QueueResult = Queue & {
+    deferredCount: number;
+    queuedCount: number;
+    activeCount: number;
+    completedCount: number;
+    table: number;
+    createdOn: Date;
+    updatedOn: Date;
+  }
+
+  type ScheduleOptions = SendOptions & { tz?: string, key?: string }
 
   interface JobPollingOptions {
     pollingIntervalSeconds?: number;
@@ -151,25 +135,11 @@ declare namespace PgBoss {
 
   interface Schedule {
     name: string;
+    key: string;
     cron: string;
+    timezone: string;
     data?: object;
-    options?: ScheduleOptions;
-  }
-
-  // source (for now): https://github.com/bendrucker/postgres-interval/blob/master/index.d.ts
-  interface PostgresInterval {
-    years?: number;
-    months?: number;
-    days?: number;
-    hours?: number;
-    minutes?: number;
-    seconds?: number;
-    milliseconds?: number;
-
-    toPostgres(): string;
-
-    toISO(): string;
-    toISOString(): string;
+    options?: SendOptions;
   }
 
   interface Job<T = object> {
@@ -186,69 +156,58 @@ declare namespace PgBoss {
     retryCount: number;
     retryDelay: number;
     retryBackoff: boolean;
+    retryDelayMax?: number;
     startAfter: Date;
     startedOn: Date;
     singletonKey: string | null;
     singletonOn: Date | null;
-    expireIn: PostgresInterval;
+    expireInSeconds: number;
+    deleteAfterSeconds: number;
     createdOn: Date;
     completedOn: Date | null;
     keepUntil: Date;
-    deadLetter: string,
-    policy: QueuePolicy,
-    output: object
+    policy: QueuePolicy;
+    deadLetter: string;
+    output: object;
   }
 
   interface JobInsert<T = object> {
-    id?: string,
+    id?: string;
     name: string;
     data?: T;
     priority?: number;
     retryLimit?: number;
     retryDelay?: number;
     retryBackoff?: boolean;
+    retryDelayMax?: number;
     startAfter?: Date | string;
     singletonKey?: string;
     singletonSeconds?: number;
     expireInSeconds?: number;
+    deleteAfterSeconds: number;
     keepUntil?: Date | string;
-    deadLetter?: string;
-  }
-
-  interface MonitorState {
-    all: number;
-    created: number;
-    retry: number;
-    active: number;
-    completed: number;
-    cancelled: number;
-    failed: number;
-  }
-
-  interface MonitorStates extends MonitorState {
-    queues: { [queueName: string]: MonitorState };
   }
 
   interface Worker {
-    id: string,
-    name: string,
-    options: WorkOptions,
-    state: 'created' | 'active' | 'stopping' | 'stopped'
-    count: number,
-    createdOn: Date,
-    lastFetchedOn: Date,
-    lastJobStartedOn: Date,
-    lastJobEndedOn: Date,
-    lastJobDuration: number,
-    lastError: object,
-    lastErrorOn: Date
+    id: string;
+    name: string;
+    options: WorkOptions;
+    state: 'created' | 'active' | 'stopping' | 'stopped';
+    count: number;
+    createdOn: Date;
+    lastFetchedOn: Date;
+    lastJobStartedOn: Date;
+    lastJobEndedOn: Date;
+    lastJobDuration: number;
+    lastError: object;
+    lastErrorOn: Date;
   }
 
   interface StopOptions {
-    close?: boolean,
-    graceful?: boolean,
-    timeout?: number,
-    wait?: boolean
+    close?: boolean;
+    graceful?: boolean;
+    timeout?: number;
+    wait?: boolean;
   }
 
   interface OffWorkOptions {
@@ -261,16 +220,9 @@ declare class PgBoss extends EventEmitter {
   constructor(connectionString: string);
   constructor(options: PgBoss.ConstructorOptions);
 
-  static getConstructionPlans(schema: string): string;
-  static getConstructionPlans(): string;
-
-  static getMigrationPlans(schema: string, version: string): string;
-  static getMigrationPlans(schema: string): string;
-  static getMigrationPlans(): string;
-
-  static getRollbackPlans(schema: string, version: string): string;
-  static getRollbackPlans(schema: string): string;
-  static getRollbackPlans(): string;
+  static getConstructionPlans(schema?: string): string;
+  static getMigrationPlans(schema?: string, version?: string): string;
+  static getRollbackPlans(schema?: string, version?: string): string;
 
   static states: PgBoss.JobStates
   static policies: PgBoss.QueuePolicies
@@ -278,18 +230,12 @@ declare class PgBoss extends EventEmitter {
   on(event: "error", handler: (error: Error) => void): this;
   off(event: "error", handler: (error: Error) => void): this;
 
-  on(event: "maintenance", handler: () => void): this;
-  off(event: "maintenance", handler: () => void): this;
-
-  on(event: "monitor-states", handler: (monitorStates: PgBoss.MonitorStates) => void): this;
-  off(event: "monitor-states", handler: (monitorStates: PgBoss.MonitorStates) => void): this;
+  on(event: "warning", handler: (warning: { message: string, data: object }) => void): this;
+  off(event: "warning", handler: (warning: { message: string, data: object }) => void): this;
 
   on(event: "wip", handler: (data: PgBoss.Worker[]) => void): this;
   off(event: "wip", handler: (data: PgBoss.Worker[]) => void): this;
 
-  on(event: "stopped", handler: () => void): this;
-  off(event: "stopped", handler: () => void): this;
-
   start(): Promise<PgBoss>;
   stop(options?: PgBoss.StopOptions): Promise<void>;
 
@@ -301,14 +247,11 @@ declare class PgBoss extends EventEmitter {
   sendAfter(name: string, data: object, options: PgBoss.SendOptions, dateString: string): Promise<string | null>;
   sendAfter(name: string, data: object, options: PgBoss.SendOptions, seconds: number): Promise<string | null>;
 
-  sendThrottled(name: string, data: object, options: PgBoss.SendOptions, seconds: number): Promise<string | null>;
-  sendThrottled(name: string, data: object, options: PgBoss.SendOptions, seconds: number, key: string): Promise<string | null>;
-
-  sendDebounced(name: string, data: object, options: PgBoss.SendOptions, seconds: number): Promise<string | null>;
-  sendDebounced(name: string, data: object, options: PgBoss.SendOptions, seconds: number, key: string): Promise<string | null>;
+  sendThrottled(name: string, data: object, options: PgBoss.SendOptions, seconds: number, key?: string): Promise<string | null>;
+  sendDebounced(name: string, data: object, options: PgBoss.SendOptions, seconds: number, key?: string): Promise<string | null>;
 
-  insert(jobs: PgBoss.JobInsert[]): Promise<void>;
-  insert(jobs: PgBoss.JobInsert[], options: PgBoss.InsertOptions): Promise<void>;
+  insert(name: string, jobs: PgBoss.JobInsert[]): Promise<void>;
+  insert(name: string, jobs: PgBoss.JobInsert[], options: PgBoss.InsertOptions): Promise<void>;
 
   fetch<T>(name: string): Promise<PgBoss.Job<T>[]>;
   fetch<T>(name: string, options: PgBoss.FetchOptions & { includeMetadata: true }): Promise<PgBoss.JobWithMetadata<T>[]>;
@@ -340,6 +283,9 @@ declare class PgBoss extends EventEmitter {
 
   deleteJob(name: string, id: string, options?: PgBoss.ConnectionOptions): Promise<void>;
   deleteJob(name: string, ids: string[], options?: PgBoss.ConnectionOptions): Promise<void>;
+  deleteQueuedJobs(name: string): Promise<void>;
+  deleteStoredJobs(name: string): Promise<void>;
+  deleteAllJobs(name: string): Promise<void>;
 
   complete(name: string, id: string, options?: PgBoss.ConnectionOptions): Promise<void>;
   complete(name: string, id: string, data: object, options?: PgBoss.ConnectionOptions): Promise<void>;
@@ -349,27 +295,22 @@ declare class PgBoss extends EventEmitter {
   fail(name: string, id: string, data: object, options?: PgBoss.ConnectionOptions): Promise<void>;
   fail(name: string, ids: string[], options?: PgBoss.ConnectionOptions): Promise<void>;
 
-  getJobById<T>(name: string, id: string, options?: PgBoss.ConnectionOptions & { includeArchive: boolean }): Promise<PgBoss.JobWithMetadata<T> | null>;
+  getJobById<T>(name: string, id: string, options?: PgBoss.ConnectionOptions): Promise<PgBoss.JobWithMetadata<T> | null>;
 
   createQueue(name: string, options?: PgBoss.Queue): Promise<void>;
   updateQueue(name: string, options?: PgBoss.Queue): Promise<void>;
   deleteQueue(name: string): Promise<void>;
-  purgeQueue(name: string): Promise<void>;
   getQueues(): Promise<PgBoss.QueueResult[]>;
   getQueue(name: string): Promise<PgBoss.QueueResult | null>;
-  getQueueSize(name: string, options?: { before: 'retry' | 'active' | 'completed' | 'cancelled' | 'failed' }): Promise<number>;
+  getQueueStats(name: string): Promise<number>;
 
-  clearStorage(): Promise<void>;
-  archive(): Promise<void>;
-  purge(): Promise<void>;
-  expire(): Promise<void>;
-  maintain(): Promise<void>;
+  supervise(name?: string): Promise<void>;
   isInstalled(): Promise<Boolean>;
   schemaVersion(): Promise<Number>;
 
   schedule(name: string, cron: string, data?: object, options?: PgBoss.ScheduleOptions): Promise<void>;
-  unschedule(name: string): Promise<void>;
-  getSchedules(): Promise<PgBoss.Schedule[]>;
+  unschedule(name: string, key?: string): Promise<void>;
+  getSchedules(name?: string, key?: string): Promise<PgBoss.Schedule[]>;
 
   getDb(): PgBoss.Db;
 }

package/version.json

@@ -1,3 +1,3 @@
 {
-  "schema": 24
+  "schema": 25
 }