pg-boss 10.0.0-beta2 → 10.0.0-beta21

package/src/timekeeper.js

@@ -2,14 +2,12 @@ const EventEmitter = require('events')
 const plans = require('./plans')
 const cronParser = require('cron-parser')
 const Attorney = require('./attorney')
-const pMap = require('p-map')
 
-const queues = {
-  CRON: '__pgboss__cron',
-  SEND_IT: '__pgboss__send_it'
+const QUEUES = {
+  SEND_IT: '__pgboss__send-it'
 }
 
-const events = {
+const EVENTS = {
   error: 'error',
   schedule: 'schedule'
 }
@@ -25,14 +23,14 @@ class Timekeeper extends EventEmitter {
     this.cronMonitorIntervalMs = config.cronMonitorIntervalSeconds * 1000
     this.clockSkew = 0
 
-    this.events = events
+    this.events = EVENTS
 
     this.getTimeCommand = plans.getTime(config.schema)
+    this.getQueueCommand = plans.getQueueByName(config.schema)
     this.getSchedulesCommand = plans.getSchedules(config.schema)
     this.scheduleCommand = plans.schedule(config.schema)
     this.unscheduleCommand = plans.unschedule(config.schema)
-    this.getCronTimeCommand = plans.getCronTime(config.schema)
-    this.setCronTimeCommand = plans.setCronTime(config.schema)
+    this.trySetCronTimeCommand = plans.trySetCronTime(config.schema)
 
     this.functions = [
       this.schedule,
@@ -49,24 +47,25 @@ class Timekeeper extends EventEmitter {
       return
     }
 
-    // cache the clock skew from the db server
+    this.stopped = false
+
     await this.cacheClockSkew()
 
-    await this.manager.createQueue(queues.CRON)
-    await this.manager.createQueue(queues.SEND_IT)
+    try {
+      await this.manager.createQueue(QUEUES.SEND_IT)
+    } catch {}
 
-    await this.manager.work(queues.CRON, { newJobCheckIntervalSeconds: this.config.cronWorkerIntervalSeconds }, (job) => this.onCron(job))
-    await this.manager.work(queues.SEND_IT, { newJobCheckIntervalSeconds: this.config.cronWorkerIntervalSeconds, teamSize: 50, teamConcurrency: 5 }, (job) => this.onSendIt(job))
+    const options = {
+      pollingIntervalSeconds: this.config.cronWorkerIntervalSeconds,
+      batchSize: 50
+    }
 
-    // uses sendDebounced() to enqueue a cron check
-    await this.checkSchedulesAsync()
+    await this.manager.work(QUEUES.SEND_IT, options, (jobs) => this.manager.insert(jobs.map(i => i.data)))
 
-    // create monitoring interval to make sure cron hasn't crashed
-    this.cronMonitorInterval = setInterval(async () => await this.monitorCron(), this.cronMonitorIntervalMs)
-    // create monitoring interval to measure and adjust for drift in clock skew
-    this.skewMonitorInterval = setInterval(async () => await this.cacheClockSkew(), this.skewMonitorIntervalMs)
+    setImmediate(() => this.onCron())
 
-    this.stopped = false
+    this.cronMonitorInterval = setInterval(async () => await this.onCron(), this.cronMonitorIntervalMs)
+    this.skewMonitorInterval = setInterval(async () => await this.cacheClockSkew(), this.skewMonitorIntervalMs)
   }
 
   async stop () {
@@ -76,8 +75,7 @@ class Timekeeper extends EventEmitter {
 
     this.stopped = true
 
-    await this.manager.offWork(queues.CRON)
-    await this.manager.offWork(queues.SEND_IT)
+    await this.manager.offWork(QUEUES.SEND_IT)
 
     if (this.skewMonitorInterval) {
       clearInterval(this.skewMonitorInterval)
@@ -90,22 +88,6 @@ class Timekeeper extends EventEmitter {
     }
   }
 
-  async monitorCron () {
-    try {
-      if (this.config.__test__force_cron_monitoring_error) {
-        throw new Error(this.config.__test__force_cron_monitoring_error)
-      }
-
-      const { secondsAgo } = await this.getCronTime()
-
-      if (secondsAgo > 60) {
-        await this.checkSchedulesAsync()
-      }
-    } catch (err) {
-      this.emit(this.events.error, err)
-    }
-  }
-
   async cacheClockSkew () {
     let skew = 0
 
@@ -134,43 +116,39 @@ class Timekeeper extends EventEmitter {
     }
   }
 
-  async checkSchedulesAsync () {
-    const opts = {
-      retryLimit: 2,
-      retentionSeconds: 60
-    }
-
-    await this.manager.sendDebounced(queues.CRON, null, opts, 60)
-  }
-
   async onCron () {
-    if (this.stopped) return
-
     try {
-      if (this.config.__test__throw_cron_processing) {
-        throw new Error(this.config.__test__throw_cron_processing)
+      if (this.stopped || this.timekeeping) return
+
+      if (this.config.__test__force_cron_monitoring_error) {
+        throw new Error(this.config.__test__force_cron_monitoring_error)
       }
 
-      const items = await this.getSchedules()
+      this.timekeeping = true
 
-      const sending = items.filter(i => this.shouldSendIt(i.cron, i.timezone))
+      const { rows } = await this.db.executeSql(this.trySetCronTimeCommand, [this.config.cronMonitorIntervalSeconds])
 
-      if (sending.length && !this.stopped) {
-        await pMap(sending, it => this.send(it), { concurrency: 5 })
+      if (rows.length === 1 && !this.stopped) {
+        await this.cron()
       }
-
-      if (this.stopped) return
-
-      // set last time cron was evaluated for downstream usage in cron monitoring
-      await this.setCronTime()
     } catch (err) {
       this.emit(this.events.error, err)
+    } finally {
+      this.timekeeping = false
     }
+  }
+
+  async cron () {
+    const schedules = await this.getSchedules()
 
-    if (this.stopped) return
+    const scheduled = schedules
+      .filter(i => this.shouldSendIt(i.cron, i.timezone))
+      .map(({ name, data, options }) =>
+        ({ name: QUEUES.SEND_IT, data: { name, data, options }, options: { singletonKey: name, singletonSeconds: 60 } }))
 
-    // uses sendDebounced() to enqueue a cron check
-    await this.checkSchedulesAsync()
+    if (scheduled.length > 0 && !this.stopped) {
+      await this.manager.insert(scheduled)
+    }
   }
 
   shouldSendIt (cron, tz) {
@@ -185,21 +163,6 @@ class Timekeeper extends EventEmitter {
     return prevDiff < 60
   }
 
-  async send (job) {
-    const options = {
-      singletonKey: job.name,
-      singletonSeconds: 60
-    }
-
-    await this.manager.send(queues.SEND_IT, job, options)
-  }
-
-  async onSendIt (job) {
-    if (this.stopped) return
-    const { name, data, options } = job.data
-    await this.manager.send(name, data, options)
-  }
-
   async getSchedules () {
     const { rows } = await this.db.executeSql(this.getSchedulesCommand)
     return rows
@@ -210,35 +173,25 @@ class Timekeeper extends EventEmitter {
 
     cronParser.parseExpression(cron, { tz })
 
-    // validation pre-check
     Attorney.checkSendArgs([name, data, options], this.config)
 
     const values = [name, cron, tz, data, options]
 
-    const result = await this.db.executeSql(this.scheduleCommand, values)
+    try {
+      await this.db.executeSql(this.scheduleCommand, values)
+    } catch (err) {
+      if (err.message.includes('foreign key')) {
+        err.message = `Queue ${name} not found`
+      }
 
-    return result ? result.rowCount : null
+      throw err
+    }
   }
 
   async unschedule (name) {
-    const result = await this.db.executeSql(this.unscheduleCommand, [name])
-    return result ? result.rowCount : null
-  }
-
-  async setCronTime () {
-    await this.db.executeSql(this.setCronTimeCommand)
-  }
-
-  async getCronTime () {
-    const { rows } = await this.db.executeSql(this.getCronTimeCommand)
-
-    let { cron_on: cronOn, seconds_ago: secondsAgo } = rows[0]
-
-    secondsAgo = secondsAgo !== null ? parseFloat(secondsAgo) : 61
-
-    return { cronOn, secondsAgo }
+    await this.db.executeSql(this.unscheduleCommand, [name])
   }
 }
 
 module.exports = Timekeeper
-module.exports.QUEUES = queues
+module.exports.QUEUES = QUEUES

package/types.d.ts

@@ -1,8 +1,24 @@
 import { EventEmitter } from 'events'
 
 declare namespace PgBoss {
+  
+  type JobStates = {
+    created : 'created',
+    retry: 'retry',
+    active: 'active',
+    completed: 'completed',
+    cancelled: 'cancelled',
+    failed: 'failed'
+  }
+
+  type QueuePolicies = {
+    standard: 'standard'
+    short: 'short',
+    singleton: 'singleton',
+    stately: 'stately'
+  }
   interface Db {
-    executeSql(text: string, values: any[]): Promise<{ rows: any[]; rowCount: number }>;
+    executeSql(text: string, values: any[]): Promise<{ rows: any[] }>;
   }
 
   interface DatabaseOptions {
@@ -91,53 +107,35 @@ declare namespace PgBoss {
   interface ConnectionOptions {
     db?: Db;
   }
-
+  
   type InsertOptions = ConnectionOptions;
-
+  
   type SendOptions = JobOptions & ExpirationOptions & RetentionOptions & RetryOptions & ConnectionOptions;
-
+  
+  type QueuePolicy = 'standard' | 'short' | 'singleton' | 'stately'
+  
+  type Queue = RetryOptions & ExpirationOptions & RetentionOptions & { name: string, policy?: QueuePolicy, deadLetter?: string }
+  type QueueResult = Queue & { createdOn: Date, updatedOn: Date }
   type ScheduleOptions = SendOptions & { tz?: string }
 
   interface JobPollingOptions {
-    newJobCheckInterval?: number;
-    newJobCheckIntervalSeconds?: number;
+    pollingIntervalSeconds?: number;
   }
 
-  interface CommonJobFetchOptions {
+  interface JobFetchOptions {
     includeMetadata?: boolean;
     priority?: boolean;
-  }
-
-  type JobFetchOptions  = CommonJobFetchOptions & {
-    teamSize?: number;
-    teamConcurrency?: number;
-    teamRefill?: boolean;
-  }
-
-  type BatchJobFetchOptions = CommonJobFetchOptions & {
-    batchSize: number;
+    batchSize?: number;
   }
 
   type WorkOptions = JobFetchOptions & JobPollingOptions
-  type BatchWorkOptions = BatchJobFetchOptions & JobPollingOptions
-
-  type FetchOptions = {
-    includeMetadata?: boolean;
-  } & ConnectionOptions;
+  type FetchOptions = JobFetchOptions & ConnectionOptions;
 
   interface WorkHandler<ReqData> {
-    (job: PgBoss.Job<ReqData>): Promise<any>;
-  }
-
-  interface BatchWorkHandler<ReqData> {
     (job: PgBoss.Job<ReqData>[]): Promise<any>;
   }
 
   interface WorkWithMetadataHandler<ReqData> {
-    (job: PgBoss.JobWithMetadata<ReqData>): Promise<any>;
-  }
-
-  interface BatchWorkWithMetadataHandler<ReqData> {
     (job: PgBoss.JobWithMetadata<ReqData>[]): Promise<any>;
   }
 
@@ -176,24 +174,26 @@ declare namespace PgBoss {
     data: T;
   }
 
-  interface JobWithMetadata<T = object> extends Job<T> {
+  interface JobWithMetadata<T = object> {
+    id: string;
+    name: string;
+    data: T;
     priority: number;
     state: 'created' | 'retry' | 'active' | 'completed' | 'cancelled' | 'failed';
-    retrylimit: number;
-    retrycount: number;
-    retrydelay: number;
-    retrybackoff: boolean;
-    startafter: Date;
-    // This is nullable in the schema, but by the time this type is reified,
-    // it will have been set.
-    startedon: Date;
-    singletonkey: string | null;
-    singletonon: Date | null;
-    expirein: PostgresInterval;
-    createdon: Date;
-    completedon: Date | null;
-    keepuntil: Date;
-    deadletter: string,
+    retryLimit: number;
+    retryCount: number;
+    retryDelay: number;
+    retryBackoff: boolean;
+    startAfter: Date;
+    startedOn: Date;
+    singletonKey: string | null;
+    singletonOn: Date | null;
+    expireIn: PostgresInterval;
+    createdOn: Date;
+    completedOn: Date | null;
+    keepUntil: Date;
+    deadLetter: string,
+    policy: QueuePolicy,
     output: object
   }
 
@@ -242,7 +242,7 @@ declare namespace PgBoss {
   }
 
   interface StopOptions {
-    destroy?: boolean,
+    close?: boolean,
     graceful?: boolean,
     timeout?: number,
     wait?: boolean
@@ -269,6 +269,9 @@ declare class PgBoss extends EventEmitter {
   static getRollbackPlans(schema: string): string;
   static getRollbackPlans(): string;
 
+  static states: PgBoss.JobStates
+  static policies: PgBoss.QueuePolicies
+
   on(event: "error", handler: (error: Error) => void): this;
   off(event: "error", handler: (error: Error) => void): this;
 
@@ -304,32 +307,24 @@ declare class PgBoss extends EventEmitter {
   insert(jobs: PgBoss.JobInsert[]): Promise<void>;
   insert(jobs: PgBoss.JobInsert[], options: PgBoss.InsertOptions): Promise<void>;
 
+  fetch<T>(name: string): Promise<PgBoss.Job<T>[]>;
+  fetch<T>(name: string, options: PgBoss.FetchOptions & { includeMetadata: true }): Promise<PgBoss.JobWithMetadata<T>[]>;
+  fetch<T>(name: string, options: PgBoss.FetchOptions): Promise<PgBoss.Job<T>[]>;
+
   work<ReqData>(name: string, handler: PgBoss.WorkHandler<ReqData>): Promise<string>;
   work<ReqData>(name: string, options: PgBoss.WorkOptions & { includeMetadata: true }, handler: PgBoss.WorkWithMetadataHandler<ReqData>): Promise<string>;
   work<ReqData>(name: string, options: PgBoss.WorkOptions, handler: PgBoss.WorkHandler<ReqData>): Promise<string>;
 
-  work<ReqData>(name: string, options: PgBoss.BatchWorkOptions & { includeMetadata: true }, handler: PgBoss.BatchWorkWithMetadataHandler<ReqData>): Promise<string>;
-  work<ReqData>(name: string, options: PgBoss.BatchWorkOptions, handler: PgBoss.BatchWorkHandler<ReqData>): Promise<string>;
-
   offWork(name: string): Promise<void>;
   offWork(options: PgBoss.OffWorkOptions): Promise<void>;
 
-  /**
-   * Notify worker that something has changed
-   * @param workerId
-   */
   notifyWorker(workerId: string): void;
 
   subscribe(event: string, name: string): Promise<void>;
   unsubscribe(event: string, name: string): Promise<void>;
-  publish(event: string): Promise<string[]>;
-  publish(event: string, data: object): Promise<string[]>;
-  publish(event: string, data: object, options: PgBoss.SendOptions): Promise<string[]>;
-
-  fetch<T>(name: string): Promise<PgBoss.Job<T> | null>;
-  fetch<T>(name: string, batchSize: number): Promise<PgBoss.Job<T>[] | null>;
-  fetch<T>(name: string, batchSize: number, options: PgBoss.FetchOptions & { includeMetadata: true }): Promise<PgBoss.JobWithMetadata<T>[] | null>;
-  fetch<T>(name: string, batchSize: number, options: PgBoss.FetchOptions): Promise<PgBoss.Job<T>[] | null>;
+  publish(event: string): Promise<void>;
+  publish(event: string, data: object): Promise<void>;
+  publish(event: string, data: object, options: PgBoss.SendOptions): Promise<void>;
 
   cancel(name: string, id: string, options?: PgBoss.ConnectionOptions): Promise<void>;
   cancel(name: string, ids: string[], options?: PgBoss.ConnectionOptions): Promise<void>;
@@ -337,6 +332,9 @@ declare class PgBoss extends EventEmitter {
   resume(name: string, id: string, options?: PgBoss.ConnectionOptions): Promise<void>;
   resume(name: string, ids: string[], options?: PgBoss.ConnectionOptions): Promise<void>;
 
+  deleteJob(name: string, id: string, options?: PgBoss.ConnectionOptions): Promise<void>;
+  deleteJob(name: string, ids: string[], options?: PgBoss.ConnectionOptions): Promise<void>;
+  
   complete(name: string, id: string, options?: PgBoss.ConnectionOptions): Promise<void>;
   complete(name: string, id: string, data: object, options?: PgBoss.ConnectionOptions): Promise<void>;
   complete(name: string, ids: string[], options?: PgBoss.ConnectionOptions): Promise<void>;
@@ -345,18 +343,23 @@ declare class PgBoss extends EventEmitter {
   fail(name: string, id: string, data: object, options?: PgBoss.ConnectionOptions): Promise<void>;
   fail(name: string, ids: string[], options?: PgBoss.ConnectionOptions): Promise<void>;
 
-  getQueueSize(name: string, options?: object): Promise<number>;
-  getJobById(name: string, id: string, options?: PgBoss.ConnectionOptions): Promise<PgBoss.JobWithMetadata | null>;
-
-  createQueue(name: string, policy: 'standard' | 'short' | 'singleton' | 'stately'): Promise<void>;
+  getJobById(name: string, id: string, options?: PgBoss.ConnectionOptions & { includeArchive: bool }): Promise<PgBoss.JobWithMetadata | null>;
+  
+  createQueue(name: string, options?: PgBoss.Queue): Promise<void>;
+  updateQueue(name: string, options?: PgBoss.Queue): Promise<void>;
   deleteQueue(name: string): Promise<void>;
   purgeQueue(name: string): Promise<void>;
-  clearStorage(): Promise<void>;
+  getQueues(): Promise<PgBoss.QueueResult[]>;
+  getQueue(name: string): Promise<PgBoss.QueueResult | null>;
+  getQueueSize(name: string, options?: { before: 'retry' | 'active' | 'completed' | 'cancelled' | 'failed' }): Promise<number>;
 
+  clearStorage(): Promise<void>;
   archive(): Promise<void>;
   purge(): Promise<void>;
   expire(): Promise<void>;
   maintain(): Promise<void>;
+  isInstalled(): Promise<Boolean>;
+  schemaVersion(): Promise<Number>;
 
   schedule(name: string, cron: string, data?: object, options?: PgBoss.ScheduleOptions): Promise<void>;
   unschedule(name: string): Promise<void>;