utneque 1.0.0

package/packages/core/src/queue/event-queue.ts

@@ -0,0 +1,318 @@
+import { CoreAnalytics } from '../analytics'
+import { groupBy } from '../utils/group-by'
+import { ON_REMOVE_FROM_FUTURE, PriorityQueue } from '../priority-queue'
+
+import { CoreContext, ContextCancelation } from '../context'
+import { Emitter } from '../emitter'
+import { Integrations, JSONObject } from '../events/interfaces'
+import { CorePlugin } from '../plugins'
+import { createTaskGroup, TaskGroup } from '../task/task-group'
+import { attempt, ensure } from './delivery'
+
+export type EventQueueEmitterContract<Ctx extends CoreContext> = {
+  message_delivered: [ctx: Ctx]
+  message_enriched: [ctx: Ctx, plugin: CorePlugin<Ctx>]
+  delivery_success: [ctx: Ctx]
+  delivery_retry: [ctx: Ctx]
+  delivery_failure: [ctx: Ctx, err: Ctx | Error | ContextCancelation]
+  flush: [ctx: Ctx, delivered: boolean]
+}
+
+export abstract class CoreEventQueue<
+  Ctx extends CoreContext = CoreContext,
+  Plugin extends CorePlugin<Ctx> = CorePlugin<Ctx>
+> extends Emitter<EventQueueEmitterContract<Ctx>> {
+  /**
+   * All event deliveries get suspended until all the tasks in this task group are complete.
+   * For example: a middleware that augments the event object should be loaded safely as a
+   * critical task, this way, event queue will wait for it to be ready before sending events.
+   *
+   * This applies to all the events already in the queue, and the upcoming ones
+   */
+  criticalTasks: TaskGroup = createTaskGroup()
+  queue: PriorityQueue<Ctx>
+  plugins: Plugin[] = []
+  failedInitializations: string[] = []
+  private flushing = false
+
+  constructor(priorityQueue: PriorityQueue<Ctx>) {
+    super()
+
+    this.queue = priorityQueue
+    this.queue.on(ON_REMOVE_FROM_FUTURE, () => {
+      this.scheduleFlush(0)
+    })
+  }
+
+  async register(
+    ctx: Ctx,
+    plugin: Plugin,
+    instance: CoreAnalytics
+  ): Promise<void> {
+    await Promise.resolve(plugin.load(ctx, instance))
+      .then(() => {
+        this.plugins.push(plugin)
+      })
+      .catch((err) => {
+        if (plugin.type === 'destination') {
+          this.failedInitializations.push(plugin.name)
+          console.warn(plugin.name, err)
+
+          ctx.log('warn', 'Failed to load destination', {
+            plugin: plugin.name,
+            error: err,
+          })
+
+          return
+        }
+
+        throw err
+      })
+  }
+
+  async deregister(
+    ctx: Ctx,
+    plugin: CorePlugin<Ctx>,
+    instance: CoreAnalytics
+  ): Promise<void> {
+    try {
+      if (plugin.unload) {
+        await Promise.resolve(plugin.unload(ctx, instance))
+      }
+
+      this.plugins = this.plugins.filter((p) => p.name !== plugin.name)
+    } catch (e) {
+      ctx.log('warn', 'Failed to unload destination', {
+        plugin: plugin.name,
+        error: e,
+      })
+    }
+  }
+
+  async dispatch(ctx: Ctx): Promise<Ctx> {
+    ctx.log('debug', 'Dispatching')
+    ctx.stats.increment('message_dispatched')
+
+    this.queue.push(ctx)
+    const willDeliver = this.subscribeToDelivery(ctx)
+    this.scheduleFlush(0)
+    return willDeliver
+  }
+
+  private async subscribeToDelivery(ctx: Ctx): Promise<Ctx> {
+    return new Promise((resolve) => {
+      const onDeliver = (flushed: Ctx, delivered: boolean): void => {
+        if (flushed.isSame(ctx)) {
+          this.off('flush', onDeliver)
+          if (delivered) {
+            resolve(flushed)
+          } else {
+            resolve(flushed)
+          }
+        }
+      }
+
+      this.on('flush', onDeliver)
+    })
+  }
+
+  async dispatchSingle(ctx: Ctx): Promise<Ctx> {
+    ctx.log('debug', 'Dispatching')
+    ctx.stats.increment('message_dispatched')
+
+    this.queue.updateAttempts(ctx)
+    ctx.attempts = 1
+
+    return this.deliver(ctx).catch((err) => {
+      const accepted = this.enqueuRetry(err, ctx)
+      if (!accepted) {
+        ctx.setFailedDelivery({ reason: err })
+        return ctx
+      }
+
+      return this.subscribeToDelivery(ctx)
+    })
+  }
+
+  isEmpty(): boolean {
+    return this.queue.length === 0
+  }
+
+  private scheduleFlush(timeout = 500): void {
+    if (this.flushing) {
+      return
+    }
+
+    this.flushing = true
+
+    setTimeout(() => {
+      // eslint-disable-next-line @typescript-eslint/no-floating-promises
+      this.flush().then(() => {
+        setTimeout(() => {
+          this.flushing = false
+
+          if (this.queue.length) {
+            this.scheduleFlush(0)
+          }
+        }, 0)
+      })
+    }, timeout)
+  }
+
+  private async deliver(ctx: Ctx): Promise<Ctx> {
+    await this.criticalTasks.done()
+
+    const start = Date.now()
+    try {
+      ctx = await this.flushOne(ctx)
+      const done = Date.now() - start
+      this.emit('delivery_success', ctx)
+      ctx.stats.gauge('delivered', done)
+      ctx.log('debug', 'Delivered', ctx.event)
+      return ctx
+    } catch (err: any) {
+      const error = err as Ctx | Error | ContextCancelation
+      ctx.log('error', 'Failed to deliver', error)
+      this.emit('delivery_failure', ctx, error)
+      ctx.stats.increment('delivery_failed')
+      throw err
+    }
+  }
+
+  private enqueuRetry(err: Error, ctx: Ctx): boolean {
+    const retriable = !(err instanceof ContextCancelation) || err.retry
+    if (!retriable) {
+      return false
+    }
+
+    return this.queue.pushWithBackoff(ctx)
+  }
+
+  async flush(): Promise<Ctx[]> {
+    if (this.queue.length === 0) {
+      return []
+    }
+
+    let ctx = this.queue.pop()
+    if (!ctx) {
+      return []
+    }
+
+    ctx.attempts = this.queue.getAttempts(ctx)
+
+    try {
+      ctx = await this.deliver(ctx)
+      this.emit('flush', ctx, true)
+    } catch (err: any) {
+      const accepted = this.enqueuRetry(err, ctx)
+
+      if (!accepted) {
+        ctx.setFailedDelivery({ reason: err })
+        this.emit('flush', ctx, false)
+      }
+
+      return []
+    }
+
+    return [ctx]
+  }
+
+  private isReady(): boolean {
+    // return this.plugins.every((p) => p.isLoaded())
+    // should we wait for every plugin to load?
+    return true
+  }
+
+  private availableExtensions(denyList: Integrations) {
+    const available = this.plugins.filter((p) => {
+      // Only filter out destination plugins or the Hightouch.io plugin
+      if (p.type !== 'destination' && p.name !== 'Hightouch.io') {
+        return true
+      }
+
+      let alternativeNameMatch: boolean | JSONObject | undefined = undefined
+      p.alternativeNames?.forEach((name) => {
+        if (denyList[name] !== undefined) {
+          alternativeNameMatch = denyList[name]
+        }
+      })
+
+      // Explicit integration option takes precedence, `All: false` does not apply to Hightouch.io
+      return (
+        denyList[p.name] ??
+        alternativeNameMatch ??
+        (p.name === 'Hightouch.io' ? true : denyList.All) !== false
+      )
+    })
+
+    const {
+      before = [],
+      enrichment = [],
+      destination = [],
+      after = [],
+    } = groupBy(available, 'type')
+
+    return {
+      before,
+      enrichment,
+      destinations: destination,
+      after,
+    }
+  }
+
+  private async flushOne(ctx: Ctx): Promise<Ctx> {
+    if (!this.isReady()) {
+      throw new Error('Not ready')
+    }
+
+    if (ctx.attempts > 1) {
+      this.emit('delivery_retry', ctx)
+    }
+
+    const { before, enrichment } = this.availableExtensions(
+      ctx.event.integrations ?? {}
+    )
+
+    for (const beforeWare of before) {
+      const temp = await ensure(ctx, beforeWare)
+      if (temp instanceof CoreContext) {
+        ctx = temp
+      }
+
+      this.emit('message_enriched', ctx, beforeWare)
+    }
+
+    for (const enrichmentWare of enrichment) {
+      const temp = await attempt(ctx, enrichmentWare)
+      if (temp instanceof CoreContext) {
+        ctx = temp
+      }
+
+      this.emit('message_enriched', ctx, enrichmentWare)
+    }
+
+    // Enrichment and before plugins can re-arrange the deny list dynamically
+    // so we need to pluck them at the end
+    const { destinations, after } = this.availableExtensions(
+      ctx.event.integrations ?? {}
+    )
+
+    await new Promise((resolve, reject) => {
+      setTimeout(() => {
+        const attempts = destinations.map((destination) =>
+          attempt(ctx, destination)
+        )
+        Promise.all(attempts).then(resolve).catch(reject)
+      }, 0)
+    })
+
+    ctx.stats.increment('message_delivered')
+
+    this.emit('message_delivered', ctx)
+
+    const afterCalls = after.map((after) => attempt(ctx, after))
+    await Promise.all(afterCalls)
+
+    return ctx
+  }
+}

package/packages/core/src/stats/__tests__/index.test.ts

@@ -0,0 +1,103 @@
+import { CoreStats } from '..'
+
+class Stats extends CoreStats {}
+
+describe(CoreStats, () => {
+  test('starts out empty', () => {
+    const stats = new Stats()
+    expect(stats.metrics).toEqual([])
+  })
+
+  test('records increments', () => {
+    const stats = new Stats()
+
+    stats.increment('m1')
+    stats.increment('m2', 2)
+    stats.increment('m3', 3, ['test:env'])
+
+    expect(stats.metrics).toEqual([
+      {
+        metric: 'm1',
+        tags: [],
+        timestamp: expect.any(Number),
+        type: 'counter',
+        value: 1,
+      },
+      {
+        metric: 'm2',
+        tags: [],
+        timestamp: expect.any(Number),
+        type: 'counter',
+        value: 2,
+      },
+      {
+        metric: 'm3',
+        tags: ['test:env'],
+        timestamp: expect.any(Number),
+        type: 'counter',
+        value: 3,
+      },
+    ])
+  })
+
+  test('records gauges', () => {
+    const stats = new Stats()
+
+    stats.gauge('m1', 1)
+    stats.gauge('m2', 2, ['test:env'])
+
+    expect(stats.metrics).toEqual([
+      {
+        metric: 'm1',
+        tags: [],
+        timestamp: expect.any(Number),
+        type: 'gauge',
+        value: 1,
+      },
+      {
+        metric: 'm2',
+        tags: ['test:env'],
+        timestamp: expect.any(Number),
+        type: 'gauge',
+        value: 2,
+      },
+    ])
+  })
+
+  test('serializes metrics to a more compact format', () => {
+    const stats = new Stats()
+
+    stats.gauge('some_gauge', 31, ['test:env'])
+    stats.increment('some_increment', 22, ['test:env'])
+
+    expect(stats.serialize()).toEqual([
+      {
+        e: expect.any(Number),
+        k: 'g',
+        m: 'some_gauge',
+        t: ['test:env'],
+        v: 31,
+      },
+      {
+        e: expect.any(Number),
+        k: 'c',
+        m: 'some_increment',
+        t: ['test:env'],
+        v: 22,
+      },
+    ])
+  })
+
+  test('flushes metrics', () => {
+    jest.spyOn(console, 'table').mockImplementationOnce(() => {})
+
+    const stats = new Stats()
+    stats.gauge('some_gauge', 31, ['test:env'])
+    stats.increment('some_increment', 22, ['test:env'])
+
+    stats.flush()
+
+    expect(stats.metrics).toEqual([])
+    expect(console.table).toHaveBeenCalled()
+  })
+})

package/packages/core/src/stats/index.ts

@@ -0,0 +1,88 @@
+type CompactMetricType = 'g' | 'c'
+
+export type CoreMetricType = 'gauge' | 'counter'
+
+export interface CoreMetric {
+  metric: string
+  value: number
+  type: CoreMetricType
+  tags: string[]
+  timestamp: number // unit milliseconds
+}
+
+export interface CompactMetric {
+  m: string // metric name
+  v: number // value
+  k: CompactMetricType
+  t: string[] // tags
+  e: number // timestamp in unit milliseconds
+}
+
+const compactMetricType = (type: CoreMetricType): CompactMetricType => {
+  const enums: Record<CoreMetricType, CompactMetricType> = {
+    gauge: 'g',
+    counter: 'c',
+  }
+  return enums[type]
+}
+
+export abstract class CoreStats {
+  metrics: CoreMetric[] = []
+  increment(metric: string, by = 1, tags?: string[]): void {
+    this.metrics.push({
+      metric,
+      value: by,
+      tags: tags ?? [],
+      type: 'counter',
+      timestamp: Date.now(),
+    })
+  }
+
+  gauge(metric: string, value: number, tags?: string[]): void {
+    this.metrics.push({
+      metric,
+      value,
+      tags: tags ?? [],
+      type: 'gauge',
+      timestamp: Date.now(),
+    })
+  }
+
+  flush(): void {
+    const formatted = this.metrics.map((m) => ({
+      ...m,
+      tags: m.tags.join(','),
+    }))
+    // ie doesn't like console.table
+    if (console.table) {
+      console.table(formatted)
+    } else {
+      console.log(formatted)
+    }
+    this.metrics = []
+  }
+
+  /**
+   * compact keys for smaller payload
+   */
+  serialize(): CompactMetric[] {
+    return this.metrics.map((m) => {
+      return {
+        m: m.metric,
+        v: m.value,
+        t: m.tags,
+        k: compactMetricType(m.type),
+        e: m.timestamp,
+      }
+    })
+  }
+}
+
+export class NullStats extends CoreStats {
+  override gauge(..._args: Parameters<CoreStats['gauge']>) {}
+  override increment(..._args: Parameters<CoreStats['increment']>) {}
+  override flush(..._args: Parameters<CoreStats['flush']>) {}
+  override serialize(..._args: Parameters<CoreStats['serialize']>) {
+    return []
+  }
+}

package/packages/core/src/task/__tests__/task-group.test.ts

@@ -0,0 +1,26 @@
+import { createTaskGroup } from '../task-group'
+
+function sleep(ms: number) {
+  return new Promise((resolve) => setTimeout(resolve, ms))
+}
+
+describe('TaskGroup', () => {
+  it('works with concurrent operations', async () => {
+    const group = createTaskGroup()
+    const a = jest.fn()
+    const b = jest.fn()
+
+    void group.run(async () => {
+      await sleep(100)
+      a()
+    })
+    void group.run(async () => {
+      await sleep(1000)
+      b()
+    })
+
+    await group.done()
+    expect(a).toBeCalled()
+    expect(b).toBeCalled()
+  })
+})

package/packages/core/src/task/task-group.ts

@@ -0,0 +1,31 @@
+import { isThenable } from '../utils/is-thenable'
+
+export type TaskGroup = {
+  done: () => Promise<void>
+  run: <Operation extends (...args: any[]) => any>(
+    op: Operation
+  ) => ReturnType<Operation>
+}
+
+export const createTaskGroup = (): TaskGroup => {
+  let taskCompletionPromise: Promise<void>
+  let resolvePromise: () => void
+  let count = 0
+
+  return {
+    done: () => taskCompletionPromise,
+    run: (op) => {
+      const returnValue = op()
+
+      if (isThenable(returnValue)) {
+        if (++count === 1) {
+          taskCompletionPromise = new Promise((res) => (resolvePromise = res))
+        }
+
+        returnValue.finally(() => --count === 0 && resolvePromise())
+      }
+
+      return returnValue
+    },
+  }
+}

package/packages/core/src/user/index.ts

@@ -0,0 +1,7 @@
+export type ID = string | null | undefined
+
+// TODO: this is a base user
+export interface User {
+  id(): ID
+  anonymousId(): ID
+}

package/packages/core/src/utils/__tests__/group-by.test.ts

@@ -0,0 +1,96 @@
+import { groupBy } from '../group-by'
+
+describe('groupBy', () => {
+  it('groups a collection by key', () => {
+    const collection = [
+      {
+        id: 'go',
+        type: 'server',
+      },
+      {
+        id: 'ruby',
+        type: 'server',
+      },
+      {
+        id: 'js',
+        type: 'browser',
+      },
+      {
+        id: 'react',
+        type: 'app',
+      },
+    ]
+
+    expect(groupBy(collection, 'type')).toEqual({
+      app: [
+        {
+          id: 'react',
+          type: 'app',
+        },
+      ],
+      browser: [
+        {
+          id: 'js',
+          type: 'browser',
+        },
+      ],
+      server: [
+        {
+          id: 'go',
+          type: 'server',
+        },
+        {
+          id: 'ruby',
+          type: 'server',
+        },
+      ],
+    })
+  })
+
+  it('accepts a function', () => {
+    const collection = [
+      {
+        id: 1,
+        type: 'server',
+      },
+      {
+        id: 2,
+        type: 'server',
+      },
+      {
+        id: 3,
+        type: 'browser',
+      },
+      {
+        id: 4,
+        type: 'app',
+      },
+    ]
+
+    const oddEven = groupBy(collection, (c) =>
+      c.id % 2 === 0 ? 'even' : 'odd'
+    )
+    expect(oddEven).toEqual({
+      even: [
+        {
+          id: 2,
+          type: 'server',
+        },
+        {
+          id: 4,
+          type: 'app',
+        },
+      ],
+      odd: [
+        {
+          id: 1,
+          type: 'server',
+        },
+        {
+          id: 3,
+          type: 'browser',
+        },
+      ],
+    })
+  })
+})

package/packages/core/src/utils/__tests__/is-plain-object.test.ts

@@ -0,0 +1,27 @@
+// Spec derived from https://github.com/jonschlinkert/is-plain-object/blob/master/test/server.js
+
+import { isPlainObject } from '../is-plain-object'
+
+describe('isPlainObject', () => {
+  it('should return `true` if the object is created by the `Object` constructor.', function () {
+    expect(isPlainObject(Object.create({}))).toBe(true)
+    expect(isPlainObject(Object.create(Object.prototype))).toBe(true)
+    expect(isPlainObject({ foo: 'bar' })).toBe(true)
+    expect(isPlainObject({})).toBe(true)
+    expect(isPlainObject(Object.create(null))).toBe(true)
+  })
+
+  it('should return `false` if the object is not created by the `Object` constructor.', function () {
+    class Foo {
+      abc = {}
+    }
+
+    expect(isPlainObject(/foo/)).toBe(false)
+    expect(isPlainObject(function () {})).toBe(false)
+    expect(isPlainObject(1)).toBe(false)
+    expect(isPlainObject(['foo', 'bar'])).toBe(false)
+    expect(isPlainObject([])).toBe(false)
+    expect(isPlainObject(new Foo())).toBe(false)
+    expect(isPlainObject(null)).toBe(false)
+  })
+})