npm - utneque - Versions diffs - 1.0.0 - Mend

utneque 1.0.0

Files changed (431) hide show

package/packages/browser/src/plugins/hightouchio/__tests__/normalize.test.ts ADDED Viewed

@@ -0,0 +1,181 @@
+import assert from 'assert'
+import cookie from 'js-cookie'
+import { HightouchioSettings } from '..'
+import { normalize } from '../normalize'
+import { Analytics } from '../../../core/analytics'
+import { HightouchEvent } from '../../../core/events'
+import { JSDOM } from 'jsdom'
+describe('before loading', () => {
+  let jsdom: JSDOM
+  beforeEach(async () => {
+    jest.restoreAllMocks()
+    jest.resetAllMocks()
+    const html = `
+    <!DOCTYPE html>
+      <head></head>
+      <body></body>
+    </html>
+    `.trim()
+    jsdom = new JSDOM(html, {
+      runScripts: 'dangerously',
+      resources: 'usable',
+      url: 'https://localhost',
+    })
+    const windowSpy = jest.spyOn(global, 'window', 'get')
+    windowSpy.mockImplementation(
+      () => jsdom.window as unknown as Window & typeof globalThis
+    )
+  })
+  let options: HightouchioSettings
+  let analytics: Analytics
+  beforeEach(() => {
+    options = { apiKey: 'foo' }
+    analytics = new Analytics({ writeKey: options.apiKey })
+    window.localStorage.clear()
+  })
+  afterEach(() => {
+    analytics.reset()
+    Object.keys(cookie.get()).map((k) => cookie.remove(k))
+    if (window.localStorage) {
+      window.localStorage.clear()
+    }
+  })
+  describe('#normalize', () => {
+    let object: HightouchEvent
+    let defaultCtx: any
+    beforeEach(() => {
+      cookie.remove('s:context.referrer')
+      defaultCtx = {
+        page: {
+          search: '',
+        },
+      }
+      object = {
+        type: 'track',
+        context: defaultCtx,
+      }
+    })
+    it('should add .anonymousId', () => {
+      analytics.user().anonymousId('anon-id')
+      normalize(analytics, object, options, {})
+      assert(object.anonymousId === 'anon-id')
+    })
+    it('should add .sentAt', () => {
+      normalize(analytics, object, options, {})
+      assert(object.sentAt)
+      // assert(type(object.sentAt) === 'date')
+    })
+    it('should add .userId', () => {
+      analytics.user().id('user-id')
+      normalize(analytics, object, options, {})
+      assert(object.userId === 'user-id')
+    })
+    it('should not replace the .timestamp', () => {
+      const timestamp = new Date()
+      object.timestamp = timestamp
+      normalize(analytics, object, options, {})
+      assert(object.timestamp === timestamp)
+    })
+    it('should not replace the .userId', () => {
+      analytics.user().id('user-id')
+      object.userId = 'existing-id'
+      normalize(analytics, object, options, {})
+      assert(object.userId === 'existing-id')
+    })
+    it('should always add .anonymousId even if .userId is given', () => {
+      object.userId = 'baz'
+      normalize(analytics, object, options, {})
+      assert(object.anonymousId?.length === 36)
+    })
+    it('should accept anonymousId being set in an event', async () => {
+      object.userId = 'baz'
+      object.anonymousId = '👻'
+      normalize(analytics, object, options, {})
+      expect(object.anonymousId).toEqual('👻')
+    })
+    it('should add .writeKey', () => {
+      normalize(analytics, object, options, {})
+      assert(object.writeKey === options.apiKey)
+    })
+    describe('unbundling', () => {
+      it('should add a list of bundled integrations', () => {
+        normalize(analytics, object, options, {
+          'Hightouch.io': {},
+          other: {
+            bundlingStatus: 'bundled',
+          },
+        })
+        assert(object)
+        assert(object._metadata)
+        assert.deepEqual(object._metadata.bundled, ['Hightouch.io', 'other'])
+      })
+      it('should add a list of bundled ids', () => {
+        normalize(
+          analytics,
+          object,
+          {
+            ...options,
+            maybeBundledConfigIds: {
+              other: ['o_123', 'o_456'],
+            },
+          },
+          {
+            'Hightouch.io': {},
+            other: {
+              bundlingStatus: 'bundled',
+            },
+          }
+        )
+        assert(object)
+        assert(object._metadata)
+        assert.deepEqual(object._metadata.bundledIds, ['o_123', 'o_456'])
+      })
+      it('should add a list of unbundled integrations when `unbundledIntegrations` is set', () => {
+        options.unbundledIntegrations = ['other2']
+        normalize(analytics, object, options, {
+          other2: {
+            bundlingStatus: 'unbundled',
+          },
+        })
+        assert(object)
+        assert(object._metadata)
+        assert.deepEqual(object._metadata.unbundled, ['other2'])
+      })
+    })
+    it('should pick up messageId from AJS', () => {
+      normalize(analytics, object, options, {}) // ajs core generates the message ID here
+      const messageId = object.messageId
+      normalize(analytics, object, options, {})
+      assert.equal(object.messageId, messageId)
+    })
+  })
+})

package/packages/browser/src/plugins/hightouchio/__tests__/retries.test.ts ADDED Viewed

@@ -0,0 +1,82 @@
+import { hightouchio, HightouchioSettings } from '..'
+import { Analytics } from '../../../core/analytics'
+// @ts-ignore isOffline mocked dependency is accused as unused
+import { isOffline } from '../../../core/connection'
+import { Plugin } from '../../../core/plugin'
+import { envEnrichment } from '../../env-enrichment'
+import { scheduleFlush } from '../schedule-flush'
+import * as PPQ from '../../../lib/priority-queue/persisted'
+import * as PQ from '../../../lib/priority-queue'
+import { Context } from '../../../core/context'
+jest.mock('../schedule-flush')
+type QueueType = 'priority' | 'persisted'
+describe('Hightouch.io retries', () => {
+  let options: HightouchioSettings
+  let analytics: Analytics
+  let hightouch: Plugin
+  let queue: (PPQ.PersistedPriorityQueue | PQ.PriorityQueue<Context>) & {
+    __type?: QueueType
+  }
+  ;[false, true].forEach((persistenceIsDisabled) => {
+    describe(`disableClientPersistence: ${persistenceIsDisabled}`, () => {
+      beforeEach(async () => {
+        jest.resetAllMocks()
+        jest.restoreAllMocks()
+        // @ts-expect-error reassign import
+        isOffline = jest.fn().mockImplementation(() => true)
+        options = { apiKey: 'foo' }
+        analytics = new Analytics(
+          { writeKey: options.apiKey },
+          {
+            retryQueue: true,
+            disableClientPersistence: persistenceIsDisabled,
+          }
+        )
+        if (persistenceIsDisabled) {
+          queue = new PQ.PriorityQueue(3, [])
+          queue['__type'] = 'priority'
+          Object.defineProperty(PQ, 'PriorityQueue', {
+            writable: true,
+            value: jest.fn().mockImplementation(() => queue),
+          })
+        } else {
+          queue = new PPQ.PersistedPriorityQueue(
+            3,
+            `${options.apiKey}:test-hightouch.io`
+          )
+          queue['__type'] = 'persisted'
+          Object.defineProperty(PPQ, 'PersistedPriorityQueue', {
+            writable: true,
+            value: jest.fn().mockImplementation(() => queue),
+          })
+        }
+        hightouch = await hightouchio(analytics, options, {})
+        await analytics.register(hightouch, envEnrichment)
+      })
+      test(`add events to the queue`, async () => {
+        jest.spyOn(queue, 'push')
+        const ctx = await analytics.track('event')
+        expect(scheduleFlush).toHaveBeenCalled()
+        /* eslint-disable  @typescript-eslint/unbound-method */
+        expect(queue.push).toHaveBeenCalled()
+        expect(queue.length).toBe(1)
+        expect(ctx.attempts).toBe(1)
+        expect(isOffline).toHaveBeenCalledTimes(2)
+        expect(queue.__type).toBe<QueueType>(
+          persistenceIsDisabled ? 'priority' : 'persisted'
+        )
+      })
+    })
+  })
+})

package/packages/browser/src/plugins/hightouchio/batched-dispatcher.ts ADDED Viewed

@@ -0,0 +1,127 @@
+import { HightouchEvent } from '../../core/events'
+import { fetch } from '../../lib/fetch'
+import { onPageChange } from '../../lib/on-page-change'
+export type BatchingDispatchConfig = {
+  size?: number
+  timeout?: number
+}
+const MAX_PAYLOAD_SIZE = 500
+function kilobytes(buffer: unknown): number {
+  const size = encodeURI(JSON.stringify(buffer)).split(/%..|./).length - 1
+  return size / 1024
+}
+/**
+ * Checks if the payload is over or close to
+ * the maximum payload size allowed by tracking
+ * API.
+ */
+function approachingTrackingAPILimit(buffer: unknown): boolean {
+  return kilobytes(buffer) >= MAX_PAYLOAD_SIZE - 50
+}
+function chunks(batch: object[]): Array<object[]> {
+  const result: object[][] = []
+  let index = 0
+  batch.forEach((item) => {
+    const size = kilobytes(result[index])
+    if (size >= 64) {
+      index++
+    }
+    if (result[index]) {
+      result[index].push(item)
+    } else {
+      result[index] = [item]
+    }
+  })
+  return result
+}
+export default function batch(
+  apiHost: string,
+  config?: BatchingDispatchConfig,
+  protocol = 'https'
+) {
+  let buffer: object[] = []
+  let pageUnloaded = false
+  const limit = config?.size ?? 10
+  const timeout = config?.timeout ?? 5000
+  function sendBatch(batch: object[]) {
+    if (batch.length === 0) {
+      return
+    }
+    const writeKey = (batch[0] as HightouchEvent)?.writeKey
+    // Remove sentAt from every event as batching only needs a single timestamp
+    const updatedBatch = batch.map((event) => {
+      const { sentAt, ...newEvent } = event as HightouchEvent
+      return newEvent
+    })
+    return fetch(`${protocol}://${apiHost}/v1/batch`, {
+      keepalive: pageUnloaded,
+      headers: {
+        Accept: 'application/json',
+        'Content-Type': 'application/json',
+      },
+      method: 'post',
+      body: JSON.stringify({
+        writeKey,
+        batch: updatedBatch,
+        sentAt: new Date().toISOString(),
+      }),
+    })
+  }
+  async function flush(): Promise<unknown> {
+    if (buffer.length) {
+      const batch = buffer
+      buffer = []
+      return sendBatch(batch)
+    }
+  }
+  let schedule: NodeJS.Timeout | undefined
+  function scheduleFlush(): void {
+    if (schedule) {
+      return
+    }
+    schedule = setTimeout(() => {
+      schedule = undefined
+      flush().catch(console.error)
+    }, timeout)
+  }
+  onPageChange((unloaded) => {
+    pageUnloaded = unloaded
+    if (pageUnloaded && buffer.length) {
+      const reqs = chunks(buffer).map(sendBatch)
+      Promise.all(reqs).catch(console.error)
+    }
+  })
+  async function dispatch(_url: string, body: object): Promise<unknown> {
+    buffer.push(body)
+    const bufferOverflow =
+      buffer.length >= limit || approachingTrackingAPILimit(buffer)
+    return bufferOverflow || pageUnloaded ? flush() : scheduleFlush()
+  }
+  return {
+    dispatch,
+  }
+}

package/packages/browser/src/plugins/hightouchio/fetch-dispatcher.ts ADDED Viewed

@@ -0,0 +1,27 @@
+import { fetch } from '../../lib/fetch'
+export type Dispatcher = (url: string, body: object) => Promise<unknown>
+export type StandardDispatcherConfig = {
+  keepalive?: boolean
+}
+export default function (config?: StandardDispatcherConfig): {
+  dispatch: Dispatcher
+} {
+  function dispatch(url: string, body: object): Promise<unknown> {
+    return fetch(url, {
+      keepalive: config?.keepalive,
+      headers: {
+        Accept: 'application/json',
+        'Content-Type': 'application/json',
+      },
+      method: 'post',
+      body: JSON.stringify(body),
+    })
+  }
+  return {
+    dispatch,
+  }
+}

package/packages/browser/src/plugins/hightouchio/index.ts ADDED Viewed

@@ -0,0 +1,147 @@
+import { Facade } from '@segment/facade'
+import { Analytics } from '../../core/analytics'
+import { LegacySettings } from '../../browser'
+import { isOffline } from '../../core/connection'
+import { Context } from '../../core/context'
+import { Plugin } from '../../core/plugin'
+import { PriorityQueue } from '../../lib/priority-queue'
+import { PersistedPriorityQueue } from '../../lib/priority-queue/persisted'
+import { toFacade } from '../../lib/to-facade'
+import batch, { BatchingDispatchConfig } from './batched-dispatcher'
+import standard, { StandardDispatcherConfig } from './fetch-dispatcher'
+import { normalize } from './normalize'
+import { scheduleFlush } from './schedule-flush'
+import { HIGHTOUCH_API_HOST } from '../../core/constants'
+type DeliveryStrategy =
+  | {
+      strategy?: 'standard'
+      config?: StandardDispatcherConfig
+    }
+  | {
+      strategy?: 'batching'
+      config?: BatchingDispatchConfig
+    }
+export type HightouchioSettings = {
+  apiKey: string
+  apiHost?: string
+  protocol?: 'http' | 'https'
+  addBundledMetadata?: boolean
+  unbundledIntegrations?: string[]
+  bundledConfigIds?: string[]
+  unbundledConfigIds?: string[]
+  maybeBundledConfigIds?: Record<string, string[]>
+  deliveryStrategy?: DeliveryStrategy
+}
+type JSON = ReturnType<Facade['json']>
+function onAlias(analytics: Analytics, json: JSON): JSON {
+  const user = analytics.user()
+  json.previousId =
+    json.previousId ?? json.from ?? user.id() ?? user.anonymousId()
+  json.userId = json.userId ?? json.to
+  delete json.from
+  delete json.to
+  return json
+}
+export function hightouchio(
+  analytics: Analytics,
+  settings?: HightouchioSettings,
+  integrations?: LegacySettings['integrations']
+): Plugin {
+  // Attach `pagehide` before buffer is created so that inflight events are added
+  // to the buffer before the buffer persists events in its own `pagehide` handler.
+  window.addEventListener('pagehide', () => {
+    buffer.push(...Array.from(inflightEvents))
+    inflightEvents.clear()
+  })
+  const writeKey = settings?.apiKey ?? ''
+  const buffer = analytics.options.disableClientPersistence
+    ? new PriorityQueue<Context>(analytics.queue.queue.maxAttempts, [])
+    : new PersistedPriorityQueue(
+        analytics.queue.queue.maxAttempts,
+        `${writeKey}:dest-Hightouch.io`
+      )
+  const inflightEvents = new Set<Context>()
+  const flushing = false
+  const apiHost = settings?.apiHost ?? HIGHTOUCH_API_HOST
+  const protocol = settings?.protocol ?? 'https'
+  const remote = `${protocol}://${apiHost}`
+  const deliveryStrategy = settings?.deliveryStrategy
+  const client =
+    deliveryStrategy?.strategy === 'batching'
+      ? batch(apiHost, deliveryStrategy.config, protocol)
+      : standard(deliveryStrategy?.config as StandardDispatcherConfig)
+  async function send(ctx: Context): Promise<Context> {
+    if (isOffline()) {
+      buffer.push(ctx)
+      // eslint-disable-next-line @typescript-eslint/no-use-before-define
+      scheduleFlush(flushing, buffer, hightouch, scheduleFlush)
+      return ctx
+    }
+    inflightEvents.add(ctx)
+    const eventType = ctx.event.type
+    let json = toFacade(ctx.event).json()
+    if (ctx.event.type === 'track') {
+      delete json.traits
+    }
+    if (ctx.event.type === 'alias') {
+      json = onAlias(analytics, json)
+    }
+    return client
+      .dispatch(
+        `${remote}/v1/${eventType}`,
+        normalize(analytics, json, settings, integrations)
+      )
+      .then(() => ctx)
+      .catch(() => {
+        buffer.pushWithBackoff(ctx)
+        // eslint-disable-next-line @typescript-eslint/no-use-before-define
+        scheduleFlush(flushing, buffer, hightouch, scheduleFlush)
+        return ctx
+      })
+      .finally(() => {
+        inflightEvents.delete(ctx)
+      })
+  }
+  const hightouch: Plugin = {
+    name: 'Hightouch.io',
+    type: 'after',
+    version: '0.1.0',
+    isLoaded: (): boolean => true,
+    load: (): Promise<void> => Promise.resolve(),
+    track: send,
+    identify: send,
+    page: send,
+    alias: send,
+    group: send,
+    screen: send,
+  }
+  // Buffer may already have items if they were previously stored in localStorage.
+  // Start flushing them immediately.
+  if (buffer.todo) {
+    scheduleFlush(flushing, buffer, hightouch, scheduleFlush)
+  }
+  return hightouch
+}

package/packages/browser/src/plugins/hightouchio/normalize.ts ADDED Viewed

@@ -0,0 +1,71 @@
+import { Analytics } from '../../core/analytics'
+import { LegacySettings } from '../../browser'
+import { SegmentFacade } from '../../lib/to-facade'
+import { HightouchioSettings } from './index'
+export function normalize(
+  analytics: Analytics,
+  json: ReturnType<SegmentFacade['json']>,
+  settings?: HightouchioSettings,
+  integrations?: LegacySettings['integrations']
+): object {
+  const user = analytics.user()
+  delete json.options
+  json.writeKey = settings?.apiKey
+  json.userId = json.userId || user.id()
+  json.anonymousId = json.anonymousId || user.anonymousId()
+  json.sentAt = new Date()
+  const failed = analytics.queue.failedInitializations || []
+  if (failed.length > 0) {
+    json._metadata = { failedInitializations: failed }
+  }
+  const bundled: string[] = []
+  const unbundled: string[] = []
+  for (const key in integrations) {
+    const integration = integrations[key]
+    if (key === 'Hightouch.io') {
+      bundled.push(key)
+    }
+    if (integration.bundlingStatus === 'bundled') {
+      bundled.push(key)
+    }
+    if (integration.bundlingStatus === 'unbundled') {
+      unbundled.push(key)
+    }
+  }
+  // This will make sure that the disabled cloud mode destinations will be
+  // included in the unbundled list.
+  for (const settingsUnbundled of settings?.unbundledIntegrations || []) {
+    if (!unbundled.includes(settingsUnbundled)) {
+      unbundled.push(settingsUnbundled)
+    }
+  }
+  const configIds = settings?.maybeBundledConfigIds ?? {}
+  const bundledConfigIds: string[] = []
+  bundled.sort().forEach((name) => {
+    ;(configIds[name] ?? []).forEach((id) => {
+      bundledConfigIds.push(id)
+    })
+  })
+  if (settings?.addBundledMetadata !== false) {
+    json._metadata = {
+      ...json._metadata,
+      bundled: bundled.sort(),
+      unbundled: unbundled.sort(),
+      bundledIds: bundledConfigIds,
+    }
+  }
+  return json
+}

package/packages/browser/src/plugins/hightouchio/schedule-flush.ts ADDED Viewed

@@ -0,0 +1,58 @@
+import { isOffline } from '../../core/connection'
+import { Context } from '../../core/context'
+import { Plugin } from '../../core/plugin'
+import { attempt } from '@ht-sdks/events-sdk-js-core'
+import { pWhile } from '../../lib/p-while'
+import { PriorityQueue } from '../../lib/priority-queue'
+async function flushQueue(
+  xt: Plugin,
+  queue: PriorityQueue<Context>
+): Promise<PriorityQueue<Context>> {
+  const failedQueue: Context[] = []
+  if (isOffline()) {
+    return queue
+  }
+  await pWhile(
+    () => queue.length > 0 && !isOffline(),
+    async () => {
+      const ctx = queue.pop()
+      if (!ctx) {
+        return
+      }
+      const result = await attempt(ctx, xt)
+      const success = result instanceof Context
+      if (!success) {
+        failedQueue.push(ctx)
+      }
+    }
+  )
+  // re-add failed tasks
+  failedQueue.map((failed) => queue.pushWithBackoff(failed))
+  return queue
+}
+export function scheduleFlush(
+  flushing: boolean,
+  buffer: PriorityQueue<Context>,
+  xt: Plugin,
+  scheduleFlush: Function
+): void {
+  if (flushing) {
+    return
+  }
+  // eslint-disable-next-line @typescript-eslint/no-misused-promises
+  setTimeout(async () => {
+    let isFlushing = true
+    // eslint-disable-next-line @typescript-eslint/no-use-before-define
+    const newBuffer = await flushQueue(xt, buffer)
+    isFlushing = false
+    if (buffer.todo > 0) {
+      scheduleFlush(isFlushing, newBuffer, xt, scheduleFlush)
+    }
+  }, Math.random() * 5000)
+}