utneque 1.0.0

package/packages/browser/qa/__tests__/smoke.test.ts

@@ -0,0 +1,170 @@
+import flat from 'flat'
+import { difference, intersection, uniq, without } from 'lodash'
+import { JSONObject } from '../../src/core/events'
+import { browser } from '../lib/browser'
+import { run } from '../lib/runner'
+import { objectSchema } from '../lib/schema'
+import { server } from '../lib/server'
+
+jest.setTimeout(100000)
+type RemovePromise<T> = T extends Promise<infer U> ? U : T
+
+if (!process.env.QA_SOURCES) {
+  throw new Error('no process.env.QA_SOURCES')
+}
+const samples = process.env.QA_SOURCES.split(',')
+
+function compareSchema(results: RemovePromise<ReturnType<typeof run>>) {
+  const classicReqs = results.classic.networkRequests
+    .filter((n) => n.url.includes('us-east-1.hightouch-events') && !n.url.endsWith('/m'))
+    .sort()
+
+  const nextReqs = results.next.networkRequests
+    .filter((n) => n.url.includes('us-east-1.hightouch-events') && !n.url.endsWith('/m'))
+    .sort()
+
+  nextReqs.forEach((req, index) => {
+    const classic = classicReqs[index]
+    if (!classic) {
+      return
+    }
+
+    expect(req.url).toEqual(classic.url)
+
+    // @ts-ignore need all sources to be rebuilt first
+    if (classic.data?._metadata) {
+      // @ts-ignore need all sources to be rebuilt first
+      delete classic.data._metadata.bundledIds
+
+      // @ts-ignore sort unbundled metadata because of a breaking change in the HightouchIO destination
+      classic.data._metadata.unbundled = uniq(
+        // @ts-ignore
+        classic.data._metadata.unbundled.sort()
+      )
+    }
+
+    expect(req.data).toContainSchema(classic.data)
+
+    const nextSchema = objectSchema(req.data as object)
+    const classicSchema = objectSchema(classic.data as object)
+
+    const intersectionKeys = without(
+      intersection(nextSchema, classicSchema),
+
+      // These are different on purpose
+      'context.library.name',
+      'context.library.version',
+
+      // We use the same website with a slightly different URL
+      'context.page.search',
+      'properties.search',
+      'context.page.url',
+      'properties.url',
+
+      'messageId',
+      'anonymousId',
+
+      // We do an integrations specific check below since 'next'
+      // may have action-destinations that 'classic' does not
+      'integrations'
+    )
+
+    expect((req.data as JSONObject).integrations).toEqual(
+      expect.objectContaining(
+        (classic.data as JSONObject).integrations
+      )
+    )
+
+    const flatNext = flat(req.data) as JSONObject
+    const flatClassic = flat(classic.data) as JSONObject
+
+    intersectionKeys.forEach((key) => {
+      const comparison = {
+        url: req.url,
+        key,
+        next: flatNext[key],
+        classic: flatClassic[key],
+      }
+
+      expect({ ...comparison, val: comparison.next }).toEqual({
+        ...comparison,
+        val: comparison.classic,
+      })
+    })
+  })
+}
+
+describe('Smoke Tests', () => {
+  // needs to be written as a string so it's not transpiled
+  const code = `(async () => {
+    await window.htevents.identify('Test', {
+      email: 'test@mctesting.org',
+    })
+
+    await window.htevents.track('Track!', {
+      leProp: 'propé',
+    })
+
+    await window.htevents.page()
+  })()`
+
+  test.concurrent.each(samples)(`smoke test (writekey: %p)`, async (writekey) => {
+    const [url, chrome] = await Promise.all([server(), browser()])
+
+    const results = await run({
+      browser: chrome,
+      script: code,
+      serverURL: url,
+      writeKey: writekey,
+    })
+
+    const classicReqs = results.classic.networkRequests
+      .map((n) => new URL(n.url).host)
+      .sort()
+
+    const nextReqs = results.next.networkRequests
+      .map((n) => new URL(n.url).host)
+      .sort()
+
+    expect(nextReqs).not.toEqual([])
+    const missing = difference(classicReqs, nextReqs)
+
+    expect(missing).toEqual([])
+    expect(nextReqs).toEqual(expect.arrayContaining(classicReqs))
+
+    const nextCookies = results.next.cookies.reduce((all, cookie) => {
+      return {
+        ...all,
+        [cookie.name]: cookie.value,
+      }
+    }, {} as Record<string, string>)
+
+    const classicCookies = results.classic.cookies.reduce((all, cookie) => {
+      return {
+        ...all,
+        [cookie.name]: cookie.value,
+      }
+    }, {} as Record<string, string>)
+
+    expect(nextCookies['htjs_user_id']).toContain('Test')
+    expect(classicCookies['htjs_user_id']).toContain('Test')
+
+    compareSchema(results)
+  })
+
+  test.concurrent.each(samples)(`obfuscated smoke test (writekey: %p)`, async (writekey) => {
+    const [url, chrome] = await Promise.all([server(true), browser()])
+    const obfuscatedresults = await run({
+      browser: chrome,
+      script: code,
+      serverURL: url,
+      writeKey: writekey,
+    })
+
+    obfuscatedresults.next.bundleRequestFailures.forEach((result) => {
+      expect(result).toBe(null)
+    })
+
+    compareSchema(obfuscatedresults)
+  })
+})

package/packages/browser/qa/lib/benchmark.ts

@@ -0,0 +1,36 @@
+import { Page } from 'playwright'
+import { Context } from '../../src/core/context'
+import { gauge } from './stats'
+
+const loadTime = (timing: PerformanceTiming) =>
+  timing.loadEventEnd - timing.loadEventStart
+
+const DOMContentLoaded = (timing: PerformanceTiming) =>
+  timing.domContentLoadedEventEnd - timing.domContentLoadedEventStart
+
+const startToInteractive = (timing: PerformanceTiming) =>
+  timing.domInteractive - timing.navigationStart
+
+export function reportMetrics(
+  next: PerformanceTiming,
+  classic: PerformanceTiming
+): Promise<Context[]> {
+  return Promise.all([
+    gauge('load_time', loadTime(next), ['release:next']),
+    gauge('start_to_interactive', startToInteractive(next), ['release:next']),
+    gauge('dom_content_loaded', DOMContentLoaded(next), ['release:next']),
+
+    gauge('load_time', loadTime(classic), ['release:classic']),
+    gauge('start_to_interactive', startToInteractive(classic), [
+      'release:classic',
+    ]),
+    gauge('dom_content_loaded', DOMContentLoaded(classic), ['release:classic']),
+  ])
+}
+
+export async function getMetrics(page: Page): Promise<PerformanceTiming> {
+  const performanceTimingJson = await page.evaluate(() =>
+    JSON.stringify(window.performance.timing)
+  )
+  return JSON.parse(performanceTimingJson) as PerformanceTiming
+}

package/packages/browser/qa/lib/browser.ts

@@ -0,0 +1,28 @@
+import playwright, { Browser } from 'playwright'
+
+const debug = process.env.DEBUG ?? false
+
+let br: Browser
+export async function browser(): Promise<Browser> {
+  if (!br) {
+    br = await playwright.chromium.launch({
+      devtools: true,
+      headless: !debug,
+      args: [
+        '--no-sandbox',
+        '--disable-setuid-sandbox',
+        '--disable-dev-shm-usage',
+        '--disable-accelerated-2d-canvas',
+        '--no-zygote',
+        '--disable-gpu',
+      ],
+    })
+  }
+
+  return br
+}
+
+!debug &&
+  process.on('exit', async () => {
+    await br?.close()
+  })

package/packages/browser/qa/lib/jest-reporter.js

@@ -0,0 +1,57 @@
+const fetch = require('node-fetch')
+const ex = require('execa')
+
+const btoa = (val) => Buffer.from(val).toString('base64')
+
+const getBranch = async () =>
+  (await ex('git', ['branch', '--show-current'])).stdout
+
+const assertEnvVariable = (name) => {
+  if (process.env[name] !== undefined) return
+  throw new Error(`process.env.${name} is undefined`)
+}
+
+async function increment(metric, value = 0, tags = []) {
+  assertEnvVariable('STATS_WRITEKEY')
+
+  const event = {
+    event: metric,
+    properties: {
+      value,
+      tags,
+      type: 'count',
+    },
+    userId: 'system',
+  }
+
+  return fetch(`https://us-east-1.hightouch-events.com/v1/track`, {
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json',
+      'User-Agent': 'analytics-node-next/latest',
+      Authorization: `Basic ${btoa(process.env.STATS_WRITEKEY)}`,
+    },
+    body: JSON.stringify(event),
+  })
+}
+
+class CustomReporter {
+  async onTestResult(_context, result) {
+    const branch = process.env.BUILDKITE_BRANCH || (await getBranch())
+
+    const metrics = result.testResults.map((res) => {
+      return increment('test_run', 1, [
+        `duration:${res.duration}`,
+        `status:${res.status}`,
+        `title:${res.title}`,
+        `full_name:${res.fullName}`,
+        `parent:${(res.ancestorTitles || []).join('.')}`,
+        `branch:${branch}`,
+      ])
+    })
+
+    await Promise.all(metrics).catch(console.error)
+  }
+}
+
+module.exports = CustomReporter

package/packages/browser/qa/lib/runner.ts

@@ -0,0 +1,142 @@
+import { Browser } from 'playwright'
+import { JSONValue } from '../../src/core/events'
+import { getMetrics } from './benchmark'
+
+const DEBUG = process.env.DEBUG ?? false
+
+type ComparisonParams = {
+  browser: Browser
+  serverURL: string
+  writeKey: string
+  script: string
+  key?: string | number | symbol,
+}
+
+export async function run(params: ComparisonParams) {
+  async function load(
+    browser: Browser,
+    isNext: boolean,
+    source: string,
+    execution: string
+  ) {
+    const networkRequests: Array<{ url: string; data: JSONValue }> = []
+    const bundleRequests: Array<string> = []
+    const context = await browser.newContext()
+    const page = await context.newPage()
+    const bundleRequestFailures: Array<any> = []
+
+    page.route('**', async (route) => {
+      const request = route.request()
+
+      if (
+        request.url().includes('cdn.hightouch-events') ||
+        request.url().includes('cdn-settings') ||
+        request.url().includes('localhost') ||
+        request.url().includes('unpkg')
+      ) {
+        bundleRequests.push(request.url())
+        if (request.url().includes('next-integration')) {
+          bundleRequestFailures.push(request.failure())
+        }
+        route.continue().catch(console.error)
+        return
+      }
+
+      if (request.resourceType() === 'script') {
+        await route.continue().catch(console.error)
+      } else {
+        try {
+          // do not actually send data
+          await route.fulfill({ body: 'ok!' }).catch(console.error)
+        } catch (_err) {
+          // there are cases where the runner finishes so fast that requests
+          // are still in flight when the browser is closing
+        }
+      }
+
+      let data: JSONValue
+      try {
+        data = JSON.parse(request.postData() ?? '{}')
+      } catch (e) {
+        data = null
+      }
+
+      const call = { url: request.url(), data }
+
+      // do not Record GA calls because it thinks ajs-next is a bot and doesn't naturally trigger requests
+      // we know GA works :)
+      if (
+        // clarity.ms uses multiple subdomains
+        call.url.includes('clarity.ms') ||
+        call.url.includes('doubleclick.net') ||
+        // bot
+        call.url.includes('googletagmanager') ||
+        // bot
+        call.url.includes('api-iam.intercom.io') ||
+        // bot
+        call.url.includes('bat.bing') ||
+        // there's no need to assert on metrics, especially as they're sampled
+        call.url.includes('us-east-1.hightouch-events.com/v1/m') ||
+        (request.method() === 'POST' && data === null)
+      ) {
+        return
+      }
+
+      networkRequests.push(call)
+    })
+
+    const url =
+      params.serverURL +
+      (isNext ? '?type=next' : '?type=classic') +
+      '&wk=' +
+      source
+
+    await page.goto(url)
+
+    await page.waitForLoadState('networkidle')
+    await page.waitForFunction(`window.htevents.initialized === true`)
+
+    // This forces every timestamp to look exactly the same.
+    // Moving this prototype manipulation after networkidle fixed a race condition around Object.freeze that interfered with certain scripts.
+    await page.evaluate(`{
+      Date.prototype.toJSON = () => "<date>";
+      Date.prototype.getTime = () => 1614653469;
+      Object.freeze(Date.prototype);
+    };`)
+
+    const codeEvaluation = await page.evaluate(execution)
+    const cookies = await context.cookies()
+    const localStorage: Record<string, string | null> = await page.evaluate(
+      () => {
+        return Object.keys(localStorage).reduce(function (obj, str) {
+          obj[str] = window.localStorage.getItem(str)
+          return obj
+        }, {} as Record<string, string | null>)
+      }
+    )
+    await page.waitForLoadState('networkidle')
+    const metrics = await getMetrics(page)
+
+    !DEBUG && (await page.close({ runBeforeUnload: true, }))
+
+    return {
+      networkRequests,
+      cookies,
+      localStorage,
+      codeEvaluation,
+      metrics,
+      bundleRequests,
+      bundleRequestFailures,
+    }
+  }
+
+  const [classic, next] = await Promise.all([
+    load(params.browser, false, params.writeKey, params.script),
+    load(params.browser, true, params.writeKey, params.script),
+  ])
+
+  return {
+    next,
+    classic,
+  }
+}

package/packages/browser/qa/lib/schema.ts

@@ -0,0 +1,59 @@
+import { difference } from 'lodash'
+import { JSONValue } from '../../src/core/events'
+import flat from 'flat'
+
+declare global {
+  namespace jest {
+    interface Matchers<R> {
+      toContainSchema(got: JSONValue): CustomMatcherResult
+    }
+  }
+}
+// Example:  ['hello', 'foo.bar.baz'] -> ['hello', 'foo', 'foo.bar', 'foo.bar.baz']
+function injectAncestorKeys(keys: string[]): string[] {
+  const allKeys = new Set(keys)
+  for (const key of keys) {
+    const parts = key.split('.')
+    if (parts.length === 1) continue
+
+    do {
+      // remove the last item from the parts
+      parts.pop()
+      const newKey = parts.join('.')
+      if (newKey) {
+        allKeys.add(newKey)
+      }
+    } while (parts.length)
+  }
+
+  return Array.from(allKeys)
+}
+
+export const objectSchema = (obj: object | string) => {
+  let parsed = obj
+  if (typeof obj === 'string') {
+    parsed = JSON.parse(obj)
+  }
+
+  const flattenedKeys = Object.keys(flat(parsed))
+  const allKeys = injectAncestorKeys(flattenedKeys)
+
+  return allKeys.sort()
+}
+
+expect.extend({
+  toContainSchema(got: object, expected: object) {
+    const gotSchema = objectSchema(got)
+    const expectedSchema = objectSchema(expected)
+    const missing = difference(expectedSchema, gotSchema)
+
+    const message = () =>
+      'Incompatible schema. Missing the following properties: \n ' +
+      missing.join('\n')
+
+    return {
+      pass: missing.length === 0,
+      message,
+    }
+  },
+})

package/packages/browser/qa/lib/server.ts

@@ -0,0 +1,54 @@
+import http from 'http'
+import fetch from 'node-fetch'
+import querystring from 'querystring'
+import handler from 'serve-handler'
+import { classic, next } from '../__fixtures__/snippets'
+
+export async function startLocalServer(
+  obfuscateBundles: boolean
+): Promise<string> {
+  const srv = http.createServer((req, res) => {
+    const query = querystring.parse(req.url!)
+
+    if (req.url!.includes('next')) {
+      res.end(next(query.wk as string, obfuscateBundles))
+      return
+    }
+
+    if (req.url!.includes('classic')) {
+      res.end(classic(query.wk as string))
+      return
+    }
+
+    handler(req, res)
+  })
+
+  return new Promise(async (resolve, reject) => {
+    const desiredPort = process.env.PORT ?? 4000
+    const desiredPath = `http://localhost:${desiredPort}`
+
+    try {
+      const ping = await fetch(desiredPath)
+      if (ping.ok) {
+        return resolve(desiredPath)
+      }
+    } catch (err) {
+      srv.on('error', reject)
+      srv.listen(desiredPort, () => {
+        // @ts-expect-error
+        const { port } = srv.address()
+        resolve(`http://localhost:${port ?? desiredPort}`)
+      })
+    }
+  })
+}
+
+let url: string
+export const server = async (obfuscateBundles = false) => {
+  if (url) {
+    return url
+  }
+
+  url = await startLocalServer(obfuscateBundles).catch(() => url)
+  return url
+}

package/packages/browser/qa/lib/stats.ts

@@ -0,0 +1,52 @@
+import { Analytics } from '../../src/index'
+import { Context } from '../../src/core/context'
+import { AnalyticsNode } from '../../src/node'
+import ex from 'execa'
+
+let analytics!: Analytics
+
+
+const getBranch = async () =>
+  (await ex('git', ['branch', '--show-current'])).stdout
+
+async function client(): Promise<Analytics> {
+
+  if (!process.env.STATS_WRITEKEY) {
+    throw new Error('no process.env.STATS_WRITEKEY')
+  }
+
+  if (analytics) {
+    return analytics
+  }
+
+  const [nodeAnalytics] = await AnalyticsNode.load({
+    writeKey: process.env.STATS_WRITEKEY,
+  })
+
+  analytics = nodeAnalytics
+  return analytics
+}
+
+export async function gauge(
+  metric: string,
+  value: number = 0,
+  tags: string[] = []
+): Promise<Context> {
+  const ajs = await client()
+
+  const branch = process.env.BUILDKITE_BRANCH || (await getBranch())
+
+  const ctx = await ajs.track(
+    metric,
+    {
+      value,
+      tags: [...tags, `branch:${branch}`],
+      type: 'gauge',
+    },
+    {
+      userId: 'system',
+    }
+  )
+
+  return ctx
+}

package/packages/browser/scripts/build-prep.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+PKG_VERSION=$(node --eval="process.stdout.write(require('./package.json').version)")
+
+cat <<EOF >src/generated/version.ts
+// This file is generated.
+export const version = '$PKG_VERSION'
+EOF

package/packages/browser/scripts/ci.sh

@@ -0,0 +1,15 @@
+#!/bin/sh
+set -e
+
+echo '--- Build bundles'
+make build
+
+echo '--- Check Size'
+yarn run -T browser size-limit
+
+echo '--- Lint files'
+make lint
+
+echo '--- Run tests'
+make test-unit
+make test-integration