utneque 1.0.0

package/packages/node/src/plugins/segmentio/publisher.ts

@@ -0,0 +1,265 @@
+import { backoff } from '@ht-sdks/events-sdk-js-core'
+import type { Context } from '../../app/context'
+import { tryCreateFormattedUrl } from '../../lib/create-url'
+import { extractPromiseParts } from '../../lib/extract-promise-parts'
+import { ContextBatch } from './context-batch'
+import { NodeEmitter } from '../../app/emitter'
+import { b64encode } from '../../lib/base-64-encode'
+import { HTTPClient, HTTPClientRequest } from '../../lib/http-client'
+
+function sleep(timeoutInMs: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, timeoutInMs))
+}
+
+function noop() {}
+
+interface PendingItem {
+  resolver: (ctx: Context) => void
+  context: Context
+}
+
+export interface PublisherProps {
+  host?: string
+  path?: string
+  flushInterval: number
+  maxEventsInBatch: number
+  maxRetries: number
+  writeKey: string
+  httpRequestTimeout?: number
+  disable?: boolean
+  httpClient: HTTPClient
+}
+
+/**
+ * The Publisher is responsible for batching events and sending them to the Hightouch API.
+ */
+export class Publisher {
+  private pendingFlushTimeout?: ReturnType<typeof setTimeout>
+  private _batch?: ContextBatch
+
+  private _flushInterval: number
+  private _maxEventsInBatch: number
+  private _maxRetries: number
+  private _auth: string
+  private _url: string
+  private _closeAndFlushPendingItemsCount?: number
+  private _httpRequestTimeout: number
+  private _emitter: NodeEmitter
+  private _disable: boolean
+  private _httpClient: HTTPClient
+  constructor(
+    {
+      host,
+      path,
+      maxRetries,
+      maxEventsInBatch,
+      flushInterval,
+      writeKey,
+      httpRequestTimeout,
+      httpClient,
+      disable,
+    }: PublisherProps,
+    emitter: NodeEmitter
+  ) {
+    this._emitter = emitter
+    this._maxRetries = maxRetries
+    this._maxEventsInBatch = Math.max(maxEventsInBatch, 1)
+    this._flushInterval = flushInterval
+    this._auth = b64encode(`${writeKey}:`)
+    this._url = tryCreateFormattedUrl(
+      host ?? 'https://us-east-1.hightouch-events.com',
+      path ?? '/v1/batch'
+    )
+    this._httpRequestTimeout = httpRequestTimeout ?? 10000
+    this._disable = Boolean(disable)
+    this._httpClient = httpClient
+  }
+
+  private createBatch(): ContextBatch {
+    this.pendingFlushTimeout && clearTimeout(this.pendingFlushTimeout)
+    const batch = new ContextBatch(this._maxEventsInBatch)
+    this._batch = batch
+    this.pendingFlushTimeout = setTimeout(() => {
+      if (batch === this._batch) {
+        this._batch = undefined
+      }
+      this.pendingFlushTimeout = undefined
+      if (batch.length) {
+        this.send(batch).catch(noop)
+      }
+    }, this._flushInterval)
+    return batch
+  }
+
+  private clearBatch() {
+    this.pendingFlushTimeout && clearTimeout(this.pendingFlushTimeout)
+    this._batch = undefined
+  }
+
+  flushAfterClose(pendingItemsCount: number) {
+    if (!pendingItemsCount) {
+      // if number of pending items is 0, there will never be anything else entering the batch, since the app is closed.
+      return
+    }
+
+    this._closeAndFlushPendingItemsCount = pendingItemsCount
+
+    // if batch is empty, there's nothing to flush, and when things come in, enqueue will handle them.
+    if (!this._batch) return
+
+    // the number of globally pending items will always be larger or the same as batch size.
+    // Any mismatch is because some globally pending items are in plugins.
+    const isExpectingNoMoreItems = this._batch.length === pendingItemsCount
+    if (isExpectingNoMoreItems) {
+      this.send(this._batch).catch(noop)
+      this.clearBatch()
+    }
+  }
+
+  /**
+   * Enqueues the context for future delivery.
+   * @param ctx - Context containing a Hightouch event.
+   * @returns a promise that resolves with the context after the event has been delivered.
+   */
+  enqueue(ctx: Context): Promise<Context> {
+    const batch = this._batch ?? this.createBatch()
+
+    const { promise: ctxPromise, resolve } = extractPromiseParts<Context>()
+
+    const pendingItem: PendingItem = {
+      context: ctx,
+      resolver: resolve,
+    }
+
+    /*
+      The following logic ensures that a batch is never orphaned,
+      and is always sent before a new batch is created.
+
+      Add an event to the existing batch.
+        Success: Check if batch is full or no more items are expected to come in (i.e. closing). If so, send batch.
+        Failure: Assume event is too big to fit in current batch - send existing batch.
+          Add an event to the new batch.
+            Success: Check if batch is full and send if it is.
+            Failure: Event exceeds maximum size (it will never fit), fail the event.
+    */
+    const addStatus = batch.tryAdd(pendingItem)
+    if (addStatus.success) {
+      const isExpectingNoMoreItems =
+        batch.length === this._closeAndFlushPendingItemsCount
+      const isFull = batch.length === this._maxEventsInBatch
+      if (isFull || isExpectingNoMoreItems) {
+        this.send(batch).catch(noop)
+        this.clearBatch()
+      }
+      return ctxPromise
+    }
+
+    // If the new item causes the maximimum event size to be exceeded, send the current batch and create a new one.
+    if (batch.length) {
+      this.send(batch).catch(noop)
+      this.clearBatch()
+    }
+
+    const fallbackBatch = this.createBatch()
+
+    const fbAddStatus = fallbackBatch.tryAdd(pendingItem)
+
+    if (fbAddStatus.success) {
+      const isExpectingNoMoreItems =
+        fallbackBatch.length === this._closeAndFlushPendingItemsCount
+      if (isExpectingNoMoreItems) {
+        this.send(fallbackBatch).catch(noop)
+        this.clearBatch()
+      }
+      return ctxPromise
+    } else {
+      // this should only occur if max event size is exceeded
+      ctx.setFailedDelivery({
+        reason: new Error(fbAddStatus.message),
+      })
+      return Promise.resolve(ctx)
+    }
+  }
+
+  private async send(batch: ContextBatch) {
+    if (this._closeAndFlushPendingItemsCount) {
+      this._closeAndFlushPendingItemsCount -= batch.length
+    }
+    const events = batch.getEvents()
+    const maxAttempts = this._maxRetries + 1
+
+    let currentAttempt = 0
+    while (currentAttempt < maxAttempts) {
+      currentAttempt++
+
+      let failureReason: unknown
+      try {
+        if (this._disable) {
+          return batch.resolveEvents()
+        }
+
+        const request: HTTPClientRequest = {
+          url: this._url,
+          method: 'POST',
+          headers: {
+            'Content-Type': 'application/json',
+            Authorization: `Basic ${this._auth}`,
+            'User-Agent': 'analytics-node-next/latest',
+          },
+          data: { batch: events, sentAt: new Date() },
+          httpRequestTimeout: this._httpRequestTimeout,
+        }
+
+        this._emitter.emit('http_request', {
+          body: request.data,
+          method: request.method,
+          url: request.url,
+          headers: request.headers,
+        })
+
+        const response = await this._httpClient.makeRequest(request)
+
+        if (response.status >= 200 && response.status < 300) {
+          // Successfully sent events, so exit!
+          batch.resolveEvents()
+          return
+        } else if (response.status === 400) {
+          // Request either malformed or size exceeded - don't retry.
+          resolveFailedBatch(
+            batch,
+            new Error(`[${response.status}] ${response.statusText}`)
+          )
+          return
+        } else {
+          // Treat other errors as transient and retry.
+          failureReason = new Error(
+            `[${response.status}] ${response.statusText}`
+          )
+        }
+      } catch (err) {
+        // Network errors get thrown, retry them.
+        failureReason = err
+      }
+
+      // Final attempt failed, update context and resolve events.
+      if (currentAttempt === maxAttempts) {
+        resolveFailedBatch(batch, failureReason)
+        return
+      }
+
+      // Retry after attempt-based backoff.
+      await sleep(
+        backoff({
+          attempt: currentAttempt,
+          minTimeout: 25,
+          maxTimeout: 1000,
+        })
+      )
+    }
+  }
+}
+
+function resolveFailedBatch(batch: ContextBatch, reason: unknown) {
+  batch.getContexts().forEach((ctx) => ctx.setFailedDelivery({ reason }))
+  batch.resolveEvents()
+}

package/packages/node/tsconfig.build.json

@@ -0,0 +1,9 @@
+{
+  "extends": "./tsconfig.json",
+  "include": ["src"],
+  "exclude": ["**/__tests__/**"],
+  "compilerOptions": {
+    "outDir": "./dist/esm",
+    "declarationDir": "./dist/types"
+  }
+}

package/packages/node/tsconfig.json

@@ -0,0 +1,10 @@
+{
+  "extends": "../../tsconfig.json",
+  "exclude": ["node_modules", "dist"],
+  "compilerOptions": {
+    "module": "ESNext",
+    "target": "es2020", // node 14
+    "moduleResolution": "node",
+    "lib": ["es2020"] // TODO: https://www.npmjs.com/package/@tsconfig/node14
+  }
+}

package/packages/test-helpers/.eslintrc.js

@@ -0,0 +1,7 @@
+/** @type { import('eslint').Linter.Config } */
+module.exports = {
+  extends: ['../../.eslintrc'],
+  env: {
+    node: true,
+  },
+}

package/packages/test-helpers/.lintstagedrc.js

@@ -0,0 +1 @@
+module.exports = require("@internal/config").lintStagedConfig

package/packages/test-helpers/jest.config.js

@@ -0,0 +1,3 @@
+const { createJestTSConfig } = require('@internal/config')
+
+module.exports = createJestTSConfig(__dirname)

package/packages/test-helpers/package.json

@@ -0,0 +1,26 @@
+{
+  "name": "@internal/test-helpers",
+  "version": "0.0.0",
+  "private": true,
+  "main": "./dist/cjs/index.js",
+  "module": "./dist/esm/index.js",
+  "types": "./dist/types/index.d.ts",
+  "scripts": {
+    ".": "yarn run -T turbo run --filter=@internal/test-helpers",
+    "lint": "yarn concurrently 'yarn:eslint .' 'yarn:tsc --noEmit'",
+    "build": "yarn concurrently 'yarn:build:*'",
+    "build:cjs": "yarn tsc -p tsconfig.build.json --outDir ./dist/cjs --module commonjs",
+    "build:esm": "yarn tsc -p tsconfig.build.json",
+    "tsc": "yarn run -T tsc",
+    "watch": "yarn build:esm --watch",
+    "eslint": "yarn run -T eslint",
+    "concurrently": "yarn run -T concurrently"
+  },
+  "dependencies": {
+    "tslib": "^2.4.1"
+  },
+  "packageManager": "yarn@3.4.1",
+  "devDependencies": {
+    "@ht-sdks/events-sdk-js-browser": "workspace:^"
+  }
+}

package/packages/test-helpers/src/analytics/cdn-settings-builder.ts

@@ -0,0 +1,79 @@
+import type { LegacySettings } from '@ht-sdks/events-sdk-js-browser'
+type RemotePlugin = NonNullable<LegacySettings['remotePlugins']>[0]
+
+export type DestinationSettingsBuilderConfig = Partial<RemotePlugin> & {
+  creationName: string
+
+  // this is not part of the RemotePlugin type -- technically, it's part of integrations
+  consentSettings?: {
+    categories: string[]
+  }
+}
+
+export class CDNSettingsBuilder {
+  private settings: LegacySettings
+  constructor({
+    writeKey,
+    baseCDNSettings,
+  }: {
+    writeKey?: string
+    baseCDNSettings?: LegacySettings
+  } = {}) {
+    const settings = baseCDNSettings || {
+      integrations: {
+        'Hightouch.io': {
+          apiKey: writeKey,
+          unbundledIntegrations: [],
+          addBundledMetadata: true,
+          maybeBundledConfigIds: {},
+          versionSettings: { version: '4.4.7', componentTypes: ['browser'] },
+          apiHost: 'us-east-1.hightouch-events.com/v1',
+        },
+      },
+      plan: {
+        track: { __default: { enabled: true, integrations: {} } },
+        identify: { __default: { enabled: true } },
+        group: { __default: { enabled: true } },
+      },
+      middlewareSettings: {} as any,
+      enabledMiddleware: {},
+      metrics: { sampleRate: 0.1, host: 'us-east-1.hightouch-events.com/v1' },
+      legacyVideoPluginsEnabled: false,
+      remotePlugins: [],
+    }
+    this.settings = settings
+  }
+
+  addActionDestinationSettings(
+    ...destinations: DestinationSettingsBuilderConfig[]
+  ) {
+    destinations.forEach((p) => {
+      const remotePlugin: RemotePlugin = {
+        creationName: p.creationName ?? 'mockCreationName',
+        libraryName: p.libraryName ?? 'mockLibraryName',
+        name: p.name ?? 'mockName',
+        url: p.url ?? 'https://mock.com/mock.js',
+        settings: { subscriptions: [], ...(p.settings || {}) },
+      }
+      this.settings.remotePlugins!.push(remotePlugin)
+      const { subscriptions, ...remotePluginSettings } = remotePlugin.settings
+
+      this.settings.integrations = {
+        ...this.settings.integrations,
+        [p.creationName]: {
+          ...remotePluginSettings,
+          ...(p.consentSettings ? { consentSettings: p.consentSettings } : {}),
+        },
+      }
+    })
+    return this
+  }
+
+  build() {
+    return this.settings
+  }
+
+  toJSON() {
+    return this.settings
+  }
+}

package/packages/test-helpers/src/analytics/index.ts

@@ -0,0 +1 @@
+export * from './cdn-settings-builder'

package/packages/test-helpers/src/index.ts

@@ -0,0 +1,2 @@
+export * from './utils'
+export * from './analytics'

package/packages/test-helpers/src/utils/index.ts

@@ -0,0 +1 @@
+export * from './sleep'

package/packages/test-helpers/src/utils/sleep.ts

@@ -0,0 +1,4 @@
+export const sleep = (time: number): Promise<void> =>
+  new Promise((resolve) => {
+    setTimeout(resolve, time)
+  })

package/packages/test-helpers/tsconfig.build.json

@@ -0,0 +1,9 @@
+{
+  "extends": "./tsconfig.json",
+  "include": ["src"],
+  "exclude": ["**/__tests__/**", "**/*.test.*"],
+  "compilerOptions": {
+    "outDir": "./dist/esm",
+    "declarationDir": "./dist/types"
+  }
+}

package/packages/test-helpers/tsconfig.json

@@ -0,0 +1,11 @@
+{
+  "extends": "../../tsconfig.json",
+  "exclude": ["node_modules", "dist"],
+  "compilerOptions": {
+    "resolveJsonModule": true,
+    "module": "esnext",
+    "target": "ES5",
+    "moduleResolution": "node",
+    "lib": ["es2020"]
+  }
+}

package/tsconfig.json

@@ -0,0 +1,21 @@
+{
+  "compilerOptions": {
+    "strict": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "noUnusedLocals": true,
+    "noUnusedParameters": true,
+    "preserveWatchOutput": true,
+    "sourceMap": true,
+    "declaration": true,
+    "declarationMap": true,
+    "importHelpers": true
+  },
+  "ts-node": {
+    "transpileOnly": true,
+    "files": true,
+    "compilerOptions": {
+      "module": "commonjs" // get rid of "Cannot use import statement outside a module" for local scripts
+    }
+  }
+}

package/turbo.json

@@ -0,0 +1,39 @@
+{
+  "$schema": "https://turborepo.org/schema.json",
+  "pipeline": {
+    "build": {
+      "dependsOn": ["^build"],
+      "inputs": [
+        "**/tsconfig*.json",
+        "**/babel.config*",
+        "**/webpack.config*",
+        "**/*.ts",
+        "**/*.tsx",
+        "!**/__tests__/**",
+        "!src/**/test-helpers/**"
+      ],
+      "outputs": ["**/dist/**", ".next/**"]
+    },
+    "test": {
+      "dependsOn": ["build"]
+    },
+    "test:int": {
+      "dependsOn": ["build"]
+    },
+    "watch": {
+      "cache": false,
+      "outputs": ["dist/**"]
+    },
+    "lint": {
+      "dependsOn": ["build"],
+      "inputs": ["**/tsconfig*.json", "**/*.ts", "**/*.tsx", "**/*.js"],
+      "outputs": []
+    },
+    "test:cloudflare-workers": {
+      "dependsOn": ["build"]
+    },
+    "test:perf-and-durability": {
+      "dependsOn": ["build"]
+    }
+  }
+}

package/typings/get-monorepo-packages.d.ts

@@ -0,0 +1,9 @@
+declare module 'get-monorepo-packages' {
+  export default function getPackages(pathToRoot: string): {
+    location: string
+    package: {
+      name: string
+      version: string
+    }
+  }[]
+}

package/typings/spawn.d.ts

@@ -0,0 +1,10 @@
+declare module '@npmcli/promise-spawn' {
+  import { EventEmitter } from 'events'
+  import { SpawnOptions } from 'child_process'
+
+  export default function spawn(
+    cmd: string,
+    args?: string[],
+    opts?: SpawnOptions
+  ): Promise<{ stdout: Buffer; code: number; stderr: Buffer }> & EventEmitter
+}