tapimo 0.8.0 → 0.10.0

package/src/Webhooks.bench.ts

@@ -0,0 +1,714 @@
+/** @module-tag localnet */
+/**
+ * Benchmarks the production poller against disposable PostgreSQL and Tempo services.
+ * Set `WEBHOOK_BENCH_COUNTS`, `WEBHOOK_BENCH_ITERATIONS`, and
+ * `WEBHOOK_BENCH_SCENARIOS` to focus a run.
+ */
+import * as http from 'node:http'
+
+import { Webhooks } from 'tapimo'
+import { bench } from 'vite-plus/test'
+
+import * as Cursor from './internal/Cursor.js'
+import * as Metrics from './Metrics.js'
+import * as TestApp from '../test/App.js'
+import * as Runtime from '../test/runtime.js'
+import * as TestViem from '../test/Viem.js'
+
+const concurrency = 8
+const endOfBlockIndex = 2_147_483_647
+const noMatchAddress = '0xffffffffffffffffffffffffffffffffffffffff'
+const runtime = Runtime.get()
+const scenarioNames = [
+  'caught-up',
+  'empty-log',
+  'empty-transaction',
+  'empty-transfer',
+  'matching-block',
+] as const
+const transactionIterations = 1
+const warmupIterations = 3
+
+type DispatchStats = {
+  calls: number
+  messages: number
+}
+
+type Fixture = {
+  count: number
+  scenarios: Partial<Record<ScenarioName, Scenario>>
+}
+
+type Measurement = {
+  cursorWriteMs: number
+  enrichMs: number
+  rpcCalls: number
+  rpcHeadMs: number
+  rpcMethods: Record<string, number>
+  rpcRequestBytes: number
+  rpcRequests: number
+  rpcResponseBytes: number
+  scanMs: number
+  subscriptionReadMs: number
+  tickMs: number
+  unattributedMs: number
+}
+
+type RpcStats = {
+  calls: number
+  methods: Record<string, number>
+  requestBytes: number
+  requests: number
+  responseBytes: number
+}
+
+type Scenario = {
+  db: ReturnType<typeof TestApp.database>
+  dispatchStats: DispatchStats
+  head: number
+  ids: readonly string[]
+  measurements: Measurement[]
+  metrics: ReturnType<typeof createMetricProbe>
+  nextHead: number
+  poller: Webhooks.createPoller.Poller
+  rpc: Awaited<ReturnType<typeof createRpcProbe>>
+  samples: number
+  startBlockNumber: number
+}
+
+type ScenarioName = (typeof scenarioNames)[number]
+
+const counts = (() => {
+  const input = process.env.WEBHOOK_BENCH_COUNTS
+  if (!input) return [1, 100, 1_000]
+  const values = [...new Set(input.split(',').map(Number))].sort((a, b) => a - b)
+  if (values.length === 0 || values.some((value) => !Number.isSafeInteger(value) || value < 1))
+    throw new Error('WEBHOOK_BENCH_COUNTS must be a comma-separated list of positive integers.')
+  return values
+})()
+
+const iterations = (() => {
+  const input = process.env.WEBHOOK_BENCH_ITERATIONS
+  if (!input) return 10
+  const value = Number(input)
+  if (!Number.isSafeInteger(value) || value < 1)
+    throw new Error('WEBHOOK_BENCH_ITERATIONS must be a positive integer.')
+  return value
+})()
+
+const selectedScenarios = (() => {
+  const input = process.env.WEBHOOK_BENCH_SCENARIOS
+  if (!input) return new Set<ScenarioName>(scenarioNames)
+  const values = [...new Set(input.split(','))]
+  const unknown = values.filter((value) => !scenarioNames.includes(value as ScenarioName))
+  if (values.length === 0 || unknown.length > 0)
+    throw new Error(
+      `WEBHOOK_BENCH_SCENARIOS must contain: ${scenarioNames.join(', ')}. Unknown: ${unknown.join(', ')}.`,
+    )
+  return new Set(values as ScenarioName[])
+})()
+
+const fixtures = new Map<number, Fixture>()
+let rpcProbe: Awaited<ReturnType<typeof createRpcProbe>> | undefined
+
+beforeAll(async () => {
+  if (runtime.mode !== 'localnet')
+    throw new Error('Run webhook benchmarks with TEMPO_ENV=localnet.')
+  rpcProbe = await createRpcProbe()
+  for (const count of counts) fixtures.set(count, await createFixture(count, rpcProbe))
+}, 180_000)
+
+afterAll(async () => {
+  reportMeasurements()
+  for (const fixture of fixtures.values())
+    for (const scenario of Object.values(fixture.scenarios)) await scenario.db.close()
+  await rpcProbe?.close()
+})
+
+describe('webhook poller', () => {
+  for (const count of counts) {
+    if (selectedScenarios.has('caught-up'))
+      bench(
+        `caught-up tick / ${count} subscriptions`,
+        async () => {
+          const scenario = getScenario(count, 'caught-up')
+          resetDispatchStats(scenario)
+          resetMeasurement(scenario)
+          const startedAt = performance.now()
+          const result = await scenario.poller.tick({
+            confirmations: 0,
+            eventTypes: ['block:created'],
+            headBlockNumber: scenario.head,
+            maxBlocks: 1,
+          })
+          recordMeasurement(scenario, performance.now() - startedAt)
+          assertDispatch(scenario, { calls: 0, messages: 0 })
+          assertResult(result, { count, dispatched: 0, scanned: 0 })
+        },
+        {
+          iterations,
+          setup: async () => {
+            const scenario = getScenario(count, 'caught-up')
+            await resetCursors(scenario, scenario.head)
+          },
+          time: 0,
+          throws: true,
+          warmupIterations,
+          warmupTime: 0,
+        },
+      )
+
+    if (selectedScenarios.has('empty-transfer'))
+      bench(
+        `empty selective transfer block / ${count} subscriptions`,
+        async () => {
+          const scenario = getScenario(count, 'empty-transfer')
+          resetDispatchStats(scenario)
+          resetMeasurement(scenario)
+          const startedAt = performance.now()
+          const result = await scenario.poller.tick({
+            confirmations: 0,
+            eventTypes: ['token:transfer'],
+            headBlockNumber: nextHead(scenario),
+            maxBlocks: 1,
+          })
+          recordMeasurement(scenario, performance.now() - startedAt)
+          assertDispatch(scenario, { calls: 0, messages: 0 })
+          assertResult(result, { count, dispatched: 0, scanned: 0 })
+        },
+        {
+          iterations,
+          setup: async () => {
+            const scenario = getScenario(count, 'empty-transfer')
+            await prepareScan(scenario)
+          },
+          time: 0,
+          throws: true,
+          warmupIterations,
+          warmupTime: 0,
+        },
+      )
+
+    if (selectedScenarios.has('empty-log'))
+      bench(
+        `empty selective log block / ${count} subscriptions`,
+        async () => {
+          const scenario = getScenario(count, 'empty-log')
+          resetDispatchStats(scenario)
+          resetMeasurement(scenario)
+          const startedAt = performance.now()
+          const result = await scenario.poller.tick({
+            confirmations: 0,
+            eventTypes: ['log:emitted'],
+            headBlockNumber: nextHead(scenario),
+            maxBlocks: 1,
+          })
+          recordMeasurement(scenario, performance.now() - startedAt)
+          assertDispatch(scenario, { calls: 0, messages: 0 })
+          assertResult(result, { count, dispatched: 0, scanned: 0 })
+        },
+        {
+          iterations,
+          setup: async () => {
+            const scenario = getScenario(count, 'empty-log')
+            await prepareScan(scenario)
+          },
+          time: 0,
+          throws: true,
+          warmupIterations,
+          warmupTime: 0,
+        },
+      )
+
+    if (selectedScenarios.has('empty-transaction'))
+      bench(
+        `empty selective transaction block / ${count} subscriptions`,
+        async () => {
+          const scenario = getScenario(count, 'empty-transaction')
+          resetDispatchStats(scenario)
+          resetMeasurement(scenario)
+          const startedAt = performance.now()
+          const result = await scenario.poller.tick({
+            confirmations: 0,
+            eventTypes: ['transaction:included'],
+            headBlockNumber: nextHead(scenario),
+            maxBlocks: 1,
+          })
+          recordMeasurement(scenario, performance.now() - startedAt)
+          assertDispatch(scenario, { calls: 0, messages: 0 })
+          assertResult(result, { count, dispatched: 0, scanned: 0 })
+        },
+        {
+          iterations: transactionIterations,
+          setup: async () => {
+            const scenario = getScenario(count, 'empty-transaction')
+            await prepareScan(scenario)
+          },
+          time: 0,
+          throws: true,
+          warmupIterations: 0,
+          warmupTime: 0,
+        },
+      )
+
+    if (selectedScenarios.has('matching-block'))
+      bench(
+        `matching block fanout / ${count} subscriptions`,
+        async () => {
+          const scenario = getScenario(count, 'matching-block')
+          resetDispatchStats(scenario)
+          resetMeasurement(scenario)
+          const startedAt = performance.now()
+          const result = await scenario.poller.tick({
+            confirmations: 0,
+            eventTypes: ['block:created'],
+            headBlockNumber: nextHead(scenario),
+            maxBlocks: 1,
+          })
+          recordMeasurement(scenario, performance.now() - startedAt)
+          assertDispatch(scenario, { calls: count, messages: count })
+          assertResult(result, { count, dispatched: count, scanned: count })
+        },
+        {
+          iterations,
+          setup: async () => {
+            const scenario = getScenario(count, 'matching-block')
+            await prepareScan(scenario)
+          },
+          time: 0,
+          throws: true,
+          warmupIterations,
+          warmupTime: 0,
+        },
+      )
+  }
+})
+
+async function createFixture(
+  count: number,
+  rpc: Awaited<ReturnType<typeof createRpcProbe>>,
+): Promise<Fixture> {
+  const client = TestViem.getClient()
+  const liveHead = Number(await client.getBlockNumber())
+  const head = Math.max(1, liveHead - 1)
+  const startBlockNumber = Math.max(0, head - Math.max(iterations, warmupIterations) - 1)
+  const scenarios: Partial<Record<ScenarioName, Scenario>> = {}
+  if (selectedScenarios.has('caught-up'))
+    scenarios['caught-up'] = await createScenario({
+      count,
+      cursorBlockNumber: head,
+      eventType: 'block:created',
+      filters: {},
+      head,
+      rpc,
+    })
+  if (selectedScenarios.has('empty-log'))
+    scenarios['empty-log'] = await createScenario({
+      count,
+      cursorBlockNumber: startBlockNumber,
+      eventType: 'log:emitted',
+      filters: { address: noMatchAddress },
+      head,
+      rpc,
+    })
+  if (selectedScenarios.has('empty-transaction'))
+    scenarios['empty-transaction'] = await createScenario({
+      count,
+      cursorBlockNumber: Math.max(0, head - transactionIterations),
+      eventType: 'transaction:included',
+      filters: { from: noMatchAddress },
+      head,
+      rpc,
+      samples: transactionIterations,
+    })
+  if (selectedScenarios.has('empty-transfer'))
+    scenarios['empty-transfer'] = await createScenario({
+      count,
+      cursorBlockNumber: startBlockNumber,
+      eventType: 'token:transfer',
+      filters: { sender: noMatchAddress },
+      head,
+      rpc,
+    })
+  if (selectedScenarios.has('matching-block'))
+    scenarios['matching-block'] = await createScenario({
+      count,
+      cursorBlockNumber: startBlockNumber,
+      eventType: 'block:created',
+      filters: {},
+      head,
+      rpc,
+    })
+  return {
+    count,
+    scenarios,
+  }
+}
+
+async function createScenario(options: createScenario.Options): Promise<Scenario> {
+  const source = TestApp.databaseFactory()
+  const db = source()
+  const dispatchStats = { calls: 0, messages: 0 }
+  const metrics = createMetricProbe()
+  const ids = await createSubscriptions({
+    count: options.count,
+    cursor: Cursor.encode([options.cursorBlockNumber, endOfBlockIndex]),
+    db,
+    eventType: options.eventType,
+    filters: options.filters,
+  })
+  return {
+    db,
+    dispatchStats,
+    head: options.head,
+    ids,
+    measurements: [],
+    metrics,
+    nextHead: options.cursorBlockNumber + 1,
+    poller: Webhooks.createPoller({
+      db: source,
+      defaultChainId: runtime.chainId,
+      dispatchBatch: async (dispatchables) => {
+        dispatchStats.calls++
+        dispatchStats.messages += dispatchables.length
+        return { ok: true }
+      },
+      metrics: metrics.sink,
+      rpc: { ...TestApp.rpc, url: options.rpc.url },
+    }),
+    rpc: options.rpc,
+    samples: options.samples ?? iterations,
+    startBlockNumber: options.cursorBlockNumber,
+  }
+}
+
+declare namespace createScenario {
+  type Options = {
+    count: number
+    cursorBlockNumber: number
+    eventType: Webhooks.EventType
+    filters: Record<string, unknown>
+    head: number
+    rpc: Awaited<ReturnType<typeof createRpcProbe>>
+    samples?: number | undefined
+  }
+}
+
+async function createSubscriptions(options: createSubscriptions.Options): Promise<string[]> {
+  const ids: string[] = []
+  for (let offset = 0; offset < options.count; offset += concurrency) {
+    const batch = Array.from(
+      { length: Math.min(concurrency, options.count - offset) },
+      async (_, index) => {
+        const subscription = await Webhooks.createSubscription(options.db, {
+          chainId: runtime.chainId,
+          destination: {
+            type: 'url',
+            url: `https://hooks.example.com/bench/${options.eventType}/${offset + index}`,
+          },
+          eventType: options.eventType,
+          filters: options.filters,
+          owner: { orgId: 'org_benchmark', type: 'api_key' },
+        })
+        await Webhooks.setCursor(options.db, subscription.id, options.cursor)
+        return subscription.id
+      },
+    )
+    ids.push(...(await Promise.all(batch)))
+  }
+  return ids
+}
+
+declare namespace createSubscriptions {
+  type Options = {
+    count: number
+    cursor: string
+    db: ReturnType<typeof TestApp.database>
+    eventType: Webhooks.EventType
+    filters: Record<string, unknown>
+  }
+}
+
+const pollStages = ['cursor_write', 'enrich', 'rpc_head', 'scan', 'subscription_read'] as const
+
+type PollStage = (typeof pollStages)[number]
+
+function createMetricProbe() {
+  let stages = emptyStages()
+  return {
+    reset() {
+      stages = emptyStages()
+    },
+    sink: Metrics.from({
+      count() {},
+      flush() {},
+      gauge() {},
+      histogram(name, value, tags) {
+        const stage = tags?.['stage']
+        if (name !== 'webhook_poller_stage_duration_ms' || !isPollStage(stage)) return
+        stages[stage] += value
+      },
+    }),
+    snapshot() {
+      return { ...stages }
+    },
+  }
+}
+
+function emptyStages(): Record<PollStage, number> {
+  return {
+    cursor_write: 0,
+    enrich: 0,
+    rpc_head: 0,
+    scan: 0,
+    subscription_read: 0,
+  }
+}
+
+function isPollStage(value: unknown): value is PollStage {
+  return typeof value === 'string' && pollStages.includes(value as PollStage)
+}
+
+async function createRpcProbe() {
+  const target = runtime.rpcUrl
+  if (!target) throw new Error('Localnet runtime is missing its RPC URL.')
+
+  let stats = emptyRpcStats()
+  const server = http.createServer((request, response) => {
+    void forwardRpc({ request, response, stats, target }).catch((error) => {
+      response.writeHead(502, { 'content-type': 'text/plain' })
+      response.end(error instanceof Error ? error.message : 'RPC probe failed.')
+    })
+  })
+  await new Promise<void>((resolve, reject) => {
+    const onError = (error: Error) => reject(error)
+    server.once('error', onError)
+    server.listen(0, '127.0.0.1', () => {
+      server.off('error', onError)
+      resolve()
+    })
+  })
+  const address = server.address()
+  if (!address || typeof address === 'string') throw new Error('RPC probe did not bind TCP.')
+
+  return {
+    close: () =>
+      new Promise<void>((resolve, reject) => {
+        server.close((error) => (error ? reject(error) : resolve()))
+      }),
+    reset() {
+      stats = emptyRpcStats()
+    },
+    snapshot(): RpcStats {
+      return { ...stats, methods: { ...stats.methods } }
+    },
+    url: `http://127.0.0.1:${address.port}`,
+  }
+}
+
+async function forwardRpc(options: {
+  request: http.IncomingMessage
+  response: http.ServerResponse
+  stats: RpcStats
+  target: string
+}): Promise<void> {
+  const { request, response, stats, target } = options
+  const chunks: Uint8Array[] = []
+  for await (const chunk of request)
+    chunks.push(typeof chunk === 'string' ? Buffer.from(chunk) : chunk)
+  const body = Buffer.concat(chunks)
+  stats.requestBytes += body.byteLength
+  stats.requests++
+  countRpcCalls(stats, body)
+
+  const headers = new Headers()
+  for (const name of ['authorization', 'content-type']) {
+    const value = request.headers[name]
+    if (typeof value === 'string') headers.set(name, value)
+  }
+  const upstream = await fetch(target, {
+    body,
+    headers,
+    method: request.method ?? 'POST',
+  })
+  const result = new Uint8Array(await upstream.arrayBuffer())
+  stats.responseBytes += result.byteLength
+  response.writeHead(upstream.status, {
+    'content-type': upstream.headers.get('content-type') ?? 'application/json',
+  })
+  response.end(result)
+}
+
+function emptyRpcStats(): RpcStats {
+  return { calls: 0, methods: {}, requestBytes: 0, requests: 0, responseBytes: 0 }
+}
+
+function countRpcCalls(stats: RpcStats, body: Uint8Array): void {
+  try {
+    const parsed = JSON.parse(Buffer.from(body).toString('utf8')) as unknown
+    for (const value of Array.isArray(parsed) ? parsed : [parsed]) {
+      if (!value || typeof value !== 'object') continue
+      const method = (value as Record<string, unknown>)['method']
+      if (typeof method !== 'string') continue
+      stats.calls++
+      stats.methods[method] = (stats.methods[method] ?? 0) + 1
+    }
+  } catch {}
+}
+
+function resetMeasurement(scenario: Scenario): void {
+  scenario.metrics.reset()
+  scenario.rpc.reset()
+}
+
+function recordMeasurement(scenario: Scenario, tickMs: number): void {
+  const rpc = scenario.rpc.snapshot()
+  const stages = scenario.metrics.snapshot()
+  const attributedMs = Object.values(stages).reduce((sum, value) => sum + value, 0)
+  scenario.measurements.push({
+    cursorWriteMs: stages.cursor_write,
+    enrichMs: stages.enrich,
+    rpcCalls: rpc.calls,
+    rpcHeadMs: stages.rpc_head,
+    rpcMethods: rpc.methods,
+    rpcRequestBytes: rpc.requestBytes,
+    rpcRequests: rpc.requests,
+    rpcResponseBytes: rpc.responseBytes,
+    scanMs: stages.scan,
+    subscriptionReadMs: stages.subscription_read,
+    tickMs,
+    unattributedMs: Math.max(0, tickMs - attributedMs),
+  })
+}
+
+function reportMeasurements(): void {
+  const summaries: Record<string, unknown>[] = []
+  for (const fixture of fixtures.values()) {
+    const scenarios = [
+      { label: 'caught-up tick', scenario: fixture.scenarios['caught-up'] },
+      { label: 'empty selective log block', scenario: fixture.scenarios['empty-log'] },
+      {
+        label: 'empty selective transaction block',
+        scenario: fixture.scenarios['empty-transaction'],
+      },
+      { label: 'empty selective transfer block', scenario: fixture.scenarios['empty-transfer'] },
+      { label: 'matching block fanout', scenario: fixture.scenarios['matching-block'] },
+    ]
+    for (const { label, scenario } of scenarios) {
+      if (!scenario) continue
+      const samples = scenario.measurements.slice(-scenario.samples)
+      summaries.push({
+        count: fixture.count,
+        cursorWriteMs: distribution(samples.map((sample) => sample.cursorWriteMs)),
+        enrichMs: distribution(samples.map((sample) => sample.enrichMs)),
+        label,
+        rpcCalls: distribution(samples.map((sample) => sample.rpcCalls)),
+        rpcHeadMs: distribution(samples.map((sample) => sample.rpcHeadMs)),
+        rpcMethods: summarizeMethods(samples),
+        rpcRequestBytes: distribution(samples.map((sample) => sample.rpcRequestBytes)),
+        rpcRequests: distribution(samples.map((sample) => sample.rpcRequests)),
+        rpcResponseBytes: distribution(samples.map((sample) => sample.rpcResponseBytes)),
+        scanMs: distribution(samples.map((sample) => sample.scanMs)),
+        subscriptionReadMs: distribution(samples.map((sample) => sample.subscriptionReadMs)),
+        tickMs: distribution(samples.map((sample) => sample.tickMs)),
+        unattributedMs: distribution(samples.map((sample) => sample.unattributedMs)),
+      })
+    }
+  }
+  console.log(`webhook benchmark instrumentation\n${JSON.stringify(summaries, null, 2)}`)
+}
+
+function summarizeMethods(samples: readonly Measurement[]): Record<string, number> {
+  const methods: Record<string, number> = {}
+  for (const sample of samples)
+    for (const [method, count] of Object.entries(sample.rpcMethods))
+      methods[method] = (methods[method] ?? 0) + count
+  for (const method of Object.keys(methods))
+    methods[method] = round(methods[method]! / samples.length)
+  return methods
+}
+
+function distribution(values: readonly number[]) {
+  const sorted = [...values].sort((a, b) => a - b)
+  const mean = sorted.reduce((sum, value) => sum + value, 0) / sorted.length
+  const percentile = (value: number) =>
+    sorted[Math.max(0, Math.ceil(sorted.length * value) - 1)] ?? 0
+  return {
+    mean: round(mean),
+    p50: round(percentile(0.5)),
+    p95: round(percentile(0.95)),
+  }
+}
+
+function round(value: number): number {
+  return Math.round(value * 100) / 100
+}
+
+async function resetCursors(scenario: Scenario, blockNumber: number): Promise<void> {
+  const cursor = Cursor.encode([blockNumber, endOfBlockIndex])
+  for (let offset = 0; offset < scenario.ids.length; offset += concurrency)
+    await Promise.all(
+      scenario.ids
+        .slice(offset, offset + concurrency)
+        .map((subscriptionId) => Webhooks.setCursor(scenario.db, subscriptionId, cursor)),
+    )
+}
+
+async function prepareScan(scenario: Scenario): Promise<void> {
+  scenario.nextHead = scenario.startBlockNumber + 1
+  await resetCursors(scenario, scenario.startBlockNumber)
+}
+
+function nextHead(scenario: Scenario): number {
+  if (scenario.nextHead > scenario.head)
+    throw new Error(`Benchmark exhausted its block window at ${scenario.head}.`)
+  return scenario.nextHead++
+}
+
+function getFixture(count: number): Fixture {
+  const fixture = fixtures.get(count)
+  if (!fixture) throw new Error(`Missing benchmark fixture for ${count} subscriptions.`)
+  return fixture
+}
+
+function getScenario(count: number, name: ScenarioName): Scenario {
+  const scenario = getFixture(count).scenarios[name]
+  if (!scenario) throw new Error(`Missing ${name} benchmark fixture for ${count} subscriptions.`)
+  return scenario
+}
+
+function assertDispatch(scenario: Scenario, expected: DispatchStats): void {
+  if (
+    scenario.dispatchStats.calls !== expected.calls ||
+    scenario.dispatchStats.messages !== expected.messages
+  )
+    throw new Error(
+      `Unexpected dispatches: ${JSON.stringify({
+        actual: scenario.dispatchStats,
+        expected,
+      })}`,
+    )
+}
+
+function assertResult(
+  result: Webhooks.createPoller.Result,
+  expected: { count: number; dispatched: number; scanned: number },
+): void {
+  if (
+    result.dispatched !== expected.dispatched ||
+    result.failed !== 0 ||
+    result.scanned !== expected.scanned ||
+    result.subscriptions !== expected.count
+  )
+    throw new Error(
+      `Unexpected poll result: ${JSON.stringify({
+        expected,
+        result,
+      })}`,
+    )
+}
+
+function resetDispatchStats(scenario: Scenario): void {
+  scenario.dispatchStats.calls = 0
+  scenario.dispatchStats.messages = 0
+}