tapimo 0.10.2 → 0.12.0

package/src/Webhooks.bench.ts

@@ -1,714 +0,0 @@
-/** @module-tag localnet */
-/**
- * Benchmarks the production poller against disposable PostgreSQL and Tempo services.
- * Set `WEBHOOK_BENCH_COUNTS`, `WEBHOOK_BENCH_ITERATIONS`, and
- * `WEBHOOK_BENCH_SCENARIOS` to focus a run.
- */
-import * as http from 'node:http'
-
-import { Webhooks } from 'tapimo'
-import { bench } from 'vite-plus/test'
-
-import * as Cursor from './internal/Cursor.js'
-import * as Metrics from './Metrics.js'
-import * as TestApp from '../test/App.js'
-import * as Runtime from '../test/runtime.js'
-import * as TestViem from '../test/Viem.js'
-
-const concurrency = 8
-const endOfBlockIndex = 2_147_483_647
-const noMatchAddress = '0xffffffffffffffffffffffffffffffffffffffff'
-const runtime = Runtime.get()
-const scenarioNames = [
-  'caught-up',
-  'empty-log',
-  'empty-transaction',
-  'empty-transfer',
-  'matching-block',
-] as const
-const transactionIterations = 1
-const warmupIterations = 3
-
-type DispatchStats = {
-  calls: number
-  messages: number
-}
-
-type Fixture = {
-  count: number
-  scenarios: Partial<Record<ScenarioName, Scenario>>
-}
-
-type Measurement = {
-  cursorWriteMs: number
-  enrichMs: number
-  rpcCalls: number
-  rpcHeadMs: number
-  rpcMethods: Record<string, number>
-  rpcRequestBytes: number
-  rpcRequests: number
-  rpcResponseBytes: number
-  scanMs: number
-  subscriptionReadMs: number
-  tickMs: number
-  unattributedMs: number
-}
-
-type RpcStats = {
-  calls: number
-  methods: Record<string, number>
-  requestBytes: number
-  requests: number
-  responseBytes: number
-}
-
-type Scenario = {
-  db: ReturnType<typeof TestApp.database>
-  dispatchStats: DispatchStats
-  head: number
-  ids: readonly string[]
-  measurements: Measurement[]
-  metrics: ReturnType<typeof createMetricProbe>
-  nextHead: number
-  poller: Webhooks.createPoller.Poller
-  rpc: Awaited<ReturnType<typeof createRpcProbe>>
-  samples: number
-  startBlockNumber: number
-}
-
-type ScenarioName = (typeof scenarioNames)[number]
-
-const counts = (() => {
-  const input = process.env.WEBHOOK_BENCH_COUNTS
-  if (!input) return [1, 100, 1_000]
-  const values = [...new Set(input.split(',').map(Number))].sort((a, b) => a - b)
-  if (values.length === 0 || values.some((value) => !Number.isSafeInteger(value) || value < 1))
-    throw new Error('WEBHOOK_BENCH_COUNTS must be a comma-separated list of positive integers.')
-  return values
-})()
-
-const iterations = (() => {
-  const input = process.env.WEBHOOK_BENCH_ITERATIONS
-  if (!input) return 10
-  const value = Number(input)
-  if (!Number.isSafeInteger(value) || value < 1)
-    throw new Error('WEBHOOK_BENCH_ITERATIONS must be a positive integer.')
-  return value
-})()
-
-const selectedScenarios = (() => {
-  const input = process.env.WEBHOOK_BENCH_SCENARIOS
-  if (!input) return new Set<ScenarioName>(scenarioNames)
-  const values = [...new Set(input.split(','))]
-  const unknown = values.filter((value) => !scenarioNames.includes(value as ScenarioName))
-  if (values.length === 0 || unknown.length > 0)
-    throw new Error(
-      `WEBHOOK_BENCH_SCENARIOS must contain: ${scenarioNames.join(', ')}. Unknown: ${unknown.join(', ')}.`,
-    )
-  return new Set(values as ScenarioName[])
-})()
-
-const fixtures = new Map<number, Fixture>()
-let rpcProbe: Awaited<ReturnType<typeof createRpcProbe>> | undefined
-
-beforeAll(async () => {
-  if (runtime.mode !== 'localnet')
-    throw new Error('Run webhook benchmarks with TEMPO_ENV=localnet.')
-  rpcProbe = await createRpcProbe()
-  for (const count of counts) fixtures.set(count, await createFixture(count, rpcProbe))
-}, 180_000)
-
-afterAll(async () => {
-  reportMeasurements()
-  for (const fixture of fixtures.values())
-    for (const scenario of Object.values(fixture.scenarios)) await scenario.db.close()
-  await rpcProbe?.close()
-})
-
-describe('webhook poller', () => {
-  for (const count of counts) {
-    if (selectedScenarios.has('caught-up'))
-      bench(
-        `caught-up tick / ${count} subscriptions`,
-        async () => {
-          const scenario = getScenario(count, 'caught-up')
-          resetDispatchStats(scenario)
-          resetMeasurement(scenario)
-          const startedAt = performance.now()
-          const result = await scenario.poller.tick({
-            confirmations: 0,
-            eventTypes: ['block:created'],
-            headBlockNumber: scenario.head,
-            maxBlocks: 1,
-          })
-          recordMeasurement(scenario, performance.now() - startedAt)
-          assertDispatch(scenario, { calls: 0, messages: 0 })
-          assertResult(result, { count, dispatched: 0, scanned: 0 })
-        },
-        {
-          iterations,
-          setup: async () => {
-            const scenario = getScenario(count, 'caught-up')
-            await resetCursors(scenario, scenario.head)
-          },
-          time: 0,
-          throws: true,
-          warmupIterations,
-          warmupTime: 0,
-        },
-      )
-
-    if (selectedScenarios.has('empty-transfer'))
-      bench(
-        `empty selective transfer block / ${count} subscriptions`,
-        async () => {
-          const scenario = getScenario(count, 'empty-transfer')
-          resetDispatchStats(scenario)
-          resetMeasurement(scenario)
-          const startedAt = performance.now()
-          const result = await scenario.poller.tick({
-            confirmations: 0,
-            eventTypes: ['token:transfer'],
-            headBlockNumber: nextHead(scenario),
-            maxBlocks: 1,
-          })
-          recordMeasurement(scenario, performance.now() - startedAt)
-          assertDispatch(scenario, { calls: 0, messages: 0 })
-          assertResult(result, { count, dispatched: 0, scanned: 0 })
-        },
-        {
-          iterations,
-          setup: async () => {
-            const scenario = getScenario(count, 'empty-transfer')
-            await prepareScan(scenario)
-          },
-          time: 0,
-          throws: true,
-          warmupIterations,
-          warmupTime: 0,
-        },
-      )
-
-    if (selectedScenarios.has('empty-log'))
-      bench(
-        `empty selective log block / ${count} subscriptions`,
-        async () => {
-          const scenario = getScenario(count, 'empty-log')
-          resetDispatchStats(scenario)
-          resetMeasurement(scenario)
-          const startedAt = performance.now()
-          const result = await scenario.poller.tick({
-            confirmations: 0,
-            eventTypes: ['log:emitted'],
-            headBlockNumber: nextHead(scenario),
-            maxBlocks: 1,
-          })
-          recordMeasurement(scenario, performance.now() - startedAt)
-          assertDispatch(scenario, { calls: 0, messages: 0 })
-          assertResult(result, { count, dispatched: 0, scanned: 0 })
-        },
-        {
-          iterations,
-          setup: async () => {
-            const scenario = getScenario(count, 'empty-log')
-            await prepareScan(scenario)
-          },
-          time: 0,
-          throws: true,
-          warmupIterations,
-          warmupTime: 0,
-        },
-      )
-
-    if (selectedScenarios.has('empty-transaction'))
-      bench(
-        `empty selective transaction block / ${count} subscriptions`,
-        async () => {
-          const scenario = getScenario(count, 'empty-transaction')
-          resetDispatchStats(scenario)
-          resetMeasurement(scenario)
-          const startedAt = performance.now()
-          const result = await scenario.poller.tick({
-            confirmations: 0,
-            eventTypes: ['transaction:included'],
-            headBlockNumber: nextHead(scenario),
-            maxBlocks: 1,
-          })
-          recordMeasurement(scenario, performance.now() - startedAt)
-          assertDispatch(scenario, { calls: 0, messages: 0 })
-          assertResult(result, { count, dispatched: 0, scanned: 0 })
-        },
-        {
-          iterations: transactionIterations,
-          setup: async () => {
-            const scenario = getScenario(count, 'empty-transaction')
-            await prepareScan(scenario)
-          },
-          time: 0,
-          throws: true,
-          warmupIterations: 0,
-          warmupTime: 0,
-        },
-      )
-
-    if (selectedScenarios.has('matching-block'))
-      bench(
-        `matching block fanout / ${count} subscriptions`,
-        async () => {
-          const scenario = getScenario(count, 'matching-block')
-          resetDispatchStats(scenario)
-          resetMeasurement(scenario)
-          const startedAt = performance.now()
-          const result = await scenario.poller.tick({
-            confirmations: 0,
-            eventTypes: ['block:created'],
-            headBlockNumber: nextHead(scenario),
-            maxBlocks: 1,
-          })
-          recordMeasurement(scenario, performance.now() - startedAt)
-          assertDispatch(scenario, { calls: count, messages: count })
-          assertResult(result, { count, dispatched: count, scanned: count })
-        },
-        {
-          iterations,
-          setup: async () => {
-            const scenario = getScenario(count, 'matching-block')
-            await prepareScan(scenario)
-          },
-          time: 0,
-          throws: true,
-          warmupIterations,
-          warmupTime: 0,
-        },
-      )
-  }
-})
-
-async function createFixture(
-  count: number,
-  rpc: Awaited<ReturnType<typeof createRpcProbe>>,
-): Promise<Fixture> {
-  const client = TestViem.getClient()
-  const liveHead = Number(await client.getBlockNumber())
-  const head = Math.max(1, liveHead - 1)
-  const startBlockNumber = Math.max(0, head - Math.max(iterations, warmupIterations) - 1)
-  const scenarios: Partial<Record<ScenarioName, Scenario>> = {}
-  if (selectedScenarios.has('caught-up'))
-    scenarios['caught-up'] = await createScenario({
-      count,
-      cursorBlockNumber: head,
-      eventType: 'block:created',
-      filters: {},
-      head,
-      rpc,
-    })
-  if (selectedScenarios.has('empty-log'))
-    scenarios['empty-log'] = await createScenario({
-      count,
-      cursorBlockNumber: startBlockNumber,
-      eventType: 'log:emitted',
-      filters: { address: noMatchAddress },
-      head,
-      rpc,
-    })
-  if (selectedScenarios.has('empty-transaction'))
-    scenarios['empty-transaction'] = await createScenario({
-      count,
-      cursorBlockNumber: Math.max(0, head - transactionIterations),
-      eventType: 'transaction:included',
-      filters: { from: noMatchAddress },
-      head,
-      rpc,
-      samples: transactionIterations,
-    })
-  if (selectedScenarios.has('empty-transfer'))
-    scenarios['empty-transfer'] = await createScenario({
-      count,
-      cursorBlockNumber: startBlockNumber,
-      eventType: 'token:transfer',
-      filters: { sender: noMatchAddress },
-      head,
-      rpc,
-    })
-  if (selectedScenarios.has('matching-block'))
-    scenarios['matching-block'] = await createScenario({
-      count,
-      cursorBlockNumber: startBlockNumber,
-      eventType: 'block:created',
-      filters: {},
-      head,
-      rpc,
-    })
-  return {
-    count,
-    scenarios,
-  }
-}
-
-async function createScenario(options: createScenario.Options): Promise<Scenario> {
-  const source = TestApp.databaseFactory()
-  const db = source()
-  const dispatchStats = { calls: 0, messages: 0 }
-  const metrics = createMetricProbe()
-  const ids = await createSubscriptions({
-    count: options.count,
-    cursor: Cursor.encode([options.cursorBlockNumber, endOfBlockIndex]),
-    db,
-    eventType: options.eventType,
-    filters: options.filters,
-  })
-  return {
-    db,
-    dispatchStats,
-    head: options.head,
-    ids,
-    measurements: [],
-    metrics,
-    nextHead: options.cursorBlockNumber + 1,
-    poller: Webhooks.createPoller({
-      db: source,
-      defaultChainId: runtime.chainId,
-      dispatchBatch: async (dispatchables) => {
-        dispatchStats.calls++
-        dispatchStats.messages += dispatchables.length
-        return { ok: true }
-      },
-      metrics: metrics.sink,
-      rpc: { ...TestApp.rpc, url: options.rpc.url },
-    }),
-    rpc: options.rpc,
-    samples: options.samples ?? iterations,
-    startBlockNumber: options.cursorBlockNumber,
-  }
-}
-
-declare namespace createScenario {
-  type Options = {
-    count: number
-    cursorBlockNumber: number
-    eventType: Webhooks.EventType
-    filters: Record<string, unknown>
-    head: number
-    rpc: Awaited<ReturnType<typeof createRpcProbe>>
-    samples?: number | undefined
-  }
-}
-
-async function createSubscriptions(options: createSubscriptions.Options): Promise<string[]> {
-  const ids: string[] = []
-  for (let offset = 0; offset < options.count; offset += concurrency) {
-    const batch = Array.from(
-      { length: Math.min(concurrency, options.count - offset) },
-      async (_, index) => {
-        const subscription = await Webhooks.createSubscription(options.db, {
-          chainId: runtime.chainId,
-          destination: {
-            type: 'url',
-            url: `https://hooks.example.com/bench/${options.eventType}/${offset + index}`,
-          },
-          eventType: options.eventType,
-          filters: options.filters,
-          owner: { orgId: 'org_benchmark', type: 'api_key' },
-        })
-        await Webhooks.setCursor(options.db, subscription.id, options.cursor)
-        return subscription.id
-      },
-    )
-    ids.push(...(await Promise.all(batch)))
-  }
-  return ids
-}
-
-declare namespace createSubscriptions {
-  type Options = {
-    count: number
-    cursor: string
-    db: ReturnType<typeof TestApp.database>
-    eventType: Webhooks.EventType
-    filters: Record<string, unknown>
-  }
-}
-
-const pollStages = ['cursor_write', 'enrich', 'rpc_head', 'scan', 'subscription_read'] as const
-
-type PollStage = (typeof pollStages)[number]
-
-function createMetricProbe() {
-  let stages = emptyStages()
-  return {
-    reset() {
-      stages = emptyStages()
-    },
-    sink: Metrics.from({
-      count() {},
-      flush() {},
-      gauge() {},
-      histogram(name, value, tags) {
-        const stage = tags?.['stage']
-        if (name !== 'webhook_poller_stage_duration_ms' || !isPollStage(stage)) return
-        stages[stage] += value
-      },
-    }),
-    snapshot() {
-      return { ...stages }
-    },
-  }
-}
-
-function emptyStages(): Record<PollStage, number> {
-  return {
-    cursor_write: 0,
-    enrich: 0,
-    rpc_head: 0,
-    scan: 0,
-    subscription_read: 0,
-  }
-}
-
-function isPollStage(value: unknown): value is PollStage {
-  return typeof value === 'string' && pollStages.includes(value as PollStage)
-}
-
-async function createRpcProbe() {
-  const target = runtime.rpcUrl
-  if (!target) throw new Error('Localnet runtime is missing its RPC URL.')
-
-  let stats = emptyRpcStats()
-  const server = http.createServer((request, response) => {
-    void forwardRpc({ request, response, stats, target }).catch((error) => {
-      response.writeHead(502, { 'content-type': 'text/plain' })
-      response.end(error instanceof Error ? error.message : 'RPC probe failed.')
-    })
-  })
-  await new Promise<void>((resolve, reject) => {
-    const onError = (error: Error) => reject(error)
-    server.once('error', onError)
-    server.listen(0, '127.0.0.1', () => {
-      server.off('error', onError)
-      resolve()
-    })
-  })
-  const address = server.address()
-  if (!address || typeof address === 'string') throw new Error('RPC probe did not bind TCP.')
-
-  return {
-    close: () =>
-      new Promise<void>((resolve, reject) => {
-        server.close((error) => (error ? reject(error) : resolve()))
-      }),
-    reset() {
-      stats = emptyRpcStats()
-    },
-    snapshot(): RpcStats {
-      return { ...stats, methods: { ...stats.methods } }
-    },
-    url: `http://127.0.0.1:${address.port}`,
-  }
-}
-
-async function forwardRpc(options: {
-  request: http.IncomingMessage
-  response: http.ServerResponse
-  stats: RpcStats
-  target: string
-}): Promise<void> {
-  const { request, response, stats, target } = options
-  const chunks: Uint8Array[] = []
-  for await (const chunk of request)
-    chunks.push(typeof chunk === 'string' ? Buffer.from(chunk) : chunk)
-  const body = Buffer.concat(chunks)
-  stats.requestBytes += body.byteLength
-  stats.requests++
-  countRpcCalls(stats, body)
-
-  const headers = new Headers()
-  for (const name of ['authorization', 'content-type']) {
-    const value = request.headers[name]
-    if (typeof value === 'string') headers.set(name, value)
-  }
-  const upstream = await fetch(target, {
-    body,
-    headers,
-    method: request.method ?? 'POST',
-  })
-  const result = new Uint8Array(await upstream.arrayBuffer())
-  stats.responseBytes += result.byteLength
-  response.writeHead(upstream.status, {
-    'content-type': upstream.headers.get('content-type') ?? 'application/json',
-  })
-  response.end(result)
-}
-
-function emptyRpcStats(): RpcStats {
-  return { calls: 0, methods: {}, requestBytes: 0, requests: 0, responseBytes: 0 }
-}
-
-function countRpcCalls(stats: RpcStats, body: Uint8Array): void {
-  try {
-    const parsed = JSON.parse(Buffer.from(body).toString('utf8')) as unknown
-    for (const value of Array.isArray(parsed) ? parsed : [parsed]) {
-      if (!value || typeof value !== 'object') continue
-      const method = (value as Record<string, unknown>)['method']
-      if (typeof method !== 'string') continue
-      stats.calls++
-      stats.methods[method] = (stats.methods[method] ?? 0) + 1
-    }
-  } catch {}
-}
-
-function resetMeasurement(scenario: Scenario): void {
-  scenario.metrics.reset()
-  scenario.rpc.reset()
-}
-
-function recordMeasurement(scenario: Scenario, tickMs: number): void {
-  const rpc = scenario.rpc.snapshot()
-  const stages = scenario.metrics.snapshot()
-  const attributedMs = Object.values(stages).reduce((sum, value) => sum + value, 0)
-  scenario.measurements.push({
-    cursorWriteMs: stages.cursor_write,
-    enrichMs: stages.enrich,
-    rpcCalls: rpc.calls,
-    rpcHeadMs: stages.rpc_head,
-    rpcMethods: rpc.methods,
-    rpcRequestBytes: rpc.requestBytes,
-    rpcRequests: rpc.requests,
-    rpcResponseBytes: rpc.responseBytes,
-    scanMs: stages.scan,
-    subscriptionReadMs: stages.subscription_read,
-    tickMs,
-    unattributedMs: Math.max(0, tickMs - attributedMs),
-  })
-}
-
-function reportMeasurements(): void {
-  const summaries: Record<string, unknown>[] = []
-  for (const fixture of fixtures.values()) {
-    const scenarios = [
-      { label: 'caught-up tick', scenario: fixture.scenarios['caught-up'] },
-      { label: 'empty selective log block', scenario: fixture.scenarios['empty-log'] },
-      {
-        label: 'empty selective transaction block',
-        scenario: fixture.scenarios['empty-transaction'],
-      },
-      { label: 'empty selective transfer block', scenario: fixture.scenarios['empty-transfer'] },
-      { label: 'matching block fanout', scenario: fixture.scenarios['matching-block'] },
-    ]
-    for (const { label, scenario } of scenarios) {
-      if (!scenario) continue
-      const samples = scenario.measurements.slice(-scenario.samples)
-      summaries.push({
-        count: fixture.count,
-        cursorWriteMs: distribution(samples.map((sample) => sample.cursorWriteMs)),
-        enrichMs: distribution(samples.map((sample) => sample.enrichMs)),
-        label,
-        rpcCalls: distribution(samples.map((sample) => sample.rpcCalls)),
-        rpcHeadMs: distribution(samples.map((sample) => sample.rpcHeadMs)),
-        rpcMethods: summarizeMethods(samples),
-        rpcRequestBytes: distribution(samples.map((sample) => sample.rpcRequestBytes)),
-        rpcRequests: distribution(samples.map((sample) => sample.rpcRequests)),
-        rpcResponseBytes: distribution(samples.map((sample) => sample.rpcResponseBytes)),
-        scanMs: distribution(samples.map((sample) => sample.scanMs)),
-        subscriptionReadMs: distribution(samples.map((sample) => sample.subscriptionReadMs)),
-        tickMs: distribution(samples.map((sample) => sample.tickMs)),
-        unattributedMs: distribution(samples.map((sample) => sample.unattributedMs)),
-      })
-    }
-  }
-  console.log(`webhook benchmark instrumentation\n${JSON.stringify(summaries, null, 2)}`)
-}
-
-function summarizeMethods(samples: readonly Measurement[]): Record<string, number> {
-  const methods: Record<string, number> = {}
-  for (const sample of samples)
-    for (const [method, count] of Object.entries(sample.rpcMethods))
-      methods[method] = (methods[method] ?? 0) + count
-  for (const method of Object.keys(methods))
-    methods[method] = round(methods[method]! / samples.length)
-  return methods
-}
-
-function distribution(values: readonly number[]) {
-  const sorted = [...values].sort((a, b) => a - b)
-  const mean = sorted.reduce((sum, value) => sum + value, 0) / sorted.length
-  const percentile = (value: number) =>
-    sorted[Math.max(0, Math.ceil(sorted.length * value) - 1)] ?? 0
-  return {
-    mean: round(mean),
-    p50: round(percentile(0.5)),
-    p95: round(percentile(0.95)),
-  }
-}
-
-function round(value: number): number {
-  return Math.round(value * 100) / 100
-}
-
-async function resetCursors(scenario: Scenario, blockNumber: number): Promise<void> {
-  const cursor = Cursor.encode([blockNumber, endOfBlockIndex])
-  for (let offset = 0; offset < scenario.ids.length; offset += concurrency)
-    await Promise.all(
-      scenario.ids
-        .slice(offset, offset + concurrency)
-        .map((subscriptionId) => Webhooks.setCursor(scenario.db, subscriptionId, cursor)),
-    )
-}
-
-async function prepareScan(scenario: Scenario): Promise<void> {
-  scenario.nextHead = scenario.startBlockNumber + 1
-  await resetCursors(scenario, scenario.startBlockNumber)
-}
-
-function nextHead(scenario: Scenario): number {
-  if (scenario.nextHead > scenario.head)
-    throw new Error(`Benchmark exhausted its block window at ${scenario.head}.`)
-  return scenario.nextHead++
-}
-
-function getFixture(count: number): Fixture {
-  const fixture = fixtures.get(count)
-  if (!fixture) throw new Error(`Missing benchmark fixture for ${count} subscriptions.`)
-  return fixture
-}
-
-function getScenario(count: number, name: ScenarioName): Scenario {
-  const scenario = getFixture(count).scenarios[name]
-  if (!scenario) throw new Error(`Missing ${name} benchmark fixture for ${count} subscriptions.`)
-  return scenario
-}
-
-function assertDispatch(scenario: Scenario, expected: DispatchStats): void {
-  if (
-    scenario.dispatchStats.calls !== expected.calls ||
-    scenario.dispatchStats.messages !== expected.messages
-  )
-    throw new Error(
-      `Unexpected dispatches: ${JSON.stringify({
-        actual: scenario.dispatchStats,
-        expected,
-      })}`,
-    )
-}
-
-function assertResult(
-  result: Webhooks.createPoller.Result,
-  expected: { count: number; dispatched: number; scanned: number },
-): void {
-  if (
-    result.dispatched !== expected.dispatched ||
-    result.failed !== 0 ||
-    result.scanned !== expected.scanned ||
-    result.subscriptions !== expected.count
-  )
-    throw new Error(
-      `Unexpected poll result: ${JSON.stringify({
-        expected,
-        result,
-      })}`,
-    )
-}
-
-function resetDispatchStats(scenario: Scenario): void {
-  scenario.dispatchStats.calls = 0
-  scenario.dispatchStats.messages = 0
-}