tapimo 0.8.0 → 0.10.0

package/src/internal/WebhookTransfer.ts

@@ -0,0 +1,426 @@
+import { encodeEventTopics, parseAbiItem, type RpcLog } from 'viem'
+import * as z from 'zod/mini'
+
+import * as Transfers from '../apps/data/routes/transfers.js'
+import * as Schema from './Schema.js'
+import * as Value from './Value.js'
+import type * as Viem from './Viem.js'
+
+/** Subscriptions matched per shared transfer scan. */
+export const scanCohortSize = 32
+
+/**
+ * Blocks per transfer `eth_getLogs` request. Firehose scans stop after a
+ * 100-row page, so small windows bound the raw RPC body retained before that cap.
+ */
+const scanChunkBlocks = 8
+
+const event = parseAbiItem(Transfers.signature)
+const topic = encodeEventTopics({ abi: [event] })[0]
+
+type Filters = z.output<typeof Transfers.schema.Predicates>
+
+type EventData = {
+  address: string
+  amount: string
+  blockNumber: number
+  logIndex: number
+  recipient: string
+  sender: string
+  timestamp: string
+  transactionHash: string
+}
+
+type Candidate = {
+  index: number
+  key: string
+  state: scan.State & { window: scan.Window }
+}
+
+type Cohort = {
+  key: string
+  states: Candidate[]
+  window: { from: number; to: number }
+}
+
+type Accumulator = {
+  full: boolean
+  items: scan.Item[]
+  state: Candidate['state']
+}
+
+/** Returns a stable key for transfer filters with equivalent RPC pushdowns. */
+function scanKey(filters_input: Record<string, unknown>): string {
+  const filters = filters_input as Filters
+  return JSON.stringify(
+    normalizeRpcValue({
+      args: logArgs(filters),
+      token: filters.token,
+    }),
+  )
+}
+
+/** Scans transfer subscriptions through shared, memory-bounded RPC cohorts. */
+export async function scan(options: scan.Options): Promise<PromiseSettledResult<scan.Page>[]> {
+  const results: PromiseSettledResult<scan.Page>[] = []
+  const candidates: Candidate[] = []
+  for (let index = 0; index < options.states.length; index++) {
+    const state = options.states[index]!
+    if (!state.window) {
+      results[index] = { status: 'fulfilled', value: { items: [] } }
+      continue
+    }
+    candidates.push({
+      index,
+      key: scanKey(state.filters),
+      state: { ...state, window: state.window },
+    })
+  }
+
+  candidates.sort(
+    (a, b) =>
+      a.key.localeCompare(b.key) ||
+      a.state.window.from - b.state.window.from ||
+      a.state.window.to - b.state.window.to,
+  )
+  const cohorts: Cohort[] = []
+  for (const candidate of candidates) {
+    const previous = cohorts.at(-1)
+    const combinedBlocks = previous
+      ? Math.max(previous.window.to, candidate.state.window.to) - previous.window.from + 1
+      : 0
+    if (
+      !previous ||
+      candidate.key !== previous.key ||
+      candidate.state.window.from > previous.window.to ||
+      combinedBlocks > options.maxBlocks ||
+      previous.states.length >= scanCohortSize
+    ) {
+      cohorts.push({
+        key: candidate.key,
+        states: [candidate],
+        window: { from: candidate.state.window.from, to: candidate.state.window.to },
+      })
+      continue
+    }
+    previous.states.push(candidate)
+    previous.window.to = Math.max(previous.window.to, candidate.state.window.to)
+  }
+
+  const settled = await mapWithLimit(cohorts, options.concurrency, (cohort) =>
+    options.schedule(async () => ({
+      cohort,
+      pages: await scanCohort({ client: options.client, cohort }),
+    })),
+  )
+  for (const entry of settled) {
+    if (entry.status === 'rejected') continue
+    for (let index = 0; index < entry.value.cohort.states.length; index++)
+      results[entry.value.cohort.states[index]!.index] = {
+        status: 'fulfilled',
+        value: entry.value.pages[index]!,
+      }
+  }
+  for (let index = 0; index < settled.length; index++) {
+    const entry = settled[index]!
+    if (entry.status !== 'rejected') continue
+    for (const candidate of cohorts[index]!.states)
+      results[candidate.index] = { reason: entry.reason, status: 'rejected' }
+  }
+  return results
+}
+
+export declare namespace scan {
+  /** One transfer match paired with its keyset position. */
+  type Item = {
+    /** Transfer position as `[blockNumber, logIndex]`. */
+    cursor: readonly [number, number]
+    /** Decoded webhook transfer payload. */
+    data: unknown
+  }
+
+  /** One bounded page returned to the poller. */
+  type Page = {
+    /** Whether another matching page remains. */
+    hasMore?: boolean | undefined
+    /** Transfer matches in ascending chain order. */
+    items: readonly Item[]
+    /** Highest block fully scanned when no matched cursor must be retained. */
+    scannedThrough?: number | undefined
+  }
+
+  /** One subscription's inputs for a shared transfer scan. */
+  type State = {
+    /** Validated transfer subscription filters. */
+    filters: Record<string, unknown>
+    /** Maximum matches retained for this fair scan page. */
+    limit: number
+    /** Bounded chain window, or undefined when the cursor is caught up. */
+    window?: Window | undefined
+  }
+
+  /** One subscription's chain range and already-dispatched position. */
+  type Window = {
+    /** First block included in the upstream scan. */
+    from: number
+    /** Last dispatched `[blockNumber, logIndex]` position. */
+    skip: readonly [number, number]
+    /** Last block included in the upstream scan. */
+    to: number
+  }
+
+  /** Inputs for scanning transfer subscriptions. */
+  type Options = {
+    /** Maximum concurrent cohort RPC scans. */
+    concurrency: number
+    /** Viem client for raw chain RPC requests. */
+    client: Viem.getClient.ReturnType
+    /** Maximum combined block range for one cohort. */
+    maxBlocks: number
+    /** Shared scheduler that bounds upstream scans across event types. */
+    schedule: <value>(fn: () => Promise<value>) => Promise<value>
+    /** Subscription states aligned with the returned results. */
+    states: readonly State[]
+  }
+}
+
+/** Runs ordered async work with bounded concurrency. */
+async function mapWithLimit<item, value>(
+  items: readonly item[],
+  limit: number,
+  fn: (item: item) => Promise<value>,
+): Promise<PromiseSettledResult<value>[]> {
+  const results: PromiseSettledResult<value>[] = []
+  let next = 0
+  await Promise.all(
+    Array.from({ length: Math.min(limit, items.length) }, async () => {
+      while (next < items.length) {
+        const index = next++
+        try {
+          results[index] = { status: 'fulfilled', value: await fn(items[index]!) }
+        } catch (reason) {
+          results[index] = { reason, status: 'rejected' }
+        }
+      }
+    }),
+  )
+  return results
+}
+
+/** Recursively canonicalizes an RPC filter value for query grouping. */
+function normalizeRpcValue(value: unknown): unknown {
+  if (typeof value === 'bigint') return `bigint:${value}`
+  if (Array.isArray(value)) return value.map(normalizeRpcValue)
+  if (value && typeof value === 'object')
+    return Object.fromEntries(
+      Object.entries(value)
+        .filter(([, item]) => item !== undefined)
+        .sort(([a], [b]) => a.localeCompare(b))
+        .map(([name, item]) => [name, normalizeRpcValue(item)]),
+    )
+  return value
+}
+
+/** Returns indexed argument combinations pushed into Transfer log reads. */
+function logArgs(filters: Filters) {
+  const args = {
+    ...(filters.sender ? { from: filters.sender } : {}),
+    ...(filters.recipient ? { to: filters.recipient } : {}),
+  }
+  return filters.address && !filters.sender && !filters.recipient
+    ? [{ from: filters.address }, { to: filters.address }]
+    : [args]
+}
+
+/** Encodes one indexed address as its left-padded event topic. */
+function addressTopic(address: string): `0x${string}` {
+  return `0x${'0'.repeat(24)}${address.slice(2).toLowerCase()}`
+}
+
+/** Builds the exact Transfer topic filter for one indexed-argument combination. */
+function topics(args: { from?: string | undefined; to?: string | undefined }) {
+  const values: (`0x${string}` | null)[] = [
+    topic,
+    args.from ? addressTopic(args.from) : null,
+    args.to ? addressTopic(args.to) : null,
+  ]
+  while (values.at(-1) === null) values.pop()
+  return values
+}
+
+/** Renders a block number as a JSON-RPC block tag. */
+function blockTag(value: number): `0x${string}` {
+  return `0x${value.toString(16)}`
+}
+
+/** Issues an untyped JSON-RPC request through the Viem client. */
+function rpc<result>(
+  client: Viem.getClient.ReturnType,
+  method: string,
+  params: readonly unknown[],
+): Promise<result> {
+  return (client.request as (args: { method: string; params: unknown[] }) => Promise<result>)({
+    method,
+    params: [...params],
+  })
+}
+
+/** Reads raw Transfer logs, splitting the either-side shortcut into two RPC filters. */
+async function getLogs(
+  client: Viem.getClient.ReturnType,
+  filters: Filters,
+  window: { from: number; to: number },
+): Promise<RpcLog[]> {
+  const pages = await Promise.all(
+    logArgs(filters).map((args) =>
+      rpc<RpcLog[]>(client, 'eth_getLogs', [
+        {
+          ...(filters.token ? { address: filters.token } : {}),
+          fromBlock: blockTag(window.from),
+          topics: topics(args),
+          toBlock: blockTag(window.to),
+        },
+      ]),
+    ),
+  )
+  const logs = new Map<string, RpcLog>()
+  for (const page of pages)
+    for (const log of page)
+      logs.set(`${log.blockNumber}:${log.logIndex}:${log.transactionHash}`, log)
+  return [...logs.values()]
+}
+
+/** Sorts raw logs into ascending `(blockNumber, logIndex)` order. */
+function byPosition(a: RpcLog, b: RpcLog): number {
+  const block = (Value.hexToNumber(a.blockNumber) ?? 0) - (Value.hexToNumber(b.blockNumber) ?? 0)
+  return block || (Value.hexToNumber(a.logIndex) ?? 0) - (Value.hexToNumber(b.logIndex) ?? 0)
+}
+
+/** True when an address is in Tempo's deterministic TIP-20 contract range. */
+function isTip20Address(address: string): boolean {
+  const value = address.toLowerCase()
+  return value >= Transfers.tip20AddressRange[0] && value < Transfers.tip20AddressRange[1]
+}
+
+/** Decodes one strict TIP-20 Transfer wire payload. */
+function decode(log: RpcLog, timestamp: string): EventData | undefined {
+  if (!isTip20Address(log.address)) return undefined
+  if (
+    log.topics.length !== 3 ||
+    log.topics[0]?.toLowerCase() !== topic.toLowerCase() ||
+    !/^0x0{24}[0-9a-fA-F]{40}$/.test(log.topics[1] ?? '') ||
+    !/^0x0{24}[0-9a-fA-F]{40}$/.test(log.topics[2] ?? '') ||
+    !/^0x[0-9a-fA-F]{64}$/.test(log.data)
+  )
+    return undefined
+  const address = Schema.TokenAddress.safeParse(log.address)
+  const recipient = Schema.Address.safeParse(`0x${log.topics[2]!.slice(-40)}`)
+  const sender = Schema.Address.safeParse(`0x${log.topics[1]!.slice(-40)}`)
+  const transactionHash = Schema.Hash.safeParse(log.transactionHash)
+  const amount = Value.fromHex(log.data)
+  const blockNumber = Value.hexToNumber(log.blockNumber)
+  const logIndex = Value.hexToNumber(log.logIndex)
+  if (
+    !address.success ||
+    !recipient.success ||
+    !sender.success ||
+    !transactionHash.success ||
+    amount === undefined ||
+    blockNumber === undefined ||
+    logIndex === undefined
+  )
+    return undefined
+  return {
+    address: address.data,
+    amount,
+    blockNumber,
+    logIndex,
+    recipient: recipient.data,
+    sender: sender.data,
+    timestamp,
+    transactionHash: transactionHash.data,
+  }
+}
+
+/** Re-checks every transfer filter after RPC topic pushdown. */
+function matches(data: EventData, filters: Filters): boolean {
+  if (filters.address && data.sender !== filters.address && data.recipient !== filters.address)
+    return false
+  if (filters.recipient && data.recipient !== filters.recipient) return false
+  if (filters.sender && data.sender !== filters.sender) return false
+  if (filters.token && data.address !== filters.token) return false
+  return true
+}
+
+/** Adds a matched row until one subscription reaches its fair-page limit. */
+function addMatch(accumulator: Accumulator, item: scan.Item): void {
+  if (accumulator.full) return
+  accumulator.items.push(item)
+  accumulator.full = accumulator.items.length >= accumulator.state.limit
+}
+
+/** Checks whether a canonical row remains eligible for one subscription. */
+function canMatch(accumulator: Accumulator, blockNumber: number, logIndex: number): boolean {
+  const { skip, to } = accumulator.state.window
+  return (
+    !accumulator.full &&
+    blockNumber <= to &&
+    (blockNumber > skip[0] || (blockNumber === skip[0] && logIndex > skip[1]))
+  )
+}
+
+/** Converts match buffers into cursor-aware poller pages. */
+function finish(accumulators: readonly Accumulator[]): scan.Page[] {
+  return accumulators.map((accumulator) => {
+    if (accumulator.full) return { hasMore: true, items: accumulator.items }
+    if (accumulator.items.length > 0) return { items: accumulator.items }
+    return { items: accumulator.items, scannedThrough: accumulator.state.window.to }
+  })
+}
+
+/** Scans one canonical transfer stream and fans rows into matching subscriptions. */
+async function scanCohort(options: {
+  client: Viem.getClient.ReturnType
+  cohort: Cohort
+}): Promise<scan.Page[]> {
+  const { client, cohort } = options
+  const accumulators = cohort.states.map<Accumulator>(({ state }) => ({
+    full: false,
+    items: [],
+    state,
+  }))
+  const timestamps = new Map<number, Promise<string>>()
+  const timestamp = (blockNumber: number) => {
+    const existing = timestamps.get(blockNumber)
+    if (existing) return existing
+    const pending = client
+      .getBlock({ blockNumber: BigInt(blockNumber), includeTransactions: false })
+      .then((block) => new Date(Number(block.timestamp) * 1_000).toISOString())
+    timestamps.set(blockNumber, pending)
+    return pending
+  }
+  const filters_source = cohort.states[0]!.state.filters as Filters
+  // RPC pushdowns bound selective responses; only the unfiltered firehose needs small range chunks.
+  const chunkBlocks = Object.values(filters_source).some(Boolean)
+    ? cohort.window.to - cohort.window.from + 1
+    : scanChunkBlocks
+
+  for (let from = cohort.window.from; from <= cohort.window.to; from += chunkBlocks) {
+    const to = Math.min(cohort.window.to, from + chunkBlocks - 1)
+    const logs = (await getLogs(client, filters_source, { from, to })).sort(byPosition)
+    for (const log of logs) {
+      const blockNumber = Value.hexToNumber(log.blockNumber)
+      const logIndex = Value.hexToNumber(log.logIndex)
+      if (blockNumber === undefined || logIndex === undefined) continue
+      const data = decode(log, await timestamp(blockNumber))
+      if (!data) continue
+      for (const accumulator of accumulators) {
+        if (!canMatch(accumulator, blockNumber, logIndex)) continue
+        if (matches(data, accumulator.state.filters as Filters))
+          addMatch(accumulator, { cursor: [blockNumber, logIndex], data })
+      }
+      if (accumulators.every((accumulator) => accumulator.full)) return finish(accumulators)
+    }
+  }
+  return finish(accumulators)
+}

package/src/internal/Webhooks.test.ts

@@ -1,5 +1,6 @@
 import { Webhooks } from 'tapimo'
 import * as TestApp from '../../test/App.js'
+import * as Metrics from '../Metrics.js'
 import * as Cursor from './Cursor.js'
 
 const apiKeyOwner = { orgId: 'org_1', type: 'api_key' } as const
@@ -327,6 +328,78 @@ describe('listActive', () => {
   })
 })
 
+describe('listActiveWithCursors', () => {
+  test('pairs each subscription with its stored cursor from one read', async () => {
+    const db = TestApp.database()
+    const anchored = await Webhooks.createSubscription(db, createInput())
+    const fresh = await Webhooks.createSubscription(db, createInput({ owner: mppOwner }))
+    await Webhooks.setCursor(db, anchored.id, Cursor.encode([100, 5]))
+
+    const pairs = await Webhooks.listActiveWithCursors(db, 4217, 'token:transfer')
+    const byId = new Map(pairs.map((pair) => [pair.subscription.id, pair.cursor]))
+    expect(Cursor.decode(byId.get(anchored.id)!, ['int', 'int'])).toEqual([100, 5])
+    expect(byId.get(fresh.id)).toBeNull()
+  })
+})
+
+describe('listActiveForChain', () => {
+  test('returns active subscriptions across event types in one read', async () => {
+    const db = TestApp.database()
+    const transfer = await Webhooks.createSubscription(db, createInput())
+    const transaction = await Webhooks.createSubscription(
+      db,
+      createInput({ eventType: 'transaction:included' }),
+    )
+    await Webhooks.createSubscription(
+      db,
+      createInput({ eventType: 'block:created', owner: mppOwner }),
+    )
+    await Webhooks.createSubscription(db, createInput({ chainId: 42431, owner: mppOwner }))
+    await Webhooks.updateSubscription(db, apiKeyOwner, transfer.id, { status: 'paused' })
+
+    const pairs = await Webhooks.listActiveForChain(db, {
+      chainId: 4217,
+      eventTypes: ['transaction:included'],
+    })
+    expect(pairs.map(({ subscription }) => subscription.id)).toEqual([transaction.id])
+  })
+})
+
+describe('maxCursorBlock', () => {
+  test('returns the freshest cursor block across event types', async () => {
+    const db = TestApp.database()
+    const transfer = await Webhooks.createSubscription(db, createInput())
+    const tx = await Webhooks.createSubscription(
+      db,
+      createInput({ eventType: 'transaction:included' }),
+    )
+    await Webhooks.setCursor(db, transfer.id, Cursor.encode([100, 5]))
+    await Webhooks.setCursor(db, tx.id, Cursor.encode([250, 0]))
+
+    expect(await Webhooks.maxCursorBlock(db, 4217)).toBe(250)
+  })
+
+  test('counts creation checkpoints at their recorded head', async () => {
+    const db = TestApp.database()
+    await Webhooks.createSubscription(db, createInput(), { startBlockNumber: 900 })
+
+    expect(await Webhooks.maxCursorBlock(db, 4217)).toBe(900)
+  })
+
+  test('ignores other chains and returns null without cursors', async () => {
+    const db = TestApp.database()
+    expect(await Webhooks.maxCursorBlock(db, 4217)).toBeNull()
+
+    const other = await Webhooks.createSubscription(db, createInput({ chainId: 42431 }))
+    await Webhooks.setCursor(db, other.id, Cursor.encode([999, 0]))
+    // A cursorless subscription on the queried chain still reads as null.
+    await Webhooks.createSubscription(db, createInput())
+
+    expect(await Webhooks.maxCursorBlock(db, 4217)).toBeNull()
+    expect(await Webhooks.maxCursorBlock(db, 42431)).toBe(999)
+  })
+})
+
 describe('cursors', () => {
   test('round-trips a per-subscription cursor', async () => {
     const db = TestApp.database()
@@ -362,6 +435,32 @@ describe('cursors', () => {
   })
 })
 
+describe('compareAndSetCursors', () => {
+  test('atomically advances a batch and reports stale rows', async () => {
+    const db = TestApp.database()
+    const initialized = await Webhooks.createSubscription(db, createInput())
+    const stale = await Webhooks.createSubscription(db, createInput())
+    const uninitialized = await Webhooks.createSubscription(db, createInput())
+    const initial = Cursor.encode([100, 1])
+    const next = Cursor.encode([101, 1])
+    await Webhooks.setCursor(db, initialized.id, initial)
+    await Webhooks.setCursor(db, stale.id, initial)
+
+    const committed = await Webhooks.compareAndSetCursors(db, {
+      cursors: [
+        { cursor: next, expectedCursor: initial, subscriptionId: initialized.id },
+        { cursor: next, expectedCursor: null, subscriptionId: stale.id },
+        { cursor: next, expectedCursor: null, subscriptionId: uninitialized.id },
+      ],
+    })
+
+    expect([...committed].toSorted()).toEqual([initialized.id, uninitialized.id].toSorted())
+    expect(await Webhooks.getCursor(db, initialized.id)).toBe(next)
+    expect(await Webhooks.getCursor(db, stale.id)).toBe(initial)
+    expect(await Webhooks.getCursor(db, uninitialized.id)).toBe(next)
+  })
+})
+
 describe('recordSuccess / recordFailure', () => {
   test('recordSuccess resets failures and stamps lastDeliveryAt', async () => {
     const db = TestApp.database()
@@ -624,6 +723,44 @@ describe('delivery log', () => {
     `)
   })
 
+  test('deliverAndRecord partitions event, enqueue, and queue latency', async () => {
+    const db = TestApp.database()
+    const created = await Webhooks.createSubscription(db, createInput())
+    const histograms = new Map<string, number>()
+    const metrics = Metrics.from({
+      count() {},
+      flush() {},
+      gauge() {},
+      histogram(name, value) {
+        histograms.set(name, value)
+      },
+    })
+    const envelope = {
+      chainId: 4217,
+      createdAt: '1970-01-01T00:00:00.500Z',
+      data: { timestamp: '1970-01-01T00:00:00.000Z' },
+      id: 'evt_null',
+      subscriptionId: created.id,
+      type: 'token:transfer',
+    } satisfies Webhooks.Envelope
+    const fetch = (async () =>
+      new Response(null, { status: 200 })) satisfies typeof globalThis.fetch
+
+    await expect(
+      Webhooks.deliverAndRecord(db, created, envelope, {
+        dequeuedAt: 1_250,
+        fetch,
+        metrics,
+        now: fixedNow,
+        queueAttempt: 2,
+        queuedAt: 1_000,
+      }),
+    ).resolves.toMatchObject({ ok: true })
+    expect(histograms.get('webhook_delivery_envelope_to_queue_ms')).toBe(500)
+    expect(histograms.get('webhook_delivery_event_to_envelope_ms')).toBe(500)
+    expect(histograms.get('webhook_delivery_queue_wait_ms')).toBe(250)
+  })
+
   test('deliverAndRecord appends a failed row with the error and status', async () => {
     const db = TestApp.database()
     const created = await Webhooks.createSubscription(db, createInput())