tapimo 0.7.0 → 0.7.2

package/src/apps/data/routes/webhooks.ts

@@ -15,7 +15,7 @@ import type * as Tidx from '../../../internal/Tidx.js'
 import * as Ttl from '../../../internal/Ttl.js'
 import * as Value from '../../../internal/Value.js'
 import type * as Viem from '../../../internal/Viem.js'
-import type * as WebhookDestination from '../../../internal/WebhookDestination.js'
+import * as WebhookDestination from '../../../internal/WebhookDestination.js'
 import * as Webhooks from '../../../internal/Webhooks.js'
 import * as Receipts from './receipts.js'
 import * as Transactions from './transactions.js'
@@ -1355,6 +1355,12 @@ export function webhooks(options: webhooks.Options = {}) {
             return Response.unsupportedChainId(c, chainId, supportedChainIds)
 
           try {
+            // Preserve destination errors before the creation-time RPC dependency.
+            WebhookDestination.assertDestination(body.destination)
+            const maxPerOwner = webhook.maxPerOwner ?? defaultMaxPerOwner
+            if ((await Webhooks.countSubscriptions(db, owner)) >= maxPerOwner)
+              throw new Webhooks.LimitExceededError(maxPerOwner)
+            const startBlockNumber = await getRpcHead(c.get('getClient')(chainId))
             const subscription = await Webhooks.createSubscription(
               db,
               {
@@ -1366,7 +1372,10 @@ export function webhooks(options: webhooks.Options = {}) {
                 owner,
                 ...(owner.type === 'mpp' ? { ttl: mppTtlMs } : {}),
               },
-              { maxPerOwner: webhook.maxPerOwner ?? defaultMaxPerOwner },
+              {
+                maxPerOwner,
+                startBlockNumber,
+              },
             )
             return c.json(
               Response.validated(schema.createWebhook.Response, toCreated(subscription)),
@@ -1878,18 +1887,17 @@ export declare namespace webhooks {
  * can equal it).
  */
 const endOfBlockIndex = 2_147_483_647
+const scanPageSize = 100
 
 /** A scanned row paired with its `(block, index)` keyset position. */
 type ScanItem = { cursor: readonly [number, number]; data: unknown }
 
 /**
- * The result of scanning one subscription for a tick. Two flavours:
+ * The result of scanning one subscription page. Two flavours:
  *
- * - **window** (the RPC scanners): every block in the confirmed window was
- *   inspected in full, so `scannedThrough` is the highest block fully scanned.
- *   The poller advances the cursor to `(scannedThrough, endOfBlockIndex)` on a
- *   clean scan even when no rows matched — a rare-filter subscription never
- *   rescans the same empty range.
+ * - **window** (the RPC scanners): `scannedThrough` is the highest block fully
+ *   scanned. A batch-limited page omits it, so the poller resumes after the last
+ *   dispatched row instead of skipping later matches.
  * - **keyset** (the TIDX legacy scanners): rows come back `LIMIT`-bounded, so
  *   the page only reports whether `hasMore` rows remain past it; the poller
  *   advances only past the last dispatched row (jumping to the window end only
@@ -1901,6 +1909,27 @@ type ScanPage = {
   scannedThrough?: number | undefined
 }
 
+/** Per-subscription state retained between fair scan passes. */
+type PollState = {
+  cursor: string
+  remaining: number
+  safeBlock: number
+  subscription: Webhooks.Subscription
+}
+
+/** Returns `candidate` only when it is lexicographically after `current`. */
+function laterCursor(current: string, candidate: string): string {
+  const before = Cursor.decode(current, ['int', 'int'])
+  const after = Cursor.decode(candidate, ['int', 'int'])
+  if (!after) return current
+  if (!before) return candidate
+  const [beforeBlock, beforeIndex] = before as [number, number]
+  const [afterBlock, afterIndex] = after as [number, number]
+  return afterBlock > beforeBlock || (afterBlock === beforeBlock && afterIndex > beforeIndex)
+    ? candidate
+    : current
+}
+
 /**
  * Polls the chain over RPC for new events and dispatches a signed envelope per
  * matching row, advancing each active subscription's `(block, index)` keyset
@@ -1921,10 +1950,12 @@ type ScanPage = {
  *
  * Cursor advances only past successfully dispatched rows, so a dispatch failure
  * pauses that subscription at the last good position and the next tick resumes.
+ * Scan failures leave that subscription unchanged; the first error is rethrown
+ * after the remaining subscriptions run.
  */
 export async function poll(options: poll.Options): Promise<poll.Result> {
   const {
-    batchSize = 100,
+    batchSize = 200,
     cache: cache_metadata = Store.memory(),
     chainId,
     client,
@@ -1967,80 +1998,134 @@ export async function poll(options: poll.Options): Promise<poll.Result> {
     scanned: 0,
     subscriptions: 0,
   }
+  const scanErrors: unknown[] = []
 
   for (const eventType of eventTypes) {
     const legacy = eventType === 'token:transfer'
     const safeBlock = legacy ? Math.max(0, (await resolveTidxHead()) - confirmations) : rpcSafeBlock
     const subscriptions = await Webhooks.listActive(db, chainId, eventType)
+    let states: PollState[] = []
     for (const subscription of subscriptions) {
       result.subscriptions++
-      const cursor = await Webhooks.getCursor(db, subscription.id)
+      let cursor = await Webhooks.getCursor(db, subscription.id)
       if (cursor === null) {
-        // First time we've seen this subscription: anchor it at the head of the
-        // confirmed window so we never backfill from genesis. Events start
-        // flowing from the next tick onward.
-        await Webhooks.setCursor(db, subscription.id, Cursor.encode([safeBlock, endOfBlockIndex]))
+        // Legacy subscriptions without a creation checkpoint anchor at the
+        // current confirmed head, avoiding a genesis backfill.
+        await Webhooks.compareAndSetCursor(db, {
+          cursor: Cursor.encode([safeBlock, endOfBlockIndex]),
+          expectedCursor: null,
+          subscriptionId: subscription.id,
+        })
         continue
       }
-      const page = await scanSubscription({
-        batchSize,
-        cache,
-        client,
+      const checkpoint = Cursor.decode(cursor, ['int'])
+      if (checkpoint) {
+        // Creation checkpoints record the live head without finalizing it. Rebase
+        // to this poller's confirmed boundary so replacement blocks stay eligible.
+        const confirmedCursor = Cursor.encode([
+          Math.max(0, (checkpoint[0] as number) - confirmations),
+          endOfBlockIndex,
+        ])
+        const committed = await Webhooks.compareAndSetCursor(db, {
+          cursor: confirmedCursor,
+          expectedCursor: cursor,
+          subscriptionId: subscription.id,
+        })
+        if (!committed) continue
+        cursor = confirmedCursor
+      }
+      const window = legacy ? undefined : scanWindow(cursor, safeBlock, maxBlocks)
+      states.push({
         cursor,
-        maxBlocks,
-        safeBlock,
+        remaining: batchSize,
+        safeBlock: window?.to ?? safeBlock,
         subscription,
-        tidx,
       })
-      result.scanned += page.items.length
-
-      let accepted: readonly [number, number] | undefined
-      let failed = false
-      for (const item of page.items) {
-        // Token metadata is best-effort for webhooks: transfers use it for display,
-        // while transactions expose fee-token references without blocking delivery.
-        const data =
-          eventType === 'token:transfer'
-            ? await withTokenMetadata(item.data, { cache: cache_metadata, chainId, client })
-            : eventType === 'transaction:included'
-              ? await withFeeTokenMetadata(item.data, { cache: cache_metadata, chainId, client })
-              : item.data
-        const envelope = Webhooks.buildEnvelope({
-          blockNumber: item.cursor[0],
-          createdAt: now(),
-          data,
-          logIndex: item.cursor[1],
-          subscription,
-        })
-        const delivered = await dispatch({ envelope, subscription })
-        if (!delivered.ok) {
-          result.failed++
-          failed = true
-          break
+    }
+
+    // Every subscription gets one bounded page before any subscription gets a
+    // continuation page, preserving fairness while allowing catch-up.
+    while (states.length > 0) {
+      const continuations: PollState[] = []
+      for (const state of states) {
+        const { subscription } = state
+        let page: ScanPage
+        try {
+          page = await scanSubscription({
+            batchSize: Math.min(scanPageSize, state.remaining),
+            cache,
+            client,
+            cursor: state.cursor,
+            maxBlocks,
+            safeBlock: state.safeBlock,
+            subscription,
+            tidx,
+          })
+        } catch (error) {
+          scanErrors.push(error)
+          continue
+        }
+        result.scanned += page.items.length
+        state.remaining -= page.items.length
+
+        let accepted: readonly [number, number] | undefined
+        let failed = false
+        for (const item of page.items) {
+          // Token metadata is best-effort for webhooks: transfers use it for display,
+          // while transactions expose fee-token references without blocking delivery.
+          const data =
+            eventType === 'token:transfer'
+              ? await withTokenMetadata(item.data, { cache: cache_metadata, chainId, client })
+              : eventType === 'transaction:included'
+                ? await withFeeTokenMetadata(item.data, { cache: cache_metadata, chainId, client })
+                : item.data
+          const envelope = Webhooks.buildEnvelope({
+            blockNumber: item.cursor[0],
+            createdAt: now(),
+            data,
+            logIndex: item.cursor[1],
+            subscription,
+          })
+          const delivered = await dispatch({ envelope, subscription })
+          if (!delivered.ok) {
+            result.failed++
+            failed = true
+            break
+          }
+          accepted = item.cursor
+          result.dispatched++
         }
-        accepted = item.cursor
-        result.dispatched++
-      }
 
-      // Advance the high-water mark. On a dispatch failure, stop at the last
-      // accepted row so the failed one retries next tick. Otherwise: window
-      // scanners jump to the end of the fully-scanned range (even with zero
-      // matches); keyset scanners advance past the last dispatched row, or — on
-      // a complete empty page — to the end of the confirmed window.
-      if (failed) {
-        if (accepted) await Webhooks.setCursor(db, subscription.id, Cursor.encode([...accepted]))
-      } else if (page.scannedThrough !== undefined)
-        await Webhooks.setCursor(
-          db,
-          subscription.id,
-          Cursor.encode([page.scannedThrough, endOfBlockIndex]),
-        )
-      else if (accepted) await Webhooks.setCursor(db, subscription.id, Cursor.encode([...accepted]))
-      else if (!page.hasMore)
-        await Webhooks.setCursor(db, subscription.id, Cursor.encode([safeBlock, endOfBlockIndex]))
+        // Persist every accepted page before another subscription begins its
+        // continuation, so a later scan failure retains completed work.
+        const candidate = (() => {
+          if (failed) return accepted ? Cursor.encode([...accepted]) : undefined
+          if (page.scannedThrough !== undefined)
+            return Cursor.encode([page.scannedThrough, endOfBlockIndex])
+          if (accepted) return Cursor.encode([...accepted])
+          if (!page.hasMore) return Cursor.encode([state.safeBlock, endOfBlockIndex])
+          return undefined
+        })()
+        const cursor = candidate ? laterCursor(state.cursor, candidate) : state.cursor
+        const progressed = cursor !== state.cursor
+        let committed = false
+        if (progressed) {
+          committed = await Webhooks.compareAndSetCursor(db, {
+            cursor,
+            expectedCursor: state.cursor,
+            subscriptionId: subscription.id,
+          })
+          if (committed) state.cursor = cursor
+        }
+
+        if (!failed && page.hasMore === true && committed && state.remaining > 0)
+          continuations.push(state)
+      }
+      states = continuations
     }
   }
 
+  if (scanErrors.length > 0) throw scanErrors[0]
   return result
 }
 
@@ -2054,7 +2139,7 @@ export declare namespace poll {
   }
   /** Options for {@link poll}. */
   type Options = {
-    /** Max matching rows scanned per subscription per tick (default 100). */
+    /** Max matching rows scanned per subscription per tick (default 200). */
     batchSize?: number | undefined
     /** Cache store for the token-metadata memoize (defaults to a fresh in-memory store; pass a persistent one to share lookups across ticks). */
     cache?: Store.Store | undefined
@@ -2269,6 +2354,7 @@ async function scanSubscription(options: {
 
   if (subscription.eventType === 'log:emitted')
     return scanLogs({
+      batchSize,
       cache,
       client,
       cursor,
@@ -2279,6 +2365,7 @@ async function scanSubscription(options: {
 
   if (subscription.eventType === 'block:created')
     return scanBlocks({
+      batchSize,
       cache,
       cursor,
       filters: parseFilters('block:created', subscription.filters),
@@ -2288,6 +2375,7 @@ async function scanSubscription(options: {
 
   if (subscription.eventType === 'transaction:included')
     return scanTxs({
+      batchSize,
       cache,
       cursor,
       filters: parseFilters('transaction:included', subscription.filters),
@@ -2302,6 +2390,9 @@ async function scanSubscription(options: {
     { tidx },
     { chainId, cursor, limit: batchSize, order: 'asc', toBlock: safeBlock, ...filters },
   )
+  // A cap-truncated fallback sample is an arbitrary subset: dispatching it
+  // would advance the cursor past unseen matches. Hold position and retry.
+  if (page.capped) return { hasMore: true, items: [] }
   return { hasMore: page.hasMore, items: page.items }
 }
 
@@ -2485,7 +2576,7 @@ type RpcBlock = Awaited<ReturnType<Viem.getClient.ReturnType['getBlock']>>
 /** Per-tick lazy cache of fetched block headers, full blocks, and receipts. */
 type ChainCache = {
   /** Full raw block (with transactions) via `eth_getBlockByNumber(n, true)`. */
-  block(n: number): Promise<Record<string, unknown> | undefined>
+  block(n: number): Promise<Record<string, unknown>>
   /** Viem-formatted block header (transaction hashes only) via `getBlock`. */
   header(n: number): Promise<RpcBlock | undefined>
   /** Receipts for a block keyed by lowercased transaction hash. */
@@ -2495,7 +2586,7 @@ type ChainCache = {
 /** Builds a per-tick {@link ChainCache} bound to a viem client. */
 function createChainCache(client: Viem.getClient.ReturnType): ChainCache {
   const headers = new Map<number, Promise<RpcBlock | undefined>>()
-  const blocks = new Map<number, Promise<Record<string, unknown> | undefined>>()
+  const blocks = new Map<number, Promise<Record<string, unknown>>>()
   const receipts = new Map<number, Promise<Map<string, Record<string, unknown>>>>()
   return {
     block(n) {
@@ -2506,7 +2597,10 @@ function createChainCache(client: Viem.getClient.ReturnType): ChainCache {
         p = rpc<Record<string, unknown> | null>(client, 'eth_getBlockByNumber', [
           blockTag(n),
           true,
-        ]).then((b) => b ?? undefined)
+        ]).then((block) => {
+          if (!block) throw new BlockUnavailableError(n)
+          return block
+        })
         blocks.set(n, p)
       }
       return p
@@ -2514,9 +2608,7 @@ function createChainCache(client: Viem.getClient.ReturnType): ChainCache {
     header(n) {
       let p = headers.get(n)
       if (!p) {
-        p = client
-          .getBlock({ blockNumber: BigInt(n), includeTransactions: false })
-          .catch(() => undefined)
+        p = client.getBlock({ blockNumber: BigInt(n), includeTransactions: false })
         headers.set(n, p)
       }
       return p
@@ -2744,6 +2836,7 @@ function decodeScanLog(log: ScanLog, event: AbiEvent): ScanLog {
  * all operators are re-checked in JS over the narrow window.
  */
 async function scanLogs(options: {
+  batchSize: number
   cache: ChainCache
   client: Viem.getClient.ReturnType
   cursor: string | undefined
@@ -2751,7 +2844,7 @@ async function scanLogs(options: {
   maxBlocks: number
   safeBlock: number
 }): Promise<ScanPage> {
-  const { cache, client, cursor, filters, maxBlocks, safeBlock } = options
+  const { batchSize, cache, client, cursor, filters, maxBlocks, safeBlock } = options
   const f = filters as z.output<typeof schema.LogFilters>
   const window = scanWindow(cursor, safeBlock, maxBlocks)
   if (!window) return { items: [], scannedThrough: undefined }
@@ -2807,7 +2900,10 @@ async function scanLogs(options: {
     const header = await cache.header(blockNumber)
     const timestamp = blockIso(header)
     const data = toLogEvent(log, timestamp, f.signature, event)
-    if (data) items.push({ cursor: [data.blockNumber, data.logIndex], data })
+    if (data) {
+      items.push({ cursor: [data.blockNumber, data.logIndex], data })
+      if (items.length >= batchSize) return { hasMore: true, items }
+    }
   }
   return { items, scannedThrough: window.to }
 }
@@ -2908,13 +3004,14 @@ function toBlockEvent(block: RpcBlock): BlockEventData | undefined {
  * empty filter is a per-block heartbeat.
  */
 async function scanBlocks(options: {
+  batchSize: number
   cache: ChainCache
   cursor: string | undefined
   filters: Record<string, unknown>
   maxBlocks: number
   safeBlock: number
 }): Promise<ScanPage> {
-  const { cache, cursor, filters, maxBlocks, safeBlock } = options
+  const { batchSize, cache, cursor, filters, maxBlocks, safeBlock } = options
   const f = filters as z.output<typeof schema.BlockFilters>
   const window = scanWindow(cursor, safeBlock, maxBlocks)
   if (!window) return { items: [], scannedThrough: undefined }
@@ -2926,7 +3023,10 @@ async function scanBlocks(options: {
     if (!block) continue
     if (!matchesBlock(block, f)) continue
     const data = toBlockEvent(block)
-    if (data) items.push({ cursor: [data.number, 0], data })
+    if (data) {
+      items.push({ cursor: [data.number, 0], data })
+      if (items.length >= batchSize) return { hasMore: true, items }
+    }
   }
   return { items, scannedThrough: window.to }
 }
@@ -2979,13 +3079,14 @@ const txFields = {
  * poller then resolves its fee-token references.
  */
 async function scanTxs(options: {
+  batchSize: number
   cache: ChainCache
   cursor: string | undefined
   filters: Record<string, unknown>
   maxBlocks: number
   safeBlock: number
 }): Promise<ScanPage> {
-  const { cache, cursor, filters, maxBlocks, safeBlock } = options
+  const { batchSize, cache, cursor, filters, maxBlocks, safeBlock } = options
   const f = filters as z.output<typeof schema.TxFilters>
   const window = scanWindow(cursor, safeBlock, maxBlocks)
   if (!window) return { items: [], scannedThrough: undefined }
@@ -2999,10 +3100,10 @@ async function scanTxs(options: {
   const items: ScanItem[] = []
   for (let n = window.from; n <= window.to; n++) {
     const block = await cache.block(n)
-    const txs = Array.isArray(block?.['transactions']) ? (block!['transactions'] as unknown[]) : []
+    const txs = Array.isArray(block['transactions']) ? (block['transactions'] as unknown[]) : []
     if (txs.length === 0) continue
     const receipts = needsReceipt || includeReceipt ? await cache.receipts(n) : undefined
-    const timestamp = asString(block?.['timestamp'])
+    const timestamp = asString(block['timestamp'])
 
     for (let i = 0; i < txs.length; i++) {
       const tx = txs[i] as Record<string, unknown>
@@ -3028,6 +3129,7 @@ async function scanTxs(options: {
           }
       }
       items.push({ cursor: [n, index], data })
+      if (items.length >= batchSize) return { hasMore: true, items }
     }
   }
   return { items, scannedThrough: window.to }
@@ -3195,3 +3297,11 @@ function unauthorized(c: Context<App.Environment>) {
     status: 401,
   })
 }
+
+class BlockUnavailableError extends Error {
+  override name = 'Webhooks.BlockUnavailableError'
+
+  constructor(blockNumber: number) {
+    super(`Block ${blockNumber} is unavailable.`)
+  }
+}