tapimo 0.8.0 → 0.10.0

package/src/apps/data/routes/transfers.ts

@@ -26,13 +26,31 @@ import * as Valuation from './valuation.js'
 // the indexer SQL.
 export const signature = 'event Transfer(address indexed from, address indexed to, uint256 value)'
 
-// TIP-20 contracts occupy the deterministic `0x20c0…` range. Full-width
-// bounds compare correctly as PostgreSQL bytea and ClickHouse hex strings.
-const tip20AddressRange = [
+/**
+ * Half-open `[start, end)` bounds of the deterministic TIP-20 contract range.
+ * Full-width so the values compare correctly as PostgreSQL bytea, ClickHouse
+ * hex strings, and lowercased JavaScript strings.
+ */
+export const tip20AddressRange = [
   '0x20c0000000000000000000000000000000000000',
   '0x20c1000000000000000000000000000000000000',
 ] as const
 
+// The global feed spans every TIP-20 token, so its newest-first query cannot
+// follow `token_transfers`' token-first sort key. Block windows bound the sort.
+const transferWindowBlocks = 100_000
+
+// Sparse global scans retain progress without monopolizing one request.
+const transferWindowScanCap = 8
+
+// Recipient-first decoded scans usually fill a page immediately. A larger raw
+// batch leaves room to discard non-TIP-20 `Transfer` events without extra I/O.
+const decodedTransferBatchSize = 256
+
+// Bound decoded recipient traversal while preserving the largest positional
+// page and its extra row for `nextCursor` detection.
+const decodedTransferRowCap = Schema.maxPageWindow + 1
+
 // A memo'd TIP-20 transfer emits `TransferWithMemo` alongside the plain
 // `Transfer`; the two rows match on `(tx_hash, from, to, amount, token)`. A
 // text memo is decoded into `memo`; an MPP attribution fingerprint memo is
@@ -42,6 +60,10 @@ const memoSignature =
 
 const tokenFields = ['token.logoUri', 'token.verified'] as const
 
+// Concurrent metadata reads collapse into one deployless multicall. Small
+// chunks keep each call under the Tempo RPC's per-call size limit.
+const tokenReadChunkSize = 10
+
 /** Zod schemas owned by the transfer handlers. */
 export namespace schema {
   /** A transfer token reference with RPC metadata. Inlined to avoid the `transfers`/`tokens` module cycle. */
@@ -363,9 +385,9 @@ export function transfers(options: transfers.Options = {}) {
 
       const options = c.req.valid('query')
       const chainId = options.chainId ?? c.get('chainId')
-      const denomination = options['valuation.currency'] ?? 'USD'
+      const denomination = options['valuation.currency']
 
-      // Cursor pages carry current valuations and curated verification data.
+      // Cursor pages can carry current valuations and curated verification data.
       if (options.cursor !== undefined) Cache.setPolicy(c, Cache.policies.metadata)
 
       // Filters shared by the page query and the opt-in total count, so the
@@ -465,14 +487,13 @@ export function transfers(options: transfers.Options = {}) {
             return suppress.has(key) ? [] : [transfer]
           })
         }
-        // Verification is ancillary to the feed. Skip empty pages and degrade
-        // cold snapshot failures to null valuations.
+        const includeLogoUri = options.include.includes('token.logoUri')
+        const includeVerified = options.include.includes('token.verified')
+        // Curated data is needed only for requested valuation or token fields.
         const snapshot =
-          data.length > 0
+          data.length > 0 && (denomination !== undefined || includeLogoUri || includeVerified)
             ? await VerifiedTokens.snapshot(c, chainId).catch(() => undefined)
             : undefined
-        const includeLogoUri = options.include.includes('token.logoUri')
-        const includeVerified = options.include.includes('token.verified')
         const uniqueTokens = Array.from(
           new Set(
             data.flatMap((transfer) => [
@@ -481,10 +502,15 @@ export function transfers(options: transfers.Options = {}) {
             ]),
           ),
         )
-        const tokensByAddress = new Map(
-          await Timing.time(c, 'tokens', () =>
-            Promise.all(
-              uniqueTokens.map(async (token) => {
+        const tokensByAddress = new Map<
+          string,
+          Awaited<ReturnType<typeof Tokens.getTokenMetadata>>
+        >()
+        await Timing.time(c, 'tokens', async () => {
+          for (let index = 0; index < uniqueTokens.length; index += tokenReadChunkSize) {
+            const group = uniqueTokens.slice(index, index + tokenReadChunkSize)
+            const entries = await Promise.all(
+              group.map(async (token) => {
                 const [logoUri, metadata] = await Promise.all([
                   includeLogoUri
                     ? Timing.time(c, 'token_logo', () =>
@@ -507,25 +533,28 @@ export function transfers(options: transfers.Options = {}) {
                   },
                 ] as const
               }),
-            ),
-          ),
-        )
+            )
+            for (const entry of entries) tokensByAddress.set(...entry)
+          }
+        })
 
         // Value each leg via its curated display currency, in the requested
         // denomination.
-        const rates = await Valuation.ratesFor(c, {
-          currencies: data.flatMap((transfer) =>
-            [
-              transfer.sourceToken.address,
-              ...(transfer.destinationToken ? [transfer.destinationToken.address] : []),
-            ].flatMap((address) => {
-              const held = snapshot?.byAddress.get(address.toLowerCase())?.currency
-              return held === undefined ? [] : [held]
-            }),
-          ),
-          denomination,
-          oracle,
-        })
+        const rates = denomination
+          ? await Valuation.ratesFor(c, {
+              currencies: data.flatMap((transfer) =>
+                [
+                  transfer.sourceToken.address,
+                  ...(transfer.destinationToken ? [transfer.destinationToken.address] : []),
+                ].flatMap((address) => {
+                  const held = snapshot?.byAddress.get(address.toLowerCase())?.currency
+                  return held === undefined ? [] : [held]
+                }),
+              ),
+              denomination,
+              oracle,
+            })
+          : undefined
         const enrich = (leg: CrossToken) => {
           const metadata = tokensByAddress.get(leg.address.toLowerCase())
           if (!metadata) throw new Error(`Token metadata unavailable for ${leg.address}`)
@@ -536,12 +565,16 @@ export function transfers(options: transfers.Options = {}) {
                 currency: metadata.currency,
                 decimals: metadata.decimals,
               }),
-              valuation: Valuation.valuationFor({
-                amount: BigInt(leg.amount),
-                denomination,
-                rates,
-                token: snapshot?.byAddress.get(leg.address.toLowerCase()),
-              }),
+              ...(denomination
+                ? {
+                    valuation: Valuation.valuationFor({
+                      amount: BigInt(leg.amount),
+                      denomination,
+                      rates,
+                      token: snapshot?.byAddress.get(leg.address.toLowerCase()),
+                    }),
+                  }
+                : {}),
             },
             token: {
               address: leg.address,
@@ -632,7 +665,7 @@ export function query(c: Context<App.Environment>, options: query.Options) {
   const { chainId, timing, ...rest } = options
   const store = c.get('store')
   const tidx = c.get('getTidx')(chainId)
-  const cursor = options.cursor ? Cursor.decode(options.cursor, ['int', 'int']) : undefined
+  const cursor = decodeTransferCursor(options.cursor)
 
   return Timing.time(c, timing, () =>
     Store.memoize(
@@ -644,7 +677,7 @@ export function query(c: Context<App.Environment>, options: query.Options) {
         }
       },
       {
-        key: `transfers:v2:${chainId}:${options.order}:${options.token ?? ''}:${options.eitherSide ?? ''}:${options.sender ?? ''}:${options.recipient ?? ''}:${options.fromBlock ?? ''}:${options.toBlock ?? ''}:${options.fromTimestamp ?? ''}:${options.toTimestamp ?? ''}:${cursor ? `cursor:${cursor[0]}:${cursor[1]}` : options.page !== undefined && options.page > 1 ? `page:${options.page}` : 'head'}:${options.limit}`,
+        key: `transfers:v2:${chainId}:${options.order}:${options.token ?? ''}:${options.eitherSide ?? ''}:${options.sender ?? ''}:${options.recipient ?? ''}:${options.fromBlock ?? ''}:${options.toBlock ?? ''}:${options.fromTimestamp ?? ''}:${options.toTimestamp ?? ''}:${cursor ? `cursor:${cursor.position[0]}:${cursor.position[1]}:${cursor.offset}` : options.page !== undefined && options.page > 1 ? `page:${options.page}` : 'head'}:${options.limit}`,
         store,
         ttl: Ttl.seconds(15),
       },
@@ -694,7 +727,7 @@ export declare namespace query {
 }
 
 /**
- * Context-free core of {@link query}: reads token-scoped rows from
+ * Context-free core of {@link query}: reads token-scoped and global rows from
  * `token_transfers`, otherwise the decoded `Transfer` CTE, then paginates and
  * decodes them. Also used by the webhook poller.
  */
@@ -706,52 +739,216 @@ export async function scan(deps: scan.Deps, options: scan.Options): Promise<scan
   // Keyset pagination: anchor the page below the previous row's `(block, log_idx)`
   // position instead of a numeric offset, so rows arriving at the head can't
   // shift items across pages. A malformed cursor falls back to the head page.
-  const cursor = options.cursor ? Cursor.decode(options.cursor, ['int', 'int']) : undefined
+  const cursor = decodeTransferCursor(options.cursor)
   // Bounded positional lane (exclusive with `cursor` at the schema): page 1 is
   // the head page, so it shares the head's SQL and cache entries; deeper pages
   // translate to a SQL OFFSET of `(page - 1) * limit` rows.
-  const offset =
+  const pageOffset =
     options.page !== undefined && options.page > 1 ? (options.page - 1) * limit : undefined
+  const offset = pageOffset ?? (cursor?.offset === 0 ? undefined : cursor?.offset)
   const address = options.eitherSide
+  const addressScoped = address !== undefined
+  const recipient = options.recipient
+  const recipientDecoded = !tokenScoped && recipient !== undefined
+  const decoded = !tokenScoped && (addressScoped || recipientDecoded)
+  const splitAddress = addressScoped && !recipientDecoded
+  // Only one-sided `"from"`/`"to"` equality shapes hit the planner's
+  // deterministic top-N kill, so only they take the capped fallback.
+  const sideFiltered =
+    addressScoped || options.sender !== undefined || options.recipient !== undefined
+  const global = !tokenScoped && !sideFiltered
 
   // Transfer events filtered by any combination of contract / sender /
   // recipient / block / timestamp. The address shortcut is split below so
   // each upstream query can use one indexed side.
-  const filters = filterClauses(options)
-  if (cursor !== undefined)
+  const filters = global
+    ? [
+        `token >= '${tip20AddressRange[0]}'`,
+        `token < '${tip20AddressRange[1]}'`,
+        ...(options.fromBlock !== undefined ? [`block_num >= ${options.fromBlock}`] : []),
+        ...(options.toBlock !== undefined ? [`block_num <= ${options.toBlock}`] : []),
+        ...(options.fromTimestamp !== undefined
+          ? [`block_timestamp >= '${formatClickHouseDateTime(options.fromTimestamp)}'`]
+          : []),
+        ...(options.toTimestamp !== undefined
+          ? [`block_timestamp <= '${formatClickHouseDateTime(options.toTimestamp)}'`]
+          : []),
+      ]
+    : decoded
+      ? [
+          ...(options.sender !== undefined ? [`"from" = '${options.sender}'`] : []),
+          ...(recipient !== undefined ? [`"to" = '${recipient}'`] : []),
+          ...(options.fromBlock !== undefined ? [`block_num >= ${options.fromBlock}`] : []),
+          ...(options.toBlock !== undefined ? [`block_num <= ${options.toBlock}`] : []),
+          ...(options.fromTimestamp !== undefined
+            ? [`block_timestamp >= '${options.fromTimestamp}'`]
+            : []),
+          ...(options.toTimestamp !== undefined
+            ? [`block_timestamp <= '${options.toTimestamp}'`]
+            : []),
+        ]
+      : filterClauses(options)
+  if (cursor !== undefined && !decoded)
     filters.push(
       Cursor.keyset([
-        { literal: Cursor.literal(cursor[0]!, 'int'), name: 'block_num', order },
-        { literal: Cursor.literal(cursor[1]!, 'int'), name: 'log_idx', order },
+        { literal: Cursor.literal(cursor.position[0], 'int'), name: 'block_num', order },
+        { literal: Cursor.literal(cursor.position[1], 'int'), name: 'log_idx', order },
       ]),
     )
-  const addressScoped = address !== undefined
   const window = addressScoped ? (offset ?? 0) + limit + 1 : limit + 1
-  // Only one-sided `"from"`/`"to"` equality shapes hit the planner's
-  // deterministic top-N kill, so only they take the capped fallback.
-  const sideFiltered =
-    addressScoped || options.sender !== undefined || options.recipient !== undefined
-  const fetchRows = async (side?: 'from' | 'to') => {
+  const fetchRows = async (options_fetch: fetchRows.Options = {}) => {
+    const side = options_fetch.side
     const sideFilters =
       side !== undefined && address !== undefined
         ? [`"${side}" = '${address}'`, ...filters]
         : filters
-    const where = sideFilters.length > 0 ? `WHERE ${sideFilters.join(' AND ')}` : ''
-    const select = `"from", "to", ${tokenScoped ? 'token AS address, amount AS value' : 'address, value'}, tx_hash, block_num, log_idx, block_timestamp`
-    const table = tokenScoped ? 'token_transfers' : 'Transfer'
-    try {
+    const queryFilters = [
+      ...sideFilters,
+      ...(options_fetch.lo !== undefined ? [`block_num >= ${options_fetch.lo}`] : []),
+      ...(options_fetch.hi !== undefined ? [`block_num <= ${options_fetch.hi}`] : []),
+    ]
+    const materialized = tokenScoped || global
+    // Keep a distinct alias until tidx.ts parses ClickHouse timestamps as UTC.
+    const select = materialized
+      ? `"from", "to", token AS address, amount AS value, tx_hash, block_num, log_idx, toString(block_timestamp, 'UTC') AS block_timestamp_utc`
+      : `"from", "to", address, value, tx_hash, block_num, log_idx, block_timestamp`
+    const table = materialized ? 'token_transfers' : 'Transfer'
+    const execute = async (
+      filters_execute: readonly string[],
+      queryLimit: number,
+      queryOffset?: number,
+    ): Promise<readonly Record<string, unknown>[]> => {
+      const where = filters_execute.length > 0 ? `WHERE ${filters_execute.join(' AND ')}` : ''
       const result = await tidx.fetch({
         chainId,
-        ...(tokenScoped ? { engine: 'clickhouse' } : { signatures: [signature] }),
+        ...(materialized ? { engine: 'clickhouse' } : { signatures: [signature] }),
         query: `
           SELECT ${select}
           FROM ${table}
           ${where}
           ORDER BY block_num ${direction}, log_idx ${direction}
-          LIMIT ${window}${!addressScoped && offset !== undefined ? ` OFFSET ${offset}` : ''}
+          LIMIT ${queryLimit}${queryOffset !== undefined ? ` OFFSET ${queryOffset}` : ''}
         ` as string,
       })
-      return { rows: result.rows, sample: false }
+      return result.rows
+    }
+
+    if (decoded) {
+      const target = (offset ?? 0) + limit + 1
+      // Filter the physical token column so ClickHouse can use the materialized table ordering.
+      const filters_materialized = [
+        `token >= '${tip20AddressRange[0]}'`,
+        `token < '${tip20AddressRange[1]}'`,
+        ...(options.sender !== undefined ? [`"from" = '${options.sender}'`] : []),
+        ...(recipient !== undefined ? [`"to" = '${recipient}'`] : []),
+        ...(options.fromBlock !== undefined ? [`block_num >= ${options.fromBlock}`] : []),
+        ...(options.toBlock !== undefined ? [`block_num <= ${options.toBlock}`] : []),
+        ...(options.fromTimestamp !== undefined
+          ? [`block_timestamp >= '${formatClickHouseDateTime(options.fromTimestamp)}'`]
+          : []),
+        ...(options.toTimestamp !== undefined
+          ? [`block_timestamp <= '${formatClickHouseDateTime(options.toTimestamp)}'`]
+          : []),
+      ]
+      if (cursor !== undefined)
+        filters_materialized.push(
+          Cursor.keyset([
+            { literal: Cursor.literal(cursor.position[0], 'int'), name: 'block_num', order },
+            { literal: Cursor.literal(cursor.position[1], 'int'), name: 'log_idx', order },
+          ]),
+        )
+      const sideFilters_materialized =
+        side !== undefined && address !== undefined
+          ? [`"${side}" = '${address}'`, ...filters_materialized]
+          : filters_materialized
+      const where_materialized =
+        sideFilters_materialized.length > 0 ? `WHERE ${sideFilters_materialized.join(' AND ')}` : ''
+      try {
+        const result = await tidx.fetch({
+          chainId,
+          engine: 'clickhouse',
+          query: `
+            SELECT DISTINCT "from", "to", token AS address, amount AS value, tx_hash, block_num, log_idx, toString(block_timestamp, 'UTC') AS block_timestamp_utc
+            FROM token_transfers
+            ${where_materialized}
+            ORDER BY block_num ${direction}, log_idx ${direction}
+            LIMIT ${target}
+          ` as string,
+        })
+        return {
+          capped: false,
+          rows: result.rows.slice(splitAddress ? 0 : (offset ?? 0), target),
+          sample: false,
+        }
+      } catch (error) {
+        // Older chains can reject materialized address scans, so retain the
+        // bounded decoded scan as a compatibility fallback.
+        if (!Tidx.isQueryRejection(error)) throw error
+      }
+
+      const rows: Record<string, unknown>[] = []
+      let anchor = cursor?.position
+      let rawRowCount = 0
+      while (rows.length < target && rawRowCount < decodedTransferRowCap) {
+        const batchLimit = Math.min(
+          Math.max(target - rows.length, decodedTransferBatchSize),
+          Schema.maxPageWindow,
+          decodedTransferRowCap - rawRowCount,
+        )
+        const batch = await (async () => {
+          if (anchor === undefined) return execute(queryFilters, batchLimit)
+
+          // Signature-decoded queries reject or mis-plan tuple keysets. Read
+          // the cursor block and strict outer block range separately.
+          const block = Cursor.literal(anchor[0]!, 'int')
+          const index = Cursor.literal(anchor[1]!, 'int')
+          const comparison = order === 'asc' ? '>' : '<'
+          const sameBlock = await execute(
+            [...queryFilters, `block_num = ${block}`, `log_idx ${comparison} ${index}`],
+            batchLimit,
+          )
+          if (sameBlock.length >= batchLimit) return sameBlock
+          const outerBlocks = await execute(
+            [...queryFilters, `block_num ${comparison} ${block}`],
+            batchLimit - sameBlock.length,
+          )
+          return [...sameBlock, ...outerBlocks]
+        })()
+
+        rawRowCount += batch.length
+        rows.push(...batch.filter(isTip20TransferRow))
+        if (batch.length < batchLimit) break
+        const last = batch.at(-1)
+        const block = Value.toNumber(last?.['block_num'])
+        const index = Value.toNumber(last?.['log_idx'])
+        if (block === undefined || index === undefined) break
+        anchor = [block, index]
+      }
+      const capped = rows.length < target && rawRowCount >= decodedTransferRowCap
+      return {
+        capped,
+        ...(capped && anchor !== undefined
+          ? {
+              resume: {
+                offset: Math.max(0, (offset ?? 0) - rows.length),
+                position: anchor,
+              },
+            }
+          : {}),
+        rows: rows.slice(splitAddress ? 0 : (offset ?? 0), target),
+        sample: false,
+      }
+    }
+
+    const queryLimit = options_fetch.limit ?? window
+    const queryOffset = options_fetch.offset ?? (!addressScoped ? offset : undefined)
+    const where = queryFilters.length > 0 ? `WHERE ${queryFilters.join(' AND ')}` : ''
+    try {
+      return {
+        capped: false,
+        rows: await execute(queryFilters, queryLimit, queryOffset),
+        sample: false,
+      }
     } catch (error) {
       // The planner kills a one-sided top-N walk when its matches sit far
       // from head (422 `db error`). Retry as a plain capped scan, returning
@@ -759,7 +956,7 @@ export async function scan(deps: scan.Deps, options: scan.Options): Promise<scan
       if (!sideFiltered || !Tidx.isDeterministicError(error)) throw error
       const result = await tidx.fetch({
         chainId,
-        ...(tokenScoped ? { engine: 'clickhouse' } : { signatures: [signature] }),
+        ...(materialized ? { engine: 'clickhouse' } : { signatures: [signature] }),
         query: `
           SELECT ${select}
           FROM ${table}
@@ -767,16 +964,93 @@ export async function scan(deps: scan.Deps, options: scan.Options): Promise<scan
           LIMIT ${Schema.countCap}
         ` as string,
       })
-      return { rows: result.rows, sample: true }
+      return { capped: result.rows.length >= Schema.countCap, rows: result.rows, sample: true }
     }
   }
 
+  const windowed = global
+  const fetchWindowedRows = async () => {
+    const indexedCeiling =
+      options.toTimestamp === undefined
+        ? await indexedHead(tidx, { chainId })
+        : await indexedBlockAtOrBefore(tidx, { chainId, timestamp: options.toTimestamp })
+    if (indexedCeiling === undefined) return { capped: false, rows: [], sample: false }
+
+    const floor = options.fromBlock ?? 0
+    const ceiling = Math.min(options.toBlock ?? indexedCeiling, indexedCeiling)
+    const target = (offset ?? 0) + limit + 1
+    const rows: Record<string, unknown>[] = []
+    const cursorBlock = cursor?.position[0]
+    let resumePosition: readonly [number, number] | undefined
+    let scannedWindows = 0
+    let anchor =
+      order === 'asc'
+        ? Math.max(floor, cursorBlock ?? floor)
+        : Math.min(ceiling, cursorBlock ?? ceiling)
+    while (
+      anchor >= floor &&
+      anchor <= ceiling &&
+      rows.length < target &&
+      scannedWindows < transferWindowScanCap
+    ) {
+      const window_lo =
+        order === 'asc' ? anchor : Math.max(floor, anchor - transferWindowBlocks + 1)
+      const window_hi =
+        order === 'desc' ? anchor : Math.min(ceiling, anchor + transferWindowBlocks - 1)
+      let windowOffset = 0
+      while (rows.length < target) {
+        const queryLimit = Math.min(target - rows.length, Schema.maxPageWindow)
+        const result = await fetchRows({
+          hi: window_hi,
+          limit: queryLimit,
+          lo: window_lo,
+          offset: windowOffset,
+        })
+        rows.push(...result.rows)
+        windowOffset += result.rows.length
+        if (result.rows.length < queryLimit) break
+      }
+      scannedWindows++
+      // Resume after the fully scanned boundary without skipping the next block.
+      resumePosition = order === 'asc' ? [window_hi, Number.MAX_SAFE_INTEGER] : [window_lo, 0]
+      anchor = order === 'asc' ? window_hi + 1 : window_lo - 1
+    }
+    const capped =
+      rows.length < target &&
+      anchor >= floor &&
+      anchor <= ceiling &&
+      scannedWindows >= transferWindowScanCap
+    return {
+      capped,
+      ...(capped && resumePosition !== undefined
+        ? {
+            resume: {
+              offset: Math.max(0, (offset ?? 0) - rows.length),
+              position: resumePosition,
+            },
+          }
+        : {}),
+      rows: rows.slice(offset ?? 0),
+      sample: false,
+    }
+  }
+
+  // With `to = recipient`, the either-side address is redundant when equal
+  // and otherwise reduces to `from = address`, so one ordered stream suffices.
+  const recipientAddressSide =
+    recipientDecoded && address !== undefined && address !== recipient ? 'from' : undefined
   // TIDX cannot plan `(from = X OR to = X)` over the decoded Transfer table.
-  // Fetch one bounded, server-filtered stream per side and collate app-side.
-  const fetched = addressScoped
-    ? await Promise.all([fetchRows('from'), fetchRows('to')])
-    : [await fetchRows()]
-  const { capped, rows } = collate(fetched, { limit: limit + 1, offset: offset ?? 0, order })
+  // Fetch one stream per side only when windowing or recipient reduction does not apply.
+  const fetched = windowed
+    ? [await fetchWindowedRows()]
+    : splitAddress
+      ? await Promise.all([fetchRows({ side: 'from' }), fetchRows({ side: 'to' })])
+      : [await fetchRows(recipientAddressSide === undefined ? {} : { side: recipientAddressSide })]
+  const { capped, resume, rows } = collate(fetched, {
+    limit: limit + 1,
+    offset: offset ?? 0,
+    order,
+  })
 
   // The next page anchors below the last fetched row's `(block, log_idx)`.
   const page = Cursor.paginate({
@@ -798,7 +1072,7 @@ export async function scan(deps: scan.Deps, options: scan.Options): Promise<scan
     const amount = Value.toIntegerString(row['value'])
     const blockNumber = Value.toNumber(row['block_num'])
     const logIndex = Value.toNumber(row['log_idx'])
-    const timestamp = Value.toIsoDateTime(row['block_timestamp'])
+    const timestamp = Value.toIsoDateTime(row['block_timestamp_utc'] ?? row['block_timestamp'])
     if (
       tokenAddress.success &&
       from.success &&
@@ -824,29 +1098,141 @@ export async function scan(deps: scan.Deps, options: scan.Options): Promise<scan
       })
   }
 
-  // A truncated sample cannot anchor keyset pagination without silently
-  // skipping unseen matches, so the page ends the list instead.
+  // Every matching row through `resume.position` was inspected, so advancing
+  // there cannot skip a match. Positional continuations retain their offset.
+  const nextCursor = resume
+    ? Cursor.encode(resume.offset > 0 ? [...resume.position, resume.offset] : [...resume.position])
+    : capped
+      ? null
+      : page.nextCursor
   return {
     capped,
-    hasMore: capped ? false : page.hasMore,
+    hasMore: resume ? true : capped ? false : page.hasMore,
     items,
     limit,
-    nextCursor: capped ? null : page.nextCursor,
+    nextCursor,
+    ...(resume ? { resumeCursor: resume.position } : {}),
+  }
+}
+
+declare namespace fetchRows {
+  /** Source-side, block-window, and pagination constraints for one upstream scan. */
+  type Options = {
+    /** Inclusive highest block number. */
+    hi?: number | undefined
+    /** Maximum rows to return. */
+    limit?: number | undefined
+    /** Inclusive lowest block number. */
+    lo?: number | undefined
+    /** Rows to skip before returning results. */
+    offset?: number | undefined
+    /** Indexed event participant side. */
+    side?: 'from' | 'to' | undefined
+  }
+}
+
+/** Reads the latest block available to TIDX for a descending window scan. */
+async function indexedHead(tidx: Tidx.Client, options: indexedHead.Options) {
+  const result = await tidx.fetch({
+    chainId: options.chainId,
+    query: 'SELECT num FROM blocks ORDER BY num DESC LIMIT 1',
+  })
+  return Value.toNumber(result.rows[0]?.['num'])
+}
+
+declare namespace indexedHead {
+  /** Chain whose indexed head should be read. */
+  type Options = {
+    /** Target chain id. */
+    chainId: number
+  }
+}
+
+/** Reads the latest indexed block at or before a timestamp. */
+async function indexedBlockAtOrBefore(tidx: Tidx.Client, options: indexedBlockAtOrBefore.Options) {
+  const { chainId, timestamp } = options
+  const result = await tidx.fetch({
+    chainId,
+    query: `
+      SELECT num
+      FROM blocks
+      WHERE timestamp <= '${Tidx.escape(timestamp)}'
+      ORDER BY num DESC
+      LIMIT 1
+    ` as string,
+  })
+  return Value.toNumber(result.rows[0]?.['num'])
+}
+
+declare namespace indexedBlockAtOrBefore {
+  /** Chain and cutoff for the highest qualifying indexed block. */
+  type Options = {
+    /** Target chain id. */
+    chainId: number
+    /** Inclusive timestamp cutoff. */
+    timestamp: string
   }
 }
 
+/** Formats an ISO timestamp for ClickHouse `DateTime64` comparisons. */
+function formatClickHouseDateTime(value: string) {
+  return value.replace('T', ' ').replace(/Z$/, '')
+}
+
 /**
  * Collates per-side transfer streams into one ordered page window: split and
  * sampled streams are deduped, globally ordered, and paged app-side, while a
- * lone engine-paged stream passes through. `capped` marks a fallback sample
- * that hit `Schema.countCap`: an arbitrary subset of that side's matches,
- * unfit to anchor pagination or exact counts.
+ * lone engine-paged stream passes through. Ordered capped streams may carry a
+ * safe raw continuation; unordered samples cannot.
  */
 export function collate(
   fetched: readonly collate.Stream[],
   options: collate.Options,
 ): collate.Result {
-  const capped = fetched.some(({ rows, sample }) => sample && rows.length >= Schema.countCap)
+  const capped = fetched.some((stream) => stream.capped)
+  const cappedStreams = fetched.filter((stream) => stream.capped)
+  const resumes = cappedStreams.flatMap((stream) => (stream.resume ? [stream.resume] : []))
+  if (
+    fetched.length > 1 &&
+    capped &&
+    !fetched.some((stream) => stream.sample) &&
+    resumes.length === cappedStreams.length
+  ) {
+    // Advance only through the least-progressed side, so the next strict
+    // keyset scan cannot rediscover any returned rows.
+    const position = resumes
+      .map((resume) => resume.position)
+      .reduce((candidate, current) => {
+        const comparison = candidate[0] - current[0] || candidate[1] - current[1]
+        if (options.order === 'asc') return comparison <= 0 ? candidate : current
+        return comparison >= 0 ? candidate : current
+      })
+    const safeRows = fetched.flatMap((stream) =>
+      stream.rows.filter((row) => {
+        const blockNumber = Value.toNumber(row['block_num'])
+        const logIndex = Value.toNumber(row['log_idx'])
+        if (blockNumber === undefined || logIndex === undefined) return false
+        const comparison = blockNumber - position[0] || logIndex - position[1]
+        return options.order === 'asc' ? comparison <= 0 : comparison >= 0
+      }),
+    )
+    const merged = mergeTransferRows(safeRows, {
+      limit: options.offset + options.limit,
+      offset: 0,
+      order: options.order,
+    })
+    const rows = merged.slice(options.offset, options.offset + options.limit)
+    if (rows.length >= options.limit) return { capped: false, rows }
+    return {
+      capped: true,
+      resume: {
+        offset: Math.max(0, options.offset - merged.length),
+        position,
+      },
+      rows,
+    }
+  }
+  const resume = fetched.length === 1 ? fetched[0]?.resume : undefined
   const rows =
     fetched.length > 1 || fetched.some(({ sample }) => sample)
       ? mergeTransferRows(
@@ -854,17 +1240,28 @@ export function collate(
           options,
         )
       : (fetched[0]?.rows ?? [])
-  return { capped, rows }
+  return { capped, ...(resume ? { resume } : {}), rows }
 }
 
 export declare namespace collate {
   /** One fetched transfer stream. */
   type Stream = {
+    /** Whether upstream bounds truncated this stream. */
+    capped: boolean
     /** Raw TIDX rows. */
     rows: readonly Record<string, unknown>[]
+    /** Ordered raw position and unmatched positional offset after a bounded scan. */
+    resume?: Resume | undefined
     /** Whether the rows are an unordered, unpaged capped fallback sample. */
     sample: boolean
   }
+  /** Progress retained after a bounded ordered scan. */
+  type Resume = {
+    /** Filtered rows that a positional continuation still needs to skip. */
+    offset: number
+    /** Last ordered position safely inspected by the bounded scan. */
+    position: readonly [blockNumber: number, logIndex: number]
+  }
   /** The page window applied to the merged stream. */
   type Options = {
     /** Maximum merged rows to return. */
@@ -876,10 +1273,12 @@ export declare namespace collate {
   }
   /** The collated page window. */
   type Result = {
-    /** Whether any fallback sample hit `Schema.countCap`. */
+    /** Whether an upstream work bound truncated any stream. */
     capped: boolean
     /** The ordered page window rows. */
     rows: readonly Record<string, unknown>[]
+    /** Ordered continuation state, absent for unordered capped samples. */
+    resume?: Resume | undefined
   }
 }
 
@@ -935,12 +1334,33 @@ export declare namespace scan {
   }
   /** A page of scanned `Transfer` rows. */
   type Page = {
-    /** Whether the page came from a cap-truncated fallback sample; such a page must not advance a poller cursor. */
+    /** Whether an upstream work bound truncated the page. */
     capped: boolean
     hasMore: boolean
     items: readonly Item[]
     limit: number
     nextCursor: string | null
+    /** Last ordered position safely inspected, when a capped scan can resume. */
+    resumeCursor?: readonly [blockNumber: number, logIndex: number] | undefined
+  }
+}
+
+/** Decodes ordinary transfer cursors and capped positional continuations. */
+function decodeTransferCursor(token: string | undefined): decodeTransferCursor.Result | undefined {
+  if (token === undefined) return undefined
+  const position = Cursor.decode(token, ['int', 'int'])
+  if (position) return { offset: 0, position: position as [blockNumber: number, logIndex: number] }
+  const continuation = Cursor.decode(token, ['int', 'int', 'int'])
+  if (!continuation) return undefined
+  const [blockNumber, logIndex, offset] = continuation as [number, number, number]
+  return { offset: Math.max(0, offset), position: [blockNumber, logIndex] }
+}
+
+declare namespace decodeTransferCursor {
+  /** Decoded raw position plus any filtered rows still to skip. */
+  type Result = {
+    offset: number
+    position: readonly [blockNumber: number, logIndex: number]
   }
 }
 
@@ -949,7 +1369,7 @@ export declare namespace scan {
  * unscoped queries constrain the decoded `Transfer` emitter to TIP-20 contracts.
  */
 function filterClauses(options: filterClauses.Options): string[] {
-  const { fromBlock, fromTimestamp, recipient, sender, toBlock, toTimestamp, token } = options
+  const { fromBlock, fromTimestamp, recipient, sender, toBlock, token, toTimestamp } = options
   const filters =
     token === undefined
       ? [`address >= '${tip20AddressRange[0]}'`, `address < '${tip20AddressRange[1]}'`]
@@ -971,6 +1391,13 @@ declare namespace filterClauses {
   >
 }
 
+function isTip20TransferRow(row: Record<string, unknown>) {
+  const address = row['address']
+  if (typeof address !== 'string') return false
+  const normalized = address.toLowerCase()
+  return normalized >= tip20AddressRange[0] && normalized < tip20AddressRange[1]
+}
+
 /**
  * Capped total-row count for the `Transfer` feed, sharing {@link query}'s
  * filters (minus pagination) so the count matches the page it annotates. Timed
@@ -1033,16 +1460,187 @@ export async function countScan(
   // rows, classify each transaction, and subtract the folded-away legs. Bounded
   // by `countCap` like the raw count, so it stays within the indexer's budget.
   if (options.crossToken) {
+    const global =
+      !tokenScoped &&
+      address === undefined &&
+      options.sender === undefined &&
+      options.recipient === undefined
+    if (global) {
+      // A capped count promises a conservative lower bound, so aggregate the
+      // deduplicated capped sample without the global block sort or 10,000-row response.
+      const materializedFilters = [
+        `token >= '${tip20AddressRange[0]}'`,
+        `token < '${tip20AddressRange[1]}'`,
+        ...(options.fromBlock !== undefined ? [`block_num >= ${options.fromBlock}`] : []),
+        ...(options.toBlock !== undefined ? [`block_num <= ${options.toBlock}`] : []),
+        ...(options.fromTimestamp !== undefined
+          ? [`block_timestamp >= '${options.fromTimestamp}'`]
+          : []),
+        ...(options.toTimestamp !== undefined
+          ? [`block_timestamp <= '${options.toTimestamp}'`]
+          : []),
+      ]
+      const result = await tidx.fetch({
+        chainId,
+        engine: 'clickhouse',
+        query: `
+          SELECT
+            count(*) AS total,
+            countIf(
+              "to" = '${Addresses.stablecoinDex}'
+              OR "from" = '${Addresses.stablecoinDex}'
+              OR "to" = '${Addresses.feeManager}'
+          ) AS suppressed
+          FROM (
+            SELECT tx_hash, log_idx, "from", "to"
+            FROM token_transfers FINAL
+            WHERE ${materializedFilters.join(' AND ')}
+            LIMIT ${Schema.countCap}
+          ) AS capped
+        ` as string,
+      })
+      const total = Value.toNumber(result.rows[0]?.['total']) ?? 0
+      const suppressed = Value.toNumber(result.rows[0]?.['suppressed']) ?? 0
+      if (total >= Schema.countCap) {
+        return {
+          totalCount: Math.max(0, total - suppressed),
+          totalCountCapped: true,
+        }
+      }
+      if (suppressed === 0) return { totalCount: total, totalCountCapped: false }
+    }
+
+    if (address !== undefined) {
+      const materializedFilters = [
+        ...(options.token === undefined
+          ? [`token >= '${tip20AddressRange[0]}'`, `token < '${tip20AddressRange[1]}'`]
+          : [`token = '${options.token}'`]),
+        ...(options.sender !== undefined ? [`"from" = '${options.sender}'`] : []),
+        ...(options.recipient !== undefined ? [`"to" = '${options.recipient}'`] : []),
+        ...(options.fromBlock !== undefined ? [`block_num >= ${options.fromBlock}`] : []),
+        ...(options.toBlock !== undefined ? [`block_num <= ${options.toBlock}`] : []),
+        ...(options.fromTimestamp !== undefined
+          ? [`block_timestamp >= '${options.fromTimestamp}'`]
+          : []),
+        ...(options.toTimestamp !== undefined
+          ? [`block_timestamp <= '${options.toTimestamp}'`]
+          : []),
+      ]
+      const sentFilters = [`"from" = '${address}'`, ...materializedFilters]
+      const receivedFilters = [
+        `"to" = '${address}'`,
+        `"from" != '${address}'`,
+        ...materializedFilters,
+      ]
+      const aggregate = await tidx.fetch({
+        chainId,
+        engine: 'clickhouse',
+        query: `
+          SELECT
+            'sent' AS side,
+            count(*) AS total,
+            countIf(
+              "to" = '${Addresses.stablecoinDex}'
+              OR "from" = '${Addresses.stablecoinDex}'
+              OR "to" = '${Addresses.feeManager}'
+            ) AS suppressed
+          FROM (
+            SELECT "from", "to"
+            FROM token_transfers
+            WHERE ${sentFilters.join(' AND ')}
+            LIMIT ${Schema.countCap}
+          ) AS sent
+          UNION ALL
+          SELECT
+            'received' AS side,
+            count(*) AS total,
+            countIf(
+              "to" = '${Addresses.stablecoinDex}'
+              OR "from" = '${Addresses.stablecoinDex}'
+              OR "to" = '${Addresses.feeManager}'
+            ) AS suppressed
+          FROM (
+            SELECT "from", "to"
+            FROM token_transfers
+            WHERE ${receivedFilters.join(' AND ')}
+            LIMIT ${Schema.countCap}
+          ) AS received
+        ` as string,
+      })
+      const sent = aggregate.rows.find((row) => Value.toText(row['side']) === 'sent')
+      const received = aggregate.rows.find((row) => Value.toText(row['side']) === 'received')
+      const sentTotal = Value.toNumber(sent?.['total']) ?? 0
+      const receivedTotal = Value.toNumber(received?.['total']) ?? 0
+      const total = sentTotal + receivedTotal
+      const suppressed =
+        (Value.toNumber(sent?.['suppressed']) ?? 0) +
+        (Value.toNumber(received?.['suppressed']) ?? 0)
+      const capped =
+        sentTotal >= Schema.countCap || receivedTotal >= Schema.countCap || total >= Schema.countCap
+      if (capped)
+        return {
+          totalCount: Math.min(Math.max(0, total - suppressed), Schema.countCap),
+          totalCountCapped: true,
+        }
+      if (suppressed === 0) return { totalCount: total, totalCountCapped: false }
+
+      // A below-cap aggregate proves these unsorted scans are complete, so the
+      // exact bundle classifier does not need a global sort.
+      const select = 'tx_hash, block_num, log_idx, "from", "to", token AS address, amount AS value'
+      const fetchRows = async (sideFilters: readonly string[]) => {
+        const result = await tidx.fetch({
+          chainId,
+          engine: 'clickhouse',
+          query: `
+            SELECT ${select}
+            FROM token_transfers
+            WHERE ${sideFilters.join(' AND ')}
+            LIMIT ${Schema.countCap}
+          ` as string,
+        })
+        return result.rows
+      }
+      const [sentRows, receivedRows] = await Promise.all([
+        fetchRows(sentFilters),
+        fetchRows(receivedFilters),
+      ])
+      const rows = [...sentRows, ...receivedRows]
+      const transactionHashes = [...legsByTransaction(rows)]
+        .filter(([, legs]) => legs.some(isBundleLeg))
+        .map(([transactionHash]) => transactionHash)
+      const transactionHashSet = new Set(transactionHashes)
+      const candidateRows = rows.filter((row) => {
+        const transactionHash = Value.toText(row['tx_hash'])
+        return (
+          transactionHash !== undefined && transactionHashSet.has(transactionHash.toLowerCase())
+        )
+      })
+      const blocks = candidateRows.flatMap((row) => {
+        const block = Value.toNumber(row['block_num'])
+        return block === undefined ? [] : [block]
+      })
+      if (blocks.length === 0 || transactionHashes.length === 0)
+        return foldCount(rows, { capped: false })
+      // Address predicates omit sibling bundle legs. Load each candidate
+      // transaction completely, then suppress only address-matched rows.
+      const classification = await tidx.fetch({
+        chainId,
+        engine: 'clickhouse',
+        query: `
+          SELECT tx_hash, block_num, log_idx, "from", "to", token AS address, amount AS value
+          FROM token_transfers
+          WHERE block_num BETWEEN ${Math.min(...blocks)} AND ${Math.max(...blocks)}
+            AND tx_hash IN (${transactionHashes.map((hash) => `'${hash}'`).join(', ')})
+        ` as string,
+      })
+      return foldCount(rows, { capped: false, classificationRows: classification.rows })
+    }
+
     // Only one-sided `"from"`/`"to"` equality shapes hit the planner's
     // deterministic top-N kill, so only they take the capped fallback.
-    const sideFiltered =
-      address !== undefined || options.sender !== undefined || options.recipient !== undefined
-    const fetchRows = async (side?: 'from' | 'to') => {
-      const sideFilters =
-        side !== undefined && address !== undefined
-          ? [`"${side}" = '${address}'`, ...filters]
-          : filters
-      const sideWhere = sideFilters.length > 0 ? `WHERE ${sideFilters.join(' AND ')}` : ''
+    const sideFiltered = options.sender !== undefined || options.recipient !== undefined
+    const fetchRows = async () => {
+      const where = filters.length > 0 ? `WHERE ${filters.join(' AND ')}` : ''
       const select = `tx_hash, block_num, log_idx, "from", "to", ${tokenScoped ? 'token AS address, amount AS value' : 'address, value'}`
       const table = tokenScoped ? 'token_transfers' : 'Transfer'
       try {
@@ -1052,12 +1650,12 @@ export async function countScan(
           query: `
             SELECT ${select}
             FROM ${table}
-            ${sideWhere}
+            ${where}
             ORDER BY block_num, log_idx
             LIMIT ${Schema.countCap}
           ` as string,
         })
-        return { rows: result.rows, sample: false }
+        return { capped: false, rows: result.rows, sample: false }
       } catch (error) {
         // Same planner kill as the page's side scan; the folded count is
         // already capped, so a plain capped scan keeps its semantics.
@@ -1068,17 +1666,14 @@ export async function countScan(
           query: `
             SELECT ${select}
             FROM ${table}
-            ${sideWhere}
+            ${where}
             LIMIT ${Schema.countCap}
           ` as string,
         })
-        return { rows: result.rows, sample: true }
+        return { capped: result.rows.length >= Schema.countCap, rows: result.rows, sample: true }
       }
     }
-    const fetched =
-      address !== undefined
-        ? await Promise.all([fetchRows('from'), fetchRows('to')])
-        : [await fetchRows()]
+    const fetched = [await fetchRows()]
     const { capped, rows } = collate(fetched, { limit: Schema.countCap, offset: 0, order: 'asc' })
     return foldCount(rows, { capped })
   }
@@ -1139,15 +1734,24 @@ export function foldCount(
   rows: readonly Record<string, unknown>[],
   options: foldCount.Options,
 ): count.Result {
-  const { capped } = options
+  const { capped, classificationRows = rows } = options
+  const rowKeys = new Set<string>()
+  for (const row of rows) {
+    const transactionHash = Value.toText(row['tx_hash'])
+    const logIndex = Value.toNumber(row['log_idx'])
+    if (transactionHash && logIndex !== undefined) rowKeys.add(eventKey(transactionHash, logIndex))
+  }
   let suppressed = 0
-  for (const legs of legsByTransaction(rows).values()) {
+  for (const [transactionHash, legs] of legsByTransaction(capped ? rows : classificationRows)) {
     if (capped) {
       suppressed += legs.filter(isBundleLeg).length
       continue
     }
     const bundle = classifyCrossToken(legs)
-    if (bundle) suppressed += bundle.suppressLogIndexes.length
+    if (bundle)
+      suppressed += bundle.suppressLogIndexes.filter((logIndex) =>
+        rowKeys.has(eventKey(transactionHash, logIndex)),
+      ).length
   }
   return {
     totalCount: rows.length - suppressed,
@@ -1160,6 +1764,8 @@ export declare namespace foldCount {
   type Options = {
     /** Whether the rows include a cap-truncated fallback sample. */
     capped: boolean
+    /** Complete transaction legs used to classify the filtered rows. */
+    classificationRows?: readonly Record<string, unknown>[] | undefined
   }
 }
 
@@ -1289,12 +1895,13 @@ type CrossTokenBundle = {
  * transfer and is left untouched.
  */
 function classifyCrossToken(legs: readonly Leg[]): CrossTokenBundle | undefined {
-  const toDex = legs.find((leg) => Address.isEqual(leg.to, Addresses.stablecoinDex))
-  const fromDex = legs.find((leg) => Address.isEqual(leg.from, Addresses.stablecoinDex))
+  const ordered = [...legs].sort((a, b) => a.logIndex - b.logIndex)
+  const toDex = ordered.find((leg) => Address.isEqual(leg.to, Addresses.stablecoinDex))
+  const fromDex = ordered.find((leg) => Address.isEqual(leg.from, Addresses.stablecoinDex))
   if (!toDex || !fromDex) return undefined
   const swapper = fromDex.to
   const destTokenAddress = fromDex.address
-  const delivery = legs.find(
+  const delivery = ordered.find(
     (leg) =>
       Address.isEqual(leg.address, destTokenAddress) &&
       Address.isEqual(leg.from, swapper) &&
@@ -1304,7 +1911,7 @@ function classifyCrossToken(legs: readonly Leg[]): CrossTokenBundle | undefined
   )
   // Require a token change so plain same-token DEX activity is not folded.
   if (!delivery || Address.isEqual(toDex.address, delivery.address)) return undefined
-  const suppressLogIndexes = legs
+  const suppressLogIndexes = ordered
     .filter((leg) => leg.logIndex !== delivery.logIndex && isBundleLeg(leg))
     .map((leg) => leg.logIndex)
   return {