tapimo 0.8.0 → 0.10.0

package/dist/apps/data/routes/transfers.js

@@ -22,18 +22,35 @@ import * as Valuation from './valuation.js';
 // signature exposes a decoded `Transfer` CTE (with `from`/`to`/`value`) to
 // the indexer SQL.
 export const signature = 'event Transfer(address indexed from, address indexed to, uint256 value)';
-// TIP-20 contracts occupy the deterministic `0x20c0…` range. Full-width
-// bounds compare correctly as PostgreSQL bytea and ClickHouse hex strings.
-const tip20AddressRange = [
+/**
+ * Half-open `[start, end)` bounds of the deterministic TIP-20 contract range.
+ * Full-width so the values compare correctly as PostgreSQL bytea, ClickHouse
+ * hex strings, and lowercased JavaScript strings.
+ */
+export const tip20AddressRange = [
     '0x20c0000000000000000000000000000000000000',
     '0x20c1000000000000000000000000000000000000',
 ];
+// The global feed spans every TIP-20 token, so its newest-first query cannot
+// follow `token_transfers`' token-first sort key. Block windows bound the sort.
+const transferWindowBlocks = 100_000;
+// Sparse global scans retain progress without monopolizing one request.
+const transferWindowScanCap = 8;
+// Recipient-first decoded scans usually fill a page immediately. A larger raw
+// batch leaves room to discard non-TIP-20 `Transfer` events without extra I/O.
+const decodedTransferBatchSize = 256;
+// Bound decoded recipient traversal while preserving the largest positional
+// page and its extra row for `nextCursor` detection.
+const decodedTransferRowCap = Schema.maxPageWindow + 1;
 // A memo'd TIP-20 transfer emits `TransferWithMemo` alongside the plain
 // `Transfer`; the two rows match on `(tx_hash, from, to, amount, token)`. A
 // text memo is decoded into `memo`; an MPP attribution fingerprint memo is
 // resolved into `attribution` instead.
 const memoSignature = 'event TransferWithMemo(address indexed from, address indexed to, uint256 amount, bytes32 indexed memo)';
 const tokenFields = ['token.logoUri', 'token.verified'];
+// Concurrent metadata reads collapse into one deployless multicall. Small
+// chunks keep each call under the Tempo RPC's per-call size limit.
+const tokenReadChunkSize = 10;
 /** Zod schemas owned by the transfer handlers. */
 export var schema;
 (function (schema) {
@@ -239,8 +256,8 @@ export function transfers(options = {}) {
             });
         const options = c.req.valid('query');
         const chainId = options.chainId ?? c.get('chainId');
-        const denomination = options['valuation.currency'] ?? 'USD';
-        // Cursor pages carry current valuations and curated verification data.
+        const denomination = options['valuation.currency'];
+        // Cursor pages can carry current valuations and curated verification data.
         if (options.cursor !== undefined)
             Cache.setPolicy(c, Cache.policies.metadata);
         // Filters shared by the page query and the opt-in total count, so the
@@ -329,49 +346,58 @@ export function transfers(options = {}) {
                     return suppress.has(key) ? [] : [transfer];
                 });
             }
-            // Verification is ancillary to the feed. Skip empty pages and degrade
-            // cold snapshot failures to null valuations.
-            const snapshot = data.length > 0
-                ? await VerifiedTokens.snapshot(c, chainId).catch(() => undefined)
-                : undefined;
             const includeLogoUri = options.include.includes('token.logoUri');
             const includeVerified = options.include.includes('token.verified');
+            // Curated data is needed only for requested valuation or token fields.
+            const snapshot = data.length > 0 && (denomination !== undefined || includeLogoUri || includeVerified)
+                ? await VerifiedTokens.snapshot(c, chainId).catch(() => undefined)
+                : undefined;
             const uniqueTokens = Array.from(new Set(data.flatMap((transfer) => [
                 transfer.sourceToken.address,
                 ...(transfer.destinationToken ? [transfer.destinationToken.address] : []),
             ])));
-            const tokensByAddress = new Map(await Timing.time(c, 'tokens', () => Promise.all(uniqueTokens.map(async (token) => {
-                const [logoUri, metadata] = await Promise.all([
-                    includeLogoUri
-                        ? Timing.time(c, 'token_logo', () => Tokens.getTokenLogo(c, { address: token, chainId })).catch(() => undefined)
-                        : undefined,
-                    Timing.time(c, 'token_metadata', () => Tokens.getTokenMetadata(c, { address: token, chainId })),
-                ]);
-                return [
-                    token.toLowerCase(),
-                    {
-                        ...metadata,
-                        logoUri: includeLogoUri
-                            ? (logoUri ??
-                                snapshot?.byAddress.get(token.toLowerCase())?.logoUri ??
-                                metadata.logoUri)
-                            : undefined,
-                    },
-                ];
-            }))));
+            const tokensByAddress = new Map();
+            await Timing.time(c, 'tokens', async () => {
+                for (let index = 0; index < uniqueTokens.length; index += tokenReadChunkSize) {
+                    const group = uniqueTokens.slice(index, index + tokenReadChunkSize);
+                    const entries = await Promise.all(group.map(async (token) => {
+                        const [logoUri, metadata] = await Promise.all([
+                            includeLogoUri
+                                ? Timing.time(c, 'token_logo', () => Tokens.getTokenLogo(c, { address: token, chainId })).catch(() => undefined)
+                                : undefined,
+                            Timing.time(c, 'token_metadata', () => Tokens.getTokenMetadata(c, { address: token, chainId })),
+                        ]);
+                        return [
+                            token.toLowerCase(),
+                            {
+                                ...metadata,
+                                logoUri: includeLogoUri
+                                    ? (logoUri ??
+                                        snapshot?.byAddress.get(token.toLowerCase())?.logoUri ??
+                                        metadata.logoUri)
+                                    : undefined,
+                            },
+                        ];
+                    }));
+                    for (const entry of entries)
+                        tokensByAddress.set(...entry);
+                }
+            });
             // Value each leg via its curated display currency, in the requested
             // denomination.
-            const rates = await Valuation.ratesFor(c, {
-                currencies: data.flatMap((transfer) => [
-                    transfer.sourceToken.address,
-                    ...(transfer.destinationToken ? [transfer.destinationToken.address] : []),
-                ].flatMap((address) => {
-                    const held = snapshot?.byAddress.get(address.toLowerCase())?.currency;
-                    return held === undefined ? [] : [held];
-                })),
-                denomination,
-                oracle,
-            });
+            const rates = denomination
+                ? await Valuation.ratesFor(c, {
+                    currencies: data.flatMap((transfer) => [
+                        transfer.sourceToken.address,
+                        ...(transfer.destinationToken ? [transfer.destinationToken.address] : []),
+                    ].flatMap((address) => {
+                        const held = snapshot?.byAddress.get(address.toLowerCase())?.currency;
+                        return held === undefined ? [] : [held];
+                    })),
+                    denomination,
+                    oracle,
+                })
+                : undefined;
             const enrich = (leg) => {
                 const metadata = tokensByAddress.get(leg.address.toLowerCase());
                 if (!metadata)
@@ -383,12 +409,16 @@ export function transfers(options = {}) {
                             currency: metadata.currency,
                             decimals: metadata.decimals,
                         }),
-                        valuation: Valuation.valuationFor({
-                            amount: BigInt(leg.amount),
-                            denomination,
-                            rates,
-                            token: snapshot?.byAddress.get(leg.address.toLowerCase()),
-                        }),
+                        ...(denomination
+                            ? {
+                                valuation: Valuation.valuationFor({
+                                    amount: BigInt(leg.amount),
+                                    denomination,
+                                    rates,
+                                    token: snapshot?.byAddress.get(leg.address.toLowerCase()),
+                                }),
+                            }
+                            : {}),
                     },
                     token: {
                         address: leg.address,
@@ -463,7 +493,7 @@ export function query(c, options) {
     const { chainId, timing, ...rest } = options;
     const store = c.get('store');
     const tidx = c.get('getTidx')(chainId);
-    const cursor = options.cursor ? Cursor.decode(options.cursor, ['int', 'int']) : undefined;
+    const cursor = decodeTransferCursor(options.cursor);
     return Timing.time(c, timing, () => Store.memoize(async () => {
         const page = await scan({ tidx }, { chainId, ...rest });
         return {
@@ -471,13 +501,13 @@ export function query(c, options) {
             nextCursor: page.nextCursor,
         };
     }, {
-        key: `transfers:v2:${chainId}:${options.order}:${options.token ?? ''}:${options.eitherSide ?? ''}:${options.sender ?? ''}:${options.recipient ?? ''}:${options.fromBlock ?? ''}:${options.toBlock ?? ''}:${options.fromTimestamp ?? ''}:${options.toTimestamp ?? ''}:${cursor ? `cursor:${cursor[0]}:${cursor[1]}` : options.page !== undefined && options.page > 1 ? `page:${options.page}` : 'head'}:${options.limit}`,
+        key: `transfers:v2:${chainId}:${options.order}:${options.token ?? ''}:${options.eitherSide ?? ''}:${options.sender ?? ''}:${options.recipient ?? ''}:${options.fromBlock ?? ''}:${options.toBlock ?? ''}:${options.fromTimestamp ?? ''}:${options.toTimestamp ?? ''}:${cursor ? `cursor:${cursor.position[0]}:${cursor.position[1]}:${cursor.offset}` : options.page !== undefined && options.page > 1 ? `page:${options.page}` : 'head'}:${options.limit}`,
         store,
         ttl: Ttl.seconds(15),
     }));
 }
 /**
- * Context-free core of {@link query}: reads token-scoped rows from
+ * Context-free core of {@link query}: reads token-scoped and global rows from
  * `token_transfers`, otherwise the decoded `Transfer` CTE, then paginates and
  * decodes them. Also used by the webhook poller.
  */
@@ -489,46 +519,193 @@ export async function scan(deps, options) {
     // Keyset pagination: anchor the page below the previous row's `(block, log_idx)`
     // position instead of a numeric offset, so rows arriving at the head can't
     // shift items across pages. A malformed cursor falls back to the head page.
-    const cursor = options.cursor ? Cursor.decode(options.cursor, ['int', 'int']) : undefined;
+    const cursor = decodeTransferCursor(options.cursor);
     // Bounded positional lane (exclusive with `cursor` at the schema): page 1 is
     // the head page, so it shares the head's SQL and cache entries; deeper pages
     // translate to a SQL OFFSET of `(page - 1) * limit` rows.
-    const offset = options.page !== undefined && options.page > 1 ? (options.page - 1) * limit : undefined;
+    const pageOffset = options.page !== undefined && options.page > 1 ? (options.page - 1) * limit : undefined;
+    const offset = pageOffset ?? (cursor?.offset === 0 ? undefined : cursor?.offset);
     const address = options.eitherSide;
+    const addressScoped = address !== undefined;
+    const recipient = options.recipient;
+    const recipientDecoded = !tokenScoped && recipient !== undefined;
+    const decoded = !tokenScoped && (addressScoped || recipientDecoded);
+    const splitAddress = addressScoped && !recipientDecoded;
+    // Only one-sided `"from"`/`"to"` equality shapes hit the planner's
+    // deterministic top-N kill, so only they take the capped fallback.
+    const sideFiltered = addressScoped || options.sender !== undefined || options.recipient !== undefined;
+    const global = !tokenScoped && !sideFiltered;
     // Transfer events filtered by any combination of contract / sender /
     // recipient / block / timestamp. The address shortcut is split below so
     // each upstream query can use one indexed side.
-    const filters = filterClauses(options);
-    if (cursor !== undefined)
+    const filters = global
+        ? [
+            `token >= '${tip20AddressRange[0]}'`,
+            `token < '${tip20AddressRange[1]}'`,
+            ...(options.fromBlock !== undefined ? [`block_num >= ${options.fromBlock}`] : []),
+            ...(options.toBlock !== undefined ? [`block_num <= ${options.toBlock}`] : []),
+            ...(options.fromTimestamp !== undefined
+                ? [`block_timestamp >= '${formatClickHouseDateTime(options.fromTimestamp)}'`]
+                : []),
+            ...(options.toTimestamp !== undefined
+                ? [`block_timestamp <= '${formatClickHouseDateTime(options.toTimestamp)}'`]
+                : []),
+        ]
+        : decoded
+            ? [
+                ...(options.sender !== undefined ? [`"from" = '${options.sender}'`] : []),
+                ...(recipient !== undefined ? [`"to" = '${recipient}'`] : []),
+                ...(options.fromBlock !== undefined ? [`block_num >= ${options.fromBlock}`] : []),
+                ...(options.toBlock !== undefined ? [`block_num <= ${options.toBlock}`] : []),
+                ...(options.fromTimestamp !== undefined
+                    ? [`block_timestamp >= '${options.fromTimestamp}'`]
+                    : []),
+                ...(options.toTimestamp !== undefined
+                    ? [`block_timestamp <= '${options.toTimestamp}'`]
+                    : []),
+            ]
+            : filterClauses(options);
+    if (cursor !== undefined && !decoded)
         filters.push(Cursor.keyset([
-            { literal: Cursor.literal(cursor[0], 'int'), name: 'block_num', order },
-            { literal: Cursor.literal(cursor[1], 'int'), name: 'log_idx', order },
+            { literal: Cursor.literal(cursor.position[0], 'int'), name: 'block_num', order },
+            { literal: Cursor.literal(cursor.position[1], 'int'), name: 'log_idx', order },
         ]));
-    const addressScoped = address !== undefined;
     const window = addressScoped ? (offset ?? 0) + limit + 1 : limit + 1;
-    // Only one-sided `"from"`/`"to"` equality shapes hit the planner's
-    // deterministic top-N kill, so only they take the capped fallback.
-    const sideFiltered = addressScoped || options.sender !== undefined || options.recipient !== undefined;
-    const fetchRows = async (side) => {
+    const fetchRows = async (options_fetch = {}) => {
+        const side = options_fetch.side;
         const sideFilters = side !== undefined && address !== undefined
             ? [`"${side}" = '${address}'`, ...filters]
             : filters;
-        const where = sideFilters.length > 0 ? `WHERE ${sideFilters.join(' AND ')}` : '';
-        const select = `"from", "to", ${tokenScoped ? 'token AS address, amount AS value' : 'address, value'}, tx_hash, block_num, log_idx, block_timestamp`;
-        const table = tokenScoped ? 'token_transfers' : 'Transfer';
-        try {
+        const queryFilters = [
+            ...sideFilters,
+            ...(options_fetch.lo !== undefined ? [`block_num >= ${options_fetch.lo}`] : []),
+            ...(options_fetch.hi !== undefined ? [`block_num <= ${options_fetch.hi}`] : []),
+        ];
+        const materialized = tokenScoped || global;
+        // Keep a distinct alias until tidx.ts parses ClickHouse timestamps as UTC.
+        const select = materialized
+            ? `"from", "to", token AS address, amount AS value, tx_hash, block_num, log_idx, toString(block_timestamp, 'UTC') AS block_timestamp_utc`
+            : `"from", "to", address, value, tx_hash, block_num, log_idx, block_timestamp`;
+        const table = materialized ? 'token_transfers' : 'Transfer';
+        const execute = async (filters_execute, queryLimit, queryOffset) => {
+            const where = filters_execute.length > 0 ? `WHERE ${filters_execute.join(' AND ')}` : '';
             const result = await tidx.fetch({
                 chainId,
-                ...(tokenScoped ? { engine: 'clickhouse' } : { signatures: [signature] }),
+                ...(materialized ? { engine: 'clickhouse' } : { signatures: [signature] }),
                 query: `
           SELECT ${select}
           FROM ${table}
           ${where}
           ORDER BY block_num ${direction}, log_idx ${direction}
-          LIMIT ${window}${!addressScoped && offset !== undefined ? ` OFFSET ${offset}` : ''}
+          LIMIT ${queryLimit}${queryOffset !== undefined ? ` OFFSET ${queryOffset}` : ''}
         `,
             });
-            return { rows: result.rows, sample: false };
+            return result.rows;
+        };
+        if (decoded) {
+            const target = (offset ?? 0) + limit + 1;
+            // Filter the physical token column so ClickHouse can use the materialized table ordering.
+            const filters_materialized = [
+                `token >= '${tip20AddressRange[0]}'`,
+                `token < '${tip20AddressRange[1]}'`,
+                ...(options.sender !== undefined ? [`"from" = '${options.sender}'`] : []),
+                ...(recipient !== undefined ? [`"to" = '${recipient}'`] : []),
+                ...(options.fromBlock !== undefined ? [`block_num >= ${options.fromBlock}`] : []),
+                ...(options.toBlock !== undefined ? [`block_num <= ${options.toBlock}`] : []),
+                ...(options.fromTimestamp !== undefined
+                    ? [`block_timestamp >= '${formatClickHouseDateTime(options.fromTimestamp)}'`]
+                    : []),
+                ...(options.toTimestamp !== undefined
+                    ? [`block_timestamp <= '${formatClickHouseDateTime(options.toTimestamp)}'`]
+                    : []),
+            ];
+            if (cursor !== undefined)
+                filters_materialized.push(Cursor.keyset([
+                    { literal: Cursor.literal(cursor.position[0], 'int'), name: 'block_num', order },
+                    { literal: Cursor.literal(cursor.position[1], 'int'), name: 'log_idx', order },
+                ]));
+            const sideFilters_materialized = side !== undefined && address !== undefined
+                ? [`"${side}" = '${address}'`, ...filters_materialized]
+                : filters_materialized;
+            const where_materialized = sideFilters_materialized.length > 0 ? `WHERE ${sideFilters_materialized.join(' AND ')}` : '';
+            try {
+                const result = await tidx.fetch({
+                    chainId,
+                    engine: 'clickhouse',
+                    query: `
+            SELECT DISTINCT "from", "to", token AS address, amount AS value, tx_hash, block_num, log_idx, toString(block_timestamp, 'UTC') AS block_timestamp_utc
+            FROM token_transfers
+            ${where_materialized}
+            ORDER BY block_num ${direction}, log_idx ${direction}
+            LIMIT ${target}
+          `,
+                });
+                return {
+                    capped: false,
+                    rows: result.rows.slice(splitAddress ? 0 : (offset ?? 0), target),
+                    sample: false,
+                };
+            }
+            catch (error) {
+                // Older chains can reject materialized address scans, so retain the
+                // bounded decoded scan as a compatibility fallback.
+                if (!Tidx.isQueryRejection(error))
+                    throw error;
+            }
+            const rows = [];
+            let anchor = cursor?.position;
+            let rawRowCount = 0;
+            while (rows.length < target && rawRowCount < decodedTransferRowCap) {
+                const batchLimit = Math.min(Math.max(target - rows.length, decodedTransferBatchSize), Schema.maxPageWindow, decodedTransferRowCap - rawRowCount);
+                const batch = await (async () => {
+                    if (anchor === undefined)
+                        return execute(queryFilters, batchLimit);
+                    // Signature-decoded queries reject or mis-plan tuple keysets. Read
+                    // the cursor block and strict outer block range separately.
+                    const block = Cursor.literal(anchor[0], 'int');
+                    const index = Cursor.literal(anchor[1], 'int');
+                    const comparison = order === 'asc' ? '>' : '<';
+                    const sameBlock = await execute([...queryFilters, `block_num = ${block}`, `log_idx ${comparison} ${index}`], batchLimit);
+                    if (sameBlock.length >= batchLimit)
+                        return sameBlock;
+                    const outerBlocks = await execute([...queryFilters, `block_num ${comparison} ${block}`], batchLimit - sameBlock.length);
+                    return [...sameBlock, ...outerBlocks];
+                })();
+                rawRowCount += batch.length;
+                rows.push(...batch.filter(isTip20TransferRow));
+                if (batch.length < batchLimit)
+                    break;
+                const last = batch.at(-1);
+                const block = Value.toNumber(last?.['block_num']);
+                const index = Value.toNumber(last?.['log_idx']);
+                if (block === undefined || index === undefined)
+                    break;
+                anchor = [block, index];
+            }
+            const capped = rows.length < target && rawRowCount >= decodedTransferRowCap;
+            return {
+                capped,
+                ...(capped && anchor !== undefined
+                    ? {
+                        resume: {
+                            offset: Math.max(0, (offset ?? 0) - rows.length),
+                            position: anchor,
+                        },
+                    }
+                    : {}),
+                rows: rows.slice(splitAddress ? 0 : (offset ?? 0), target),
+                sample: false,
+            };
+        }
+        const queryLimit = options_fetch.limit ?? window;
+        const queryOffset = options_fetch.offset ?? (!addressScoped ? offset : undefined);
+        const where = queryFilters.length > 0 ? `WHERE ${queryFilters.join(' AND ')}` : '';
+        try {
+            return {
+                capped: false,
+                rows: await execute(queryFilters, queryLimit, queryOffset),
+                sample: false,
+            };
         }
         catch (error) {
             // The planner kills a one-sided top-N walk when its matches sit far
@@ -538,7 +715,7 @@ export async function scan(deps, options) {
                 throw error;
             const result = await tidx.fetch({
                 chainId,
-                ...(tokenScoped ? { engine: 'clickhouse' } : { signatures: [signature] }),
+                ...(materialized ? { engine: 'clickhouse' } : { signatures: [signature] }),
                 query: `
           SELECT ${select}
           FROM ${table}
@@ -546,15 +723,84 @@ export async function scan(deps, options) {
           LIMIT ${Schema.countCap}
         `,
             });
-            return { rows: result.rows, sample: true };
+            return { capped: result.rows.length >= Schema.countCap, rows: result.rows, sample: true };
         }
     };
+    const windowed = global;
+    const fetchWindowedRows = async () => {
+        const indexedCeiling = options.toTimestamp === undefined
+            ? await indexedHead(tidx, { chainId })
+            : await indexedBlockAtOrBefore(tidx, { chainId, timestamp: options.toTimestamp });
+        if (indexedCeiling === undefined)
+            return { capped: false, rows: [], sample: false };
+        const floor = options.fromBlock ?? 0;
+        const ceiling = Math.min(options.toBlock ?? indexedCeiling, indexedCeiling);
+        const target = (offset ?? 0) + limit + 1;
+        const rows = [];
+        const cursorBlock = cursor?.position[0];
+        let resumePosition;
+        let scannedWindows = 0;
+        let anchor = order === 'asc'
+            ? Math.max(floor, cursorBlock ?? floor)
+            : Math.min(ceiling, cursorBlock ?? ceiling);
+        while (anchor >= floor &&
+            anchor <= ceiling &&
+            rows.length < target &&
+            scannedWindows < transferWindowScanCap) {
+            const window_lo = order === 'asc' ? anchor : Math.max(floor, anchor - transferWindowBlocks + 1);
+            const window_hi = order === 'desc' ? anchor : Math.min(ceiling, anchor + transferWindowBlocks - 1);
+            let windowOffset = 0;
+            while (rows.length < target) {
+                const queryLimit = Math.min(target - rows.length, Schema.maxPageWindow);
+                const result = await fetchRows({
+                    hi: window_hi,
+                    limit: queryLimit,
+                    lo: window_lo,
+                    offset: windowOffset,
+                });
+                rows.push(...result.rows);
+                windowOffset += result.rows.length;
+                if (result.rows.length < queryLimit)
+                    break;
+            }
+            scannedWindows++;
+            // Resume after the fully scanned boundary without skipping the next block.
+            resumePosition = order === 'asc' ? [window_hi, Number.MAX_SAFE_INTEGER] : [window_lo, 0];
+            anchor = order === 'asc' ? window_hi + 1 : window_lo - 1;
+        }
+        const capped = rows.length < target &&
+            anchor >= floor &&
+            anchor <= ceiling &&
+            scannedWindows >= transferWindowScanCap;
+        return {
+            capped,
+            ...(capped && resumePosition !== undefined
+                ? {
+                    resume: {
+                        offset: Math.max(0, (offset ?? 0) - rows.length),
+                        position: resumePosition,
+                    },
+                }
+                : {}),
+            rows: rows.slice(offset ?? 0),
+            sample: false,
+        };
+    };
+    // With `to = recipient`, the either-side address is redundant when equal
+    // and otherwise reduces to `from = address`, so one ordered stream suffices.
+    const recipientAddressSide = recipientDecoded && address !== undefined && address !== recipient ? 'from' : undefined;
     // TIDX cannot plan `(from = X OR to = X)` over the decoded Transfer table.
-    // Fetch one bounded, server-filtered stream per side and collate app-side.
-    const fetched = addressScoped
-        ? await Promise.all([fetchRows('from'), fetchRows('to')])
-        : [await fetchRows()];
-    const { capped, rows } = collate(fetched, { limit: limit + 1, offset: offset ?? 0, order });
+    // Fetch one stream per side only when windowing or recipient reduction does not apply.
+    const fetched = windowed
+        ? [await fetchWindowedRows()]
+        : splitAddress
+            ? await Promise.all([fetchRows({ side: 'from' }), fetchRows({ side: 'to' })])
+            : [await fetchRows(recipientAddressSide === undefined ? {} : { side: recipientAddressSide })];
+    const { capped, resume, rows } = collate(fetched, {
+        limit: limit + 1,
+        offset: offset ?? 0,
+        order,
+    });
     // The next page anchors below the last fetched row's `(block, log_idx)`.
     const page = Cursor.paginate({
         rows,
@@ -574,7 +820,7 @@ export async function scan(deps, options) {
         const amount = Value.toIntegerString(row['value']);
         const blockNumber = Value.toNumber(row['block_num']);
         const logIndex = Value.toNumber(row['log_idx']);
-        const timestamp = Value.toIsoDateTime(row['block_timestamp']);
+        const timestamp = Value.toIsoDateTime(row['block_timestamp_utc'] ?? row['block_timestamp']);
         if (tokenAddress.success &&
             from.success &&
             to.success &&
@@ -597,29 +843,103 @@ export async function scan(deps, options) {
                 },
             });
     }
-    // A truncated sample cannot anchor keyset pagination without silently
-    // skipping unseen matches, so the page ends the list instead.
+    // Every matching row through `resume.position` was inspected, so advancing
+    // there cannot skip a match. Positional continuations retain their offset.
+    const nextCursor = resume
+        ? Cursor.encode(resume.offset > 0 ? [...resume.position, resume.offset] : [...resume.position])
+        : capped
+            ? null
+            : page.nextCursor;
     return {
         capped,
-        hasMore: capped ? false : page.hasMore,
+        hasMore: resume ? true : capped ? false : page.hasMore,
         items,
         limit,
-        nextCursor: capped ? null : page.nextCursor,
+        nextCursor,
+        ...(resume ? { resumeCursor: resume.position } : {}),
     };
 }
+/** Reads the latest block available to TIDX for a descending window scan. */
+async function indexedHead(tidx, options) {
+    const result = await tidx.fetch({
+        chainId: options.chainId,
+        query: 'SELECT num FROM blocks ORDER BY num DESC LIMIT 1',
+    });
+    return Value.toNumber(result.rows[0]?.['num']);
+}
+/** Reads the latest indexed block at or before a timestamp. */
+async function indexedBlockAtOrBefore(tidx, options) {
+    const { chainId, timestamp } = options;
+    const result = await tidx.fetch({
+        chainId,
+        query: `
+      SELECT num
+      FROM blocks
+      WHERE timestamp <= '${Tidx.escape(timestamp)}'
+      ORDER BY num DESC
+      LIMIT 1
+    `,
+    });
+    return Value.toNumber(result.rows[0]?.['num']);
+}
+/** Formats an ISO timestamp for ClickHouse `DateTime64` comparisons. */
+function formatClickHouseDateTime(value) {
+    return value.replace('T', ' ').replace(/Z$/, '');
+}
 /**
  * Collates per-side transfer streams into one ordered page window: split and
  * sampled streams are deduped, globally ordered, and paged app-side, while a
- * lone engine-paged stream passes through. `capped` marks a fallback sample
- * that hit `Schema.countCap`: an arbitrary subset of that side's matches,
- * unfit to anchor pagination or exact counts.
+ * lone engine-paged stream passes through. Ordered capped streams may carry a
+ * safe raw continuation; unordered samples cannot.
  */
 export function collate(fetched, options) {
-    const capped = fetched.some(({ rows, sample }) => sample && rows.length >= Schema.countCap);
+    const capped = fetched.some((stream) => stream.capped);
+    const cappedStreams = fetched.filter((stream) => stream.capped);
+    const resumes = cappedStreams.flatMap((stream) => (stream.resume ? [stream.resume] : []));
+    if (fetched.length > 1 &&
+        capped &&
+        !fetched.some((stream) => stream.sample) &&
+        resumes.length === cappedStreams.length) {
+        // Advance only through the least-progressed side, so the next strict
+        // keyset scan cannot rediscover any returned rows.
+        const position = resumes
+            .map((resume) => resume.position)
+            .reduce((candidate, current) => {
+            const comparison = candidate[0] - current[0] || candidate[1] - current[1];
+            if (options.order === 'asc')
+                return comparison <= 0 ? candidate : current;
+            return comparison >= 0 ? candidate : current;
+        });
+        const safeRows = fetched.flatMap((stream) => stream.rows.filter((row) => {
+            const blockNumber = Value.toNumber(row['block_num']);
+            const logIndex = Value.toNumber(row['log_idx']);
+            if (blockNumber === undefined || logIndex === undefined)
+                return false;
+            const comparison = blockNumber - position[0] || logIndex - position[1];
+            return options.order === 'asc' ? comparison <= 0 : comparison >= 0;
+        }));
+        const merged = mergeTransferRows(safeRows, {
+            limit: options.offset + options.limit,
+            offset: 0,
+            order: options.order,
+        });
+        const rows = merged.slice(options.offset, options.offset + options.limit);
+        if (rows.length >= options.limit)
+            return { capped: false, rows };
+        return {
+            capped: true,
+            resume: {
+                offset: Math.max(0, options.offset - merged.length),
+                position,
+            },
+            rows,
+        };
+    }
+    const resume = fetched.length === 1 ? fetched[0]?.resume : undefined;
     const rows = fetched.length > 1 || fetched.some(({ sample }) => sample)
         ? mergeTransferRows(fetched.flatMap(({ rows }) => rows), options)
         : (fetched[0]?.rows ?? []);
-    return { capped, rows };
+    return { capped, ...(resume ? { resume } : {}), rows };
 }
 /** Deduplicates and globally orders bounded `from`/`to` transfer streams. */
 function mergeTransferRows(rows, options) {
@@ -643,12 +963,25 @@ function mergeTransferRows(rows, options) {
     });
     return positioned.slice(options.offset, options.offset + options.limit).map(({ row }) => row);
 }
+/** Decodes ordinary transfer cursors and capped positional continuations. */
+function decodeTransferCursor(token) {
+    if (token === undefined)
+        return undefined;
+    const position = Cursor.decode(token, ['int', 'int']);
+    if (position)
+        return { offset: 0, position: position };
+    const continuation = Cursor.decode(token, ['int', 'int', 'int']);
+    if (!continuation)
+        return undefined;
+    const [blockNumber, logIndex, offset] = continuation;
+    return { offset: Math.max(0, offset), position: [blockNumber, logIndex] };
+}
 /**
  * Builds shared transfer filters. Token-scoped queries target `token_transfers`;
  * unscoped queries constrain the decoded `Transfer` emitter to TIP-20 contracts.
  */
 function filterClauses(options) {
-    const { fromBlock, fromTimestamp, recipient, sender, toBlock, toTimestamp, token } = options;
+    const { fromBlock, fromTimestamp, recipient, sender, toBlock, token, toTimestamp } = options;
     const filters = token === undefined
         ? [`address >= '${tip20AddressRange[0]}'`, `address < '${tip20AddressRange[1]}'`]
         : [`token = '${token}'`];
@@ -666,6 +999,13 @@ function filterClauses(options) {
         filters.push(`block_timestamp <= '${toTimestamp}'`);
     return filters;
 }
+function isTip20TransferRow(row) {
+    const address = row['address'];
+    if (typeof address !== 'string')
+        return false;
+    const normalized = address.toLowerCase();
+    return normalized >= tip20AddressRange[0] && normalized < tip20AddressRange[1];
+}
 /**
  * Capped total-row count for the `Transfer` feed, sharing {@link query}'s
  * filters (minus pagination) so the count matches the page it annotates. Timed
@@ -698,14 +1038,181 @@ export async function countScan(deps, options) {
     // rows, classify each transaction, and subtract the folded-away legs. Bounded
     // by `countCap` like the raw count, so it stays within the indexer's budget.
     if (options.crossToken) {
+        const global = !tokenScoped &&
+            address === undefined &&
+            options.sender === undefined &&
+            options.recipient === undefined;
+        if (global) {
+            // A capped count promises a conservative lower bound, so aggregate the
+            // deduplicated capped sample without the global block sort or 10,000-row response.
+            const materializedFilters = [
+                `token >= '${tip20AddressRange[0]}'`,
+                `token < '${tip20AddressRange[1]}'`,
+                ...(options.fromBlock !== undefined ? [`block_num >= ${options.fromBlock}`] : []),
+                ...(options.toBlock !== undefined ? [`block_num <= ${options.toBlock}`] : []),
+                ...(options.fromTimestamp !== undefined
+                    ? [`block_timestamp >= '${options.fromTimestamp}'`]
+                    : []),
+                ...(options.toTimestamp !== undefined
+                    ? [`block_timestamp <= '${options.toTimestamp}'`]
+                    : []),
+            ];
+            const result = await tidx.fetch({
+                chainId,
+                engine: 'clickhouse',
+                query: `
+          SELECT
+            count(*) AS total,
+            countIf(
+              "to" = '${Addresses.stablecoinDex}'
+              OR "from" = '${Addresses.stablecoinDex}'
+              OR "to" = '${Addresses.feeManager}'
+          ) AS suppressed
+          FROM (
+            SELECT tx_hash, log_idx, "from", "to"
+            FROM token_transfers FINAL
+            WHERE ${materializedFilters.join(' AND ')}
+            LIMIT ${Schema.countCap}
+          ) AS capped
+        `,
+            });
+            const total = Value.toNumber(result.rows[0]?.['total']) ?? 0;
+            const suppressed = Value.toNumber(result.rows[0]?.['suppressed']) ?? 0;
+            if (total >= Schema.countCap) {
+                return {
+                    totalCount: Math.max(0, total - suppressed),
+                    totalCountCapped: true,
+                };
+            }
+            if (suppressed === 0)
+                return { totalCount: total, totalCountCapped: false };
+        }
+        if (address !== undefined) {
+            const materializedFilters = [
+                ...(options.token === undefined
+                    ? [`token >= '${tip20AddressRange[0]}'`, `token < '${tip20AddressRange[1]}'`]
+                    : [`token = '${options.token}'`]),
+                ...(options.sender !== undefined ? [`"from" = '${options.sender}'`] : []),
+                ...(options.recipient !== undefined ? [`"to" = '${options.recipient}'`] : []),
+                ...(options.fromBlock !== undefined ? [`block_num >= ${options.fromBlock}`] : []),
+                ...(options.toBlock !== undefined ? [`block_num <= ${options.toBlock}`] : []),
+                ...(options.fromTimestamp !== undefined
+                    ? [`block_timestamp >= '${options.fromTimestamp}'`]
+                    : []),
+                ...(options.toTimestamp !== undefined
+                    ? [`block_timestamp <= '${options.toTimestamp}'`]
+                    : []),
+            ];
+            const sentFilters = [`"from" = '${address}'`, ...materializedFilters];
+            const receivedFilters = [
+                `"to" = '${address}'`,
+                `"from" != '${address}'`,
+                ...materializedFilters,
+            ];
+            const aggregate = await tidx.fetch({
+                chainId,
+                engine: 'clickhouse',
+                query: `
+          SELECT
+            'sent' AS side,
+            count(*) AS total,
+            countIf(
+              "to" = '${Addresses.stablecoinDex}'
+              OR "from" = '${Addresses.stablecoinDex}'
+              OR "to" = '${Addresses.feeManager}'
+            ) AS suppressed
+          FROM (
+            SELECT "from", "to"
+            FROM token_transfers
+            WHERE ${sentFilters.join(' AND ')}
+            LIMIT ${Schema.countCap}
+          ) AS sent
+          UNION ALL
+          SELECT
+            'received' AS side,
+            count(*) AS total,
+            countIf(
+              "to" = '${Addresses.stablecoinDex}'
+              OR "from" = '${Addresses.stablecoinDex}'
+              OR "to" = '${Addresses.feeManager}'
+            ) AS suppressed
+          FROM (
+            SELECT "from", "to"
+            FROM token_transfers
+            WHERE ${receivedFilters.join(' AND ')}
+            LIMIT ${Schema.countCap}
+          ) AS received
+        `,
+            });
+            const sent = aggregate.rows.find((row) => Value.toText(row['side']) === 'sent');
+            const received = aggregate.rows.find((row) => Value.toText(row['side']) === 'received');
+            const sentTotal = Value.toNumber(sent?.['total']) ?? 0;
+            const receivedTotal = Value.toNumber(received?.['total']) ?? 0;
+            const total = sentTotal + receivedTotal;
+            const suppressed = (Value.toNumber(sent?.['suppressed']) ?? 0) +
+                (Value.toNumber(received?.['suppressed']) ?? 0);
+            const capped = sentTotal >= Schema.countCap || receivedTotal >= Schema.countCap || total >= Schema.countCap;
+            if (capped)
+                return {
+                    totalCount: Math.min(Math.max(0, total - suppressed), Schema.countCap),
+                    totalCountCapped: true,
+                };
+            if (suppressed === 0)
+                return { totalCount: total, totalCountCapped: false };
+            // A below-cap aggregate proves these unsorted scans are complete, so the
+            // exact bundle classifier does not need a global sort.
+            const select = 'tx_hash, block_num, log_idx, "from", "to", token AS address, amount AS value';
+            const fetchRows = async (sideFilters) => {
+                const result = await tidx.fetch({
+                    chainId,
+                    engine: 'clickhouse',
+                    query: `
+            SELECT ${select}
+            FROM token_transfers
+            WHERE ${sideFilters.join(' AND ')}
+            LIMIT ${Schema.countCap}
+          `,
+                });
+                return result.rows;
+            };
+            const [sentRows, receivedRows] = await Promise.all([
+                fetchRows(sentFilters),
+                fetchRows(receivedFilters),
+            ]);
+            const rows = [...sentRows, ...receivedRows];
+            const transactionHashes = [...legsByTransaction(rows)]
+                .filter(([, legs]) => legs.some(isBundleLeg))
+                .map(([transactionHash]) => transactionHash);
+            const transactionHashSet = new Set(transactionHashes);
+            const candidateRows = rows.filter((row) => {
+                const transactionHash = Value.toText(row['tx_hash']);
+                return (transactionHash !== undefined && transactionHashSet.has(transactionHash.toLowerCase()));
+            });
+            const blocks = candidateRows.flatMap((row) => {
+                const block = Value.toNumber(row['block_num']);
+                return block === undefined ? [] : [block];
+            });
+            if (blocks.length === 0 || transactionHashes.length === 0)
+                return foldCount(rows, { capped: false });
+            // Address predicates omit sibling bundle legs. Load each candidate
+            // transaction completely, then suppress only address-matched rows.
+            const classification = await tidx.fetch({
+                chainId,
+                engine: 'clickhouse',
+                query: `
+          SELECT tx_hash, block_num, log_idx, "from", "to", token AS address, amount AS value
+          FROM token_transfers
+          WHERE block_num BETWEEN ${Math.min(...blocks)} AND ${Math.max(...blocks)}
+            AND tx_hash IN (${transactionHashes.map((hash) => `'${hash}'`).join(', ')})
+        `,
+            });
+            return foldCount(rows, { capped: false, classificationRows: classification.rows });
+        }
         // Only one-sided `"from"`/`"to"` equality shapes hit the planner's
         // deterministic top-N kill, so only they take the capped fallback.
-        const sideFiltered = address !== undefined || options.sender !== undefined || options.recipient !== undefined;
-        const fetchRows = async (side) => {
-            const sideFilters = side !== undefined && address !== undefined
-                ? [`"${side}" = '${address}'`, ...filters]
-                : filters;
-            const sideWhere = sideFilters.length > 0 ? `WHERE ${sideFilters.join(' AND ')}` : '';
+        const sideFiltered = options.sender !== undefined || options.recipient !== undefined;
+        const fetchRows = async () => {
+            const where = filters.length > 0 ? `WHERE ${filters.join(' AND ')}` : '';
             const select = `tx_hash, block_num, log_idx, "from", "to", ${tokenScoped ? 'token AS address, amount AS value' : 'address, value'}`;
             const table = tokenScoped ? 'token_transfers' : 'Transfer';
             try {
@@ -715,12 +1222,12 @@ export async function countScan(deps, options) {
                     query: `
             SELECT ${select}
             FROM ${table}
-            ${sideWhere}
+            ${where}
             ORDER BY block_num, log_idx
             LIMIT ${Schema.countCap}
           `,
                 });
-                return { rows: result.rows, sample: false };
+                return { capped: false, rows: result.rows, sample: false };
             }
             catch (error) {
                 // Same planner kill as the page's side scan; the folded count is
@@ -733,16 +1240,14 @@ export async function countScan(deps, options) {
                     query: `
             SELECT ${select}
             FROM ${table}
-            ${sideWhere}
+            ${where}
             LIMIT ${Schema.countCap}
           `,
                 });
-                return { rows: result.rows, sample: true };
+                return { capped: result.rows.length >= Schema.countCap, rows: result.rows, sample: true };
             }
         };
-        const fetched = address !== undefined
-            ? await Promise.all([fetchRows('from'), fetchRows('to')])
-            : [await fetchRows()];
+        const fetched = [await fetchRows()];
         const { capped, rows } = collate(fetched, { limit: Schema.countCap, offset: 0, order: 'asc' });
         return foldCount(rows, { capped });
     }
@@ -786,16 +1291,23 @@ export async function countScan(deps, options) {
  * fee leg instead, keeping the reported lower bound from overcounting.
  */
 export function foldCount(rows, options) {
-    const { capped } = options;
+    const { capped, classificationRows = rows } = options;
+    const rowKeys = new Set();
+    for (const row of rows) {
+        const transactionHash = Value.toText(row['tx_hash']);
+        const logIndex = Value.toNumber(row['log_idx']);
+        if (transactionHash && logIndex !== undefined)
+            rowKeys.add(eventKey(transactionHash, logIndex));
+    }
     let suppressed = 0;
-    for (const legs of legsByTransaction(rows).values()) {
+    for (const [transactionHash, legs] of legsByTransaction(capped ? rows : classificationRows)) {
         if (capped) {
             suppressed += legs.filter(isBundleLeg).length;
             continue;
         }
         const bundle = classifyCrossToken(legs);
         if (bundle)
-            suppressed += bundle.suppressLogIndexes.length;
+            suppressed += bundle.suppressLogIndexes.filter((logIndex) => rowKeys.has(eventKey(transactionHash, logIndex))).length;
     }
     return {
         totalCount: rows.length - suppressed,
@@ -876,13 +1388,14 @@ function isBundleLeg(leg) {
  * transfer and is left untouched.
  */
 function classifyCrossToken(legs) {
-    const toDex = legs.find((leg) => Address.isEqual(leg.to, Addresses.stablecoinDex));
-    const fromDex = legs.find((leg) => Address.isEqual(leg.from, Addresses.stablecoinDex));
+    const ordered = [...legs].sort((a, b) => a.logIndex - b.logIndex);
+    const toDex = ordered.find((leg) => Address.isEqual(leg.to, Addresses.stablecoinDex));
+    const fromDex = ordered.find((leg) => Address.isEqual(leg.from, Addresses.stablecoinDex));
     if (!toDex || !fromDex)
         return undefined;
     const swapper = fromDex.to;
     const destTokenAddress = fromDex.address;
-    const delivery = legs.find((leg) => Address.isEqual(leg.address, destTokenAddress) &&
+    const delivery = ordered.find((leg) => Address.isEqual(leg.address, destTokenAddress) &&
         Address.isEqual(leg.from, swapper) &&
         !Address.isEqual(leg.to, Addresses.stablecoinDex) &&
         !Address.isEqual(leg.to, swapper) &&
@@ -890,7 +1403,7 @@ function classifyCrossToken(legs) {
     // Require a token change so plain same-token DEX activity is not folded.
     if (!delivery || Address.isEqual(toDex.address, delivery.address))
         return undefined;
-    const suppressLogIndexes = legs
+    const suppressLogIndexes = ordered
         .filter((leg) => leg.logIndex !== delivery.logIndex && isBundleLeg(leg))
         .map((leg) => leg.logIndex);
     return {