tapimo 0.7.0 → 0.7.2

package/dist/apps/data/routes/webhooks.js

@@ -11,6 +11,7 @@ import * as Schema from '../../../internal/Schema.js';
 import * as Store from '../../../internal/Store.js';
 import * as Ttl from '../../../internal/Ttl.js';
 import * as Value from '../../../internal/Value.js';
+import * as WebhookDestination from '../../../internal/WebhookDestination.js';
 import * as Webhooks from '../../../internal/Webhooks.js';
 import * as Receipts from './receipts.js';
 import * as Transactions from './transactions.js';
@@ -852,6 +853,12 @@ export function webhooks(options = {}) {
         if (!supportedChainIds.has(chainId))
             return Response.unsupportedChainId(c, chainId, supportedChainIds);
         try {
+            // Preserve destination errors before the creation-time RPC dependency.
+            WebhookDestination.assertDestination(body.destination);
+            const maxPerOwner = webhook.maxPerOwner ?? defaultMaxPerOwner;
+            if ((await Webhooks.countSubscriptions(db, owner)) >= maxPerOwner)
+                throw new Webhooks.LimitExceededError(maxPerOwner);
+            const startBlockNumber = await getRpcHead(c.get('getClient')(chainId));
             const subscription = await Webhooks.createSubscription(db, {
                 chainId,
                 ...(body.context === undefined ? {} : { context: body.context }),
@@ -860,7 +867,10 @@ export function webhooks(options = {}) {
                 filters: body.filters,
                 owner,
                 ...(owner.type === 'mpp' ? { ttl: mppTtlMs } : {}),
-            }, { maxPerOwner: webhook.maxPerOwner ?? defaultMaxPerOwner });
+            }, {
+                maxPerOwner,
+                startBlockNumber,
+            });
             return c.json(Response.validated(schema.createWebhook.Response, toCreated(subscription)), 200);
         }
         catch (cause) {
@@ -1306,6 +1316,21 @@ export function webhooks(options = {}) {
  * can equal it).
  */
 const endOfBlockIndex = 2_147_483_647;
+const scanPageSize = 100;
+/** Returns `candidate` only when it is lexicographically after `current`. */
+function laterCursor(current, candidate) {
+    const before = Cursor.decode(current, ['int', 'int']);
+    const after = Cursor.decode(candidate, ['int', 'int']);
+    if (!after)
+        return current;
+    if (!before)
+        return candidate;
+    const [beforeBlock, beforeIndex] = before;
+    const [afterBlock, afterIndex] = after;
+    return afterBlock > beforeBlock || (afterBlock === beforeBlock && afterIndex > beforeIndex)
+        ? candidate
+        : current;
+}
 /**
  * Polls the chain over RPC for new events and dispatches a signed envelope per
  * matching row, advancing each active subscription's `(block, index)` keyset
@@ -1326,9 +1351,11 @@ const endOfBlockIndex = 2_147_483_647;
  *
  * Cursor advances only past successfully dispatched rows, so a dispatch failure
  * pauses that subscription at the last good position and the next tick resumes.
+ * Scan failures leave that subscription unchanged; the first error is rethrown
+ * after the remaining subscriptions run.
  */
 export async function poll(options) {
-    const { batchSize = 100, cache: cache_metadata = Store.memory(), chainId, client, confirmations = 2, db, eventTypes = ['token:transfer', 'transaction:included', 'log:emitted', 'block:created'], maxBlocks = 1_000, tidx, } = options;
+    const { batchSize = 200, cache: cache_metadata = Store.memory(), chainId, client, confirmations = 2, db, eventTypes = ['token:transfer', 'transaction:included', 'log:emitted', 'block:created'], maxBlocks = 1_000, tidx, } = options;
     const now = options.now ?? (() => new Date());
     // Default (self-host) transport: deliver inline and record the outcome so a
     // sustained-failure subscription auto-disables. On Cloudflare, callers pass a
@@ -1358,74 +1385,134 @@ export async function poll(options) {
         scanned: 0,
         subscriptions: 0,
     };
+    const scanErrors = [];
     for (const eventType of eventTypes) {
         const legacy = eventType === 'token:transfer';
         const safeBlock = legacy ? Math.max(0, (await resolveTidxHead()) - confirmations) : rpcSafeBlock;
         const subscriptions = await Webhooks.listActive(db, chainId, eventType);
+        let states = [];
         for (const subscription of subscriptions) {
             result.subscriptions++;
-            const cursor = await Webhooks.getCursor(db, subscription.id);
+            let cursor = await Webhooks.getCursor(db, subscription.id);
             if (cursor === null) {
-                // First time we've seen this subscription: anchor it at the head of the
-                // confirmed window so we never backfill from genesis. Events start
-                // flowing from the next tick onward.
-                await Webhooks.setCursor(db, subscription.id, Cursor.encode([safeBlock, endOfBlockIndex]));
+                // Legacy subscriptions without a creation checkpoint anchor at the
+                // current confirmed head, avoiding a genesis backfill.
+                await Webhooks.compareAndSetCursor(db, {
+                    cursor: Cursor.encode([safeBlock, endOfBlockIndex]),
+                    expectedCursor: null,
+                    subscriptionId: subscription.id,
+                });
                 continue;
             }
-            const page = await scanSubscription({
-                batchSize,
-                cache,
-                client,
+            const checkpoint = Cursor.decode(cursor, ['int']);
+            if (checkpoint) {
+                // Creation checkpoints record the live head without finalizing it. Rebase
+                // to this poller's confirmed boundary so replacement blocks stay eligible.
+                const confirmedCursor = Cursor.encode([
+                    Math.max(0, checkpoint[0] - confirmations),
+                    endOfBlockIndex,
+                ]);
+                const committed = await Webhooks.compareAndSetCursor(db, {
+                    cursor: confirmedCursor,
+                    expectedCursor: cursor,
+                    subscriptionId: subscription.id,
+                });
+                if (!committed)
+                    continue;
+                cursor = confirmedCursor;
+            }
+            const window = legacy ? undefined : scanWindow(cursor, safeBlock, maxBlocks);
+            states.push({
                 cursor,
-                maxBlocks,
-                safeBlock,
+                remaining: batchSize,
+                safeBlock: window?.to ?? safeBlock,
                 subscription,
-                tidx,
             });
-            result.scanned += page.items.length;
-            let accepted;
-            let failed = false;
-            for (const item of page.items) {
-                // Token metadata is best-effort for webhooks: transfers use it for display,
-                // while transactions expose fee-token references without blocking delivery.
-                const data = eventType === 'token:transfer'
-                    ? await withTokenMetadata(item.data, { cache: cache_metadata, chainId, client })
-                    : eventType === 'transaction:included'
-                        ? await withFeeTokenMetadata(item.data, { cache: cache_metadata, chainId, client })
-                        : item.data;
-                const envelope = Webhooks.buildEnvelope({
-                    blockNumber: item.cursor[0],
-                    createdAt: now(),
-                    data,
-                    logIndex: item.cursor[1],
-                    subscription,
-                });
-                const delivered = await dispatch({ envelope, subscription });
-                if (!delivered.ok) {
-                    result.failed++;
-                    failed = true;
-                    break;
+        }
+        // Every subscription gets one bounded page before any subscription gets a
+        // continuation page, preserving fairness while allowing catch-up.
+        while (states.length > 0) {
+            const continuations = [];
+            for (const state of states) {
+                const { subscription } = state;
+                let page;
+                try {
+                    page = await scanSubscription({
+                        batchSize: Math.min(scanPageSize, state.remaining),
+                        cache,
+                        client,
+                        cursor: state.cursor,
+                        maxBlocks,
+                        safeBlock: state.safeBlock,
+                        subscription,
+                        tidx,
+                    });
                 }
-                accepted = item.cursor;
-                result.dispatched++;
-            }
-            // Advance the high-water mark. On a dispatch failure, stop at the last
-            // accepted row so the failed one retries next tick. Otherwise: window
-            // scanners jump to the end of the fully-scanned range (even with zero
-            // matches); keyset scanners advance past the last dispatched row, or — on
-            // a complete empty page — to the end of the confirmed window.
-            if (failed) {
-                if (accepted)
-                    await Webhooks.setCursor(db, subscription.id, Cursor.encode([...accepted]));
+                catch (error) {
+                    scanErrors.push(error);
+                    continue;
+                }
+                result.scanned += page.items.length;
+                state.remaining -= page.items.length;
+                let accepted;
+                let failed = false;
+                for (const item of page.items) {
+                    // Token metadata is best-effort for webhooks: transfers use it for display,
+                    // while transactions expose fee-token references without blocking delivery.
+                    const data = eventType === 'token:transfer'
+                        ? await withTokenMetadata(item.data, { cache: cache_metadata, chainId, client })
+                        : eventType === 'transaction:included'
+                            ? await withFeeTokenMetadata(item.data, { cache: cache_metadata, chainId, client })
+                            : item.data;
+                    const envelope = Webhooks.buildEnvelope({
+                        blockNumber: item.cursor[0],
+                        createdAt: now(),
+                        data,
+                        logIndex: item.cursor[1],
+                        subscription,
+                    });
+                    const delivered = await dispatch({ envelope, subscription });
+                    if (!delivered.ok) {
+                        result.failed++;
+                        failed = true;
+                        break;
+                    }
+                    accepted = item.cursor;
+                    result.dispatched++;
+                }
+                // Persist every accepted page before another subscription begins its
+                // continuation, so a later scan failure retains completed work.
+                const candidate = (() => {
+                    if (failed)
+                        return accepted ? Cursor.encode([...accepted]) : undefined;
+                    if (page.scannedThrough !== undefined)
+                        return Cursor.encode([page.scannedThrough, endOfBlockIndex]);
+                    if (accepted)
+                        return Cursor.encode([...accepted]);
+                    if (!page.hasMore)
+                        return Cursor.encode([state.safeBlock, endOfBlockIndex]);
+                    return undefined;
+                })();
+                const cursor = candidate ? laterCursor(state.cursor, candidate) : state.cursor;
+                const progressed = cursor !== state.cursor;
+                let committed = false;
+                if (progressed) {
+                    committed = await Webhooks.compareAndSetCursor(db, {
+                        cursor,
+                        expectedCursor: state.cursor,
+                        subscriptionId: subscription.id,
+                    });
+                    if (committed)
+                        state.cursor = cursor;
+                }
+                if (!failed && page.hasMore === true && committed && state.remaining > 0)
+                    continuations.push(state);
             }
-            else if (page.scannedThrough !== undefined)
-                await Webhooks.setCursor(db, subscription.id, Cursor.encode([page.scannedThrough, endOfBlockIndex]));
-            else if (accepted)
-                await Webhooks.setCursor(db, subscription.id, Cursor.encode([...accepted]));
-            else if (!page.hasMore)
-                await Webhooks.setCursor(db, subscription.id, Cursor.encode([safeBlock, endOfBlockIndex]));
+            states = continuations;
         }
     }
+    if (scanErrors.length > 0)
+        throw scanErrors[0];
     return result;
 }
 /**
@@ -1557,6 +1644,7 @@ async function scanSubscription(options) {
     const chainId = subscription.chainId;
     if (subscription.eventType === 'log:emitted')
         return scanLogs({
+            batchSize,
             cache,
             client,
             cursor,
@@ -1566,6 +1654,7 @@ async function scanSubscription(options) {
         });
     if (subscription.eventType === 'block:created')
         return scanBlocks({
+            batchSize,
             cache,
             cursor,
             filters: parseFilters('block:created', subscription.filters),
@@ -1574,6 +1663,7 @@ async function scanSubscription(options) {
         });
     if (subscription.eventType === 'transaction:included')
         return scanTxs({
+            batchSize,
             cache,
             cursor,
             filters: parseFilters('transaction:included', subscription.filters),
@@ -1585,6 +1675,10 @@ async function scanSubscription(options) {
         return { hasMore: false, items: [] };
     const filters = transferFilters(subscription.filters);
     const page = await Transfers.scan({ tidx }, { chainId, cursor, limit: batchSize, order: 'asc', toBlock: safeBlock, ...filters });
+    // A cap-truncated fallback sample is an arbitrary subset: dispatching it
+    // would advance the cursor past unseen matches. Hold position and retry.
+    if (page.capped)
+        return { hasMore: true, items: [] };
     return { hasMore: page.hasMore, items: page.items };
 }
 const filterSchemas = {
@@ -1762,7 +1856,11 @@ function createChainCache(client) {
                 p = rpc(client, 'eth_getBlockByNumber', [
                     blockTag(n),
                     true,
-                ]).then((b) => b ?? undefined);
+                ]).then((block) => {
+                    if (!block)
+                        throw new BlockUnavailableError(n);
+                    return block;
+                });
                 blocks.set(n, p);
             }
             return p;
@@ -1770,9 +1868,7 @@ function createChainCache(client) {
         header(n) {
             let p = headers.get(n);
             if (!p) {
-                p = client
-                    .getBlock({ blockNumber: BigInt(n), includeTransactions: false })
-                    .catch(() => undefined);
+                p = client.getBlock({ blockNumber: BigInt(n), includeTransactions: false });
                 headers.set(n, p);
             }
             return p;
@@ -1956,7 +2052,7 @@ function decodeScanLog(log, event) {
  * all operators are re-checked in JS over the narrow window.
  */
 async function scanLogs(options) {
-    const { cache, client, cursor, filters, maxBlocks, safeBlock } = options;
+    const { batchSize, cache, client, cursor, filters, maxBlocks, safeBlock } = options;
     const f = filters;
     const window = scanWindow(cursor, safeBlock, maxBlocks);
     if (!window)
@@ -2011,8 +2107,11 @@ async function scanLogs(options) {
         const header = await cache.header(blockNumber);
         const timestamp = blockIso(header);
         const data = toLogEvent(log, timestamp, f.signature, event);
-        if (data)
+        if (data) {
             items.push({ cursor: [data.blockNumber, data.logIndex], data });
+            if (items.length >= batchSize)
+                return { hasMore: true, items };
+        }
     }
     return { items, scannedThrough: window.to };
 }
@@ -2096,7 +2195,7 @@ function toBlockEvent(block) {
  * empty filter is a per-block heartbeat.
  */
 async function scanBlocks(options) {
-    const { cache, cursor, filters, maxBlocks, safeBlock } = options;
+    const { batchSize, cache, cursor, filters, maxBlocks, safeBlock } = options;
     const f = filters;
     const window = scanWindow(cursor, safeBlock, maxBlocks);
     if (!window)
@@ -2111,8 +2210,11 @@ async function scanBlocks(options) {
         if (!matchesBlock(block, f))
             continue;
         const data = toBlockEvent(block);
-        if (data)
+        if (data) {
             items.push({ cursor: [data.number, 0], data });
+            if (items.length >= batchSize)
+                return { hasMore: true, items };
+        }
     }
     return { items, scannedThrough: window.to };
 }
@@ -2159,7 +2261,7 @@ const txFields = {
  * poller then resolves its fee-token references.
  */
 async function scanTxs(options) {
-    const { cache, cursor, filters, maxBlocks, safeBlock } = options;
+    const { batchSize, cache, cursor, filters, maxBlocks, safeBlock } = options;
     const f = filters;
     const window = scanWindow(cursor, safeBlock, maxBlocks);
     if (!window)
@@ -2172,11 +2274,11 @@ async function scanTxs(options) {
     const items = [];
     for (let n = window.from; n <= window.to; n++) {
         const block = await cache.block(n);
-        const txs = Array.isArray(block?.['transactions']) ? block['transactions'] : [];
+        const txs = Array.isArray(block['transactions']) ? block['transactions'] : [];
         if (txs.length === 0)
             continue;
         const receipts = needsReceipt || includeReceipt ? await cache.receipts(n) : undefined;
-        const timestamp = asString(block?.['timestamp']);
+        const timestamp = asString(block['timestamp']);
         for (let i = 0; i < txs.length; i++) {
             const tx = txs[i];
             const index = Value.hexToNumber(tx['transactionIndex']) ?? i;
@@ -2205,6 +2307,8 @@ async function scanTxs(options) {
                     };
             }
             items.push({ cursor: [n, index], data });
+            if (items.length >= batchSize)
+                return { hasMore: true, items };
         }
     }
     return { items, scannedThrough: window.to };
@@ -2362,4 +2466,10 @@ function unauthorized(c) {
         status: 401,
     });
 }
+class BlockUnavailableError extends Error {
+    name = 'Webhooks.BlockUnavailableError';
+    constructor(blockNumber) {
+        super(`Block ${blockNumber} is unavailable.`);
+    }
+}
 //# sourceMappingURL=webhooks.js.map