ponder 0.14.13 → 0.15.0

package/dist/esm/runtime/historical.js

@@ -1,15 +1,18 @@
 import { ShutdownError } from '../internal/errors.js';
 import { _eth_getBlockByNumber } from '../rpc/actions.js';
 import { buildEvents, decodeEvents } from './events.js';
-import { isAddressFactory } from './filter.js';
 import { createHistoricalSync } from '../sync-historical/index.js';
+import { createSyncStore } from '../sync-store/index.js';
 import { MAX_CHECKPOINT, ZERO_CHECKPOINT, decodeCheckpoint, encodeCheckpoint, min, } from '../utils/checkpoint.js';
+import { estimate } from '../utils/estimate.js';
 import { formatPercentage } from '../utils/format.js';
-import { bufferAsyncGenerator, mergeAsyncGenerators, } from '../utils/generators.js';
-import { intervalDifference, intervalIntersectionMany, intervalSum, intervalUnion, } from '../utils/interval.js';
+import { bufferAsyncGenerator, createCallbackGenerator, mergeAsyncGenerators, } from '../utils/generators.js';
+import { intervalSum } from '../utils/interval.js';
 import { partition } from '../utils/partition.js';
+import { promiseWithResolvers } from '../utils/promiseWithResolvers.js';
 import { startClock } from '../utils/timer.js';
 import { hexToNumber } from "viem";
+import { getRequiredIntervals, getRequiredIntervalsWithFilters, } from "./index.js";
 import { initEventGenerator, initRefetchEvents } from "./init.js";
 import { getOmnichainCheckpoint } from "./omnichain.js";
 export async function* getHistoricalEventsOmnichain(params) {
@@ -19,7 +22,7 @@ export async function* getHistoricalEventsOmnichain(params) {
     while (true) {
         const eventGenerators = Array.from(params.perChainSync.entries()).map(async function* ([chain, { syncProgress, childAddresses, cachedIntervals },]) {
             const rpc = params.indexingBuild.rpcs[params.indexingBuild.chains.findIndex((c) => c.id === chain.id)];
-            const sources = params.indexingBuild.sources.filter(({ filter }) => filter.chainId === chain.id);
+            const eventCallbacks = params.indexingBuild.eventCallbacks[params.indexingBuild.chains.findIndex((c) => c.id === chain.id)];
             const crashRecoveryCheckpoint = params.crashRecoveryCheckpoint?.find(({ chainId }) => chainId === chain.id)?.checkpoint;
             const to = min(syncProgress.getCheckpoint({ tag: "finalized" }), syncProgress.getCheckpoint({ tag: "end" }));
             const omnichainTo = min(getOmnichainCheckpoint({
@@ -42,7 +45,10 @@ export async function* getHistoricalEventsOmnichain(params) {
                     from = crashRecoveryCheckpoint;
                 }
                 else {
-                    const fromBlock = await params.syncStore.getSafeCrashRecoveryBlock({
+                    const fromBlock = await createSyncStore({
+                        common: params.common,
+                        qb: params.database.syncQB,
+                    }).getSafeCrashRecoveryBlock({
                         chainId: chain.id,
                         timestamp: Number(decodeCheckpoint(crashRecoveryCheckpoint).blockTimestamp),
                     });
@@ -63,8 +69,8 @@ export async function* getHistoricalEventsOmnichain(params) {
                 const cursor = perChainCursor.get(chain);
                 // Yield pending events from previous iterations. Note that it is possible to
                 // previous pending events to still be pending after the catchup.
-                const events = pendingEvents.filter((event) => event.chainId === chain.id && event.checkpoint <= omnichainTo);
-                pendingEvents = pendingEvents.filter((event) => (event.chainId === chain.id &&
+                const events = pendingEvents.filter((event) => event.chain.id === chain.id && event.checkpoint <= omnichainTo);
+                pendingEvents = pendingEvents.filter((event) => (event.chain.id === chain.id &&
                     event.checkpoint <= omnichainTo) === false);
                 const blockRange = [
                     Number(decodeCheckpoint(cursor).blockNumber),
@@ -97,7 +103,7 @@ export async function* getHistoricalEventsOmnichain(params) {
                 indexingBuild: params.indexingBuild,
                 chain,
                 rpc,
-                sources,
+                eventCallbacks,
                 childAddresses,
                 syncProgress,
                 cachedIntervals,
@@ -105,12 +111,12 @@ export async function* getHistoricalEventsOmnichain(params) {
                 to,
                 limit: Math.round(params.common.options.syncEventsQuerySize /
                     (params.indexingBuild.chains.length + 1)) + 6,
-                syncStore: params.syncStore,
+                database: params.database,
                 isCatchup,
             });
             for await (let { events: rawEvents, checkpoint, blockRange, } of eventGenerator) {
                 const endClock = startClock();
-                let events = decodeEvents(params.common, sources, rawEvents);
+                let events = decodeEvents(params.common, chain, eventCallbacks, rawEvents);
                 params.common.logger.trace({
                     msg: "Decoded events",
                     chain: chain.name,
@@ -218,7 +224,7 @@ export async function* getHistoricalEventsMultichain(params) {
     while (true) {
         const eventGenerators = Array.from(params.perChainSync.entries()).map(async function* ([chain, { syncProgress, childAddresses, cachedIntervals },]) {
             const rpc = params.indexingBuild.rpcs[params.indexingBuild.chains.findIndex((c) => c.id === chain.id)];
-            const sources = params.indexingBuild.sources.filter(({ filter }) => filter.chainId === chain.id);
+            const eventCallbacks = params.indexingBuild.eventCallbacks[params.indexingBuild.chains.findIndex((c) => c.id === chain.id)];
             const crashRecoveryCheckpoint = params.crashRecoveryCheckpoint?.find(({ chainId }) => chainId === chain.id)?.checkpoint;
             const to = min(syncProgress.getCheckpoint({ tag: "finalized" }), syncProgress.getCheckpoint({ tag: "end" }));
             let from;
@@ -234,7 +240,10 @@ export async function* getHistoricalEventsMultichain(params) {
                     from = crashRecoveryCheckpoint;
                 }
                 else {
-                    const fromBlock = await params.syncStore.getSafeCrashRecoveryBlock({
+                    const fromBlock = await createSyncStore({
+                        common: params.common,
+                        qb: params.database.syncQB,
+                    }).getSafeCrashRecoveryBlock({
                         chainId: chain.id,
                         timestamp: Number(decodeCheckpoint(crashRecoveryCheckpoint).blockTimestamp),
                     });
@@ -276,7 +285,7 @@ export async function* getHistoricalEventsMultichain(params) {
                 indexingBuild: params.indexingBuild,
                 chain,
                 rpc,
-                sources,
+                eventCallbacks,
                 childAddresses,
                 syncProgress,
                 cachedIntervals,
@@ -284,12 +293,12 @@ export async function* getHistoricalEventsMultichain(params) {
                 to,
                 limit: Math.round(params.common.options.syncEventsQuerySize /
                     (params.indexingBuild.chains.length + 1)) + 6,
-                syncStore: params.syncStore,
+                database: params.database,
                 isCatchup,
             });
             for await (const { events: rawEvents, checkpoint, blockRange, } of eventGenerator) {
                 const endClock = startClock();
-                let events = decodeEvents(params.common, sources, rawEvents);
+                let events = decodeEvents(params.common, chain, eventCallbacks, rawEvents);
                 params.common.logger.trace({
                     msg: "Decoded events",
                     chain: chain.name,
@@ -367,25 +376,142 @@ export async function* getHistoricalEventsMultichain(params) {
         isCatchup = true;
     }
 }
+export async function* getHistoricalEventsIsolated(params) {
+    let isCatchup = false;
+    let lastUnfinalizedRefetch = Date.now();
+    let cursor;
+    while (true) {
+        const rpc = params.indexingBuild.rpcs[params.indexingBuild.chains.findIndex((c) => c.id === params.chain.id)];
+        const eventCallbacks = params.indexingBuild.eventCallbacks[params.indexingBuild.chains.findIndex((c) => c.id === params.chain.id)];
+        const crashRecoveryCheckpoint = params.crashRecoveryCheckpoint?.find(({ chainId }) => chainId === params.chain.id)?.checkpoint;
+        const to = min(params.syncProgress.getCheckpoint({ tag: "finalized" }), params.syncProgress.getCheckpoint({ tag: "end" }));
+        let from;
+        if (isCatchup === false) {
+            // In order to speed up the "extract" phase when there is a crash recovery,
+            // the beginning cursor is moved forwards. This only works when `crashRecoveryCheckpoint`
+            // is defined.
+            if (crashRecoveryCheckpoint === undefined) {
+                from = params.syncProgress.getCheckpoint({ tag: "start" });
+            }
+            else if (Number(decodeCheckpoint(crashRecoveryCheckpoint).chainId) ===
+                params.chain.id) {
+                from = crashRecoveryCheckpoint;
+            }
+            else {
+                from = params.syncProgress.getCheckpoint({ tag: "start" });
+            }
+        }
+        else {
+            from = encodeCheckpoint({
+                ...ZERO_CHECKPOINT,
+                blockTimestamp: decodeCheckpoint(cursor).blockTimestamp,
+                chainId: decodeCheckpoint(cursor).chainId,
+                blockNumber: decodeCheckpoint(cursor).blockNumber + 1n,
+            });
+            if (from > to)
+                return;
+        }
+        params.common.logger.info({
+            msg: "Started backfill indexing",
+            chain: params.chain.name,
+            chain_id: params.chain.id,
+            block_range: JSON.stringify([
+                Number(decodeCheckpoint(from).blockNumber),
+                Number(decodeCheckpoint(to).blockNumber),
+            ]),
+        });
+        const eventGenerator = await initEventGenerator({
+            common: params.common,
+            indexingBuild: params.indexingBuild,
+            chain: params.chain,
+            rpc,
+            eventCallbacks,
+            childAddresses: params.childAddresses,
+            syncProgress: params.syncProgress,
+            cachedIntervals: params.cachedIntervals,
+            from,
+            to,
+            limit: Math.round(params.common.options.syncEventsQuerySize /
+                (params.indexingBuild.chains.length + 1)) + 6,
+            database: params.database,
+            isCatchup,
+        });
+        for await (const { events: rawEvents, checkpoint, blockRange, } of eventGenerator) {
+            const endClock = startClock();
+            let events = decodeEvents(params.common, params.chain, eventCallbacks, rawEvents);
+            params.common.logger.trace({
+                msg: "Decoded events",
+                chain: params.chain.name,
+                chain_id: params.chain.id,
+                event_count: events.length,
+                duration: endClock(),
+            });
+            params.common.metrics.ponder_historical_extract_duration.inc({ step: "decode" }, endClock());
+            // Removes events that have a checkpoint earlier than (or equal to)
+            // the crash recovery checkpoint.
+            if (crashRecoveryCheckpoint) {
+                const [left, right] = partition(events, (event) => event.checkpoint <= crashRecoveryCheckpoint);
+                events = right;
+                if (left.length > 0) {
+                    params.common.logger.trace({
+                        msg: "Filtered events before crash recovery checkpoint",
+                        chain: params.chain.name,
+                        chain_id: params.chain.id,
+                        event_count: left.length,
+                        checkpoint: crashRecoveryCheckpoint,
+                    });
+                }
+            }
+            yield { chainId: params.chain.id, events, checkpoint, blockRange };
+        }
+        cursor = to;
+        if (Date.now() - lastUnfinalizedRefetch < 30000) {
+            break;
+        }
+        lastUnfinalizedRefetch = Date.now();
+        const context = {
+            logger: params.common.logger.child({ action: "refetch_finalized_block" }),
+        };
+        const endClock = startClock();
+        const finalizedBlock = await _eth_getBlockByNumber(rpc, { blockTag: "latest" }, context).then((latest) => _eth_getBlockByNumber(rpc, {
+            blockNumber: Math.max(hexToNumber(latest.number) - params.chain.finalityBlockCount, 0),
+        }, context));
+        const finalizedBlockNumber = hexToNumber(finalizedBlock.number);
+        params.common.logger.debug({
+            msg: "Refetched finalized block for backfill cutover",
+            chain: params.chain.name,
+            chain_id: params.chain.id,
+            finalized_block: finalizedBlockNumber,
+            duration: endClock(),
+        });
+        if (hexToNumber(finalizedBlock.number) -
+            hexToNumber(params.syncProgress.finalized.number) <=
+            params.chain.finalityBlockCount) {
+            break;
+        }
+        params.syncProgress.finalized = finalizedBlock;
+        isCatchup = true;
+    }
+}
 export async function refetchHistoricalEvents(params) {
     const events = new Array(params.events.length);
-    for (const chain of PONDER_INDEXING_BUILD.chains) {
+    for (const chain of params.indexingBuild.chains) {
         const { childAddresses } = params.perChainSync.get(chain);
         // Note: All filters are refetched, no matter if they are resolved or not.
-        const sources = params.indexingBuild.sources.filter(({ filter }) => filter.chainId === chain.id);
-        const chainEvents = params.events.filter((event) => event.chainId === chain.id);
+        const eventCallbacks = params.indexingBuild.eventCallbacks[params.indexingBuild.chains.findIndex((c) => c.id === chain.id)];
+        const chainEvents = params.events.filter((event) => event.chain.id === chain.id);
         if (chainEvents.length === 0)
             continue;
         const rawEvents = await initRefetchEvents({
             common: params.common,
             chain,
             childAddresses,
-            sources,
+            eventCallbacks,
             events: chainEvents,
             syncStore: params.syncStore,
         });
         const endClock = startClock();
-        const refetchedEvents = decodeEvents(params.common, sources, rawEvents);
+        const refetchedEvents = decodeEvents(params.common, chain, eventCallbacks, rawEvents);
         params.common.logger.trace({
             msg: "Decoded events",
             chain: chain.name,
@@ -397,7 +523,7 @@ export async function refetchHistoricalEvents(params) {
         let i = 0;
         let j = 0;
         while (i < params.events.length && j < refetchedEvents.length) {
-            if (params.events[i]?.chainId === chain.id) {
+            if (params.events[i]?.chain.id === chain.id) {
                 events[i] = refetchedEvents[j];
                 i++;
                 j++;
@@ -419,7 +545,7 @@ export async function refetchLocalEvents(params) {
     while (cursor <= toBlock) {
         const queryEndClock = startClock();
         const { blocks, logs, transactions, transactionReceipts, traces, cursor: queryCursor, } = await params.syncStore.getEventData({
-            filters: params.sources.map(({ filter }) => filter),
+            filters: params.eventCallbacks.map(({ filter }) => filter),
             fromBlock: cursor,
             toBlock,
             chainId: params.chain.id,
@@ -427,7 +553,7 @@ export async function refetchLocalEvents(params) {
         });
         const endClock = startClock();
         const rawEvents = buildEvents({
-            sources: params.sources,
+            eventCallbacks: params.eventCallbacks,
             blocks,
             logs,
             transactions,
@@ -465,6 +591,10 @@ export async function refetchLocalEvents(params) {
     return events;
 }
 export async function* getLocalEventGenerator(params) {
+    const syncStore = createSyncStore({
+        common: params.common,
+        qb: params.database.syncQB,
+    });
     const fromBlock = Number(decodeCheckpoint(params.from).blockNumber);
     const toBlock = Number(decodeCheckpoint(params.to).blockNumber);
     let cursor = fromBlock;
@@ -472,8 +602,8 @@ export async function* getLocalEventGenerator(params) {
     for await (const syncCursor of bufferAsyncGenerator(localSyncGenerator, Number.POSITIVE_INFINITY)) {
         while (cursor <= Math.min(syncCursor, toBlock)) {
             const queryEndClock = startClock();
-            const { blocks, logs, transactions, transactionReceipts, traces, cursor: queryCursor, } = await params.syncStore.getEventData({
-                filters: params.sources.map(({ filter }) => filter),
+            const { blocks, logs, transactions, transactionReceipts, traces, cursor: queryCursor, } = await syncStore.getEventData({
+                filters: params.eventCallbacks.map(({ filter }) => filter),
                 fromBlock: cursor,
                 toBlock: Math.min(syncCursor, toBlock),
                 chainId: params.chain.id,
@@ -481,7 +611,7 @@ export async function* getLocalEventGenerator(params) {
             });
             const endClock = startClock();
             const events = buildEvents({
-                sources: params.sources,
+                eventCallbacks: params.eventCallbacks,
                 blocks,
                 logs,
                 transactions,
@@ -528,7 +658,7 @@ export async function* getLocalEventGenerator(params) {
 export async function* getLocalSyncGenerator(params) {
     const backfillEndClock = startClock();
     const label = { chain: params.chain.name };
-    let cursor = hexToNumber(params.syncProgress.start.number);
+    let first = hexToNumber(params.syncProgress.start.number);
     const last = params.syncProgress.end === undefined
         ? params.syncProgress.finalized
         : hexToNumber(params.syncProgress.end.number) >
@@ -561,41 +691,12 @@ export async function* getLocalSyncGenerator(params) {
         hexToNumber(params.syncProgress.start.number),
         hexToNumber(last.number),
     ];
-    const requiredIntervals = Array.from(params.cachedIntervals.entries()).flatMap(([filter, fragmentIntervals]) => {
-        const filterIntervals = [
-            [
-                filter.fromBlock ?? 0,
-                Math.min(filter.toBlock ?? Number.POSITIVE_INFINITY, totalInterval[1]),
-            ],
-        ];
-        switch (filter.type) {
-            case "log":
-                if (isAddressFactory(filter.address)) {
-                    filterIntervals.push([
-                        filter.address.fromBlock ?? 0,
-                        Math.min(filter.address.toBlock ?? Number.POSITIVE_INFINITY, totalInterval[1]),
-                    ]);
-                }
-                break;
-            case "trace":
-            case "transaction":
-            case "transfer":
-                if (isAddressFactory(filter.fromAddress)) {
-                    filterIntervals.push([
-                        filter.fromAddress.fromBlock ?? 0,
-                        Math.min(filter.fromAddress.toBlock ?? Number.POSITIVE_INFINITY, totalInterval[1]),
-                    ]);
-                }
-                if (isAddressFactory(filter.toAddress)) {
-                    filterIntervals.push([
-                        filter.toAddress.fromBlock ?? 0,
-                        Math.min(filter.toAddress.toBlock ?? Number.POSITIVE_INFINITY, totalInterval[1]),
-                    ]);
-                }
-        }
-        return intervalDifference(intervalUnion(filterIntervals), intervalIntersectionMany(fragmentIntervals.map(({ intervals }) => intervals)));
+    const requiredIntervals = getRequiredIntervals({
+        filters: params.eventCallbacks.map(({ filter }) => filter),
+        interval: totalInterval,
+        cachedIntervals: params.cachedIntervals,
     });
-    const required = intervalSum(intervalUnion(requiredIntervals));
+    const required = intervalSum(requiredIntervals);
     const total = totalInterval[1] - totalInterval[0] + 1;
     params.common.metrics.ponder_historical_total_blocks.set(label, total);
     params.common.metrics.ponder_historical_cached_blocks.set(label, total - required);
@@ -627,7 +728,7 @@ export async function* getLocalSyncGenerator(params) {
                 cache_rate: formatPercentage((total - required) / total),
             });
         }
-        cursor = hexToNumber(params.syncProgress.current.number) + 1;
+        first = hexToNumber(params.syncProgress.current.number) + 1;
     }
     else {
         params.common.logger.info({
@@ -638,88 +739,163 @@ export async function* getLocalSyncGenerator(params) {
         });
     }
     const historicalSync = createHistoricalSync(params);
-    while (true) {
-        // Select a range of blocks to sync bounded by `finalizedBlock`.
-        // It is important for devEx that the interval is not too large, because
-        // time spent syncing ≈ time before indexing function feedback.
-        const interval = [
-            Math.min(cursor, hexToNumber(last.number)),
-            Math.min(cursor + estimateRange, hexToNumber(last.number)),
-        ];
+    const { callback: intervalCallback, generator: intervalGenerator } = createCallbackGenerator();
+    intervalCallback({
+        interval: [
+            first,
+            Math.min(first + estimateRange, hexToNumber(last.number)),
+        ],
+        promise: Promise.resolve(),
+    });
+    /**
+     * @returns `true` if any data was inserted into the database.
+     */
+    async function syncInterval({ interval, promise, }) {
         const endClock = startClock();
-        const durationTimer = setTimeout(() => {
-            params.common.logger.warn({
-                msg: "Fetching backfill JSON-RPC data is taking longer than expected",
-                chain: params.chain.name,
-                chain_id: params.chain.id,
-                block_range: JSON.stringify(interval),
-                duration: endClock(),
-            });
-        }, 5000);
-        let synced;
-        try {
-            synced = await historicalSync.sync(interval);
-        }
-        catch (error) {
-            if (error instanceof ShutdownError) {
+        const isSyncComplete = interval[1] === hexToNumber(last.number);
+        const { intervals: requiredIntervals, factoryIntervals: requiredFactoryIntervals, } = getRequiredIntervalsWithFilters({
+            interval,
+            filters: params.eventCallbacks.map(({ filter }) => filter),
+            cachedIntervals: params.cachedIntervals,
+        });
+        let closestToTipBlock;
+        if (requiredIntervals.length > 0 || requiredFactoryIntervals.length > 0) {
+            const pwr = promiseWithResolvers();
+            const durationTimer = setTimeout(() => {
+                params.common.logger.warn({
+                    msg: "Fetching backfill JSON-RPC data is taking longer than expected",
+                    chain: params.chain.name,
+                    chain_id: params.chain.id,
+                    block_range: JSON.stringify(interval),
+                    duration: endClock(),
+                });
+            }, params.common.options.command === "dev" ? 10000 : 50000);
+            closestToTipBlock = await params.database.syncQB
+                .transaction(async (tx) => {
+                const syncStore = createSyncStore({ common: params.common, qb: tx });
+                const logs = await historicalSync.syncBlockRangeData({
+                    interval,
+                    requiredIntervals,
+                    requiredFactoryIntervals,
+                    syncStore,
+                });
+                // Wait for the previous interval to complete `syncBlockData`.
+                await promise;
+                if (isSyncComplete === false) {
+                    // Queue the next interval
+                    intervalCallback({
+                        interval: [
+                            Math.min(interval[1] + 1, hexToNumber(last.number)),
+                            Math.min(interval[1] + 1 + estimateRange, hexToNumber(last.number)),
+                        ],
+                        promise: pwr.promise,
+                    });
+                }
+                const closestToTipBlock = await historicalSync.syncBlockData({
+                    interval,
+                    requiredIntervals,
+                    logs,
+                    syncStore,
+                });
+                if (params.chain.disableCache === false) {
+                    await syncStore.insertIntervals({
+                        intervals: requiredIntervals,
+                        factoryIntervals: requiredFactoryIntervals,
+                        chainId: params.chain.id,
+                    });
+                }
+                return closestToTipBlock;
+            })
+                .catch((error) => {
+                if (error instanceof ShutdownError) {
+                    throw error;
+                }
+                params.common.logger.warn({
+                    msg: "Failed to fetch backfill JSON-RPC data",
+                    chain: params.chain.name,
+                    chain_id: params.chain.id,
+                    block_range: JSON.stringify(interval),
+                    duration: endClock(),
+                    error,
+                });
                 throw error;
-            }
-            params.common.logger.warn({
-                msg: "Failed to fetch backfill JSON-RPC data",
-                chain: params.chain.name,
-                chain_id: params.chain.id,
-                block_range: JSON.stringify(interval),
-                duration: endClock(),
-                error,
             });
-            throw error;
-        }
-        clearTimeout(durationTimer);
-        // Update cursor to record progress
-        cursor = interval[1] + 1;
-        // `synced` will be undefined if a cache hit occur in `historicalSync.sync()`.
-        if (synced === undefined) {
-            // If the all known blocks are synced, then update `syncProgress.current`, else
-            // progress to the next iteration.
-            if (interval[1] === hexToNumber(last.number)) {
-                params.syncProgress.current = last;
-            }
-            else {
-                continue;
-            }
-        }
-        else {
-            if (interval[1] === hexToNumber(last.number)) {
-                params.syncProgress.current = last;
-            }
-            else {
-                params.syncProgress.current = synced;
-            }
+            clearTimeout(durationTimer);
             const duration = endClock();
-            params.common.metrics.ponder_sync_block.set(label, hexToNumber(params.syncProgress.current.number));
-            params.common.metrics.ponder_sync_block_timestamp.set(label, hexToNumber(params.syncProgress.current.timestamp));
-            params.common.metrics.ponder_historical_completed_blocks.inc(label, interval[1] - interval[0] + 1);
             // Use the duration and interval of the last call to `sync` to update estimate
-            // 25 <= estimate(new) <= estimate(prev) * 2 <= 100_000
-            estimateRange = Math.min(Math.max(25, Math.round((1000 * (interval[1] - interval[0])) / duration)), estimateRange * 2, 100000);
+            estimateRange = estimate({
+                from: interval[0],
+                to: interval[1],
+                target: params.common.options.command === "dev" ? 2000 : 10000,
+                result: duration,
+                min: 25,
+                max: 100000,
+                prev: estimateRange,
+                maxIncrease: 1.5,
+            });
             params.common.logger.trace({
                 msg: "Updated block range estimate for fetching backfill JSON-RPC data",
                 chain: params.chain.name,
                 chain_id: params.chain.id,
                 range: estimateRange,
             });
+            // Resolve promise so the next interval can continue.
+            pwr.resolve();
         }
-        yield hexToNumber(params.syncProgress.current.number);
-        if (params.syncProgress.isEnd() || params.syncProgress.isFinalized()) {
-            params.common.logger.info({
-                msg: "Finished fetching backfill JSON-RPC data",
-                chain: params.chain.name,
-                chain_id: params.chain.id,
-                duration: backfillEndClock(),
+        else {
+            // Wait for the previous interval to complete `syncBlockData`.
+            await promise;
+            if (isSyncComplete === false) {
+                // Queue the next interval
+                intervalCallback({
+                    interval: [
+                        Math.min(interval[1] + 1, hexToNumber(last.number)),
+                        Math.min(interval[1] + 1 + estimateRange, hexToNumber(last.number)),
+                    ],
+                    promise: Promise.resolve(),
+                });
+            }
+        }
+        if (interval[1] === hexToNumber(last.number)) {
+            params.syncProgress.current = last;
+        }
+        if (closestToTipBlock) {
+            params.common.metrics.ponder_sync_block.set(label, hexToNumber(closestToTipBlock.number));
+            params.common.metrics.ponder_sync_block_timestamp.set(label, hexToNumber(closestToTipBlock.timestamp));
+        }
+        else {
+            params.common.metrics.ponder_sync_block.set(label, interval[1]);
+        }
+        params.common.metrics.ponder_historical_completed_blocks.inc(label, interval[1] - interval[0] + 1);
+        return requiredIntervals.length > 0;
+    }
+    const { callback, generator } = createCallbackGenerator();
+    (async () => {
+        for await (const { interval, promise } of intervalGenerator) {
+            // Note: this relies on the invariant that `syncInterval`
+            // will always resolve promises in the order it was called.
+            syncInterval({ interval, promise }).then((didInsertData) => {
+                const isDone = interval[1] === hexToNumber(last.number);
+                if (didInsertData || isDone) {
+                    callback({ value: interval[1], done: false });
+                }
+                if (isDone) {
+                    callback({ value: undefined, done: true });
+                }
             });
-            return;
         }
+    })();
+    for await (const result of generator) {
+        if (result.done)
+            break;
+        yield result.value;
     }
+    params.common.logger.info({
+        msg: "Finished fetching backfill JSON-RPC data",
+        chain: params.chain.name,
+        chain_id: params.chain.id,
+        duration: backfillEndClock(),
+    });
 }
 /**
  * Merges multiple event generators into a single generator while preserving
@@ -746,7 +922,7 @@ export async function* mergeAsyncGeneratorsWithEventOrder(generators) {
                     ];
                     mergedResults.push({
                         events: left,
-                        chainId: event.chainId,
+                        chainId: event.chain.id,
                         checkpoint: right.length > 0 ? event.checkpoint : result.value.checkpoint,
                         blockRange,
                     });