ponder 0.14.13 → 0.15.0

package/dist/esm/sync-historical/index.js

@@ -1,41 +1,16 @@
 import { _debug_traceBlockByNumber, _eth_getBlockByNumber, _eth_getBlockReceipts, _eth_getLogs, _eth_getTransactionReceipt, validateLogsAndBlock, validateReceiptsAndBlock, validateTracesAndBlock, validateTransactionsAndBlock, } from '../rpc/actions.js';
-import { getChildAddress, isAddressFactory, isAddressMatched, isLogFactoryMatched, isTraceFilterMatched, isTransactionFilterMatched, isTransferFilterMatched, } from '../runtime/filter.js';
-import { recoverFilter } from '../runtime/fragments.js';
-import { getChunks, intervalBounds, intervalDifference, intervalRange, intervalUnion, } from '../utils/interval.js';
+import { getChildAddress, isAddressFactory, isAddressMatched, isBlockFilterMatched, isBlockInFilter, isLogFactoryMatched, isLogFilterMatched, isTraceFilterMatched, isTransactionFilterMatched, isTransferFilterMatched, } from '../runtime/filter.js';
+import { getChunks, intervalBounds, intervalRange, } from '../utils/interval.js';
+import { promiseAllSettledWithThrow } from '../utils/promiseAllSettledWithThrow.js';
+import { createQueue } from '../utils/queue.js';
 import { startClock } from '../utils/timer.js';
 import { getLogsRetryHelper } from "@ponder/utils";
-import { hexToBigInt, hexToNumber, toHex, zeroHash, } from "viem";
+import { hexToNumber, toHex, zeroHash, } from "viem";
 export const createHistoricalSync = (args) => {
     /**
      * Flag to fetch transaction receipts through _eth_getBlockReceipts (true) or _eth_getTransactionReceipt (false)
      */
     let isBlockReceipts = true;
-    /**
-     * Blocks that have already been extracted.
-     * Note: All entries are deleted at the end of each call to `sync()`.
-     */
-    const blockCache = new Map();
-    /**
-     * Traces that have already been fetched.
-     * Note: All entries are deleted at the end of each call to `sync()`.
-     */
-    const traceCache = new Map();
-    /**
-     * Transactions that should be saved to the sync-store.
-     * Note: All entries are deleted at the end of each call to `sync()`.
-     */
-    const transactionsCache = new Set();
-    /**
-     * Block transaction receipts that have already been fetched.
-     * Note: All entries are deleted at the end of each call to `sync()`.
-     */
-    const blockReceiptsCache = new Map();
-    /**
-     * Transaction receipts that have already been fetched.
-     * Note: All entries are deleted at the end of each call to `sync()`.
-     */
-    const transactionReceiptsCache = new Map();
-    const childAddressesCache = new Map();
     /**
      * Data about the range passed to "eth_getLogs" share among all log
      * filters and log factories.
@@ -43,30 +18,23 @@ export const createHistoricalSync = (args) => {
     let logsRequestMetadata = {
         estimatedRange: 500,
     };
-    // Closest-to-tip block that has been synced.
-    let latestBlock;
-    ////////
-    // Helper functions for sync tasks
-    ////////
     /**
      * Split "eth_getLogs" requests into ranges inferred from errors
      * and batch requests.
      */
-    const syncLogsDynamic = async ({ filter, address, interval, }, context) => {
-        //  Use the recommended range if available, else don't chunk the interval at all.
+    const syncLogsDynamic = async ({ address, topic0, topic1, topic2, topic3, interval }, context) => {
+        // Use the recommended range if available, else don't chunk the interval at all.
         const intervals = getChunks({
             interval,
             maxChunkSize: logsRequestMetadata.confirmedRange ??
                 logsRequestMetadata.estimatedRange,
         });
-        const topics = "eventSelector" in filter
-            ? [filter.eventSelector]
-            : [
-                filter.topic0 ?? null,
-                filter.topic1 ?? null,
-                filter.topic2 ?? null,
-                filter.topic3 ?? null,
-            ];
+        const topics = [
+            topic0 ?? null,
+            topic1 ?? null,
+            topic2 ?? null,
+            topic3 ?? null,
+        ];
         // Note: the `topics` field is very fragile for many rpc providers, and
         // cannot handle extra "null" topics
         if (topics[3] === null) {
@@ -136,7 +104,7 @@ export const createHistoricalSync = (args) => {
                     ? range
                     : undefined,
             };
-            return syncLogsDynamic({ address, interval, filter }, context);
+            return syncLogsDynamic({ address, topic0, topic1, topic2, topic3, interval }, context);
         })))).then((logs) => logs.flat());
         /**
          * Dynamically increase the range used in "eth_getLogs" if an
@@ -147,58 +115,18 @@ export const createHistoricalSync = (args) => {
         }
         return logs;
     };
-    /**
-     * Extract block, using `blockCache` to avoid fetching
-     * the same block twice. Also, update `latestBlock`.
-     *
-     * @param number Block to be extracted
-     *
-     * Note: This function could more accurately skip chain requests by taking
-     * advantage of `syncStore.hasBlock` and `syncStore.hasTransaction`.
-     */
-    const syncBlock = async (number, context) => {
-        let block;
-        /**
-         * `blockCache` contains all blocks that have been extracted during the
-         * current call to `sync()`. If `number` is present in `blockCache` use it,
-         * otherwise, request the block and add it to `blockCache` and the sync-store.
-         */
-        if (blockCache.has(number)) {
-            block = await blockCache.get(number);
-        }
-        else {
-            const _block = _eth_getBlockByNumber(args.rpc, { blockNumber: toHex(number) }, { ...context, retryNullBlockRequest: true });
-            blockCache.set(number, _block);
-            block = await _block;
-            // Update `latestBlock` if `block` is closer to tip.
-            if (hexToBigInt(block.number) >= hexToBigInt(latestBlock?.number ?? "0x0")) {
-                latestBlock = block;
-            }
-        }
-        return block;
-    };
-    const syncTrace = async (block, context) => {
-        if (traceCache.has(block)) {
-            return traceCache.get(block);
-        }
-        else {
-            const traces = _debug_traceBlockByNumber(args.rpc, { blockNumber: block }, context);
-            traceCache.set(block, traces);
-            return traces;
-        }
-    };
     const syncTransactionReceipts = async (block, transactionHashes, context) => {
         if (transactionHashes.size === 0) {
             return [];
         }
         if (isBlockReceipts === false) {
-            const transactionReceipts = await Promise.all(Array.from(transactionHashes).map((hash) => syncTransactionReceipt(hash, context)));
+            const transactionReceipts = await Promise.all(Array.from(transactionHashes).map((hash) => _eth_getTransactionReceipt(args.rpc, { hash }, context)));
             validateReceiptsAndBlock(transactionReceipts, block, "eth_getTransactionReceipt", "number");
             return transactionReceipts;
         }
         let blockReceipts;
         try {
-            blockReceipts = await syncBlockReceipts(block, context);
+            blockReceipts = await _eth_getBlockReceipts(args.rpc, { blockHash: block.hash }, context);
         }
         catch (_error) {
             const error = _error;
@@ -216,34 +144,18 @@ export const createHistoricalSync = (args) => {
         const transactionReceipts = blockReceipts.filter((receipt) => transactionHashes.has(receipt.transactionHash));
         return transactionReceipts;
     };
-    const syncTransactionReceipt = async (transaction, context) => {
-        if (transactionReceiptsCache.has(transaction)) {
-            return transactionReceiptsCache.get(transaction);
-        }
-        else {
-            const receipt = _eth_getTransactionReceipt(args.rpc, { hash: transaction }, context);
-            transactionReceiptsCache.set(transaction, receipt);
-            return receipt;
-        }
-    };
-    const syncBlockReceipts = async (block, context) => {
-        if (blockReceiptsCache.has(block.hash)) {
-            return blockReceiptsCache.get(block.hash);
-        }
-        else {
-            const blockReceipts = _eth_getBlockReceipts(args.rpc, { blockHash: block.hash }, context);
-            blockReceiptsCache.set(block.hash, blockReceipts);
-            return blockReceipts;
-        }
-    };
-    /** Extract and insert the log-based addresses that match `filter` + `interval`. */
-    const syncLogFactory = async (factory, interval, context) => {
+    /**
+     * Fetch child addresses for `factory` within `interval`
+     *
+     * @dev Newly fetched child addresses are added into `args.childAddresses`
+     */
+    const syncAddressFactory = async (factory, interval, context) => {
         const logs = await syncLogsDynamic({
-            filter: factory,
-            interval,
             address: factory.address,
+            topic0: factory.eventSelector,
+            interval,
         }, context);
-        const childAddresses = childAddressesCache.get(factory) ?? new Map();
+        const childAddresses = new Map();
         const childAddressesRecord = args.childAddresses.get(factory.id);
         for (const log of logs) {
             if (isLogFactoryMatched({ factory, log })) {
@@ -257,72 +169,130 @@ export const createHistoricalSync = (args) => {
                 }
             }
         }
-        // Note: `factory` must refer to the same original `factory` in `filter`
-        // and not be a recovered factory from `recoverFilter`.
-        childAddressesCache.set(factory, childAddresses);
+        return childAddresses;
     };
-    /**
-     * Return all addresses that match `filter` after extracting addresses
-     * that match `filter` and `interval`.
-     */
-    const syncAddressFactory = async (factory, interval, context) => {
-        const factoryInterval = [
-            Math.max(factory.fromBlock ?? 0, interval[0]),
-            Math.min(factory.toBlock ?? Number.POSITIVE_INFINITY, interval[1]),
-        ];
-        if (factoryInterval[0] <= factoryInterval[1]) {
-            await syncLogFactory(factory, factoryInterval, context);
-        }
-        // Note: `factory` must refer to the same original `factory` in `filter`
-        // and not be a recovered factory from `recoverFilter`.
-        return args.childAddresses.get(factory.id);
-    };
-    ////////
-    // Helper function for filter types
-    ////////
-    const syncLogFilter = async (filter, interval, context) => {
-        let logs;
-        if (isAddressFactory(filter.address)) {
-            const childAddresses = await syncAddressFactory(filter.address, interval, context);
-            // Note: Exit early when only the factory needs to be synced
-            if ((filter.fromBlock ?? 0) > interval[1])
-                return;
-            logs = await syncLogsDynamic({
-                filter,
-                interval,
-                address: childAddresses.size >=
-                    args.common.options.factoryAddressCountThreshold
-                    ? undefined
-                    : Array.from(childAddresses.keys()),
-            }, context);
-            logs = logs.filter((log) => isAddressMatched({
-                address: log.address,
-                blockNumber: hexToNumber(log.blockNumber),
-                childAddresses,
+    return {
+        async syncBlockRangeData({ interval, requiredIntervals, requiredFactoryIntervals, syncStore, }) {
+            const context = {
+                logger: args.common.logger.child({ action: "fetch_block_data" }),
+            };
+            const endClock = startClock();
+            const childAddresses = new Map();
+            const logs = [];
+            // Dedupe factory intervals by factory id
+            const factoryIntervalsById = new Map();
+            for (const { factory, interval } of requiredFactoryIntervals) {
+                if (factoryIntervalsById.has(factory.id)) {
+                    const existingInterval = factoryIntervalsById.get(factory.id).interval;
+                    factoryIntervalsById.get(factory.id).interval = intervalBounds([
+                        existingInterval,
+                        interval,
+                    ]);
+                }
+                else {
+                    factoryIntervalsById.set(factory.id, { factory, interval });
+                }
+            }
+            requiredFactoryIntervals = Array.from(factoryIntervalsById.values());
+            await Promise.all(requiredFactoryIntervals.map(async ({ factory, interval }) => {
+                childAddresses.set(factory.id, await syncAddressFactory(factory, interval, context));
             }));
-        }
-        else {
-            logs = await syncLogsDynamic({
-                filter,
-                interval,
-                address: filter.address,
-            }, context);
-        }
-        await args.syncStore.insertLogs({ logs, chainId: args.chain.id }, context);
-        const logsPerBlock = new Map();
-        for (const log of logs) {
-            const blockNumber = hexToNumber(log.blockNumber);
-            if (logsPerBlock.has(blockNumber) === false) {
-                logsPerBlock.set(blockNumber, []);
+            const mergedEthGetLogsParams = new Map();
+            const singleEthGetLogsParams = [];
+            for (const { filter, interval } of requiredIntervals) {
+                if (filter.type !== "log")
+                    continue;
+                const hasAddress = filter.address !== undefined;
+                const hasTopic1 = filter.topic1 !== undefined;
+                const hasTopic2 = filter.topic2 !== undefined;
+                const hasTopic3 = filter.topic3 !== undefined;
+                if (hasAddress === false || hasTopic1 || hasTopic2 || hasTopic3) {
+                    if (isAddressFactory(filter.address)) {
+                        const childAddresses = args.childAddresses.get(filter.address.id);
+                        singleEthGetLogsParams.push({
+                            address: childAddresses.size >=
+                                args.common.options.factoryAddressCountThreshold
+                                ? undefined
+                                : Array.from(childAddresses.keys()),
+                            topic0: filter.topic0,
+                            topic1: filter.topic1,
+                            topic2: filter.topic2,
+                            topic3: filter.topic3,
+                            interval,
+                        });
+                    }
+                    else {
+                        singleEthGetLogsParams.push({
+                            address: filter.address,
+                            topic0: filter.topic0,
+                            topic1: filter.topic1,
+                            topic2: filter.topic2,
+                            topic3: filter.topic3,
+                            interval,
+                        });
+                    }
+                    continue;
+                }
+                let addressKey;
+                if (isAddressFactory(filter.address)) {
+                    addressKey = filter.address.id;
+                }
+                else if (Array.isArray(filter.address)) {
+                    addressKey = filter.address.join("_");
+                }
+                else {
+                    addressKey = filter.address;
+                }
+                if (mergedEthGetLogsParams.has(addressKey) === false) {
+                    if (isAddressFactory(filter.address)) {
+                        const childAddresses = args.childAddresses.get(filter.address.id);
+                        mergedEthGetLogsParams.set(addressKey, {
+                            address: childAddresses.size >=
+                                args.common.options.factoryAddressCountThreshold
+                                ? undefined
+                                : Array.from(childAddresses.keys()),
+                            topic0: filter.topic0,
+                            topic1: filter.topic1,
+                            topic2: filter.topic2,
+                            topic3: filter.topic3,
+                            interval,
+                        });
+                    }
+                    else {
+                        mergedEthGetLogsParams.set(addressKey, {
+                            address: filter.address,
+                            topic0: filter.topic0,
+                            topic1: filter.topic1,
+                            topic2: filter.topic2,
+                            topic3: filter.topic3,
+                            interval,
+                        });
+                    }
+                }
+                else {
+                    const existingInterval = mergedEthGetLogsParams.get(addressKey).interval;
+                    const existingTopic0 = mergedEthGetLogsParams.get(addressKey)
+                        .topic0;
+                    mergedEthGetLogsParams.get(addressKey).topic0 = [
+                        ...(Array.isArray(existingTopic0)
+                            ? existingTopic0
+                            : [existingTopic0]),
+                        filter.topic0,
+                    ];
+                    mergedEthGetLogsParams.get(addressKey).interval = intervalBounds([
+                        existingInterval,
+                        interval,
+                    ]);
+                }
             }
-            logsPerBlock.get(blockNumber).push(log);
-        }
-        const blocks = await Promise.all(Array.from(logsPerBlock.keys()).map((number) => syncBlock(number, context)));
-        // Validate that logs point to the valid transaction hash in the block
-        for (const block of blocks) {
-            const logs = logsPerBlock.get(hexToNumber(block.number));
-            validateTransactionsAndBlock(block, "number");
-            validateLogsAndBlock(logs, block, "number");
+            const ethGetLogsParams = [
+                ...singleEthGetLogsParams,
+                ...Array.from(mergedEthGetLogsParams.values()),
+            ];
+            await Promise.all(ethGetLogsParams.map(async (params) => {
+                const _logs = await syncLogsDynamic(params, context);
+                logs.push(..._logs);
+            }));
             for (const log of logs) {
                 if (log.transactionHash === zeroHash) {
                     args.common.logger.warn({
@@ -330,318 +300,307 @@ export const createHistoricalSync = (args) => {
                         action: "fetch_block_data",
                         chain: args.chain.name,
                         chain_id: args.chain.id,
-                        number: hexToNumber(block.number),
-                        hash: block.hash,
+                        number: hexToNumber(log.blockNumber),
+                        hash: log.blockHash,
                         logIndex: hexToNumber(log.logIndex),
                     });
                 }
             }
-        }
-        const transactionHashes = new Set(logs.map((l) => l.transactionHash));
-        for (const hash of transactionHashes) {
-            transactionsCache.add(hash);
-        }
-        if (filter.hasTransactionReceipt) {
-            const transactionReceipts = await Promise.all(blocks.map((block) => {
-                const blockTransactionHashes = new Set();
-                for (const log of logsPerBlock.get(hexToNumber(block.number))) {
-                    if (log.transactionHash !== zeroHash) {
-                        blockTransactionHashes.add(log.transactionHash);
-                    }
-                }
-                return syncTransactionReceipts(block, blockTransactionHashes, context);
-            })).then((receipts) => receipts.flat());
-            await args.syncStore.insertTransactionReceipts({
-                transactionReceipts,
-                chainId: args.chain.id,
-            }, context);
-        }
-    };
-    const syncBlockFilter = async (filter, interval, context) => {
-        const baseOffset = (interval[0] - filter.offset) % filter.interval;
-        const offset = baseOffset === 0 ? 0 : filter.interval - baseOffset;
-        // Determine which blocks are matched by the block filter.
-        const requiredBlocks = [];
-        for (let b = interval[0] + offset; b <= interval[1]; b += filter.interval) {
-            requiredBlocks.push(b);
-        }
-        await Promise.all(requiredBlocks.map(async (number) => {
-            const block = await syncBlock(number, context);
-            validateTransactionsAndBlock(block, "number");
-            return block;
-        }));
-    };
-    const syncTransactionFilter = async (filter, interval, context) => {
-        const fromChildAddresses = isAddressFactory(filter.fromAddress)
-            ? await syncAddressFactory(filter.fromAddress, interval, context)
-            : undefined;
-        const toChildAddresses = isAddressFactory(filter.toAddress)
-            ? await syncAddressFactory(filter.toAddress, interval, context)
-            : undefined;
-        // Note: Exit early when only the factory needs to be synced
-        if ((filter.fromBlock ?? 0) > interval[1])
-            return;
-        const blocks = await Promise.all(intervalRange(interval).map((number) => syncBlock(number, context)));
-        const transactionHashes = new Set();
-        const requiredBlocks = new Set();
-        for (const block of blocks) {
-            validateTransactionsAndBlock(block, "number");
-            for (const transaction of block.transactions) {
-                if (isTransactionFilterMatched({ filter, transaction }) === false) {
-                    continue;
-                }
-                if (isAddressFactory(filter.fromAddress) &&
-                    isAddressMatched({
-                        address: transaction.from,
-                        blockNumber: Number(block.number),
-                        childAddresses: fromChildAddresses,
-                    }) === false) {
-                    continue;
-                }
-                if (isAddressFactory(filter.toAddress) &&
-                    isAddressMatched({
-                        address: transaction.to ?? undefined,
-                        blockNumber: Number(block.number),
-                        childAddresses: toChildAddresses,
-                    }) === false) {
-                    continue;
-                }
-                transactionHashes.add(transaction.hash);
-                requiredBlocks.add(block);
-            }
-        }
-        for (const hash of transactionHashes) {
-            transactionsCache.add(hash);
-        }
-        const transactionReceipts = await Promise.all(Array.from(requiredBlocks).map((block) => {
-            const blockTransactionHashes = new Set(block.transactions
-                .filter((t) => transactionHashes.has(t.hash))
-                .map((t) => t.hash));
-            return syncTransactionReceipts(block, blockTransactionHashes, context);
-        })).then((receipts) => receipts.flat());
-        await args.syncStore.insertTransactionReceipts({
-            transactionReceipts,
-            chainId: args.chain.id,
-        }, context);
-    };
-    const syncTraceOrTransferFilter = async (filter, interval, context) => {
-        const fromChildAddresses = isAddressFactory(filter.fromAddress)
-            ? await syncAddressFactory(filter.fromAddress, interval, context)
-            : undefined;
-        const toChildAddresses = isAddressFactory(filter.toAddress)
-            ? await syncAddressFactory(filter.toAddress, interval, context)
-            : undefined;
-        // Note: Exit early when only the factory needs to be synced
-        if ((filter.fromBlock ?? 0) > interval[1])
-            return;
-        const requiredBlocks = new Set();
-        const traces = await Promise.all(intervalRange(interval).map(async (number) => {
-            let traces = await syncTrace(number, context);
-            // remove unmatched traces
-            traces = traces.filter((trace) => {
-                if (filter.type === "trace" &&
-                    isTraceFilterMatched({
-                        filter,
-                        trace: trace.trace,
-                        block: { number: BigInt(number) },
-                    }) === false) {
-                    return false;
-                }
-                if (filter.type === "transfer" &&
-                    isTransferFilterMatched({
-                        filter,
-                        trace: trace.trace,
-                        block: { number: BigInt(number) },
-                    }) === false) {
-                    return false;
-                }
-                if (isAddressFactory(filter.fromAddress) &&
-                    isAddressMatched({
-                        address: trace.trace.from,
-                        blockNumber: number,
-                        childAddresses: fromChildAddresses,
-                    }) === false) {
-                    return false;
-                }
-                if (isAddressFactory(filter.toAddress) &&
-                    isAddressMatched({
-                        address: trace.trace.to,
-                        blockNumber: number,
-                        childAddresses: toChildAddresses,
-                    }) === false) {
-                    return false;
-                }
-                return true;
-            });
-            if (traces.length === 0)
-                return [];
-            const block = await syncBlock(number, context);
-            validateTransactionsAndBlock(block, "number");
-            validateTracesAndBlock(traces, block, "number");
-            requiredBlocks.add(block);
-            const transactionsByHash = new Map();
-            for (const transaction of block.transactions) {
-                transactionsByHash.set(transaction.hash, transaction);
+            let childAddressCount = 0;
+            for (const { size } of childAddresses.values()) {
+                childAddressCount += size;
             }
-            return traces.map((trace) => {
-                const transaction = transactionsByHash.get(trace.transactionHash);
-                transactionsCache.add(trace.transactionHash);
-                return { trace, transaction, block };
-            });
-        })).then((traces) => traces.flat());
-        await args.syncStore.insertTraces({
-            traces,
-            chainId: args.chain.id,
-        }, context);
-        if (filter.hasTransactionReceipt) {
-            const transactionReceipts = await Promise.all(Array.from(requiredBlocks).map((block) => {
-                const blockTransactionHashes = new Set(traces
-                    .filter((t) => t.block.hash === block.hash)
-                    .map((t) => t.transaction.hash));
-                return syncTransactionReceipts(block, blockTransactionHashes, context);
-            })).then((receipts) => receipts.flat());
-            await args.syncStore.insertTransactionReceipts({
-                transactionReceipts,
+            args.common.logger.debug({
+                msg: "Fetched block range data",
+                chain: args.chain.name,
+                chain_id: args.chain.id,
+                block_range: JSON.stringify(interval),
+                log_count: logs.length,
+                child_address_count: childAddressCount,
+                duration: endClock(),
+            }, ["chain", "block_range"]);
+            await promiseAllSettledWithThrow(Array.from(childAddresses.entries()).map(([factoryId, childAddresses]) => syncStore.insertChildAddresses({
+                factory: factoryIntervalsById.get(factoryId).factory,
+                childAddresses,
                 chainId: args.chain.id,
-            }, context);
-        }
-    };
-    return {
-        async sync(_interval) {
+            }, context)));
+            return logs;
+        },
+        async syncBlockData({ syncStore, interval, requiredIntervals, logs }) {
             const context = {
                 logger: args.common.logger.child({ action: "fetch_block_data" }),
             };
             const endClock = startClock();
-            const intervalsToSync = [];
-            // Determine the requests that need to be made, and which intervals need to be inserted.
-            // Fragments are used to create a minimal filter, to avoid refetching data even if a filter
-            // is only partially synced.
-            for (const { filter } of args.sources) {
-                let filterIntervals = [
-                    [
-                        Math.max(filter.fromBlock ?? 0, _interval[0]),
-                        Math.min(filter.toBlock ?? Number.POSITIVE_INFINITY, _interval[1]),
-                    ],
-                ];
+            const blockFilters = [];
+            const transactionFilters = [];
+            const traceFilters = [];
+            const logFilters = [];
+            const transferFilters = [];
+            for (const { filter } of requiredIntervals) {
                 switch (filter.type) {
-                    case "log":
-                        if (isAddressFactory(filter.address)) {
-                            filterIntervals.push([
-                                Math.max(filter.address.fromBlock ?? 0, _interval[0]),
-                                Math.min(filter.address.toBlock ?? Number.POSITIVE_INFINITY, _interval[1]),
-                            ]);
-                        }
+                    case "block": {
+                        blockFilters.push(filter);
                         break;
-                    case "trace":
-                    case "transaction":
-                    case "transfer":
-                        if (isAddressFactory(filter.fromAddress)) {
-                            filterIntervals.push([
-                                Math.max(filter.fromAddress.fromBlock ?? 0, _interval[0]),
-                                Math.min(filter.fromAddress.toBlock ?? Number.POSITIVE_INFINITY, _interval[1]),
-                            ]);
-                        }
-                        if (isAddressFactory(filter.toAddress)) {
-                            filterIntervals.push([
-                                Math.max(filter.toAddress.fromBlock ?? 0, _interval[0]),
-                                Math.min(filter.toAddress.toBlock ?? Number.POSITIVE_INFINITY, _interval[1]),
-                            ]);
-                        }
-                }
-                filterIntervals = filterIntervals.filter(([start, end]) => start <= end);
-                if (filterIntervals.length === 0) {
-                    continue;
-                }
-                filterIntervals = intervalUnion(filterIntervals);
-                const completedIntervals = args.cachedIntervals.get(filter);
-                const requiredIntervals = [];
-                for (const { fragment, intervals: fragmentIntervals, } of completedIntervals) {
-                    const requiredFragmentIntervals = intervalDifference(filterIntervals, fragmentIntervals);
-                    if (requiredFragmentIntervals.length > 0) {
-                        requiredIntervals.push({
-                            fragment,
-                            intervals: requiredFragmentIntervals,
-                        });
                     }
-                }
-                if (requiredIntervals.length > 0) {
-                    const requiredInterval = intervalBounds(requiredIntervals.flatMap(({ intervals }) => intervals));
-                    const requiredFilter = recoverFilter(filter, requiredIntervals.map(({ fragment }) => fragment));
-                    intervalsToSync.push({
-                        filter: requiredFilter,
-                        interval: requiredInterval,
-                    });
-                }
-            }
-            await Promise.all(intervalsToSync.map(async ({ filter, interval }) => {
-                // Request last block of interval
-                const blockPromise = syncBlock(interval[1], context);
-                switch (filter.type) {
-                    case "log": {
-                        await syncLogFilter(filter, interval, context);
+                    case "transaction": {
+                        transactionFilters.push(filter);
                         break;
                     }
-                    case "block": {
-                        await syncBlockFilter(filter, interval, context);
+                    case "trace": {
+                        traceFilters.push(filter);
                         break;
                     }
-                    case "transaction": {
-                        await syncTransactionFilter(filter, interval, context);
+                    case "log": {
+                        logFilters.push(filter);
                         break;
                     }
-                    case "trace":
                     case "transfer": {
-                        await syncTraceOrTransferFilter(filter, interval, context);
+                        transferFilters.push(filter);
                         break;
                     }
                 }
-                await blockPromise;
-            }));
-            const blocks = await Promise.all(blockCache.values());
-            await Promise.all([
-                args.syncStore.insertBlocks({ blocks, chainId: args.chain.id }, context),
-                args.syncStore.insertTransactions({
-                    transactions: blocks.flatMap((block) => block.transactions.filter(({ hash }) => transactionsCache.has(hash))),
-                    chainId: args.chain.id,
-                }, context),
-                ...Array.from(childAddressesCache.entries()).map(([factory, childAddresses]) => args.syncStore.insertChildAddresses({
-                    factory,
-                    childAddresses,
-                    chainId: args.chain.id,
-                }, context)),
-            ]);
-            // Add corresponding intervals to the sync-store
-            // Note: this should happen after so the database doesn't become corrupted
-            if (args.chain.disableCache === false) {
-                await args.syncStore.insertIntervals({
-                    intervals: intervalsToSync,
-                    chainId: args.chain.id,
-                }, context);
             }
-            let childAddressCount = 0;
-            for (const childAddresses of childAddressesCache.values()) {
-                childAddressCount += childAddresses.size;
+            const perBlockLogs = new Map();
+            for (const log of logs) {
+                const blockNumber = hexToNumber(log.blockNumber);
+                if (perBlockLogs.has(blockNumber) === false) {
+                    perBlockLogs.set(blockNumber, []);
+                }
+                perBlockLogs.get(blockNumber).push(log);
+            }
+            let closestToTipBlock;
+            const syncBlockData = async (blockNumber) => {
+                let block;
+                const requiredTransactions = new Set();
+                const requiredTransactionReceipts = new Set();
+                ////////
+                // Logs
+                ////////
+                let logs = [];
+                if (perBlockLogs.has(blockNumber)) {
+                    block = await _eth_getBlockByNumber(args.rpc, { blockNumber }, context);
+                    logs = perBlockLogs.get(blockNumber).filter((log) => {
+                        let isMatched = false;
+                        for (const filter of logFilters) {
+                            if (isLogFilterMatched({ filter, log }) &&
+                                (isAddressFactory(filter.address)
+                                    ? isAddressMatched({
+                                        address: log.address,
+                                        blockNumber,
+                                        childAddresses: args.childAddresses.get(filter.address.id),
+                                    })
+                                    : true)) {
+                                isMatched = true;
+                                requiredTransactions.add(log.transactionHash);
+                                if (filter.hasTransactionReceipt) {
+                                    requiredTransactionReceipts.add(log.transactionHash);
+                                    // skip to next log
+                                    break;
+                                }
+                            }
+                        }
+                        return isMatched;
+                    });
+                    if (logs.length > 0) {
+                        validateLogsAndBlock(logs, block, "number");
+                    }
+                }
+                ////////
+                // Traces
+                ////////
+                const shouldRequestTraces = traceFilters.some((filter) => isBlockInFilter(filter, blockNumber)) ||
+                    transferFilters.some((filter) => isBlockInFilter(filter, blockNumber));
+                let traces = [];
+                if (shouldRequestTraces) {
+                    if (block === undefined) {
+                        [block, traces] = await Promise.all([
+                            _eth_getBlockByNumber(args.rpc, { blockNumber }, context),
+                            _debug_traceBlockByNumber(args.rpc, { blockNumber }, context),
+                        ]);
+                    }
+                    else {
+                        traces = await _debug_traceBlockByNumber(args.rpc, { blockNumber }, context);
+                    }
+                    traces = traces.filter((trace) => {
+                        let isMatched = false;
+                        for (const filter of transferFilters) {
+                            if (isTransferFilterMatched({
+                                filter,
+                                trace: trace.trace,
+                                block: { number: BigInt(blockNumber) },
+                            }) &&
+                                (isAddressFactory(filter.fromAddress)
+                                    ? isAddressMatched({
+                                        address: trace.trace.from,
+                                        blockNumber,
+                                        childAddresses: args.childAddresses.get(filter.fromAddress.id),
+                                    })
+                                    : true) &&
+                                (isAddressFactory(filter.toAddress)
+                                    ? isAddressMatched({
+                                        address: trace.trace.to,
+                                        blockNumber,
+                                        childAddresses: args.childAddresses.get(filter.toAddress.id),
+                                    })
+                                    : true)) {
+                                isMatched = true;
+                                requiredTransactions.add(trace.transactionHash);
+                                if (filter.hasTransactionReceipt) {
+                                    requiredTransactionReceipts.add(trace.transactionHash);
+                                    // skip to next trace
+                                    break;
+                                }
+                            }
+                        }
+                        for (const filter of traceFilters) {
+                            if (isTraceFilterMatched({
+                                filter,
+                                trace: trace.trace,
+                                block: { number: BigInt(blockNumber) },
+                            }) &&
+                                (isAddressFactory(filter.fromAddress)
+                                    ? isAddressMatched({
+                                        address: trace.trace.from,
+                                        blockNumber,
+                                        childAddresses: args.childAddresses.get(filter.fromAddress.id),
+                                    })
+                                    : true) &&
+                                (isAddressFactory(filter.toAddress)
+                                    ? isAddressMatched({
+                                        address: trace.trace.to,
+                                        blockNumber,
+                                        childAddresses: args.childAddresses.get(filter.toAddress.id),
+                                    })
+                                    : true)) {
+                                isMatched = true;
+                                requiredTransactions.add(trace.transactionHash);
+                                if (filter.hasTransactionReceipt) {
+                                    requiredTransactionReceipts.add(trace.transactionHash);
+                                    // skip to next trace
+                                    break;
+                                }
+                            }
+                        }
+                        return isMatched;
+                    });
+                    if (traces.length > 0) {
+                        validateTracesAndBlock(traces, block, "number");
+                    }
+                }
+                ////////
+                // Block
+                ////////
+                if (block === undefined &&
+                    blockFilters.some((filter) => isBlockFilterMatched({
+                        filter,
+                        block: { number: BigInt(blockNumber) },
+                    }))) {
+                    block = await _eth_getBlockByNumber(args.rpc, { blockNumber }, context);
+                }
+                ////////
+                // Transactions
+                ////////
+                // Return early if no data is fetched
+                if (block === undefined &&
+                    transactionFilters.some((filter) => isBlockInFilter(filter, blockNumber)) === false) {
+                    return;
+                }
+                if (block === undefined) {
+                    block = await _eth_getBlockByNumber(args.rpc, { blockNumber }, context);
+                }
+                if (closestToTipBlock === undefined ||
+                    hexToNumber(block.number) > hexToNumber(closestToTipBlock.number)) {
+                    closestToTipBlock = block;
+                }
+                const transactions = block.transactions.filter((transaction) => {
+                    let isMatched = requiredTransactions.has(transaction.hash);
+                    for (const filter of transactionFilters) {
+                        if (isTransactionFilterMatched({ filter, transaction }) &&
+                            (isAddressFactory(filter.fromAddress)
+                                ? isAddressMatched({
+                                    address: transaction.from,
+                                    blockNumber,
+                                    childAddresses: args.childAddresses.get(filter.fromAddress.id),
+                                })
+                                : true) &&
+                            (isAddressFactory(filter.toAddress)
+                                ? isAddressMatched({
+                                    address: transaction.to ?? undefined,
+                                    blockNumber,
+                                    childAddresses: args.childAddresses.get(filter.toAddress.id),
+                                })
+                                : true)) {
+                            requiredTransactionReceipts.add(transaction.hash);
+                            isMatched = true;
+                        }
+                    }
+                    return isMatched;
+                });
+                if (transactions.length > 0) {
+                    validateTransactionsAndBlock(block, "number");
+                }
+                const transactionsByHash = new Map();
+                for (const transaction of transactions) {
+                    transactionsByHash.set(transaction.hash, transaction);
+                }
+                ////////
+                // Transaction Receipts
+                ////////
+                const transactionReceipts = await syncTransactionReceipts(block, requiredTransactionReceipts);
+                blockCount += 1;
+                transactionCount += transactions.length;
+                receiptCount += transactionReceipts.length;
+                traceCount += traces.length;
+                // Free memory of all unused transactions
+                block.transactions = transactions;
+                await promiseAllSettledWithThrow([
+                    syncStore.insertBlocks({ blocks: [block], chainId: args.chain.id }),
+                    syncStore.insertTransactions({
+                        transactions,
+                        chainId: args.chain.id,
+                    }),
+                    syncStore.insertTransactionReceipts({
+                        transactionReceipts,
+                        chainId: args.chain.id,
+                    }),
+                    syncStore.insertTraces({
+                        traces: traces.map((trace) => ({
+                            trace,
+                            block: block,
+                            transaction: transactionsByHash.get(trace.transactionHash),
+                        })),
+                        chainId: args.chain.id,
+                    }),
+                    syncStore.insertLogs({ logs, chainId: args.chain.id }),
+                ]);
+            };
+            let blockCount = 0;
+            let transactionCount = 0;
+            let receiptCount = 0;
+            let traceCount = 0;
+            // Same memory usage as `sync-realtime`.
+            const MAX_BLOCKS_IN_MEM = Math.max(args.chain.finalityBlockCount * 2, 100);
+            if (requiredIntervals.length > 0) {
+                const queue = createQueue({
+                    browser: false,
+                    initialStart: true,
+                    concurrency: MAX_BLOCKS_IN_MEM,
+                    worker: syncBlockData,
+                });
+                await Promise.all(intervalRange(interval).map((blockNumber) => queue.add(blockNumber)));
             }
             args.common.logger.debug({
                 msg: "Fetched block data",
                 chain: args.chain.name,
                 chain_id: args.chain.id,
-                block_range: JSON.stringify(_interval),
-                block_count: blockCache.size,
-                transaction_count: transactionsCache.size,
-                // receipt_count:
-                // log_count:
-                trace_count: traceCache.size,
-                child_address_count: childAddressCount,
+                block_range: JSON.stringify(interval),
+                block_count: blockCount,
+                transaction_count: transactionCount,
+                receipt_count: receiptCount,
+                trace_count: traceCount,
                 duration: endClock(),
             }, ["chain", "block_range"]);
-            blockCache.clear();
-            traceCache.clear();
-            transactionsCache.clear();
-            blockReceiptsCache.clear();
-            transactionReceiptsCache.clear();
-            childAddressesCache.clear();
-            return latestBlock;
+            return closestToTipBlock;
         },
     };
 };