ponder 0.14.13 → 0.15.0

package/src/sync-historical/index.ts

@@ -4,11 +4,7 @@ import type {
   Chain,
   Factory,
   FactoryId,
-  FilterWithoutBlocks,
-  Fragment,
-  LogFactory,
   LogFilter,
-  Source,
   SyncBlock,
   SyncLog,
   SyncTrace,
@@ -34,29 +30,36 @@ import {
   getChildAddress,
   isAddressFactory,
   isAddressMatched,
+  isBlockFilterMatched,
+  isBlockInFilter,
   isLogFactoryMatched,
+  isLogFilterMatched,
   isTraceFilterMatched,
   isTransactionFilterMatched,
   isTransferFilterMatched,
 } from "@/runtime/filter.js";
-import { recoverFilter } from "@/runtime/fragments.js";
-import type { CachedIntervals } from "@/runtime/index.js";
+import type {
+  ChildAddresses,
+  IntervalWithFactory,
+  IntervalWithFilter,
+} from "@/runtime/index.js";
 import type { SyncStore } from "@/sync-store/index.js";
 import {
   type Interval,
   getChunks,
   intervalBounds,
-  intervalDifference,
   intervalRange,
-  intervalUnion,
 } from "@/utils/interval.js";
+import { promiseAllSettledWithThrow } from "@/utils/promiseAllSettledWithThrow.js";
+import { createQueue } from "@/utils/queue.js";
 import { startClock } from "@/utils/timer.js";
 import { getLogsRetryHelper } from "@ponder/utils";
 import {
   type Address,
   type Hash,
+  type Hex,
+  type LogTopic,
   type RpcError,
-  hexToBigInt,
   hexToNumber,
   toHex,
   zeroHash,
@@ -64,20 +67,30 @@ import {
 
 export type HistoricalSync = {
   /**
-   * Extract raw data for `interval` and return the closest-to-tip block
-   * that is synced.
+   * Sync block data that can be queried for a range of blocks (logs).
+   */
+  syncBlockRangeData(params: {
+    interval: Interval;
+    requiredIntervals: IntervalWithFilter[];
+    requiredFactoryIntervals: IntervalWithFactory[];
+    syncStore: SyncStore;
+  }): Promise<SyncLog[]>;
+  /**
+   * Sync block data that must be queried for a single block (block, transactions, receipts, traces).
    */
-  sync(interval: Interval): Promise<SyncBlock | undefined>;
+  syncBlockData(params: {
+    interval: Interval;
+    requiredIntervals: IntervalWithFilter[];
+    logs: SyncLog[];
+    syncStore: SyncStore;
+  }): Promise<SyncBlock | undefined>;
 };
 
 type CreateHistoricalSyncParameters = {
   common: Common;
   chain: Chain;
   rpc: Rpc;
-  sources: Source[];
   childAddresses: Map<FactoryId, Map<Address, number>>;
-  cachedIntervals: CachedIntervals;
-  syncStore: SyncStore;
 };
 
 export const createHistoricalSync = (
@@ -87,36 +100,6 @@ export const createHistoricalSync = (
    * Flag to fetch transaction receipts through _eth_getBlockReceipts (true) or _eth_getTransactionReceipt (false)
    */
   let isBlockReceipts = true;
-  /**
-   * Blocks that have already been extracted.
-   * Note: All entries are deleted at the end of each call to `sync()`.
-   */
-  const blockCache = new Map<number, Promise<SyncBlock>>();
-  /**
-   * Traces that have already been fetched.
-   * Note: All entries are deleted at the end of each call to `sync()`.
-   */
-  const traceCache = new Map<number, Promise<SyncTrace[]>>();
-  /**
-   * Transactions that should be saved to the sync-store.
-   * Note: All entries are deleted at the end of each call to `sync()`.
-   */
-  const transactionsCache = new Set<Hash>();
-  /**
-   * Block transaction receipts that have already been fetched.
-   * Note: All entries are deleted at the end of each call to `sync()`.
-   */
-  const blockReceiptsCache = new Map<Hash, Promise<SyncTransactionReceipt[]>>();
-  /**
-   * Transaction receipts that have already been fetched.
-   * Note: All entries are deleted at the end of each call to `sync()`.
-   */
-  const transactionReceiptsCache = new Map<
-    Hash,
-    Promise<SyncTransactionReceipt>
-  >();
-
-  const childAddressesCache = new Map<LogFactory, Map<Address, number>>();
 
   /**
    * Data about the range passed to "eth_getLogs" share among all log
@@ -131,32 +114,28 @@ export const createHistoricalSync = (
     estimatedRange: 500,
   };
 
-  // Closest-to-tip block that has been synced.
-  let latestBlock: SyncBlock | undefined;
-
   ////////
   // Helper functions for sync tasks
   ////////
 
+  type EthGetLogsParams = {
+    address: Address | Address[] | undefined;
+    topic0?: LogTopic;
+    topic1?: LogTopic;
+    topic2?: LogTopic;
+    topic3?: LogTopic;
+    interval: Interval;
+  };
+
   /**
    * Split "eth_getLogs" requests into ranges inferred from errors
    * and batch requests.
    */
   const syncLogsDynamic = async (
-    {
-      filter,
-      address,
-      interval,
-    }: {
-      filter: LogFilter | LogFactory;
-      interval: Interval;
-      /** Explicitly set because of the complexity of factory contracts. */
-      address: Address | Address[] | undefined;
-    },
+    { address, topic0, topic1, topic2, topic3, interval }: EthGetLogsParams,
     context?: Parameters<Rpc["request"]>[1],
   ): Promise<SyncLog[]> => {
-    //  Use the recommended range if available, else don't chunk the interval at all.
-
+    // Use the recommended range if available, else don't chunk the interval at all.
     const intervals = getChunks({
       interval,
       maxChunkSize:
@@ -164,15 +143,12 @@ export const createHistoricalSync = (
         logsRequestMetadata.estimatedRange,
     });
 
-    const topics =
-      "eventSelector" in filter
-        ? [filter.eventSelector]
-        : [
-            filter.topic0 ?? null,
-            filter.topic1 ?? null,
-            filter.topic2 ?? null,
-            filter.topic3 ?? null,
-          ];
+    const topics = [
+      topic0 ?? null,
+      topic1 ?? null,
+      topic2 ?? null,
+      topic3 ?? null,
+    ];
 
     // Note: the `topics` field is very fragile for many rpc providers, and
     // cannot handle extra "null" topics
@@ -257,7 +233,10 @@ export const createHistoricalSync = (
                 : undefined,
             };
 
-            return syncLogsDynamic({ address, interval, filter }, context);
+            return syncLogsDynamic(
+              { address, topic0, topic1, topic2, topic3, interval },
+              context,
+            );
           }),
         ),
       ),
@@ -277,66 +256,6 @@ export const createHistoricalSync = (
     return logs;
   };
 
-  /**
-   * Extract block, using `blockCache` to avoid fetching
-   * the same block twice. Also, update `latestBlock`.
-   *
-   * @param number Block to be extracted
-   *
-   * Note: This function could more accurately skip chain requests by taking
-   * advantage of `syncStore.hasBlock` and `syncStore.hasTransaction`.
-   */
-  const syncBlock = async (
-    number: number,
-    context?: Parameters<Rpc["request"]>[1],
-  ): Promise<SyncBlock> => {
-    let block: SyncBlock;
-
-    /**
-     * `blockCache` contains all blocks that have been extracted during the
-     * current call to `sync()`. If `number` is present in `blockCache` use it,
-     * otherwise, request the block and add it to `blockCache` and the sync-store.
-     */
-
-    if (blockCache.has(number)) {
-      block = await blockCache.get(number)!;
-    } else {
-      const _block = _eth_getBlockByNumber(
-        args.rpc,
-        { blockNumber: toHex(number) },
-        { ...context, retryNullBlockRequest: true },
-      );
-      blockCache.set(number, _block);
-      block = await _block;
-
-      // Update `latestBlock` if `block` is closer to tip.
-      if (
-        hexToBigInt(block.number) >= hexToBigInt(latestBlock?.number ?? "0x0")
-      ) {
-        latestBlock = block;
-      }
-    }
-
-    return block;
-  };
-
-  const syncTrace = async (
-    block: number,
-    context?: Parameters<Rpc["request"]>[1],
-  ) => {
-    if (traceCache.has(block)) {
-      return traceCache.get(block)!;
-    } else {
-      const traces = _debug_traceBlockByNumber(
-        args.rpc,
-        { blockNumber: block },
-        context,
-      );
-      traceCache.set(block, traces);
-      return traces;
-    }
-  };
-
   const syncTransactionReceipts = async (
     block: SyncBlock,
     transactionHashes: Set<Hash>,
@@ -349,7 +268,7 @@ export const createHistoricalSync = (
     if (isBlockReceipts === false) {
       const transactionReceipts = await Promise.all(
         Array.from(transactionHashes).map((hash) =>
-          syncTransactionReceipt(hash, context),
+          _eth_getTransactionReceipt(args.rpc, { hash }, context),
         ),
       );
 
@@ -365,7 +284,11 @@ export const createHistoricalSync = (
 
     let blockReceipts: SyncTransactionReceipt[];
     try {
-      blockReceipts = await syncBlockReceipts(block, context);
+      blockReceipts = await _eth_getBlockReceipts(
+        args.rpc,
+        { blockHash: block.hash },
+        context,
+      );
     } catch (_error) {
       const error = _error as Error;
       args.common.logger.warn({
@@ -394,58 +317,26 @@ export const createHistoricalSync = (
     return transactionReceipts;
   };
 
-  const syncTransactionReceipt = async (
-    transaction: Hash,
-    context?: Parameters<Rpc["request"]>[1],
-  ) => {
-    if (transactionReceiptsCache.has(transaction)) {
-      return transactionReceiptsCache.get(transaction)!;
-    } else {
-      const receipt = _eth_getTransactionReceipt(
-        args.rpc,
-        { hash: transaction },
-        context,
-      );
-      transactionReceiptsCache.set(transaction, receipt);
-      return receipt;
-    }
-  };
-
-  const syncBlockReceipts = async (
-    block: SyncBlock,
-    context?: Parameters<Rpc["request"]>[1],
-  ) => {
-    if (blockReceiptsCache.has(block.hash)) {
-      return blockReceiptsCache.get(block.hash)!;
-    } else {
-      const blockReceipts = _eth_getBlockReceipts(
-        args.rpc,
-        { blockHash: block.hash },
-        context,
-      );
-      blockReceiptsCache.set(block.hash, blockReceipts);
-      return blockReceipts;
-    }
-  };
-
-  /** Extract and insert the log-based addresses that match `filter` + `interval`. */
-  const syncLogFactory = async (
-    factory: LogFactory,
+  /**
+   * Fetch child addresses for `factory` within `interval`
+   *
+   * @dev Newly fetched child addresses are added into `args.childAddresses`
+   */
+  const syncAddressFactory = async (
+    factory: Factory,
     interval: Interval,
     context?: Parameters<Rpc["request"]>[1],
-  ) => {
+  ): Promise<Map<Address, number>> => {
     const logs = await syncLogsDynamic(
       {
-        filter: factory,
-        interval,
         address: factory.address,
+        topic0: factory.eventSelector,
+        interval,
       },
       context,
     );
 
-    const childAddresses =
-      childAddressesCache.get(factory) ?? new Map<Address, number>();
-
+    const childAddresses = new Map<Address, number>();
     const childAddressesRecord = args.childAddresses.get(factory.id)!;
 
     for (const log of logs) {
@@ -464,108 +355,160 @@ export const createHistoricalSync = (
       }
     }
 
-    // Note: `factory` must refer to the same original `factory` in `filter`
-    // and not be a recovered factory from `recoverFilter`.
-    childAddressesCache.set(factory, childAddresses);
+    return childAddresses;
   };
 
-  /**
-   * Return all addresses that match `filter` after extracting addresses
-   * that match `filter` and `interval`.
-   */
-  const syncAddressFactory = async (
-    factory: Factory,
-    interval: Interval,
-    context?: Parameters<Rpc["request"]>[1],
-  ): Promise<Map<Address, number>> => {
-    const factoryInterval: Interval = [
-      Math.max(factory.fromBlock ?? 0, interval[0]),
-      Math.min(factory.toBlock ?? Number.POSITIVE_INFINITY, interval[1]),
-    ];
-
-    if (factoryInterval[0] <= factoryInterval[1]) {
-      await syncLogFactory(factory, factoryInterval, context);
-    }
-
-    // Note: `factory` must refer to the same original `factory` in `filter`
-    // and not be a recovered factory from `recoverFilter`.
-    return args.childAddresses.get(factory.id)!;
-  };
+  return {
+    async syncBlockRangeData({
+      interval,
+      requiredIntervals,
+      requiredFactoryIntervals,
+      syncStore,
+    }) {
+      const context = {
+        logger: args.common.logger.child({ action: "fetch_block_data" }),
+      };
+      const endClock = startClock();
+      const childAddresses: ChildAddresses = new Map();
+      const logs: SyncLog[] = [];
+
+      // Dedupe factory intervals by factory id
+
+      const factoryIntervalsById: Map<
+        Factory["id"],
+        { factory: Factory; interval: Interval }
+      > = new Map();
+
+      for (const { factory, interval } of requiredFactoryIntervals) {
+        if (factoryIntervalsById.has(factory.id)) {
+          const existingInterval = factoryIntervalsById.get(
+            factory.id,
+          )!.interval;
+
+          factoryIntervalsById.get(factory.id)!.interval = intervalBounds([
+            existingInterval,
+            interval,
+          ]);
+        } else {
+          factoryIntervalsById.set(factory.id, { factory, interval });
+        }
+      }
 
-  ////////
-  // Helper function for filter types
-  ////////
+      requiredFactoryIntervals = Array.from(factoryIntervalsById.values());
 
-  const syncLogFilter = async (
-    filter: LogFilter,
-    interval: Interval,
-    context?: Parameters<Rpc["request"]>[1],
-  ) => {
-    let logs: SyncLog[];
-    if (isAddressFactory(filter.address)) {
-      const childAddresses = await syncAddressFactory(
-        filter.address,
-        interval,
-        context,
+      await Promise.all(
+        requiredFactoryIntervals.map(async ({ factory, interval }) => {
+          childAddresses.set(
+            factory.id,
+            await syncAddressFactory(factory, interval, context)!,
+          );
+        }),
       );
 
-      // Note: Exit early when only the factory needs to be synced
-      if ((filter.fromBlock ?? 0) > interval[1]) return;
-
-      logs = await syncLogsDynamic(
-        {
-          filter,
-          interval,
-          address:
-            childAddresses.size >=
-            args.common.options.factoryAddressCountThreshold
-              ? undefined
-              : Array.from(childAddresses.keys()),
-        },
-        context,
-      );
+      const mergedEthGetLogsParams: Map<string, EthGetLogsParams> = new Map();
+      const singleEthGetLogsParams: EthGetLogsParams[] = [];
+
+      for (const { filter, interval } of requiredIntervals) {
+        if (filter.type !== "log") continue;
+
+        const hasAddress = filter.address !== undefined;
+        const hasTopic1 = filter.topic1 !== undefined;
+        const hasTopic2 = filter.topic2 !== undefined;
+        const hasTopic3 = filter.topic3 !== undefined;
+
+        if (hasAddress === false || hasTopic1 || hasTopic2 || hasTopic3) {
+          if (isAddressFactory(filter.address)) {
+            const childAddresses = args.childAddresses.get(filter.address.id)!;
+            singleEthGetLogsParams.push({
+              address:
+                childAddresses.size >=
+                args.common.options.factoryAddressCountThreshold
+                  ? undefined
+                  : Array.from(childAddresses.keys()),
+              topic0: filter.topic0,
+              topic1: filter.topic1,
+              topic2: filter.topic2,
+              topic3: filter.topic3,
+              interval,
+            });
+          } else {
+            singleEthGetLogsParams.push({
+              address: filter.address,
+              topic0: filter.topic0,
+              topic1: filter.topic1,
+              topic2: filter.topic2,
+              topic3: filter.topic3,
+              interval,
+            });
+          }
 
-      logs = logs.filter((log) =>
-        isAddressMatched({
-          address: log.address,
-          blockNumber: hexToNumber(log.blockNumber),
-          childAddresses,
-        }),
-      );
-    } else {
-      logs = await syncLogsDynamic(
-        {
-          filter,
-          interval,
-          address: filter.address,
-        },
-        context,
-      );
-    }
+          continue;
+        }
 
-    await args.syncStore.insertLogs({ logs, chainId: args.chain.id }, context);
+        let addressKey: string;
+        if (isAddressFactory(filter.address)) {
+          addressKey = filter.address.id;
+        } else if (Array.isArray(filter.address)) {
+          addressKey = filter.address.join("_");
+        } else {
+          addressKey = filter.address as Address;
+        }
 
-    const logsPerBlock = new Map<number, SyncLog[]>();
-    for (const log of logs) {
-      const blockNumber = hexToNumber(log.blockNumber);
-      if (logsPerBlock.has(blockNumber) === false) {
-        logsPerBlock.set(blockNumber, []);
+        if (mergedEthGetLogsParams.has(addressKey) === false) {
+          if (isAddressFactory(filter.address)) {
+            const childAddresses = args.childAddresses.get(filter.address.id)!;
+            mergedEthGetLogsParams.set(addressKey, {
+              address:
+                childAddresses.size >=
+                args.common.options.factoryAddressCountThreshold
+                  ? undefined
+                  : Array.from(childAddresses.keys()),
+              topic0: filter.topic0,
+              topic1: filter.topic1,
+              topic2: filter.topic2,
+              topic3: filter.topic3,
+              interval,
+            });
+          } else {
+            mergedEthGetLogsParams.set(addressKey, {
+              address: filter.address,
+              topic0: filter.topic0,
+              topic1: filter.topic1,
+              topic2: filter.topic2,
+              topic3: filter.topic3,
+              interval,
+            });
+          }
+        } else {
+          const existingInterval =
+            mergedEthGetLogsParams.get(addressKey)!.interval;
+          const existingTopic0 = mergedEthGetLogsParams.get(addressKey)!
+            .topic0 as Hex | Hex[];
+
+          mergedEthGetLogsParams.get(addressKey)!.topic0 = [
+            ...(Array.isArray(existingTopic0)
+              ? existingTopic0
+              : [existingTopic0]),
+            filter.topic0,
+          ];
+          mergedEthGetLogsParams.get(addressKey)!.interval = intervalBounds([
+            existingInterval,
+            interval,
+          ]);
+        }
       }
-      logsPerBlock.get(blockNumber)!.push(log);
-    }
-
-    const blocks = await Promise.all(
-      Array.from(logsPerBlock.keys()).map((number) =>
-        syncBlock(number, context),
-      ),
-    );
 
-    // Validate that logs point to the valid transaction hash in the block
-    for (const block of blocks) {
-      const logs = logsPerBlock.get(hexToNumber(block.number))!;
+      const ethGetLogsParams = [
+        ...singleEthGetLogsParams,
+        ...Array.from(mergedEthGetLogsParams.values()),
+      ];
 
-      validateTransactionsAndBlock(block, "number");
-      validateLogsAndBlock(logs, block, "number");
+      await Promise.all(
+        ethGetLogsParams.map(async (params) => {
+          const _logs = await syncLogsDynamic(params, context);
+          logs.push(..._logs);
+        }),
+      );
 
       for (const log of logs) {
         if (log.transactionHash === zeroHash) {
@@ -574,503 +517,424 @@ export const createHistoricalSync = (
             action: "fetch_block_data",
             chain: args.chain.name,
             chain_id: args.chain.id,
-            number: hexToNumber(block.number),
-            hash: block.hash,
+            number: hexToNumber(log.blockNumber),
+            hash: log.blockHash,
             logIndex: hexToNumber(log.logIndex),
           });
         }
       }
-    }
-
-    const transactionHashes = new Set(logs.map((l) => l.transactionHash));
-    for (const hash of transactionHashes) {
-      transactionsCache.add(hash);
-    }
-
-    if (filter.hasTransactionReceipt) {
-      const transactionReceipts = await Promise.all(
-        blocks.map((block) => {
-          const blockTransactionHashes = new Set<Hash>();
-
-          for (const log of logsPerBlock.get(hexToNumber(block.number))!) {
-            if (log.transactionHash !== zeroHash) {
-              blockTransactionHashes.add(log.transactionHash);
-            }
-          }
 
-          return syncTransactionReceipts(
-            block,
-            blockTransactionHashes,
-            context,
-          );
-        }),
-      ).then((receipts) => receipts.flat());
+      let childAddressCount = 0;
+      for (const { size } of childAddresses.values()) {
+        childAddressCount += size;
+      }
 
-      await args.syncStore.insertTransactionReceipts(
+      args.common.logger.debug(
         {
-          transactionReceipts,
-          chainId: args.chain.id,
+          msg: "Fetched block range data",
+          chain: args.chain.name,
+          chain_id: args.chain.id,
+          block_range: JSON.stringify(interval),
+          log_count: logs.length,
+          child_address_count: childAddressCount,
+          duration: endClock(),
         },
-        context,
+        ["chain", "block_range"],
       );
-    }
-  };
-
-  const syncBlockFilter = async (
-    filter: BlockFilter,
-    interval: Interval,
-    context?: Parameters<Rpc["request"]>[1],
-  ) => {
-    const baseOffset = (interval[0] - filter.offset) % filter.interval;
-    const offset = baseOffset === 0 ? 0 : filter.interval - baseOffset;
-
-    // Determine which blocks are matched by the block filter.
-    const requiredBlocks: number[] = [];
-    for (let b = interval[0] + offset; b <= interval[1]; b += filter.interval) {
-      requiredBlocks.push(b);
-    }
-
-    await Promise.all(
-      requiredBlocks.map(async (number) => {
-        const block = await syncBlock(number, context);
-        validateTransactionsAndBlock(block, "number");
-        return block;
-      }),
-    );
-  };
-
-  const syncTransactionFilter = async (
-    filter: TransactionFilter,
-    interval: Interval,
-    context?: Parameters<Rpc["request"]>[1],
-  ) => {
-    const fromChildAddresses = isAddressFactory(filter.fromAddress)
-      ? await syncAddressFactory(filter.fromAddress, interval, context)
-      : undefined;
-
-    const toChildAddresses = isAddressFactory(filter.toAddress)
-      ? await syncAddressFactory(filter.toAddress, interval, context)
-      : undefined;
-
-    // Note: Exit early when only the factory needs to be synced
-    if ((filter.fromBlock ?? 0) > interval[1]) return;
 
-    const blocks = await Promise.all(
-      intervalRange(interval).map((number) => syncBlock(number, context)),
-    );
+      await promiseAllSettledWithThrow(
+        Array.from(childAddresses.entries()).map(
+          ([factoryId, childAddresses]) =>
+            syncStore.insertChildAddresses(
+              {
+                factory: factoryIntervalsById.get(factoryId)!.factory,
+                childAddresses,
+                chainId: args.chain.id,
+              },
+              context,
+            ),
+        ),
+      );
 
-    const transactionHashes: Set<Hash> = new Set();
-    const requiredBlocks: Set<SyncBlock> = new Set();
+      return logs;
+    },
+    async syncBlockData({ syncStore, interval, requiredIntervals, logs }) {
+      const context = {
+        logger: args.common.logger.child({ action: "fetch_block_data" }),
+      };
+      const endClock = startClock();
 
-    for (const block of blocks) {
-      validateTransactionsAndBlock(block, "number");
+      const blockFilters: BlockFilter[] = [];
+      const transactionFilters: TransactionFilter[] = [];
+      const traceFilters: TraceFilter[] = [];
+      const logFilters: LogFilter[] = [];
+      const transferFilters: TransferFilter[] = [];
 
-      for (const transaction of block.transactions) {
-        if (isTransactionFilterMatched({ filter, transaction }) === false) {
-          continue;
+      for (const { filter } of requiredIntervals) {
+        switch (filter.type) {
+          case "block": {
+            blockFilters.push(filter as BlockFilter);
+            break;
+          }
+          case "transaction": {
+            transactionFilters.push(filter as TransactionFilter);
+            break;
+          }
+          case "trace": {
+            traceFilters.push(filter as TraceFilter);
+            break;
+          }
+          case "log": {
+            logFilters.push(filter as LogFilter);
+            break;
+          }
+          case "transfer": {
+            transferFilters.push(filter as TransferFilter);
+            break;
+          }
         }
+      }
 
-        if (
-          isAddressFactory(filter.fromAddress) &&
-          isAddressMatched({
-            address: transaction.from,
-            blockNumber: Number(block.number),
-            childAddresses: fromChildAddresses!,
-          }) === false
-        ) {
-          continue;
+      const perBlockLogs = new Map<number, SyncLog[]>();
+      for (const log of logs) {
+        const blockNumber = hexToNumber(log.blockNumber);
+        if (perBlockLogs.has(blockNumber) === false) {
+          perBlockLogs.set(blockNumber, []);
         }
+        perBlockLogs.get(blockNumber)!.push(log);
+      }
 
-        if (
-          isAddressFactory(filter.toAddress) &&
-          isAddressMatched({
-            address: transaction.to ?? undefined,
-            blockNumber: Number(block.number),
-            childAddresses: toChildAddresses!,
-          }) === false
-        ) {
-          continue;
-        }
+      let closestToTipBlock: SyncBlock | undefined;
 
-        transactionHashes.add(transaction.hash);
-        requiredBlocks.add(block);
-      }
-    }
+      const syncBlockData = async (blockNumber: number) => {
+        let block: SyncBlock | undefined;
 
-    for (const hash of transactionHashes) {
-      transactionsCache.add(hash);
-    }
+        const requiredTransactions = new Set<Hash>();
+        const requiredTransactionReceipts = new Set<Hash>();
 
-    const transactionReceipts = await Promise.all(
-      Array.from(requiredBlocks).map((block) => {
-        const blockTransactionHashes = new Set(
-          block.transactions
-            .filter((t) => transactionHashes.has(t.hash))
-            .map((t) => t.hash),
-        );
-        return syncTransactionReceipts(block, blockTransactionHashes, context);
-      }),
-    ).then((receipts) => receipts.flat());
+        ////////
+        // Logs
+        ////////
 
-    await args.syncStore.insertTransactionReceipts(
-      {
-        transactionReceipts,
-        chainId: args.chain.id,
-      },
-      context,
-    );
-  };
+        let logs: SyncLog[] = [];
+        if (perBlockLogs.has(blockNumber)) {
+          block = await _eth_getBlockByNumber(
+            args.rpc,
+            { blockNumber },
+            context,
+          );
 
-  const syncTraceOrTransferFilter = async (
-    filter: TraceFilter | TransferFilter,
-    interval: Interval,
-    context?: Parameters<Rpc["request"]>[1],
-  ) => {
-    const fromChildAddresses = isAddressFactory(filter.fromAddress)
-      ? await syncAddressFactory(filter.fromAddress, interval, context)
-      : undefined;
-
-    const toChildAddresses = isAddressFactory(filter.toAddress)
-      ? await syncAddressFactory(filter.toAddress, interval, context)
-      : undefined;
-
-    // Note: Exit early when only the factory needs to be synced
-    if ((filter.fromBlock ?? 0) > interval[1]) return;
-
-    const requiredBlocks: Set<SyncBlock> = new Set();
-    const traces = await Promise.all(
-      intervalRange(interval).map(async (number) => {
-        let traces = await syncTrace(number, context);
-
-        // remove unmatched traces
-        traces = traces.filter((trace) => {
-          if (
-            filter.type === "trace" &&
-            isTraceFilterMatched({
-              filter,
-              trace: trace.trace,
-              block: { number: BigInt(number) },
-            }) === false
-          ) {
-            return false;
-          }
+          logs = perBlockLogs.get(blockNumber)!.filter((log) => {
+            let isMatched = false;
+
+            for (const filter of logFilters) {
+              if (
+                isLogFilterMatched({ filter, log }) &&
+                (isAddressFactory(filter.address)
+                  ? isAddressMatched({
+                      address: log.address,
+                      blockNumber,
+                      childAddresses: args.childAddresses.get(
+                        filter.address.id,
+                      )!,
+                    })
+                  : true)
+              ) {
+                isMatched = true;
+
+                requiredTransactions.add(log.transactionHash);
+                if (filter.hasTransactionReceipt) {
+                  requiredTransactionReceipts.add(log.transactionHash);
+
+                  // skip to next log
+                  break;
+                }
+              }
+            }
 
-          if (
-            filter.type === "transfer" &&
-            isTransferFilterMatched({
-              filter,
-              trace: trace.trace,
-              block: { number: BigInt(number) },
-            }) === false
-          ) {
-            return false;
-          }
+            return isMatched;
+          });
 
-          if (
-            isAddressFactory(filter.fromAddress) &&
-            isAddressMatched({
-              address: trace.trace.from,
-              blockNumber: number,
-              childAddresses: fromChildAddresses!,
-            }) === false
-          ) {
-            return false;
+          if (logs.length > 0) {
+            validateLogsAndBlock(logs, block!, "number");
           }
+        }
 
-          if (
-            isAddressFactory(filter.toAddress) &&
-            isAddressMatched({
-              address: trace.trace.to,
-              blockNumber: number,
-              childAddresses: toChildAddresses!,
-            }) === false
-          ) {
-            return false;
-          }
+        ////////
+        // Traces
+        ////////
 
-          return true;
-        });
+        const shouldRequestTraces =
+          traceFilters.some((filter) => isBlockInFilter(filter, blockNumber)) ||
+          transferFilters.some((filter) =>
+            isBlockInFilter(filter, blockNumber),
+          );
 
-        if (traces.length === 0) return [];
+        let traces: SyncTrace[] = [];
+        if (shouldRequestTraces) {
+          if (block === undefined) {
+            [block, traces] = await Promise.all([
+              _eth_getBlockByNumber(args.rpc, { blockNumber }, context),
+              _debug_traceBlockByNumber(args.rpc, { blockNumber }, context),
+            ]);
+          } else {
+            traces = await _debug_traceBlockByNumber(
+              args.rpc,
+              { blockNumber },
+              context,
+            );
+          }
 
-        const block = await syncBlock(number, context);
+          traces = traces.filter((trace) => {
+            let isMatched = false;
+            for (const filter of transferFilters) {
+              if (
+                isTransferFilterMatched({
+                  filter,
+                  trace: trace.trace,
+                  block: { number: BigInt(blockNumber) },
+                }) &&
+                (isAddressFactory(filter.fromAddress)
+                  ? isAddressMatched({
+                      address: trace.trace.from,
+                      blockNumber,
+                      childAddresses: args.childAddresses.get(
+                        filter.fromAddress.id,
+                      )!,
+                    })
+                  : true) &&
+                (isAddressFactory(filter.toAddress)
+                  ? isAddressMatched({
+                      address: trace.trace.to,
+                      blockNumber,
+                      childAddresses: args.childAddresses.get(
+                        filter.toAddress.id,
+                      )!,
+                    })
+                  : true)
+              ) {
+                isMatched = true;
+                requiredTransactions.add(trace.transactionHash);
+                if (filter.hasTransactionReceipt) {
+                  requiredTransactionReceipts.add(trace.transactionHash);
+                  // skip to next trace
+                  break;
+                }
+              }
+            }
 
-        validateTransactionsAndBlock(block, "number");
-        validateTracesAndBlock(traces, block, "number");
+            for (const filter of traceFilters) {
+              if (
+                isTraceFilterMatched({
+                  filter,
+                  trace: trace.trace,
+                  block: { number: BigInt(blockNumber) },
+                }) &&
+                (isAddressFactory(filter.fromAddress)
+                  ? isAddressMatched({
+                      address: trace.trace.from,
+                      blockNumber,
+                      childAddresses: args.childAddresses.get(
+                        filter.fromAddress.id,
+                      )!,
+                    })
+                  : true) &&
+                (isAddressFactory(filter.toAddress)
+                  ? isAddressMatched({
+                      address: trace.trace.to,
+                      blockNumber,
+                      childAddresses: args.childAddresses.get(
+                        filter.toAddress.id,
+                      )!,
+                    })
+                  : true)
+              ) {
+                isMatched = true;
+                requiredTransactions.add(trace.transactionHash);
+                if (filter.hasTransactionReceipt) {
+                  requiredTransactionReceipts.add(trace.transactionHash);
+                  // skip to next trace
+                  break;
+                }
+              }
+            }
 
-        requiredBlocks.add(block);
+            return isMatched;
+          });
 
-        const transactionsByHash = new Map<Hash, SyncTransaction>();
-        for (const transaction of block.transactions) {
-          transactionsByHash.set(transaction.hash, transaction);
+          if (traces.length > 0) {
+            validateTracesAndBlock(traces, block, "number");
+          }
         }
 
-        return traces.map((trace) => {
-          const transaction = transactionsByHash.get(trace.transactionHash)!;
-          transactionsCache.add(trace.transactionHash);
-
-          return { trace, transaction, block };
-        });
-      }),
-    ).then((traces) => traces.flat());
-
-    await args.syncStore.insertTraces(
-      {
-        traces,
-        chainId: args.chain.id,
-      },
-      context,
-    );
+        ////////
+        // Block
+        ////////
 
-    if (filter.hasTransactionReceipt) {
-      const transactionReceipts = await Promise.all(
-        Array.from(requiredBlocks).map((block) => {
-          const blockTransactionHashes = new Set(
-            traces
-              .filter((t) => t.block.hash === block.hash)
-              .map((t) => t.transaction.hash),
-          );
-          return syncTransactionReceipts(
-            block,
-            blockTransactionHashes,
+        if (
+          block === undefined &&
+          blockFilters.some((filter) =>
+            isBlockFilterMatched({
+              filter,
+              block: { number: BigInt(blockNumber) },
+            }),
+          )
+        ) {
+          block = await _eth_getBlockByNumber(
+            args.rpc,
+            { blockNumber },
             context,
           );
-        }),
-      ).then((receipts) => receipts.flat());
-
-      await args.syncStore.insertTransactionReceipts(
-        {
-          transactionReceipts,
-          chainId: args.chain.id,
-        },
-        context,
-      );
-    }
-  };
-
-  return {
-    async sync(_interval) {
-      const context = {
-        logger: args.common.logger.child({ action: "fetch_block_data" }),
-      };
-      const endClock = startClock();
-
-      const intervalsToSync: {
-        interval: Interval;
-        filter: FilterWithoutBlocks;
-      }[] = [];
-
-      // Determine the requests that need to be made, and which intervals need to be inserted.
-      // Fragments are used to create a minimal filter, to avoid refetching data even if a filter
-      // is only partially synced.
-
-      for (const { filter } of args.sources) {
-        let filterIntervals: Interval[] = [
-          [
-            Math.max(filter.fromBlock ?? 0, _interval[0]),
-            Math.min(filter.toBlock ?? Number.POSITIVE_INFINITY, _interval[1]),
-          ],
-        ];
-
-        switch (filter.type) {
-          case "log":
-            if (isAddressFactory(filter.address)) {
-              filterIntervals.push([
-                Math.max(filter.address.fromBlock ?? 0, _interval[0]),
-                Math.min(
-                  filter.address.toBlock ?? Number.POSITIVE_INFINITY,
-                  _interval[1],
-                ),
-              ]);
-            }
-            break;
-          case "trace":
-          case "transaction":
-          case "transfer":
-            if (isAddressFactory(filter.fromAddress)) {
-              filterIntervals.push([
-                Math.max(filter.fromAddress.fromBlock ?? 0, _interval[0]),
-                Math.min(
-                  filter.fromAddress.toBlock ?? Number.POSITIVE_INFINITY,
-                  _interval[1],
-                ),
-              ]);
-            }
-
-            if (isAddressFactory(filter.toAddress)) {
-              filterIntervals.push([
-                Math.max(filter.toAddress.fromBlock ?? 0, _interval[0]),
-                Math.min(
-                  filter.toAddress.toBlock ?? Number.POSITIVE_INFINITY,
-                  _interval[1],
-                ),
-              ]);
-            }
         }
 
-        filterIntervals = filterIntervals.filter(
-          ([start, end]) => start <= end,
-        );
+        ////////
+        // Transactions
+        ////////
 
-        if (filterIntervals.length === 0) {
-          continue;
+        // Return early if no data is fetched
+        if (
+          block === undefined &&
+          transactionFilters.some((filter) =>
+            isBlockInFilter(filter, blockNumber),
+          ) === false
+        ) {
+          return;
         }
 
-        filterIntervals = intervalUnion(filterIntervals);
-
-        const completedIntervals = args.cachedIntervals.get(filter)!;
-        const requiredIntervals: {
-          fragment: Fragment;
-          intervals: Interval[];
-        }[] = [];
-
-        for (const {
-          fragment,
-          intervals: fragmentIntervals,
-        } of completedIntervals) {
-          const requiredFragmentIntervals = intervalDifference(
-            filterIntervals,
-            fragmentIntervals,
+        if (block === undefined) {
+          block = await _eth_getBlockByNumber(
+            args.rpc,
+            { blockNumber },
+            context,
           );
+        }
 
-          if (requiredFragmentIntervals.length > 0) {
-            requiredIntervals.push({
-              fragment,
-              intervals: requiredFragmentIntervals,
-            });
-          }
+        if (
+          closestToTipBlock === undefined ||
+          hexToNumber(block.number) > hexToNumber(closestToTipBlock.number)
+        ) {
+          closestToTipBlock = block;
         }
 
-        if (requiredIntervals.length > 0) {
-          const requiredInterval = intervalBounds(
-            requiredIntervals.flatMap(({ intervals }) => intervals),
-          );
+        const transactions = block.transactions.filter((transaction) => {
+          let isMatched = requiredTransactions.has(transaction.hash);
+          for (const filter of transactionFilters) {
+            if (
+              isTransactionFilterMatched({ filter, transaction }) &&
+              (isAddressFactory(filter.fromAddress)
+                ? isAddressMatched({
+                    address: transaction.from,
+                    blockNumber,
+                    childAddresses: args.childAddresses.get(
+                      filter.fromAddress.id,
+                    )!,
+                  })
+                : true) &&
+              (isAddressFactory(filter.toAddress)
+                ? isAddressMatched({
+                    address: transaction.to ?? undefined,
+                    blockNumber,
+                    childAddresses: args.childAddresses.get(
+                      filter.toAddress.id,
+                    )!,
+                  })
+                : true)
+            ) {
+              requiredTransactionReceipts.add(transaction.hash);
+              isMatched = true;
+            }
+          }
+          return isMatched;
+        });
 
-          const requiredFilter = recoverFilter(
-            filter,
-            requiredIntervals.map(({ fragment }) => fragment),
-          );
+        if (transactions.length > 0) {
+          validateTransactionsAndBlock(block, "number");
+        }
 
-          intervalsToSync.push({
-            filter: requiredFilter,
-            interval: requiredInterval,
-          });
+        const transactionsByHash = new Map<Hash, SyncTransaction>();
+        for (const transaction of transactions) {
+          transactionsByHash.set(transaction.hash, transaction);
         }
-      }
 
-      await Promise.all(
-        intervalsToSync.map(async ({ filter, interval }) => {
-          // Request last block of interval
-          const blockPromise = syncBlock(interval[1], context);
-
-          switch (filter.type) {
-            case "log": {
-              await syncLogFilter(filter as LogFilter, interval, context);
-              break;
-            }
+        ////////
+        // Transaction Receipts
+        ////////
 
-            case "block": {
-              await syncBlockFilter(filter as BlockFilter, interval, context);
-              break;
-            }
+        const transactionReceipts = await syncTransactionReceipts(
+          block,
+          requiredTransactionReceipts,
+        );
 
-            case "transaction": {
-              await syncTransactionFilter(
-                filter as TransactionFilter,
-                interval,
-                context,
-              );
-              break;
-            }
+        blockCount += 1;
+        transactionCount += transactions.length;
+        receiptCount += transactionReceipts.length;
+        traceCount += traces.length;
 
-            case "trace":
-            case "transfer": {
-              await syncTraceOrTransferFilter(
-                filter as TraceFilter | TransferFilter,
-                interval,
-                context,
-              );
-              break;
-            }
-          }
+        // Free memory of all unused transactions
+        block.transactions = transactions;
 
-          await blockPromise;
-        }),
+        await promiseAllSettledWithThrow([
+          syncStore.insertBlocks({ blocks: [block], chainId: args.chain.id }),
+          syncStore.insertTransactions({
+            transactions,
+            chainId: args.chain.id,
+          }),
+          syncStore.insertTransactionReceipts({
+            transactionReceipts,
+            chainId: args.chain.id,
+          }),
+          syncStore.insertTraces({
+            traces: traces.map((trace) => ({
+              trace,
+              block: block!,
+              transaction: transactionsByHash.get(trace.transactionHash)!,
+            })),
+            chainId: args.chain.id,
+          }),
+          syncStore.insertLogs({ logs, chainId: args.chain.id }),
+        ]);
+      };
+
+      let blockCount = 0;
+      let transactionCount = 0;
+      let receiptCount = 0;
+      let traceCount = 0;
+
+      // Same memory usage as `sync-realtime`.
+      const MAX_BLOCKS_IN_MEM = Math.max(
+        args.chain.finalityBlockCount * 2,
+        100,
       );
 
-      const blocks = await Promise.all(blockCache.values());
+      if (requiredIntervals.length > 0) {
+        const queue = createQueue({
+          browser: false,
+          initialStart: true,
+          concurrency: MAX_BLOCKS_IN_MEM,
+          worker: syncBlockData,
+        });
 
-      await Promise.all([
-        args.syncStore.insertBlocks(
-          { blocks, chainId: args.chain.id },
-          context,
-        ),
-        args.syncStore.insertTransactions(
-          {
-            transactions: blocks.flatMap((block) =>
-              block.transactions.filter(({ hash }) =>
-                transactionsCache.has(hash),
-              ),
-            ),
-            chainId: args.chain.id,
-          },
-          context,
-        ),
-        ...Array.from(childAddressesCache.entries()).map(
-          ([factory, childAddresses]) =>
-            args.syncStore.insertChildAddresses(
-              {
-                factory,
-                childAddresses,
-                chainId: args.chain.id,
-              },
-              context,
-            ),
-        ),
-      ]);
-
-      // Add corresponding intervals to the sync-store
-      // Note: this should happen after so the database doesn't become corrupted
-      if (args.chain.disableCache === false) {
-        await args.syncStore.insertIntervals(
-          {
-            intervals: intervalsToSync,
-            chainId: args.chain.id,
-          },
-          context,
+        await Promise.all(
+          intervalRange(interval).map((blockNumber) => queue.add(blockNumber)),
         );
       }
 
-      let childAddressCount = 0;
-      for (const childAddresses of childAddressesCache.values()) {
-        childAddressCount += childAddresses.size;
-      }
-
       args.common.logger.debug(
         {
           msg: "Fetched block data",
           chain: args.chain.name,
           chain_id: args.chain.id,
-          block_range: JSON.stringify(_interval),
-          block_count: blockCache.size,
-          transaction_count: transactionsCache.size,
-          // receipt_count:
-          // log_count:
-          trace_count: traceCache.size,
-          child_address_count: childAddressCount,
+          block_range: JSON.stringify(interval),
+          block_count: blockCount,
+          transaction_count: transactionCount,
+          receipt_count: receiptCount,
+          trace_count: traceCount,
           duration: endClock(),
         },
         ["chain", "block_range"],
       );
 
-      blockCache.clear();
-      traceCache.clear();
-      transactionsCache.clear();
-      blockReceiptsCache.clear();
-      transactionReceiptsCache.clear();
-      childAddressesCache.clear();
-
-      return latestBlock;
+      return closestToTipBlock;
     },
   };
 };