ponder 0.17.12 → 0.18.0

package/src/sync-historical/index.ts

@@ -14,13 +14,13 @@ import type {
   BlockFilter,
   Chain,
   Factory,
+  FactoryFragmentId,
   FactoryId,
   LogFilter,
   SyncBlock,
   SyncLog,
   SyncTrace,
   SyncTransaction,
-  SyncTransactionReceipt,
   TraceFilter,
   TransactionFilter,
   TransferFilter,
@@ -28,9 +28,8 @@ import type {
 import {
   debug_traceBlockByNumber,
   eth_getBlockByNumber,
-  eth_getBlockReceipts,
   eth_getLogs,
-  eth_getTransactionReceipt,
+  eth_getTransactionReceipts,
   validateLogsAndBlock,
   validateReceiptsAndBlock,
   validateTracesAndBlock,
@@ -50,8 +49,8 @@ import {
   isTransferFilterMatched,
   mergeLogFiltersToRequests,
 } from "@/runtime/filter.js";
+import { getFactoryFragmentId } from "@/runtime/fragments.js";
 import type {
-  ChildAddresses,
   IntervalWithFactory,
   IntervalWithFilter,
 } from "@/runtime/index.js";
@@ -100,11 +99,6 @@ type CreateHistoricalSyncParameters = {
 export const createHistoricalSync = (
   args: CreateHistoricalSyncParameters,
 ): HistoricalSync => {
-  /**
-   * Flag to fetch transaction receipts through eth_getBlockReceipts (true) or eth_getTransactionReceipt (false)
-   */
-  let isBlockReceipts = true;
-
   /**
    * Data about the range passed to "eth_getLogs" share among all log
    * filters and log factories.
@@ -222,95 +216,16 @@ export const createHistoricalSync = (
     return logs;
   };
 
-  const syncTransactionReceipts = async (
-    block: SyncBlock,
-    transactionHashes: Set<Hash>,
-    context?: Parameters<Rpc["request"]>[1],
-  ): Promise<SyncTransactionReceipt[]> => {
-    if (transactionHashes.size === 0) {
-      return [];
-    }
-
-    if (isBlockReceipts === false) {
-      const transactionReceipts = await Promise.all(
-        Array.from(transactionHashes).map(async (hash) => {
-          const receipt = await eth_getTransactionReceipt(
-            args.rpc,
-            [hash],
-            context,
-          );
-
-          validateReceiptsAndBlock(
-            [receipt],
-            block,
-            {
-              method: "eth_getTransactionReceipt",
-              params: [hash],
-            },
-            {
-              method: "eth_getBlockByNumber",
-              params: [block.number, true],
-            },
-          );
-
-          return receipt;
-        }),
-      );
-
-      return transactionReceipts;
-    }
-
-    let blockReceipts: SyncTransactionReceipt[];
-    try {
-      blockReceipts = await eth_getBlockReceipts(
-        args.rpc,
-        [block.hash],
-        context,
-      );
-    } catch (_error) {
-      const error = _error as Error;
-      args.common.logger.warn({
-        msg: "Caught eth_getBlockReceipts error, switching to eth_getTransactionReceipt method",
-        action: "fetch_block_data",
-        chain: args.chain.name,
-        chain_id: args.chain.id,
-        error,
-      });
-
-      isBlockReceipts = false;
-      return syncTransactionReceipts(block, transactionHashes, context);
-    }
-
-    validateReceiptsAndBlock(
-      blockReceipts,
-      block,
-      {
-        method: "eth_getBlockReceipts",
-        params: [block.hash],
-      },
-      {
-        method: "eth_getBlockByNumber",
-        params: [block.number, true],
-      },
-    );
-
-    const transactionReceipts = blockReceipts.filter((receipt) =>
-      transactionHashes.has(receipt.transactionHash),
-    );
-
-    return transactionReceipts;
-  };
-
   /**
    * Fetch child addresses for `factory` within `interval`
    *
-   * @dev Newly fetched child addresses are added into `args.childAddresses`
+   * @dev Fetched child addresses are added into `args.childAddresses`
    */
   const syncAddressFactory = async (
     factory: Factory,
     interval: Interval,
     context?: Parameters<Rpc["request"]>[1],
-  ): Promise<Map<Address, number>> => {
+  ): Promise<Map<FactoryFragmentId, Map<Address, number>>> => {
     const logs = await syncLogsDynamic(
       {
         address: factory.address,
@@ -321,7 +236,7 @@ export const createHistoricalSync = (
       context,
     );
 
-    const childAddresses = new Map<Address, number>();
+    const childAddresses = new Map<FactoryFragmentId, Map<Address, number>>();
     const factoryChildAddresses = args.childAddresses.get(factory.id)!;
 
     const childAddressDecodeFailureIds = new Set<string>();
@@ -355,15 +270,31 @@ export const createHistoricalSync = (
             throw error;
           }
         }
-        const existingBlockNumber = factoryChildAddresses.get(address);
-        const newBlockNumber = hexToNumber(log.blockNumber);
+        const blockNumber = hexToNumber(log.blockNumber);
+
+        // Note: Child addresses are stored per parent, so a child address that is
+        // already known from a different parent must also be stored for this parent.
+
+        const fragmentId = getFactoryFragmentId(factory, log.address);
+        if (childAddresses.has(fragmentId) === false) {
+          childAddresses.set(fragmentId, new Map());
+        }
+        const fragmentChildAddresses = childAddresses.get(fragmentId)!;
+
+        const fragmentBlockNumber = fragmentChildAddresses.get(address);
+        if (
+          fragmentBlockNumber === undefined ||
+          fragmentBlockNumber > blockNumber
+        ) {
+          fragmentChildAddresses.set(address, blockNumber);
+        }
 
+        const existingBlockNumber = factoryChildAddresses.get(address);
         if (
           existingBlockNumber === undefined ||
-          existingBlockNumber > newBlockNumber
+          existingBlockNumber > blockNumber
         ) {
-          childAddresses.set(address, newBlockNumber);
-          factoryChildAddresses.set(address, newBlockNumber);
+          factoryChildAddresses.set(address, blockNumber);
         }
       }
     }
@@ -390,7 +321,7 @@ export const createHistoricalSync = (
         logger: args.common.logger.child({ action: "fetch_block_data" }),
       };
       const endClock = startClock();
-      const childAddresses: ChildAddresses = new Map();
+      const childAddresses = new Map<FactoryFragmentId, Map<Address, number>>();
 
       // Dedupe factory intervals by factory id
 
@@ -418,10 +349,33 @@ export const createHistoricalSync = (
 
       await Promise.all(
         requiredFactoryIntervals.map(async ({ factory, interval }) => {
-          childAddresses.set(
-            factory.id,
-            await syncAddressFactory(factory, interval, context)!,
+          const factoryChildAddresses = await syncAddressFactory(
+            factory,
+            interval,
+            context,
           );
+
+          // Note: Factories can share a `factory_log` fragment.
+
+          for (const [
+            fragmentId,
+            fragmentChildAddresses,
+          ] of factoryChildAddresses) {
+            if (childAddresses.has(fragmentId) === false) {
+              childAddresses.set(fragmentId, fragmentChildAddresses);
+              continue;
+            }
+
+            const existing = childAddresses.get(fragmentId)!;
+            for (const [address, blockNumber] of fragmentChildAddresses) {
+              if (
+                existing.has(address) === false ||
+                existing.get(address)! > blockNumber
+              ) {
+                existing.set(address, blockNumber);
+              }
+            }
+          }
         }),
       );
 
@@ -478,18 +432,9 @@ export const createHistoricalSync = (
         ["chain", "block_range"],
       );
 
-      await promiseAllSettledWithThrow(
-        Array.from(childAddresses.entries()).map(
-          ([factoryId, childAddresses]) =>
-            syncStore.insertChildAddresses(
-              {
-                factory: factoryIntervalsById.get(factoryId)!.factory,
-                childAddresses,
-                chainId: args.chain.id,
-              },
-              context,
-            ),
-        ),
+      await syncStore.insertChildAddresses(
+        { childAddresses, chainId: args.chain.id },
+        context,
       );
 
       return logs;
@@ -839,9 +784,24 @@ export const createHistoricalSync = (
         // Transaction Receipts
         ////////
 
-        const transactionReceipts = await syncTransactionReceipts(
-          block,
-          requiredTransactionReceipts,
+        const receiptResponses = await eth_getTransactionReceipts(
+          args.rpc,
+          {
+            blockHash: block.hash,
+            transactionHashes: requiredTransactionReceipts,
+          },
+          context,
+        );
+        const transactionReceipts = receiptResponses.flatMap(
+          ({ receipts, request }) => {
+            validateReceiptsAndBlock(receipts, block, request, {
+              method: "eth_getBlockByNumber",
+              params: [block.number, true],
+            });
+            return receipts.filter((receipt) =>
+              requiredTransactionReceipts.has(receipt.transactionHash),
+            );
+          },
         );
 
         blockCount += 1;
@@ -879,11 +839,7 @@ export const createHistoricalSync = (
       let receiptCount = 0;
       let traceCount = 0;
 
-      // Same memory usage as `sync-realtime`.
-      const MAX_BLOCKS_IN_MEM = Math.max(
-        args.chain.finalityBlockCount * 2,
-        100,
-      );
+      const MAX_BLOCKS_IN_MEM = 100;
 
       if (requiredIntervals.length > 0) {
         const queue = createQueue({

package/src/sync-realtime/index.ts

@@ -12,6 +12,7 @@ import type {
   Chain,
   EventCallback,
   Factory,
+  FactoryFragmentId,
   FactoryId,
   Filter,
   LightBlock,
@@ -30,9 +31,8 @@ import {
   debug_traceBlockByHash,
   eth_getBlockByHash,
   eth_getBlockByNumber,
-  eth_getBlockReceipts,
   eth_getLogs,
-  eth_getTransactionReceipt,
+  eth_getTransactionReceipts,
   validateLogsAndBlock,
   validateReceiptsAndBlock,
   validateTracesAndBlock,
@@ -51,6 +51,10 @@ import {
   isTransactionFilterMatched,
   isTransferFilterMatched,
 } from "@/runtime/filter.js";
+import {
+  getFactoryFragmentId,
+  getFactoryFragmentIds,
+} from "@/runtime/fragments.js";
 import type { SyncProgress } from "@/runtime/index.js";
 import { isAsyncExecutionChain } from "@/utils/finality.js";
 import { createLock } from "@/utils/mutex.js";
@@ -79,7 +83,8 @@ export type BlockWithEventData = {
   transactionReceipts: SyncTransactionReceipt[];
   logs: SyncLog[];
   traces: SyncTrace[];
-  childAddresses: Map<Factory, Set<Address>>;
+  /** Child addresses found in the block per `factory_log` fragment ID. */
+  childAddresses: Map<FactoryFragmentId, Set<Address>>;
 };
 
 export type RealtimeSyncEvent =
@@ -106,7 +111,6 @@ const MAX_QUEUED_BLOCKS = 50;
 export const createRealtimeSync = (
   args: CreateRealtimeSyncParameters,
 ): RealtimeSync => {
-  let isBlockReceipts = true;
   let finalizedBlock: LightBlock = args.syncProgress.finalized;
   const childAddresses = args.childAddresses;
   /** Annotates `childAddresses` for efficient lookup by block number */
@@ -129,6 +133,7 @@ export const createRealtimeSync = (
 
   const factories: Factory[] = [];
   const factoryIds = new Set<FactoryId>();
+  const factoryFragmentIds = new Map<Factory, FactoryFragmentId[]>();
   const logFilters: LogFilter[] = [];
   const traceFilters: TraceFilter[] = [];
   const transactionFilters: TransactionFilter[] = [];
@@ -167,104 +172,10 @@ export const createRealtimeSync = (
       if (factoryIds.has(factory.id)) continue;
       factoryIds.add(factory.id);
       factories.push(factory);
+      factoryFragmentIds.set(factory, getFactoryFragmentIds(factory));
     }
   }
 
-  const syncTransactionReceipts = async (
-    block: SyncBlock,
-    transactionHashes: Set<Hash>,
-    ethGetBlockMethod: "eth_getBlockByHash" | "eth_getBlockByNumber",
-    context?: Parameters<Rpc["request"]>[1],
-  ): Promise<SyncTransactionReceipt[]> => {
-    if (transactionHashes.size === 0) {
-      return [];
-    }
-
-    if (isBlockReceipts === false) {
-      const transactionReceipts = await Promise.all(
-        Array.from(transactionHashes).map(async (hash) => {
-          const receipt = await eth_getTransactionReceipt(
-            args.rpc,
-            [hash],
-            context,
-          );
-
-          validateReceiptsAndBlock(
-            [receipt],
-            block,
-            {
-              method: "eth_getTransactionReceipt",
-              params: [hash],
-            },
-            ethGetBlockMethod === "eth_getBlockByNumber"
-              ? {
-                  method: "eth_getBlockByNumber",
-                  params: [block.number, true],
-                }
-              : {
-                  method: "eth_getBlockByHash",
-                  params: [block.hash, true],
-                },
-          );
-
-          return receipt;
-        }),
-      );
-
-      return transactionReceipts;
-    }
-
-    let blockReceipts: SyncTransactionReceipt[];
-    try {
-      blockReceipts = await eth_getBlockReceipts(
-        args.rpc,
-        [block.hash],
-        context,
-      );
-    } catch (_error) {
-      const error = _error as Error;
-      args.common.logger.warn({
-        msg: "Caught eth_getBlockReceipts error, switching to eth_getTransactionReceipt method",
-        action: "fetch block data",
-        chain: args.chain.name,
-        chain_id: args.chain.id,
-        error,
-      });
-
-      isBlockReceipts = false;
-      return syncTransactionReceipts(
-        block,
-        transactionHashes,
-        ethGetBlockMethod,
-        context,
-      );
-    }
-
-    validateReceiptsAndBlock(
-      blockReceipts,
-      block,
-      {
-        method: "eth_getBlockReceipts",
-        params: [block.hash],
-      },
-      ethGetBlockMethod === "eth_getBlockByNumber"
-        ? {
-            method: "eth_getBlockByNumber",
-            params: [block.number, true],
-          }
-        : {
-            method: "eth_getBlockByHash",
-            params: [block.hash, true],
-          },
-    );
-
-    const transactionReceipts = blockReceipts.filter((receipt) =>
-      transactionHashes.has(receipt.transactionHash),
-    );
-
-    return transactionReceipts;
-  };
-
   const getLatestUnfinalizedBlock = () => {
     if (unfinalizedBlocks.length === 0) {
       return finalizedBlock;
@@ -475,14 +386,13 @@ export const createRealtimeSync = (
     ////////
 
     // Record `blockChildAddresses` that contain factory child addresses
-    const blockChildAddresses = new Map<Factory, Set<Address>>();
+    const blockChildAddresses: BlockWithEventData["childAddresses"] = new Map();
 
     const childAddressDecodeFailureIds = new Set<string>();
     let childAddressDecodeFailureCount = 0;
     let childAddressDecodeSuccessCount = 0;
 
     for (const factory of factories) {
-      blockChildAddresses.set(factory, new Set<Address>());
       for (const log of logs) {
         if (isLogFactoryMatched({ factory, log })) {
           let address: Address;
@@ -510,7 +420,11 @@ export const createRealtimeSync = (
               throw error;
             }
           }
-          blockChildAddresses.get(factory)!.add(address);
+          const fragmentId = getFactoryFragmentId(factory, log.address);
+          if (blockChildAddresses.has(fragmentId) === false) {
+            blockChildAddresses.set(fragmentId, new Set());
+          }
+          blockChildAddresses.get(fragmentId)!.add(address);
         }
       }
     }
@@ -659,12 +573,26 @@ export const createRealtimeSync = (
     // Transaction Receipts
     ////////
 
-    const transactionReceipts = await syncTransactionReceipts(
-      block,
-      requiredTransactionReceipts,
-      ethGetBlockMethod,
+    const receiptResponses = await eth_getTransactionReceipts(
+      args.rpc,
+      { blockHash: block.hash, transactionHashes: requiredTransactionReceipts },
       context,
     );
+    const transactionReceipts = receiptResponses.flatMap(
+      ({ receipts, request }) => {
+        validateReceiptsAndBlock(
+          receipts,
+          block,
+          request,
+          ethGetBlockMethod === "eth_getBlockByNumber"
+            ? { method: "eth_getBlockByNumber", params: [block.number, true] }
+            : { method: "eth_getBlockByHash", params: [block.hash, true] },
+        );
+        return receipts.filter((receipt) =>
+          requiredTransactionReceipts.has(receipt.transactionHash),
+        );
+      },
+    );
 
     let childAddressCount = 0;
     for (const childAddresses of blockChildAddresses.values()) {
@@ -712,15 +640,17 @@ export const createRealtimeSync = (
     matchedFilters: Set<Filter>;
   } => {
     // Update `childAddresses`
-    for (const factory of factories) {
-      const knownAddresses = childAddresses.get(factory.id)!;
-      const blockAddresses = blockChildAddresses.get(factory)!;
-      for (const address of blockAddresses) {
-        // Retain only addresses first discovered in this block in the persistence and reorg delta.
-        if (knownAddresses.has(address)) {
-          blockAddresses.delete(address);
-        } else {
-          knownAddresses.set(address, hexToNumber(block.number));
+    if (blockChildAddresses.size > 0) {
+      for (const factory of factories) {
+        const knownAddresses = childAddresses.get(factory.id)!;
+        for (const fragmentId of factoryFragmentIds.get(factory)!) {
+          const blockAddresses = blockChildAddresses.get(fragmentId);
+          if (blockAddresses === undefined) continue;
+          for (const address of blockAddresses) {
+            if (knownAddresses.has(address) === false) {
+              knownAddresses.set(address, hexToNumber(block.number));
+            }
+          }
         }
       }
     }
@@ -964,15 +894,21 @@ export const createRealtimeSync = (
 
     // remove reorged blocks from `childAddresses`
     for (const block of reorgedBlocks) {
+      const blockNumber = hexToNumber(block.number);
+      const blockChildAddresses = childAddressesPerBlock.get(blockNumber)!;
       for (const factory of factories) {
-        const addresses = childAddressesPerBlock
-          .get(hexToNumber(block.number))!
-          .get(factory)!;
-        for (const address of addresses) {
-          childAddresses.get(factory.id)!.delete(address);
+        const knownAddresses = childAddresses.get(factory.id)!;
+        for (const fragmentId of factoryFragmentIds.get(factory)!) {
+          const blockAddresses = blockChildAddresses.get(fragmentId);
+          if (blockAddresses === undefined) continue;
+          for (const address of blockAddresses) {
+            if (knownAddresses.get(address) === blockNumber) {
+              knownAddresses.delete(address);
+            }
+          }
         }
       }
-      childAddressesPerBlock.delete(hexToNumber(block.number));
+      childAddressesPerBlock.delete(blockNumber);
     }
 
     return {
@@ -1156,20 +1092,24 @@ export const createRealtimeSync = (
       blockCallback,
     };
 
-    // Determine if a new range has become finalized by evaluating if the
-    // latest block number is 2 * finalityBlockCount >= finalized block number.
-    // Essentially, there is a range the width of finalityBlockCount that is entirely
-    // finalized.
-
+    // Finalize the oldest contiguous blocks that are outside the timestamp
+    // based reorg window.
     const blockMovesFinality =
-      hexToNumber(block.number) >=
-      hexToNumber(finalizedBlock.number) + 2 * args.chain.finalityBlockCount;
+      hexToNumber(block.timestamp) -
+        hexToNumber(unfinalizedBlocks[0]!.timestamp) >=
+      args.chain.reorgWindow;
     if (blockMovesFinality) {
-      const pendingFinalizedBlock = unfinalizedBlocks.find(
-        (lb) =>
-          hexToNumber(lb.number) ===
-          hexToNumber(block.number) - args.chain.finalityBlockCount,
-      )!;
+      let pendingFinalizedBlock = unfinalizedBlocks[0]!;
+      for (const candidate of unfinalizedBlocks) {
+        if (
+          hexToNumber(block.timestamp) - hexToNumber(candidate.timestamp) >=
+          args.chain.reorgWindow
+        ) {
+          pendingFinalizedBlock = candidate;
+        } else {
+          break;
+        }
+      }
 
       args.common.logger.debug({
         msg: "Removed finalized blocks from local chain",

package/src/sync-store/encode.ts

@@ -136,7 +136,7 @@ export const encodeTrace = ({
   chainId: BigInt(chainId),
   blockNumber: hexToBigInt(block.number),
   transactionIndex: hexToNumber(transaction.transactionIndex),
-  traceIndex: trace.trace.index,
+  traceAddress: trace.trace.traceAddress,
   from: toLowerCase(trace.trace.from),
   to: trace.trace.to ? toLowerCase(trace.trace.to) : null,
   input: trace.trace.input,
@@ -145,9 +145,4 @@ export const encodeTrace = ({
   type: trace.trace.type,
   gas: hexToBigInt(trace.trace.gas),
   gasUsed: hexToBigInt(trace.trace.gasUsed),
-  error: trace.trace.error ? trace.trace.error.replace(/\0/g, "") : null,
-  revertReason: trace.trace.revertReason
-    ? trace.trace.revertReason.replace(/\0/g, "")
-    : null,
-  subcalls: trace.trace.subcalls,
 });