ponder 0.17.6 → 0.17.8

package/src/runtime/filter.ts

@@ -1,4 +1,4 @@
-import { type Address, hexToNumber } from "viem";
+import { type Address, type Hex, hexToNumber, numberToHex } from "viem";
 import type {
   BlockFilter,
   Factory,
@@ -23,6 +23,8 @@ import type {
   TransactionFilter,
   TransferFilter,
 } from "@/internal/types.js";
+import { type RequestParameters, sanitizeLogTopics } from "@/rpc/index.js";
+import type { ChildAddresses, IntervalWithFilter } from "@/runtime/index.js";
 import type {
   Block,
   Log,
@@ -30,6 +32,7 @@ import type {
   Transaction,
   TransactionReceipt,
 } from "@/types/eth.js";
+import type { Interval } from "@/utils/interval.js";
 import { toLowerCase } from "@/utils/lowercase.js";
 
 /** Returns true if `address` is an address filter. */
@@ -46,6 +49,179 @@ export const isAddressFactory = (
   return Array.isArray(address) ? isAddressFactory(address[0]) : true;
 };
 
+type EthGetLogsRequest = Extract<
+  RequestParameters,
+  { method: "eth_getLogs" }
+> & {
+  params: [
+    Extract<
+      Extract<RequestParameters, { method: "eth_getLogs" }>["params"][0],
+      { blockHash?: undefined }
+    > & { fromBlock: Hex; toBlock: Hex },
+  ];
+};
+
+const logRequestKeys = [
+  "address",
+  "topic0",
+  "topic1",
+  "topic2",
+  "topic3",
+] as const;
+
+const getLogRequestValue = (
+  request: EthGetLogsRequest,
+  key: (typeof logRequestKeys)[number],
+): Hex | Hex[] | undefined => {
+  if (key === "address") return request.params[0].address;
+
+  const topic = request.params[0].topics?.[Number(key.slice(-1))];
+  return topic === null ? undefined : topic;
+};
+
+/** Returns `true` if `left` and `right` overlap by at least 80% of the smaller interval's length. */
+const hasSufficientLogIntervalOverlap = (
+  left: Interval,
+  right: Interval,
+): boolean => {
+  const overlapStart = Math.max(left[0], right[0]);
+  const overlapEnd = Math.min(left[1], right[1]);
+  if (overlapStart > overlapEnd) return false;
+
+  const overlapLength = overlapEnd - overlapStart + 1;
+  const smallerIntervalLength = Math.min(
+    left[1] - left[0] + 1,
+    right[1] - right[0] + 1,
+  );
+
+  return overlapLength / smallerIntervalLength >= 0.8;
+};
+
+const mergeLogFilterValue = <value extends Hex>(
+  left: value | value[] | undefined,
+  right: value | value[] | undefined,
+): value[] | undefined =>
+  left === undefined || right === undefined
+    ? undefined
+    : [
+        ...new Set([
+          ...(Array.isArray(left) ? left : [left]),
+          ...(Array.isArray(right) ? right : [right]),
+        ]),
+      ];
+
+/**
+ * Merge `LogFilter`s into the smallest possible set of "eth_getLogs" requests.
+ *
+ * @dev Two filters are merged into a single request when:
+ *   1. their required intervals overlap by at least 80% (of the smaller
+ *      interval's length), and
+ *   2. at most one of `address`, `topic0`, `topic1`, `topic2`, or `topic3`
+ *      differs between them.
+ * The one differing dimension (if any) is unioned into an array.
+ *
+ * @dev Factory addresses are resolved into concrete child addresses (or
+ * `undefined`, above `factoryAddressCountThreshold`) before merging, so
+ * requests originating from different factories can still be merged.
+ */
+export const mergeLogFiltersToRequests = (
+  filters: IntervalWithFilter[],
+  childAddresses: ChildAddresses,
+  factoryAddressCountThreshold: number,
+): EthGetLogsRequest[] => {
+  const requests: EthGetLogsRequest[] = [];
+
+  for (const { filter, interval } of filters) {
+    if (filter.type !== "log") continue;
+
+    let address: Hex | Hex[] | undefined;
+    if (isAddressFactory(filter.address)) {
+      const addresses = childAddresses.get(filter.address.id)!;
+      if (addresses.size === 0) continue;
+
+      address =
+        addresses.size >= factoryAddressCountThreshold
+          ? undefined
+          : Array.from(addresses.keys());
+    } else {
+      address = filter.address;
+    }
+
+    const candidate: EthGetLogsRequest = {
+      method: "eth_getLogs",
+      params: [
+        {
+          address,
+          topics: sanitizeLogTopics([
+            filter.topic0,
+            filter.topic1 ?? null,
+            filter.topic2 ?? null,
+            filter.topic3 ?? null,
+          ]),
+          fromBlock: numberToHex(interval[0]),
+          toBlock: numberToHex(interval[1]),
+        },
+      ],
+    };
+
+    let isMerged = false;
+
+    for (const request of requests) {
+      if (
+        hasSufficientLogIntervalOverlap(
+          [
+            hexToNumber(request.params[0].fromBlock),
+            hexToNumber(request.params[0].toBlock),
+          ],
+          interval,
+        ) === false
+      ) {
+        continue;
+      }
+
+      const numberOfDifferences = logRequestKeys.filter(
+        (key) =>
+          JSON.stringify(getLogRequestValue(request, key)) !==
+          JSON.stringify(getLogRequestValue(candidate, key)),
+      ).length;
+      if (numberOfDifferences > 1) continue;
+
+      const requestParams = request.params[0];
+      for (const key of logRequestKeys) {
+        const requestValue = getLogRequestValue(request, key);
+        const candidateValue = getLogRequestValue(candidate, key);
+        if (JSON.stringify(requestValue) === JSON.stringify(candidateValue)) {
+          continue;
+        }
+
+        const mergedValue = mergeLogFilterValue(requestValue, candidateValue);
+        if (key === "address") {
+          requestParams.address = mergedValue;
+        } else {
+          const topics = [...(requestParams.topics ?? [])];
+          topics[Number(key.slice(-1))] = mergedValue ?? null;
+          requestParams.topics = sanitizeLogTopics(topics);
+        }
+      }
+
+      requestParams.fromBlock = numberToHex(
+        Math.min(hexToNumber(requestParams.fromBlock), interval[0]),
+      );
+      requestParams.toBlock = numberToHex(
+        Math.max(hexToNumber(requestParams.toBlock), interval[1]),
+      );
+      isMerged = true;
+      break;
+    }
+
+    if (isMerged === false) {
+      requests.push(candidate);
+    }
+  }
+
+  return requests;
+};
+
 export const getChildAddress = ({
   log,
   factory,

package/src/sync-historical/index.ts

@@ -4,7 +4,6 @@ import {
   type Hash,
   type Hex,
   hexToNumber,
-  type LogTopic,
   numberToHex,
   type RpcError,
   toHex,
@@ -37,7 +36,7 @@ import {
   validateTracesAndBlock,
   validateTransactionsAndBlock,
 } from "@/rpc/actions.js";
-import { type Rpc, sanitizeLogTopics } from "@/rpc/index.js";
+import type { RequestParameters, Rpc } from "@/rpc/index.js";
 import {
   getChildAddress,
   isAddressFactory,
@@ -49,6 +48,7 @@ import {
   isTraceFilterMatched,
   isTransactionFilterMatched,
   isTransferFilterMatched,
+  mergeLogFiltersToRequests,
 } from "@/runtime/filter.js";
 import type {
   ChildAddresses,
@@ -79,6 +79,7 @@ export type HistoricalSync = {
   }): Promise<SyncLog[]>;
   /**
    * Sync block data that must be queried for a single block (block, transactions, receipts, traces).
+   * @returns Closest-to-tip synced block.
    */
   syncBlockData(params: {
     interval: Interval;
@@ -120,48 +121,37 @@ export const createHistoricalSync = (
   // Helper functions for sync tasks
   ////////
 
-  type EthGetLogsParams = {
-    address: Address | Address[] | undefined;
-    topic0?: LogTopic;
-    topic1?: LogTopic;
-    topic2?: LogTopic;
-    topic3?: LogTopic;
-    interval: Interval;
-  };
-
   /**
    * Split "eth_getLogs" requests into ranges inferred from errors
    * and batch requests.
    */
   const syncLogsDynamic = async (
-    { address, topic0, topic1, topic2, topic3, interval }: EthGetLogsParams,
+    params: Extract<RequestParameters, { method: "eth_getLogs" }>["params"][0],
     context?: Parameters<Rpc["request"]>[1],
   ): Promise<SyncLog[]> => {
+    const { address, topics } = params;
+
     const intervals = getChunks({
-      interval,
+      interval: [
+        hexToNumber(params.fromBlock as Hex),
+        hexToNumber(params.toBlock as Hex),
+      ],
       maxChunkSize:
         args.chain.ethGetLogsBlockRange ??
         logsRequestMetadata.confirmedRange ??
         logsRequestMetadata.estimatedRange,
     });
 
-    const topics = sanitizeLogTopics([
-      topic0 ?? null,
-      topic1 ?? null,
-      topic2 ?? null,
-      topic3 ?? null,
-    ])!;
-
     const logs = await Promise.all(
-      intervals.map((interval) =>
+      intervals.map(([fromBlock, toBlock]) =>
         eth_getLogs(
           args.rpc,
           [
             {
               address,
               topics,
-              fromBlock: numberToHex(interval[0]),
-              toBlock: numberToHex(interval[1]),
+              fromBlock: numberToHex(fromBlock),
+              toBlock: numberToHex(toBlock),
             },
           ],
           context,
@@ -177,8 +167,8 @@ export const createHistoricalSync = (
               {
                 address,
                 topics,
-                fromBlock: toHex(interval[0]),
-                toBlock: toHex(interval[1]),
+                fromBlock: toHex(fromBlock),
+                toBlock: toHex(toBlock),
               },
             ],
             error: error as RpcError,
@@ -205,7 +195,12 @@ export const createHistoricalSync = (
           };
 
           return syncLogsDynamic(
-            { address, topic0, topic1, topic2, topic3, interval },
+            {
+              address,
+              topics,
+              fromBlock: numberToHex(fromBlock),
+              toBlock: numberToHex(toBlock),
+            },
             context,
           );
         }),
@@ -318,14 +313,15 @@ export const createHistoricalSync = (
     const logs = await syncLogsDynamic(
       {
         address: factory.address,
-        topic0: factory.eventSelector,
-        interval,
+        topics: [factory.eventSelector],
+        fromBlock: numberToHex(interval[0]),
+        toBlock: numberToHex(interval[1]),
       },
       context,
     );
 
     const childAddresses = new Map<Address, number>();
-    const childAddressesRecord = args.childAddresses.get(factory.id)!;
+    const factoryChildAddresses = args.childAddresses.get(factory.id)!;
 
     const childAddressDecodeFailureIds = new Set<string>();
     let childAddressDecodeFailureCount = 0;
@@ -358,7 +354,7 @@ export const createHistoricalSync = (
             throw error;
           }
         }
-        const existingBlockNumber = childAddressesRecord.get(address);
+        const existingBlockNumber = factoryChildAddresses.get(address);
         const newBlockNumber = hexToNumber(log.blockNumber);
 
         if (
@@ -366,7 +362,7 @@ export const createHistoricalSync = (
           existingBlockNumber > newBlockNumber
         ) {
           childAddresses.set(address, newBlockNumber);
-          childAddressesRecord.set(address, newBlockNumber);
+          factoryChildAddresses.set(address, newBlockNumber);
         }
       }
     }
@@ -428,132 +424,17 @@ export const createHistoricalSync = (
         }),
       );
 
-      const mergedEthGetLogsParams: Map<string, EthGetLogsParams> = new Map();
-      const singleEthGetLogsParams: EthGetLogsParams[] = [];
-
-      for (const { filter, interval } of requiredIntervals) {
-        if (filter.type !== "log") continue;
-
-        const hasAddress = filter.address !== undefined;
-        const hasTopic1 = filter.topic1 !== undefined && filter.topic1 !== null;
-        const hasTopic2 = filter.topic2 !== undefined && filter.topic2 !== null;
-        const hasTopic3 = filter.topic3 !== undefined && filter.topic3 !== null;
-
-        if (hasAddress === false || hasTopic1 || hasTopic2 || hasTopic3) {
-          if (isAddressFactory(filter.address)) {
-            const childAddresses = args.childAddresses.get(filter.address.id)!;
-
-            if (childAddresses.size === 0) {
-              continue;
-            }
-
-            singleEthGetLogsParams.push({
-              address:
-                childAddresses.size >=
-                args.common.options.factoryAddressCountThreshold
-                  ? undefined
-                  : Array.from(childAddresses.keys()),
-              topic0: filter.topic0,
-              topic1: filter.topic1,
-              topic2: filter.topic2,
-              topic3: filter.topic3,
-              interval,
-            });
-          } else {
-            singleEthGetLogsParams.push({
-              address: filter.address,
-              topic0: filter.topic0,
-              topic1: filter.topic1,
-              topic2: filter.topic2,
-              topic3: filter.topic3,
-              interval,
-            });
-          }
-
-          continue;
-        }
-
-        let addressKey: string;
-        if (isAddressFactory(filter.address)) {
-          addressKey = filter.address.id;
-        } else if (Array.isArray(filter.address)) {
-          addressKey = filter.address.join("_");
-        } else {
-          addressKey = filter.address as Address;
-        }
-
-        if (mergedEthGetLogsParams.has(addressKey) === false) {
-          if (isAddressFactory(filter.address)) {
-            const childAddresses = args.childAddresses.get(filter.address.id)!;
-
-            if (childAddresses.size === 0) {
-              continue;
-            }
-
-            mergedEthGetLogsParams.set(addressKey, {
-              address:
-                childAddresses.size >=
-                args.common.options.factoryAddressCountThreshold
-                  ? undefined
-                  : Array.from(childAddresses.keys()),
-              topic0: filter.topic0,
-              topic1: filter.topic1,
-              topic2: filter.topic2,
-              topic3: filter.topic3,
-              interval,
-            });
-          } else {
-            mergedEthGetLogsParams.set(addressKey, {
-              address: filter.address,
-              topic0: filter.topic0,
-              topic1: filter.topic1,
-              topic2: filter.topic2,
-              topic3: filter.topic3,
-              interval,
-            });
-          }
-        } else {
-          const existingInterval =
-            mergedEthGetLogsParams.get(addressKey)!.interval;
-          const existingTopic0 = mergedEthGetLogsParams.get(addressKey)!
-            .topic0 as Hex | Hex[];
-
-          if (Array.isArray(existingTopic0)) {
-            if (existingTopic0.includes(filter.topic0) === false) {
-              mergedEthGetLogsParams.get(addressKey)!.topic0 = [
-                ...existingTopic0,
-                filter.topic0,
-              ];
-            }
-          } else {
-            if (existingTopic0 !== filter.topic0) {
-              mergedEthGetLogsParams.get(addressKey)!.topic0 = [
-                existingTopic0,
-                filter.topic0,
-              ];
-            }
-          }
-
-          mergedEthGetLogsParams.get(addressKey)!.interval = intervalBounds([
-            existingInterval,
-            interval,
-          ]);
-        }
-      }
-
-      const ethGetLogsParams = dedupe(
-        [
-          ...singleEthGetLogsParams,
-          ...Array.from(mergedEthGetLogsParams.values()),
-        ],
-        (params) => JSON.stringify(params),
+      const ethGetLogsRequests = mergeLogFiltersToRequests(
+        requiredIntervals,
+        args.childAddresses,
+        args.common.options.factoryAddressCountThreshold,
       );
 
       let logs: SyncLog[] = [];
 
       await Promise.all(
-        ethGetLogsParams.map(async (params) => {
-          const _logs = await syncLogsDynamic(params, context);
+        ethGetLogsRequests.map(async (request) => {
+          const _logs = await syncLogsDynamic(request.params[0], context);
           for (const log of _logs) {
             logs.push(log);
           }

package/src/sync-store/index.ts

@@ -351,11 +351,15 @@ export const createSyncStore = ({
         const batchSize = 200;
 
         for (let i = 0; i < queries.length; i += batchSize) {
+          const batch = queries.slice(i, i + batchSize);
           const _rows = await qb.wrap(
             { label: "select_intervals" },
-            () =>
+            () => {
+              if (batch.length === 1) return batch[0]!.execute();
+
               // @ts-expect-error
-              unionAll(...queries.slice(i, i + batchSize)),
+              return unionAll(...batch);
+            },
             context,
           );