ponder 0.18.0 → 0.18.2

package/src/sync-realtime/bloom.ts

@@ -1,102 +1,152 @@
-import { type Hex, hexToBytes, hexToNumber, keccak256 } from "viem";
-import type { LogFilter, SyncBlock } from "@/internal/types.js";
-import {
-  getFilterFromBlock,
-  getFilterToBlock,
-  isAddressFactory,
-} from "@/runtime/filter.js";
+import { type Hex, keccak256 } from "viem";
+import type { Factory, LogFilter } from "@/internal/types.js";
+import { isAddressFactory } from "@/runtime/filter.js";
 
 export const zeroLogsBloom =
   "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000";
 
 const BLOOM_SIZE_BYTES = 256;
 
-export const isInBloom = (_bloom: Hex, input: Hex): boolean => {
-  const bloom = hexToBytes(_bloom);
-  const hash = hexToBytes(keccak256(input));
-
-  for (const i of [0, 2, 4]) {
-    const bit = (hash[i + 1]! + (hash[i]! << 8)) & 0x7ff;
-    if (
-      (bloom[BLOOM_SIZE_BYTES - 1 - Math.floor(bit / 8)]! &
-        (1 << (bit % 8))) ===
-      0
-    )
-      return false;
+export type BloomBits = {
+  byte0: number;
+  mask0: number;
+  byte1: number;
+  mask1: number;
+  byte2: number;
+  mask2: number;
+};
+
+/**
+ * A `LogFilter` with each address and topic converted to `BloomBits`.
+ *
+ * `undefined` matches every bloom. An array matches if any item is in the bloom.
+ */
+export type LogFilterBloom = {
+  address: BloomBits[] | undefined;
+  topics: (BloomBits[] | undefined)[];
+};
+
+export const getBloomBits = (input: Hex): BloomBits => {
+  const hash = keccak256(input, "bytes");
+  const bytes = [0, 0, 0];
+  const masks = [0, 0, 0];
+
+  for (let i = 0; i < 3; i++) {
+    const bit = (hash[i * 2 + 1]! + (hash[i * 2]! << 8)) & 0x7ff;
+    bytes[i] = BLOOM_SIZE_BYTES - 1 - Math.floor(bit / 8);
+    masks[i] = 1 << (bit % 8);
   }
 
-  return true;
+  return {
+    byte0: bytes[0]!,
+    mask0: masks[0]!,
+    byte1: bytes[1]!,
+    mask1: masks[1]!,
+    byte2: bytes[2]!,
+    mask2: masks[2]!,
+  };
+};
+
+export const isBloomBitsInBloom = (
+  bloom: Uint8Array,
+  bits: BloomBits,
+): boolean =>
+  (bloom[bits.byte0]! & bits.mask0) !== 0 &&
+  (bloom[bits.byte1]! & bits.mask1) !== 0 &&
+  (bloom[bits.byte2]! & bits.mask2) !== 0;
+
+export const isInBloom = (bloom: Uint8Array, input: Hex): boolean =>
+  isBloomBitsInBloom(bloom, getBloomBits(input));
+
+const isAnyInBloom = (
+  bloom: Uint8Array,
+  bits: BloomBits[] | undefined,
+): boolean =>
+  bits === undefined || bits.some((b) => isBloomBitsInBloom(bloom, b));
+
+const toBloomBits = (
+  input: Hex | Hex[] | null | undefined,
+): BloomBits[] | undefined => {
+  if (input === null || input === undefined) return undefined;
+  if (Array.isArray(input)) return input.map(getBloomBits);
+  return [getBloomBits(input)];
+};
+
+/**
+ * Convert `filter` to a `LogFilterBloom`. Call once for each filter,
+ * so that the inputs are not hashed again for each block.
+ */
+export const getLogFilterBloom = (filter: LogFilter): LogFilterBloom => {
+  let address: BloomBits[] | undefined;
+
+  if (isAddressFactory(filter.address)) {
+    address = undefined;
+  } else if (Array.isArray(filter.address) && filter.address.length === 0) {
+    address = undefined;
+  } else {
+    address = toBloomBits(filter.address);
+  }
+
+  return {
+    address,
+    topics: [filter.topic0, filter.topic1, filter.topic2, filter.topic3].map(
+      toBloomBits,
+    ),
+  };
 };
 
 /**
  * Return true if `filter` is in `bloom`.
  *
- * A filter with an address of type `LogFactory` is matched
- * if the address filter is matched (new child contract) or the log
- * filter is matched (log on child contract).
+ * A filter with an address of type `LogFactory` matches all addresses.
+ * Use `isFactoryInBloom` to check for new child contracts.
  *
  * Note: False positives are possible.
  */
 export function isFilterInBloom({
-  block,
+  bloom,
   filter,
 }: {
-  block: Pick<SyncBlock, "number" | "logsBloom">;
-  filter: LogFilter;
+  bloom: Uint8Array;
+  filter: LogFilterBloom;
 }): boolean {
-  // Return `false` for out of range blocks
-  if (
-    hexToNumber(block.number) < getFilterFromBlock(filter) ||
-    hexToNumber(block.number) > getFilterToBlock(filter)
-  ) {
-    return false;
-  }
+  return (
+    isAnyInBloom(bloom, filter.address) &&
+    filter.topics.every((topic) => isAnyInBloom(bloom, topic))
+  );
+}
 
-  const isTopicsInBloom = [
-    filter.topic0,
-    filter.topic1,
-    filter.topic2,
-    filter.topic3,
-  ].every((topic) => {
-    if (topic === null || topic === undefined) {
-      return true;
-    } else if (Array.isArray(topic)) {
-      return topic.some((t) => isInBloom(block.logsBloom, t));
-    } else {
-      return isInBloom(block.logsBloom, topic);
-    }
-  });
-
-  let isAddressInBloom: boolean;
-
-  if (filter.address === undefined) isAddressInBloom = true;
-  else if (isAddressFactory(filter.address)) {
-    // Return true if the `Factory` is matched.
-    if (
-      (filter.address.address === undefined ||
-        (Array.isArray(filter.address.address)
-          ? filter.address.address.some((address) =>
-              isInBloom(block.logsBloom, address),
-            )
-          : isInBloom(block.logsBloom, filter.address.address))) &&
-      isInBloom(block.logsBloom, filter.address.eventSelector)
-    ) {
-      return true;
-    }
-
-    isAddressInBloom = true;
-  } else if (Array.isArray(filter.address)) {
-    if (filter.address.length === 0) {
-      isAddressInBloom = true;
-    } else {
-      isAddressInBloom = filter.address.some((address) =>
-        isInBloom(block.logsBloom, address),
-      );
-    }
-  } else {
-    // single address case
-    isAddressInBloom = isInBloom(block.logsBloom, filter.address);
-  }
+/**
+ * A `Factory` with its address and event selector converted to `BloomBits`.
+ */
+export type FactoryBloom = {
+  address: BloomBits[] | undefined;
+  eventSelector: BloomBits;
+};
 
-  return isAddressInBloom && isTopicsInBloom;
+/**
+ * Convert `factory` to a `FactoryBloom`. Call once for each factory,
+ * so that the inputs are not hashed again for each block.
+ */
+export const getFactoryBloom = (factory: Factory): FactoryBloom => ({
+  address: toBloomBits(factory.address),
+  eventSelector: getBloomBits(factory.eventSelector),
+});
+
+/**
+ * Return true if a log that matches `factory` can be in `bloom`.
+ *
+ * Note: False positives are possible.
+ */
+export function isFactoryInBloom({
+  bloom,
+  factory,
+}: {
+  bloom: Uint8Array;
+  factory: FactoryBloom;
+}): boolean {
+  return (
+    isAnyInBloom(bloom, factory.address) &&
+    isBloomBitsInBloom(bloom, factory.eventSelector)
+  );
 }

package/src/sync-realtime/index.ts

@@ -1,6 +1,8 @@
 import {
   type Address,
   type Hash,
+  type Hex,
+  hexToBytes,
   hexToNumber,
   numberToHex,
   zeroHash,
@@ -45,11 +47,13 @@ import {
   isAddressFactory,
   isAddressMatched,
   isBlockFilterMatched,
+  isBlockInFilter,
   isLogFactoryMatched,
   isLogFilterMatched,
   isTraceFilterMatched,
   isTransactionFilterMatched,
   isTransferFilterMatched,
+  mergeLogFiltersToBlockRequest,
 } from "@/runtime/filter.js";
 import {
   getFactoryFragmentId,
@@ -60,7 +64,14 @@ import { isAsyncExecutionChain } from "@/utils/finality.js";
 import { createLock } from "@/utils/mutex.js";
 import { range } from "@/utils/range.js";
 import { startClock } from "@/utils/timer.js";
-import { isFilterInBloom, isInBloom, zeroLogsBloom } from "./bloom.js";
+import {
+  getFactoryBloom,
+  getLogFilterBloom,
+  isFactoryInBloom,
+  isFilterInBloom,
+  isInBloom,
+  zeroLogsBloom,
+} from "./bloom.js";
 
 export type RealtimeSync = {
   /**
@@ -176,6 +187,9 @@ export const createRealtimeSync = (
     }
   }
 
+  const logFilterBlooms = logFilters.map(getLogFilterBloom);
+  const factoryBlooms = factories.map(getFactoryBloom);
+
   const getLatestUnfinalizedBlock = () => {
     if (unfinalizedBlocks.length === 0) {
       return finalizedBlock;
@@ -211,33 +225,45 @@ export const createRealtimeSync = (
     // Logs
     ////////
 
-    // "eth_getLogs" calls can be skipped if no filters match `newHeadBlock.logsBloom`.
     // Async-execution chains cannot reliably use blooms to filter or validate logs.
+    // A zero `logsBloom` is treated as missing.
     const isAsyncExecution = isAsyncExecutionChain(args.chain.id);
-    const shouldRequestLogs = isAsyncExecution
-      ? logFilters.length > 0
-      : maybeBlockHeader.logsBloom === zeroLogsBloom ||
-        logFilters.some((filter) =>
-          isFilterInBloom({ block: maybeBlockHeader, filter }),
-        );
+    const blockNumber = hexToNumber(maybeBlockHeader.number);
+    const bloom = hexToBytes(maybeBlockHeader.logsBloom);
+
+    const logsRequestFilter = mergeLogFiltersToBlockRequest({
+      logFilters: logFilters.filter(
+        (logFilter, i) =>
+          isBlockInFilter(logFilter, blockNumber) &&
+          (isAsyncExecution ||
+            maybeBlockHeader.logsBloom === zeroLogsBloom ||
+            isFilterInBloom({ bloom, filter: logFilterBlooms[i]! })),
+      ),
+      factories: factories.filter(
+        (factory, i) =>
+          isBlockInFilter(factory, blockNumber) &&
+          (isAsyncExecution ||
+            maybeBlockHeader.logsBloom === zeroLogsBloom ||
+            isFactoryInBloom({ bloom, factory: factoryBlooms[i]! })),
+      ),
+    });
+    const shouldRequestLogs = logsRequestFilter !== undefined;
 
     let logs: SyncLog[] = [];
     if (shouldRequestLogs) {
+      const logsRequestParams = {
+        blockHash: maybeBlockHeader.hash,
+        address: logsRequestFilter.address,
+        topics: logsRequestFilter.topics,
+      };
+
       if (block === undefined) {
         [block, logs] = await Promise.all([
           eth_getBlockByHash(args.rpc, [maybeBlockHeader.hash, true], context),
-          eth_getLogs(
-            args.rpc,
-            [{ blockHash: maybeBlockHeader.hash }],
-            context,
-          ),
+          eth_getLogs(args.rpc, [logsRequestParams], context),
         ]);
       } else {
-        logs = await eth_getLogs(
-          args.rpc,
-          [{ blockHash: block.hash }],
-          context,
-        );
+        logs = await eth_getLogs(args.rpc, [logsRequestParams], context);
       }
 
       validateLogsAndBlock(
@@ -245,7 +271,7 @@ export const createRealtimeSync = (
         block,
         {
           method: "eth_getLogs",
-          params: [{ blockHash: block.hash }],
+          params: [logsRequestParams],
         },
         ethGetBlockMethod === "eth_getBlockByNumber"
           ? {
@@ -256,60 +282,8 @@ export const createRealtimeSync = (
               method: "eth_getBlockByHash",
               params: [block.hash, true],
             },
-        isAsyncExecutionChain(args.chain.id),
       );
 
-      if (isAsyncExecutionChain(args.chain.id) === false) {
-        // Note: Exact `logsBloom` validations were considered too strict to add to `validateLogsAndBlock`.
-        let isInvalidLogsBloom = false;
-        for (const log of logs) {
-          if (isInBloom(block.logsBloom, log.address) === false) {
-            isInvalidLogsBloom = true;
-          }
-
-          if (
-            log.topics[0] &&
-            isInBloom(block.logsBloom, log.topics[0]) === false
-          ) {
-            isInvalidLogsBloom = true;
-          }
-
-          if (
-            log.topics[1] &&
-            isInBloom(block.logsBloom, log.topics[1]) === false
-          ) {
-            isInvalidLogsBloom = true;
-          }
-
-          if (
-            log.topics[2] &&
-            isInBloom(block.logsBloom, log.topics[2]) === false
-          ) {
-            isInvalidLogsBloom = true;
-          }
-
-          if (
-            log.topics[3] &&
-            isInBloom(block.logsBloom, log.topics[3]) === false
-          ) {
-            isInvalidLogsBloom = true;
-          }
-
-          if (isInvalidLogsBloom) {
-            args.common.logger.warn({
-              msg: "Detected inconsistent RPC responses. Log not found in block.logsBloom.",
-              action: "fetch_block_data",
-              chain: args.chain.name,
-              chain_id: args.chain.id,
-              number: hexToNumber(block.number),
-              hash: block.hash,
-              logIndex: hexToNumber(log.logIndex),
-            });
-            break;
-          }
-        }
-      }
-
       for (const log of logs) {
         if (log.transactionHash === zeroHash) {
           args.common.logger.warn({
@@ -325,7 +299,10 @@ export const createRealtimeSync = (
       }
     }
 
-    if (shouldRequestLogs === false && logFilters.length > 0) {
+    if (
+      shouldRequestLogs === false &&
+      (logFilters.length > 0 || factories.length > 0)
+    ) {
       args.common.logger.trace({
         msg: "Skipped eth_getLogs request due to bloom filter result",
         action: "fetch_block_data",
@@ -387,6 +364,7 @@ export const createRealtimeSync = (
 
     // Record `blockChildAddresses` that contain factory child addresses
     const blockChildAddresses: BlockWithEventData["childAddresses"] = new Map();
+    const factoryLogs = new Set<SyncLog>();
 
     const childAddressDecodeFailureIds = new Set<string>();
     let childAddressDecodeFailureCount = 0;
@@ -395,6 +373,7 @@ export const createRealtimeSync = (
     for (const factory of factories) {
       for (const log of logs) {
         if (isLogFactoryMatched({ factory, log })) {
+          factoryLogs.add(log);
           let address: Address;
           try {
             address = getChildAddress({ log, factory });
@@ -462,6 +441,69 @@ export const createRealtimeSync = (
       return isMatched;
     });
 
+    if (isAsyncExecution === false && block !== undefined) {
+      // Note: Exact `logsBloom` validations were considered too strict to add to `validateLogsAndBlock`.
+      const logsBloom = hexToBytes(block.logsBloom);
+      const checkedInputs = new Set<Hex>();
+      let isInvalidLogsBloom = false;
+      for (const log of [...factoryLogs, ...logs]) {
+        if (
+          checkedInputs.has(log.address) === false &&
+          isInBloom(logsBloom, log.address) === false
+        ) {
+          isInvalidLogsBloom = true;
+        }
+
+        if (
+          log.topics[0] &&
+          checkedInputs.has(log.topics[0]) === false &&
+          isInBloom(logsBloom, log.topics[0]) === false
+        ) {
+          isInvalidLogsBloom = true;
+        }
+
+        if (
+          log.topics[1] &&
+          checkedInputs.has(log.topics[1]) === false &&
+          isInBloom(logsBloom, log.topics[1]) === false
+        ) {
+          isInvalidLogsBloom = true;
+        }
+
+        if (
+          log.topics[2] &&
+          checkedInputs.has(log.topics[2]) === false &&
+          isInBloom(logsBloom, log.topics[2]) === false
+        ) {
+          isInvalidLogsBloom = true;
+        }
+
+        if (
+          log.topics[3] &&
+          checkedInputs.has(log.topics[3]) === false &&
+          isInBloom(logsBloom, log.topics[3]) === false
+        ) {
+          isInvalidLogsBloom = true;
+        }
+
+        if (isInvalidLogsBloom) {
+          args.common.logger.warn({
+            msg: "Detected inconsistent RPC responses. Log not found in block.logsBloom.",
+            action: "fetch_block_data",
+            chain: args.chain.name,
+            chain_id: args.chain.id,
+            number: hexToNumber(block.number),
+            hash: block.hash,
+            logIndex: hexToNumber(log.logIndex),
+          });
+          break;
+        }
+
+        checkedInputs.add(log.address);
+        for (const topic of log.topics) checkedInputs.add(topic);
+      }
+    }
+
     // Initial weak trace filtering before full filtering with factory addresses in handleBlock
     traces = traces.filter((trace) => {
       let isMatched = false;

package/src/utils/copy.ts

@@ -5,75 +5,94 @@ import util from "node:util";
  */
 export const COPY_ON_WRITE = Symbol.for("ponder:copyOnWrite");
 
-const copyOnWriteCopies = new WeakMap<object, object>();
+/**
+ * Target of a copy-on-write proxy.
+ *
+ * Note: The handler is shared by all proxies, so the state of each proxy
+ * is stored in its target.
+ */
+class CopyOnWriteTarget {
+  obj: object;
+  copiedObject: object | undefined;
+
+  constructor(obj: object) {
+    this.obj = obj;
+    this.copiedObject = undefined;
+  }
 
-function inspectCopyOnWrite(this: object) {
-  return copyOnWriteCopies.get(this) ?? (this as any)[COPY_ON_WRITE] ?? this;
+  // Note: `this` is the proxy when called by `util.inspect`.
+  [util.inspect.custom]() {
+    // @ts-expect-error
+    return this[COPY_ON_WRITE] ?? this.copiedObject ?? this.obj;
+  }
 }
 
+const getCopiedObject = (target: CopyOnWriteTarget) => {
+  if (target.copiedObject === undefined) {
+    target.copiedObject = structuredClone(target.obj);
+  }
+  return target.copiedObject;
+};
+
+const copyOnWriteHandler: ProxyHandler<CopyOnWriteTarget> = {
+  get(target, prop, receiver) {
+    if (prop === COPY_ON_WRITE) {
+      return target.copiedObject ?? target.obj;
+    }
+    let result = Reflect.get(target.copiedObject ?? target.obj, prop, receiver);
+
+    if (
+      typeof result === "object" &&
+      result !== null &&
+      target.copiedObject === undefined
+    ) {
+      result = Reflect.get(getCopiedObject(target), prop, receiver);
+    }
+
+    return result;
+  },
+  set(target, prop, newValue, receiver) {
+    return Reflect.set(getCopiedObject(target), prop, newValue, receiver);
+  },
+  deleteProperty(target, prop) {
+    return Reflect.deleteProperty(getCopiedObject(target), prop);
+  },
+  defineProperty(target, prop, descriptor) {
+    return Reflect.defineProperty(getCopiedObject(target), prop, descriptor);
+  },
+  ownKeys(target) {
+    return Reflect.ownKeys(target.copiedObject ?? target.obj);
+  },
+  has(target, prop) {
+    return Reflect.has(target.copiedObject ?? target.obj, prop);
+  },
+  getOwnPropertyDescriptor(target, prop) {
+    return Reflect.getOwnPropertyDescriptor(
+      target.copiedObject ?? target.obj,
+      prop,
+    );
+  },
+  getPrototypeOf(target) {
+    return Reflect.getPrototypeOf(target.copiedObject ?? target.obj);
+  },
+};
+
 /**
- * Create a copy-on-write proxy for an object.
+ * Create a copy-on-write proxy for a plain object.
+ *
+ * @dev Arrays and other exotic objects are not supported. The proxy target
+ * is a `CopyOnWriteTarget`, not `obj`, so the proxy invariants break for
+ * non-configurable properties such as `Array.length`:
+ * `Object.keys()` and spread throw a `TypeError`, and `Array.isArray()`
+ * returns `false`.
  */
-export const copyOnWrite = <T extends object>(obj: T): T => {
-  let copiedObject: T | undefined;
-
-  Object.defineProperty(obj, util.inspect.custom, {
-    value: inspectCopyOnWrite,
-    configurable: true,
-  });
-
-  const proxy = new Proxy<T>(obj, {
-    get(target, prop, receiver) {
-      if (prop === COPY_ON_WRITE) {
-        return copiedObject ?? target;
-      }
-      let result = Reflect.get(copiedObject ?? target, prop, receiver);
-
-      if (
-        typeof result === "object" &&
-        result !== null &&
-        copiedObject === undefined
-      ) {
-        copiedObject = structuredClone(target);
-        copyOnWriteCopies.set(proxy, copiedObject);
-        result = Reflect.get(copiedObject, prop, receiver);
-      }
-
-      return result;
-    },
-    set(target, prop, newValue, receiver) {
-      if (copiedObject === undefined) {
-        copiedObject = structuredClone(target);
-        copyOnWriteCopies.set(proxy, copiedObject);
-      }
-      return Reflect.set(copiedObject, prop, newValue, receiver);
-    },
-    deleteProperty(target, prop) {
-      if (copiedObject === undefined) {
-        copiedObject = structuredClone(target);
-        copyOnWriteCopies.set(proxy, copiedObject);
-      }
-      return Reflect.deleteProperty(copiedObject, prop);
-    },
-    defineProperty(target, prop, descriptor) {
-      if (copiedObject === undefined) {
-        copiedObject = structuredClone(target);
-        copyOnWriteCopies.set(proxy, copiedObject);
-      }
-      return Reflect.defineProperty(copiedObject, prop, descriptor);
-    },
-    ownKeys(target) {
-      return Reflect.ownKeys(copiedObject ?? target);
-    },
-    has(target, prop) {
-      return Reflect.has(copiedObject ?? target, prop);
-    },
-    getOwnPropertyDescriptor(target, prop) {
-      return Reflect.getOwnPropertyDescriptor(copiedObject ?? target, prop);
-    },
-  });
-
-  return proxy;
+export const copyOnWrite = <T extends { [key: string]: unknown }>(
+  obj: T,
+): T => {
+  return new Proxy(
+    new CopyOnWriteTarget(obj),
+    copyOnWriteHandler,
+  ) as unknown as T;
 };
 
 /**