ponder 0.9.2 → 0.9.3

package/src/sync-historical/index.ts

@@ -61,7 +61,6 @@ export type HistoricalSync = {
    * that is synced.
    */
   sync(interval: Interval): Promise<SyncBlock | undefined>;
-  kill(): void;
 };
 
 type CreateHistoricalSyncParameters = {
@@ -76,7 +75,6 @@ type CreateHistoricalSyncParameters = {
 export const createHistoricalSync = async (
   args: CreateHistoricalSyncParameters,
 ): Promise<HistoricalSync> => {
-  let isKilled = false;
   /**
    * Flag to fetch transaction receipts through _eth_getBlockReceipts (true) or _eth_getTransactionReceipt (false)
    */
@@ -434,8 +432,6 @@ export const createHistoricalSync = async (
       address: filter.address,
     });
 
-    if (isKilled) return;
-
     // Insert `logs` into the sync-store
     await args.syncStore.insertLogs({
       logs: logs.map((log) => ({ log })),
@@ -478,12 +474,8 @@ export const createHistoricalSync = async (
       ? await syncAddressFactory(filter.address, interval)
       : filter.address;
 
-    if (isKilled) return;
-
     const logs = await syncLogsDynamic({ filter, interval, address });
 
-    if (isKilled) return;
-
     const blocks = await Promise.all(
       logs.map((log) => syncBlock(hexToNumber(log.blockNumber))),
     );
@@ -523,16 +515,12 @@ export const createHistoricalSync = async (
       transactionsCache.add(hash);
     }
 
-    if (isKilled) return;
-
     await args.syncStore.insertLogs({
       logs: logs.map((log, i) => ({ log, block: blocks[i]! })),
       shouldUpdateCheckpoint: true,
       chainId: args.network.chainId,
     });
 
-    if (isKilled) return;
-
     if (shouldGetTransactionReceipt(filter)) {
       const transactionReceipts = await Promise.all(
         Array.from(requiredBlocks).map((blockHash) => {
@@ -555,8 +543,6 @@ export const createHistoricalSync = async (
         }),
       ).then((receipts) => receipts.flat());
 
-      if (isKilled) return;
-
       await args.syncStore.insertTransactionReceipts({
         transactionReceipts,
         chainId: args.network.chainId,
@@ -595,14 +581,10 @@ export const createHistoricalSync = async (
         )
       : undefined;
 
-    if (isKilled) return;
-
     const blocks = await Promise.all(
       intervalRange(interval).map((number) => syncBlock(number)),
     );
 
-    if (isKilled) return;
-
     const transactionHashes: Set<Hash> = new Set();
     const requiredBlocks: Set<SyncBlock> = new Set();
 
@@ -627,8 +609,6 @@ export const createHistoricalSync = async (
       transactionsCache.add(hash);
     }
 
-    if (isKilled) return;
-
     const transactionReceipts = await Promise.all(
       Array.from(requiredBlocks).map((block) => {
         const blockTransactionHashes = new Set(
@@ -640,8 +620,6 @@ export const createHistoricalSync = async (
       }),
     ).then((receipts) => receipts.flat());
 
-    if (isKilled) return;
-
     await args.syncStore.insertTransactionReceipts({
       transactionReceipts,
       chainId: args.network.chainId,
@@ -715,15 +693,11 @@ export const createHistoricalSync = async (
       }),
     ).then((traces) => traces.flat());
 
-    if (isKilled) return;
-
     await args.syncStore.insertTraces({
       traces,
       chainId: args.network.chainId,
     });
 
-    if (isKilled) return;
-
     if (shouldGetTransactionReceipt(filter)) {
       const transactionReceipts = await Promise.all(
         Array.from(requiredBlocks).map((blockHash) => {
@@ -736,8 +710,6 @@ export const createHistoricalSync = async (
         }),
       ).then((receipts) => receipts.flat());
 
-      if (isKilled) return;
-
       await args.syncStore.insertTransactionReceipts({
         transactionReceipts,
         chainId: args.network.chainId,
@@ -858,14 +830,10 @@ export const createHistoricalSync = async (
             return;
           }
 
-          if (isKilled) return;
-
           await blockPromise;
         }),
       );
 
-      if (isKilled) return;
-
       const blocks = await Promise.all(blockCache.values());
 
       await Promise.all([
@@ -883,8 +851,6 @@ export const createHistoricalSync = async (
         }),
       ]);
 
-      if (isKilled) return;
-
       // Add corresponding intervals to the sync-store
       // Note: this should happen after so the database doesn't become corrupted
       if (args.network.disableCache === false) {
@@ -902,8 +868,5 @@ export const createHistoricalSync = async (
 
       return latestBlock;
     },
-    kill() {
-      isKilled = true;
-    },
   };
 };

package/src/sync-realtime/index.ts

@@ -1,4 +1,5 @@
 import type { Common } from "@/internal/common.js";
+import { ShutdownError } from "@/internal/errors.js";
 import type {
   BlockFilter,
   Factory,
@@ -30,6 +31,7 @@ import type {
   SyncTransaction,
   SyncTransactionReceipt,
 } from "@/types/sync.js";
+import { mutex } from "@/utils/mutex.js";
 import { range } from "@/utils/range.js";
 import type { RequestQueue } from "@/utils/requestQueue.js";
 import {
@@ -42,7 +44,7 @@ import {
 } from "@/utils/rpc.js";
 import { startClock } from "@/utils/timer.js";
 import { wait } from "@/utils/wait.js";
-import { type Queue, createQueue } from "@ponder/common";
+import type { Queue } from "@ponder/common";
 import { type Address, type Hash, hexToNumber, zeroHash } from "viem";
 import { isFilterInBloom, zeroLogsBloom } from "./bloom.js";
 
@@ -51,10 +53,10 @@ export type RealtimeSync = {
     syncProgress: Pick<SyncProgress, "finalized">;
     initialChildAddresses: Map<Factory, Set<Address>>;
   }): Promise<Queue<void, BlockWithEventData>>;
-  kill(): Promise<void>;
   unfinalizedBlocks: LightBlock[];
   finalizedChildAddresses: Map<Factory, Set<Address>>;
   unfinalizedChildAddresses: Map<Factory, Set<Address>>;
+  kill: () => void;
 };
 
 type CreateRealtimeSyncParameters = {
@@ -102,7 +104,6 @@ export const createRealtimeSync = (
   ////////
   // state
   ////////
-  let isKilled = false;
   let isBlockReceipts = true;
   let finalizedBlock: LightBlock;
   let finalizedChildAddresses: Map<Factory, Set<Address>>;
@@ -115,7 +116,7 @@ export const createRealtimeSync = (
    * `parentHash` => `hash`.
    */
   let unfinalizedBlocks: LightBlock[] = [];
-  let queue: Queue<void, BlockWithEventData & { endClock?: () => number }>;
+  // let queue: Queue<void, BlockWithEventData & { endClock?: () => number }>;
   let consecutiveErrors = 0;
   let interval: NodeJS.Timeout | undefined;
 
@@ -433,7 +434,7 @@ export const createRealtimeSync = (
         service: "realtime",
         msg: `Finalized ${hexToNumber(pendingFinalizedBlock.number) - hexToNumber(finalizedBlock.number) + 1} '${
           args.network.name
-        }' blocks from ${hexToNumber(finalizedBlock.number) + 1} to ${hexToNumber(pendingFinalizedBlock.number)}`,
+        }' blocks [${hexToNumber(finalizedBlock.number) + 1}, ${hexToNumber(pendingFinalizedBlock.number)}]`,
       });
 
       const finalizedBlocks = unfinalizedBlocks.filter(
@@ -490,11 +491,6 @@ export const createRealtimeSync = (
 
       await args.onEvent({ type: "finalize", block: pendingFinalizedBlock });
     }
-
-    args.common.logger.debug({
-      service: "realtime",
-      msg: `Finished syncing '${args.network.name}' block ${hexToNumber(block.number)}`,
-    });
   };
 
   /**
@@ -551,9 +547,9 @@ export const createRealtimeSync = (
 
     args.common.logger.warn({
       service: "realtime",
-      msg: `Reconciled ${reorgedBlocks.length}-block reorg on '${
+      msg: `Reconciled ${reorgedBlocks.length}-block '${
         args.network.name
-      }' with common ancestor block ${hexToNumber(commonAncestor.number)}`,
+      }' reorg with common ancestor block ${hexToNumber(commonAncestor.number)}`,
     });
 
     // recompute `unfinalizedChildAddresses`
@@ -691,11 +687,11 @@ export const createRealtimeSync = (
           if (log.transactionHash === zeroHash) {
             args.common.logger.warn({
               service: "sync",
-              msg: `Detected log with empty transaction hash in block ${block.hash} at log index ${hexToNumber(log.logIndex)}. This is expected for some networks like ZKsync.`,
+              msg: `Detected '${args.network.name}' log with empty transaction hash in block ${block.hash} at log index ${hexToNumber(log.logIndex)}. This is expected for some networks like ZKsync.`,
             });
           } else {
             throw new Error(
-              `Detected inconsistent RPC responses. 'log.transactionHash' ${log.transactionHash} not found in 'block.transactions' ${block.hash}`,
+              `Detected inconsistent '${args.network.name}' RPC responses. 'log.transactionHash' ${log.transactionHash} not found in 'block.transactions' ${block.hash}`,
             );
           }
         }
@@ -708,7 +704,7 @@ export const createRealtimeSync = (
     ) {
       args.common.logger.debug({
         service: "realtime",
-        msg: `Skipped fetching logs for '${args.network.name}' block ${hexToNumber(block.number)} due to bloom filter result`,
+        msg: `Skipped fetching '${args.network.name}' logs for block ${hexToNumber(block.number)} due to bloom filter result`,
       });
     }
 
@@ -782,7 +778,7 @@ export const createRealtimeSync = (
           if (log.transactionHash === zeroHash) {
             args.common.logger.warn({
               service: "sync",
-              msg: `Detected log with empty transaction hash in block ${block.hash} at log index ${hexToNumber(log.logIndex)}. This is expected for some networks like ZKsync.`,
+              msg: `Detected '${args.network.name}' log with empty transaction hash in block ${block.hash} at log index ${hexToNumber(log.logIndex)}. This is expected for some networks like ZKsync.`,
             });
           } else {
             requiredTransactions.add(log.transactionHash);
@@ -908,11 +904,11 @@ export const createRealtimeSync = (
        * 4) Block is behind the last processed. This is a sign that
        *    a reorg has occurred.
        */
-      queue = createQueue({
-        browser: false,
-        concurrency: 1,
-        initialStart: true,
-        worker: async ({ block, ...rest }) => {
+      const processBlock = mutex(
+        async ({
+          block,
+          ...rest
+        }: BlockWithEventData & { endClock?: () => number }) => {
           const latestBlock = getLatestUnfinalizedBlock();
 
           // We already saw and handled this block. No-op.
@@ -931,7 +927,7 @@ export const createRealtimeSync = (
             if (hexToNumber(latestBlock.number) >= hexToNumber(block.number)) {
               await handleReorg(block);
 
-              queue.clear();
+              processBlock.clear();
               return;
             }
 
@@ -961,24 +957,19 @@ export const createRealtimeSync = (
                 service: "realtime",
                 msg: `Fetched ${missingBlockRange.length} missing '${
                   args.network.name
-                }' blocks from ${hexToNumber(latestBlock.number) + 1} to ${Math.min(
+                }' blocks [${hexToNumber(latestBlock.number) + 1}, ${Math.min(
                   hexToNumber(block.number),
                   hexToNumber(latestBlock.number) + MAX_QUEUED_BLOCKS,
-                )}`,
+                )}]`,
               });
 
-              // This is needed to ensure proper `kill()` behavior. When the service
-              // is killed, nothing should be added to the queue, or else `onIdle()`
-              // will never resolve.
-              if (isKilled) return;
-
-              queue.clear();
+              processBlock.clear();
 
               for (const pendingBlock of pendingBlocks) {
-                queue.add(pendingBlock);
+                processBlock(pendingBlock);
               }
 
-              queue.add({ block, ...rest });
+              processBlock({ block, ...rest });
 
               return;
             }
@@ -986,7 +977,7 @@ export const createRealtimeSync = (
             // Check if a reorg occurred by validating the chain of block hashes.
             if (block.parentHash !== latestBlock.hash) {
               await handleReorg(block);
-              queue.clear();
+              processBlock.clear();
               return;
             }
 
@@ -999,10 +990,12 @@ export const createRealtimeSync = (
 
             return;
           } catch (_error) {
-            if (isKilled) return;
-
             const error = _error as Error;
 
+            if (args.common.shutdown.isKilled) {
+              throw new ShutdownError();
+            }
+
             args.common.logger.warn({
               service: "realtime",
               msg: `Failed to process '${args.network.name}' block ${hexToNumber(block.number)}`,
@@ -1022,7 +1015,7 @@ export const createRealtimeSync = (
 
             // Remove all blocks from the queue. This protects against an
             // erroneous block causing a fatal error.
-            queue.clear();
+            processBlock.clear();
 
             // After a certain number of attempts, emit a fatal error.
             if (++consecutiveErrors === ERROR_TIMEOUT.length) {
@@ -1036,7 +1029,7 @@ export const createRealtimeSync = (
             }
           }
         },
-      });
+      );
 
       const enqueue = async () => {
         try {
@@ -1044,6 +1037,11 @@ export const createRealtimeSync = (
             blockTag: "latest",
           });
 
+          args.common.logger.debug({
+            service: "realtime",
+            msg: `Received latest '${args.network.name}' block ${hexToNumber(block.number)}`,
+          });
+
           const latestBlock = getLatestUnfinalizedBlock();
 
           // We already saw and handled this block. No-op.
@@ -1062,12 +1060,14 @@ export const createRealtimeSync = (
 
           consecutiveErrors = 0;
 
-          return queue.add({ ...blockWithEventData, endClock });
+          return processBlock({ ...blockWithEventData, endClock });
         } catch (_error) {
-          if (isKilled) return;
-
           const error = _error as Error;
 
+          if (args.common.shutdown.isKilled) {
+            throw new ShutdownError();
+          }
+
           args.common.logger.warn({
             service: "realtime",
             msg: `Failed to fetch latest '${args.network.name}' block`,
@@ -1089,15 +1089,12 @@ export const createRealtimeSync = (
 
       interval = setInterval(enqueue, args.network.pollingInterval);
 
+      args.common.shutdown.add(() => {
+        clearInterval(interval);
+      });
+
       // Note: this is done just for testing.
-      return enqueue().then(() => queue);
-    },
-    async kill() {
-      clearInterval(interval);
-      isKilled = true;
-      queue?.pause();
-      queue?.clear();
-      await queue?.onIdle();
+      return enqueue().then(() => processBlock);
     },
     get unfinalizedBlocks() {
       return unfinalizedBlocks;
@@ -1108,5 +1105,8 @@ export const createRealtimeSync = (
     get unfinalizedChildAddresses() {
       return unfinalizedChildAddresses;
     },
+    async kill() {
+      clearInterval(interval);
+    },
   };
 };

package/src/sync-store/encoding.ts

@@ -8,9 +8,9 @@ import type {
 } from "@/types/sync.js";
 import {
   EVENT_TYPES,
+  MAX_CHECKPOINT,
+  ZERO_CHECKPOINT,
   encodeCheckpoint,
-  maxCheckpoint,
-  zeroCheckpoint,
 } from "@/utils/checkpoint.js";
 import { toLowerCase } from "@/utils/lowercase.js";
 import type { ColumnType, Insertable } from "kysely";
@@ -55,9 +55,9 @@ export const encodeBlock = ({
       blockTimestamp: hexToNumber(block.timestamp),
       chainId: BigInt(chainId),
       blockNumber: hexToBigInt(block.number),
-      transactionIndex: maxCheckpoint.transactionIndex,
+      transactionIndex: MAX_CHECKPOINT.transactionIndex,
       eventType: EVENT_TYPES.blocks,
-      eventIndex: zeroCheckpoint.eventIndex,
+      eventIndex: ZERO_CHECKPOINT.eventIndex,
     }),
     baseFeePerGas: block.baseFeePerGas
       ? hexToBigInt(block.baseFeePerGas)
@@ -183,7 +183,7 @@ export const encodeTransaction = ({
       blockNumber: hexToBigInt(transaction.blockNumber),
       transactionIndex: hexToBigInt(transaction.transactionIndex),
       eventType: EVENT_TYPES.transactions,
-      eventIndex: zeroCheckpoint.eventIndex,
+      eventIndex: ZERO_CHECKPOINT.eventIndex,
     }),
     chainId,
     blockHash: transaction.blockHash,

package/src/sync-store/index.ts

@@ -118,10 +118,7 @@ export type SyncStore = {
     blocks: Pick<LightBlock, "number">[];
     chainId: number;
   }): Promise<void>;
-  pruneByChain(args: {
-    fromBlock: number;
-    chainId: number;
-  }): Promise<void>;
+  pruneByChain(args: { chainId: number }): Promise<void>;
 };
 
 const logFactorySQL = (
@@ -902,38 +899,23 @@ export const createSyncStore = ({
           .execute();
       },
     ),
-  pruneByChain: async ({ fromBlock, chainId }) =>
+  pruneByChain: async ({ chainId }) =>
     database.wrap({ method: "pruneByChain", includeTraceLogs: true }, () =>
       database.qb.sync.transaction().execute(async (tx) => {
-        await tx
-          .deleteFrom("logs")
-          .where("chainId", "=", chainId)
-          .where("blockNumber", ">=", fromBlock.toString())
-          .execute();
-        await tx
-          .deleteFrom("blocks")
-          .where("chainId", "=", chainId)
-          .where("number", ">=", fromBlock.toString())
-          .execute();
+        await tx.deleteFrom("logs").where("chainId", "=", chainId).execute();
+        await tx.deleteFrom("blocks").where("chainId", "=", chainId).execute();
         await tx
           .deleteFrom("rpc_request_results")
           .where("chain_id", "=", chainId)
-          .where("block_number", ">=", fromBlock.toString())
-          .execute();
-        await tx
-          .deleteFrom("traces")
-          .where("chainId", "=", chainId)
-          .where("blockNumber", ">=", fromBlock.toString())
           .execute();
+        await tx.deleteFrom("traces").where("chainId", "=", chainId).execute();
         await tx
           .deleteFrom("transactions")
           .where("chainId", "=", chainId)
-          .where("blockNumber", ">=", fromBlock.toString())
           .execute();
         await tx
           .deleteFrom("transactionReceipts")
           .where("chainId", "=", chainId)
-          .where("blockNumber", ">=", fromBlock.toString())
           .execute();
       }),
     ),

package/src/ui/index.ts

@@ -10,11 +10,7 @@ export function createUi({ common }: { common: Common }) {
   const ui = buildUiState();
   const { render, unmount } = setupInkApp(ui);
 
-  let isKilled = false;
-
   const renderInterval = setInterval(async () => {
-    if (isKilled) return;
-
     ui.sync = await getSyncProgress(common.metrics);
     ui.indexing = await getIndexingProgress(common.metrics);
     ui.app = await getAppProgress(common.metrics);
@@ -26,13 +22,8 @@ export function createUi({ common }: { common: Common }) {
     render(ui);
   }, 100);
 
-  const kill = () => {
-    isKilled = true;
+  common.shutdown.add(async () => {
     clearInterval(renderInterval);
     unmount();
-  };
-
-  return {
-    kill,
-  };
+  });
 }

package/src/utils/checkpoint.ts

@@ -105,7 +105,7 @@ export const decodeCheckpoint = (checkpoint: string): Checkpoint => {
   };
 };
 
-export const zeroCheckpoint: Checkpoint = {
+export const ZERO_CHECKPOINT: Checkpoint = {
   blockTimestamp: 0,
   chainId: 0n,
   blockNumber: 0n,
@@ -114,7 +114,7 @@ export const zeroCheckpoint: Checkpoint = {
   eventIndex: 0n,
 };
 
-export const maxCheckpoint: Checkpoint = {
+export const MAX_CHECKPOINT: Checkpoint = {
   blockTimestamp: 99999_99999,
   chainId: 9999_9999_9999_9999n,
   blockNumber: 9999_9999_9999_9999n,
@@ -123,6 +123,10 @@ export const maxCheckpoint: Checkpoint = {
   eventIndex: 9999_9999_9999_9999n,
 };
 
+export const ZERO_CHECKPOINT_STRING = encodeCheckpoint(ZERO_CHECKPOINT);
+export const MAX_CHECKPOINT_STRING = encodeCheckpoint(MAX_CHECKPOINT);
+
+/**
 /**
  * Returns true if two checkpoints are equal.
  */
@@ -154,4 +158,14 @@ export const checkpointMin = (...checkpoints: Checkpoint[]) =>
     return isCheckpointGreaterThan(min, checkpoint) ? checkpoint : min;
   });
 
-export const LATEST = encodeCheckpoint(maxCheckpoint);
+export const LATEST = MAX_CHECKPOINT_STRING;
+
+/** Compute the minimum checkpoint, filtering out undefined */
+export const min = (...checkpoints: (string | undefined)[]) => {
+  return checkpoints.reduce((acc, cur) => {
+    if (cur === undefined) return acc;
+    if (acc === undefined) return cur;
+    if (acc < cur) return acc;
+    return cur;
+  })!;
+};

package/src/utils/chunk.ts

@@ -0,0 +1,7 @@
+export const chunk = <T>(array: T[], size: number): T[][] => {
+  const chunks = [];
+  for (let i = 0; i < array.length; i += size) {
+    chunks.push(array.slice(i, i + size));
+  }
+  return chunks;
+};

package/src/utils/generators.ts

@@ -1,5 +1,11 @@
 import { promiseWithResolvers } from "@ponder/common";
 
+/**
+ * Merges multiple async generators into a single async generator.
+ *
+ * @param generators - The generators to merge.
+ * @returns A single async generator that yields results from all input generators.
+ */
 export async function* mergeAsyncGenerators<T>(
   generators: AsyncGenerator<T>[],
 ): AsyncGenerator<T> {
@@ -25,3 +31,63 @@ export async function* mergeAsyncGenerators<T>(
     }
   }
 }
+
+/**
+ * Buffers the results of an async generator.
+ *
+ * @param generator - The generator to buffer.
+ * @param size - The size of the buffer.
+ * @returns An async generator that yields results from the input generator.
+ */
+export async function* bufferAsyncGenerator<T>(
+  generator: AsyncGenerator<T>,
+  size: number,
+): AsyncGenerator<T> {
+  const buffer: T[] = [];
+  let done = false;
+
+  let pwr1 = promiseWithResolvers<void>();
+  let pwr2 = promiseWithResolvers<void>();
+
+  (async () => {
+    for await (const result of generator) {
+      buffer.push(result);
+
+      pwr1.resolve();
+
+      if (buffer.length > size) await pwr2.promise;
+      pwr2 = promiseWithResolvers<void>();
+    }
+    done = true;
+    pwr1.resolve();
+  })();
+
+  while (done === false || buffer.length > 0) {
+    if (buffer.length > 0) {
+      pwr2.resolve();
+
+      yield buffer.shift()!;
+    } else {
+      await pwr1.promise;
+      pwr1 = promiseWithResolvers<void>();
+    }
+  }
+}
+
+/**
+ * Drains an async generator into an array.
+ *
+ * @param asyncGenerator - The async generator to drain.
+ * @returns An array of results from the input generator.
+ */
+export async function drainAsyncGenerator<T>(
+  asyncGenerator: AsyncGenerator<T>,
+): Promise<T[]> {
+  const result: T[] = [];
+
+  for await (const events of asyncGenerator) {
+    result.push(events);
+  }
+
+  return result;
+}