ponder 0.9.2 → 0.9.4-debug.1

package/src/sync/index.ts

@@ -1,9 +1,11 @@
 import type { Common } from "@/internal/common.js";
 import type {
   Factory,
+  Filter,
   IndexingBuild,
   Network,
   RawEvent,
+  Seconds,
   Source,
   Status,
 } from "@/internal/types.js";
@@ -20,14 +22,19 @@ import type { SyncStore } from "@/sync-store/index.js";
 import type { LightBlock, SyncBlock } from "@/types/sync.js";
 import {
   type Checkpoint,
+  MAX_CHECKPOINT,
+  ZERO_CHECKPOINT,
+  ZERO_CHECKPOINT_STRING,
   decodeCheckpoint,
   encodeCheckpoint,
-  maxCheckpoint,
-  zeroCheckpoint,
+  min,
 } from "@/utils/checkpoint.js";
 import { estimate } from "@/utils/estimate.js";
-import { formatEta, formatPercentage } from "@/utils/format.js";
-import { mergeAsyncGenerators } from "@/utils/generators.js";
+import { formatPercentage } from "@/utils/format.js";
+import {
+  bufferAsyncGenerator,
+  mergeAsyncGenerators,
+} from "@/utils/generators.js";
 import {
   type Interval,
   intervalDifference,
@@ -37,31 +44,23 @@ import {
   sortIntervals,
 } from "@/utils/interval.js";
 import { intervalUnion } from "@/utils/interval.js";
+import { createMutex } from "@/utils/mutex.js";
 import { never } from "@/utils/never.js";
-import { type RequestQueue, createRequestQueue } from "@/utils/requestQueue.js";
+import { partition } from "@/utils/partition.js";
+import type { RequestQueue } from "@/utils/requestQueue.js";
+import { _eth_getBlockByNumber } from "@/utils/rpc.js";
 import { startClock } from "@/utils/timer.js";
-import { type Queue, createQueue } from "@ponder/common";
-import {
-  type Address,
-  type Hash,
-  type Transport,
-  hexToBigInt,
-  hexToNumber,
-  toHex,
-} from "viem";
-import { _eth_getBlockByNumber } from "../utils/rpc.js";
+import { zipperMany } from "@/utils/zipper.js";
+import { type Address, type Hash, hexToBigInt, hexToNumber, toHex } from "viem";
 import { buildEvents } from "./events.js";
 import { isAddressFactory } from "./filter.js";
-import { cachedTransport } from "./transport.js";
 
 export type Sync = {
-  getEvents(): AsyncGenerator<{ events: RawEvent[]; checkpoint: string }>;
+  getEvents(): AsyncGenerator<RawEvent[]>;
   startRealtime(): Promise<void>;
   getStatus(): Status;
-  getStartCheckpoint(): string;
+  seconds: Seconds;
   getFinalizedCheckpoint(): string;
-  getCachedTransport(network: Network): Transport;
-  kill(): Promise<void>;
 };
 
 export type RealtimeEvent =
@@ -70,20 +69,22 @@ export type RealtimeEvent =
       checkpoint: string;
       status: Status;
       events: RawEvent[];
+      network: Network;
     }
   | {
       type: "reorg";
       checkpoint: string;
+      network: Network;
     }
   | {
       type: "finalize";
       checkpoint: string;
+      network: Network;
     };
 
 export type SyncProgress = {
   start: SyncBlock | LightBlock;
   end: SyncBlock | LightBlock | undefined;
-  cached: SyncBlock | LightBlock | undefined;
   current: SyncBlock | LightBlock | undefined;
   finalized: SyncBlock | LightBlock;
 };
@@ -107,7 +108,7 @@ export const blockToCheckpoint = (
   rounding: "up" | "down",
 ): Checkpoint => {
   return {
-    ...(rounding === "up" ? maxCheckpoint : zeroCheckpoint),
+    ...(rounding === "up" ? MAX_CHECKPOINT : ZERO_CHECKPOINT),
     blockTimestamp: hexToNumber(block.timestamp),
     chainId: BigInt(chainId),
     blockNumber: hexToBigInt(block.number),
@@ -153,34 +154,24 @@ const getHistoricalLast = (
       : syncProgress.end;
 };
 
-/** Compute the minimum checkpoint, filtering out undefined */
-const min = (...checkpoints: (string | undefined)[]) => {
-  return checkpoints.reduce((acc, cur) => {
-    if (cur === undefined) return acc;
-    if (acc === undefined) return cur;
-    if (acc < cur) return acc;
-    return cur;
-  })!;
-};
-
 export const splitEvents = (
   events: RawEvent[],
 ): { checkpoint: string; events: RawEvent[] }[] => {
-  let prevHash: Hash | undefined;
+  let hash: Hash | undefined;
   const result: { checkpoint: string; events: RawEvent[] }[] = [];
 
   for (const event of events) {
-    if (prevHash === undefined || prevHash !== event.block.hash) {
+    if (hash === undefined || hash !== event.block.hash) {
       result.push({
         checkpoint: encodeCheckpoint({
-          ...maxCheckpoint,
+          ...MAX_CHECKPOINT,
           blockTimestamp: Number(event.block.timestamp),
           chainId: BigInt(event.chainId),
           blockNumber: event.block.number,
         }),
         events: [],
       });
-      prevHash = event.block.hash;
+      hash = event.block.hash;
     }
 
     result[result.length - 1]!.events.push(event);
@@ -219,153 +210,40 @@ export const getChainCheckpoint = ({
   );
 };
 
-type CreateSyncParameters = {
+export const createSync = async (params: {
   common: Common;
   indexingBuild: Pick<IndexingBuild, "sources" | "networks">;
+  requestQueues: RequestQueue[];
   syncStore: SyncStore;
   onRealtimeEvent(event: RealtimeEvent): Promise<void>;
   onFatalError(error: Error): void;
   initialCheckpoint: string;
-};
-
-export const createSync = async (args: CreateSyncParameters): Promise<Sync> => {
+  ordering: "omnichain" | "multichain";
+}): Promise<Sync> => {
   const perNetworkSync = new Map<
     Network,
     {
-      requestQueue: RequestQueue;
       syncProgress: SyncProgress;
       historicalSync: HistoricalSync;
       realtimeSync: RealtimeSync;
-      realtimeQueue: Queue<void, RealtimeSyncEvent>;
-      unfinalizedBlocks: Omit<
-        Extract<RealtimeSyncEvent, { type: "block" }>,
-        "type"
-      >[];
     }
   >();
-  /** Events that have been executed but not finalized. */
-  let executedEvents: RawEvent[] = [];
-  /** Events that have not been executed yet. */
-  let pendingEvents: RawEvent[] = [];
-  const status: Status = {};
-  let isKilled = false;
-  // Realtime events across all chains that can't be passed to the parent function
-  // because the overall checkpoint hasn't caught up to the events yet.
-
-  // Instantiate `localSyncData` and `status`
-  await Promise.all(
-    args.indexingBuild.networks.map(async (network) => {
-      const requestQueue = createRequestQueue({
-        network,
-        common: args.common,
-      });
-      const sources = args.indexingBuild.sources.filter(
-        ({ filter }) => filter.chainId === network.chainId,
-      );
-
-      const { start, end, finalized } = await syncDiagnostic({
-        common: args.common,
-        sources,
-        requestQueue,
-        network,
-      });
-
-      // Invalidate sync cache for devnet sources
-      if (network.disableCache) {
-        args.common.logger.warn({
-          service: "sync",
-          msg: `Deleting cache records for '${network.name}' from block ${hexToNumber(start.number)}`,
-        });
-
-        await args.syncStore.pruneByChain({
-          fromBlock: hexToNumber(start.number),
-          chainId: network.chainId,
-        });
-      }
-
-      const historicalSync = await createHistoricalSync({
-        common: args.common,
-        sources,
-        syncStore: args.syncStore,
-        requestQueue,
-        network,
-        onFatalError: args.onFatalError,
-      });
-
-      const realtimeQueue = createQueue({
-        initialStart: true,
-        browser: false,
-        concurrency: 1,
-        worker: async (event: RealtimeSyncEvent) =>
-          onRealtimeSyncEvent({ event, network }),
-      });
-
-      const realtimeSync = createRealtimeSync({
-        common: args.common,
-        sources,
-        requestQueue,
-        network,
-        onEvent: (event) =>
-          realtimeQueue.add(event).catch((error) => {
-            args.common.logger.error({
-              service: "sync",
-              msg: `Fatal error: Unable to process ${event.type} event`,
-              error,
-            });
-            args.onFatalError(error);
-          }),
-        onFatalError: args.onFatalError,
-      });
-
-      const cached = await getCachedBlock({
-        sources,
-        requestQueue,
-        historicalSync,
-      });
-
-      // Update "ponder_sync_block" metric
-      if (cached !== undefined) {
-        args.common.metrics.ponder_sync_block.set(
-          { network: network.name },
-          hexToNumber(cached.number),
-        );
-      }
-
-      const syncProgress: SyncProgress = {
-        start,
-        end,
-        finalized,
-        cached,
-        current: cached,
-      };
-
-      args.common.metrics.ponder_sync_is_realtime.set(
-        { network: network.name },
-        0,
-      );
-      args.common.metrics.ponder_sync_is_complete.set(
-        { network: network.name },
-        0,
-      );
 
-      perNetworkSync.set(network, {
-        requestQueue,
-        syncProgress,
-        historicalSync,
-        realtimeSync,
-        realtimeQueue,
-        unfinalizedBlocks: [],
-      });
-      status[network.name] = { block: null, ready: false };
-    }),
-  );
+  const getMultichainCheckpoint = ({
+    tag,
+    network,
+  }: { tag: "start" | "end" | "current" | "finalized"; network: Network }):
+    | string
+    | undefined => {
+    const syncProgress = perNetworkSync.get(network)!.syncProgress;
+    return getChainCheckpoint({ syncProgress, network, tag });
+  };
 
-  /**
-   * Returns the minimum checkpoint across all chains.
-   */
-  const getOmnichainCheckpoint = (
-    tag: "start" | "end" | "current" | "finalized",
-  ): string | undefined => {
+  const getOmnichainCheckpoint = ({
+    tag,
+  }: { tag: "start" | "end" | "current" | "finalized" }):
+    | string
+    | undefined => {
     const checkpoints = Array.from(perNetworkSync.entries()).map(
       ([network, { syncProgress }]) =>
         getChainCheckpoint({ syncProgress, network, tag }),
@@ -392,30 +270,29 @@ export const createSync = async (args: CreateSyncParameters): Promise<Sync> => {
         timestamp: decodeCheckpoint(checkpoint).blockTimestamp,
         number: Number(decodeCheckpoint(checkpoint).blockNumber),
       };
-    } else {
-      let i = events.length - 1;
-      while (i >= 0) {
-        const event = events[i]!;
+      return;
+    }
 
-        if (network.chainId === event.chainId) {
-          status[network.name]!.block = {
-            timestamp: decodeCheckpoint(event.checkpoint).blockTimestamp,
-            number: Number(decodeCheckpoint(event.checkpoint).blockNumber),
-          };
-        }
+    let i = events.length - 1;
+    while (i >= 0) {
+      const event = events[i]!;
 
-        i--;
+      if (network.chainId === event.chainId) {
+        status[network.name]!.block = {
+          timestamp: decodeCheckpoint(event.checkpoint).blockTimestamp,
+          number: Number(decodeCheckpoint(event.checkpoint).blockNumber),
+        };
+        return;
       }
+
+      i--;
     }
   };
 
   const updateRealtimeStatus = ({
     checkpoint,
     network,
-  }: {
-    checkpoint: string;
-    network: Network;
-  }) => {
+  }: { checkpoint: string; network: Network }) => {
     const localBlock = perNetworkSync
       .get(network)!
       .realtimeSync.unfinalizedBlocks.findLast(
@@ -431,483 +308,501 @@ export const createSync = async (args: CreateSyncParameters): Promise<Sync> => {
     }
   };
 
-  /**
-   * Estimate optimal range (seconds) to query at a time, eventually
-   * used to determine `to` passed to `getEvents`
-   */
-  let estimateSeconds = 1_000;
-  /**
-   * Omnichain `getEvents`
-   *
-   * Extract all events across `args.networks` ordered by checkpoint.
-   * The generator is "completed" when all event have been extracted
-   * before the minimum finalized checkpoint (supremum).
-   *
-   * Note: `syncStore.getEvents` is used to order between multiple
-   * networks. This approach is not future proof.
-   */
   async function* getEvents() {
-    let latestFinalizedFetch = Date.now();
-
-    /**
-     * Calculate start checkpoint, if `initialCheckpoint` is non-zero,
-     * use that. Otherwise, use `startBlock`
-     */
-    const start =
-      args.initialCheckpoint !== encodeCheckpoint(zeroCheckpoint)
-        ? args.initialCheckpoint
-        : getOmnichainCheckpoint("start")!;
-
-    // Cursor used to track progress.
-    let from = start;
-
-    let showLogs = true;
-    while (true) {
-      const syncGenerator = mergeAsyncGenerators(
-        Array.from(perNetworkSync.entries()).map(
-          ([network, { syncProgress, historicalSync }]) =>
-            localHistoricalSyncGenerator({
-              common: args.common,
-              network,
-              syncProgress,
-              historicalSync,
-              showLogs,
-            }),
-        ),
-      );
-
-      // Only show logs on the first iteration
-      showLogs = false;
-
-      for await (const _ of syncGenerator) {
-        /**
-         * `current` is used to calculate the `to` checkpoint, if any
-         * network hasn't yet ingested a block, run another iteration of this loop.
-         * It is an invariant that `latestBlock` will eventually be defined.
-         */
-        if (
-          Array.from(perNetworkSync.values()).some(
-            ({ syncProgress }) => syncProgress.current === undefined,
-          )
-        ) {
-          continue;
-        }
+    let cursor =
+      params.initialCheckpoint !== ZERO_CHECKPOINT_STRING
+        ? params.initialCheckpoint
+        : getOmnichainCheckpoint({ tag: "start" })!;
+
+    const to = min(
+      getOmnichainCheckpoint({ tag: "end" }),
+      getOmnichainCheckpoint({ tag: "finalized" }),
+    );
 
-        // Calculate the mininum "current" checkpoint, limited by "finalized" and "end"
-        const to = min(
-          getOmnichainCheckpoint("end"),
-          getOmnichainCheckpoint("finalized"),
-          getOmnichainCheckpoint("current"),
+    const eventGenerators = Array.from(perNetworkSync.entries()).map(
+      ([network, { syncProgress, historicalSync }]) => {
+        const sources = params.indexingBuild.sources.filter(
+          ({ filter }) => filter.chainId === network.chainId,
         );
 
-        let consecutiveErrors = 0;
-
-        /*
-         * Extract events with `syncStore.getEvents()`, paginating to
-         * avoid loading too many events into memory.
-         */
-        while (true) {
-          if (isKilled) return;
-          if (from >= to) break;
-          const getEventsMaxBatchSize = args.common.options.syncEventsQuerySize;
-
-          // convert `estimateSeconds` to checkpoint
-          const estimatedTo = encodeCheckpoint({
-            ...zeroCheckpoint,
-            blockTimestamp: Math.min(
-              decodeCheckpoint(from).blockTimestamp + estimateSeconds,
-              maxCheckpoint.blockTimestamp,
-            ),
-          });
-
-          try {
-            const { events, cursor } = await args.syncStore.getEvents({
-              filters: args.indexingBuild.sources.map(({ filter }) => filter),
-              from,
-              to: to < estimatedTo ? to : estimatedTo,
-              limit: getEventsMaxBatchSize,
-            });
-
-            args.common.logger.debug({
-              service: "sync",
-              msg: `Fetched ${events.length} events from the database for a ${formatEta(estimateSeconds * 1000)} range from timestamp ${decodeCheckpoint(from).blockTimestamp}`,
-            });
-
-            for (const network of args.indexingBuild.networks) {
-              updateHistoricalStatus({ events, checkpoint: cursor, network });
-            }
-
-            estimateSeconds = estimate({
-              from: decodeCheckpoint(from).blockTimestamp,
-              to: decodeCheckpoint(cursor).blockTimestamp,
-              target: getEventsMaxBatchSize,
-              result: events.length,
-              min: 10,
-              max: 86_400,
-              prev: estimateSeconds,
-              maxIncrease: 1.08,
-            });
+        const localSyncGenerator = getLocalSyncGenerator({
+          common: params.common,
+          network,
+          syncProgress,
+          historicalSync,
+        });
 
-            consecutiveErrors = 0;
-            yield { events, checkpoint: to };
-            from = cursor;
-          } catch (error) {
-            // Handle errors by reducing the requested range by 10x
-            estimateSeconds = Math.max(10, Math.round(estimateSeconds / 10));
+        const localEventGenerator = getLocalEventGenerator({
+          common: params.common,
+          network,
+          syncStore: params.syncStore,
+          sources,
+          localSyncGenerator,
+          from:
+            params.initialCheckpoint !== ZERO_CHECKPOINT_STRING
+              ? params.initialCheckpoint
+              : getChainCheckpoint({ syncProgress, network, tag: "start" })!,
+          to,
+          limit: Math.round(
+            params.common.options.syncEventsQuerySize /
+              (params.indexingBuild.networks.length * 2),
+          ),
+        });
 
-            args.common.logger.debug({
-              service: "sync",
-              msg: `Failed to fetch events from the database, retrying with a ${formatEta(estimateSeconds * 1000)} range`,
-            });
+        return bufferAsyncGenerator(localEventGenerator, 1);
+      },
+    );
 
-            if (++consecutiveErrors > 4) throw error;
-          }
-        }
+    const mergeAsync =
+      params.ordering === "multichain"
+        ? mergeAsyncGenerators
+        : mergeAsyncGeneratorsWithEventOrder;
+
+    for await (const { events, checkpoint } of mergeAsync(eventGenerators)) {
+      if (params.ordering === "multichain") {
+        const network = params.indexingBuild.networks.find(
+          (network) =>
+            network.chainId === Number(decodeCheckpoint(checkpoint).chainId),
+        )!;
+        params.common.logger.debug({
+          service: "sync",
+          msg: `Sequenced ${events.length} '${network.name}' events for timestamp range [${decodeCheckpoint(cursor).blockTimestamp}, ${decodeCheckpoint(checkpoint).blockTimestamp}]`,
+        });
+      } else {
+        params.common.logger.debug({
+          service: "sync",
+          msg: `Sequenced ${events.length} events for timestamp range [${decodeCheckpoint(cursor).blockTimestamp}, ${decodeCheckpoint(checkpoint).blockTimestamp}]`,
+        });
       }
 
-      /** `true` if all networks have synced all known finalized blocks.  */
-      const allHistoricalSyncExhaustive = Array.from(
-        perNetworkSync.values(),
-      ).every(({ syncProgress }) => {
-        if (isSyncEnd(syncProgress)) return true;
-
-        // Determine if `finalized` block is considered "stale"
-        const staleSeconds = (Date.now() - latestFinalizedFetch) / 1_000;
-        if (staleSeconds <= args.common.options.syncHandoffStaleSeconds) {
-          return true;
-        }
-
-        return false;
-      });
-
-      if (allHistoricalSyncExhaustive) break;
-
-      /** At least one network has a `finalized` block that is considered "stale". */
-
-      latestFinalizedFetch = Date.now();
-
-      await Promise.all(
-        Array.from(perNetworkSync.entries()).map(
-          async ([network, { requestQueue, syncProgress }]) => {
-            args.common.logger.debug({
-              service: "sync",
-              msg: `Refetching '${network.name}' finalized block`,
-            });
-
-            const latestBlock = await _eth_getBlockByNumber(requestQueue, {
-              blockTag: "latest",
-            });
-
-            const finalizedBlockNumber = Math.max(
-              0,
-              hexToNumber(latestBlock.number) - network.finalityBlockCount,
-            );
+      for (const network of params.indexingBuild.networks) {
+        updateHistoricalStatus({ events, checkpoint, network });
+      }
+      yield events;
+      cursor = checkpoint;
+    }
+  }
 
-            syncProgress.finalized = await _eth_getBlockByNumber(requestQueue, {
-              blockNumber: finalizedBlockNumber,
-            });
+  /** Events that have been executed but not finalized. */
+  let executedEvents: RawEvent[] = [];
+  /** Events that have not been executed. */
+  let pendingEvents: RawEvent[] = [];
 
-            const historicalLast = getHistoricalLast(syncProgress);
+  const realtimeMutex = createMutex();
 
-            // Set metric "ponder_historical_total_blocks"
-            args.common.metrics.ponder_historical_total_blocks.set(
-              { network: network.name },
-              hexToNumber(historicalLast.number) -
-                hexToNumber(syncProgress.start.number) +
-                1,
-            );
-          },
-        ),
-      );
-    }
-  }
+  const checkpoints = {
+    // Note: `checkpoints.current` not used in multichain ordering
+    current: ZERO_CHECKPOINT_STRING,
+    finalized: ZERO_CHECKPOINT_STRING,
+  };
 
-  /**
-   * Omnichain `onRealtimeSyncEvent`
-   *
-   * Handle callback events across all `args.networks`, and raising these
-   * events to `args.onRealtimeEvent` while maintaining checkpoint ordering.
-   */
-  const onRealtimeSyncEvent = async ({
-    network,
-    event,
-  }: { network: Network; event: RealtimeSyncEvent }) => {
-    const { syncProgress, realtimeSync, unfinalizedBlocks } =
-      perNetworkSync.get(network)!;
+  // Note: `latencyTimers` not used in multichain ordering
+  const latencyTimers = new Map<string, () => number>();
 
+  const onRealtimeSyncEvent = (
+    event: RealtimeSyncEvent,
+    {
+      network,
+      syncProgress,
+      realtimeSync,
+    }: {
+      network: Network;
+      syncProgress: SyncProgress;
+      realtimeSync: RealtimeSync;
+    },
+  ): void => {
     switch (event.type) {
-      /**
-       * Handle a new block being ingested.
-       */
       case "block": {
-        // Update local sync, record checkpoint before and after
-        const from = getOmnichainCheckpoint("current")!;
-        syncProgress.current = event.block;
-        const to = getOmnichainCheckpoint("current")!;
-
-        // Update "ponder_sync_block" metric
-        args.common.metrics.ponder_sync_block.set(
-          { network: network.name },
-          hexToNumber(syncProgress.current.number),
-        );
-
-        const newEvents = buildEvents({
-          sources: args.indexingBuild.sources,
+        const events = buildEvents({
+          sources: params.indexingBuild.sources,
           chainId: network.chainId,
           blockWithEventData: event,
           finalizedChildAddresses: realtimeSync.finalizedChildAddresses,
           unfinalizedChildAddresses: realtimeSync.unfinalizedChildAddresses,
         });
 
-        unfinalizedBlocks.push(event);
-        pendingEvents.push(...newEvents);
+        params.common.logger.debug({
+          service: "sync",
+          msg: `Extracted ${events.length} '${network.name}' events for block ${hexToNumber(event.block.number)}`,
+        });
 
-        if (to > from) {
-          for (const network of args.indexingBuild.networks) {
-            updateRealtimeStatus({ checkpoint: to, network });
-          }
+        if (params.ordering === "multichain") {
+          // Note: `checkpoints.current` not used in multichain ordering
+          const checkpoint = getMultichainCheckpoint({
+            tag: "current",
+            network,
+          })!;
 
-          // Move events from pending to executed
+          status[network.name]!.block = {
+            timestamp: hexToNumber(event.block.timestamp),
+            number: hexToNumber(event.block.number),
+          };
 
-          const events = pendingEvents
-            .filter((event) => event.checkpoint < to)
-            .sort((a, b) => (a.checkpoint < b.checkpoint ? -1 : 1));
+          const readyEvents = events.concat(pendingEvents);
+          pendingEvents = [];
+          executedEvents = executedEvents.concat(readyEvents);
 
-          pendingEvents = pendingEvents.filter(
-            ({ checkpoint }) => checkpoint > to,
-          );
-          executedEvents.push(...events);
+          params.common.logger.debug({
+            service: "sync",
+            msg: `Sequenced ${readyEvents.length} '${network.name}' events for block ${hexToNumber(event.block.number)}`,
+          });
 
-          args
+          params
             .onRealtimeEvent({
               type: "block",
-              checkpoint: to,
+              checkpoint,
               status: structuredClone(status),
-              events,
+              events: readyEvents.sort((a, b) =>
+                a.checkpoint < b.checkpoint ? -1 : 1,
+              ),
+              network,
             })
             .then(() => {
-              if (events.length > 0 && isKilled === false) {
-                args.common.logger.info({
-                  service: "app",
-                  msg: `Indexed ${events.length} events`,
-                });
+              // update `ponder_realtime_latency` metric
+              if (event.endClock) {
+                params.common.metrics.ponder_realtime_latency.observe(
+                  { network: network.name },
+                  event.endClock(),
+                );
               }
+            });
+        } else {
+          const from = checkpoints.current;
+          checkpoints.current = getOmnichainCheckpoint({ tag: "current" })!;
+          const to = getOmnichainCheckpoint({ tag: "current" })!;
+
+          if (event.endClock !== undefined) {
+            latencyTimers.set(
+              encodeCheckpoint(
+                blockToCheckpoint(event.block, network.chainId, "up"),
+              ),
+              event.endClock,
+            );
+          }
 
-              // update `ponder_realtime_latency` metric
-              for (const network of args.indexingBuild.networks) {
-                for (const { block, endClock } of perNetworkSync.get(network)!
-                  .unfinalizedBlocks) {
-                  const checkpoint = encodeCheckpoint(
-                    blockToCheckpoint(block, network.chainId, "up"),
-                  );
-                  if (checkpoint > from && checkpoint <= to && endClock) {
-                    args.common.metrics.ponder_realtime_latency.observe(
+          if (to > from) {
+            for (const network of params.indexingBuild.networks) {
+              updateRealtimeStatus({ checkpoint: to, network });
+            }
+
+            // Move ready events from pending to executed
+
+            const readyEvents = pendingEvents
+              .concat(events)
+              .filter(({ checkpoint }) => checkpoint < to);
+            pendingEvents = pendingEvents
+              .concat(events)
+              .filter(({ checkpoint }) => checkpoint > to);
+            executedEvents = executedEvents.concat(readyEvents);
+
+            params.common.logger.debug({
+              service: "sync",
+              msg: `Sequenced ${readyEvents.length} '${network.name}' events for timestamp range [${decodeCheckpoint(from).blockTimestamp}, ${decodeCheckpoint(to).blockTimestamp}]`,
+            });
+
+            params
+              .onRealtimeEvent({
+                type: "block",
+                checkpoint: to,
+                status: structuredClone(status),
+                events: readyEvents.sort((a, b) =>
+                  a.checkpoint < b.checkpoint ? -1 : 1,
+                ),
+                network,
+              })
+              .then(() => {
+                // update `ponder_realtime_latency` metric
+                for (const [checkpoint, timer] of latencyTimers) {
+                  if (checkpoint > from && checkpoint <= to) {
+                    const chainId = Number(
+                      decodeCheckpoint(checkpoint).chainId,
+                    );
+                    const network = params.indexingBuild.networks.find(
+                      (network) => network.chainId === chainId,
+                    )!;
+                    params.common.metrics.ponder_realtime_latency.observe(
                       { network: network.name },
-                      endClock(),
+                      timer(),
                     );
                   }
                 }
-              }
-            });
+              });
+          } else {
+            pendingEvents = pendingEvents.concat(events);
+          }
         }
 
         break;
       }
-      /**
-       * Handle a new block being finalized.
-       */
-      case "finalize": {
-        // Newly finalized range
-        const interval = [
-          hexToNumber(syncProgress.finalized.number),
-          hexToNumber(event.block.number),
-        ] satisfies Interval;
 
-        // Update local sync, record checkpoint before and after
-        const prev = getOmnichainCheckpoint("finalized")!;
-        syncProgress.finalized = event.block;
-        const checkpoint = getOmnichainCheckpoint("finalized")!;
+      case "finalize": {
+        const from = checkpoints.finalized;
+        checkpoints.finalized = getOmnichainCheckpoint({ tag: "finalized" })!;
+        const to = getOmnichainCheckpoint({ tag: "finalized" })!;
 
         if (
+          params.ordering === "omnichain" &&
           getChainCheckpoint({ syncProgress, network, tag: "finalized" })! >
-          getOmnichainCheckpoint("current")!
+            getOmnichainCheckpoint({ tag: "current" })!
         ) {
-          args.common.logger.warn({
+          params.common.logger.warn({
             service: "sync",
-            msg: `Finalized block for '${network.name}' has surpassed overall indexing checkpoint`,
+            msg: `Finalized '${network.name}' block has surpassed overall indexing checkpoint`,
           });
         }
 
         // Remove all finalized data
 
-        const finalizedBlocks = unfinalizedBlocks.filter(
-          ({ block }) =>
-            hexToNumber(block.number) <= hexToNumber(event.block.number),
-        );
+        executedEvents = executedEvents.filter((e) => e.checkpoint > to);
 
-        perNetworkSync.get(network)!.unfinalizedBlocks =
-          unfinalizedBlocks.filter(
-            ({ block }) =>
-              hexToNumber(block.number) > hexToNumber(event.block.number),
-          );
+        // Raise event to parent function (runtime)
+        if (to > from) {
+          params.onRealtimeEvent({
+            type: "finalize",
+            checkpoint: to,
+            network,
+          });
+        }
+
+        break;
+      }
+
+      case "reorg": {
+        // Remove all reorged data
+
+        let reorgedEvents = 0;
+
+        const isReorgedEvent = ({ chainId, block }: RawEvent) => {
+          if (
+            chainId === network.chainId &&
+            Number(block.number) > hexToNumber(event.block.number)
+          ) {
+            reorgedEvents++;
+            return true;
+          }
+          return false;
+        };
 
+        pendingEvents = pendingEvents.filter(
+          (e) => isReorgedEvent(e) === false,
+        );
         executedEvents = executedEvents.filter(
-          (e) => e.checkpoint > checkpoint,
+          (e) => isReorgedEvent(e) === false,
         );
 
-        // Add finalized blocks, logs, transactions, receipts, and traces to the sync-store.
+        params.common.logger.debug({
+          service: "sync",
+          msg: `Removed ${reorgedEvents} reorged '${network.name}' events`,
+        });
 
-        await Promise.all([
-          args.syncStore.insertBlocks({
-            blocks: finalizedBlocks
-              .filter(({ hasMatchedFilter }) => hasMatchedFilter)
-              .map(({ block }) => block),
-            chainId: network.chainId,
-          }),
-          args.syncStore.insertLogs({
-            logs: finalizedBlocks.flatMap(({ logs, block }) =>
-              logs.map((log) => ({ log, block })),
-            ),
-            shouldUpdateCheckpoint: true,
-            chainId: network.chainId,
-          }),
-          args.syncStore.insertLogs({
-            logs: finalizedBlocks.flatMap(({ factoryLogs }) =>
-              factoryLogs.map((log) => ({ log })),
-            ),
-            shouldUpdateCheckpoint: false,
-            chainId: network.chainId,
-          }),
-          args.syncStore.insertTransactions({
-            transactions: finalizedBlocks.flatMap(({ transactions, block }) =>
-              transactions.map((transaction) => ({
-                transaction,
-                block,
-              })),
-            ),
-            chainId: network.chainId,
-          }),
-          args.syncStore.insertTransactionReceipts({
-            transactionReceipts: finalizedBlocks.flatMap(
-              ({ transactionReceipts }) => transactionReceipts,
-            ),
-            chainId: network.chainId,
-          }),
-          args.syncStore.insertTraces({
-            traces: finalizedBlocks.flatMap(({ traces, block, transactions }) =>
-              traces.map((trace) => ({
-                trace,
-                block,
-                transaction: transactions.find(
-                  (t) => t.hash === trace.transactionHash,
-                )!,
-              })),
-            ),
-            chainId: network.chainId,
-          }),
-        ]);
+        if (params.ordering === "multichain") {
+          // Note: `checkpoints.current` not used in multichain ordering
+          const checkpoint = getMultichainCheckpoint({
+            tag: "current",
+            network,
+          })!;
 
-        // Add corresponding intervals to the sync-store
-        // Note: this should happen after so the database doesn't become corrupted
+          // Move events from executed to pending
 
-        if (network.disableCache === false) {
-          await args.syncStore.insertIntervals({
-            intervals: args.indexingBuild.sources
-              .filter(({ filter }) => filter.chainId === network.chainId)
-              .map(({ filter }) => ({ filter, interval })),
-            chainId: network.chainId,
+          const events = executedEvents.filter(
+            (e) => e.checkpoint > checkpoint,
+          );
+          executedEvents = executedEvents.filter(
+            (e) => e.checkpoint < checkpoint,
+          );
+          pendingEvents = pendingEvents.concat(events);
+
+          params.common.logger.debug({
+            service: "sync",
+            msg: `Rescheduled ${events.length} reorged events`,
           });
-        }
 
-        // Raise event to parent function (runtime)
-        if (checkpoint > prev) {
-          args.onRealtimeEvent({ type: "finalize", checkpoint });
-        }
+          params.onRealtimeEvent({ type: "reorg", checkpoint, network });
+        } else {
+          const from = checkpoints.current;
+          checkpoints.current = getOmnichainCheckpoint({ tag: "current" })!;
+          const to = getOmnichainCheckpoint({ tag: "current" })!;
 
-        /**
-         * The realtime service can be killed if `endBlock` is
-         * defined has become finalized.
-         */
-        if (isSyncEnd(syncProgress)) {
-          args.common.metrics.ponder_sync_is_realtime.set(
-            { network: network.name },
-            0,
-          );
-          args.common.metrics.ponder_sync_is_complete.set(
-            { network: network.name },
-            1,
-          );
-          args.common.logger.info({
+          // Move events from executed to pending
+
+          const events = executedEvents.filter((e) => e.checkpoint > to);
+          executedEvents = executedEvents.filter((e) => e.checkpoint < to);
+          pendingEvents = pendingEvents.concat(events);
+
+          params.common.logger.debug({
             service: "sync",
-            msg: `Synced final end block for '${network.name}' (${hexToNumber(syncProgress.end!.number)}), killing realtime sync service`,
+            msg: `Rescheduled ${events.length} reorged events`,
           });
-          realtimeSync.kill();
+
+          if (to < from) {
+            params.onRealtimeEvent({
+              type: "reorg",
+              checkpoint: to,
+              network,
+            });
+          }
         }
+
         break;
       }
-      /**
-       * Handle a reorg with a new common ancestor block being found.
-       */
-      case "reorg": {
-        syncProgress.current = event.block;
-        // Note: this checkpoint is <= the previous checkpoint
-        const checkpoint = getOmnichainCheckpoint("current")!;
 
-        // Update "ponder_sync_block" metric
-        args.common.metrics.ponder_sync_block.set(
-          { network: network.name },
-          hexToNumber(syncProgress.current.number),
-        );
+      default:
+        never(event);
+    }
+  };
 
-        // Remove all reorged data
+  await Promise.all(
+    params.indexingBuild.networks.map(async (network, index) => {
+      const requestQueue = params.requestQueues[index]!;
 
-        perNetworkSync.get(network)!.unfinalizedBlocks =
-          unfinalizedBlocks.filter(
-            ({ block }) =>
-              hexToNumber(block.number) <= hexToNumber(event.block.number),
-          );
+      const sources = params.indexingBuild.sources.filter(
+        ({ filter }) => filter.chainId === network.chainId,
+      );
 
-        const isReorgedEvent = ({ chainId, block }: RawEvent) =>
-          chainId === network.chainId &&
-          Number(block.number) > hexToNumber(event.block.number);
+      // Invalidate sync cache for devnet sources
+      if (network.disableCache) {
+        params.common.logger.warn({
+          service: "sync",
+          msg: `Deleting cache records for '${network.name}'`,
+        });
 
-        pendingEvents = pendingEvents.filter(
-          (e) => isReorgedEvent(e) === false,
-        );
-        executedEvents = executedEvents.filter(
-          (e) => isReorgedEvent(e) === false,
-        );
+        await params.syncStore.pruneByChain({
+          chainId: network.chainId,
+        });
+      }
 
-        // Move events from executed to pending
+      const historicalSync = await createHistoricalSync({
+        common: params.common,
+        sources,
+        syncStore: params.syncStore,
+        requestQueue,
+        network,
+        onFatalError: params.onFatalError,
+      });
 
-        const events = executedEvents.filter((e) => e.checkpoint > checkpoint);
-        executedEvents = executedEvents.filter(
-          (e) => e.checkpoint < checkpoint,
-        );
-        pendingEvents.push(...events);
+      const syncProgress = await getLocalSyncProgress({
+        common: params.common,
+        network,
+        sources,
+        requestQueue,
+        intervalsCache: historicalSync.intervalsCache,
+      });
 
-        await args.syncStore.pruneRpcRequestResult({
-          chainId: network.chainId,
-          blocks: event.reorgedBlocks,
-        });
+      const realtimeSync = createRealtimeSync({
+        common: params.common,
+        sources,
+        requestQueue,
+        network,
+        onEvent: realtimeMutex((event) =>
+          perChainOnRealtimeSyncEvent(event)
+            .then((event) => {
+              onRealtimeSyncEvent(event, {
+                network,
+                syncProgress,
+                realtimeSync,
+              });
 
-        // Raise event to parent function (runtime)
-        args.onRealtimeEvent({ type: "reorg", checkpoint });
+              if (isSyncFinalized(syncProgress) && isSyncEnd(syncProgress)) {
+                // The realtime service can be killed if `endBlock` is
+                // defined has become finalized.
+
+                params.common.metrics.ponder_sync_is_realtime.set(
+                  { network: network.name },
+                  0,
+                );
+                params.common.metrics.ponder_sync_is_complete.set(
+                  { network: network.name },
+                  1,
+                );
+                params.common.logger.info({
+                  service: "sync",
+                  msg: `Killing '${network.name}' live indexing because the end block ${hexToNumber(syncProgress.end!.number)} has been finalized`,
+                });
+                realtimeSync.kill();
+              }
+            })
+            .catch((error) => {
+              params.common.logger.error({
+                service: "sync",
+                msg: `Fatal error: Unable to process ${event.type} event`,
+                error,
+              });
+              params.onFatalError(error);
+            }),
+        ),
+        onFatalError: params.onFatalError,
+      });
 
-        break;
-      }
+      params.common.metrics.ponder_sync_is_realtime.set(
+        { network: network.name },
+        0,
+      );
+      params.common.metrics.ponder_sync_is_complete.set(
+        { network: network.name },
+        0,
+      );
 
-      default:
-        never(event);
+      perNetworkSync.set(network, {
+        syncProgress,
+        historicalSync,
+        realtimeSync,
+      });
+
+      const perChainOnRealtimeSyncEvent = getPerChainOnRealtimeSyncEvent({
+        common: params.common,
+        network,
+        sources,
+        syncStore: params.syncStore,
+        syncProgress,
+      });
+    }),
+  );
+
+  const status: Status = {};
+  const seconds: Seconds = {};
+
+  for (const network of params.indexingBuild.networks) {
+    status[network.name] = { block: null, ready: false };
+  }
+
+  if (params.ordering === "multichain") {
+    for (const network of params.indexingBuild.networks) {
+      seconds[network.name] = {
+        start: decodeCheckpoint(
+          getMultichainCheckpoint({ tag: "start", network })!,
+        ).blockTimestamp,
+        end: decodeCheckpoint(
+          min(
+            getOmnichainCheckpoint({ tag: "end" }),
+            getOmnichainCheckpoint({ tag: "finalized" }),
+          ),
+        ).blockTimestamp,
+        cached: decodeCheckpoint(params.initialCheckpoint).blockTimestamp,
+      };
     }
-  };
+  } else {
+    for (const network of params.indexingBuild.networks) {
+      seconds[network.name] = {
+        start: decodeCheckpoint(getOmnichainCheckpoint({ tag: "start" })!)
+          .blockTimestamp,
+        end: decodeCheckpoint(
+          min(
+            getOmnichainCheckpoint({ tag: "end" }),
+            getOmnichainCheckpoint({ tag: "finalized" }),
+          ),
+        ).blockTimestamp,
+        cached: decodeCheckpoint(params.initialCheckpoint).blockTimestamp,
+      };
+    }
+  }
+
   return {
     getEvents,
     async startRealtime() {
-      for (const network of args.indexingBuild.networks) {
+      for (const network of params.indexingBuild.networks) {
         const { syncProgress, realtimeSync } = perNetworkSync.get(network)!;
 
-        const filters = args.indexingBuild.sources
+        const filters = params.indexingBuild.sources
           .filter(({ filter }) => filter.chainId === network.chainId)
           .map(({ filter }) => filter);
 
@@ -921,7 +816,7 @@ export const createSync = async (args: CreateSyncParameters): Promise<Sync> => {
         // finalized checkpoint and add them to pendingEvents. These events are synced during
         // the historical phase, but must be indexed in the realtime phase because events
         // synced in realtime on other chains might be ordered before them.
-        const from = getOmnichainCheckpoint("finalized")!;
+        const from = getOmnichainCheckpoint({ tag: "finalized" })!;
 
         const finalized = getChainCheckpoint({
           syncProgress,
@@ -936,17 +831,27 @@ export const createSync = async (args: CreateSyncParameters): Promise<Sync> => {
         const to = min(finalized, end);
 
         if (to > from) {
-          const events = await args.syncStore.getEvents({ filters, from, to });
-          pendingEvents.push(...events.events);
+          const events = await params.syncStore.getEvents({
+            filters,
+            from,
+            to,
+          });
+
+          params.common.logger.debug({
+            service: "sync",
+            msg: `Extracted and scheduled ${events.events.length} '${network.name}' events`,
+          });
+
+          pendingEvents = pendingEvents.concat(events.events);
         }
 
         if (isSyncEnd(syncProgress)) {
-          args.common.metrics.ponder_sync_is_complete.set(
+          params.common.metrics.ponder_sync_is_complete.set(
             { network: network.name },
             1,
           );
         } else {
-          args.common.metrics.ponder_sync_is_realtime.set(
+          params.common.metrics.ponder_sync_is_realtime.set(
             { network: network.name },
             1,
           );
@@ -954,225 +859,389 @@ export const createSync = async (args: CreateSyncParameters): Promise<Sync> => {
           const initialChildAddresses = new Map<Factory, Set<Address>>();
 
           for (const filter of filters) {
-            if ("address" in filter && isAddressFactory(filter.address)) {
-              const addresses = await args.syncStore.getChildAddresses({
-                filter: filter.address,
-              });
+            switch (filter.type) {
+              case "log":
+                if (isAddressFactory(filter.address)) {
+                  const addresses = await params.syncStore.getChildAddresses({
+                    filter: filter.address,
+                  });
+
+                  initialChildAddresses.set(filter.address, new Set(addresses));
+                }
+                break;
+
+              case "transaction":
+              case "transfer":
+              case "trace":
+                if (isAddressFactory(filter.fromAddress)) {
+                  const addresses = await params.syncStore.getChildAddresses({
+                    filter: filter.fromAddress,
+                  });
+
+                  initialChildAddresses.set(
+                    filter.fromAddress,
+                    new Set(addresses),
+                  );
+                }
+
+                if (isAddressFactory(filter.toAddress)) {
+                  const addresses = await params.syncStore.getChildAddresses({
+                    filter: filter.toAddress,
+                  });
 
-              initialChildAddresses.set(filter.address, new Set(addresses));
+                  initialChildAddresses.set(
+                    filter.toAddress,
+                    new Set(addresses),
+                  );
+                }
+
+                break;
             }
           }
 
+          params.common.logger.debug({
+            service: "sync",
+            msg: `Initialized '${network.name}' realtime sync with ${initialChildAddresses.size} factory child addresses`,
+          });
+
           realtimeSync.start({ syncProgress, initialChildAddresses });
         }
       }
     },
-    getStartCheckpoint() {
-      return getOmnichainCheckpoint("start")!;
-    },
-    getFinalizedCheckpoint() {
-      return getOmnichainCheckpoint("finalized")!;
-    },
     getStatus() {
       return status;
     },
-    getCachedTransport(network) {
-      const { requestQueue } = perNetworkSync.get(network)!;
-      return cachedTransport({ requestQueue, syncStore: args.syncStore });
-    },
-    async kill() {
-      isKilled = true;
-      const promises: Promise<void>[] = [];
-      for (const network of args.indexingBuild.networks) {
-        const { historicalSync, realtimeSync, realtimeQueue } =
-          perNetworkSync.get(network)!;
-        historicalSync.kill();
-        realtimeQueue.pause();
-        realtimeQueue.clear();
-        promises.push(realtimeQueue.onIdle());
-        promises.push(realtimeSync.kill());
-      }
-      await Promise.all(promises);
+    seconds,
+    getFinalizedCheckpoint() {
+      return getOmnichainCheckpoint({ tag: "finalized" })!;
     },
   };
 };
 
-/** ... */
-export const syncDiagnostic = async ({
+export const getPerChainOnRealtimeSyncEvent = ({
   common,
-  sources,
   network,
-  requestQueue,
+  sources,
+  syncStore,
+  syncProgress,
 }: {
   common: Common;
-  sources: Source[];
   network: Network;
-  requestQueue: RequestQueue;
+  sources: Source[];
+  syncStore: SyncStore;
+  syncProgress: SyncProgress;
 }) => {
-  /** Earliest `startBlock` among all `filters` */
-  const start = Math.min(...sources.map(({ filter }) => filter.fromBlock ?? 0));
-  /**
-   * Latest `endBlock` among all filters. `undefined` if at least one
-   * of the filters doesn't have an `endBlock`.
-   */
-  const end = sources.some(({ filter }) => filter.toBlock === undefined)
-    ? undefined
-    : Math.max(...sources.map(({ filter }) => filter.toBlock!));
-
-  const [remoteChainId, startBlock, latestBlock] = await Promise.all([
-    requestQueue.request({ method: "eth_chainId" }),
-    _eth_getBlockByNumber(requestQueue, { blockNumber: start }),
-    _eth_getBlockByNumber(requestQueue, { blockTag: "latest" }),
-  ]);
-
-  const endBlock =
-    end === undefined
-      ? undefined
-      : end > hexToBigInt(latestBlock.number)
-        ? ({
-            number: toHex(end),
-            hash: "0x",
-            parentHash: "0x",
-            timestamp: toHex(maxCheckpoint.blockTimestamp),
-          } as LightBlock)
-        : await _eth_getBlockByNumber(requestQueue, { blockNumber: end });
+  let unfinalizedBlocks: Omit<
+    Extract<RealtimeSyncEvent, { type: "block" }>,
+    "type"
+  >[] = [];
 
-  // Warn if the config has a different chainId than the remote.
-  if (hexToNumber(remoteChainId) !== network.chainId) {
-    common.logger.warn({
-      service: "sync",
-      msg: `Remote chain ID (${remoteChainId}) does not match configured chain ID (${network.chainId}) for network "${network.name}"`,
-    });
-  }
+  return async (event: RealtimeSyncEvent): Promise<RealtimeSyncEvent> => {
+    switch (event.type) {
+      case "block": {
+        syncProgress.current = event.block;
 
-  const finalizedBlockNumber = Math.max(
-    0,
-    hexToNumber(latestBlock.number) - network.finalityBlockCount,
-  );
+        common.logger.debug({
+          service: "sync",
+          msg: `Updated '${network.name}' current block to ${hexToNumber(event.block.number)}`,
+        });
 
-  const finalizedBlock = await _eth_getBlockByNumber(requestQueue, {
-    blockNumber: finalizedBlockNumber,
-  });
+        common.metrics.ponder_sync_block.set(
+          { network: network.name },
+          hexToNumber(syncProgress.current.number),
+        );
 
-  return {
-    start: startBlock,
-    end: endBlock,
-    finalized: finalizedBlock,
+        unfinalizedBlocks.push(event);
+
+        return event;
+      }
+
+      case "finalize": {
+        const finalizedInterval = [
+          hexToNumber(syncProgress.finalized.number),
+          hexToNumber(event.block.number),
+        ] satisfies Interval;
+
+        syncProgress.finalized = event.block;
+
+        common.logger.debug({
+          service: "sync",
+          msg: `Updated '${network.name}' finalized block to ${hexToNumber(event.block.number)}`,
+        });
+
+        // Remove all finalized data
+
+        const finalizedBlocks = unfinalizedBlocks.filter(
+          ({ block }) =>
+            hexToNumber(block.number) <= hexToNumber(event.block.number),
+        );
+
+        unfinalizedBlocks = unfinalizedBlocks.filter(
+          ({ block }) =>
+            hexToNumber(block.number) > hexToNumber(event.block.number),
+        );
+
+        // Add finalized blocks, logs, transactions, receipts, and traces to the sync-store.
+
+        await Promise.all([
+          syncStore.insertBlocks({
+            blocks: finalizedBlocks
+              .filter(({ hasMatchedFilter }) => hasMatchedFilter)
+              .map(({ block }) => block),
+            chainId: network.chainId,
+          }),
+          syncStore.insertLogs({
+            logs: finalizedBlocks.flatMap(({ logs, block }) =>
+              logs.map((log) => ({ log, block })),
+            ),
+            shouldUpdateCheckpoint: true,
+            chainId: network.chainId,
+          }),
+          syncStore.insertLogs({
+            logs: finalizedBlocks.flatMap(({ factoryLogs }) =>
+              factoryLogs.map((log) => ({ log })),
+            ),
+            shouldUpdateCheckpoint: false,
+            chainId: network.chainId,
+          }),
+          syncStore.insertTransactions({
+            transactions: finalizedBlocks.flatMap(({ transactions, block }) =>
+              transactions.map((transaction) => ({
+                transaction,
+                block,
+              })),
+            ),
+            chainId: network.chainId,
+          }),
+          syncStore.insertTransactionReceipts({
+            transactionReceipts: finalizedBlocks.flatMap(
+              ({ transactionReceipts }) => transactionReceipts,
+            ),
+            chainId: network.chainId,
+          }),
+          syncStore.insertTraces({
+            traces: finalizedBlocks.flatMap(({ traces, block, transactions }) =>
+              traces.map((trace) => ({
+                trace,
+                block,
+                transaction: transactions.find(
+                  (t) => t.hash === trace.transactionHash,
+                )!,
+              })),
+            ),
+            chainId: network.chainId,
+          }),
+        ]);
+
+        // Add corresponding intervals to the sync-store
+        // Note: this should happen after insertion so the database doesn't become corrupted
+
+        if (network.disableCache === false) {
+          const syncedIntervals: {
+            interval: Interval;
+            filter: Filter;
+          }[] = [];
+
+          for (const { filter } of sources) {
+            const intervals = intervalIntersection(
+              [finalizedInterval],
+              [
+                [
+                  filter.fromBlock ?? 0,
+                  filter.toBlock ?? Number.POSITIVE_INFINITY,
+                ],
+              ],
+            );
+
+            for (const interval of intervals) {
+              syncedIntervals.push({ interval, filter });
+            }
+          }
+
+          await syncStore.insertIntervals({
+            intervals: syncedIntervals,
+            chainId: network.chainId,
+          });
+        }
+
+        return event;
+      }
+
+      case "reorg": {
+        syncProgress.current = event.block;
+
+        common.logger.debug({
+          service: "sync",
+          msg: `Updated '${network.name}' current block to ${hexToNumber(event.block.number)}`,
+        });
+
+        common.metrics.ponder_sync_block.set(
+          { network: network.name },
+          hexToNumber(syncProgress.current.number),
+        );
+
+        // Remove all reorged data
+
+        unfinalizedBlocks = unfinalizedBlocks.filter(
+          ({ block }) =>
+            hexToNumber(block.number) <= hexToNumber(event.block.number),
+        );
+
+        await syncStore.pruneRpcRequestResult({
+          chainId: network.chainId,
+          blocks: event.reorgedBlocks,
+        });
+
+        return event;
+      }
+    }
   };
 };
 
-/** Returns the closest-to-tip block that has been synced for all `sources`. */
-export const getCachedBlock = ({
-  sources,
-  requestQueue,
-  historicalSync,
-}: {
+export async function* getLocalEventGenerator(params: {
+  common: Common;
+  network: Network;
+  syncStore: SyncStore;
   sources: Source[];
-  requestQueue: RequestQueue;
-  historicalSync: HistoricalSync;
-}): Promise<SyncBlock | LightBlock> | undefined => {
-  const latestCompletedBlocks = sources.map(({ filter }) => {
-    const requiredInterval = [
-      filter.fromBlock ?? 0,
-      filter.toBlock ?? Number.POSITIVE_INFINITY,
-    ] satisfies Interval;
-    const fragmentIntervals = historicalSync.intervalsCache.get(filter)!;
+  localSyncGenerator: AsyncGenerator<string>;
+  from: string;
+  to: string;
+  limit: number;
+}): AsyncGenerator<{ events: RawEvent[]; checkpoint: string }> {
+  let cursor = params.from;
+  // Estimate optimal range (seconds) to query at a time, eventually
+  // used to determine `to` passed to `getEvents`.
+  let estimateSeconds = 1_000;
 
-    const completedIntervals = sortIntervals(
-      intervalIntersection(
-        [requiredInterval],
-        intervalIntersectionMany(
-          fragmentIntervals.map(({ intervals }) => intervals),
+  params.common.logger.debug({
+    service: "sync",
+    msg: `Initialized '${params.network.name}' extract query for timestamp range [${decodeCheckpoint(params.from).blockTimestamp}, ${decodeCheckpoint(params.to).blockTimestamp}]`,
+  });
+
+  for await (const syncCheckpoint of bufferAsyncGenerator(
+    params.localSyncGenerator,
+    Number.POSITIVE_INFINITY,
+  )) {
+    let consecutiveErrors = 0;
+    while (cursor < min(syncCheckpoint, params.to)) {
+      const estimateCheckpoint = encodeCheckpoint({
+        ...ZERO_CHECKPOINT,
+        chainId: BigInt(params.network.chainId),
+        blockTimestamp: Math.min(
+          decodeCheckpoint(cursor).blockTimestamp + estimateSeconds,
+          MAX_CHECKPOINT.blockTimestamp,
         ),
-      ),
-    );
+      });
+      const to = min(syncCheckpoint, estimateCheckpoint, params.to);
+      try {
+        const { events, cursor: queryCursor } =
+          await params.syncStore.getEvents({
+            filters: params.sources.map(({ filter }) => filter),
+            from: cursor,
+            to,
+            limit: params.limit,
+          });
 
-    if (completedIntervals.length === 0) return undefined;
+        params.common.logger.debug({
+          service: "sync",
+          msg: `Extracted ${events.length} '${params.network.name}' events for timestamp range [${decodeCheckpoint(cursor).blockTimestamp}, ${decodeCheckpoint(queryCursor).blockTimestamp}]`,
+        });
 
-    const earliestCompletedInterval = completedIntervals[0]!;
-    if (earliestCompletedInterval[0] !== (filter.fromBlock ?? 0)) {
-      return undefined;
-    }
-    return earliestCompletedInterval[1];
-  });
+        estimateSeconds = estimate({
+          from: decodeCheckpoint(cursor).blockTimestamp,
+          to: decodeCheckpoint(queryCursor).blockTimestamp,
+          target: params.limit,
+          result: events.length,
+          min: 10,
+          max: 86_400,
+          prev: estimateSeconds,
+          maxIncrease: 1.08,
+        });
 
-  const minCompletedBlock = Math.min(
-    ...(latestCompletedBlocks.filter(
-      (block) => block !== undefined,
-    ) as number[]),
-  );
+        params.common.logger.debug({
+          service: "sync",
+          msg: `Updated '${params.network.name}' extract query estimate to ${estimateSeconds} seconds`,
+        });
 
-  /**  Filter i has known progress if a completed interval is found or if
-   * `_latestCompletedBlocks[i]` is undefined but `sources[i].filter.fromBlock`
-   * is > `_minCompletedBlock`.
-   */
-  if (
-    latestCompletedBlocks.every(
-      (block, i) =>
-        block !== undefined ||
-        (sources[i]!.filter.fromBlock ?? 0) > minCompletedBlock,
-    )
-  ) {
-    return _eth_getBlockByNumber(requestQueue, {
-      blockNumber: minCompletedBlock,
-    });
-  }
+        consecutiveErrors = 0;
+        cursor = queryCursor;
+        yield { events, checkpoint: cursor };
+      } catch (error) {
+        params.common.logger.warn({
+          service: "sync",
+          msg: `Failed '${params.network.name}' extract query for timestamp range [${decodeCheckpoint(cursor).blockTimestamp}, ${decodeCheckpoint(to).blockTimestamp}]`,
+          error: error as Error,
+        });
 
-  return undefined;
-};
+        // Handle errors by reducing the requested range by 10x
+        estimateSeconds = Math.max(10, Math.round(estimateSeconds / 10));
+
+        params.common.logger.debug({
+          service: "sync",
+          msg: `Updated '${params.network.name}' getEvents query estimate to ${estimateSeconds} seconds`,
+        });
 
-/** Predictive pagination and metrics for `historicalSync.sync()` */
-export async function* localHistoricalSyncGenerator({
+        if (++consecutiveErrors > 4) throw error;
+      }
+    }
+  }
+}
+
+export async function* getLocalSyncGenerator({
   common,
   network,
   syncProgress,
   historicalSync,
-  showLogs,
 }: {
   common: Common;
   network: Network;
   syncProgress: SyncProgress;
   historicalSync: HistoricalSync;
-  showLogs: boolean;
-}): AsyncGenerator {
-  // Return immediately if the `syncProgress.start` is unfinalized
+}): AsyncGenerator<string> {
+  const label = { network: network.name };
+
+  let cursor = hexToNumber(syncProgress.start.number);
+  const last = getHistoricalLast(syncProgress);
+
+  // Estimate optimal range (blocks) to sync at a time, eventually to be used to
+  // determine `interval` passed to `historicalSync.sync()`.
+  let estimateRange = 25;
+
+  // Handle two special cases:
+  // 1. `syncProgress.start` > `syncProgress.finalized`
+  // 2. `cached` is defined
+
   if (
     hexToNumber(syncProgress.start.number) >
     hexToNumber(syncProgress.finalized.number)
   ) {
     syncProgress.current = syncProgress.finalized;
 
-    // Update "ponder_sync_block" metric
+    common.logger.warn({
+      service: "sync",
+      msg: `Skipped '${network.name}' historical sync because the start block is unfinalized`,
+    });
+
     common.metrics.ponder_sync_block.set(
-      { network: network.name },
+      label,
       hexToNumber(syncProgress.current.number),
     );
-
-    if (showLogs) {
-      common.logger.warn({
-        service: "historical",
-        msg: `Skipped historical sync for '${network.name}' because the start block is not finalized`,
-      });
-    }
-
-    const label = { network: network.name };
-    // Set "ponder_historical_total_blocks"
     common.metrics.ponder_historical_total_blocks.set(label, 0);
-    // Set "ponder_historical_sync_cached_blocks"
     common.metrics.ponder_historical_cached_blocks.set(label, 0);
 
     return;
   }
 
-  const historicalLast = getHistoricalLast(syncProgress);
-
-  // Intialize metrics
-
   const totalInterval = [
     hexToNumber(syncProgress.start.number),
-    hexToNumber(historicalLast.number),
+    hexToNumber(last.number),
   ] satisfies Interval;
 
+  common.logger.debug({
+    service: "sync",
+    msg: `Initialized '${network.name}' historical sync for block range [${totalInterval[0]}, ${totalInterval[1]}]`,
+  });
+
   const requiredIntervals = Array.from(
     historicalSync.intervalsCache.entries(),
   ).flatMap(([filter, fragmentIntervals]) =>
@@ -1193,110 +1262,91 @@ export async function* localHistoricalSyncGenerator({
   );
 
   const required = intervalSum(intervalUnion(requiredIntervals));
-
   const total = totalInterval[1] - totalInterval[0] + 1;
 
-  const label = { network: network.name };
-  // Set "ponder_historical_total_blocks"
   common.metrics.ponder_historical_total_blocks.set(label, total);
-  // Set "ponder_historical_sync_cached_blocks"
   common.metrics.ponder_historical_cached_blocks.set(label, total - required);
 
-  if (showLogs) {
-    common.logger.info({
-      service: "historical",
-      msg: `Started syncing '${network.name}' with ${formatPercentage(
-        (total - required) / total,
-      )} cached`,
-    });
-  }
+  // Handle cache hit
+  if (syncProgress.current !== undefined) {
+    common.metrics.ponder_sync_block.set(
+      label,
+      hexToNumber(syncProgress.current.number),
+    );
 
-  /**
-   * Estimate optimal range (blocks) to sync at a time, eventually to be used to
-   * determine `interval` passed to `historicalSync.sync()`.
-   */
-  let estimateRange = 25;
-  // Cursor to track progress.
-  let fromBlock = hexToNumber(syncProgress.start.number);
-
-  /**
-   * Handle a cache hit by fast forwarding and potentially exiting.
-   * A cache hit can either be: (listed by priority)
-   *   1) recovering progress from earlier invocations with different `finalized` blocks
-   *   2) recovering progress from the interval cache
-   */
-  if (
-    syncProgress.current !== undefined &&
-    (syncProgress.cached === undefined ||
-      hexToNumber(syncProgress.current.number) >
-        hexToNumber(syncProgress.cached.number))
-  ) {
-    fromBlock = hexToNumber(syncProgress.current.number) + 1;
-  } else if (syncProgress.cached !== undefined) {
     // `getEvents` can make progress without calling `sync`, so immediately "yield"
-    yield;
-
-    if (
-      hexToNumber(syncProgress.cached.number) ===
-      hexToNumber(historicalLast.number)
-    ) {
-      if (showLogs) {
-        common.logger.info({
-          service: "historical",
-          msg: `Skipped historical sync for '${network.name}' because all blocks are cached.`,
-        });
-      }
+    yield encodeCheckpoint(
+      blockToCheckpoint(syncProgress.current, network.chainId, "up"),
+    );
+
+    if (hexToNumber(syncProgress.current.number) === hexToNumber(last.number)) {
+      common.logger.info({
+        service: "sync",
+        msg: `Skipped '${network.name}' historical sync because all blocks are cached`,
+      });
       return;
+    } else {
+      common.logger.info({
+        service: "sync",
+        msg: `Started '${network.name}' historical sync with ${formatPercentage(
+          (total - required) / total,
+        )} cached`,
+      });
     }
 
-    fromBlock = hexToNumber(syncProgress.cached.number) + 1;
+    cursor = hexToNumber(syncProgress.current.number) + 1;
+  } else {
+    common.logger.info({
+      service: "historical",
+      msg: `Started '${network.name}' historical sync with 0% cached`,
+    });
   }
 
   while (true) {
-    /**
-     * Select a range of blocks to sync bounded by `finalizedBlock`.
-     *
-     * It is important for devEx that the interval is not too large, because
-     * time spent syncing ≈ time before indexing function feedback.
-     */
+    // Select a range of blocks to sync bounded by `finalizedBlock`.
+    // It is important for devEx that the interval is not too large, because
+    // time spent syncing ≈ time before indexing function feedback.
+
     const interval: Interval = [
-      Math.min(fromBlock, hexToNumber(historicalLast.number)),
-      Math.min(fromBlock + estimateRange, hexToNumber(historicalLast.number)),
+      Math.min(cursor, hexToNumber(last.number)),
+      Math.min(cursor + estimateRange, hexToNumber(last.number)),
     ];
 
     const endClock = startClock();
 
-    const syncBlock = await historicalSync.sync(interval);
+    const synced = await historicalSync.sync(interval);
+
+    common.logger.debug({
+      service: "sync",
+      msg: `Synced ${interval[1] - interval[0] + 1} '${network.name}' blocks in range [${interval[0]}, ${interval[1]}]`,
+    });
 
     // Update cursor to record progress
-    fromBlock = interval[1] + 1;
-
-    if (syncBlock === undefined) {
-      /**
-       * `syncBlock` will be undefined if a cache hit occur in `historicalSync.sync()`.
-       * If the all known blocks are synced, then update `syncProgress.current`, else
-       * progress to the next iteration.
-       */
-      if (interval[1] === hexToNumber(historicalLast.number)) {
-        syncProgress.current = historicalLast;
+    cursor = interval[1] + 1;
+
+    // `synced` will be undefined if a cache hit occur in `historicalSync.sync()`.
+
+    if (synced === undefined) {
+      // If the all known blocks are synced, then update `syncProgress.current`, else
+      // progress to the next iteration.
+      if (interval[1] === hexToNumber(last.number)) {
+        syncProgress.current = last;
       } else {
         continue;
       }
     } else {
-      if (interval[1] === hexToNumber(historicalLast.number)) {
-        syncProgress.current = historicalLast;
+      if (interval[1] === hexToNumber(last.number)) {
+        syncProgress.current = last;
       } else {
-        syncProgress.current = syncBlock;
+        syncProgress.current = synced;
       }
 
       const duration = endClock();
 
-      // Update "ponder_sync_block" metric
       common.metrics.ponder_sync_block.set(
         label,
-        hexToNumber(syncProgress.current.number),
+        hexToNumber(syncProgress.current!.number),
       );
-
       common.metrics.ponder_historical_duration.observe(label, duration);
       common.metrics.ponder_historical_completed_blocks.inc(
         label,
@@ -1313,12 +1363,203 @@ export async function* localHistoricalSyncGenerator({
         estimateRange * 2,
         100_000,
       );
+
+      common.logger.debug({
+        service: "sync",
+        msg: `Updated '${network.name}' historical sync estimate to ${estimateRange} blocks`,
+      });
     }
 
-    yield;
+    yield encodeCheckpoint(
+      blockToCheckpoint(syncProgress.current!, network.chainId, "up"),
+    );
 
     if (isSyncEnd(syncProgress) || isSyncFinalized(syncProgress)) {
+      common.logger.info({
+        service: "sync",
+        msg: `Completed '${network.name}' historical sync`,
+      });
       return;
     }
   }
 }
+
+export const getLocalSyncProgress = async ({
+  common,
+  sources,
+  network,
+  requestQueue,
+  intervalsCache,
+}: {
+  common: Common;
+  sources: Source[];
+  network: Network;
+  requestQueue: RequestQueue;
+  intervalsCache: HistoricalSync["intervalsCache"];
+}): Promise<SyncProgress> => {
+  const syncProgress = {} as SyncProgress;
+  const filters = sources.map(({ filter }) => filter);
+
+  // Earliest `fromBlock` among all `filters`
+  const start = Math.min(...filters.map((filter) => filter.fromBlock ?? 0));
+  const cached = getCachedBlock({ filters, intervalsCache });
+
+  const diagnostics = await Promise.all(
+    cached === undefined
+      ? [
+          requestQueue.request({ method: "eth_chainId" }),
+          _eth_getBlockByNumber(requestQueue, { blockTag: "latest" }),
+          _eth_getBlockByNumber(requestQueue, { blockNumber: start }),
+        ]
+      : [
+          requestQueue.request({ method: "eth_chainId" }),
+          _eth_getBlockByNumber(requestQueue, { blockTag: "latest" }),
+          _eth_getBlockByNumber(requestQueue, { blockNumber: start }),
+          _eth_getBlockByNumber(requestQueue, { blockNumber: cached }),
+        ],
+  );
+
+  const finalized = Math.max(
+    0,
+    hexToNumber(diagnostics[1].number) - network.finalityBlockCount,
+  );
+  syncProgress.finalized = await _eth_getBlockByNumber(requestQueue, {
+    blockNumber: finalized,
+  });
+  syncProgress.start = diagnostics[2];
+  if (diagnostics.length === 4) {
+    syncProgress.current = diagnostics[3];
+  }
+
+  // Warn if the config has a different chainId than the remote.
+  if (hexToNumber(diagnostics[0]) !== network.chainId) {
+    common.logger.warn({
+      service: "sync",
+      msg: `Remote chain ID (${diagnostics[0]}) does not match configured chain ID (${network.chainId}) for network "${network.name}"`,
+    });
+  }
+
+  if (filters.some((filter) => filter.toBlock === undefined)) {
+    return syncProgress;
+  }
+
+  // Latest `toBlock` among all `filters`
+  const end = Math.max(...filters.map((filter) => filter.toBlock!));
+
+  if (end > hexToNumber(diagnostics[1].number)) {
+    syncProgress.end = {
+      number: toHex(end),
+      hash: "0x",
+      parentHash: "0x",
+      timestamp: toHex(MAX_CHECKPOINT.blockTimestamp),
+    } satisfies LightBlock;
+  } else {
+    syncProgress.end = await _eth_getBlockByNumber(requestQueue, {
+      blockNumber: end,
+    });
+  }
+
+  return syncProgress;
+};
+
+/** Returns the closest-to-tip block that has been synced for all `sources`. */
+export const getCachedBlock = ({
+  filters,
+  intervalsCache,
+}: {
+  filters: Filter[];
+  intervalsCache: HistoricalSync["intervalsCache"];
+}): number | undefined => {
+  const latestCompletedBlocks = filters.map((filter) => {
+    const requiredInterval = [
+      filter.fromBlock ?? 0,
+      filter.toBlock ?? Number.POSITIVE_INFINITY,
+    ] satisfies Interval;
+    const fragmentIntervals = intervalsCache.get(filter)!;
+
+    const completedIntervals = sortIntervals(
+      intervalIntersection(
+        [requiredInterval],
+        intervalIntersectionMany(
+          fragmentIntervals.map(({ intervals }) => intervals),
+        ),
+      ),
+    );
+
+    if (completedIntervals.length === 0) return undefined;
+
+    const earliestCompletedInterval = completedIntervals[0]!;
+    if (earliestCompletedInterval[0] !== (filter.fromBlock ?? 0)) {
+      return undefined;
+    }
+    return earliestCompletedInterval[1];
+  });
+
+  const minCompletedBlock = Math.min(
+    ...(latestCompletedBlocks.filter(
+      (block) => block !== undefined,
+    ) as number[]),
+  );
+
+  //  Filter i has known progress if a completed interval is found or if
+  // `_latestCompletedBlocks[i]` is undefined but `filters[i].fromBlock`
+  // is > `_minCompletedBlock`.
+
+  if (
+    latestCompletedBlocks.every(
+      (block, i) =>
+        block !== undefined || (filters[i]!.fromBlock ?? 0) > minCompletedBlock,
+    )
+  ) {
+    return minCompletedBlock;
+  }
+
+  return undefined;
+};
+
+/**
+ * Merges multiple event generators into a single generator while preserving
+ * the order of events.
+ *
+ * @param generators - Generators to merge.
+ * @returns A single generator that yields events from all generators.
+ */
+export async function* mergeAsyncGeneratorsWithEventOrder(
+  generators: AsyncGenerator<{ events: RawEvent[]; checkpoint: string }>[],
+): AsyncGenerator<{ events: RawEvent[]; checkpoint: string }> {
+  const results = await Promise.all(generators.map((gen) => gen.next()));
+
+  while (results.some((res) => res.done !== true)) {
+    const supremum = min(
+      ...results.map((res) => (res.done ? undefined : res.value.checkpoint)),
+    );
+
+    const eventArrays: RawEvent[][] = [];
+
+    for (const result of results) {
+      if (result.done === false) {
+        const [left, right] = partition(
+          result.value.events,
+          (event) => event.checkpoint <= supremum,
+        );
+
+        eventArrays.push(left);
+        result.value.events = right;
+      }
+    }
+
+    const events = zipperMany(eventArrays).sort((a, b) =>
+      a.checkpoint < b.checkpoint ? -1 : 1,
+    );
+
+    const index = results.findIndex(
+      (res) => res.done === false && res.value.checkpoint === supremum,
+    );
+
+    const resultPromise = generators[index]!.next();
+    if (events.length > 0) {
+      yield { events, checkpoint: supremum };
+    }
+    results[index] = await resultPromise;
+  }
+}