ponder 0.14.13 → 0.15.0

package/src/runtime/realtime.ts

@@ -1,11 +1,12 @@
+import type { Database } from "@/database/index.js";
 import type { Common } from "@/internal/common.js";
 import type {
   Chain,
   Event,
+  EventCallback,
   Factory,
   Filter,
   IndexingBuild,
-  Source,
   SyncBlock,
   SyncBlockHeader,
 } from "@/internal/types.js";
@@ -23,7 +24,7 @@ import {
   type RealtimeSyncEvent,
   createRealtimeSync,
 } from "@/sync-realtime/index.js";
-import type { SyncStore } from "@/sync-store/index.js";
+import { createSyncStore } from "@/sync-store/index.js";
 import {
   ZERO_CHECKPOINT_STRING,
   blockToCheckpoint,
@@ -36,9 +37,11 @@ import {
   mergeAsyncGenerators,
 } from "@/utils/generators.js";
 import { type Interval, intervalIntersection } from "@/utils/interval.js";
+import { promiseAllSettledWithThrow } from "@/utils/promiseAllSettledWithThrow.js";
 import { promiseWithResolvers } from "@/utils/promiseWithResolvers.js";
 import { startClock } from "@/utils/timer.js";
 import { type Address, hexToNumber } from "viem";
+import { getFilterFactories } from "./filter.js";
 import type { ChildAddresses, SyncProgress } from "./index.js";
 import { getOmnichainCheckpoint } from "./omnichain.js";
 
@@ -57,7 +60,7 @@ export async function* getRealtimeEventsOmnichain(params: {
   common: Common;
   indexingBuild: Pick<
     IndexingBuild,
-    "sources" | "chains" | "rpcs" | "finalizedBlocks"
+    "eventCallbacks" | "chains" | "rpcs" | "finalizedBlocks"
   >;
   perChainSync: Map<
     Chain,
@@ -70,7 +73,7 @@ export async function* getRealtimeEventsOmnichain(params: {
       >[];
     }
   >;
-  syncStore: SyncStore;
+  database: Database;
   pendingEvents: Event[];
 }): AsyncGenerator<RealtimeEvent> {
   const eventGenerators = Array.from(params.perChainSync.entries())
@@ -92,9 +95,10 @@ export async function* getRealtimeEventsOmnichain(params: {
 
       const rpc =
         params.indexingBuild.rpcs[params.indexingBuild.chains.indexOf(chain)]!;
-      const sources = params.indexingBuild.sources.filter(
-        ({ filter }) => filter.chainId === chain.id,
-      );
+      const eventCallbacks =
+        params.indexingBuild.eventCallbacks[
+          params.indexingBuild.chains.findIndex((c) => c.id === chain.id)
+        ]!;
 
       params.common.metrics.ponder_sync_is_realtime.set(
         { chain: chain.name },
@@ -119,10 +123,10 @@ export async function* getRealtimeEventsOmnichain(params: {
           common: params.common,
           chain,
           rpc,
-          sources,
+          eventCallbacks,
           syncProgress,
           childAddresses,
-          syncStore: params.syncStore,
+          database: params.database,
         }),
         100,
         bufferCallback,
@@ -150,23 +154,24 @@ export async function* getRealtimeEventsOmnichain(params: {
     const { syncProgress, childAddresses, unfinalizedBlocks } =
       params.perChainSync.get(chain)!;
 
-    const sources = params.indexingBuild.sources.filter(
-      ({ filter }) => filter.chainId === chain.id,
-    );
+    const eventCallbacks =
+      params.indexingBuild.eventCallbacks[
+        params.indexingBuild.chains.findIndex((c) => c.id === chain.id)
+      ]!;
 
     await handleRealtimeSyncEvent(event, {
       common: params.common,
       chain,
-      sources,
+      eventCallbacks,
       syncProgress,
       unfinalizedBlocks,
-      syncStore: params.syncStore,
+      database: params.database,
     });
 
     switch (event.type) {
       case "block": {
         const events = buildEvents({
-          sources,
+          eventCallbacks,
           chainId: chain.id,
           blocks: [syncBlockToInternal({ block: event.block })],
           logs: event.logs.map((log) => syncLogToInternal({ log })),
@@ -198,7 +203,12 @@ export async function* getRealtimeEventsOmnichain(params: {
           event_count: events.length,
         });
 
-        const decodedEvents = decodeEvents(params.common, sources, events);
+        const decodedEvents = decodeEvents(
+          params.common,
+          chain,
+          eventCallbacks,
+          events,
+        );
 
         params.common.logger.trace({
           msg: "Decoded block events",
@@ -274,7 +284,7 @@ export async function* getRealtimeEventsOmnichain(params: {
       case "reorg": {
         const isReorgedEvent = (_event: Event) => {
           if (
-            _event.chainId === chain.id &&
+            _event.chain.id === chain.id &&
             Number(_event.event.block.number) > hexToNumber(event.block.number)
           ) {
             return true;
@@ -317,7 +327,7 @@ export async function* getRealtimeEventsMultichain(params: {
   common: Common;
   indexingBuild: Pick<
     IndexingBuild,
-    "sources" | "chains" | "rpcs" | "finalizedBlocks"
+    "eventCallbacks" | "chains" | "rpcs" | "finalizedBlocks"
   >;
   perChainSync: Map<
     Chain,
@@ -330,7 +340,7 @@ export async function* getRealtimeEventsMultichain(params: {
       >[];
     }
   >;
-  syncStore: SyncStore;
+  database: Database;
 }): AsyncGenerator<RealtimeEvent> {
   const eventGenerators = Array.from(params.perChainSync.entries())
     .map(([chain, { syncProgress, childAddresses }]) => {
@@ -351,9 +361,10 @@ export async function* getRealtimeEventsMultichain(params: {
 
       const rpc =
         params.indexingBuild.rpcs[params.indexingBuild.chains.indexOf(chain)]!;
-      const sources = params.indexingBuild.sources.filter(
-        ({ filter }) => filter.chainId === chain.id,
-      );
+      const eventCallbacks =
+        params.indexingBuild.eventCallbacks[
+          params.indexingBuild.chains.findIndex((c) => c.id === chain.id)
+        ]!;
 
       params.common.metrics.ponder_sync_is_realtime.set(
         { chain: chain.name },
@@ -378,10 +389,10 @@ export async function* getRealtimeEventsMultichain(params: {
           common: params.common,
           chain,
           rpc,
-          sources,
+          eventCallbacks,
           syncProgress,
           childAddresses,
-          syncStore: params.syncStore,
+          database: params.database,
         }),
         100,
         bufferCallback,
@@ -405,23 +416,24 @@ export async function* getRealtimeEventsMultichain(params: {
     const { syncProgress, childAddresses, unfinalizedBlocks } =
       params.perChainSync.get(chain)!;
 
-    const sources = params.indexingBuild.sources.filter(
-      ({ filter }) => filter.chainId === chain.id,
-    );
+    const eventCallbacks =
+      params.indexingBuild.eventCallbacks[
+        params.indexingBuild.chains.findIndex((c) => c.id === chain.id)
+      ]!;
 
     await handleRealtimeSyncEvent(event, {
       common: params.common,
       chain,
-      sources,
+      eventCallbacks,
       syncProgress,
       unfinalizedBlocks,
-      syncStore: params.syncStore,
+      database: params.database,
     });
 
     switch (event.type) {
       case "block": {
         const events = buildEvents({
-          sources,
+          eventCallbacks,
           chainId: chain.id,
           blocks: [syncBlockToInternal({ block: event.block })],
           logs: event.logs.map((log) => syncLogToInternal({ log })),
@@ -453,7 +465,12 @@ export async function* getRealtimeEventsMultichain(params: {
           event_count: events.length,
         });
 
-        const decodedEvents = decodeEvents(params.common, sources, events);
+        const decodedEvents = decodeEvents(
+          params.common,
+          chain,
+          eventCallbacks,
+          events,
+        );
 
         params.common.logger.trace({
           msg: "Decoded block events",
@@ -498,7 +515,7 @@ export async function* getRealtimeEventsMultichain(params: {
 
         for (const [index, event] of executedEvents.entries()) {
           const _chain = params.indexingBuild.chains.find(
-            (c) => c.id === event.chainId,
+            (c) => c.id === event.chain.id,
           )!;
           const _checkpoint = params.perChainSync
             .get(_chain)!
@@ -531,7 +548,7 @@ export async function* getRealtimeEventsMultichain(params: {
       case "reorg": {
         const isReorgedEvent = (_event: Event) => {
           if (
-            _event.chainId === chain.id &&
+            _event.chain.id === chain.id &&
             Number(_event.event.block.number) > hexToNumber(event.block.number)
           ) {
             return true;
@@ -544,7 +561,7 @@ export async function* getRealtimeEventsMultichain(params: {
         // index of the first reorged event
         let reorgIndex: number | undefined = undefined;
         for (const [index, event] of executedEvents.entries()) {
-          if (event.chainId === chain.id && event.checkpoint > checkpoint) {
+          if (event.chain.id === chain.id && event.checkpoint > checkpoint) {
             reorgIndex = index;
             break;
           }
@@ -574,14 +591,179 @@ export async function* getRealtimeEventsMultichain(params: {
   }
 }
 
+export async function* getRealtimeEventsIsolated(params: {
+  common: Common;
+  indexingBuild: Pick<
+    IndexingBuild,
+    "eventCallbacks" | "chains" | "rpcs" | "finalizedBlocks"
+  >;
+  chain: Chain;
+  syncProgress: SyncProgress;
+  childAddresses: ChildAddresses;
+  unfinalizedBlocks: Omit<
+    Extract<RealtimeSyncEvent, { type: "block" }>,
+    "type"
+  >[];
+  database: Database;
+}): AsyncGenerator<RealtimeEvent> {
+  if (params.syncProgress.isEnd()) {
+    params.common.logger.info({
+      msg: "Skipped live indexing (chain only requires backfill indexing)",
+      chain: params.chain.name,
+      chain_id: params.chain.id,
+      end_block: hexToNumber(params.syncProgress.end!.number),
+    });
+
+    params.common.metrics.ponder_sync_is_complete.set(
+      { chain: params.chain.name },
+      1,
+    );
+    return;
+  }
+
+  const rpc =
+    params.indexingBuild.rpcs[
+      params.indexingBuild.chains.indexOf(params.chain)
+    ]!;
+  const eventCallbacks =
+    params.indexingBuild.eventCallbacks[
+      params.indexingBuild.chains.indexOf(params.chain)
+    ]!;
+
+  params.common.metrics.ponder_sync_is_realtime.set(
+    { chain: params.chain.name },
+    1,
+  );
+
+  const bufferCallback = (bufferSize: number) => {
+    // Note: Only log when the buffer size is greater than 1 because
+    // a buffer size of 1 is not backpressure.
+    if (bufferSize === 1) return;
+    params.common.logger.trace({
+      msg: "Detected live indexing backpressure",
+      chain: params.chain.name,
+      chain_id: params.chain.id,
+      buffer_size: bufferSize,
+      indexing_step: "order block events",
+    });
+  };
+
+  const eventGenerator = bufferAsyncGenerator(
+    getRealtimeEventGenerator({
+      common: params.common,
+      chain: params.chain,
+      rpc,
+      eventCallbacks,
+      syncProgress: params.syncProgress,
+      childAddresses: params.childAddresses,
+      database: params.database,
+    }),
+    100,
+    bufferCallback,
+  );
+
+  for await (const { chain, event } of eventGenerator) {
+    await handleRealtimeSyncEvent(event, {
+      common: params.common,
+      chain,
+      eventCallbacks,
+      syncProgress: params.syncProgress,
+      unfinalizedBlocks: params.unfinalizedBlocks,
+      database: params.database,
+    });
+
+    switch (event.type) {
+      case "block": {
+        const rawEvents = buildEvents({
+          eventCallbacks,
+          chainId: chain.id,
+          blocks: [syncBlockToInternal({ block: event.block })],
+          logs: event.logs.map((log) => syncLogToInternal({ log })),
+          transactions: event.transactions.map((transaction) =>
+            syncTransactionToInternal({ transaction }),
+          ),
+          transactionReceipts: event.transactionReceipts.map(
+            (transactionReceipt) =>
+              syncTransactionReceiptToInternal({ transactionReceipt }),
+          ),
+          traces: event.traces.map((trace) =>
+            syncTraceToInternal({
+              trace,
+              block: event.block,
+              transaction: event.transactions.find(
+                (t) => t.hash === trace.transactionHash,
+              )!,
+            }),
+          ),
+          childAddresses: params.childAddresses,
+        });
+
+        params.common.logger.trace({
+          msg: "Constructed events from block",
+          chain: chain.name,
+          chain_id: chain.id,
+          number: hexToNumber(event.block.number),
+          hash: event.block.hash,
+          event_count: rawEvents.length,
+        });
+
+        const events = decodeEvents(
+          params.common,
+          chain,
+          eventCallbacks,
+          rawEvents,
+        );
+
+        params.common.logger.trace({
+          msg: "Decoded block events",
+          chain: chain.name,
+          chain_id: chain.id,
+          number: hexToNumber(event.block.number),
+          hash: event.block.hash,
+          event_count: events.length,
+        });
+
+        const checkpoint = params.syncProgress.getCheckpoint({
+          tag: "current",
+        });
+
+        yield {
+          type: "block",
+          events,
+          chain,
+          checkpoint,
+          blockCallback: event.blockCallback,
+        };
+        break;
+      }
+      case "finalize": {
+        const checkpoint = params.syncProgress.getCheckpoint({
+          tag: "finalized",
+        });
+
+        yield { type: "finalize", chain, checkpoint };
+        break;
+      }
+      case "reorg": {
+        const checkpoint = params.syncProgress.getCheckpoint({
+          tag: "current",
+        });
+
+        yield { type: "reorg", chain, checkpoint };
+        break;
+      }
+    }
+  }
+}
+
 export async function* getRealtimeEventGenerator(params: {
   common: Common;
   chain: Chain;
   rpc: Rpc;
-  sources: Source[];
+  eventCallbacks: EventCallback[];
   syncProgress: SyncProgress;
   childAddresses: ChildAddresses;
-  syncStore: SyncStore;
+  database: Database;
 }) {
   const realtimeSync = createRealtimeSync(params);
 
@@ -689,13 +871,13 @@ export async function handleRealtimeSyncEvent(
   params: {
     common: Common;
     chain: Chain;
-    sources: Source[];
+    eventCallbacks: EventCallback[];
     syncProgress: SyncProgress;
     unfinalizedBlocks: Omit<
       Extract<RealtimeSyncEvent, { type: "block" }>,
       "type"
     >[];
-    syncStore: SyncStore;
+    database: Database;
   },
 ) {
   switch (event.type) {
@@ -763,96 +945,110 @@ export async function handleRealtimeSyncEvent(
         logger: params.common.logger.child({ action: "finalize_block_range" }),
       };
 
-      await Promise.all([
-        params.syncStore.insertBlocks(
-          {
-            blocks: finalizedBlocks
-              .filter(({ hasMatchedFilter }) => hasMatchedFilter)
-              .map(({ block }) => block),
-            chainId: params.chain.id,
-          },
-          context,
-        ),
-        params.syncStore.insertTransactions(
-          {
-            transactions: finalizedBlocks.flatMap(
-              ({ transactions }) => transactions,
-            ),
-            chainId: params.chain.id,
-          },
-          context,
-        ),
-        params.syncStore.insertTransactionReceipts(
-          {
-            transactionReceipts: finalizedBlocks.flatMap(
-              ({ transactionReceipts }) => transactionReceipts,
-            ),
-            chainId: params.chain.id,
-          },
-          context,
-        ),
-        params.syncStore.insertLogs(
-          {
-            logs: finalizedBlocks.flatMap(({ logs }) => logs),
-            chainId: params.chain.id,
-          },
-          context,
-        ),
-        params.syncStore.insertTraces(
-          {
-            traces: finalizedBlocks.flatMap(({ traces, block, transactions }) =>
-              traces.map((trace) => ({
-                trace,
-                block: block as SyncBlock, // SyncBlock is expected for traces.length !== 0
-                transaction: transactions.find(
-                  (t) => t.hash === trace.transactionHash,
-                )!,
-              })),
-            ),
-            chainId: params.chain.id,
-          },
-          context,
-        ),
-        ...Array.from(childAddresses.entries()).map(
-          ([factory, childAddresses]) =>
-            params.syncStore.insertChildAddresses(
-              {
-                factory,
-                childAddresses,
-                chainId: params.chain.id,
-              },
-              context,
-            ),
-        ),
-      ]);
-
-      // Add corresponding intervals to the sync-store
-      // Note: this should happen after insertion so the database doesn't become corrupted
+      await params.database.syncQB.transaction(
+        async (tx) => {
+          const syncStore = createSyncStore({ common: params.common, qb: tx });
 
-      const syncedIntervals: {
-        interval: Interval;
-        filter: Filter;
-      }[] = [];
-
-      for (const { filter } of params.sources) {
-        const intervals = intervalIntersection(
-          [finalizedInterval],
-          [[filter.fromBlock ?? 0, filter.toBlock ?? Number.POSITIVE_INFINITY]],
-        );
+          await promiseAllSettledWithThrow([
+            syncStore.insertBlocks({
+              blocks: finalizedBlocks
+                .filter(({ hasMatchedFilter }) => hasMatchedFilter)
+                .map(({ block }) => block),
+              chainId: params.chain.id,
+            }),
+            syncStore.insertTransactions({
+              transactions: finalizedBlocks.flatMap(
+                ({ transactions }) => transactions,
+              ),
+              chainId: params.chain.id,
+            }),
+            syncStore.insertTransactionReceipts({
+              transactionReceipts: finalizedBlocks.flatMap(
+                ({ transactionReceipts }) => transactionReceipts,
+              ),
+              chainId: params.chain.id,
+            }),
+            syncStore.insertLogs({
+              logs: finalizedBlocks.flatMap(({ logs }) => logs),
+              chainId: params.chain.id,
+            }),
+            syncStore.insertTraces({
+              traces: finalizedBlocks.flatMap(
+                ({ traces, block, transactions }) =>
+                  traces.map((trace) => ({
+                    trace,
+                    block: block as SyncBlock, // SyncBlock is expected for traces.length !== 0
+                    transaction: transactions.find(
+                      (t) => t.hash === trace.transactionHash,
+                    )!,
+                  })),
+              ),
+              chainId: params.chain.id,
+            }),
+            ...Array.from(childAddresses.entries()).map(
+              ([factory, childAddresses]) =>
+                syncStore.insertChildAddresses({
+                  factory,
+                  childAddresses,
+                  chainId: params.chain.id,
+                }),
+            ),
+          ]);
+
+          const intervals: {
+            interval: Interval;
+            filter: Filter;
+          }[] = [];
+
+          const factoryIntervals: {
+            interval: Interval;
+            factory: Factory;
+          }[] = [];
+
+          for (const { filter } of params.eventCallbacks) {
+            const completedIntervals = intervalIntersection(
+              [finalizedInterval],
+              [
+                [
+                  filter.fromBlock ?? 0,
+                  filter.toBlock ?? Number.POSITIVE_INFINITY,
+                ],
+              ],
+            );
+
+            for (const interval of completedIntervals) {
+              intervals.push({ interval, filter });
+            }
 
-        for (const interval of intervals) {
-          syncedIntervals.push({ interval, filter });
-        }
-      }
+            for (const factory of getFilterFactories(filter)) {
+              const completedIntervals = intervalIntersection(
+                [finalizedInterval],
+                [
+                  [
+                    factory.fromBlock ?? 0,
+                    factory.toBlock ?? Number.POSITIVE_INFINITY,
+                  ],
+                ],
+              );
+
+              for (const interval of completedIntervals) {
+                factoryIntervals.push({ interval, factory });
+              }
+            }
+          }
 
-      await params.syncStore.insertIntervals(
-        {
-          intervals: syncedIntervals,
-          chainId: params.chain.id,
+          await syncStore.insertIntervals(
+            {
+              intervals,
+              factoryIntervals,
+              chainId: params.chain.id,
+            },
+            context,
+          );
         },
+        undefined,
         context,
       );
-
       break;
     }
     case "reorg": {
@@ -878,7 +1074,10 @@ export async function handleRealtimeSyncEvent(
         } else break;
       }
 
-      await params.syncStore.pruneRpcRequestResults(
+      await createSyncStore({
+        common: params.common,
+        qb: params.database.syncQB,
+      }).pruneRpcRequestResults(
         {
           chainId: params.chain.id,
           blocks: event.reorgedBlocks,