ponder 0.14.13 → 0.15.0

package/src/runtime/multichain.ts

@@ -1,12 +1,13 @@
-import { runCodegen } from "@/bin/utils/codegen.js";
 import {
   commitBlock,
   createIndexes,
+  createLiveQueryTriggers,
   createTriggers,
   createViews,
+  dropLiveQueryTriggers,
   dropTriggers,
-  finalize,
-  revert,
+  finalizeMultichain,
+  revertMultichain,
 } from "@/database/actions.js";
 import {
   type Database,
@@ -86,17 +87,15 @@ export async function runMultichain({
   crashRecoveryCheckpoint: CrashRecoveryCheckpoint;
   database: Database;
 }) {
-  runCodegen({ common });
-
   const columnAccessPattern = createColumnAccessPattern({
     indexingBuild,
   });
-  const syncStore = createSyncStore({ common, database });
+  const syncStore = createSyncStore({ common, qb: database.syncQB });
 
   const PONDER_CHECKPOINT = getPonderCheckpointTable(namespaceBuild.schema);
   const PONDER_META = getPonderMetaTable(namespaceBuild.schema);
 
-  let eventCount = getEventCount(indexingBuild.indexingFunctions);
+  const eventCount = getEventCount(indexingBuild.indexingFunctions);
 
   const cachedViemClient = createCachedViemClient({
     common,
@@ -122,7 +121,6 @@ export async function runMultichain({
     common,
     indexingBuild,
     client: cachedViemClient,
-    eventCount,
     indexingErrorHandler,
     columnAccessPattern,
   });
@@ -157,18 +155,17 @@ export async function runMultichain({
 
   await Promise.all(
     indexingBuild.chains.map(async (chain) => {
-      const sources = indexingBuild.sources.filter(
-        ({ filter }) => filter.chainId === chain.id,
-      );
+      const eventCallbacks =
+        indexingBuild.eventCallbacks[indexingBuild.chains.indexOf(chain)]!;
 
       const cachedIntervals = await getCachedIntervals({
         chain,
-        sources,
+        filters: eventCallbacks.map(({ filter }) => filter),
         syncStore,
       });
       const syncProgress = await initSyncProgress({
         common,
-        sources,
+        filters: eventCallbacks.map(({ filter }) => filter),
         chain,
         rpc: indexingBuild.rpcs[indexingBuild.chains.indexOf(chain)]!,
         finalizedBlock:
@@ -176,7 +173,7 @@ export async function runMultichain({
         cachedIntervals,
       });
       const childAddresses = await getChildAddresses({
-        sources,
+        filters: eventCallbacks.map(({ filter }) => filter),
         syncStore,
       });
       const unfinalizedBlocks: Omit<
@@ -302,14 +299,14 @@ export async function runMultichain({
     // underlying metrics collection is actually synchronous
     // https://github.com/siimon/prom-client/blob/master/lib/histogram.js#L102-L125
     const { eta, progress } = await getAppProgress(common.metrics);
-    if (eta === undefined || progress === undefined) {
+    if (eta === undefined && progress === undefined) {
       return;
     }
 
     common.logger.info({
       msg: "Updated backfill indexing progress",
-      progress: formatPercentage(progress),
-      estimate: formatEta(eta * 1_000),
+      progress: progress === undefined ? undefined : formatPercentage(progress),
+      estimate: eta === undefined ? undefined : formatEta(eta * 1_000),
     });
   }, 5_000);
 
@@ -331,7 +328,7 @@ export async function runMultichain({
       indexingBuild,
       crashRecoveryCheckpoint,
       perChainSync,
-      syncStore,
+      database,
     }),
     (params) => {
       common.metrics.ponder_historical_concurrency_group_duration.inc(
@@ -364,7 +361,9 @@ export async function runMultichain({
     let endClock = startClock();
     await database.userQB.transaction(
       async (tx) => {
-        const initialEventCount = structuredClone(eventCount);
+        const initialCompletedEvents = structuredClone(
+          await common.metrics.ponder_indexing_completed_events.get(),
+        );
 
         try {
           historicalIndexingStore.qb = tx;
@@ -453,7 +452,12 @@ export async function runMultichain({
           );
           endClock = startClock();
         } catch (error) {
-          eventCount = initialEventCount;
+          for (const value of initialCompletedEvents.values) {
+            common.metrics.ponder_indexing_completed_events.set(
+              value.labels,
+              value.value,
+            );
+          }
           indexingCache.invalidate();
           indexingCache.clear();
 
@@ -526,7 +530,7 @@ export async function runMultichain({
       ),
     );
     common.metrics.ponder_indexing_timestamp.set(
-      { chain: chain.name },
+      label,
       seconds[chain.name]!.end,
     );
   }
@@ -563,6 +567,7 @@ export async function runMultichain({
   endClock = startClock();
 
   await createTriggers(database.adminQB, { tables });
+  await createLiveQueryTriggers(database.adminQB, { namespaceBuild, tables });
 
   common.logger.debug({
     msg: "Created database triggers",
@@ -583,6 +588,8 @@ export async function runMultichain({
     });
   }
 
+  endClock = startClock();
+
   await database.adminQB.wrap({ label: "update_ready" }, (db) =>
     db
       .update(PONDER_META)
@@ -616,7 +623,7 @@ export async function runMultichain({
       common,
       indexingBuild,
       perChainSync,
-      syncStore,
+      database,
     }),
     100,
     bufferCallback,
@@ -628,93 +635,113 @@ export async function runMultichain({
         };
         const endClock = startClock();
 
-        if (event.events.length > 0) {
-          // Events must be run block-by-block, so that `database.commitBlock` can accurately
-          // update the temporary `checkpoint` value set in the trigger.
-
-          const perBlockEvents = splitEvents(event.events);
-
-          for (const { checkpoint, events } of perBlockEvents) {
-            await database.userQB.transaction(
-              async (tx) => {
-                const chain = indexingBuild.chains.find(
-                  (chain) =>
-                    chain.id === Number(decodeCheckpoint(checkpoint).chainId),
-                )!;
-
-                try {
-                  realtimeIndexingStore.qb = tx;
-                  realtimeIndexingStore.isProcessingEvents = true;
-
-                  common.logger.trace({
-                    msg: "Processing block events",
-                    chain: chain.name,
-                    chain_id: chain.id,
-                    number: Number(decodeCheckpoint(checkpoint).blockNumber),
-                    event_count: events.length,
-                  });
-
-                  await indexing.processRealtimeEvents({
-                    events,
-                    db: realtimeIndexingStore,
-                  });
+        await database.userQB.transaction(
+          async (tx) => {
+            if (database.userQB.$dialect === "postgres") {
+              await tx.wrap(
+                (tx) =>
+                  tx.execute(
+                    "CREATE TEMP TABLE live_query_tables (table_name TEXT PRIMARY KEY) ON COMMIT DROP",
+                  ),
+                context,
+              );
+            } else {
+              await tx.wrap(
+                (tx) =>
+                  tx.execute(
+                    "CREATE TEMP TABLE IF NOT EXISTS live_query_tables (table_name TEXT PRIMARY KEY)",
+                  ),
+                context,
+              );
+            }
 
-                  common.logger.trace({
-                    msg: "Processed block events",
+            // Events must be run block-by-block, so that `database.commitBlock` can accurately
+            // update the temporary `checkpoint` value set in the trigger.
+            for (const { checkpoint, events } of splitEvents(event.events)) {
+              const chain = indexingBuild.chains.find(
+                (chain) =>
+                  chain.id === Number(decodeCheckpoint(checkpoint).chainId),
+              )!;
+
+              try {
+                realtimeIndexingStore.qb = tx;
+                realtimeIndexingStore.isProcessingEvents = true;
+
+                common.logger.trace({
+                  msg: "Processing block events",
+                  chain: chain.name,
+                  chain_id: chain.id,
+                  number: Number(decodeCheckpoint(checkpoint).blockNumber),
+                  event_count: events.length,
+                });
+
+                await indexing.processRealtimeEvents({
+                  events,
+                  db: realtimeIndexingStore,
+                });
+
+                common.logger.trace({
+                  msg: "Processed block events",
+                  chain: chain.name,
+                  chain_id: chain.id,
+                  number: Number(decodeCheckpoint(checkpoint).blockNumber),
+                  event_count: events.length,
+                });
+
+                realtimeIndexingStore.isProcessingEvents = false;
+
+                await Promise.all(
+                  tables.map((table) =>
+                    commitBlock(tx, { table, checkpoint, preBuild }, context),
+                  ),
+                );
+
+                common.logger.trace({
+                  msg: "Committed reorg data for block",
+                  chain: chain.name,
+                  chain_id: chain.id,
+                  number: Number(decodeCheckpoint(checkpoint).blockNumber),
+                  event_count: events.length,
+                  checkpoint,
+                });
+              } catch (error) {
+                if (error instanceof NonRetryableUserError === false) {
+                  common.logger.warn({
+                    msg: "Failed to index block",
                     chain: chain.name,
                     chain_id: chain.id,
                     number: Number(decodeCheckpoint(checkpoint).blockNumber),
-                    event_count: events.length,
+                    error: error,
                   });
+                }
 
-                  realtimeIndexingStore.isProcessingEvents = false;
-
-                  await Promise.all(
-                    tables.map((table) =>
-                      commitBlock(tx, { table, checkpoint }, context),
-                    ),
-                  );
+                throw error;
+              }
 
-                  common.logger.trace({
-                    msg: "Committed reorg data for block",
-                    chain: chain.name,
-                    chain_id: chain.id,
-                    number: Number(decodeCheckpoint(checkpoint).blockNumber),
-                    event_count: events.length,
-                    checkpoint,
-                  });
+              common.metrics.ponder_indexing_timestamp.set(
+                { chain: chain.name },
+                Number(decodeCheckpoint(checkpoint).blockTimestamp),
+              );
+            }
 
-                  common.metrics.ponder_indexing_timestamp.set(
-                    { chain: chain.name },
-                    Number(decodeCheckpoint(checkpoint).blockTimestamp),
-                  );
-                } catch (error) {
-                  if (error instanceof NonRetryableUserError === false) {
-                    common.logger.warn({
-                      msg: "Failed to index block",
-                      chain: chain.name,
-                      chain_id: chain.id,
-                      number: Number(decodeCheckpoint(checkpoint).blockNumber),
-                      error: error,
-                    });
-                  }
-
-                  throw error;
-                }
-              },
-              undefined,
+            await tx.wrap(
+              { label: "update_checkpoints" },
+              (db) =>
+                db
+                  .update(PONDER_CHECKPOINT)
+                  .set({ latestCheckpoint: event.checkpoint })
+                  .where(eq(PONDER_CHECKPOINT.chainName, event.chain.name)),
               context,
             );
-          }
-        }
 
-        await database.userQB.wrap(
-          { label: "update_checkpoints" },
-          (db) =>
-            db
-              .update(PONDER_CHECKPOINT)
-              .set({ latestCheckpoint: event.checkpoint })
-              .where(eq(PONDER_CHECKPOINT.chainName, event.chain.name)),
+            if (database.userQB.$dialect === "pglite") {
+              await tx.wrap(
+                (tx) => tx.execute("TRUNCATE TABLE live_query_tables"),
+                context,
+              );
+            }
+          },
+          undefined,
           context,
         );
 
@@ -743,13 +770,17 @@ export async function runMultichain({
         await database.userQB.transaction(
           async (tx) => {
             await dropTriggers(tx, { tables }, context);
+            await dropLiveQueryTriggers(
+              tx,
+              { namespaceBuild, tables },
+              context,
+            );
 
-            const counts = await revert(
+            const counts = await revertMultichain(
               tx,
               {
                 checkpoint: event.checkpoint,
                 tables,
-                preBuild,
               },
               context,
             );
@@ -763,6 +794,11 @@ export async function runMultichain({
             }
 
             await createTriggers(tx, { tables }, context);
+            await createLiveQueryTriggers(
+              tx,
+              { namespaceBuild, tables },
+              context,
+            );
           },
           undefined,
           context,
@@ -784,12 +820,11 @@ export async function runMultichain({
         };
         const endClock = startClock();
 
-        await finalize(
+        await finalizeMultichain(
           database.userQB,
           {
             checkpoint: event.checkpoint,
             tables,
-            preBuild,
             namespaceBuild,
           },
           context,