ponder 0.9.2 → 0.9.3

package/src/bin/utils/{shutdown.ts → exit.ts}

@@ -1,34 +1,26 @@
 import os from "node:os";
 import readline from "node:readline";
 import type { Common } from "@/internal/common.js";
-import { IgnorableError } from "@/internal/errors.js";
+import { ShutdownError } from "@/internal/errors.js";
 
 const SHUTDOWN_GRACE_PERIOD_MS = 5_000;
 
-/**
- * Sets up shutdown handlers for the process. Accepts additional cleanup logic to run.
- */
-export function setupShutdown({
+/** Sets up shutdown handlers for the process. Accepts additional cleanup logic to run. */
+export const createExit = ({
   common,
-  cleanup,
 }: {
-  common: Common;
-  cleanup: () => Promise<void>;
-}) {
+  common: Pick<Common, "logger" | "telemetry" | "shutdown">;
+}) => {
   let isShuttingDown = false;
 
-  const shutdown = async ({
-    reason,
-    code,
-  }: { reason: string; code: 0 | 1 }) => {
+  const exit = async ({ reason, code }: { reason: string; code: 0 | 1 }) => {
     if (isShuttingDown) return;
     isShuttingDown = true;
-    setTimeout(async () => {
+    const timeout = setTimeout(async () => {
       common.logger.fatal({
         service: "process",
         msg: "Failed to shutdown within 5 seconds, terminating (exit code 1)",
       });
-      await common.logger.kill();
       process.exit(1);
     }, SHUTDOWN_GRACE_PERIOD_MS);
 
@@ -43,15 +35,14 @@ export function setupShutdown({
       properties: { duration_seconds: process.uptime() },
     });
 
-    await cleanup();
+    await common.shutdown.kill();
+    clearTimeout(timeout);
 
-    const level = code === 0 ? "info" : "fatal";
-    common.logger[level]({
-      service: "process",
-      msg: `Finished shutdown sequence, terminating (exit code ${code})`,
-    });
+    if (process.stdin.isTTY) {
+      process.stdin.setRawMode(false);
+      process.stdin.pause();
+    }
 
-    await common.logger.kill();
     process.exit(code);
   };
 
@@ -61,36 +52,31 @@ export function setupShutdown({
       output: process.stdout,
     });
     readlineInterface.on("SIGINT", () =>
-      shutdown({ reason: "Received SIGINT", code: 0 }),
+      exit({ reason: "Received SIGINT", code: 0 }),
     );
   }
 
-  process.on("SIGINT", () => shutdown({ reason: "Received SIGINT", code: 0 }));
-  process.on("SIGTERM", () =>
-    shutdown({ reason: "Received SIGTERM", code: 0 }),
-  );
-  process.on("SIGQUIT", () =>
-    shutdown({ reason: "Received SIGQUIT", code: 0 }),
-  );
-
+  process.on("SIGINT", () => exit({ reason: "Received SIGINT", code: 0 }));
+  process.on("SIGTERM", () => exit({ reason: "Received SIGTERM", code: 0 }));
+  process.on("SIGQUIT", () => exit({ reason: "Received SIGQUIT", code: 0 }));
   process.on("uncaughtException", (error: Error) => {
-    if (error instanceof IgnorableError) return;
+    if (error instanceof ShutdownError) return;
     common.logger.error({
       service: "process",
       msg: "Caught uncaughtException event",
       error,
     });
-    shutdown({ reason: "Received uncaughtException", code: 1 });
+    exit({ reason: "Received uncaughtException", code: 1 });
   });
   process.on("unhandledRejection", (error: Error) => {
-    if (error instanceof IgnorableError) return;
+    if (error instanceof ShutdownError) return;
     common.logger.error({
       service: "process",
       msg: "Caught unhandledRejection event",
       error,
     });
-    shutdown({ reason: "Received unhandledRejection", code: 1 });
+    exit({ reason: "Received unhandledRejection", code: 1 });
   });
 
-  return shutdown;
-}
+  return exit;
+};

package/src/bin/utils/run.ts

@@ -6,22 +6,24 @@ import { createRealtimeIndexingStore } from "@/indexing-store/realtime.js";
 import { createIndexingService } from "@/indexing/index.js";
 import type { Common } from "@/internal/common.js";
 import { getAppProgress } from "@/internal/metrics.js";
-import type { Event, IndexingBuild, SchemaBuild } from "@/internal/types.js";
+import type { IndexingBuild, PreBuild, SchemaBuild } from "@/internal/types.js";
 import { createSyncStore } from "@/sync-store/index.js";
 import { decodeEvents } from "@/sync/events.js";
 import { type RealtimeEvent, createSync, splitEvents } from "@/sync/index.js";
 import {
+  ZERO_CHECKPOINT_STRING,
   decodeCheckpoint,
-  encodeCheckpoint,
-  zeroCheckpoint,
 } from "@/utils/checkpoint.js";
+import { chunk } from "@/utils/chunk.js";
 import { formatEta, formatPercentage } from "@/utils/format.js";
+import { createMutex } from "@/utils/mutex.js";
 import { never } from "@/utils/never.js";
-import { createQueue } from "@ponder/common";
+import { createRequestQueue } from "@/utils/requestQueue.js";
 
 /** Starts the sync and indexing services for the specified build. */
 export async function run({
   common,
+  preBuild,
   schemaBuild,
   indexingBuild,
   database,
@@ -29,146 +31,178 @@ export async function run({
   onReloadableError,
 }: {
   common: Common;
+  preBuild: PreBuild;
   schemaBuild: SchemaBuild;
   indexingBuild: IndexingBuild;
   database: Database;
   onFatalError: (error: Error) => void;
   onReloadableError: (error: Error) => void;
 }) {
-  let isKilled = false;
-
   const initialCheckpoint = await database.recoverCheckpoint();
+  await database.migrateSync();
+
+  const requestQueues = indexingBuild.networks.map((network) =>
+    createRequestQueue({ network, common }),
+  );
+
   const syncStore = createSyncStore({ common, database });
   const metadataStore = getMetadataStore({ database });
 
-  // This can be a long-running operation, so it's best to do it after
-  // starting the server so the app can become responsive more quickly.
-  await database.migrateSync();
-
-  runCodegen({ common });
+  const realtimeMutex = createMutex();
 
-  // Note: can throw
   const sync = await createSync({
     common,
     indexingBuild,
+    requestQueues,
     syncStore,
-    // Note: this is not great because it references the
-    // `realtimeQueue` which isn't defined yet
     onRealtimeEvent: (realtimeEvent) => {
-      return realtimeQueue.add(realtimeEvent);
+      if (realtimeEvent.type === "reorg") {
+        realtimeMutex.clear();
+      }
+
+      return onRealtimeEvent(realtimeEvent);
     },
     onFatalError,
     initialCheckpoint,
-  });
-
-  const handleEvents = async (events: Event[], checkpoint: string) => {
-    if (events.length === 0) return { status: "success" } as const;
-
-    indexingService.updateTotalSeconds(decodeCheckpoint(checkpoint));
-
-    return await indexingService.processEvents({ events });
-  };
-
-  const realtimeQueue = createQueue({
-    initialStart: true,
-    browser: false,
-    concurrency: 1,
-    worker: async (event: RealtimeEvent) => {
-      switch (event.type) {
-        case "block": {
-          // Events must be run block-by-block, so that `database.complete` can accurately
-          // update the temporary `checkpoint` value set in the trigger.
-          for (const { checkpoint, events } of splitEvents(event.events)) {
-            const result = await handleEvents(
-              decodeEvents(common, indexingBuild.sources, events),
-              event.checkpoint,
-            );
-
-            if (result.status === "error") onReloadableError(result.error);
-
-            // Set reorg table `checkpoint` column for newly inserted rows.
-            await database.complete({ checkpoint });
-          }
-
-          await metadataStore.setStatus(event.status);
-
-          break;
-        }
-        case "reorg":
-          await database.removeTriggers();
-          await database.revert({ checkpoint: event.checkpoint });
-          await database.createTriggers();
-
-          break;
-
-        case "finalize":
-          await database.finalize({ checkpoint: event.checkpoint });
-          break;
-
-        default:
-          never(event);
-      }
-    },
+    ordering: preBuild.ordering,
   });
 
   const indexingService = createIndexingService({
     common,
     indexingBuild,
-    sync,
+    requestQueues,
+    syncStore,
   });
 
   const historicalIndexingStore = createHistoricalIndexingStore({
     common,
     schemaBuild,
     database,
-    initialCheckpoint,
+    isDatabaseEmpty: initialCheckpoint === ZERO_CHECKPOINT_STRING,
   });
 
   indexingService.setIndexingStore(historicalIndexingStore);
 
+  runCodegen({ common });
+
   await metadataStore.setStatus(sync.getStatus());
 
-  const start = async () => {
-    // If the initial checkpoint is zero, we need to run setup events.
-    if (encodeCheckpoint(zeroCheckpoint) === initialCheckpoint) {
-      const result = await indexingService.processSetupEvents({
-        sources: indexingBuild.sources,
-        networks: indexingBuild.networks,
-      });
-      if (result.status === "killed") {
-        return;
-      } else if (result.status === "error") {
-        onReloadableError(result.error);
-        return;
-      }
-    }
+  for (const network of indexingBuild.networks) {
+    const label = { network: network.name };
+    common.metrics.ponder_historical_total_indexing_seconds.set(
+      label,
+      Math.max(
+        sync.seconds[network.name]!.end - sync.seconds[network.name]!.start,
+        0,
+      ),
+    );
+    common.metrics.ponder_historical_cached_indexing_seconds.set(
+      label,
+      Math.max(
+        sync.seconds[network.name]!.cached - sync.seconds[network.name]!.start,
+        0,
+      ),
+    );
+    common.metrics.ponder_historical_completed_indexing_seconds.set(label, 0);
+    common.metrics.ponder_indexing_timestamp.set(
+      label,
+      Math.max(
+        sync.seconds[network.name]!.cached,
+        sync.seconds[network.name]!.start,
+      ),
+    );
+  }
 
-    // Track the last processed checkpoint, used to set metrics
-    let end: string | undefined;
-    let lastFlush = Date.now();
+  // Reset the start timestamp so the eta estimate doesn't include
+  // the startup time.
+  common.metrics.start_timestamp = Date.now();
 
-    // Run historical indexing until complete.
-    for await (const { events, checkpoint } of sync.getEvents()) {
-      end = checkpoint;
+  // If the initial checkpoint is zero, we need to run setup events.
+  if (initialCheckpoint === ZERO_CHECKPOINT_STRING) {
+    const result = await indexingService.processSetupEvents({
+      sources: indexingBuild.sources,
+      networks: indexingBuild.networks,
+    });
 
+    if (result.status === "error") {
+      onReloadableError(result.error);
+      return;
+    }
+  }
+
+  let lastFlush = Date.now();
+
+  // Run historical indexing until complete.
+  for await (const events of sync.getEvents()) {
+    if (events.length > 0) {
       const decodedEvents = decodeEvents(common, indexingBuild.sources, events);
-      const result = await handleEvents(decodedEvents, checkpoint);
+      const eventChunks = chunk(decodedEvents, 93);
+      common.logger.debug({
+        service: "app",
+        msg: `Decoded ${decodedEvents.length} events`,
+      });
+      for (const eventChunk of eventChunks) {
+        const result = await indexingService.processEvents({
+          events: eventChunk,
+        });
+
+        if (result.status === "error") {
+          onReloadableError(result.error);
+          return;
+        }
+
+        const checkpoint = decodeCheckpoint(
+          eventChunk[eventChunk.length - 1]!.checkpoint,
+        );
+
+        if (preBuild.ordering === "multichain") {
+          const network = indexingBuild.networks.find(
+            (network) => network.chainId === Number(checkpoint.chainId),
+          )!;
+          common.metrics.ponder_historical_completed_indexing_seconds.set(
+            { network: network.name },
+            Math.max(
+              checkpoint.blockTimestamp - sync.seconds[network.name]!.start,
+              0,
+            ),
+          );
+          common.metrics.ponder_indexing_timestamp.set(
+            { network: network.name },
+            checkpoint.blockTimestamp,
+          );
+        } else {
+          for (const network of indexingBuild.networks) {
+            common.metrics.ponder_historical_completed_indexing_seconds.set(
+              { network: network.name },
+              Math.max(
+                checkpoint.blockTimestamp - sync.seconds[network.name]!.start,
+                0,
+              ),
+            );
+            common.metrics.ponder_indexing_timestamp.set(
+              { network: network.name },
+              checkpoint.blockTimestamp,
+            );
+          }
+        }
+
+        // Note: allows for terminal and logs to be updated
+        await new Promise(setImmediate);
+      }
 
       // underlying metrics collection is actually synchronous
       // https://github.com/siimon/prom-client/blob/master/lib/histogram.js#L102-L125
       const { eta, progress } = await getAppProgress(common.metrics);
-      if (events.length > 0) {
-        if (eta === undefined || progress === undefined) {
-          common.logger.info({
-            service: "app",
-            msg: `Indexed ${events.length} events`,
-          });
-        } else {
-          common.logger.info({
-            service: "app",
-            msg: `Indexed ${events.length} events with ${formatPercentage(progress)} complete and ${formatEta(eta)} remaining`,
-          });
-        }
+      if (eta === undefined || progress === undefined) {
+        common.logger.info({
+          service: "app",
+          msg: `Indexed ${decodedEvents.length} events`,
+        });
+      } else {
+        common.logger.info({
+          service: "app",
+          msg: `Indexed ${decodedEvents.length} events with ${formatPercentage(progress)} complete and ${formatEta(eta * 1_000)} remaining`,
+        });
       }
 
       // Persist the indexing store to the db if it is too full. The `finalized`
@@ -193,13 +227,9 @@ export async function run({
           });
         }
 
-        await database.finalize({
-          checkpoint: encodeCheckpoint(zeroCheckpoint),
-        });
+        await database.finalize({ checkpoint: ZERO_CHECKPOINT_STRING });
         await historicalIndexingStore.flush();
-        await database.complete({
-          checkpoint: encodeCheckpoint(zeroCheckpoint),
-        });
+        await database.complete({ checkpoint: ZERO_CHECKPOINT_STRING });
         await database.finalize({
           checkpoint: events[events.length - 1]!.checkpoint,
         });
@@ -210,86 +240,156 @@ export async function run({
           msg: "Completed flush",
         });
       }
-
-      await metadataStore.setStatus(sync.getStatus());
-      if (result.status === "killed") {
-        return;
-      } else if (result.status === "error") {
-        onReloadableError(result.error);
-        return;
-      }
     }
 
-    if (isKilled) return;
+    await metadataStore.setStatus(sync.getStatus());
+  }
 
-    // Persist the indexing store to the db. The `finalized`
-    // checkpoint is used as a mutex. Any rows in the reorg table that may
-    // have been written because of raw sql access are deleted. Also must truncate
-    // the reorg tables that may have been written because of raw sql access.
+  // Persist the indexing store to the db. The `finalized`
+  // checkpoint is used as a mutex. Any rows in the reorg table that may
+  // have been written because of raw sql access are deleted. Also must truncate
+  // the reorg tables that may have been written because of raw sql access.
 
-    common.logger.debug({
-      service: "indexing",
-      msg: "Completed all historical events, starting final flush",
-    });
+  common.logger.debug({
+    service: "indexing",
+    msg: "Completed all historical events, starting final flush",
+  });
 
-    await database.finalize({ checkpoint: encodeCheckpoint(zeroCheckpoint) });
-    await historicalIndexingStore.flush();
-    await database.complete({ checkpoint: encodeCheckpoint(zeroCheckpoint) });
-    await database.finalize({ checkpoint: sync.getFinalizedCheckpoint() });
-
-    // Manually update metrics to fix a UI bug that occurs when the end
-    // checkpoint is between the last processed event and the finalized
-    // checkpoint.
-    const start = sync.getStartCheckpoint();
-    common.metrics.ponder_indexing_completed_seconds.set(
-      decodeCheckpoint(end ?? start).blockTimestamp -
-        decodeCheckpoint(start).blockTimestamp,
+  await database.finalize({ checkpoint: ZERO_CHECKPOINT_STRING });
+  await historicalIndexingStore.flush();
+  await database.complete({ checkpoint: ZERO_CHECKPOINT_STRING });
+  await database.finalize({ checkpoint: sync.getFinalizedCheckpoint() });
+
+  // Manually update metrics to fix a UI bug that occurs when the end
+  // checkpoint is between the last processed event and the finalized
+  // checkpoint.
+
+  for (const network of indexingBuild.networks) {
+    const label = { network: network.name };
+    common.metrics.ponder_historical_completed_indexing_seconds.set(
+      label,
+      Math.max(
+        sync.seconds[network.name]!.end - sync.seconds[network.name]!.start,
+        0,
+      ),
     );
-    common.metrics.ponder_indexing_total_seconds.set(
-      decodeCheckpoint(end ?? start).blockTimestamp -
-        decodeCheckpoint(start).blockTimestamp,
-    );
-    common.metrics.ponder_indexing_completed_timestamp.set(
-      decodeCheckpoint(end ?? start).blockTimestamp,
+    common.metrics.ponder_indexing_timestamp.set(
+      { network: network.name },
+      sync.seconds[network.name]!.end,
     );
+  }
 
-    // Become healthy
-    common.logger.info({
-      service: "indexing",
-      msg: "Completed historical indexing",
-    });
+  // Become healthy
+  common.logger.info({
+    service: "indexing",
+    msg: "Completed historical indexing",
+  });
 
-    await database.createIndexes();
-    await database.createTriggers();
+  const onRealtimeEvent = realtimeMutex(async (event: RealtimeEvent) => {
+    switch (event.type) {
+      case "block": {
+        if (event.events.length > 0) {
+          // Events must be run block-by-block, so that `database.complete` can accurately
+          // update the temporary `checkpoint` value set in the trigger.
 
-    indexingService.setIndexingStore(
-      createRealtimeIndexingStore({
-        common,
-        schemaBuild,
-        database,
-      }),
-    );
+          const perBlockEvents = splitEvents(event.events);
 
-    await sync.startRealtime();
+          common.logger.debug({
+            service: "app",
+            msg: `Partitioned events into ${perBlockEvents.length} blocks`,
+          });
 
-    await metadataStore.setStatus(sync.getStatus());
+          for (const { checkpoint, events } of perBlockEvents) {
+            const network = indexingBuild.networks.find(
+              (network) =>
+                network.chainId ===
+                Number(decodeCheckpoint(checkpoint).chainId),
+            )!;
+
+            const decodedEvents = decodeEvents(
+              common,
+              indexingBuild.sources,
+              events,
+            );
 
-    common.logger.info({
-      service: "server",
-      msg: "Started returning 200 responses from /ready endpoint",
-    });
-  };
-
-  const startPromise = start();
-
-  return async () => {
-    isKilled = true;
-    indexingService.kill();
-    await sync.kill();
-    realtimeQueue.pause();
-    realtimeQueue.clear();
-    await realtimeQueue.onIdle();
-    await startPromise;
-    await database.unlock();
-  };
+            common.logger.debug({
+              service: "app",
+              msg: `Decoded ${decodedEvents.length} '${network.name}' events for block ${Number(decodeCheckpoint(checkpoint).blockNumber)}`,
+            });
+
+            const result = await indexingService.processEvents({
+              events: decodedEvents,
+            });
+
+            common.logger.info({
+              service: "app",
+              msg: `Indexed ${decodedEvents.length} '${network.name}' events for block ${Number(decodeCheckpoint(checkpoint).blockNumber)}`,
+            });
+
+            if (result.status === "error") onReloadableError(result.error);
+
+            // Set reorg table `checkpoint` column for newly inserted rows.
+            await database.complete({ checkpoint });
+
+            if (preBuild.ordering === "multichain") {
+              const network = indexingBuild.networks.find(
+                (network) =>
+                  network.chainId ===
+                  Number(decodeCheckpoint(checkpoint).chainId),
+              )!;
+
+              common.metrics.ponder_indexing_timestamp.set(
+                { network: network.name },
+                decodeCheckpoint(checkpoint).blockTimestamp,
+              );
+            } else {
+              for (const network of indexingBuild.networks) {
+                common.metrics.ponder_indexing_timestamp.set(
+                  { network: network.name },
+                  decodeCheckpoint(checkpoint).blockTimestamp,
+                );
+              }
+            }
+          }
+        }
+
+        await metadataStore.setStatus(event.status);
+
+        break;
+      }
+      case "reorg":
+        await database.removeTriggers();
+        await database.revert({ checkpoint: event.checkpoint });
+        await database.createTriggers();
+
+        break;
+
+      case "finalize":
+        await database.finalize({ checkpoint: event.checkpoint });
+        break;
+
+      default:
+        never(event);
+    }
+  });
+
+  await database.createIndexes();
+  await database.createTriggers();
+
+  indexingService.setIndexingStore(
+    createRealtimeIndexingStore({
+      common,
+      schemaBuild,
+      database,
+    }),
+  );
+
+  await sync.startRealtime();
+
+  await metadataStore.setStatus(sync.getStatus());
+
+  common.logger.info({
+    service: "server",
+    msg: "Started returning 200 responses from /ready endpoint",
+  });
 }

package/src/bin/utils/runServer.ts

@@ -11,9 +11,5 @@ export async function runServer(params: {
   apiBuild: ApiBuild;
   database: Database;
 }) {
-  const server = await createServer(params);
-
-  return async () => {
-    await server.kill();
-  };
+  await createServer(params);
 }