ponder 0.9.2 → 0.9.3

package/src/build/index.ts

@@ -72,7 +72,6 @@ export type Build = {
   startDev: (params: {
     onReload: (kind: "indexing" | "api") => void;
   }) => void;
-  kill: () => Promise<void>;
 };
 
 export const createBuild = async ({
@@ -133,6 +132,8 @@ export const createBuild = async ({
     plugins: [viteTsconfigPathsPlugin(), vitePluginPonder(common.options)],
   });
 
+  common.shutdown.add(() => viteDevServer.close());
+
   // This is Vite boilerplate (initializes the Rollup container).
   await viteDevServer.pluginContainer.buildStart({});
 
@@ -373,6 +374,7 @@ export const createBuild = async ({
         status: "success",
         result: {
           databaseConfig: preBuild.databaseConfig,
+          ordering: preBuild.ordering,
         },
       } as const;
     },
@@ -585,13 +587,6 @@ export const createBuild = async ({
 
       viteDevServer.watcher.on("change", onFileChange);
     },
-    async kill() {
-      await viteDevServer?.close();
-      common.logger.debug({
-        service: "build",
-        msg: "Killed build service",
-      });
-    },
   } satisfies Build;
 
   return build;

package/src/build/pre.ts

@@ -18,6 +18,7 @@ export function buildPre({
   options: Pick<Options, "rootDir" | "ponderDir">;
 }): {
   databaseConfig: DatabaseConfig;
+  ordering: NonNullable<Config["ordering"]>;
   logs: { level: "warn" | "info" | "debug"; msg: string }[];
 } {
   const logs: { level: "warn" | "info" | "debug"; msg: string }[] = [];
@@ -112,6 +113,7 @@ export function buildPre({
   return {
     databaseConfig,
     logs,
+    ordering: config.ordering ?? "omnichain",
   };
 }
 
@@ -131,6 +133,7 @@ export function safeBuildPre({
     return {
       status: "success",
       databaseConfig: result.databaseConfig,
+      ordering: result.ordering,
       logs: result.logs,
     } as const;
   } catch (_error) {

package/src/config/index.ts

@@ -5,20 +5,22 @@ import type { AddressConfig } from "./address.js";
 import type { GetEventFilter } from "./eventFilter.js";
 
 export type Config = {
+  database?: DatabaseConfig;
+  ordering?: "omnichain" | "multichain";
   networks: { [networkName: string]: NetworkConfig<unknown> };
   contracts: { [contractName: string]: GetContract };
   accounts: { [accountName: string]: AccountConfig<unknown> };
-  database?: DatabaseConfig;
   blocks: {
     [sourceName: string]: GetBlockFilter<unknown>;
   };
 };
 
 export type CreateConfigReturnType<networks, contracts, accounts, blocks> = {
+  database?: DatabaseConfig;
+  ordering?: "omnichain" | "multichain";
   networks: networks;
   contracts: contracts;
   accounts: accounts;
-  database?: DatabaseConfig;
   blocks: blocks;
 };
 
@@ -29,6 +31,7 @@ export const createConfig = <
   const blocks = {},
 >(config: {
   database?: DatabaseConfig;
+  ordering?: "omnichain" | "multichain";
   // TODO: add jsdoc to these properties.
   networks: NetworksConfig<Narrow<networks>>;
   contracts?: ContractsConfig<networks, Narrow<contracts>>;

package/src/database/index.ts

@@ -2,7 +2,7 @@ import { randomUUID } from "node:crypto";
 import { getPrimaryKeyColumns, getTableNames } from "@/drizzle/index.js";
 import { getColumnCasing } from "@/drizzle/kit/index.js";
 import type { Common } from "@/internal/common.js";
-import { IgnorableError, NonRetryableError } from "@/internal/errors.js";
+import { NonRetryableError, ShutdownError } from "@/internal/errors.js";
 import type {
   IndexingBuild,
   NamespaceBuild,
@@ -17,10 +17,9 @@ import {
 } from "@/sync-store/migrations.js";
 import type { Drizzle } from "@/types/db.js";
 import {
+  MAX_CHECKPOINT_STRING,
+  ZERO_CHECKPOINT_STRING,
   decodeCheckpoint,
-  encodeCheckpoint,
-  maxCheckpoint,
-  zeroCheckpoint,
 } from "@/utils/checkpoint.js";
 import { formatEta } from "@/utils/format.js";
 import { createPool, createReadonlyPool } from "@/utils/pg.js";
@@ -62,8 +61,6 @@ export type Database = {
   revert(args: { checkpoint: string }): Promise<void>;
   finalize(args: { checkpoint: string }): Promise<void>;
   complete(args: { checkpoint: string }): Promise<void>;
-  unlock(): Promise<void>;
-  kill(): Promise<void>;
 };
 
 export type PonderApp = {
@@ -91,8 +88,8 @@ export type PonderInternalSchema = {
 } & {
   [_: ReturnType<typeof getTableNames>[number]["reorg"]]: unknown & {
     operation_id: number;
-    checkpoint: string;
     operation: 0 | 1 | 2;
+    checkpoint: string;
   };
 };
 
@@ -127,11 +124,10 @@ export const createDatabase = async ({
 }: {
   common: Common;
   namespace: NamespaceBuild;
-  preBuild: PreBuild;
+  preBuild: Pick<PreBuild, "databaseConfig">;
   schemaBuild: Omit<SchemaBuild, "graphqlSchema">;
 }): Promise<Database> => {
   let heartbeatInterval: NodeJS.Timeout | undefined;
-  let isKilled = false;
 
   ////////
   // Create schema, drivers, roles, and query builders
@@ -155,6 +151,22 @@ export const createDatabase = async ({
           : preBuild.databaseConfig.instance,
     };
 
+    common.shutdown.add(async () => {
+      clearInterval(heartbeatInterval);
+
+      await qb.internal
+        .updateTable("_ponder_meta")
+        .where("key", "=", "app")
+        .set({
+          value: sql`jsonb_set(value, '{is_locked}', to_jsonb(0))`,
+        })
+        .execute();
+
+      if (dialect === "pglite") {
+        await (driver as PGliteDriver).instance.close();
+      }
+    });
+
     const kyselyDialect = createPgliteKyselyDialect(driver.instance);
 
     await driver.instance.query(`CREATE SCHEMA IF NOT EXISTS "${namespace}"`);
@@ -251,6 +263,27 @@ export const createDatabase = async ({
       listen: undefined,
     } as PostgresDriver;
 
+    common.shutdown.add(async () => {
+      clearInterval(heartbeatInterval);
+
+      await qb.internal
+        .updateTable("_ponder_meta")
+        .where("key", "=", "app")
+        .set({
+          value: sql`jsonb_set(value, '{is_locked}', to_jsonb(0))`,
+        })
+        .execute();
+
+      const d = driver as PostgresDriver;
+      d.listen?.release();
+      await Promise.all([
+        d.internal.end(),
+        d.user.end(),
+        d.readonly.end(),
+        d.sync.end(),
+      ]);
+    });
+
     await driver.internal.query(`CREATE SCHEMA IF NOT EXISTS "${namespace}"`);
 
     qb = {
@@ -340,20 +373,15 @@ export const createDatabase = async ({
   // Helpers
   ////////
 
-  /**
-   * Undo operations in user tables by using the "reorg" tables.
-   *
-   * Note: "reorg" tables may contain operations that have not been applied to the
-   *       underlying tables, but only be 1 operation at most.
-   */
+  /** Undo operations in user tables by using the "reorg" tables. */
   const revert = async ({
+    tx,
     tableName,
     checkpoint,
-    tx,
   }: {
+    tx: Transaction<PonderInternalSchema>;
     tableName: ReturnType<typeof getTableNames>[number];
     checkpoint: string;
-    tx: Transaction<PonderInternalSchema>;
   }) => {
     const primaryKeyColumns = getPrimaryKeyColumns(
       schemaBuild.schema[tableName.js] as PgTable,
@@ -432,7 +460,8 @@ export const createDatabase = async ({
     });
   };
 
-  let checkpoint: string | undefined;
+  /** 'true' if `migrate` created new tables. */
+  let createdTables: boolean;
 
   const database = {
     driver,
@@ -458,15 +487,27 @@ export const createDatabase = async ({
         }
 
         try {
+          if (common.shutdown.isKilled) {
+            throw new ShutdownError();
+          }
+
           const result = await fn();
           common.metrics.ponder_database_method_duration.observe(
             { method: options.method },
             endClock(),
           );
+
+          if (common.shutdown.isKilled) {
+            throw new ShutdownError();
+          }
           return result;
         } catch (_error) {
           const error = _error as Error;
 
+          if (common.shutdown.isKilled) {
+            throw new ShutdownError();
+          }
+
           common.metrics.ponder_database_method_duration.observe(
             { method: options.method },
             endClock(),
@@ -475,14 +516,6 @@ export const createDatabase = async ({
             method: options.method,
           });
 
-          if (isKilled) {
-            common.logger.trace({
-              service: "database",
-              msg: `Ignored error during '${options.method}' database method, service is killed (id=${id})`,
-            });
-            throw new IgnorableError();
-          }
-
           if (!hasError) {
             hasError = true;
             firstError = error;
@@ -839,7 +872,7 @@ export const createDatabase = async ({
               .executeTakeFirst()
               .then((row) => row?.value);
 
-            let createdTables = false;
+            createdTables = false;
 
             if (previousApp === undefined) {
               await createEnums();
@@ -850,7 +883,7 @@ export const createDatabase = async ({
               (process.env.PONDER_EXPERIMENTAL_DB === "platform" &&
                 previousApp.build_id !== buildId) ||
               (process.env.PONDER_EXPERIMENTAL_DB === "platform" &&
-                previousApp.checkpoint === encodeCheckpoint(zeroCheckpoint))
+                previousApp.checkpoint === ZERO_CHECKPOINT_STRING)
             ) {
               for (const tableName of getTableNames(schemaBuild.schema)) {
                 await tx.schema
@@ -887,14 +920,12 @@ export const createDatabase = async ({
 
               // write metadata
 
-              checkpoint = encodeCheckpoint(zeroCheckpoint);
-
               const newApp = {
                 is_locked: 1,
                 is_dev: common.options.command === "dev" ? 1 : 0,
                 heartbeat_at: Date.now(),
                 build_id: buildId,
-                checkpoint: encodeCheckpoint(zeroCheckpoint),
+                checkpoint: ZERO_CHECKPOINT_STRING,
                 table_names: getTableNames(schemaBuild.schema).map(
                   ({ sql }) => sql,
                 ),
@@ -1012,9 +1043,8 @@ export const createDatabase = async ({
       }, common.options.databaseHeartbeatInterval);
     },
     async recoverCheckpoint() {
-      if (checkpoint !== undefined) {
-        return checkpoint;
-      }
+      // new tables are empty
+      if (createdTables) return ZERO_CHECKPOINT_STRING;
 
       return this.wrap(
         { method: "recoverCheckpoint", includeTraceLogs: true },
@@ -1027,7 +1057,7 @@ export const createDatabase = async ({
               .executeTakeFirstOrThrow()
               .then((row) => row.value);
 
-            if (app.checkpoint === encodeCheckpoint(zeroCheckpoint)) {
+            if (app.checkpoint === ZERO_CHECKPOINT_STRING) {
               for (const tableName of getTableNames(schemaBuild.schema)) {
                 await sql
                   .raw(
@@ -1108,13 +1138,13 @@ RETURNS TRIGGER AS $$
 BEGIN
   IF TG_OP = 'INSERT' THEN
     INSERT INTO "${namespace}"."${tableName.reorg}" (${columnNames.join(",")}, operation, checkpoint)
-    VALUES (${columnNames.map((name) => `NEW.${name}`).join(",")}, 0, '${encodeCheckpoint(maxCheckpoint)}');
+    VALUES (${columnNames.map((name) => `NEW.${name}`).join(",")}, 0, '${MAX_CHECKPOINT_STRING}');
   ELSIF TG_OP = 'UPDATE' THEN
     INSERT INTO "${namespace}"."${tableName.reorg}" (${columnNames.join(",")}, operation, checkpoint)
-    VALUES (${columnNames.map((name) => `OLD.${name}`).join(",")}, 1, '${encodeCheckpoint(maxCheckpoint)}');
+    VALUES (${columnNames.map((name) => `OLD.${name}`).join(",")}, 1, '${MAX_CHECKPOINT_STRING}');
   ELSIF TG_OP = 'DELETE' THEN
     INSERT INTO "${namespace}"."${tableName.reorg}" (${columnNames.join(",")}, operation, checkpoint)
-    VALUES (${columnNames.map((name) => `OLD.${name}`).join(",")}, 2, '${encodeCheckpoint(maxCheckpoint)}');
+    VALUES (${columnNames.map((name) => `OLD.${name}`).join(",")}, 2, '${MAX_CHECKPOINT_STRING}');
   END IF;
   RETURN NULL;
 END;
@@ -1151,13 +1181,9 @@ $$ LANGUAGE plpgsql
       await this.wrap({ method: "revert", includeTraceLogs: true }, () =>
         Promise.all(
           getTableNames(schemaBuild.schema).map((tableName) =>
-            qb.internal.transaction().execute((tx) =>
-              revert({
-                tableName,
-                checkpoint,
-                tx,
-              }),
-            ),
+            qb.internal
+              .transaction()
+              .execute((tx) => revert({ tx, tableName, checkpoint })),
           ),
         ),
       );
@@ -1201,7 +1227,7 @@ $$ LANGUAGE plpgsql
               await qb.internal
                 .updateTable(tableName.reorg)
                 .set({ checkpoint })
-                .where("checkpoint", "=", encodeCheckpoint(maxCheckpoint))
+                .where("checkpoint", "=", MAX_CHECKPOINT_STRING)
                 .execute();
             },
           ),
@@ -1224,32 +1250,6 @@ $$ LANGUAGE plpgsql
         },
       );
     },
-    async kill() {
-      isKilled = true;
-
-      if (dialect === "pglite") {
-        const d = driver as PGliteDriver;
-        await d.instance.close();
-      }
-
-      if (dialect === "pglite_test") {
-        // no-op, allow test harness to clean up the instance
-      }
-
-      if (dialect === "postgres") {
-        const d = driver as PostgresDriver;
-        d.listen?.release();
-        await d.internal.end();
-        await d.user.end();
-        await d.readonly.end();
-        await d.sync.end();
-      }
-
-      common.logger.debug({
-        service: "database",
-        msg: "Closed connection to database",
-      });
-    },
   } satisfies Database;
 
   // @ts-ignore

package/src/drizzle/kit/index.ts

@@ -109,9 +109,9 @@ const createReorgTableStatement = (statement: JsonCreateTableStatement) => {
       generatePgSnapshot(
         [
           pgTable("", {
-            operation_id: serial("operation_id").notNull().primaryKey(),
-            operation: integer("operation").notNull(),
-            checkpoint: varchar("checkpoint", {
+            operation_id: serial().notNull().primaryKey(),
+            operation: integer().notNull(),
+            checkpoint: varchar({
               length: 75,
             }).notNull(),
           }),

package/src/indexing/index.ts

@@ -1,20 +1,16 @@
 import { type Extend, extend } from "@/utils/extend.js";
 import {
   create,
-  kill,
   processEvents,
   processSetupEvents,
   setIndexingStore,
-  updateTotalSeconds,
 } from "./service.js";
 import type { Context, Service } from "./service.js";
 
 const methods = {
   create,
-  kill,
   processEvents,
   processSetupEvents,
-  updateTotalSeconds,
   setIndexingStore,
 };
 

package/src/indexing/service.ts

@@ -1,5 +1,6 @@
 import type { IndexingStore } from "@/indexing-store/index.js";
 import type { Common } from "@/internal/common.js";
+import { ShutdownError } from "@/internal/errors.js";
 import type {
   ContractSource,
   Event,
@@ -10,23 +11,25 @@ import type {
   SetupEvent,
   Source,
 } from "@/internal/types.js";
+import type { SyncStore } from "@/sync-store/index.js";
 import { isAddressFactory } from "@/sync/filter.js";
-import type { Sync } from "@/sync/index.js";
+import { cachedTransport } from "@/sync/transport.js";
 import type { Db } from "@/types/db.js";
 import type { Block, Log, Trace, Transaction } from "@/types/eth.js";
 import type { DeepPartial } from "@/types/utils.js";
 import {
-  type Checkpoint,
+  ZERO_CHECKPOINT,
   decodeCheckpoint,
   encodeCheckpoint,
-  zeroCheckpoint,
 } from "@/utils/checkpoint.js";
 import { prettyPrint } from "@/utils/print.js";
+import type { RequestQueue } from "@/utils/requestQueue.js";
 import { startClock } from "@/utils/timer.js";
-import type { Abi, Address } from "viem";
-import { checksumAddress, createClient } from "viem";
+import { type Abi, type Address, createClient } from "viem";
+import { checksumAddress } from "viem";
 import { addStackTrace } from "./addStackTrace.js";
-import { type ReadOnlyClient, getPonderActions } from "./ponderActions.js";
+import type { ReadOnlyClient } from "./ponderActions.js";
+import { getPonderActions } from "./ponderActions.js";
 
 export type Context = {
   network: { chainId: number; name: string };
@@ -49,12 +52,9 @@ export type Service = {
   indexingFunctions: IndexingFunctions;
 
   // state
-  isKilled: boolean;
-
   eventCount: {
     [eventName: string]: number;
   };
-  startCheckpoint: Checkpoint;
 
   /**
    * Reduce memory usage by reserving space for objects ahead of time
@@ -76,14 +76,16 @@ export type Service = {
 export const create = ({
   common,
   indexingBuild: { sources, networks, indexingFunctions },
-  sync,
+  requestQueues,
+  syncStore,
 }: {
   common: Common;
   indexingBuild: Pick<
     IndexingBuild,
     "sources" | "networks" | "indexingFunctions"
   >;
-  sync: Sync;
+  requestQueues: RequestQueue[];
+  syncStore: SyncStore;
 }): Service => {
   const contextState: Service["currentEvent"]["contextState"] = {
     blockNumber: undefined!,
@@ -139,10 +141,12 @@ export const create = ({
   }
 
   // build clientByChainId
-  for (const network of networks) {
-    const transport = sync.getCachedTransport(network);
+  for (let i = 0; i < networks.length; i++) {
+    const network = networks[i]!;
+    const requestQueue = requestQueues[i]!;
+
     clientByChainId[network.chainId] = createClient({
-      transport,
+      transport: cachedTransport({ requestQueue, syncStore }),
       chain: network.chain,
       // @ts-ignore
     }).extend(getPonderActions(contextState));
@@ -157,9 +161,7 @@ export const create = ({
   return {
     common,
     indexingFunctions,
-    isKilled: false,
     eventCount,
-    startCheckpoint: decodeCheckpoint(sync.getStartCheckpoint()),
     currentEvent: {
       contextState,
       context: {
@@ -184,11 +186,7 @@ export const processSetupEvents = async (
     sources: Source[];
     networks: Network[];
   },
-): Promise<
-  | { status: "error"; error: Error }
-  | { status: "success" }
-  | { status: "killed" }
-> => {
+): Promise<{ status: "error"; error: Error } | { status: "success" }> => {
   for (const eventName of Object.keys(indexingService.indexingFunctions)) {
     if (!eventName.endsWith(":setup")) continue;
 
@@ -204,8 +202,6 @@ export const processSetupEvents = async (
 
       if (source === undefined) continue;
 
-      if (indexingService.isKilled) return { status: "killed" };
-
       indexingService.eventCount[eventName]!++;
 
       const result = await executeSetup(indexingService, {
@@ -213,7 +209,7 @@ export const processSetupEvents = async (
           type: "setup",
           chainId: network.chainId,
           checkpoint: encodeCheckpoint({
-            ...zeroCheckpoint,
+            ...ZERO_CHECKPOINT,
             chainId: BigInt(network.chainId),
             blockNumber: BigInt(source.filter.fromBlock ?? 0),
           }),
@@ -236,14 +232,8 @@ export const processSetupEvents = async (
 export const processEvents = async (
   indexingService: Service,
   { events }: { events: Event[] },
-): Promise<
-  | { status: "error"; error: Error }
-  | { status: "success" }
-  | { status: "killed" }
-> => {
+): Promise<{ status: "error"; error: Error } | { status: "success" }> => {
   for (let i = 0; i < events.length; i++) {
-    if (indexingService.isKilled) return { status: "killed" };
-
     const event = events[i]!;
 
     indexingService.eventCount[event.name]!++;
@@ -262,40 +252,8 @@ export const processEvents = async (
       service: "indexing",
       msg: `Completed indexing function (event="${event.name}", checkpoint=${event.checkpoint})`,
     });
-
-    // periodically update metrics
-    if (i % 93 === 0) {
-      updateCompletedEvents(indexingService);
-
-      const eventTimestamp = decodeCheckpoint(event.checkpoint).blockTimestamp;
-
-      indexingService.common.metrics.ponder_indexing_completed_seconds.set(
-        eventTimestamp - indexingService.startCheckpoint.blockTimestamp,
-      );
-      indexingService.common.metrics.ponder_indexing_completed_timestamp.set(
-        eventTimestamp,
-      );
-
-      // Note: allows for terminal and logs to be updated
-      await new Promise(setImmediate);
-    }
   }
 
-  // set completed seconds
-  if (events.length > 0) {
-    const lastEventInBatchTimestamp = decodeCheckpoint(
-      events[events.length - 1]!.checkpoint,
-    ).blockTimestamp;
-
-    indexingService.common.metrics.ponder_indexing_completed_seconds.set(
-      lastEventInBatchTimestamp -
-        indexingService.startCheckpoint.blockTimestamp,
-    );
-    indexingService.common.metrics.ponder_indexing_completed_timestamp.set(
-      lastEventInBatchTimestamp,
-    );
-  }
-  // set completed events
   updateCompletedEvents(indexingService);
 
   return { status: "success" };
@@ -314,24 +272,6 @@ export const setIndexingStore = (
   };
 };
 
-export const kill = (indexingService: Service) => {
-  indexingService.common.logger.debug({
-    service: "indexing",
-    msg: "Killed indexing service",
-  });
-  indexingService.isKilled = true;
-};
-
-export const updateTotalSeconds = (
-  indexingService: Service,
-  endCheckpoint: Checkpoint,
-) => {
-  indexingService.common.metrics.ponder_indexing_total_seconds.set(
-    endCheckpoint.blockTimestamp -
-      indexingService.startCheckpoint.blockTimestamp,
-  );
-};
-
 const updateCompletedEvents = (indexingService: Service) => {
   for (const event of Object.keys(indexingService.eventCount)) {
     const metricLabel = {
@@ -347,11 +287,7 @@ const updateCompletedEvents = (indexingService: Service) => {
 const executeSetup = async (
   indexingService: Service,
   { event }: { event: SetupEvent },
-): Promise<
-  | { status: "error"; error: Error }
-  | { status: "success" }
-  | { status: "killed" }
-> => {
+): Promise<{ status: "error"; error: Error } | { status: "success" }> => {
   const {
     common,
     indexingFunctions,
@@ -382,9 +318,12 @@ const executeSetup = async (
       endClock(),
     );
   } catch (_error) {
-    if (indexingService.isKilled) return { status: "killed" };
     const error = _error instanceof Error ? _error : new Error(String(_error));
 
+    if (common.shutdown.isKilled) {
+      throw new ShutdownError();
+    }
+
     addStackTrace(error, common.options);
     addErrorMeta(error, toErrorMeta(event));
 
@@ -406,11 +345,7 @@ const executeSetup = async (
 const executeEvent = async (
   indexingService: Service,
   { event }: { event: Event },
-): Promise<
-  | { status: "error"; error: Error }
-  | { status: "success" }
-  | { status: "killed" }
-> => {
+): Promise<{ status: "error"; error: Error } | { status: "success" }> => {
   const {
     common,
     indexingFunctions,
@@ -442,9 +377,12 @@ const executeEvent = async (
       endClock(),
     );
   } catch (_error) {
-    if (indexingService.isKilled) return { status: "killed" };
     const error = _error instanceof Error ? _error : new Error(String(_error));
 
+    if (common.shutdown.isKilled) {
+      throw new ShutdownError();
+    }
+
     addStackTrace(error, common.options);
     addErrorMeta(error, toErrorMeta(event));