ponder 0.17.12 → 0.18.0

package/src/database/index.ts

@@ -1,8 +1,10 @@
+import fs from "node:fs";
 import type { PGlite } from "@electric-sql/pglite";
 import {
   eq,
   getTableName,
   getViewName,
+  inArray,
   isTable,
   isView,
   sql,
@@ -10,7 +12,6 @@ import {
 import { drizzle as drizzleNodePg } from "drizzle-orm/node-postgres";
 import { pgSchema, pgTable } from "drizzle-orm/pg-core";
 import { drizzle as drizzlePglite } from "drizzle-orm/pglite";
-import { Kysely, Migrator, PostgresDialect, WithSchemaPlugin } from "kysely";
 import type { Pool } from "pg";
 import prometheus from "prom-client";
 import { hexToBigInt } from "viem";
@@ -22,11 +23,7 @@ import {
   getReorgTableName,
 } from "@/drizzle/onchain.js";
 import type { Common } from "@/internal/common.js";
-import {
-  MigrationError,
-  NonRetryableUserError,
-  ShutdownError,
-} from "@/internal/errors.js";
+import { MigrationError, UniqueConstraintError } from "@/internal/errors.js";
 import type {
   CrashRecoveryCheckpoint,
   IndexingBuild,
@@ -34,12 +31,11 @@ import type {
   PreBuild,
   SchemaBuild,
 } from "@/internal/types.js";
-import { buildMigrationProvider } from "@/sync-store/migrations.js";
 import * as PONDER_SYNC from "@/sync-store/schema.js";
 import { decodeCheckpoint } from "@/utils/checkpoint.js";
 import { formatEta } from "@/utils/format.js";
 import { createPool, createReadonlyPool } from "@/utils/pg.js";
-import { createPglite, createPgliteKyselyDialect } from "@/utils/pglite.js";
+import { createPglite } from "@/utils/pglite.js";
 import { startClock } from "@/utils/timer.js";
 import { wait } from "@/utils/wait.js";
 import {
@@ -48,7 +44,7 @@ import {
   dropLiveQueryTriggers,
   dropTriggers,
 } from "./actions.js";
-import { createQB, parseDbError, type QB } from "./queryBuilder.js";
+import { createQB, type QB } from "./queryBuilder.js";
 
 export type Database = {
   driver: PostgresDriver | PGliteDriver;
@@ -91,6 +87,11 @@ export const VIEWS = pgSchema("information_schema").table("views", (t) => ({
   table_schema: t.text().notNull(),
 }));
 
+const LEGACY_MIGRATIONS = pgSchema("ponder_sync").table(
+  "kysely_migration",
+  (t) => ({ name: t.text().primaryKey() }),
+);
+
 export const PONDER_META_TABLE_NAME = "_ponder_meta";
 /**
  * @dev Used since version 0.11.
@@ -420,90 +421,132 @@ export const createDatabase = ({
     userQB,
     readonlyQB,
     async migrateSync() {
-      const kysely = new Kysely({
-        dialect:
-          dialect === "postgres"
-            ? new PostgresDialect({ pool: (driver as PostgresDriver).admin })
-            : createPgliteKyselyDialect((driver as PGliteDriver).instance),
-        log(event) {
-          if (event.level === "query") {
-            common.metrics.ponder_postgres_query_total.inc({ pool: "migrate" });
-          }
-        },
-        plugins: [new WithSchemaPlugin("ponder_sync")],
-      });
-
-      const migrationProvider = buildMigrationProvider(common.logger);
-      const migrator = new Migrator({
-        db: kysely,
-        provider: migrationProvider,
-        migrationTableSchema: "ponder_sync",
-      });
-
-      // Note: inline operation of database wrapper because this is the only place where kysely is used
-      for (let i = 0; i <= 9; i++) {
-        const endClock = startClock();
-        try {
-          const { error } = await migrator.migrateToLatest();
-          if (error) throw error;
-
-          common.metrics.ponder_database_method_duration.observe(
-            { method: "migrate_sync" },
-            endClock(),
-          );
-
-          return;
-        } catch (_error) {
-          const error = parseDbError(_error);
-
-          if (common.shutdown.isKilled) {
-            throw new ShutdownError();
-          }
-
-          common.metrics.ponder_database_method_duration.observe(
-            { method: "migrate_sync" },
-            endClock(),
-          );
-          common.metrics.ponder_database_method_error_total.inc({
-            method: "migrate_sync",
-          });
-
-          common.logger.warn({
-            msg: "Failed database query",
-            query: "migrate_sync",
-            retry_count: i,
-            error,
-          });
+      // Use a separate Drizzle instance because createQB skips PGlite transactions.
+      const db =
+        driver.dialect === "postgres"
+          ? drizzleNodePg(driver.admin, { casing: "snake_case" })
+          : drizzlePglite(driver.instance, { casing: "snake_case" });
+
+      const migratedSchemas = await adminQB
+        .wrap({ label: "migrate_sync" }, () =>
+          db.transaction(
+            async (tx) => {
+              await tx.execute(sql`SET LOCAL statement_timeout = 3600000`);
+
+              const dbSchemas = await tx
+                .select()
+                .from(SCHEMATA)
+                .where(
+                  inArray(SCHEMATA.schemaName, [
+                    ...PONDER_SYNC.PONDER_SYNC_SCHEMAS,
+                  ]),
+                )
+                .then((result) => result.map((r) => r.schemaName));
+
+              if (dbSchemas.includes("ponder_sync")) {
+                const migrations = await tx.select().from(LEGACY_MIGRATIONS);
+
+                if (
+                  migrations.some(
+                    (migration) =>
+                      migration.name === "2025_02_26_1_rpc_request_results",
+                  ) === false
+                ) {
+                  throw new MigrationError(
+                    '"ponder_sync" migration failed. Please first update to v0.10 or "DROP SCHEMA ponder_sync CASCADE".',
+                  );
+                }
+              }
 
-          if (error instanceof NonRetryableUserError) {
-            common.logger.warn({
-              msg: "Failed database query",
-              query: "migrate_sync",
-              error,
-            });
-            throw error;
-          }
+              const migratedSchemas: { schema: string; duration: number }[] =
+                [];
+              // Note: Run the migrations after the latest existing schema. A new
+              // database only creates the latest schema, and does not copy data.
+              let latestIndex = -1;
+              for (let i = 0; i < PONDER_SYNC.PONDER_SYNC_SCHEMAS.length; i++) {
+                if (dbSchemas.includes(PONDER_SYNC.PONDER_SYNC_SCHEMAS[i]!)) {
+                  latestIndex = i;
+                }
+              }
+              const isNewDatabase = latestIndex === -1;
+              const schemasToMigrate = isNewDatabase
+                ? [PONDER_SYNC.PONDER_SYNC_SCHEMA]
+                : PONDER_SYNC.PONDER_SYNC_SCHEMAS.slice(latestIndex + 1);
+
+              for (const schema of schemasToMigrate) {
+                const endSchemaClock = startClock();
+
+                const files = isNewDatabase
+                  ? ["create.sql"]
+                  : ["create.sql", "copy.sql"];
+                const statements = files.flatMap((file) =>
+                  fs
+                    .readFileSync(
+                      new URL(
+                        `../sync-store/sql/${schema}/${file}`,
+                        import.meta.url,
+                      ),
+                      "utf-8",
+                    )
+                    .split("--> statement-breakpoint")
+                    .map((statement) => statement.trim())
+                    .filter((statement) => statement.length > 0),
+                );
 
-          if (i === 9) {
-            common.logger.warn({
-              msg: "Failed database query",
-              query: "migrate_sync",
-              retry_count: i,
-              error,
-            });
-            throw error;
+                common.logger.info({
+                  msg: `Started migrating '${schema}' schema`,
+                  statement_count: statements.length,
+                });
+
+                for (let i = 0; i < statements.length; i++) {
+                  const statement = statements[i]!;
+                  // Log only the first non-comment line; CREATE TABLE statements span many lines.
+                  const summary = (
+                    statement
+                      .split("\n")
+                      .find((line) => line.startsWith("--") === false) ?? ""
+                  ).slice(0, 120);
+
+                  common.logger.debug({
+                    msg: "Started migration statement",
+                    schema,
+                    statement_index: i,
+                    statement: summary,
+                  });
+
+                  const endClock = startClock();
+                  await tx.execute(sql.raw(statement));
+
+                  common.logger.debug({
+                    msg: "Completed migration statement",
+                    schema,
+                    statement_index: i,
+                    statement: summary,
+                    duration: endClock(),
+                  });
+                }
+                migratedSchemas.push({ schema, duration: endSchemaClock() });
+              }
+              return migratedSchemas;
+            },
+            { isolationLevel: "repeatable read" },
+          ),
+        )
+        .catch((error) => {
+          // Note: Concurrent startups wait on `CREATE SCHEMA` until the first one
+          // commits, then fail with a unique violation. The first startup migrated
+          // the schemas.
+          if (
+            error instanceof UniqueConstraintError &&
+            error.message.includes("pg_namespace_nspname_index")
+          ) {
+            return [];
           }
+          throw error;
+        });
 
-          const duration = 125 * 2 ** i;
-          common.logger.debug({
-            msg: "Failed database query",
-            query: "migrate_sync",
-            retry_count: i,
-            retry_delay: duration,
-            error,
-          });
-          await wait(duration);
-        }
+      for (const { schema, duration } of migratedSchemas) {
+        common.logger.info({ msg: `Migrated '${schema}' schema`, duration });
       }
     },
     async migrate({ buildId, chains, finalizedBlocks }) {

package/src/index.ts

@@ -129,3 +129,8 @@ export {
 } from "@/drizzle/onchain.js";
 export { graphql } from "@/graphql/middleware.js";
 export type { ReadonlyDrizzle } from "@/types/db.js";
+export {
+  type Checkpoint,
+  decodeCheckpoint,
+  encodeCheckpoint,
+} from "@/utils/checkpoint.js";

package/src/indexing/client.ts

@@ -734,26 +734,30 @@ export const createCachedViemClient = ({
             ({ ev }) => ev > DB_PREDICTION_THRESHOLD,
           );
 
-          common.metrics.ponder_indexing_rpc_prefetch_total.inc(
-            {
-              chain: chain.name,
-              method: "eth_call",
-              type: "database",
-            },
-            dbRequests.length,
-          );
+          // Note: Without the rpc request cache, all requests miss the database.
+          let cachedResults: (string | undefined)[] = [];
+          if (chain.cacheRpcRequests) {
+            common.metrics.ponder_indexing_rpc_prefetch_total.inc(
+              {
+                chain: chain.name,
+                method: "eth_call",
+                type: "database",
+              },
+              dbRequests.length,
+            );
 
-          const cachedResults = await syncStore.getRpcRequestResults(
-            {
-              requests: dbRequests.map(({ request }) => request),
-              chainId,
-            },
-            context,
-          );
+            cachedResults = await syncStore.getRpcRequestResults(
+              {
+                requests: dbRequests.map(({ request }) => request),
+                chainId,
+              },
+              context,
+            );
+          }
 
           for (let i = 0; i < dbRequests.length; i++) {
             const request = dbRequests[i]!;
-            const cachedResult = cachedResults[i]!;
+            const cachedResult = cachedResults[i];
 
             if (cachedResult !== undefined) {
               cache
@@ -916,14 +920,16 @@ export const cachedTransport =
             (request) => results.has(request) === false,
           );
 
-          const dbResults = await syncStore.getRpcRequestResults(
-            { requests: dbRequests, chainId: chain.id },
-            context,
-          );
+          const dbResults = chain.cacheRpcRequests
+            ? await syncStore.getRpcRequestResults(
+                { requests: dbRequests, chainId: chain.id },
+                context,
+              )
+            : [];
 
           for (let i = 0; i < dbRequests.length; i++) {
             const request = dbRequests[i]!;
-            const result = dbResults[i]!;
+            const result = dbResults[i];
 
             if (result !== undefined) {
               common.metrics.ponder_indexing_rpc_requests_total.inc({
@@ -1004,19 +1010,21 @@ export const cachedTransport =
 
           // Note: insertRpcRequestResults errors can be ignored and not awaited, since
           // the response is already fetched.
-          syncStore
-            .insertRpcRequestResults(
-              {
-                requests: Array.from(requestsToInsert).map((request) => ({
-                  request,
-                  blockNumber: encodedBlockNumber,
-                  result: JSON.stringify(results.get(request)!.returnData),
-                })),
-                chainId: chain.id,
-              },
-              context,
-            )
-            .catch(() => {});
+          if (chain.cacheRpcRequests) {
+            syncStore
+              .insertRpcRequestResults(
+                {
+                  requests: Array.from(requestsToInsert).map((request) => ({
+                    request,
+                    blockNumber: encodedBlockNumber,
+                    result: JSON.stringify(results.get(request)!.returnData),
+                  })),
+                  chainId: chain.id,
+                },
+                context,
+              )
+              .catch(() => {});
+          }
 
           // Note: at this point, it is an invariant that either `allowFailure` is true or
           // there are no failed requests.
@@ -1073,7 +1081,10 @@ export const cachedTransport =
 
               if (result instanceof Error) throw result;
 
-              if (UNCACHED_RESPONSES.includes(result) === false) {
+              if (
+                chain.cacheRpcRequests &&
+                UNCACHED_RESPONSES.includes(result) === false
+              ) {
                 // Note: insertRpcRequestResults errors can be ignored and not awaited, since
                 // the response is already fetched.
                 syncStore
@@ -1105,10 +1116,12 @@ export const cachedTransport =
             return decodeResponse(cachedResult);
           }
 
-          const [cachedResult] = await syncStore.getRpcRequestResults(
-            { requests: [body], chainId: chain.id },
-            context,
-          );
+          const [cachedResult] = chain.cacheRpcRequests
+            ? await syncStore.getRpcRequestResults(
+                { requests: [body], chainId: chain.id },
+                context,
+              )
+            : [undefined];
 
           if (cachedResult !== undefined) {
             common.metrics.ponder_indexing_rpc_requests_total.inc({
@@ -1128,7 +1141,10 @@ export const cachedTransport =
 
           const response = await rpc.request(body, context);
 
-          if (UNCACHED_RESPONSES.includes(response) === false) {
+          if (
+            chain.cacheRpcRequests &&
+            UNCACHED_RESPONSES.includes(response) === false
+          ) {
             // Note: insertRpcRequestResults errors can be ignored and not awaited, since
             // the response is already fetched.
             syncStore

package/src/indexing/index.ts

@@ -970,7 +970,10 @@ const logText = (log?: DeepPartial<UserLog>) =>
 
 const traceText = (trace?: DeepPartial<UserTrace>) =>
   `Trace:\n${prettyPrint({
-    traceIndex: trace?.traceIndex,
+    traceAddress:
+      trace?.traceAddress === undefined
+        ? undefined
+        : JSON.stringify(trace.traceAddress),
     from: trace?.from,
     to: trace?.to,
   })}`;

package/src/internal/types.ts

@@ -60,10 +60,6 @@ export type Filter =
 
 /**
  * Filter that matches addresses.
- *
- * @dev This object is used as a unique constraint in the `ponder_sync.factories` table.
- * Any changes to the type must be backwards compatible and probably requires updating
- * `syncStore.getChildAddresses` and `syncStore.insertChildAddresses`.
  */
 export type Factory = LogFactory;
 export type FilterAddress<
@@ -91,7 +87,6 @@ export type TransactionFilter<
   sourceId: string;
   fromAddress: FilterAddress<fromFactory>;
   toAddress: FilterAddress<toFactory>;
-  includeReverted: boolean;
   fromBlock: number | undefined;
   toBlock: number | undefined;
   hasTransactionReceipt: true;
@@ -113,7 +108,6 @@ export type TraceFilter<
   toAddress: FilterAddress<toFactory>;
   functionSelector: Hex;
   callType: Trace["type"] | undefined;
-  includeReverted: boolean;
   fromBlock: number | undefined;
   toBlock: number | undefined;
   hasTransactionReceipt: boolean;
@@ -156,7 +150,6 @@ export type TransferFilter<
   sourceId: string;
   fromAddress: FilterAddress<fromFactory>;
   toAddress: FilterAddress<toFactory>;
-  includeReverted: boolean;
   fromBlock: number | undefined;
   toBlock: number | undefined;
   hasTransactionReceipt: boolean;
@@ -190,12 +183,14 @@ export type FragmentAddress =
       address: Address | null;
       eventSelector: Factory["eventSelector"];
       childAddressLocation: Factory["childAddressLocation"];
+      fromBlock: number | null;
+      toBlock: number | null;
     }
   | null;
 
 export type FragmentAddressId =
   | Address
-  | `${Address | null}_${Factory["eventSelector"]}_${Factory["childAddressLocation"]}`
+  | `${Address | null}_${Factory["eventSelector"]}_${Factory["childAddressLocation"]}_${number | null}_${number | null}`
   | null;
 export type FragmentTopic = Hex | null;
 
@@ -262,7 +257,10 @@ export type FragmentId =
   /** factory_log_{chainId}_{address}_{eventSelector}_{childAddressLocation}_{fromBlock}_{toBlock} */
   | `factory_log_${number}_${Address | null}_${Factory["eventSelector"]}_${Factory["childAddressLocation"]}_${number | null}_${number | null}`;
 
+export type FactoryFragmentId = Extract<FragmentId, `factory_${string}`>;
+
 // Contract
+
 export type Contract = {
   abi: Abi;
   address?: Address | readonly Address[];
@@ -302,8 +300,8 @@ export type Chain = {
   rpc: string | string[] | Transport;
   ws: string | undefined;
   pollingInterval: number;
-  finalityBlockCount: number;
-  disableCache: boolean;
+  reorgWindow: number;
+  cacheRpcRequests: boolean;
   ethGetLogsBlockRange: number | undefined;
   viemChain: ViemChain | undefined;
 };
@@ -410,7 +408,7 @@ export type SyncBlockHeader = Omit<SyncBlock, "transactions"> & {
 export type SyncTransaction = RpcTransaction<false>;
 export type SyncTransactionReceipt = RpcTransactionReceipt;
 export type SyncTrace = {
-  trace: DebugTrace["result"] & { index: number; subcalls: number };
+  trace: DebugTrace["result"] & { traceAddress: number[] };
   transactionHash: DebugTrace["txHash"];
 };
 export type SyncLog = ViemLog<Hex, Hex, false>;
@@ -435,8 +433,7 @@ export type RequiredTraceColumns =
   | "output"
   | "value"
   | "type"
-  | "error"
-  | "traceIndex";
+  | "traceAddress";
 export type RequiredLogColumns = keyof Log;
 
 export type RequiredInternalBlockColumns = RequiredBlockColumns;