ponder 0.14.13 → 0.15.0

package/src/database/index.ts

@@ -1,9 +1,12 @@
-import { getTableNames } from "@/drizzle/index.js";
+import { getPartitionName, getReorgTableName } from "@/drizzle/index.js";
 import {
   SHARED_OPERATION_ID_SEQUENCE,
-  getReorgTable,
   sqlToReorgTableName,
 } from "@/drizzle/kit/index.js";
+import {
+  getLiveQueryNotifyProcedureName,
+  getLiveQueryProcedureName,
+} from "@/drizzle/onchain.js";
 import type { Common } from "@/internal/common.js";
 import {
   MigrationError,
@@ -41,7 +44,12 @@ import { Kysely, Migrator, PostgresDialect, WithSchemaPlugin } from "kysely";
 import type { Pool } from "pg";
 import prometheus from "prom-client";
 import { hexToBigInt } from "viem";
-import { crashRecovery } from "./actions.js";
+import {
+  crashRecovery,
+  createLiveQueryProcedures,
+  dropLiveQueryTriggers,
+  dropTriggers,
+} from "./actions.js";
 import { type QB, createQB, parseDbError } from "./queryBuilder.js";
 
 export type Database = {
@@ -178,7 +186,7 @@ export const createDatabase = ({
 }: {
   common: Common;
   namespace: NamespaceBuild;
-  preBuild: Pick<PreBuild, "databaseConfig">;
+  preBuild: Pick<PreBuild, "databaseConfig" | "ordering">;
   schemaBuild: Omit<SchemaBuild, "graphqlSchema">;
 }): Database => {
   let heartbeatInterval: NodeJS.Timeout | undefined;
@@ -489,27 +497,50 @@ export const createDatabase = ({
 
       const createTables = async (tx: QB) => {
         for (let i = 0; i < schemaBuild.statements.tables.sql.length; i++) {
-          await tx
-            .wrap(
-              (tx) => tx.execute(schemaBuild.statements.tables.sql[i]!),
-              context,
-            )
-            .catch((_error) => {
-              const error = _error as Error;
-              if (!error.message.includes("already exists")) throw error;
-              const e = new MigrationError(
-                `Unable to create table "${namespace.schema}"."${schemaBuild.statements.tables.json[i]!.tableName}" because a table with that name already exists.`,
+          try {
+            const schemaName = schemaBuild.statements.tables.json[i]!.schema;
+            const tableName = schemaBuild.statements.tables.json[i]!.tableName;
+
+            if (
+              preBuild.ordering === "experimental_isolated" &&
+              tableName.startsWith("_reorg__") === false
+            ) {
+              const sql = schemaBuild.statements.tables.sql[i]!;
+              await tx.wrap((tx) =>
+                tx.execute(
+                  `${sql.slice(0, sql.length - 2)} PARTITION BY LIST (chain_id);`,
+                ),
               );
-              e.stack = undefined;
-              throw e;
-            });
+
+              for (const chain of chains) {
+                await tx.wrap((tx) =>
+                  tx.execute(
+                    `CREATE TABLE "${schemaName}"."${getPartitionName(tableName, chain.id)}" PARTITION OF "${schemaName}"."${tableName}" FOR VALUES IN (${chain.id});`,
+                  ),
+                );
+              }
+            } else {
+              await tx.wrap(
+                (tx) => tx.execute(schemaBuild.statements.tables.sql[i]!),
+                context,
+              );
+            }
+          } catch (_error) {
+            let error = _error as Error;
+            if (!error.message.includes("already exists")) throw error;
+            error = new MigrationError(
+              `Unable to create table '${namespace.schema}'.'${schemaBuild.statements.tables.json[i]!.tableName}' because a table with that name already exists.`,
+            );
+            error.stack = undefined;
+            throw error;
+          }
         }
 
         for (const table of tables) {
           await tx.wrap(
             (tx) =>
               tx.execute(
-                `CREATE INDEX IF NOT EXISTS "${getTableName(table)}_checkpoint_index" ON "${namespace.schema}"."${getTableName(getReorgTable(table))}" ("checkpoint")`,
+                `CREATE INDEX IF NOT EXISTS "${getTableName(table)}_checkpoint_index" ON "${namespace.schema}"."${getReorgTableName(table)}" ("checkpoint")`,
               ),
             context,
           );
@@ -589,40 +620,6 @@ CREATE TABLE IF NOT EXISTS "${namespace.schema}"."_ponder_checkpoint" (
             ),
           context,
         );
-
-        const trigger = "status_trigger";
-        const notification = "status_notify()";
-        const channel = `${namespace.schema}_status_channel`;
-
-        await tx.wrap(
-          (tx) =>
-            tx.execute(
-              `
-CREATE OR REPLACE FUNCTION "${namespace.schema}".${notification}
-RETURNS TRIGGER
-LANGUAGE plpgsql
-AS $$
-BEGIN
-NOTIFY "${channel}";
-RETURN NULL;
-END;
-$$;`,
-            ),
-          context,
-        );
-
-        await tx.wrap(
-          (tx) =>
-            tx.execute(
-              `
-CREATE OR REPLACE TRIGGER "${trigger}"
-AFTER INSERT OR UPDATE OR DELETE
-ON "${namespace.schema}"._ponder_checkpoint
-FOR EACH STATEMENT
-EXECUTE PROCEDURE "${namespace.schema}".${notification};`,
-            ),
-          context,
-        );
       };
 
       const tryAcquireLockAndMigrate = () =>
@@ -680,6 +677,11 @@ EXECUTE PROCEDURE "${namespace.schema}".${notification};`,
             await createEnums(tx);
             await createTables(tx);
             await createViews(tx);
+            await createLiveQueryProcedures(
+              tx,
+              { namespaceBuild: namespace },
+              context,
+            );
 
             common.logger.info({
               msg: "Created database tables",
@@ -800,7 +802,24 @@ EXECUTE PROCEDURE "${namespace.schema}".${notification};`,
               context,
             );
 
+            await tx.wrap((tx) =>
+              tx.execute(
+                `DROP FUNCTION IF EXISTS "${namespace.schema}".${getLiveQueryProcedureName()}`,
+              ),
+            );
+
+            await tx.wrap((tx) =>
+              tx.execute(
+                `DROP FUNCTION IF EXISTS "${namespace.schema}".${getLiveQueryNotifyProcedureName()}`,
+              ),
+            );
+
             await createAdminObjects(tx);
+            await createLiveQueryProcedures(
+              tx,
+              { namespaceBuild: namespace },
+              context,
+            );
 
             common.logger.debug({
               msg: "Reset internal database objects",
@@ -902,12 +921,20 @@ EXECUTE PROCEDURE "${namespace.schema}".${notification};`,
 
           // Remove triggers
 
-          for (const table of tables) {
-            await tx.wrap(
-              (tx) =>
-                tx.execute(
-                  `DROP TRIGGER IF EXISTS "${getTableNames(table).trigger}" ON "${namespace.schema}"."${getTableName(table)}"`,
-                ),
+          if (preBuild.ordering === "experimental_isolated") {
+            for (const { chainId } of checkpoints) {
+              await dropTriggers(tx, { tables, chainId });
+              await dropLiveQueryTriggers(
+                tx,
+                { namespaceBuild: namespace, tables, chainId },
+                context,
+              );
+            }
+          } else {
+            await dropTriggers(tx, { tables });
+            await dropLiveQueryTriggers(
+              tx,
+              { namespaceBuild: namespace, tables },
               context,
             );
           }

package/src/database/queryBuilder.ts

@@ -1,3 +1,4 @@
+import crypto from "node:crypto";
 import type { Common } from "@/internal/common.js";
 import { BaseError } from "@/internal/errors.js";
 import {

package/src/drizzle/index.ts

@@ -6,14 +6,22 @@ import {
 } from "drizzle-orm/pg-core";
 import { getColumnCasing, sqlToReorgTableName } from "./kit/index.js";
 
-export const getTableNames = (table: PgTable) => {
-  const sql = getTableName(table);
+export const getPartitionName = (table: string | PgTable, chainId: number) => {
+  return `${typeof table === "string" ? table : getTableName(table)}_${chainId}`;
+};
 
-  return {
-    reorg: sqlToReorgTableName(sql),
-    trigger: sqlToReorgTableName(sql),
-    triggerFn: `operation_reorg__${sql}()`,
-  };
+export const getReorgTableName = (table: string | PgTable) => {
+  return sqlToReorgTableName(
+    typeof table === "string" ? table : getTableName(table),
+  );
+};
+
+export const getReorgTriggerName = () => {
+  return "reorg_trigger";
+};
+
+export const getReorgProcedureName = (table: PgTable) => {
+  return `operation_reorg__${getTableName(table)}()`;
 };
 
 export const getPrimaryKeyColumns = (

package/src/drizzle/onchain.ts

@@ -38,6 +38,25 @@ import {
 } from "./json.js";
 import { PgTextBuilder, type PgTextBuilderInitial } from "./text.js";
 
+export const getLiveQueryTriggerName = () => {
+  return "live_query_trigger";
+};
+export const getLiveQueryProcedureName = () => {
+  return "live_query_procedure()";
+};
+export const getLiveQueryChannelName = (schema: string) => {
+  return `${schema}_live_query_channel`;
+};
+export const getLiveQueryNotifyTriggerName = () => {
+  return "live_query_notify_trigger";
+};
+export const getViewsLiveQueryNotifyTriggerName = (schema?: string) => {
+  return `${schema}_live_query_notify_trigger`;
+};
+export const getLiveQueryNotifyProcedureName = () => {
+  return "live_query_notify_procedure()";
+};
+
 /** @internal */
 function getColumnNameAndConfig<
   TConfig extends Record<string, any> | undefined,

package/src/indexing/client.ts

@@ -453,32 +453,32 @@ export const createCachedViemClient = ({
       hasConstant,
     }: { pattern: ProfilePattern; hasConstant: boolean }) => {
       const profilePatternKey = getProfilePatternKey(pattern);
+      const eventName = (event as Event).eventCallback.name;
 
-      if (profile.get(event.name)!.has(profilePatternKey)) {
-        profile.get(event.name)!.get(profilePatternKey)!.count++;
+      if (profile.get(eventName)!.has(profilePatternKey)) {
+        profile.get(eventName)!.get(profilePatternKey)!.count++;
 
         if (hasConstant) {
-          profileConstantLRU.get(event.name)!.delete(profilePatternKey);
-          profileConstantLRU.get(event.name)!.add(profilePatternKey);
+          profileConstantLRU.get(eventName)!.delete(profilePatternKey);
+          profileConstantLRU.get(eventName)!.add(profilePatternKey);
         }
       } else {
         profile
-          .get(event.name)!
+          .get(eventName)!
           .set(profilePatternKey, { pattern, hasConstant, count: 1 });
 
         if (hasConstant) {
-          profileConstantLRU.get(event.name)!.add(profilePatternKey);
+          profileConstantLRU.get(eventName)!.add(profilePatternKey);
           if (
-            profileConstantLRU.get(event.name)!.size >
-            MAX_CONSTANT_PATTERN_COUNT
+            profileConstantLRU.get(eventName)!.size > MAX_CONSTANT_PATTERN_COUNT
           ) {
             const firstKey = profileConstantLRU
-              .get(event.name)!
+              .get(eventName)!
               .keys()
               .next().value;
             if (firstKey) {
-              profile.get(event.name)!.delete(firstKey);
-              profileConstantLRU.get(event.name)!.delete(firstKey);
+              profile.get(eventName)!.delete(firstKey);
+              profileConstantLRU.get(eventName)!.delete(firstKey);
             }
           }
         }
@@ -500,11 +500,13 @@ export const createCachedViemClient = ({
         if (
           event.type !== "setup" &&
           userBlockNumber === undefined &&
-          eventCount[event.name]! % SAMPLING_RATE === 1
+          eventCount[event.eventCallback.name]! % SAMPLING_RATE === 1
         ) {
-          if (profile.has(event.name) === false) {
-            profile.set(event.name, new Map());
-            profileConstantLRU.set(event.name, new Set());
+          const eventName = event.eventCallback.name;
+
+          if (profile.has(eventName) === false) {
+            profile.set(eventName, new Map());
+            profileConstantLRU.set(eventName, new Set());
           }
 
           // profile "readContract" and "multicall" actions
@@ -514,7 +516,7 @@ export const createCachedViemClient = ({
               args: { ...args, cache } as Parameters<
                 PonderActions["readContract"]
               >[0],
-              hints: Array.from(profile.get(event.name)!.values()),
+              hints: Array.from(profile.get(eventName)!.values()),
             });
             if (recordPatternResult) {
               addProfilePattern(recordPatternResult);
@@ -529,7 +531,7 @@ export const createCachedViemClient = ({
                 const recordPatternResult = recordProfilePattern({
                   event: event,
                   args: contract,
-                  hints: Array.from(profile.get(event.name)!.values()),
+                  hints: Array.from(profile.get(eventName)!.values()),
                 });
                 if (recordPatternResult) {
                   addProfilePattern(recordPatternResult);
@@ -564,6 +566,10 @@ export const createCachedViemClient = ({
             // @ts-ignore
             return await action(...args);
           } catch (error) {
+            const eventName =
+              event.type === "setup"
+                ? event.setupCallback.name
+                : event.eventCallback.name;
             if (
               (error instanceof BlockNotFoundError === false &&
                 error instanceof TransactionNotFoundError === false &&
@@ -576,12 +582,12 @@ export const createCachedViemClient = ({
               (args[0] as RetryableOptions).retryEmptyResponse === false
             ) {
               const chain = indexingBuild.chains.find(
-                (n) => n.id === event.chainId,
+                (n) => n.id === event.chain.id,
               )!;
               common.logger.warn({
                 msg: "Failed 'context.client' action",
                 action: actionName,
-                event: event.name,
+                event: eventName,
                 chain: chain.name,
                 chain_id: chain.id,
                 retry_count: i,
@@ -593,12 +599,12 @@ export const createCachedViemClient = ({
 
             const duration = BASE_DURATION * 2 ** i;
             const chain = indexingBuild.chains.find(
-              (n) => n.id === event.chainId,
+              (n) => n.id === event.chain.id,
             )!;
             common.logger.warn({
               msg: "Failed 'context.client' action",
               action: actionName,
-              event: event.name,
+              event: eventName,
               chain: chain.name,
               chain_id: chain.id,
               retry_count: i,
@@ -683,10 +689,13 @@ export const createCachedViemClient = ({
       const prediction: { ev: number; request: Request }[] = [];
 
       for (const event of events) {
-        if (profile.has(event.name)) {
-          for (const [, { pattern, count }] of profile.get(event.name)!) {
+        if (profile.has(event.eventCallback.name)) {
+          for (const [, { pattern, count }] of profile.get(
+            event.eventCallback.name,
+          )!) {
             // Expected value of times the prediction will be used.
-            const ev = (count * SAMPLING_RATE) / eventCount[event.name]!;
+            const ev =
+              (count * SAMPLING_RATE) / eventCount[event.eventCallback.name]!;
             prediction.push({
               ev,
               request: recoverProfilePattern(pattern, event),
@@ -808,10 +817,14 @@ export const cachedTransport =
   ({ chain: viemChain }) =>
     custom({
       async request({ method, params }) {
+        const _event = event();
         const context = {
           logger: common.logger.child({
             action: "cache JSON-RPC request",
-            event: event().name,
+            event:
+              _event.type === "setup"
+                ? _event.setupCallback.name
+                : _event.eventCallback.name,
           }),
         };
         const body = { method, params };