ponder 0.11.21 → 0.11.22

package/src/database/queryBuilder.ts

@@ -1,319 +0,0 @@
-import type { Common } from "@/internal/common.js";
-import {
-  NonRetryableError,
-  ShutdownError,
-  TransactionError,
-} from "@/internal/errors.js";
-import {
-  BigIntSerializationError,
-  CheckConstraintError,
-  NotNullConstraintError,
-  UniqueConstraintError,
-  getBaseError,
-} from "@/internal/errors.js";
-import type { Schema } from "@/internal/types.js";
-import type { Drizzle } from "@/types/db.js";
-import { startClock } from "@/utils/timer.js";
-import { wait } from "@/utils/wait.js";
-import type { PGlite } from "@electric-sql/pglite";
-import {
-  type PgDatabase,
-  PgDialect,
-  type PgQueryResultHKT,
-  type PgTransactionConfig,
-} from "drizzle-orm/pg-core";
-import pg from "pg";
-
-const RETRY_COUNT = 9;
-const BASE_DURATION = 125;
-const SQL_LENGTH_LIMIT = 50;
-
-/**
- * Query builder with built-in retry logic, logging, and metrics.
- */
-export type QB<
-  TSchema extends Schema = Schema,
-  TClient extends PGlite | pg.Pool | pg.PoolClient =
-    | PGlite
-    | pg.Pool
-    | pg.PoolClient,
-> = ((label?: string) => Omit<Drizzle<TSchema>, "transaction"> & {
-  transaction<T>(
-    transaction: (tx: QB<TSchema, TClient>) => Promise<T>,
-    config?: PgTransactionConfig,
-  ): Promise<T>;
-}) &
-  (
-    | { $dialect: "pglite"; $client: PGlite }
-    | { $dialect: "postgres"; $client: pg.Pool | pg.PoolClient }
-  );
-
-export const parseSqlError = (e: any): Error => {
-  let error = getBaseError(e);
-
-  if (error?.message?.includes("violates not-null constraint")) {
-    error = new NotNullConstraintError(error.message);
-  } else if (error?.message?.includes("violates unique constraint")) {
-    error = new UniqueConstraintError(error.message);
-  } else if (error?.message?.includes("violates check constraint")) {
-    error = new CheckConstraintError(error.message);
-  } else if (
-    error?.message?.includes("Do not know how to serialize a BigInt")
-  ) {
-    error = new BigIntSerializationError(error.message);
-    error.meta.push(
-      "Hint:\n  The JSON column type does not support BigInt values. Use the replaceBigInts() helper function before inserting into the database. Docs: https://ponder.sh/docs/api-reference/ponder-utils#replacebigints",
-    );
-  } else if (error?.message?.includes("does not exist")) {
-    error = new NonRetryableError(error.message);
-  }
-
-  return error;
-};
-
-/**
- * Create a query builder.
- *
- * @example
- * ```ts
- * const qb = createQB(common, drizzle(pool, { casing: "snake_case" }));
- * const result = await qb.label("test").select().from(accounts);
- * ```
- */
-export const createQB = <
-  TSchema extends Schema = { [name: string]: never },
-  TClient extends PGlite | pg.Pool | pg.PoolClient =
-    | PGlite
-    | pg.Pool
-    | pg.PoolClient,
->(
-  createDb: () => PgDatabase<PgQueryResultHKT, TSchema> & { $client: TClient },
-  { common, isAdmin }: { common: Common; isAdmin?: boolean },
-): QB<TSchema, TClient> => {
-  const dialect = new PgDialect({ casing: "snake_case" });
-  let txLabel: string | undefined = undefined;
-
-  const wrap = async <T>(
-    label: string | undefined,
-    fn: () => Promise<T>,
-    sql: string,
-  ): Promise<T> => {
-    // First error thrown is often the most useful
-    let firstError: any;
-    let hasError = false;
-
-    for (let i = 0; i <= RETRY_COUNT; i++) {
-      const endClock = startClock();
-      const id = crypto.randomUUID().slice(0, 8);
-
-      if (label) {
-        common.logger.trace({
-          service: "database",
-          msg: `Started '${label}' database method (id=${id})`,
-        });
-      }
-
-      try {
-        if (common.shutdown.isKilled && isAdmin === false) {
-          throw new ShutdownError();
-        }
-
-        const result = await fn();
-        if (label) {
-          common.metrics.ponder_database_method_duration.observe(
-            { method: label },
-            endClock(),
-          );
-        }
-
-        if (common.shutdown.isKilled && isAdmin === false) {
-          throw new ShutdownError();
-        }
-
-        return result;
-      } catch (e) {
-        const error = parseSqlError(e);
-
-        if (common.shutdown.isKilled) {
-          throw new ShutdownError();
-        }
-
-        if (label) {
-          common.metrics.ponder_database_method_duration.observe(
-            { method: label },
-            endClock(),
-          );
-          common.metrics.ponder_database_method_error_total.inc({
-            method: label,
-          });
-        }
-
-        if (!hasError) {
-          hasError = true;
-          firstError = error;
-        }
-
-        if (
-          error instanceof NonRetryableError &&
-          error instanceof TransactionError === false
-        ) {
-          common.logger.warn({
-            service: "database",
-            msg: `Failed '${label ?? sql}' database query (id=${id})`,
-            error,
-          });
-          throw error;
-        }
-
-        if (i === RETRY_COUNT) {
-          common.logger.warn({
-            service: "database",
-            msg: `Failed '${label ?? sql}' database query after '${i + 1}' attempts (id=${id})`,
-            error,
-          });
-          throw firstError;
-        }
-
-        const duration = BASE_DURATION * 2 ** i;
-        common.logger.debug({
-          service: "database",
-          msg: `Failed '${label ?? sql}' database query, retrying after ${duration} milliseconds (id=${id})`,
-          error,
-        });
-        await wait(duration);
-      } finally {
-        if (label) {
-          common.logger.trace({
-            service: "database",
-            msg: `Completed '${label}' database method in ${Math.round(endClock())}ms (id=${id})`,
-          });
-        }
-      }
-    }
-
-    throw "unreachable";
-  };
-
-  const assignClient = (qb: QB<TSchema, TClient>, client: TClient) => {
-    if (client instanceof pg.Pool || client instanceof pg.Client) {
-      Object.assign(qb, { $dialect: "postgres" });
-    } else {
-      Object.assign(qb, { $dialect: "pglite" });
-    }
-
-    Object.assign(qb, { $client: client });
-  };
-
-  const wrapTx = (db: PgDatabase<PgQueryResultHKT, TSchema>) => {
-    const _transaction = db.transaction.bind(db);
-    db.transaction = async (...args) => {
-      const callback = args[0];
-      args[0] = async (_tx) => {
-        wrapTx(_tx);
-
-        const previousLabel = txLabel;
-
-        const tx = (label?: string) => {
-          txLabel = label;
-          return _tx;
-        };
-
-        // @ts-expect-error
-        assignClient(tx, _tx.session.client);
-        // @ts-expect-error
-        const result = await callback(tx);
-
-        txLabel = previousLabel;
-        return result;
-      };
-      return _transaction(...args);
-    };
-  };
-
-  const qb = ((label: string | undefined) => {
-    const db = createDb();
-    const isClient = db.$client instanceof pg.Client;
-
-    // non-transaction queries (retryable)
-
-    const execute = db._.session.execute.bind(db._.session);
-    db._.session.execute = async (...args) => {
-      return wrap(
-        label,
-        () => execute(...args),
-        dialect.sqlToQuery(args[0]).sql.slice(0, SQL_LENGTH_LIMIT),
-      );
-    };
-
-    const prepareQuery = db._.session.prepareQuery.bind(db._.session);
-    db._.session.prepareQuery = (...args) => {
-      const result = prepareQuery(...args);
-      const execute = result.execute.bind(result);
-      result.execute = async (..._args) => {
-        return wrap(
-          label,
-          () => execute(..._args),
-          args[0].sql.slice(0, SQL_LENGTH_LIMIT),
-        );
-      };
-      return result;
-    };
-
-    // transaction queries (non-retryable)
-
-    wrapTx(db);
-    txLabel = label;
-
-    const transaction = db._.session.transaction.bind(db._.session);
-    db._.session.transaction = async (...args) => {
-      const callback = args[0];
-      args[0] = async (..._args) => {
-        const tx = _args[0] as PgDatabase<PgQueryResultHKT, TSchema>;
-        const txExecute = isClient
-          ? execute
-          : tx._.session.execute.bind(tx._.session);
-        // @ts-expect-error
-        tx._.session.execute = async (...args) => {
-          return wrap(
-            txLabel,
-            () =>
-              txExecute(...args).catch((error) => {
-                throw new TransactionError(error.message);
-              }),
-            dialect.sqlToQuery(args[0]).sql.slice(0, SQL_LENGTH_LIMIT),
-          );
-        };
-
-        const txPrepareQuery = isClient
-          ? prepareQuery
-          : tx._.session.prepareQuery.bind(tx._.session);
-        // @ts-ignore
-        tx._.session.prepareQuery = (...args) => {
-          const result = txPrepareQuery(...args);
-          const execute = result.execute.bind(result);
-          result.execute = async (..._args) => {
-            return wrap(
-              txLabel,
-              () =>
-                execute(..._args).catch((error) => {
-                  throw new TransactionError(error.message);
-                }),
-              args[0].sql.slice(0, SQL_LENGTH_LIMIT),
-            );
-          };
-          return result;
-        };
-
-        return callback(..._args);
-      };
-
-      return wrap(label, () => transaction(...args), "begin");
-    };
-
-    return db;
-  }) as unknown as QB<TSchema, TClient>;
-
-  assignClient(qb, createDb().$client);
-
-  return qb;
-};

package/src/database/utils.ts

@@ -1,140 +0,0 @@
-import { getPrimaryKeyColumns } from "@/drizzle/index.js";
-import { getTableNames } from "@/drizzle/index.js";
-import { getColumnCasing, getReorgTable } from "@/drizzle/kit/index.js";
-import type { SchemaBuild } from "@/internal/types.js";
-import { MAX_CHECKPOINT_STRING } from "@/utils/checkpoint.js";
-import { eq, getTableColumns, getTableName, lte, sql } from "drizzle-orm";
-import { type PgTable, getTableConfig } from "drizzle-orm/pg-core";
-import type { QB } from "./queryBuilder.js";
-
-export const createIndexes = async (
-  qb: QB,
-  { statements }: { statements: SchemaBuild["statements"] },
-) => {
-  for (const statement of statements.indexes.sql) {
-    await qb("create_indexes").transaction(async (tx) => {
-      // 60 minutes
-      await tx().execute("SET statement_timeout = 3600000;");
-      await tx().execute(statement);
-    });
-  }
-};
-
-export const createTrigger = async (qb: QB, { table }: { table: PgTable }) => {
-  const schema = getTableConfig(table).schema ?? "public";
-  const columns = getTableColumns(table);
-
-  const columnNames = Object.values(columns).map(
-    (column) => `"${getColumnCasing(column, "snake_case")}"`,
-  );
-
-  await qb("create_trigger_function").execute(
-    sql.raw(`
-CREATE OR REPLACE FUNCTION "${schema}".${getTableNames(table).triggerFn}
-RETURNS TRIGGER AS $$
-BEGIN
-IF TG_OP = 'INSERT' THEN
-INSERT INTO "${schema}"."${getTableName(getReorgTable(table))}" (${columnNames.join(",")}, operation, checkpoint)
-VALUES (${columnNames.map((name) => `NEW.${name}`).join(",")}, 0, '${MAX_CHECKPOINT_STRING}');
-ELSIF TG_OP = 'UPDATE' THEN
-INSERT INTO "${schema}"."${getTableName(getReorgTable(table))}" (${columnNames.join(",")}, operation, checkpoint)
-VALUES (${columnNames.map((name) => `OLD.${name}`).join(",")}, 1, '${MAX_CHECKPOINT_STRING}');
-ELSIF TG_OP = 'DELETE' THEN
-INSERT INTO "${schema}"."${getTableName(getReorgTable(table))}" (${columnNames.join(",")}, operation, checkpoint)
-VALUES (${columnNames.map((name) => `OLD.${name}`).join(",")}, 2, '${MAX_CHECKPOINT_STRING}');
-END IF;
-RETURN NULL;
-END;
-$$ LANGUAGE plpgsql`),
-  );
-
-  await qb("create_trigger").execute(
-    sql.raw(`
-CREATE OR REPLACE TRIGGER "${getTableNames(table).trigger}"
-AFTER INSERT OR UPDATE OR DELETE ON "${schema}"."${getTableName(table)}"
-FOR EACH ROW EXECUTE FUNCTION "${schema}".${getTableNames(table).triggerFn};
-`),
-  );
-};
-
-export const dropTrigger = async (qb: QB, { table }: { table: PgTable }) => {
-  const schema = getTableConfig(table).schema ?? "public";
-
-  await qb("drop_trigger").execute(
-    sql.raw(
-      `DROP TRIGGER IF EXISTS "${getTableNames(table).trigger}" ON "${schema}"."${getTableName(table)}"`,
-    ),
-  );
-};
-
-export const revert = async (
-  qb: QB,
-  { checkpoint, table }: { checkpoint: string; table: PgTable },
-): Promise<number> => {
-  const primaryKeyColumns = getPrimaryKeyColumns(table);
-  const schema = getTableConfig(table).schema ?? "public";
-
-  const result = await qb("revert").execute(
-    sql.raw(`
-WITH reverted1 AS (
-DELETE FROM "${schema}"."${getTableName(getReorgTable(table))}"
-WHERE checkpoint > '${checkpoint}' RETURNING *
-), reverted2 AS (
-SELECT ${primaryKeyColumns.map(({ sql }) => `"${sql}"`).join(", ")}, MIN(operation_id) AS operation_id FROM reverted1
-GROUP BY ${primaryKeyColumns.map(({ sql }) => `"${sql}"`).join(", ")}
-), reverted3 AS (
-SELECT ${Object.values(getTableColumns(table))
-      .map((column) => `reverted1."${getColumnCasing(column, "snake_case")}"`)
-      .join(", ")}, reverted1.operation FROM reverted2
-INNER JOIN reverted1
-ON ${primaryKeyColumns.map(({ sql }) => `reverted2."${sql}" = reverted1."${sql}"`).join("AND ")}
-AND reverted2.operation_id = reverted1.operation_id
-), inserted AS (
-DELETE FROM "${schema}"."${getTableName(table)}" as t
-WHERE EXISTS (
-SELECT * FROM reverted3
-WHERE ${primaryKeyColumns.map(({ sql }) => `t."${sql}" = reverted3."${sql}"`).join("AND ")}
-AND OPERATION = 0
-)
-RETURNING *
-), updated_or_deleted AS (
-INSERT INTO  "${schema}"."${getTableName(table)}"
-SELECT ${Object.values(getTableColumns(table))
-      .map((column) => `"${getColumnCasing(column, "snake_case")}"`)
-      .join(", ")} FROM reverted3
-WHERE operation = 1 OR operation = 2
-ON CONFLICT (${primaryKeyColumns.map(({ sql }) => `"${sql}"`).join(", ")})
-DO UPDATE SET
-${Object.values(getTableColumns(table))
-  .map(
-    (column) =>
-      `"${getColumnCasing(column, "snake_case")}" = EXCLUDED."${getColumnCasing(column, "snake_case")}"`,
-  )
-  .join(", ")}
-RETURNING *
-) SELECT COUNT(*) FROM reverted1 as count;
-`),
-  );
-
-  return result.rows[0]!.count as number;
-};
-
-export const finalize = async (
-  qb: QB,
-  { checkpoint, table }: { checkpoint: string; table: PgTable },
-) => {
-  await qb("finalize")
-    .delete(getReorgTable(table))
-    .where(lte(getReorgTable(table).checkpoint, checkpoint));
-};
-
-export const commitBlock = async (
-  qb: QB,
-  { checkpoint, table }: { checkpoint: string; table: PgTable },
-) => {
-  const reorgTable = getReorgTable(table);
-  await qb("commit_block")
-    .update(reorgTable)
-    .set({ checkpoint })
-    .where(eq(reorgTable.checkpoint, MAX_CHECKPOINT_STRING));
-};

package/src/drizzle/json.ts

@@ -1,154 +0,0 @@
-import { BigIntSerializationError, getBaseError } from "@/internal/errors.js";
-import { type ColumnBaseConfig, entityKind } from "drizzle-orm";
-import type {
-  ColumnBuilderBaseConfig,
-  ColumnBuilderRuntimeConfig,
-  MakeColumnConfig,
-} from "drizzle-orm/column-builder";
-import {
-  type AnyPgTable,
-  PgColumn,
-  PgColumnBuilder,
-} from "drizzle-orm/pg-core";
-
-export type PgJsonBuilderInitial<TName extends string> = PgJsonBuilder<{
-  name: TName;
-  dataType: "json";
-  columnType: "PgJson";
-  data: unknown;
-  driverParam: string;
-  enumValues: undefined;
-}>;
-
-export class PgJsonBuilder<
-  T extends ColumnBuilderBaseConfig<"json", "PgJson">,
-> extends PgColumnBuilder<T> {
-  static readonly [entityKind]: string = "PgJsonBuilder";
-
-  constructor(name: T["name"]) {
-    super(name, "json", "PgJson");
-  }
-
-  /** @internal */
-  // @ts-ignore
-  override build<TTableName extends string>(
-    table: AnyPgTable<{ name: TTableName }>,
-  ): PgJson<MakeColumnConfig<T, TTableName>> {
-    return new PgJson<MakeColumnConfig<T, TTableName>>(
-      table,
-      this.config as ColumnBuilderRuntimeConfig<any, any>,
-    );
-  }
-}
-
-export class PgJson<
-  T extends ColumnBaseConfig<"json", "PgJson">,
-> extends PgColumn<T> {
-  static readonly [entityKind]: string = "PgJson";
-
-  getSQLType(): string {
-    return "json";
-  }
-
-  override mapToDriverValue(value: T["data"]): string {
-    try {
-      return JSON.stringify(value);
-    } catch (_error) {
-      let error = getBaseError(_error);
-      if (error?.message?.includes("Do not know how to serialize a BigInt")) {
-        error = new BigIntSerializationError(error.message);
-        error.meta.push(
-          "Hint:\n  The JSON column type does not support BigInt values. Use the replaceBigInts() helper function before inserting into the database. Docs: https://ponder.sh/docs/api-reference/ponder-utils#replacebigints",
-        );
-      }
-
-      throw error;
-    }
-  }
-
-  override mapFromDriverValue(value: T["data"] | string): T["data"] {
-    if (typeof value === "string") {
-      try {
-        return JSON.parse(value);
-      } catch {
-        return value as T["data"];
-      }
-    }
-    return value;
-  }
-}
-
-export type PgJsonbBuilderInitial<TName extends string> = PgJsonbBuilder<{
-  name: TName;
-  dataType: "json";
-  columnType: "PgJsonb";
-  data: unknown;
-  driverParam: unknown;
-  enumValues: undefined;
-}>;
-
-export class PgJsonbBuilder<
-  T extends ColumnBuilderBaseConfig<"json", "PgJsonb">,
-> extends PgColumnBuilder<T> {
-  static override readonly [entityKind]: string = "PgJsonbBuilder";
-
-  constructor(name: T["name"]) {
-    super(name, "json", "PgJsonb");
-  }
-
-  /** @internal */
-  // @ts-ignore
-  override build<TTableName extends string>(
-    table: AnyPgTable<{ name: TTableName }>,
-  ): PgJsonb<MakeColumnConfig<T, TTableName>> {
-    return new PgJsonb<MakeColumnConfig<T, TTableName>>(
-      table,
-      this.config as ColumnBuilderRuntimeConfig<any, any>,
-    );
-  }
-}
-
-export class PgJsonb<
-  T extends ColumnBaseConfig<"json", "PgJsonb">,
-> extends PgColumn<T> {
-  static override readonly [entityKind]: string = "PgJsonb";
-
-  // biome-ignore lint/complexity/noUselessConstructor: <explanation>
-  constructor(
-    table: AnyPgTable<{ name: T["tableName"] }>,
-    config: PgJsonbBuilder<T>["config"],
-  ) {
-    super(table, config);
-  }
-
-  getSQLType(): string {
-    return "jsonb";
-  }
-
-  override mapToDriverValue(value: T["data"]): string {
-    try {
-      return JSON.stringify(value);
-    } catch (_error) {
-      let error = getBaseError(_error);
-      if (error?.message?.includes("Do not know how to serialize a BigInt")) {
-        error = new BigIntSerializationError(error.message);
-        error.meta.push(
-          "Hint:\n  The JSON column type does not support BigInt values. Use the replaceBigInts() helper function before inserting into the database. Docs: https://ponder.sh/docs/api-reference/ponder-utils#replacebigints",
-        );
-      }
-
-      throw error;
-    }
-  }
-
-  override mapFromDriverValue(value: T["data"] | string): T["data"] {
-    if (typeof value === "string") {
-      try {
-        return JSON.parse(value);
-      } catch {
-        return value as T["data"];
-      }
-    }
-    return value;
-  }
-}