ponder 0.11.21 → 0.11.22

package/src/drizzle/index.ts

@@ -1,12 +1,13 @@
-import { getTableColumns, getTableName } from "drizzle-orm";
+import { type TableConfig, getTableColumns, getTableName } from "drizzle-orm";
 import {
   type PgColumn,
   type PgTable,
+  type PgTableWithColumns,
   getTableConfig,
 } from "drizzle-orm/pg-core";
 import { getColumnCasing, sqlToReorgTableName } from "./kit/index.js";
 
-export const getTableNames = (table: PgTable) => {
+export const getTableNames = (table: PgTableWithColumns<TableConfig>) => {
   const sql = getTableName(table);
 
   return {

package/src/drizzle/kit/index.ts

@@ -1,4 +1,4 @@
-import { SQL, getTableName, is } from "drizzle-orm";
+import { SQL, type TableConfig, getTableName, is } from "drizzle-orm";
 import { CasingCache, toCamelCase, toSnakeCase } from "drizzle-orm/casing";
 import {
   type AnyPgTable,
@@ -9,6 +9,7 @@ import {
   PgSchema,
   type PgSequence,
   PgTable,
+  type PgTableWithColumns,
   PgView,
   getTableConfig,
   index,
@@ -39,7 +40,9 @@ export type SqlStatements = {
 export const sqlToReorgTableName = (tableName: string) =>
   `_reorg__${tableName}`;
 
-export const getReorgTable = (table: PgTable) => {
+export const getReorgTable = <config extends TableConfig>(
+  table: PgTableWithColumns<config>,
+) => {
   const schema = getTableConfig(table).schema;
 
   if (schema && schema !== "public") {

package/src/drizzle/onchain.ts

@@ -26,12 +26,6 @@ import {
 import { PgBigintBuilder, type PgBigintBuilderInitial } from "./bigint.js";
 import { PgBytesBuilder, type PgBytesBuilderInitial } from "./bytes.js";
 import { PgHexBuilder, type PgHexBuilderInitial } from "./hex.js";
-import {
-  PgJsonBuilder,
-  type PgJsonBuilderInitial,
-  PgJsonbBuilder,
-  type PgJsonbBuilderInitial,
-} from "./json.js";
 import { PgTextBuilder, type PgTextBuilderInitial } from "./text.js";
 
 /** @internal */
@@ -62,22 +56,6 @@ export function bigint(columnName?: string) {
   return new PgBigintBuilder(columnName ?? "");
 }
 
-export function json(): PgJsonBuilderInitial<"">;
-export function json<name extends string>(
-  name: name,
-): PgJsonBuilderInitial<name>;
-export function json(name?: string) {
-  return new PgJsonBuilder(name ?? "");
-}
-
-export function jsonb(): PgJsonbBuilderInitial<"">;
-export function jsonb<name extends string>(
-  name: name,
-): PgJsonbBuilderInitial<name>;
-export function jsonb(name?: string) {
-  return new PgJsonbBuilder(name ?? "");
-}
-
 // @ts-ignore
 export function bytes(): PgBytesBuilderInitial<"">;
 export function bytes<name extends string>(
@@ -143,7 +121,7 @@ export type BuildExtraConfigColumns<
 
 export type PgColumnsBuilders = Omit<
   _PgColumnsBuilders,
-  "bigint" | "serial" | "smallserial" | "bigserial" | "json" | "jsonb"
+  "bigint" | "serial" | "smallserial" | "bigserial"
 > & {
   /**
    * Create an 8 byte number column.
@@ -188,8 +166,6 @@ export type PgColumnsBuilders = Omit<
    * }));
    */
   bytes: typeof bytes;
-  json: typeof json;
-  jsonb: typeof jsonb;
 };
 
 /**
@@ -297,7 +273,7 @@ function pgTableWithSchema<
 
   const parsedColumns: columns =
     typeof columns === "function"
-      ? columns({ ...restColumns, int8, hex, bigint, bytes, text, json, jsonb })
+      ? columns({ ...restColumns, int8, hex, bigint, bytes, text })
       : columns;
 
   const builtColumns = Object.fromEntries(

package/src/graphql/index.ts

@@ -1,7 +1,8 @@
-import type { QB } from "@/database/queryBuilder.js";
+import type { Database } from "@/database/index.js";
 import type { OnchainTable } from "@/drizzle/onchain.js";
 import { normalizeColumn } from "@/indexing-store/utils.js";
 import type { Schema, Status } from "@/internal/types.js";
+import type { Drizzle, ReadonlyDrizzle } from "@/types/db.js";
 import { decodeCheckpoint } from "@/utils/checkpoint.js";
 import { never } from "@/utils/never.js";
 import { deserialize, serialize } from "@/utils/serialize.js";
@@ -36,7 +37,6 @@ import {
   notInArray,
   notLike,
   or,
-  sql,
 } from "drizzle-orm";
 import {
   type PgEnum,
@@ -68,8 +68,9 @@ import { GraphQLJSON } from "./json.js";
 
 type Parent = Record<string, any>;
 type Context = {
-  qb: QB<{ [key: string]: OnchainTable }>;
   getDataLoader: ReturnType<typeof buildDataLoaderCache>;
+  getCheckpoints: Database["getCheckpoints"];
+  drizzle: Drizzle<{ [key: string]: OnchainTable }>;
 };
 
 type PluralArgs = {
@@ -334,7 +335,7 @@ export function buildGraphQLSchema({
 
                 return executePluralQuery(
                   referencedTable,
-                  context.qb,
+                  context.drizzle,
                   args,
                   includeTotalCount,
                   relationalConditions,
@@ -414,7 +415,12 @@ export function buildGraphQLSchema({
       resolve: async (_parent, args: PluralArgs, context, info) => {
         const includeTotalCount = selectionIncludesField(info, "totalCount");
 
-        return executePluralQuery(table, context.qb, args, includeTotalCount);
+        return executePluralQuery(
+          table,
+          context.drizzle,
+          args,
+          includeTotalCount,
+        );
       },
     };
   }
@@ -422,27 +428,15 @@ export function buildGraphQLSchema({
   queryFields._meta = {
     type: GraphQLMeta,
     resolve: async (_source, _args, context) => {
-      // Note: This is done to avoid non-browser compatible dependencies
-      const checkpoints = (await context
-        .qb()
-        .execute(
-          sql`SELECT chain_name, chain_id, latest_checkpoint, safe_checkpoint from _ponder_checkpoint`,
-        )
-        .then((res) => res.rows)) as {
-        chain_name: string;
-        chain_id: number;
-        latest_checkpoint: string;
-        safe_checkpoint: string;
-      }[];
-
+      const checkpoints = await context.getCheckpoints();
       const status: Status = {};
-      for (const { chain_name, chain_id, latest_checkpoint } of checkpoints) {
-        status[chain_name] = {
-          id: chain_id,
+      for (const { chainName, chainId, latestCheckpoint } of checkpoints) {
+        status[chainName] = {
+          id: chainId,
           block: {
-            number: Number(decodeCheckpoint(latest_checkpoint).blockNumber),
+            number: Number(decodeCheckpoint(latestCheckpoint).blockNumber),
             timestamp: Number(
-              decodeCheckpoint(latest_checkpoint).blockTimestamp,
+              decodeCheckpoint(latestCheckpoint).blockTimestamp,
             ),
           },
         };
@@ -566,13 +560,13 @@ const innerType = (
 
 async function executePluralQuery(
   table: TableRelationalConfig,
-  qb: QB<{ [key: string]: OnchainTable }>,
+  drizzle: Drizzle<{ [key: string]: OnchainTable }>,
   args: PluralArgs,
   includeTotalCount: boolean,
   extraConditions: (SQL | undefined)[] = [],
 ) {
-  const rawTable = qb()._.fullSchema[table.tsName];
-  const baseQuery = qb().query[table.tsName];
+  const rawTable = drizzle._.fullSchema[table.tsName];
+  const baseQuery = drizzle.query[table.tsName];
   if (rawTable === undefined || baseQuery === undefined)
     throw new Error(`Internal error: Table "${table.tsName}" not found in RQB`);
 
@@ -616,7 +610,7 @@ async function executePluralQuery(
   let hasNextPage = false;
 
   const totalCountPromise = includeTotalCount
-    ? qb()
+    ? drizzle
         .select({ count: count() })
         .from(rawTable)
         .where(and(...whereConditions, ...extraConditions))
@@ -1019,15 +1013,16 @@ function buildCursorCondition(
   return buildCondition(0);
 }
 
-export function buildDataLoaderCache(qb: QB) {
+export function buildDataLoaderCache({
+  drizzle,
+}: { drizzle: ReadonlyDrizzle<Schema> }) {
   const dataLoaderMap = new Map<
     TableRelationalConfig,
     DataLoader<string, any> | undefined
   >();
   return ({ table }: { table: TableRelationalConfig }) => {
-    const baseQuery = (qb as QB<{ [key: string]: OnchainTable }>)().query[
-      table.tsName
-    ];
+    const baseQuery = (drizzle as Drizzle<{ [key: string]: OnchainTable }>)
+      .query[table.tsName];
     if (baseQuery === undefined)
       throw new Error(
         `Internal error: Unknown table "${table.tsName}" in data loader cache`,

package/src/graphql/middleware.ts

@@ -28,7 +28,7 @@ import { buildDataLoaderCache, buildGraphQLSchema } from "./index.js";
  *
  */
 export const graphql = (
-  { schema }: { db: ReadonlyDrizzle<Schema>; schema: Schema },
+  { db, schema }: { db: ReadonlyDrizzle<Schema>; schema: Schema },
   {
     maxOperationTokens = 1000,
     maxOperationDepth = 100,
@@ -53,11 +53,13 @@ export const graphql = (
     graphqlEndpoint: "*", // Disable built-in route validation, use Hono routing instead
     schema: graphqlSchema,
     context: () => {
-      const getDataLoader = buildDataLoaderCache(
-        globalThis.PONDER_DATABASE.readonlyQB,
-      );
+      const getDataLoader = buildDataLoaderCache({ drizzle: db });
 
-      return { qb: globalThis.PONDER_DATABASE.readonlyQB, getDataLoader };
+      return {
+        getDataLoader,
+        getCheckpoints: () => globalThis.PONDER_DATABASE.getCheckpoints(),
+        drizzle: db,
+      };
     },
     maskedErrors: process.env.NODE_ENV === "production",
     logging: false,

package/src/indexing-store/cache.ts

@@ -1,6 +1,5 @@
 import { Readable } from "node:stream";
 import { pipeline } from "node:stream/promises";
-import type { QB } from "@/database/queryBuilder.js";
 import { getPrimaryKeyColumns } from "@/drizzle/index.js";
 import { getColumnCasing } from "@/drizzle/kit/index.js";
 import { addErrorMeta, toErrorMeta } from "@/indexing/index.js";
@@ -9,11 +8,14 @@ import { FlushError } from "@/internal/errors.js";
 import type {
   CrashRecoveryCheckpoint,
   Event,
+  Schema,
   SchemaBuild,
 } from "@/internal/types.js";
+import type { Drizzle } from "@/types/db.js";
 import { dedupe } from "@/utils/dedupe.js";
 import { prettyPrint } from "@/utils/print.js";
 import { startClock } from "@/utils/timer.js";
+import { PGlite } from "@electric-sql/pglite";
 import {
   type Column,
   type Table,
@@ -22,13 +24,13 @@ import {
   getTableName,
   is,
   or,
-  sql,
 } from "drizzle-orm";
 import {
   PgTable,
   type PgTableWithColumns,
   getTableConfig,
 } from "drizzle-orm/pg-core";
+import type { PoolClient } from "pg";
 import copy from "pg-copy-streams";
 import {
   getProfilePatternKey,
@@ -45,10 +47,11 @@ export type IndexingCache = {
   /**
    * Returns the entry for `table` with `key`.
    */
-  get: (params: { table: Table; key: object }) =>
-    | Row
-    | null
-    | Promise<Row | null>;
+  get: (params: {
+    table: Table;
+    key: object;
+    db: Drizzle<Schema>;
+  }) => Row | null | Promise<Row | null>;
   /**
    * Sets the entry for `table` with `key` to `row`.
    */
@@ -61,17 +64,27 @@ export type IndexingCache = {
   /**
    * Deletes the entry for `table` with `key`.
    */
-  delete: (params: { table: Table; key: object }) => boolean | Promise<boolean>;
+  delete: (params: {
+    table: Table;
+    key: object;
+    db: Drizzle<Schema>;
+  }) => boolean | Promise<boolean>;
   /**
    * Writes all temporary data to the database.
    *
    * @param params.tableNames - If provided, only flush the tables in the set.
    */
-  flush: (params?: { tableNames?: Set<string> }) => Promise<void>;
+  flush: (params: {
+    client: PoolClient | PGlite;
+    tableNames?: Set<string>;
+  }) => Promise<void>;
   /**
    * Predict and load rows that will be accessed in the next event batch.
    */
-  prefetch: (params: { events: Event[] }) => Promise<void>;
+  prefetch: (params: {
+    events: Event[];
+    db: Drizzle<Schema>;
+  }) => Promise<void>;
   /**
    * Remove spillover and buffer entries.
    */
@@ -85,7 +98,6 @@ export type IndexingCache = {
    */
   clear: () => void;
   event: Event | undefined;
-  qb: QB;
 };
 
 const SAMPLING_RATE = 10;
@@ -219,15 +231,15 @@ export const getCopyText = (table: Table, rows: Row[]) => {
   return results.join("\n");
 };
 
-export const getCopyHelper = (qb: QB) => {
-  if (qb.$dialect === "pglite") {
+export const getCopyHelper = ({ client }: { client: PoolClient | PGlite }) => {
+  if (client instanceof PGlite) {
     return async (table: Table, text: string, includeSchema = true) => {
       const target = includeSchema
         ? `"${getTableConfig(table).schema ?? "public"}"."${getTableName(
             table,
           )}"`
         : `"${getTableName(table)}"`;
-      await qb.$client.query(`COPY ${target} FROM '/dev/blob'`, [], {
+      await client.query(`COPY ${target} FROM '/dev/blob'`, [], {
         blob: new Blob([text]),
       });
     };
@@ -240,7 +252,7 @@ export const getCopyHelper = (qb: QB) => {
         : `"${getTableName(table)}"`;
       await pipeline(
         Readable.from(text),
-        qb.$client.query(copy.from(`COPY ${target} FROM STDIN`)),
+        client.query(copy.from(`COPY ${target} FROM STDIN`)),
       );
     };
   }
@@ -291,7 +303,6 @@ export const createIndexingCache = ({
    */
   let cacheBytes = 0;
   let event: Event | undefined;
-  let qb: QB = undefined!;
   let isCacheComplete = crashRecoveryCheckpoint === undefined;
   const primaryKeyCache = new Map<Table, [string, Column][]>();
 
@@ -330,7 +341,7 @@ export const createIndexingCache = ({
         updateBuffer.get(table)!.has(ck)
       );
     },
-    async get({ table, key }) {
+    async get({ table, key, db }) {
       if (event && eventCount[event.name]! % SAMPLING_RATE === 1) {
         if (profile.has(event.name) === false) {
           profile.set(event.name, new Map());
@@ -402,7 +413,7 @@ export const createIndexingCache = ({
 
       const endClock = startClock();
 
-      const result = await qb()
+      const result = await db
         .select()
         .from(table)
         .where(getWhereCondition(table, key))
@@ -443,7 +454,7 @@ export const createIndexingCache = ({
 
       return row;
     },
-    async delete({ table, key }) {
+    async delete({ table, key, db }) {
       const ck = getCacheKey(table, key);
 
       const inInsertBuffer = insertBuffer.get(table)!.delete(ck);
@@ -451,7 +462,7 @@ export const createIndexingCache = ({
 
       cache.get(table)!.delete(ck);
 
-      const inDb = await qb()
+      const inDb = await db
         .delete(table)
         .where(getWhereCondition(table, key))
         .returning()
@@ -459,8 +470,8 @@ export const createIndexingCache = ({
 
       return inInsertBuffer || inUpdateBuffer || inDb;
     },
-    async flush({ tableNames } = {}) {
-      const copy = getCopyHelper(qb);
+    async flush({ client, tableNames }) {
+      const copy = getCopyHelper({ client });
 
       const shouldRecordBytes = isCacheComplete;
 
@@ -480,7 +491,8 @@ export const createIndexingCache = ({
         if (insertValues.length > 0) {
           const endClock = startClock();
 
-          await qb().execute(sql.raw("SAVEPOINT flush"));
+          // @ts-ignore
+          await client.query("SAVEPOINT flush");
 
           try {
             const text = getCopyText(
@@ -496,13 +508,15 @@ export const createIndexingCache = ({
             const result = await recoverBatchError(
               insertValues,
               async (values) => {
-                await qb().execute(sql.raw("ROLLBACK to flush"));
+                // @ts-ignore
+                await client.query("ROLLBACK to flush");
                 const text = getCopyText(
                   table,
                   values.map(({ row }) => row),
                 );
                 await copy(table, text);
-                await qb().execute(sql.raw("SAVEPOINT flush"));
+                // @ts-ignore
+                await client.query("SAVEPOINT flush");
               },
             );
 
@@ -615,8 +629,10 @@ export const createIndexingCache = ({
 
           const endClock = startClock();
 
-          await qb().execute(sql.raw(createTempTableQuery));
-          await qb().execute(sql.raw("SAVEPOINT flush"));
+          // @ts-ignore
+          await client.query(createTempTableQuery);
+          // @ts-ignore
+          await client.query("SAVEPOINT flush");
 
           try {
             const text = getCopyText(
@@ -632,13 +648,15 @@ export const createIndexingCache = ({
             const result = await recoverBatchError(
               updateValues,
               async (values) => {
-                await qb().execute(sql.raw("ROLLBACK to flush"));
+                // @ts-ignore
+                await client.query("ROLLBACK to flush");
                 const text = getCopyText(
                   table,
                   values.map(({ row }) => row),
                 );
                 await copy(table, text, false);
-                await qb().execute(sql.raw("SAVEPOINT flush"));
+                // @ts-ignore
+                await client.query("SAVEPOINT flush");
               },
             );
 
@@ -691,7 +709,8 @@ export const createIndexingCache = ({
             );
           }
 
-          await qb().execute(sql.raw(updateQuery));
+          // @ts-ignore
+          await client.query(updateQuery);
 
           common.metrics.ponder_indexing_cache_query_duration.observe(
             {
@@ -718,11 +737,12 @@ export const createIndexingCache = ({
         }
 
         if (insertValues.length > 0 || updateValues.length > 0) {
-          await qb().execute(sql.raw("RELEASE flush"));
+          // @ts-ignore
+          await client.query("RELEASE flush");
         }
       }
     },
-    async prefetch({ events }) {
+    async prefetch({ events, db }) {
       if (isCacheComplete) {
         if (cacheBytes < common.options.indexingCacheMaxBytes) {
           return;
@@ -803,7 +823,7 @@ export const createIndexingCache = ({
             if (conditions.length === 0) return;
             const endClock = startClock();
 
-            await qb()
+            await db
               .select()
               .from(table)
               .where(or(...conditions))
@@ -878,8 +898,5 @@ export const createIndexingCache = ({
     set event(_event: Event | undefined) {
       event = _event;
     },
-    set qb(_qb: QB) {
-      qb = _qb;
-    },
   };
 };