ponder 0.14.13 → 0.15.0

package/src/internal/options.ts

@@ -31,6 +31,8 @@ export type Options = {
   databaseHeartbeatTimeout: number;
   databaseMaxQueryParameters: number;
 
+  maxThreads: number;
+
   factoryAddressCountThreshold: number;
 
   indexingCacheMaxBytes: number;
@@ -103,6 +105,8 @@ export const buildOptions = ({ cliOptions }: { cliOptions: CliOptions }) => {
     // Half of the max query parameters for PGlite
     databaseMaxQueryParameters: 16_000,
 
+    maxThreads: 4,
+
     factoryAddressCountThreshold: 1_000,
 
     rpcMaxConcurrency: 256,

package/src/internal/telemetry.ts

@@ -295,7 +295,7 @@ export function buildPayload({
 
   return {
     database_kind: preBuild?.databaseConfig.kind,
-    contract_count: indexingBuild?.sources.length ?? 0,
+    contract_count: 0,
     network_count: indexingBuild?.chains.length ?? 0,
     table_count,
     indexing_function_count,

package/src/internal/types.ts

@@ -9,7 +9,6 @@ import type {
   Transfer,
 } from "@/types/eth.js";
 import type { PartialExcept, Prettify } from "@/types/utils.js";
-import type { AbiEvents, AbiFunctions } from "@/utils/abi.js";
 import type { Trace as DebugTrace } from "@/utils/debug.js";
 import type { PGliteOptions } from "@/utils/pglite.js";
 import type { PGlite } from "@electric-sql/pglite";
@@ -17,6 +16,8 @@ import type { Hono } from "hono";
 import type { PoolConfig } from "pg";
 import type {
   Abi,
+  AbiEvent,
+  AbiFunction,
   Address,
   BlockTag,
   Hex,
@@ -40,18 +41,13 @@ export type DatabaseConfig =
 // Indexing
 
 /** Indexing functions as defined in `ponder.on()` */
-export type RawIndexingFunctions = {
+export type IndexingFunctions = {
   /** Name of the event */
   name: string;
   /** Callback function */
   fn: (...args: any) => any;
 }[];
 
-/** Indexing functions for event callbacks */
-export type IndexingFunctions = {
-  [eventName: string]: (...args: any) => any;
-};
-
 // Filters
 
 /** Filter definition based on the fundamental data model of the Ethereum blockchain. */
@@ -61,12 +57,6 @@ export type Filter =
   | TransferFilter
   | TransactionFilter
   | TraceFilter;
-export type FilterWithoutBlocks =
-  | Omit<BlockFilter, "fromBlock" | "toBlock">
-  | Omit<TransactionFilter, "fromBlock" | "toBlock">
-  | Omit<TraceFilter, "fromBlock" | "toBlock">
-  | Omit<LogFilter, "fromBlock" | "toBlock">
-  | Omit<TransferFilter, "fromBlock" | "toBlock">;
 
 /** Filter that matches addresses. */
 export type Factory = LogFactory;
@@ -77,6 +67,7 @@ export type FilterAddress<
 export type BlockFilter = {
   type: "block";
   chainId: number;
+  sourceId: string;
   interval: number;
   offset: number;
   fromBlock: number | undefined;
@@ -91,6 +82,7 @@ export type TransactionFilter<
 > = {
   type: "transaction";
   chainId: number;
+  sourceId: string;
   fromAddress: FilterAddress<fromFactory>;
   toAddress: FilterAddress<toFactory>;
   includeReverted: boolean;
@@ -110,9 +102,10 @@ export type TraceFilter<
 > = {
   type: "trace";
   chainId: number;
+  sourceId: string;
   fromAddress: FilterAddress<fromFactory>;
   toAddress: FilterAddress<toFactory>;
-  functionSelector: Hex | Hex[] | undefined;
+  functionSelector: Hex;
   callType: Trace["type"] | undefined;
   includeReverted: boolean;
   fromBlock: number | undefined;
@@ -131,8 +124,9 @@ export type LogFilter<
 > = {
   type: "log";
   chainId: number;
+  sourceId: string;
   address: FilterAddress<factory>;
-  topic0: LogTopic;
+  topic0: Hex;
   topic1: LogTopic;
   topic2: LogTopic;
   topic3: LogTopic;
@@ -153,6 +147,7 @@ export type TransferFilter<
 > = {
   type: "transfer";
   chainId: number;
+  sourceId: string;
   fromAddress: FilterAddress<fromFactory>;
   toAddress: FilterAddress<toFactory>;
   includeReverted: boolean;
@@ -215,14 +210,14 @@ export type Fragment =
       chainId: number;
       fromAddress: FragmentAddress;
       toAddress: FragmentAddress;
-      functionSelector: Hex | null;
+      functionSelector: Hex;
       includeTransactionReceipts: boolean;
     }
   | {
       type: "log";
       chainId: number;
       address: FragmentAddress;
-      topic0: FragmentTopic;
+      topic0: Hex;
       topic1: FragmentTopic;
       topic2: FragmentTopic;
       topic3: FragmentTopic;
@@ -234,6 +229,13 @@ export type Fragment =
       fromAddress: FragmentAddress;
       toAddress: FragmentAddress;
       includeTransactionReceipts: boolean;
+    }
+  | {
+      type: "factory_log";
+      chainId: number;
+      address: Address;
+      eventSelector: Factory["eventSelector"];
+      childAddressLocation: Factory["childAddressLocation"];
     };
 
 /** Minimum slice of a {@link Filter} */
@@ -247,53 +249,40 @@ export type FragmentId =
   /** log_{chainId}_{address}_{topic0}_{topic1}_{topic2}_{topic3}_{includeReceipts} */
   | `log_${number}_${FragmentAddressId}_${FragmentTopic}_${FragmentTopic}_${FragmentTopic}_${FragmentTopic}_${0 | 1}`
   /** transfer_{chainId}_{fromAddress}_{toAddress}_{includeReceipts} */
-  | `transfer_${number}_${FragmentAddressId}_${FragmentAddressId}_${0 | 1}`;
-
-// Sources
-
-/** Event source that matches {@link Event}s containing an underlying filter and metadata. */
-export type Source = ContractSource | AccountSource | BlockSource;
-
-export type ContractSource<
-  filter extends "log" | "trace" = "log" | "trace",
-  factory extends Factory | undefined = Factory | undefined,
-  fromFactory extends Factory | undefined = Factory | undefined,
-  toFactory extends Factory | undefined = Factory | undefined,
-> = {
-  filter: filter extends "log"
-    ? LogFilter<factory>
-    : TraceFilter<fromFactory, toFactory>;
-} & ContractMetadata;
-
-export type AccountSource<
-  filter extends "transaction" | "transfer" = "transaction" | "transfer",
-  fromFactory extends Factory | undefined = Factory | undefined,
-  toFactory extends Factory | undefined = Factory | undefined,
-> = {
-  filter: filter extends "transaction"
-    ? TransactionFilter<fromFactory, toFactory>
-    : TransferFilter<fromFactory, toFactory>;
-} & AccountMetadata;
+  | `transfer_${number}_${FragmentAddressId}_${FragmentAddressId}_${0 | 1}`
+  /** factory_log_{chainId}_{address}_{eventSelector}_{childAddressLocation} */
+  | `factory_log_${number}_${Address}_${Factory["eventSelector"]}_${Factory["childAddressLocation"]}`;
 
-export type BlockSource = { filter: BlockFilter } & BlockMetadata;
-
-export type ContractMetadata = {
-  type: "contract";
+// Contract
+export type Contract = {
   abi: Abi;
-  abiEvents: AbiEvents;
-  abiFunctions: AbiFunctions;
-  name: string;
-  chain: Chain;
+  address?: Address | readonly Address[];
+  startBlock?: number;
+  endBlock?: number;
 };
-export type AccountMetadata = {
-  type: "account";
+
+// Event Callback
+
+export type EventCallback = {
+  filter: Filter;
   name: string;
+  fn: (...args: any) => any;
   chain: Chain;
-};
-export type BlockMetadata = {
-  type: "block";
+} & (
+  | {
+      type: "contract";
+      abiItem: AbiEvent | AbiFunction;
+      metadata: { safeName: string; abi: Abi };
+    }
+  | { type: "account"; direction: "from" | "to" }
+  | { type: "block" }
+);
+
+export type SetupCallback = {
   name: string;
+  fn: (...args: any) => any;
   chain: Chain;
+  block: number | undefined;
 };
 
 // Chain
@@ -327,7 +316,7 @@ export type PreBuild = {
   /** Database type and configuration */
   databaseConfig: DatabaseConfig;
   /** Ordering of events */
-  ordering: "omnichain" | "multichain";
+  ordering: "omnichain" | "multichain" | "experimental_isolated";
 };
 
 export type SchemaBuild = {
@@ -339,16 +328,22 @@ export type SchemaBuild = {
 export type IndexingBuild = {
   /** Ten character hex string identifier. */
   buildId: string;
-  /** Sources to index. */
-  sources: Source[];
   /** Chains to index. */
   chains: Chain[];
   /** RPCs for all `chains`. */
   rpcs: Rpc[];
   /** Finalized blocks for all `chains`. */
   finalizedBlocks: LightBlock[];
-  /** Event callbacks for all `sources`.  */
+  /** Event callbacks for all `chains`.  */
+  eventCallbacks: EventCallback[][];
+  /** Setup callbacks for all `chains`. */
+  setupCallbacks: SetupCallback[][];
+  /** Indexing functions registered with `ponder.on()`. */
   indexingFunctions: IndexingFunctions;
+  /** Contracts for all `chains`. */
+  contracts: {
+    [name: string]: Contract;
+  }[];
 };
 
 export type ApiBuild = {
@@ -489,9 +484,9 @@ export type UserLog = Log;
 // Events
 
 export type RawEvent = {
-  chainId: number;
-  sourceIndex: number;
   checkpoint: string;
+  chainId: number;
+  eventCallbackIndex: number;
   log?: UserLog;
   block: UserBlock;
   transaction?: UserTransaction;
@@ -508,22 +503,18 @@ export type Event =
 
 export type SetupEvent = {
   type: "setup";
-  chainId: number;
   checkpoint: string;
-
-  /** `${source.name}:setup` */
-  name: string;
+  chain: Chain;
+  setupCallback: SetupCallback;
 
   block: bigint;
 };
 
 export type BlockEvent = {
   type: "block";
-  chainId: number;
   checkpoint: string;
-
-  /** `${source.name}:block` */
-  name: string;
+  chain: Chain;
+  eventCallback: EventCallback;
 
   event: {
     id: string;
@@ -533,11 +524,9 @@ export type BlockEvent = {
 
 export type TransactionEvent = {
   type: "transaction";
-  chainId: number;
   checkpoint: string;
-
-  /** `${source.name}.{safeName}()` */
-  name: string;
+  chain: Chain;
+  eventCallback: EventCallback;
 
   event: {
     id: string;
@@ -549,11 +538,9 @@ export type TransactionEvent = {
 
 export type TraceEvent = {
   type: "trace";
-  chainId: number;
   checkpoint: string;
-
-  /** `${source.name}:transfer:from` | `${source.name}:transfer:to` */
-  name: string;
+  chain: Chain;
+  eventCallback: EventCallback;
 
   event: {
     id: string;
@@ -568,11 +555,9 @@ export type TraceEvent = {
 
 export type LogEvent = {
   type: "log";
-  chainId: number;
   checkpoint: string;
-
-  /** `${source.name}:${safeName}` */
-  name: string;
+  chain: Chain;
+  eventCallback: EventCallback;
 
   event: {
     id: string;
@@ -586,11 +571,9 @@ export type LogEvent = {
 
 export type TransferEvent = {
   type: "transfer";
-  chainId: number;
   checkpoint: string;
-
-  /** `${source.name}:transfer:from` | `${source.name}:transfer:to` */
-  name: string;
+  chain: Chain;
+  eventCallback: EventCallback;
 
   event: {
     id: string;

package/src/rpc/http.ts

@@ -0,0 +1,164 @@
+import { URL } from "node:url";
+import type { Common } from "@/internal/common.js";
+import type { Chain } from "@/internal/types.js";
+import { HttpRequestError, RpcRequestError, TimeoutError } from "viem";
+import {
+  type HttpRequestParameters,
+  type HttpRequestReturnType,
+  type HttpRpcClientOptions,
+  stringify,
+} from "viem/utils";
+
+export type RpcRequest = {
+  jsonrpc?: "2.0" | undefined;
+  method: string;
+  params?: any | undefined;
+  id?: number | undefined;
+};
+
+export type HttpRpcClient = {
+  request<body extends RpcRequest>(
+    params: HttpRequestParameters<body>,
+  ): Promise<HttpRequestReturnType<body>>;
+};
+
+export function getHttpRpcClient(
+  url: string,
+  options: HttpRpcClientOptions & { common: Common; chain: Chain },
+): HttpRpcClient {
+  const { common, chain } = options;
+  const timeoutMs = options?.timeout ?? 10_000;
+  let id = 1;
+  return {
+    async request(params) {
+      // biome-ignore lint/suspicious/noAsyncPromiseExecutor: <explanation>
+      return new Promise(async (resolve, reject) => {
+        let isTimeoutRejected = false;
+        const { body } = params;
+
+        const fetchOptions = {
+          ...(params.fetchOptions ?? {}),
+        };
+
+        const { headers, method } = fetchOptions;
+
+        let reader: ReadableStreamDefaultReader<Uint8Array> | undefined;
+        const controller = new AbortController();
+        const timeoutId = setTimeout(async () => {
+          isTimeoutRejected = true;
+          controller.abort();
+
+          reject(new TimeoutError({ body, url }));
+
+          if (reader) {
+            common.logger.warn({
+              msg: "JSON-RPC request timed out reading response body",
+              chain: chain.name,
+              chain_id: chain.id,
+              hostname: new URL(url).hostname,
+              // @ts-ignore
+              request_id: headers ? headers["X-Request-ID"] : undefined,
+              method: body.method,
+              request: JSON.stringify(body),
+              duration: timeoutMs,
+            });
+            try {
+              await reader.cancel("Timeout");
+            } catch {}
+          }
+        }, timeoutMs);
+
+        try {
+          const init: RequestInit = {
+            body: stringify({
+              jsonrpc: "2.0",
+              id: body.id ?? id++,
+              ...body,
+            }),
+            headers: {
+              "Content-Type": "application/json",
+              ...headers,
+            },
+            method: method || "POST",
+            signal: controller.signal,
+          };
+          const request = new Request(url, init);
+          const response = await fetch(request);
+
+          reader = response.body?.getReader()!;
+          const chunks: Uint8Array[] = [];
+          let totalLength = 0;
+
+          try {
+            while (true) {
+              const { done, value } = await reader.read();
+              if (done) break;
+              chunks.push(value);
+              totalLength += value.length;
+            }
+          } finally {
+            reader.releaseLock();
+            reader = undefined;
+          }
+
+          if (isTimeoutRejected) return;
+
+          let offset = 0;
+          const fullData = new Uint8Array(totalLength);
+          for (const chunk of chunks) {
+            fullData.set(chunk, offset);
+            offset += chunk.length;
+          }
+
+          const text = new TextDecoder().decode(fullData);
+
+          let data: any = text;
+          try {
+            data = JSON.parse(data || "{}");
+          } catch (err) {
+            if (response.ok) throw err;
+            data = { error: data };
+          }
+
+          clearTimeout(timeoutId);
+
+          if (!response.ok) {
+            reject(
+              new HttpRequestError({
+                body,
+                details: stringify(data.error) || response.statusText,
+                headers: response.headers,
+                status: response.status,
+                url,
+              }),
+            );
+            return;
+          }
+
+          if (data.error) {
+            reject(
+              new RpcRequestError({
+                body,
+                error: data.error,
+                url: url,
+              }),
+            );
+          } else {
+            resolve(data.result);
+          }
+        } catch (_error) {
+          const error = _error as Error;
+          clearTimeout(timeoutId);
+
+          if (isTimeoutRejected) return;
+
+          if (error.name === "AbortError") {
+            reject(new TimeoutError({ body, url }));
+          }
+          if (error instanceof HttpRequestError) reject(error);
+          reject(new HttpRequestError({ body, cause: error, url }));
+        }
+      });
+    },
+  };
+}

package/src/rpc/index.ts

@@ -1,4 +1,4 @@
-import crypto from "node:crypto";
+import crypto, { type UUID } from "node:crypto";
 import url from "node:url";
 import type { Common } from "@/internal/common.js";
 import type { Logger } from "@/internal/logger.js";
@@ -16,7 +16,6 @@ import {
   getLogsRetryHelper,
 } from "@ponder/utils";
 import {
-  http,
   BlockNotFoundError,
   type EIP1193Parameters,
   type EIP1193RequestFn,
@@ -30,12 +29,14 @@ import {
   type RpcError,
   type RpcTransactionReceipt,
   TimeoutError,
+  custom,
   hexToNumber,
   isHex,
   webSocket,
 } from "viem";
 import { WebSocket } from "ws";
 import type { DebugRpcSchema } from "../utils/debug.js";
+import { getHttpRpcClient } from "./http.js";
 
 export type RpcSchema = [
   ...PublicRpcSchema,
@@ -176,16 +177,32 @@ export const createRpc = ({
 }: { common: Common; chain: Chain; concurrency?: number }): Rpc => {
   let backends: { request: EIP1193RequestFn<RpcSchema>; hostname: string }[];
 
+  let requestId: UUID | undefined;
+
   if (typeof chain.rpc === "string") {
     const protocol = new url.URL(chain.rpc).protocol;
     const hostname = new url.URL(chain.rpc).hostname;
     if (protocol === "https:" || protocol === "http:") {
+      const httpRpcClient = getHttpRpcClient(chain.rpc, {
+        common,
+        chain,
+        timeout: 10_000,
+      });
       backends = [
         {
-          request: http(chain.rpc)({
+          request: custom({
+            request(body) {
+              if (requestId) {
+                return httpRpcClient.request({
+                  body,
+                  fetchOptions: { headers: { "X-Request-ID": requestId } },
+                });
+              }
+              return httpRpcClient.request({ body });
+            },
+          })({
             chain: chain.viemChain,
             retryCount: 0,
-            timeout: 10_000,
           }).request,
           hostname,
         },
@@ -210,11 +227,25 @@ export const createRpc = ({
       const hostname = new url.URL(chain.rpc).hostname;
 
       if (protocol === "https:" || protocol === "http:") {
+        const httpRpcClient = getHttpRpcClient(rpc, {
+          common,
+          chain,
+          timeout: 10_000,
+        });
         return {
-          request: http(rpc)({
+          request: custom({
+            request(body) {
+              if (requestId) {
+                return httpRpcClient.request({
+                  body,
+                  fetchOptions: { headers: { "X-Request-ID": requestId } },
+                });
+              }
+              return httpRpcClient.request({ body });
+            },
+          })({
             chain: chain.viemChain,
             retryCount: 0,
-            timeout: 10_000,
           }).request,
           hostname,
         };
@@ -433,7 +464,7 @@ export const createRpc = ({
         clearTimeout(t);
         const getBucketDuration = endClock();
         endClock = startClock();
-        const id = crypto.randomUUID().slice(0, 8);
+        const id = crypto.randomUUID();
 
         const surpassTimeout = setTimeout(() => {
           logger.warn({
@@ -469,6 +500,7 @@ export const createRpc = ({
             bucket.rps[bucket.rps.length - 1]!.count++;
           }
 
+          requestId = id;
           const response = await bucket.request(body);
 
           if (response === undefined) {