ponder 0.9.2 → 0.9.4-debug.1

package/src/indexing/service.ts

@@ -1,5 +1,6 @@
 import type { IndexingStore } from "@/indexing-store/index.js";
 import type { Common } from "@/internal/common.js";
+import { ShutdownError } from "@/internal/errors.js";
 import type {
   ContractSource,
   Event,
@@ -10,23 +11,25 @@ import type {
   SetupEvent,
   Source,
 } from "@/internal/types.js";
+import type { SyncStore } from "@/sync-store/index.js";
 import { isAddressFactory } from "@/sync/filter.js";
-import type { Sync } from "@/sync/index.js";
+import { cachedTransport } from "@/sync/transport.js";
 import type { Db } from "@/types/db.js";
 import type { Block, Log, Trace, Transaction } from "@/types/eth.js";
 import type { DeepPartial } from "@/types/utils.js";
 import {
-  type Checkpoint,
+  ZERO_CHECKPOINT,
   decodeCheckpoint,
   encodeCheckpoint,
-  zeroCheckpoint,
 } from "@/utils/checkpoint.js";
 import { prettyPrint } from "@/utils/print.js";
+import type { RequestQueue } from "@/utils/requestQueue.js";
 import { startClock } from "@/utils/timer.js";
-import type { Abi, Address } from "viem";
-import { checksumAddress, createClient } from "viem";
+import { type Abi, type Address, createClient } from "viem";
+import { checksumAddress } from "viem";
 import { addStackTrace } from "./addStackTrace.js";
-import { type ReadOnlyClient, getPonderActions } from "./ponderActions.js";
+import type { ReadOnlyClient } from "./ponderActions.js";
+import { getPonderActions } from "./ponderActions.js";
 
 export type Context = {
   network: { chainId: number; name: string };
@@ -49,12 +52,9 @@ export type Service = {
   indexingFunctions: IndexingFunctions;
 
   // state
-  isKilled: boolean;
-
   eventCount: {
     [eventName: string]: number;
   };
-  startCheckpoint: Checkpoint;
 
   /**
    * Reduce memory usage by reserving space for objects ahead of time
@@ -76,14 +76,16 @@ export type Service = {
 export const create = ({
   common,
   indexingBuild: { sources, networks, indexingFunctions },
-  sync,
+  requestQueues,
+  syncStore,
 }: {
   common: Common;
   indexingBuild: Pick<
     IndexingBuild,
     "sources" | "networks" | "indexingFunctions"
   >;
-  sync: Sync;
+  requestQueues: RequestQueue[];
+  syncStore: SyncStore;
 }): Service => {
   const contextState: Service["currentEvent"]["contextState"] = {
     blockNumber: undefined!,
@@ -139,10 +141,12 @@ export const create = ({
   }
 
   // build clientByChainId
-  for (const network of networks) {
-    const transport = sync.getCachedTransport(network);
+  for (let i = 0; i < networks.length; i++) {
+    const network = networks[i]!;
+    const requestQueue = requestQueues[i]!;
+
     clientByChainId[network.chainId] = createClient({
-      transport,
+      transport: cachedTransport({ requestQueue, syncStore }),
       chain: network.chain,
       // @ts-ignore
     }).extend(getPonderActions(contextState));
@@ -157,9 +161,7 @@ export const create = ({
   return {
     common,
     indexingFunctions,
-    isKilled: false,
     eventCount,
-    startCheckpoint: decodeCheckpoint(sync.getStartCheckpoint()),
     currentEvent: {
       contextState,
       context: {
@@ -184,11 +186,7 @@ export const processSetupEvents = async (
     sources: Source[];
     networks: Network[];
   },
-): Promise<
-  | { status: "error"; error: Error }
-  | { status: "success" }
-  | { status: "killed" }
-> => {
+): Promise<{ status: "error"; error: Error } | { status: "success" }> => {
   for (const eventName of Object.keys(indexingService.indexingFunctions)) {
     if (!eventName.endsWith(":setup")) continue;
 
@@ -204,8 +202,6 @@ export const processSetupEvents = async (
 
       if (source === undefined) continue;
 
-      if (indexingService.isKilled) return { status: "killed" };
-
       indexingService.eventCount[eventName]!++;
 
       const result = await executeSetup(indexingService, {
@@ -213,7 +209,7 @@ export const processSetupEvents = async (
           type: "setup",
           chainId: network.chainId,
           checkpoint: encodeCheckpoint({
-            ...zeroCheckpoint,
+            ...ZERO_CHECKPOINT,
             chainId: BigInt(network.chainId),
             blockNumber: BigInt(source.filter.fromBlock ?? 0),
           }),
@@ -236,14 +232,8 @@ export const processSetupEvents = async (
 export const processEvents = async (
   indexingService: Service,
   { events }: { events: Event[] },
-): Promise<
-  | { status: "error"; error: Error }
-  | { status: "success" }
-  | { status: "killed" }
-> => {
+): Promise<{ status: "error"; error: Error } | { status: "success" }> => {
   for (let i = 0; i < events.length; i++) {
-    if (indexingService.isKilled) return { status: "killed" };
-
     const event = events[i]!;
 
     indexingService.eventCount[event.name]!++;
@@ -262,40 +252,8 @@ export const processEvents = async (
       service: "indexing",
       msg: `Completed indexing function (event="${event.name}", checkpoint=${event.checkpoint})`,
     });
-
-    // periodically update metrics
-    if (i % 93 === 0) {
-      updateCompletedEvents(indexingService);
-
-      const eventTimestamp = decodeCheckpoint(event.checkpoint).blockTimestamp;
-
-      indexingService.common.metrics.ponder_indexing_completed_seconds.set(
-        eventTimestamp - indexingService.startCheckpoint.blockTimestamp,
-      );
-      indexingService.common.metrics.ponder_indexing_completed_timestamp.set(
-        eventTimestamp,
-      );
-
-      // Note: allows for terminal and logs to be updated
-      await new Promise(setImmediate);
-    }
   }
 
-  // set completed seconds
-  if (events.length > 0) {
-    const lastEventInBatchTimestamp = decodeCheckpoint(
-      events[events.length - 1]!.checkpoint,
-    ).blockTimestamp;
-
-    indexingService.common.metrics.ponder_indexing_completed_seconds.set(
-      lastEventInBatchTimestamp -
-        indexingService.startCheckpoint.blockTimestamp,
-    );
-    indexingService.common.metrics.ponder_indexing_completed_timestamp.set(
-      lastEventInBatchTimestamp,
-    );
-  }
-  // set completed events
   updateCompletedEvents(indexingService);
 
   return { status: "success" };
@@ -314,24 +272,6 @@ export const setIndexingStore = (
   };
 };
 
-export const kill = (indexingService: Service) => {
-  indexingService.common.logger.debug({
-    service: "indexing",
-    msg: "Killed indexing service",
-  });
-  indexingService.isKilled = true;
-};
-
-export const updateTotalSeconds = (
-  indexingService: Service,
-  endCheckpoint: Checkpoint,
-) => {
-  indexingService.common.metrics.ponder_indexing_total_seconds.set(
-    endCheckpoint.blockTimestamp -
-      indexingService.startCheckpoint.blockTimestamp,
-  );
-};
-
 const updateCompletedEvents = (indexingService: Service) => {
   for (const event of Object.keys(indexingService.eventCount)) {
     const metricLabel = {
@@ -347,11 +287,7 @@ const updateCompletedEvents = (indexingService: Service) => {
 const executeSetup = async (
   indexingService: Service,
   { event }: { event: SetupEvent },
-): Promise<
-  | { status: "error"; error: Error }
-  | { status: "success" }
-  | { status: "killed" }
-> => {
+): Promise<{ status: "error"; error: Error } | { status: "success" }> => {
   const {
     common,
     indexingFunctions,
@@ -382,9 +318,12 @@ const executeSetup = async (
       endClock(),
     );
   } catch (_error) {
-    if (indexingService.isKilled) return { status: "killed" };
     const error = _error instanceof Error ? _error : new Error(String(_error));
 
+    if (common.shutdown.isKilled) {
+      throw new ShutdownError();
+    }
+
     addStackTrace(error, common.options);
     addErrorMeta(error, toErrorMeta(event));
 
@@ -406,11 +345,7 @@ const executeSetup = async (
 const executeEvent = async (
   indexingService: Service,
   { event }: { event: Event },
-): Promise<
-  | { status: "error"; error: Error }
-  | { status: "success" }
-  | { status: "killed" }
-> => {
+): Promise<{ status: "error"; error: Error } | { status: "success" }> => {
   const {
     common,
     indexingFunctions,
@@ -442,9 +377,12 @@ const executeEvent = async (
       endClock(),
     );
   } catch (_error) {
-    if (indexingService.isKilled) return { status: "killed" };
     const error = _error instanceof Error ? _error : new Error(String(_error));
 
+    if (common.shutdown.isKilled) {
+      throw new ShutdownError();
+    }
+
     addStackTrace(error, common.options);
     addErrorMeta(error, toErrorMeta(event));
 

package/src/indexing-store/historical.ts

@@ -13,7 +13,6 @@ import {
   UniqueConstraintError,
 } from "@/internal/errors.js";
 import type { SchemaBuild } from "@/internal/types.js";
-import { encodeCheckpoint, zeroCheckpoint } from "@/utils/checkpoint.js";
 import { prettyPrint } from "@/utils/print.js";
 import { createQueue } from "@ponder/common";
 import {
@@ -146,12 +145,12 @@ export const createHistoricalIndexingStore = ({
   common,
   schemaBuild: { schema },
   database,
-  initialCheckpoint,
+  isDatabaseEmpty,
 }: {
   common: Common;
   schemaBuild: Pick<SchemaBuild, "schema">;
   database: Database;
-  initialCheckpoint: string;
+  isDatabaseEmpty: boolean;
 }): IndexingStore<"historical"> => {
   // Operation queue to make sure all queries are run in order, circumventing race conditions
   const queue = createQueue<unknown, () => Promise<unknown>>({
@@ -302,7 +301,6 @@ export const createHistoricalIndexingStore = ({
     return size;
   };
 
-  let isDatabaseEmpty = initialCheckpoint === encodeCheckpoint(zeroCheckpoint);
   /** Estimated number of bytes used by cache. */
   let cacheBytes = 0;
   /** LRU counter. */

package/src/internal/common.ts

@@ -1,6 +1,7 @@
 import type { Logger } from "./logger.js";
 import type { MetricsService } from "./metrics.js";
 import type { Options } from "./options.js";
+import type { Shutdown } from "./shutdown.js";
 import type { Telemetry } from "./telemetry.js";
 
 export type Common = {
@@ -8,4 +9,5 @@ export type Common = {
   logger: Logger;
   metrics: MetricsService;
   telemetry: Telemetry;
+  shutdown: Shutdown;
 };

package/src/internal/errors.ts

@@ -35,15 +35,6 @@ export class NonRetryableError extends BaseError {
   }
 }
 
-export class IgnorableError extends BaseError {
-  override name = "IgnorableError";
-
-  constructor(message?: string | undefined) {
-    super(message);
-    Object.setPrototypeOf(this, IgnorableError.prototype);
-  }
-}
-
 // Indexing store errors
 
 export class StoreError extends NonRetryableError {
@@ -126,3 +117,12 @@ export class FlushError extends NonRetryableError {
     Object.setPrototypeOf(this, FlushError.prototype);
   }
 }
+
+export class ShutdownError extends NonRetryableError {
+  override name = "ShutdownError";
+
+  constructor(message?: string | undefined) {
+    super(message);
+    Object.setPrototypeOf(this, ShutdownError.prototype);
+  }
+}

package/src/internal/logger.ts

@@ -72,7 +72,7 @@ export function createLogger({
     trace(options: Omit<Log, "level" | "time">) {
       logger.trace(options);
     },
-    async kill() {},
+    flush: () => new Promise((resolve) => logger.flush(resolve)),
   };
 }
 

package/src/internal/metrics.ts

@@ -17,14 +17,17 @@ const httpRequestSizeBytes = [
 
 export class MetricsService {
   registry: prometheus.Registry;
+  start_timestamp: number;
+  rps: { [network: string]: { count: number; timestamp: number }[] };
 
-  ponder_indexing_total_seconds: prometheus.Gauge;
-  ponder_indexing_completed_seconds: prometheus.Gauge;
-  ponder_indexing_completed_events: prometheus.Gauge<"event">;
+  ponder_historical_total_indexing_seconds: prometheus.Gauge<"network">;
+  ponder_historical_cached_indexing_seconds: prometheus.Gauge<"network">;
+  ponder_historical_completed_indexing_seconds: prometheus.Gauge<"network">;
 
-  ponder_indexing_completed_timestamp: prometheus.Gauge;
-  ponder_indexing_has_error: prometheus.Gauge;
+  ponder_indexing_timestamp: prometheus.Gauge<"network">;
+  ponder_indexing_has_error: prometheus.Gauge<"network">;
 
+  ponder_indexing_completed_events: prometheus.Gauge<"event">;
   ponder_indexing_function_duration: prometheus.Histogram<"event">;
   ponder_indexing_abi_decoding_duration: prometheus.Histogram;
 
@@ -64,15 +67,25 @@ export class MetricsService {
 
   constructor() {
     this.registry = new prometheus.Registry();
+    this.start_timestamp = Date.now();
+    this.rps = {};
 
-    this.ponder_indexing_total_seconds = new prometheus.Gauge({
-      name: "ponder_indexing_total_seconds",
+    this.ponder_historical_total_indexing_seconds = new prometheus.Gauge({
+      name: "ponder_historical_total_indexing_seconds",
       help: "Total number of seconds that are required",
+      labelNames: ["network"] as const,
+      registers: [this.registry],
+    });
+    this.ponder_historical_cached_indexing_seconds = new prometheus.Gauge({
+      name: "ponder_historical_cached_indexing_seconds",
+      help: "Number of seconds that have been cached",
+      labelNames: ["network"] as const,
       registers: [this.registry],
     });
-    this.ponder_indexing_completed_seconds = new prometheus.Gauge({
-      name: "ponder_indexing_completed_seconds",
+    this.ponder_historical_completed_indexing_seconds = new prometheus.Gauge({
+      name: "ponder_historical_completed_indexing_seconds",
       help: "Number of seconds that have been completed",
+      labelNames: ["network"] as const,
       registers: [this.registry],
     });
     this.ponder_indexing_completed_events = new prometheus.Gauge({
@@ -81,9 +94,10 @@ export class MetricsService {
       labelNames: ["network", "event"] as const,
       registers: [this.registry],
     });
-    this.ponder_indexing_completed_timestamp = new prometheus.Gauge({
-      name: "ponder_indexing_completed_timestamp",
+    this.ponder_indexing_timestamp = new prometheus.Gauge({
+      name: "ponder_indexing_timestamp",
       help: "Timestamp through which all events have been completed",
+      labelNames: ["network"] as const,
       registers: [this.registry],
     });
     this.ponder_indexing_has_error = new prometheus.Gauge({
@@ -245,10 +259,14 @@ export class MetricsService {
   }
 
   resetIndexingMetrics() {
-    this.ponder_indexing_total_seconds.reset();
-    this.ponder_indexing_completed_seconds.reset();
+    this.start_timestamp = Date.now();
+    this.rps = {};
+
+    this.ponder_historical_total_indexing_seconds.reset();
+    this.ponder_historical_cached_indexing_seconds.reset();
+    this.ponder_historical_completed_indexing_seconds.reset();
     this.ponder_indexing_completed_events.reset();
-    this.ponder_indexing_completed_timestamp.reset();
+    this.ponder_indexing_timestamp.reset();
     this.ponder_indexing_has_error.reset();
     this.ponder_indexing_function_duration.reset();
     this.ponder_indexing_abi_decoding_duration.reset();
@@ -284,8 +302,6 @@ export class MetricsService {
   }
 }
 
-const rps: { [network: string]: { count: number; timestamp: number }[] } = {};
-
 export async function getSyncProgress(metrics: MetricsService): Promise<
   {
     networkName: string;
@@ -338,14 +354,14 @@ export async function getSyncProgress(metrics: MetricsService): Promise<
   }
 
   for (const [networkName, count] of Object.entries(requestCount)) {
-    if (rps[networkName] === undefined) {
-      rps[networkName] = [{ count, timestamp: Date.now() }];
+    if (metrics.rps[networkName] === undefined) {
+      metrics.rps[networkName] = [{ count, timestamp: Date.now() }];
     } else {
-      rps[networkName]!.push({ count, timestamp: Date.now() });
+      metrics.rps[networkName]!.push({ count, timestamp: Date.now() });
     }
 
-    if (rps[networkName]!.length > 100) {
-      rps[networkName]!.shift();
+    if (metrics.rps[networkName]!.length > 100) {
+      metrics.rps[networkName]!.shift();
     }
   }
 
@@ -365,9 +381,9 @@ export async function getSyncProgress(metrics: MetricsService): Promise<
     // The ETA is low quality if we've completed only one or two blocks.
     const eta = completedBlocks >= 3 ? total - elapsed : undefined;
 
-    const _length = rps[labels.network!]!.length;
-    const _firstRps = rps[labels.network!]![0]!;
-    const _lastRps = rps[labels.network!]![_length - 1]!;
+    const _length = metrics.rps[labels.network!]!.length;
+    const _firstRps = metrics.rps[labels.network!]![0]!;
+    const _lastRps = metrics.rps[labels.network!]![_length - 1]!;
 
     const requests = _lastRps.count - (_length > 1 ? _firstRps.count : 0);
     const seconds =
@@ -389,16 +405,33 @@ export async function getIndexingProgress(metrics: MetricsService) {
     .values[0]?.value;
   const hasError = hasErrorMetric === 1;
 
-  const totalSeconds =
-    (await metrics.ponder_indexing_total_seconds.get()).values[0]?.value ?? 0;
+  const sum = (x: number[]) => x.reduce((a, b) => a + b, 0);
+  const max = (x: number[]) => x.reduce((a, b) => Math.max(a, b), 0);
+
+  const totalSeconds = await metrics.ponder_historical_total_indexing_seconds
+    .get()
+    .then(({ values }) => values.map(({ value }) => value))
+    .then(sum);
+  const cachedSeconds = await metrics.ponder_historical_cached_indexing_seconds
+    .get()
+    .then(({ values }) => values.map(({ value }) => value))
+    .then(sum);
   const completedSeconds =
-    (await metrics.ponder_indexing_completed_seconds.get()).values[0]?.value ??
-    0;
-  const completedToTimestamp =
-    (await metrics.ponder_indexing_completed_timestamp.get()).values[0]!
-      .value ?? 0;
+    await metrics.ponder_historical_completed_indexing_seconds
+      .get()
+      .then(({ values }) => values.map(({ value }) => value))
+      .then(sum);
+  const timestamp = await metrics.ponder_indexing_timestamp
+    .get()
+    .then(({ values }) => values.map(({ value }) => value))
+    .then(max);
 
-  const progress = totalSeconds === 0 ? 0 : completedSeconds / totalSeconds;
+  const progress =
+    timestamp === 0
+      ? 0
+      : totalSeconds === 0
+        ? 1
+        : (completedSeconds + cachedSeconds) / totalSeconds;
 
   const indexingCompletedEventsMetric = (
     await metrics.ponder_indexing_completed_events.get()
@@ -433,10 +466,10 @@ export async function getIndexingProgress(metrics: MetricsService) {
   return {
     hasError,
     overall: {
-      completedSeconds,
       totalSeconds,
+      cachedSeconds,
+      completedSeconds,
       progress,
-      completedToTimestamp,
       totalEvents,
     },
     events,
@@ -444,86 +477,25 @@ export async function getIndexingProgress(metrics: MetricsService) {
 }
 
 export async function getAppProgress(metrics: MetricsService): Promise<{
-  mode: "historical" | "realtime" | "complete" | undefined;
+  mode: "historical" | "realtime" | undefined;
   progress: number;
   eta: number | undefined;
 }> {
-  const sync = await getSyncProgress(metrics);
   const indexing = await getIndexingProgress(metrics);
-  const decodingSum = await metrics.ponder_indexing_abi_decoding_duration
-    .get()
-    .then(
-      (m) =>
-        m.values.find(
-          (v) => v.metricName === "ponder_indexing_abi_decoding_duration_sum",
-        )?.value,
-    );
-  const getEventsSum = await metrics.ponder_database_method_duration
-    .get()
-    .then(
-      (m) =>
-        m.values.find(
-          (v) =>
-            v.labels.method === "getEvents" &&
-            v.metricName === "ponder_database_method_duration_sum",
-        )?.value,
-    );
-  const indexingSum = indexing.events.reduce(
-    (acc, cur) => acc + cur.averageDuration * cur.count,
-    0,
-  );
-
-  let maxSync: (typeof sync)[number] | undefined;
-  for (const networkSync of sync) {
-    if (
-      maxSync === undefined ||
-      maxSync.eta === undefined ||
-      (networkSync.eta && networkSync.eta > maxSync.eta)
-    ) {
-      maxSync = networkSync;
-    }
-  }
 
   const remainingSeconds =
-    indexing.overall.totalSeconds - indexing.overall.completedSeconds;
+    indexing.overall.totalSeconds -
+    (indexing.overall.completedSeconds + indexing.overall.cachedSeconds);
+  const elapsedSeconds = (Date.now() - metrics.start_timestamp) / 1_000;
 
-  const indexingEta =
+  const eta =
     indexing.overall.completedSeconds === 0
-      ? undefined
-      : (((decodingSum ?? 0) + (getEventsSum ?? 0) + indexingSum) *
-          remainingSeconds) /
-        indexing.overall.completedSeconds;
-
-  const eta = sync.every((n) => n.progress === 1)
-    ? indexingEta
-    : maxSync?.eta === undefined && indexingEta === undefined
-      ? undefined
-      : maxSync?.eta === undefined && maxSync?.progress !== undefined
-        ? undefined
-        : Math.max(maxSync?.eta ?? 0, indexingEta ?? 0);
-
-  // Edge case: If all matched events occurred in the same unix timestamp (second), progress will
-  // be zero, even though indexing is complete. When this happens, totalEvents will be non-zero.
-  const indexingProgress =
-    indexing.overall.progress === 0 && indexing.overall.totalEvents > 0
-      ? 1
-      : indexing.overall.progress;
-
-  const progress = sync.every((n) => n.progress === 1)
-    ? indexingProgress
-    : maxSync?.progress === undefined
       ? 0
-      : maxSync!.progress * indexingProgress;
+      : (elapsedSeconds / indexing.overall.completedSeconds) * remainingSeconds;
 
   return {
-    mode: sync.some((n) => n.status === "realtime")
-      ? "realtime"
-      : sync.every((n) => n.status === "complete")
-        ? "complete"
-        : sync.length === 0
-          ? undefined
-          : "historical",
-    progress,
+    mode: indexing.overall.progress === 1 ? "realtime" : "historical",
+    progress: indexing.overall.progress,
     eta,
   };
 }

package/src/internal/shutdown.ts

@@ -0,0 +1,25 @@
+export type Shutdown = {
+  add: (callback: () => unknown | Promise<unknown>) => void;
+  kill: () => Promise<void>;
+  isKilled: boolean;
+  abortController: AbortController;
+};
+
+export const createShutdown = (): Shutdown => {
+  const abortController = new AbortController();
+  const callbacks: (() => unknown | Promise<unknown>)[] = [];
+
+  return {
+    add: (callback) => {
+      callbacks.push(callback);
+    },
+    kill: async () => {
+      abortController.abort();
+      await Promise.all(callbacks.map((callback) => callback()));
+    },
+    get isKilled() {
+      return abortController.signal.aborted;
+    },
+    abortController,
+  };
+};