ponder 0.9.2 → 0.9.3

package/src/indexing-store/historical.ts

@@ -13,7 +13,6 @@ import {
   UniqueConstraintError,
 } from "@/internal/errors.js";
 import type { SchemaBuild } from "@/internal/types.js";
-import { encodeCheckpoint, zeroCheckpoint } from "@/utils/checkpoint.js";
 import { prettyPrint } from "@/utils/print.js";
 import { createQueue } from "@ponder/common";
 import {
@@ -146,12 +145,12 @@ export const createHistoricalIndexingStore = ({
   common,
   schemaBuild: { schema },
   database,
-  initialCheckpoint,
+  isDatabaseEmpty,
 }: {
   common: Common;
   schemaBuild: Pick<SchemaBuild, "schema">;
   database: Database;
-  initialCheckpoint: string;
+  isDatabaseEmpty: boolean;
 }): IndexingStore<"historical"> => {
   // Operation queue to make sure all queries are run in order, circumventing race conditions
   const queue = createQueue<unknown, () => Promise<unknown>>({
@@ -302,7 +301,6 @@ export const createHistoricalIndexingStore = ({
     return size;
   };
 
-  let isDatabaseEmpty = initialCheckpoint === encodeCheckpoint(zeroCheckpoint);
   /** Estimated number of bytes used by cache. */
   let cacheBytes = 0;
   /** LRU counter. */

package/src/internal/common.ts

@@ -1,6 +1,7 @@
 import type { Logger } from "./logger.js";
 import type { MetricsService } from "./metrics.js";
 import type { Options } from "./options.js";
+import type { Shutdown } from "./shutdown.js";
 import type { Telemetry } from "./telemetry.js";
 
 export type Common = {
@@ -8,4 +9,5 @@ export type Common = {
   logger: Logger;
   metrics: MetricsService;
   telemetry: Telemetry;
+  shutdown: Shutdown;
 };

package/src/internal/errors.ts

@@ -35,15 +35,6 @@ export class NonRetryableError extends BaseError {
   }
 }
 
-export class IgnorableError extends BaseError {
-  override name = "IgnorableError";
-
-  constructor(message?: string | undefined) {
-    super(message);
-    Object.setPrototypeOf(this, IgnorableError.prototype);
-  }
-}
-
 // Indexing store errors
 
 export class StoreError extends NonRetryableError {
@@ -126,3 +117,12 @@ export class FlushError extends NonRetryableError {
     Object.setPrototypeOf(this, FlushError.prototype);
   }
 }
+
+export class ShutdownError extends NonRetryableError {
+  override name = "ShutdownError";
+
+  constructor(message?: string | undefined) {
+    super(message);
+    Object.setPrototypeOf(this, ShutdownError.prototype);
+  }
+}

package/src/internal/logger.ts

@@ -72,7 +72,7 @@ export function createLogger({
     trace(options: Omit<Log, "level" | "time">) {
       logger.trace(options);
     },
-    async kill() {},
+    flush: () => new Promise((resolve) => logger.flush(resolve)),
   };
 }
 

package/src/internal/metrics.ts

@@ -17,14 +17,17 @@ const httpRequestSizeBytes = [
 
 export class MetricsService {
   registry: prometheus.Registry;
+  start_timestamp: number;
+  rps: { [network: string]: { count: number; timestamp: number }[] };
 
-  ponder_indexing_total_seconds: prometheus.Gauge;
-  ponder_indexing_completed_seconds: prometheus.Gauge;
-  ponder_indexing_completed_events: prometheus.Gauge<"event">;
+  ponder_historical_total_indexing_seconds: prometheus.Gauge<"network">;
+  ponder_historical_cached_indexing_seconds: prometheus.Gauge<"network">;
+  ponder_historical_completed_indexing_seconds: prometheus.Gauge<"network">;
 
-  ponder_indexing_completed_timestamp: prometheus.Gauge;
-  ponder_indexing_has_error: prometheus.Gauge;
+  ponder_indexing_timestamp: prometheus.Gauge<"network">;
+  ponder_indexing_has_error: prometheus.Gauge<"network">;
 
+  ponder_indexing_completed_events: prometheus.Gauge<"event">;
   ponder_indexing_function_duration: prometheus.Histogram<"event">;
   ponder_indexing_abi_decoding_duration: prometheus.Histogram;
 
@@ -64,15 +67,25 @@ export class MetricsService {
 
   constructor() {
     this.registry = new prometheus.Registry();
+    this.start_timestamp = Date.now();
+    this.rps = {};
 
-    this.ponder_indexing_total_seconds = new prometheus.Gauge({
-      name: "ponder_indexing_total_seconds",
+    this.ponder_historical_total_indexing_seconds = new prometheus.Gauge({
+      name: "ponder_historical_total_indexing_seconds",
       help: "Total number of seconds that are required",
+      labelNames: ["network"] as const,
+      registers: [this.registry],
+    });
+    this.ponder_historical_cached_indexing_seconds = new prometheus.Gauge({
+      name: "ponder_historical_cached_indexing_seconds",
+      help: "Number of seconds that have been cached",
+      labelNames: ["network"] as const,
       registers: [this.registry],
     });
-    this.ponder_indexing_completed_seconds = new prometheus.Gauge({
-      name: "ponder_indexing_completed_seconds",
+    this.ponder_historical_completed_indexing_seconds = new prometheus.Gauge({
+      name: "ponder_historical_completed_indexing_seconds",
       help: "Number of seconds that have been completed",
+      labelNames: ["network"] as const,
       registers: [this.registry],
     });
     this.ponder_indexing_completed_events = new prometheus.Gauge({
@@ -81,9 +94,10 @@ export class MetricsService {
       labelNames: ["network", "event"] as const,
       registers: [this.registry],
     });
-    this.ponder_indexing_completed_timestamp = new prometheus.Gauge({
-      name: "ponder_indexing_completed_timestamp",
+    this.ponder_indexing_timestamp = new prometheus.Gauge({
+      name: "ponder_indexing_timestamp",
       help: "Timestamp through which all events have been completed",
+      labelNames: ["network"] as const,
       registers: [this.registry],
     });
     this.ponder_indexing_has_error = new prometheus.Gauge({
@@ -245,10 +259,14 @@ export class MetricsService {
   }
 
   resetIndexingMetrics() {
-    this.ponder_indexing_total_seconds.reset();
-    this.ponder_indexing_completed_seconds.reset();
+    this.start_timestamp = Date.now();
+    this.rps = {};
+
+    this.ponder_historical_total_indexing_seconds.reset();
+    this.ponder_historical_cached_indexing_seconds.reset();
+    this.ponder_historical_completed_indexing_seconds.reset();
     this.ponder_indexing_completed_events.reset();
-    this.ponder_indexing_completed_timestamp.reset();
+    this.ponder_indexing_timestamp.reset();
     this.ponder_indexing_has_error.reset();
     this.ponder_indexing_function_duration.reset();
     this.ponder_indexing_abi_decoding_duration.reset();
@@ -284,8 +302,6 @@ export class MetricsService {
   }
 }
 
-const rps: { [network: string]: { count: number; timestamp: number }[] } = {};
-
 export async function getSyncProgress(metrics: MetricsService): Promise<
   {
     networkName: string;
@@ -338,14 +354,14 @@ export async function getSyncProgress(metrics: MetricsService): Promise<
   }
 
   for (const [networkName, count] of Object.entries(requestCount)) {
-    if (rps[networkName] === undefined) {
-      rps[networkName] = [{ count, timestamp: Date.now() }];
+    if (metrics.rps[networkName] === undefined) {
+      metrics.rps[networkName] = [{ count, timestamp: Date.now() }];
     } else {
-      rps[networkName]!.push({ count, timestamp: Date.now() });
+      metrics.rps[networkName]!.push({ count, timestamp: Date.now() });
     }
 
-    if (rps[networkName]!.length > 100) {
-      rps[networkName]!.shift();
+    if (metrics.rps[networkName]!.length > 100) {
+      metrics.rps[networkName]!.shift();
     }
   }
 
@@ -365,9 +381,9 @@ export async function getSyncProgress(metrics: MetricsService): Promise<
     // The ETA is low quality if we've completed only one or two blocks.
     const eta = completedBlocks >= 3 ? total - elapsed : undefined;
 
-    const _length = rps[labels.network!]!.length;
-    const _firstRps = rps[labels.network!]![0]!;
-    const _lastRps = rps[labels.network!]![_length - 1]!;
+    const _length = metrics.rps[labels.network!]!.length;
+    const _firstRps = metrics.rps[labels.network!]![0]!;
+    const _lastRps = metrics.rps[labels.network!]![_length - 1]!;
 
     const requests = _lastRps.count - (_length > 1 ? _firstRps.count : 0);
     const seconds =
@@ -389,16 +405,33 @@ export async function getIndexingProgress(metrics: MetricsService) {
     .values[0]?.value;
   const hasError = hasErrorMetric === 1;
 
-  const totalSeconds =
-    (await metrics.ponder_indexing_total_seconds.get()).values[0]?.value ?? 0;
+  const sum = (x: number[]) => x.reduce((a, b) => a + b, 0);
+  const max = (x: number[]) => x.reduce((a, b) => Math.max(a, b), 0);
+
+  const totalSeconds = await metrics.ponder_historical_total_indexing_seconds
+    .get()
+    .then(({ values }) => values.map(({ value }) => value))
+    .then(sum);
+  const cachedSeconds = await metrics.ponder_historical_cached_indexing_seconds
+    .get()
+    .then(({ values }) => values.map(({ value }) => value))
+    .then(sum);
   const completedSeconds =
-    (await metrics.ponder_indexing_completed_seconds.get()).values[0]?.value ??
-    0;
-  const completedToTimestamp =
-    (await metrics.ponder_indexing_completed_timestamp.get()).values[0]!
-      .value ?? 0;
+    await metrics.ponder_historical_completed_indexing_seconds
+      .get()
+      .then(({ values }) => values.map(({ value }) => value))
+      .then(sum);
+  const timestamp = await metrics.ponder_indexing_timestamp
+    .get()
+    .then(({ values }) => values.map(({ value }) => value))
+    .then(max);
 
-  const progress = totalSeconds === 0 ? 0 : completedSeconds / totalSeconds;
+  const progress =
+    timestamp === 0
+      ? 0
+      : totalSeconds === 0
+        ? 1
+        : (completedSeconds + cachedSeconds) / totalSeconds;
 
   const indexingCompletedEventsMetric = (
     await metrics.ponder_indexing_completed_events.get()
@@ -433,10 +466,10 @@ export async function getIndexingProgress(metrics: MetricsService) {
   return {
     hasError,
     overall: {
-      completedSeconds,
       totalSeconds,
+      cachedSeconds,
+      completedSeconds,
       progress,
-      completedToTimestamp,
       totalEvents,
     },
     events,
@@ -444,86 +477,25 @@ export async function getIndexingProgress(metrics: MetricsService) {
 }
 
 export async function getAppProgress(metrics: MetricsService): Promise<{
-  mode: "historical" | "realtime" | "complete" | undefined;
+  mode: "historical" | "realtime" | undefined;
   progress: number;
   eta: number | undefined;
 }> {
-  const sync = await getSyncProgress(metrics);
   const indexing = await getIndexingProgress(metrics);
-  const decodingSum = await metrics.ponder_indexing_abi_decoding_duration
-    .get()
-    .then(
-      (m) =>
-        m.values.find(
-          (v) => v.metricName === "ponder_indexing_abi_decoding_duration_sum",
-        )?.value,
-    );
-  const getEventsSum = await metrics.ponder_database_method_duration
-    .get()
-    .then(
-      (m) =>
-        m.values.find(
-          (v) =>
-            v.labels.method === "getEvents" &&
-            v.metricName === "ponder_database_method_duration_sum",
-        )?.value,
-    );
-  const indexingSum = indexing.events.reduce(
-    (acc, cur) => acc + cur.averageDuration * cur.count,
-    0,
-  );
-
-  let maxSync: (typeof sync)[number] | undefined;
-  for (const networkSync of sync) {
-    if (
-      maxSync === undefined ||
-      maxSync.eta === undefined ||
-      (networkSync.eta && networkSync.eta > maxSync.eta)
-    ) {
-      maxSync = networkSync;
-    }
-  }
 
   const remainingSeconds =
-    indexing.overall.totalSeconds - indexing.overall.completedSeconds;
+    indexing.overall.totalSeconds -
+    (indexing.overall.completedSeconds + indexing.overall.cachedSeconds);
+  const elapsedSeconds = (Date.now() - metrics.start_timestamp) / 1_000;
 
-  const indexingEta =
+  const eta =
     indexing.overall.completedSeconds === 0
-      ? undefined
-      : (((decodingSum ?? 0) + (getEventsSum ?? 0) + indexingSum) *
-          remainingSeconds) /
-        indexing.overall.completedSeconds;
-
-  const eta = sync.every((n) => n.progress === 1)
-    ? indexingEta
-    : maxSync?.eta === undefined && indexingEta === undefined
-      ? undefined
-      : maxSync?.eta === undefined && maxSync?.progress !== undefined
-        ? undefined
-        : Math.max(maxSync?.eta ?? 0, indexingEta ?? 0);
-
-  // Edge case: If all matched events occurred in the same unix timestamp (second), progress will
-  // be zero, even though indexing is complete. When this happens, totalEvents will be non-zero.
-  const indexingProgress =
-    indexing.overall.progress === 0 && indexing.overall.totalEvents > 0
-      ? 1
-      : indexing.overall.progress;
-
-  const progress = sync.every((n) => n.progress === 1)
-    ? indexingProgress
-    : maxSync?.progress === undefined
       ? 0
-      : maxSync!.progress * indexingProgress;
+      : (elapsedSeconds / indexing.overall.completedSeconds) * remainingSeconds;
 
   return {
-    mode: sync.some((n) => n.status === "realtime")
-      ? "realtime"
-      : sync.every((n) => n.status === "complete")
-        ? "complete"
-        : sync.length === 0
-          ? undefined
-          : "historical",
-    progress,
+    mode: indexing.overall.progress === 1 ? "realtime" : "historical",
+    progress: indexing.overall.progress,
     eta,
   };
 }

package/src/internal/shutdown.ts

@@ -0,0 +1,25 @@
+export type Shutdown = {
+  add: (callback: () => unknown | Promise<unknown>) => void;
+  kill: () => Promise<void>;
+  isKilled: boolean;
+  abortController: AbortController;
+};
+
+export const createShutdown = (): Shutdown => {
+  const abortController = new AbortController();
+  const callbacks: (() => unknown | Promise<unknown>)[] = [];
+
+  return {
+    add: (callback) => {
+      callbacks.push(callback);
+    },
+    kill: async () => {
+      abortController.abort();
+      await Promise.all(callbacks.map((callback) => callback()));
+    },
+    get isKilled() {
+      return abortController.signal.aborted;
+    },
+    abortController,
+  };
+};

package/src/internal/telemetry.ts

@@ -6,11 +6,12 @@ import path from "node:path";
 import { promisify } from "node:util";
 import type { Options } from "@/internal/options.js";
 import { startClock } from "@/utils/timer.js";
-import { wait } from "@/utils/wait.js";
 import { createQueue } from "@ponder/common";
 import Conf from "conf";
 import { type PM, detect, getNpmVersion } from "detect-package-manager";
+import { ShutdownError } from "./errors.js";
 import type { Logger } from "./logger.js";
+import type { Shutdown } from "./shutdown.js";
 import type { IndexingBuild } from "./types.js";
 import type { PreBuild, SchemaBuild } from "./types.js";
 
@@ -65,12 +66,12 @@ export type Telemetry = ReturnType<typeof createTelemetry>;
 export function createTelemetry({
   options,
   logger,
-}: { options: Options; logger: Logger }) {
+  shutdown,
+}: { options: Options; logger: Logger; shutdown: Shutdown }) {
   if (options.telemetryDisabled) {
     return {
       record: (_event: TelemetryEvent) => {},
       flush: async () => {},
-      kill: async () => {},
     };
   }
 
@@ -156,13 +157,12 @@ export function createTelemetry({
   let context: Awaited<ReturnType<typeof buildContext>> | undefined = undefined;
   const contextPromise = buildContext();
 
-  const controller = new AbortController();
-  let isKilled = false;
-
   const queue = createQueue({
     initialStart: true,
     concurrency: 10,
     worker: async (event: TelemetryEvent) => {
+      if (shutdown.isKilled) return;
+
       const endClock = startClock();
       try {
         if (context === undefined) context = await contextPromise;
@@ -182,7 +182,6 @@ export function createTelemetry({
           method: "POST",
           headers: { "Content-Type": "application/json" },
           body,
-          signal: controller.signal,
         });
         logger.trace({
           service: "telemetry",
@@ -190,6 +189,11 @@ export function createTelemetry({
         });
       } catch (error_) {
         const error = error_ as Error;
+
+        if (shutdown.isKilled) {
+          throw new ShutdownError();
+        }
+
         logger.trace({
           service: "telemetry",
           msg: `Failed to send '${event.name}' event after ${endClock()}ms`,
@@ -200,7 +204,6 @@ export function createTelemetry({
   });
 
   const record = (event: TelemetryEvent) => {
-    if (isKilled) return;
     queue.add(event);
   };
 
@@ -211,21 +214,16 @@ export function createTelemetry({
     });
   }, HEARTBEAT_INTERVAL_MS);
 
+  shutdown.add(() => {
+    clearInterval(heartbeatInterval);
+  });
+
   // Note that this method is only used for testing.
   const flush = async () => {
     await queue.onIdle();
   };
 
-  const kill = async () => {
-    clearInterval(heartbeatInterval);
-    isKilled = true;
-    // If there are any events in the queue that have not started, drop them.
-    queue.clear();
-    // Wait at most 1 second for any in-flight events to complete.
-    await Promise.race([queue.onIdle(), wait(1_000)]);
-  };
-
-  return { record, flush, kill };
+  return { record, flush };
 }
 
 async function getPackageManager() {

package/src/internal/types.ts

@@ -284,8 +284,8 @@ export type BlockMetadata = {
 export type Network = {
   name: string;
   chainId: number;
-  chain: Chain;
   transport: ReturnType<Transport>;
+  chain: Chain;
   pollingInterval: number;
   maxRequestsPerSecond: number;
   finalityBlockCount: number;
@@ -306,6 +306,8 @@ export type NamespaceBuild = string;
 export type PreBuild = {
   /** Database type and configuration */
   databaseConfig: DatabaseConfig;
+  /** Ordering of events */
+  ordering: "omnichain" | "multichain";
 };
 
 export type SchemaBuild = {
@@ -344,6 +346,12 @@ export type Status = {
   };
 };
 
+// Seconds
+
+export type Seconds = {
+  [network: string]: { start: number; end: number; cached: number };
+};
+
 // Events
 
 export type RawEvent = {

package/src/server/index.ts

@@ -13,7 +13,6 @@ import { onError } from "./error.js";
 
 export type Server = {
   hono: Hono;
-  kill: () => Promise<void>;
 };
 
 export async function createServer({
@@ -145,8 +144,7 @@ export async function createServer({
     gracefulTerminationTimeout: 1000,
   });
 
-  return {
-    hono,
-    kill: () => terminator.terminate(),
-  };
+  common.shutdown.add(() => terminator.terminate());
+
+  return { hono };
 }

package/src/sync/events.ts

@@ -23,9 +23,9 @@ import type {
 } from "@/types/sync.js";
 import {
   EVENT_TYPES,
+  MAX_CHECKPOINT,
+  ZERO_CHECKPOINT,
   encodeCheckpoint,
-  maxCheckpoint,
-  zeroCheckpoint,
 } from "@/utils/checkpoint.js";
 import { never } from "@/utils/never.js";
 import { startClock } from "@/utils/timer.js";
@@ -329,9 +329,9 @@ export const buildEvents = ({
               blockTimestamp: hexToNumber(block.timestamp),
               chainId: BigInt(filter.chainId),
               blockNumber: hexToBigInt(block.number),
-              transactionIndex: maxCheckpoint.transactionIndex,
+              transactionIndex: MAX_CHECKPOINT.transactionIndex,
               eventType: EVENT_TYPES.blocks,
-              eventIndex: zeroCheckpoint.eventIndex,
+              eventIndex: ZERO_CHECKPOINT.eventIndex,
             }),
             block: convertBlock(block),
             log: undefined,

package/src/sync/filter.ts

@@ -484,6 +484,7 @@ export const defaultTransactionReceiptInclude: `transactionReceipt.${keyof Trans
 
 const defaultTraceInclude: `trace.${keyof UserTrace}`[] = [
   "trace.id",
+  "trace.traceIndex",
   "trace.type",
   "trace.from",
   "trace.to",