ponder 0.14.13 → 0.15.0

package/src/internal/metrics.ts

@@ -1,7 +1,12 @@
+import { type Worker, parentPort } from "node:worker_threads";
+import {
+  type PromiseWithResolvers,
+  promiseWithResolvers,
+} from "@/utils/promiseWithResolvers.js";
 import { truncate } from "@/utils/truncate.js";
 import { getTableName, isTable } from "drizzle-orm";
 import prometheus from "prom-client";
-import type { IndexingBuild, SchemaBuild } from "./types.js";
+import type { IndexingBuild, PreBuild, SchemaBuild } from "./types.js";
 
 const sometimesIODurationMs = [
   0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 5, 10, 50, 100, 500, 1_000, 5_000,
@@ -17,6 +22,9 @@ const httpRequestSizeBytes = [
   10_000_000,
 ];
 
+const GET_METRICS_REQ = "prom-client:getMetricsReq";
+const GET_METRICS_RES = "prom-client:getMetricsRes";
+
 export class MetricsService {
   registry: prometheus.Registry;
   start_timestamp: number;
@@ -124,12 +132,14 @@ export class MetricsService {
       help: "Ponder version information",
       labelNames: ["version", "major", "minor", "patch"] as const,
       registers: [this.registry],
+      aggregator: "first",
     });
     this.ponder_settings_info = new prometheus.Gauge({
       name: "ponder_settings_info",
       help: "Ponder settings information",
       labelNames: ["ordering", "database", "command"] as const,
       registers: [this.registry],
+      aggregator: "first",
     });
 
     this.ponder_historical_concurrency_group_duration = new prometheus.Gauge({
@@ -137,18 +147,21 @@ export class MetricsService {
       help: "Duration of historical concurrency groups",
       labelNames: ["group"] as const,
       registers: [this.registry],
+      aggregator: "sum",
     });
     this.ponder_historical_extract_duration = new prometheus.Gauge({
       name: "ponder_historical_extract_duration",
       help: "Duration of historical extract phase",
       labelNames: ["step"] as const,
       registers: [this.registry],
+      aggregator: "sum",
     });
     this.ponder_historical_transform_duration = new prometheus.Gauge({
       name: "ponder_historical_transform_duration",
       help: "Duration of historical transform phase",
       labelNames: ["step"] as const,
       registers: [this.registry],
+      aggregator: "sum",
     });
 
     this.ponder_historical_start_timestamp_seconds = new prometheus.Gauge({
@@ -156,12 +169,14 @@ export class MetricsService {
       help: "Timestamp at which historical indexing started",
       labelNames: ["chain"] as const,
       registers: [this.registry],
+      aggregator: "min",
     });
     this.ponder_historical_end_timestamp_seconds = new prometheus.Gauge({
       name: "ponder_historical_end_timestamp_seconds",
       help: "Timestamp at which historical indexing ended",
       labelNames: ["chain"] as const,
       registers: [this.registry],
+      aggregator: "max",
     });
 
     this.ponder_historical_total_indexing_seconds = new prometheus.Gauge({
@@ -169,30 +184,35 @@ export class MetricsService {
       help: "Total number of seconds that are required",
       labelNames: ["chain"] as const,
       registers: [this.registry],
+      aggregator: "sum",
     });
     this.ponder_historical_cached_indexing_seconds = new prometheus.Gauge({
       name: "ponder_historical_cached_indexing_seconds",
       help: "Number of seconds that have been cached",
       labelNames: ["chain"] as const,
       registers: [this.registry],
+      aggregator: "sum",
     });
     this.ponder_historical_completed_indexing_seconds = new prometheus.Gauge({
       name: "ponder_historical_completed_indexing_seconds",
       help: "Number of seconds that have been completed",
       labelNames: ["chain"] as const,
       registers: [this.registry],
+      aggregator: "sum",
     });
     this.ponder_indexing_completed_events = new prometheus.Gauge({
       name: "ponder_indexing_completed_events",
       help: "Number of events that have been processed",
       labelNames: ["chain", "event"] as const,
       registers: [this.registry],
+      aggregator: "sum",
     });
     this.ponder_indexing_timestamp = new prometheus.Gauge({
       name: "ponder_indexing_timestamp",
       help: "Timestamp through which all events have been completed",
       labelNames: ["chain"] as const,
       registers: [this.registry],
+      aggregator: "first",
     });
     this.ponder_indexing_function_duration = new prometheus.Histogram({
       name: "ponder_indexing_function_duration",
@@ -200,6 +220,7 @@ export class MetricsService {
       labelNames: ["chain", "event"] as const,
       buckets: sometimesIODurationMs,
       registers: [this.registry],
+      aggregator: "sum",
     });
     this.ponder_indexing_cache_query_duration = new prometheus.Histogram({
       name: "ponder_indexing_cache_query_duration",
@@ -207,6 +228,7 @@ export class MetricsService {
       labelNames: ["table", "method"] as const,
       buckets: alwaysIODurationMs,
       registers: [this.registry],
+      aggregator: "sum",
     });
     this.ponder_indexing_rpc_action_duration = new prometheus.Histogram({
       name: "ponder_indexing_rpc_action_duration",
@@ -214,36 +236,42 @@ export class MetricsService {
       labelNames: ["action"] as const,
       buckets: sometimesIODurationMs,
       registers: [this.registry],
+      aggregator: "sum",
     });
     this.ponder_indexing_rpc_prefetch_total = new prometheus.Counter({
       name: "ponder_indexing_rpc_prefetch_total",
       help: "Number of RPC prefetches",
       labelNames: ["chain", "method", "type"] as const,
       registers: [this.registry],
+      aggregator: "sum",
     });
     this.ponder_indexing_rpc_requests_total = new prometheus.Counter({
       name: "ponder_indexing_rpc_requests_total",
       help: "Number of RPC requests",
       labelNames: ["chain", "method", "type"] as const,
       registers: [this.registry],
+      aggregator: "sum",
     });
     this.ponder_indexing_cache_requests_total = new prometheus.Counter({
       name: "ponder_indexing_cache_requests_total",
       help: "Number of cache accesses",
       labelNames: ["table", "type"] as const,
       registers: [this.registry],
+      aggregator: "sum",
     });
     this.ponder_indexing_store_queries_total = new prometheus.Counter({
       name: "ponder_indexing_store_queries_total",
       help: "Number of indexing store operations",
       labelNames: ["table", "method"] as const,
       registers: [this.registry],
+      aggregator: "sum",
     });
     this.ponder_indexing_store_raw_sql_duration = new prometheus.Histogram({
       name: "ponder_indexing_store_raw_sql_duration",
       help: "Duration of raw SQL store operations",
       buckets: alwaysIODurationMs,
       registers: [this.registry],
+      aggregator: "sum",
     });
 
     this.ponder_sync_block = new prometheus.Gauge({
@@ -251,24 +279,28 @@ export class MetricsService {
       help: "Closest-to-tip synced block number",
       labelNames: ["chain"] as const,
       registers: [this.registry],
+      aggregator: "max",
     });
     this.ponder_sync_block_timestamp = new prometheus.Gauge({
       name: "ponder_sync_block_timestamp",
       help: "Closest-to-tip synced block timestamp",
       labelNames: ["chain"] as const,
       registers: [this.registry],
+      aggregator: "max",
     });
     this.ponder_sync_is_realtime = new prometheus.Gauge({
       name: "ponder_sync_is_realtime",
       help: "Boolean (0 or 1) indicating if the sync is realtime mode",
       labelNames: ["chain"] as const,
       registers: [this.registry],
+      aggregator: "max",
     });
     this.ponder_sync_is_complete = new prometheus.Gauge({
       name: "ponder_sync_is_complete",
       help: "Boolean (0 or 1) indicating if the sync has synced all blocks",
       labelNames: ["chain"] as const,
       registers: [this.registry],
+      aggregator: "max",
     });
 
     this.ponder_historical_total_blocks = new prometheus.Gauge({
@@ -276,18 +308,21 @@ export class MetricsService {
       help: "Number of blocks required for the historical sync",
       labelNames: ["chain"] as const,
       registers: [this.registry],
+      aggregator: "max",
     });
     this.ponder_historical_cached_blocks = new prometheus.Gauge({
       name: "ponder_historical_cached_blocks",
       help: "Number of blocks that were found in the cache for the historical sync",
       labelNames: ["chain"] as const,
       registers: [this.registry],
+      aggregator: "max",
     });
     this.ponder_historical_completed_blocks = new prometheus.Gauge({
       name: "ponder_historical_completed_blocks",
       help: "Number of blocks that have been processed for the historical sync",
       labelNames: ["chain", "source", "type"] as const,
       registers: [this.registry],
+      aggregator: "max",
     });
 
     this.ponder_realtime_reorg_total = new prometheus.Counter({
@@ -295,6 +330,7 @@ export class MetricsService {
       help: "Count of how many re-orgs have occurred",
       labelNames: ["chain"] as const,
       registers: [this.registry],
+      aggregator: "sum",
     });
     this.ponder_realtime_latency = new prometheus.Histogram({
       name: "ponder_realtime_latency",
@@ -305,6 +341,7 @@ export class MetricsService {
         1_000_000,
       ],
       registers: [this.registry],
+      aggregator: "sum",
     });
     this.ponder_realtime_block_arrival_latency = new prometheus.Histogram({
       name: "ponder_realtime_block_arrival_latency",
@@ -315,6 +352,7 @@ export class MetricsService {
         1_000_000,
       ],
       registers: [this.registry],
+      aggregator: "sum",
     });
 
     this.ponder_database_method_duration = new prometheus.Histogram({
@@ -323,18 +361,21 @@ export class MetricsService {
       labelNames: ["service", "method"] as const,
       buckets: alwaysIODurationMs,
       registers: [this.registry],
+      aggregator: "sum",
     });
     this.ponder_database_method_error_total = new prometheus.Counter({
       name: "ponder_database_method_error_total",
       help: "Total number of errors encountered during database operations",
       labelNames: ["service", "method"] as const,
       registers: [this.registry],
+      aggregator: "sum",
     });
     this.ponder_http_server_active_requests = new prometheus.Gauge({
       name: "ponder_http_server_active_requests",
       help: "Number of active HTTP server requests",
       labelNames: ["method", "path"] as const,
       registers: [this.registry],
+      aggregator: "sum",
     });
     this.ponder_http_server_request_duration_ms = new prometheus.Histogram({
       name: "ponder_http_server_request_duration_ms",
@@ -342,6 +383,7 @@ export class MetricsService {
       labelNames: ["method", "path", "status"] as const,
       buckets: alwaysIODurationMs,
       registers: [this.registry],
+      aggregator: "sum",
     });
     this.ponder_http_server_request_size_bytes = new prometheus.Histogram({
       name: "ponder_http_server_request_size_bytes",
@@ -349,6 +391,7 @@ export class MetricsService {
       labelNames: ["method", "path", "status"] as const,
       buckets: httpRequestSizeBytes,
       registers: [this.registry],
+      aggregator: "sum",
     });
     this.ponder_http_server_response_size_bytes = new prometheus.Histogram({
       name: "ponder_http_server_response_size_bytes",
@@ -356,6 +399,7 @@ export class MetricsService {
       labelNames: ["method", "path", "status"] as const,
       buckets: httpRequestSizeBytes,
       registers: [this.registry],
+      aggregator: "sum",
     });
 
     this.ponder_rpc_request_duration = new prometheus.Histogram({
@@ -364,6 +408,7 @@ export class MetricsService {
       labelNames: ["chain", "method"] as const,
       buckets: alwaysIODurationMs,
       registers: [this.registry],
+      aggregator: "sum",
     });
 
     this.ponder_rpc_request_error_total = new prometheus.Counter({
@@ -371,6 +416,7 @@ export class MetricsService {
       help: "Total count of failed RPC requests",
       labelNames: ["chain", "method"] as const,
       registers: [this.registry],
+      aggregator: "sum",
     });
 
     this.ponder_postgres_query_total = new prometheus.Counter({
@@ -378,6 +424,7 @@ export class MetricsService {
       help: "Total number of queries submitted to the database",
       labelNames: ["pool"] as const,
       registers: [this.registry],
+      aggregator: "sum",
     });
 
     prometheus.collectDefaultMetrics({
@@ -391,8 +438,12 @@ export class MetricsService {
    * Get string representation for all metrics.
    * @returns Metrics encoded using Prometheus v0.0.4 format.
    */
-  async getMetrics() {
-    return await this.registry.metrics();
+  getMetrics() {
+    return this.registry.metrics();
+  }
+
+  async getRegistry() {
+    return this.registry;
   }
 
   initializeIndexingMetrics({
@@ -404,8 +455,8 @@ export class MetricsService {
   }) {
     const tables = Object.values(schemaBuild.schema).filter(isTable);
 
-    for (const event of Object.keys(indexingBuild.indexingFunctions)) {
-      this.ponder_indexing_completed_events.inc({ event }, 0);
+    for (const { name: eventName } of indexingBuild.indexingFunctions) {
+      this.ponder_indexing_completed_events.inc({ event: eventName }, 0);
     }
 
     for (const table of tables) {
@@ -477,6 +528,168 @@ export class MetricsService {
   }
 }
 
+type MetricsAggregationRequest = {
+  type: typeof GET_METRICS_REQ;
+  requestId: number;
+};
+type MetricsAggregationResponse = {
+  type: typeof GET_METRICS_RES;
+  requestId: number;
+  error?: string;
+  metrics: prometheus.MetricObjectWithValues<prometheus.MetricValue<string>>[];
+};
+
+export class AggregateMetricsService extends MetricsService {
+  workers: Worker[];
+  requests: Map<
+    number,
+    {
+      responses: prometheus.MetricObjectWithValues<
+        prometheus.MetricValue<string>
+      >[][];
+      workerIds: number[];
+      pending: number;
+      pwr: PromiseWithResolvers<void>;
+    }
+  >;
+  requestId: number;
+  mainThreadMetrics: MetricsService;
+
+  constructor(mainThreadMetrics: MetricsService, workers: Worker[]) {
+    super();
+
+    this.mainThreadMetrics = mainThreadMetrics;
+    this.workers = workers;
+    this.requests = new Map();
+    this.requestId = 0;
+
+    for (const worker of workers) {
+      worker.on("message", (message: MetricsAggregationResponse) => {
+        if (message.type === GET_METRICS_RES) {
+          const request = this.requests.get(message.requestId);
+
+          if (request === undefined) return;
+
+          if (message.error) {
+            request.pwr.reject(new Error(message.error));
+            return;
+          }
+
+          request.responses.push(message.metrics);
+          request.workerIds.push(worker.threadId);
+          request.pending--;
+          if (request.pending === 0) {
+            request.pwr.resolve();
+          }
+        }
+      });
+    }
+  }
+
+  override async getMetrics() {
+    const requestId = this.requestId++;
+    const pwr = promiseWithResolvers<void>();
+
+    this.requests.set(requestId, {
+      responses: [],
+      workerIds: [],
+      pending: this.workers.length,
+      pwr,
+    });
+
+    for (const worker of this.workers) {
+      worker.postMessage({
+        type: GET_METRICS_REQ,
+        requestId,
+      } satisfies MetricsAggregationRequest);
+    }
+
+    await pwr.promise;
+
+    const request = this.requests.get(requestId)!;
+    this.requests.delete(requestId);
+
+    // Sort response by worker id for consistent metrics
+    const responseIndexSort = new Array(this.workers.length)
+      .fill(0)
+      .map((_, index) => index)
+      .sort((a, b) => request.workerIds[a]! - request.workerIds[b]!);
+
+    return prometheus.AggregatorRegistry.aggregate([
+      ...responseIndexSort.map((index) => request.responses[index]!),
+      await this.registry.getMetricsAsJSON(),
+      await this.mainThreadMetrics.registry.getMetricsAsJSON(),
+    ]).metrics();
+  }
+
+  override async getRegistry() {
+    const requestId = this.requestId++;
+    const pwr = promiseWithResolvers<void>();
+
+    this.requests.set(requestId, {
+      responses: [],
+      workerIds: [],
+      pending: this.workers.length,
+      pwr,
+    });
+
+    for (const worker of this.workers) {
+      worker.postMessage({
+        type: GET_METRICS_REQ,
+        requestId,
+      } satisfies MetricsAggregationRequest);
+    }
+
+    await pwr.promise;
+
+    const request = this.requests.get(requestId)!;
+    this.requests.delete(requestId);
+
+    // Sort response by worker id for consistent metrics
+    const responseIndexSort = new Array(this.workers.length)
+      .fill(0)
+      .map((_, index) => index)
+      .sort((a, b) => request.workerIds[a]! - request.workerIds[b]!);
+
+    return prometheus.AggregatorRegistry.aggregate([
+      ...responseIndexSort.map((index) => request.responses[index]!),
+      await this.registry.getMetricsAsJSON(),
+      await this.mainThreadMetrics.registry.getMetricsAsJSON(),
+    ]) as prometheus.Registry;
+  }
+
+  // Note: `resetIndexingMetrics` and `resetApiMetrics` are never called with `AggregateMetricsService`.
+}
+
+export class IsolatedMetricsService extends MetricsService {
+  constructor() {
+    super();
+
+    if (parentPort) {
+      parentPort.on("message", (message: MetricsAggregationRequest) => {
+        if (message.type === GET_METRICS_REQ) {
+          this.registry
+            .getMetricsAsJSON()
+            .then((metrics) => {
+              parentPort!.postMessage({
+                type: GET_METRICS_RES,
+                requestId: message.requestId,
+                metrics,
+              });
+            })
+            .catch((error) => {
+              parentPort!.postMessage({
+                type: GET_METRICS_RES,
+                requestId: message.requestId,
+                error: error.message,
+              });
+            });
+        }
+      });
+    }
+  }
+}
+
 const extractMetric = (
   metric: prometheus.MetricObjectWithValues<prometheus.MetricValue<"chain">>,
   chain: string,
@@ -596,44 +809,48 @@ export async function getAppProgress(metrics: MetricsService): Promise<{
   progress: number | undefined;
   eta: number | undefined;
 }> {
-  const totalSecondsMetric =
-    await metrics.ponder_historical_total_indexing_seconds.get();
-  const cachedSecondsMetric =
-    await metrics.ponder_historical_cached_indexing_seconds.get();
-  const completedSecondsMetric =
-    await metrics.ponder_historical_completed_indexing_seconds.get();
-  const timestampMetric = await metrics.ponder_indexing_timestamp.get();
-
-  const ordering: "multichain" | "omnichain" | undefined =
-    await metrics.ponder_settings_info
-      .get()
-      .then((metric) => metric.values[0]?.labels.ordering as any);
-  if (ordering === undefined) {
-    return {
-      mode: "backfill",
-      progress: undefined,
-      eta: undefined,
-    };
-  } else if (ordering === "multichain") {
-    const perChainAppProgress: Awaited<ReturnType<typeof getAppProgress>>[] =
-      [];
-
-    for (const chainName of totalSecondsMetric.values.map(
-      ({ labels }) => labels.chain as string,
-    )) {
-      const totalSeconds = extractMetric(totalSecondsMetric, chainName);
-      const cachedSeconds = extractMetric(cachedSecondsMetric, chainName);
-      const completedSeconds = extractMetric(completedSecondsMetric, chainName);
-      const timestamp = extractMetric(timestampMetric, chainName);
-
-      if (
-        totalSeconds === undefined ||
-        cachedSeconds === undefined ||
-        completedSeconds === undefined ||
-        timestamp === undefined
-      ) {
-        continue;
-      }
+  // Note: `getRegistry` must be used because this function is used with "experimental_isolated" ordering.
+  const registry = await metrics.getRegistry();
+
+  const totalSecondsMetric = await registry
+    .getSingleMetric("ponder_historical_total_indexing_seconds")!
+    .get();
+  const cachedSecondsMetric = await registry
+    .getSingleMetric("ponder_historical_cached_indexing_seconds")!
+    .get();
+  const completedSecondsMetric = await registry
+    .getSingleMetric("ponder_historical_completed_indexing_seconds")!
+    .get();
+  const timestampMetric = await registry
+    .getSingleMetric("ponder_indexing_timestamp")!
+    .get();
+
+  const settingsMetric = await registry
+    .getSingleMetric("ponder_settings_info")!
+    .get();
+  const ordering: PreBuild["ordering"] | undefined = settingsMetric?.values[0]
+    ?.labels.ordering as any;
+
+  switch (ordering) {
+    case undefined:
+      return {
+        mode: "backfill",
+        progress: undefined,
+        eta: undefined,
+      };
+    case "omnichain": {
+      const totalSeconds = totalSecondsMetric?.values
+        .map(({ value }) => value)
+        .reduce((prev, curr) => prev + curr, 0);
+      const cachedSeconds = cachedSecondsMetric?.values
+        .map(({ value }) => value)
+        .reduce((prev, curr) => prev + curr, 0);
+      const completedSeconds = completedSecondsMetric?.values
+        .map(({ value }) => value)
+        .reduce((prev, curr) => prev + curr, 0);
+      const timestamp = timestampMetric?.values
+        .map(({ value }) => value)
+        .reduce((prev, curr) => Math.max(prev, curr), 0);
 
       const progress =
         timestamp === 0
@@ -642,79 +859,92 @@ export async function getAppProgress(metrics: MetricsService): Promise<{
             ? 1
             : (completedSeconds + cachedSeconds) / totalSeconds;
 
-      if (!metrics.progressMetadata[chainName]) {
-        metrics.progressMetadata[chainName] = {
-          batches: [{ elapsedSeconds: 0, completedSeconds: 0 }],
-          previousTimestamp: Date.now(),
-          previousCompletedSeconds: 0,
-          rate: 0,
-        };
-      }
-
-      const eta: number | undefined = calculateEta(
-        metrics.progressMetadata[chainName]!,
-        totalSeconds,
-        cachedSeconds,
-        completedSeconds,
-      );
-      perChainAppProgress.push({
+      return {
         mode: progress === 1 ? "live" : "backfill",
-        progress,
-        eta,
-      });
+        progress: progress,
+        eta: calculateEta(
+          metrics.progressMetadata.general!,
+          totalSeconds,
+          cachedSeconds,
+          completedSeconds,
+        ),
+      };
     }
+    case "multichain":
+    case "experimental_isolated": {
+      const perChainAppProgress: Awaited<ReturnType<typeof getAppProgress>>[] =
+        [];
+
+      for (const chainName of totalSecondsMetric?.values.map(
+        ({ labels }) => labels.chain as string,
+      ) ?? []) {
+        const totalSeconds = extractMetric(totalSecondsMetric, chainName);
+        const cachedSeconds = extractMetric(cachedSecondsMetric, chainName);
+        const completedSeconds = extractMetric(
+          completedSecondsMetric,
+          chainName,
+        );
+        const timestamp = extractMetric(timestampMetric, chainName);
+
+        if (
+          totalSeconds === undefined ||
+          cachedSeconds === undefined ||
+          completedSeconds === undefined ||
+          timestamp === undefined
+        ) {
+          continue;
+        }
 
-    return perChainAppProgress.reduce(
-      (prev, curr) => ({
-        mode: curr.mode === "backfill" ? curr.mode : prev.mode,
-        progress:
-          prev.progress === undefined || curr.progress === undefined
-            ? undefined
-            : Math.min(prev.progress, curr.progress),
-        eta:
-          curr.progress === 1
-            ? prev.eta
-            : prev.eta === undefined || curr.eta === undefined
-              ? undefined
-              : Math.max(prev.eta, curr.eta),
-      }),
-      {
-        mode: "live",
-        progress: 1,
-        eta: 0,
-      },
-    ) as any;
-  } else {
-    const totalSeconds = totalSecondsMetric.values
-      .map(({ value }) => value)
-      .reduce((prev, curr) => prev + curr, 0);
-    const cachedSeconds = cachedSecondsMetric.values
-      .map(({ value }) => value)
-      .reduce((prev, curr) => prev + curr, 0);
-    const completedSeconds = completedSecondsMetric.values
-      .map(({ value }) => value)
-      .reduce((prev, curr) => prev + curr, 0);
-    const timestamp = timestampMetric.values
-      .map(({ value }) => value)
-      .reduce((prev, curr) => Math.max(prev, curr), 0);
+        const progress =
+          timestamp === 0
+            ? 0
+            : totalSeconds === 0
+              ? 1
+              : (completedSeconds + cachedSeconds) / totalSeconds;
+
+        if (!metrics.progressMetadata[chainName]) {
+          metrics.progressMetadata[chainName] = {
+            batches: [{ elapsedSeconds: 0, completedSeconds: 0 }],
+            previousTimestamp: Date.now(),
+            previousCompletedSeconds: 0,
+            rate: 0,
+          };
+        }
 
-    const progress =
-      timestamp === 0
-        ? 0
-        : totalSeconds === 0
-          ? 1
-          : (completedSeconds + cachedSeconds) / totalSeconds;
+        const eta: number | undefined = calculateEta(
+          metrics.progressMetadata[chainName]!,
+          totalSeconds,
+          cachedSeconds,
+          completedSeconds,
+        );
+        perChainAppProgress.push({
+          mode: progress === 1 ? "live" : "backfill",
+          progress,
+          eta,
+        });
+      }
 
-    return {
-      mode: progress === 1 ? "live" : "backfill",
-      progress: progress,
-      eta: calculateEta(
-        metrics.progressMetadata.general!,
-        totalSeconds,
-        cachedSeconds,
-        completedSeconds,
-      ),
-    };
+      return perChainAppProgress.reduce(
+        (prev, curr) => ({
+          mode: curr.mode === "backfill" ? curr.mode : prev.mode,
+          progress:
+            prev.progress === undefined || curr.progress === undefined
+              ? undefined
+              : Math.min(prev.progress, curr.progress),
+          eta:
+            curr.progress === 1
+              ? prev.eta
+              : prev.eta === undefined || curr.eta === undefined
+                ? undefined
+                : Math.max(prev.eta, curr.eta),
+        }),
+        {
+          mode: "live",
+          progress: 1,
+          eta: 0,
+        },
+      );
+    }
   }
 }