ponder 0.14.13 → 0.15.0

package/dist/esm/internal/metrics.js

@@ -1,3 +1,5 @@
+import { parentPort } from "node:worker_threads";
+import { promiseWithResolvers, } from '../utils/promiseWithResolvers.js';
 import { truncate } from '../utils/truncate.js';
 import { getTableName, isTable } from "drizzle-orm";
 import prometheus from "prom-client";
@@ -12,6 +14,8 @@ const httpRequestSizeBytes = [
     10, 100, 1000, 5000, 10000, 50000, 100000, 500000, 1000000, 5000000,
     10000000,
 ];
+const GET_METRICS_REQ = "prom-client:getMetricsReq";
+const GET_METRICS_RES = "prom-client:getMetricsRes";
 export class MetricsService {
     constructor() {
         Object.defineProperty(this, "registry", {
@@ -314,72 +318,84 @@ export class MetricsService {
             help: "Ponder version information",
             labelNames: ["version", "major", "minor", "patch"],
             registers: [this.registry],
+            aggregator: "first",
         });
         this.ponder_settings_info = new prometheus.Gauge({
             name: "ponder_settings_info",
             help: "Ponder settings information",
             labelNames: ["ordering", "database", "command"],
             registers: [this.registry],
+            aggregator: "first",
         });
         this.ponder_historical_concurrency_group_duration = new prometheus.Gauge({
             name: "ponder_historical_concurrency_group_duration",
             help: "Duration of historical concurrency groups",
             labelNames: ["group"],
             registers: [this.registry],
+            aggregator: "sum",
         });
         this.ponder_historical_extract_duration = new prometheus.Gauge({
             name: "ponder_historical_extract_duration",
             help: "Duration of historical extract phase",
             labelNames: ["step"],
             registers: [this.registry],
+            aggregator: "sum",
         });
         this.ponder_historical_transform_duration = new prometheus.Gauge({
             name: "ponder_historical_transform_duration",
             help: "Duration of historical transform phase",
             labelNames: ["step"],
             registers: [this.registry],
+            aggregator: "sum",
         });
         this.ponder_historical_start_timestamp_seconds = new prometheus.Gauge({
             name: "ponder_historical_start_timestamp_seconds",
             help: "Timestamp at which historical indexing started",
             labelNames: ["chain"],
             registers: [this.registry],
+            aggregator: "min",
         });
         this.ponder_historical_end_timestamp_seconds = new prometheus.Gauge({
             name: "ponder_historical_end_timestamp_seconds",
             help: "Timestamp at which historical indexing ended",
             labelNames: ["chain"],
             registers: [this.registry],
+            aggregator: "max",
         });
         this.ponder_historical_total_indexing_seconds = new prometheus.Gauge({
             name: "ponder_historical_total_indexing_seconds",
             help: "Total number of seconds that are required",
             labelNames: ["chain"],
             registers: [this.registry],
+            aggregator: "sum",
         });
         this.ponder_historical_cached_indexing_seconds = new prometheus.Gauge({
             name: "ponder_historical_cached_indexing_seconds",
             help: "Number of seconds that have been cached",
             labelNames: ["chain"],
             registers: [this.registry],
+            aggregator: "sum",
         });
         this.ponder_historical_completed_indexing_seconds = new prometheus.Gauge({
             name: "ponder_historical_completed_indexing_seconds",
             help: "Number of seconds that have been completed",
             labelNames: ["chain"],
             registers: [this.registry],
+            aggregator: "sum",
         });
         this.ponder_indexing_completed_events = new prometheus.Gauge({
             name: "ponder_indexing_completed_events",
             help: "Number of events that have been processed",
             labelNames: ["chain", "event"],
             registers: [this.registry],
+            aggregator: "sum",
         });
         this.ponder_indexing_timestamp = new prometheus.Gauge({
             name: "ponder_indexing_timestamp",
             help: "Timestamp through which all events have been completed",
             labelNames: ["chain"],
             registers: [this.registry],
+            aggregator: "first",
         });
         this.ponder_indexing_function_duration = new prometheus.Histogram({
             name: "ponder_indexing_function_duration",
@@ -387,6 +403,7 @@ export class MetricsService {
             labelNames: ["chain", "event"],
             buckets: sometimesIODurationMs,
             registers: [this.registry],
+            aggregator: "sum",
         });
         this.ponder_indexing_cache_query_duration = new prometheus.Histogram({
             name: "ponder_indexing_cache_query_duration",
@@ -394,6 +411,7 @@ export class MetricsService {
             labelNames: ["table", "method"],
             buckets: alwaysIODurationMs,
             registers: [this.registry],
+            aggregator: "sum",
         });
         this.ponder_indexing_rpc_action_duration = new prometheus.Histogram({
             name: "ponder_indexing_rpc_action_duration",
@@ -401,84 +419,98 @@ export class MetricsService {
             labelNames: ["action"],
             buckets: sometimesIODurationMs,
             registers: [this.registry],
+            aggregator: "sum",
         });
         this.ponder_indexing_rpc_prefetch_total = new prometheus.Counter({
             name: "ponder_indexing_rpc_prefetch_total",
             help: "Number of RPC prefetches",
             labelNames: ["chain", "method", "type"],
             registers: [this.registry],
+            aggregator: "sum",
         });
         this.ponder_indexing_rpc_requests_total = new prometheus.Counter({
             name: "ponder_indexing_rpc_requests_total",
             help: "Number of RPC requests",
             labelNames: ["chain", "method", "type"],
             registers: [this.registry],
+            aggregator: "sum",
         });
         this.ponder_indexing_cache_requests_total = new prometheus.Counter({
             name: "ponder_indexing_cache_requests_total",
             help: "Number of cache accesses",
             labelNames: ["table", "type"],
             registers: [this.registry],
+            aggregator: "sum",
         });
         this.ponder_indexing_store_queries_total = new prometheus.Counter({
             name: "ponder_indexing_store_queries_total",
             help: "Number of indexing store operations",
             labelNames: ["table", "method"],
             registers: [this.registry],
+            aggregator: "sum",
         });
         this.ponder_indexing_store_raw_sql_duration = new prometheus.Histogram({
             name: "ponder_indexing_store_raw_sql_duration",
             help: "Duration of raw SQL store operations",
             buckets: alwaysIODurationMs,
             registers: [this.registry],
+            aggregator: "sum",
         });
         this.ponder_sync_block = new prometheus.Gauge({
             name: "ponder_sync_block",
             help: "Closest-to-tip synced block number",
             labelNames: ["chain"],
             registers: [this.registry],
+            aggregator: "max",
         });
         this.ponder_sync_block_timestamp = new prometheus.Gauge({
             name: "ponder_sync_block_timestamp",
             help: "Closest-to-tip synced block timestamp",
             labelNames: ["chain"],
             registers: [this.registry],
+            aggregator: "max",
         });
         this.ponder_sync_is_realtime = new prometheus.Gauge({
             name: "ponder_sync_is_realtime",
             help: "Boolean (0 or 1) indicating if the sync is realtime mode",
             labelNames: ["chain"],
             registers: [this.registry],
+            aggregator: "max",
         });
         this.ponder_sync_is_complete = new prometheus.Gauge({
             name: "ponder_sync_is_complete",
             help: "Boolean (0 or 1) indicating if the sync has synced all blocks",
             labelNames: ["chain"],
             registers: [this.registry],
+            aggregator: "max",
         });
         this.ponder_historical_total_blocks = new prometheus.Gauge({
             name: "ponder_historical_total_blocks",
             help: "Number of blocks required for the historical sync",
             labelNames: ["chain"],
             registers: [this.registry],
+            aggregator: "max",
         });
         this.ponder_historical_cached_blocks = new prometheus.Gauge({
             name: "ponder_historical_cached_blocks",
             help: "Number of blocks that were found in the cache for the historical sync",
             labelNames: ["chain"],
             registers: [this.registry],
+            aggregator: "max",
         });
         this.ponder_historical_completed_blocks = new prometheus.Gauge({
             name: "ponder_historical_completed_blocks",
             help: "Number of blocks that have been processed for the historical sync",
             labelNames: ["chain", "source", "type"],
             registers: [this.registry],
+            aggregator: "max",
         });
         this.ponder_realtime_reorg_total = new prometheus.Counter({
             name: "ponder_realtime_reorg_total",
             help: "Count of how many re-orgs have occurred",
             labelNames: ["chain"],
             registers: [this.registry],
+            aggregator: "sum",
         });
         this.ponder_realtime_latency = new prometheus.Histogram({
             name: "ponder_realtime_latency",
@@ -489,6 +521,7 @@ export class MetricsService {
                 1000000,
             ],
             registers: [this.registry],
+            aggregator: "sum",
         });
         this.ponder_realtime_block_arrival_latency = new prometheus.Histogram({
             name: "ponder_realtime_block_arrival_latency",
@@ -499,6 +532,7 @@ export class MetricsService {
                 1000000,
             ],
             registers: [this.registry],
+            aggregator: "sum",
         });
         this.ponder_database_method_duration = new prometheus.Histogram({
             name: "ponder_database_method_duration",
@@ -506,18 +540,21 @@ export class MetricsService {
             labelNames: ["service", "method"],
             buckets: alwaysIODurationMs,
             registers: [this.registry],
+            aggregator: "sum",
         });
         this.ponder_database_method_error_total = new prometheus.Counter({
             name: "ponder_database_method_error_total",
             help: "Total number of errors encountered during database operations",
             labelNames: ["service", "method"],
             registers: [this.registry],
+            aggregator: "sum",
         });
         this.ponder_http_server_active_requests = new prometheus.Gauge({
             name: "ponder_http_server_active_requests",
             help: "Number of active HTTP server requests",
             labelNames: ["method", "path"],
             registers: [this.registry],
+            aggregator: "sum",
         });
         this.ponder_http_server_request_duration_ms = new prometheus.Histogram({
             name: "ponder_http_server_request_duration_ms",
@@ -525,6 +562,7 @@ export class MetricsService {
             labelNames: ["method", "path", "status"],
             buckets: alwaysIODurationMs,
             registers: [this.registry],
+            aggregator: "sum",
         });
         this.ponder_http_server_request_size_bytes = new prometheus.Histogram({
             name: "ponder_http_server_request_size_bytes",
@@ -532,6 +570,7 @@ export class MetricsService {
             labelNames: ["method", "path", "status"],
             buckets: httpRequestSizeBytes,
             registers: [this.registry],
+            aggregator: "sum",
         });
         this.ponder_http_server_response_size_bytes = new prometheus.Histogram({
             name: "ponder_http_server_response_size_bytes",
@@ -539,6 +578,7 @@ export class MetricsService {
             labelNames: ["method", "path", "status"],
             buckets: httpRequestSizeBytes,
             registers: [this.registry],
+            aggregator: "sum",
         });
         this.ponder_rpc_request_duration = new prometheus.Histogram({
             name: "ponder_rpc_request_duration",
@@ -546,18 +586,21 @@ export class MetricsService {
             labelNames: ["chain", "method"],
             buckets: alwaysIODurationMs,
             registers: [this.registry],
+            aggregator: "sum",
         });
         this.ponder_rpc_request_error_total = new prometheus.Counter({
             name: "ponder_rpc_request_error_total",
             help: "Total count of failed RPC requests",
             labelNames: ["chain", "method"],
             registers: [this.registry],
+            aggregator: "sum",
         });
         this.ponder_postgres_query_total = new prometheus.Counter({
             name: "ponder_postgres_query_total",
             help: "Total number of queries submitted to the database",
             labelNames: ["pool"],
             registers: [this.registry],
+            aggregator: "sum",
         });
         prometheus.collectDefaultMetrics({
             register: this.registry,
@@ -569,13 +612,16 @@ export class MetricsService {
      * Get string representation for all metrics.
      * @returns Metrics encoded using Prometheus v0.0.4 format.
      */
-    async getMetrics() {
-        return await this.registry.metrics();
+    getMetrics() {
+        return this.registry.metrics();
+    }
+    async getRegistry() {
+        return this.registry;
     }
     initializeIndexingMetrics({ indexingBuild, schemaBuild, }) {
         const tables = Object.values(schemaBuild.schema).filter(isTable);
-        for (const event of Object.keys(indexingBuild.indexingFunctions)) {
-            this.ponder_indexing_completed_events.inc({ event }, 0);
+        for (const { name: eventName } of indexingBuild.indexingFunctions) {
+            this.ponder_indexing_completed_events.inc({ event: eventName }, 0);
         }
         for (const table of tables) {
             for (const type of ["complete", "hit", "miss"]) {
@@ -634,6 +680,143 @@ export class MetricsService {
         this.ponder_http_server_response_size_bytes.reset();
     }
 }
+export class AggregateMetricsService extends MetricsService {
+    constructor(mainThreadMetrics, workers) {
+        super();
+        Object.defineProperty(this, "workers", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "requests", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "requestId", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "mainThreadMetrics", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.mainThreadMetrics = mainThreadMetrics;
+        this.workers = workers;
+        this.requests = new Map();
+        this.requestId = 0;
+        for (const worker of workers) {
+            worker.on("message", (message) => {
+                if (message.type === GET_METRICS_RES) {
+                    const request = this.requests.get(message.requestId);
+                    if (request === undefined)
+                        return;
+                    if (message.error) {
+                        request.pwr.reject(new Error(message.error));
+                        return;
+                    }
+                    request.responses.push(message.metrics);
+                    request.workerIds.push(worker.threadId);
+                    request.pending--;
+                    if (request.pending === 0) {
+                        request.pwr.resolve();
+                    }
+                }
+            });
+        }
+    }
+    async getMetrics() {
+        const requestId = this.requestId++;
+        const pwr = promiseWithResolvers();
+        this.requests.set(requestId, {
+            responses: [],
+            workerIds: [],
+            pending: this.workers.length,
+            pwr,
+        });
+        for (const worker of this.workers) {
+            worker.postMessage({
+                type: GET_METRICS_REQ,
+                requestId,
+            });
+        }
+        await pwr.promise;
+        const request = this.requests.get(requestId);
+        this.requests.delete(requestId);
+        // Sort response by worker id for consistent metrics
+        const responseIndexSort = new Array(this.workers.length)
+            .fill(0)
+            .map((_, index) => index)
+            .sort((a, b) => request.workerIds[a] - request.workerIds[b]);
+        return prometheus.AggregatorRegistry.aggregate([
+            ...responseIndexSort.map((index) => request.responses[index]),
+            await this.registry.getMetricsAsJSON(),
+            await this.mainThreadMetrics.registry.getMetricsAsJSON(),
+        ]).metrics();
+    }
+    async getRegistry() {
+        const requestId = this.requestId++;
+        const pwr = promiseWithResolvers();
+        this.requests.set(requestId, {
+            responses: [],
+            workerIds: [],
+            pending: this.workers.length,
+            pwr,
+        });
+        for (const worker of this.workers) {
+            worker.postMessage({
+                type: GET_METRICS_REQ,
+                requestId,
+            });
+        }
+        await pwr.promise;
+        const request = this.requests.get(requestId);
+        this.requests.delete(requestId);
+        // Sort response by worker id for consistent metrics
+        const responseIndexSort = new Array(this.workers.length)
+            .fill(0)
+            .map((_, index) => index)
+            .sort((a, b) => request.workerIds[a] - request.workerIds[b]);
+        return prometheus.AggregatorRegistry.aggregate([
+            ...responseIndexSort.map((index) => request.responses[index]),
+            await this.registry.getMetricsAsJSON(),
+            await this.mainThreadMetrics.registry.getMetricsAsJSON(),
+        ]);
+    }
+}
+export class IsolatedMetricsService extends MetricsService {
+    constructor() {
+        super();
+        if (parentPort) {
+            parentPort.on("message", (message) => {
+                if (message.type === GET_METRICS_REQ) {
+                    this.registry
+                        .getMetricsAsJSON()
+                        .then((metrics) => {
+                        parentPort.postMessage({
+                            type: GET_METRICS_RES,
+                            requestId: message.requestId,
+                            metrics,
+                        });
+                    })
+                        .catch((error) => {
+                        parentPort.postMessage({
+                            type: GET_METRICS_RES,
+                            requestId: message.requestId,
+                            error: error.message,
+                        });
+                    });
+                }
+            });
+        }
+    }
+}
 const extractMetric = (metric, chain) => {
     return metric.values.find((m) => m.labels.chain === chain)?.value;
 };
@@ -714,92 +897,106 @@ export async function getIndexingProgress(metrics) {
     };
 }
 export async function getAppProgress(metrics) {
-    const totalSecondsMetric = await metrics.ponder_historical_total_indexing_seconds.get();
-    const cachedSecondsMetric = await metrics.ponder_historical_cached_indexing_seconds.get();
-    const completedSecondsMetric = await metrics.ponder_historical_completed_indexing_seconds.get();
-    const timestampMetric = await metrics.ponder_indexing_timestamp.get();
-    const ordering = await metrics.ponder_settings_info
-        .get()
-        .then((metric) => metric.values[0]?.labels.ordering);
-    if (ordering === undefined) {
-        return {
-            mode: "backfill",
-            progress: undefined,
-            eta: undefined,
-        };
-    }
-    else if (ordering === "multichain") {
-        const perChainAppProgress = [];
-        for (const chainName of totalSecondsMetric.values.map(({ labels }) => labels.chain)) {
-            const totalSeconds = extractMetric(totalSecondsMetric, chainName);
-            const cachedSeconds = extractMetric(cachedSecondsMetric, chainName);
-            const completedSeconds = extractMetric(completedSecondsMetric, chainName);
-            const timestamp = extractMetric(timestampMetric, chainName);
-            if (totalSeconds === undefined ||
-                cachedSeconds === undefined ||
-                completedSeconds === undefined ||
-                timestamp === undefined) {
-                continue;
-            }
+    // Note: `getRegistry` must be used because this function is used with "experimental_isolated" ordering.
+    const registry = await metrics.getRegistry();
+    const totalSecondsMetric = await registry
+        .getSingleMetric("ponder_historical_total_indexing_seconds")
+        .get();
+    const cachedSecondsMetric = await registry
+        .getSingleMetric("ponder_historical_cached_indexing_seconds")
+        .get();
+    const completedSecondsMetric = await registry
+        .getSingleMetric("ponder_historical_completed_indexing_seconds")
+        .get();
+    const timestampMetric = await registry
+        .getSingleMetric("ponder_indexing_timestamp")
+        .get();
+    const settingsMetric = await registry
+        .getSingleMetric("ponder_settings_info")
+        .get();
+    const ordering = settingsMetric?.values[0]
+        ?.labels.ordering;
+    switch (ordering) {
+        case undefined:
+            return {
+                mode: "backfill",
+                progress: undefined,
+                eta: undefined,
+            };
+        case "omnichain": {
+            const totalSeconds = totalSecondsMetric?.values
+                .map(({ value }) => value)
+                .reduce((prev, curr) => prev + curr, 0);
+            const cachedSeconds = cachedSecondsMetric?.values
+                .map(({ value }) => value)
+                .reduce((prev, curr) => prev + curr, 0);
+            const completedSeconds = completedSecondsMetric?.values
+                .map(({ value }) => value)
+                .reduce((prev, curr) => prev + curr, 0);
+            const timestamp = timestampMetric?.values
+                .map(({ value }) => value)
+                .reduce((prev, curr) => Math.max(prev, curr), 0);
             const progress = timestamp === 0
                 ? 0
                 : totalSeconds === 0
                     ? 1
                     : (completedSeconds + cachedSeconds) / totalSeconds;
-            if (!metrics.progressMetadata[chainName]) {
-                metrics.progressMetadata[chainName] = {
-                    batches: [{ elapsedSeconds: 0, completedSeconds: 0 }],
-                    previousTimestamp: Date.now(),
-                    previousCompletedSeconds: 0,
-                    rate: 0,
-                };
-            }
-            const eta = calculateEta(metrics.progressMetadata[chainName], totalSeconds, cachedSeconds, completedSeconds);
-            perChainAppProgress.push({
+            return {
                 mode: progress === 1 ? "live" : "backfill",
-                progress,
-                eta,
-            });
+                progress: progress,
+                eta: calculateEta(metrics.progressMetadata.general, totalSeconds, cachedSeconds, completedSeconds),
+            };
         }
-        return perChainAppProgress.reduce((prev, curr) => ({
-            mode: curr.mode === "backfill" ? curr.mode : prev.mode,
-            progress: prev.progress === undefined || curr.progress === undefined
-                ? undefined
-                : Math.min(prev.progress, curr.progress),
-            eta: curr.progress === 1
-                ? prev.eta
-                : prev.eta === undefined || curr.eta === undefined
+        case "multichain":
+        case "experimental_isolated": {
+            const perChainAppProgress = [];
+            for (const chainName of totalSecondsMetric?.values.map(({ labels }) => labels.chain) ?? []) {
+                const totalSeconds = extractMetric(totalSecondsMetric, chainName);
+                const cachedSeconds = extractMetric(cachedSecondsMetric, chainName);
+                const completedSeconds = extractMetric(completedSecondsMetric, chainName);
+                const timestamp = extractMetric(timestampMetric, chainName);
+                if (totalSeconds === undefined ||
+                    cachedSeconds === undefined ||
+                    completedSeconds === undefined ||
+                    timestamp === undefined) {
+                    continue;
+                }
+                const progress = timestamp === 0
+                    ? 0
+                    : totalSeconds === 0
+                        ? 1
+                        : (completedSeconds + cachedSeconds) / totalSeconds;
+                if (!metrics.progressMetadata[chainName]) {
+                    metrics.progressMetadata[chainName] = {
+                        batches: [{ elapsedSeconds: 0, completedSeconds: 0 }],
+                        previousTimestamp: Date.now(),
+                        previousCompletedSeconds: 0,
+                        rate: 0,
+                    };
+                }
+                const eta = calculateEta(metrics.progressMetadata[chainName], totalSeconds, cachedSeconds, completedSeconds);
+                perChainAppProgress.push({
+                    mode: progress === 1 ? "live" : "backfill",
+                    progress,
+                    eta,
+                });
+            }
+            return perChainAppProgress.reduce((prev, curr) => ({
+                mode: curr.mode === "backfill" ? curr.mode : prev.mode,
+                progress: prev.progress === undefined || curr.progress === undefined
                     ? undefined
-                    : Math.max(prev.eta, curr.eta),
-        }), {
-            mode: "live",
-            progress: 1,
-            eta: 0,
-        });
-    }
-    else {
-        const totalSeconds = totalSecondsMetric.values
-            .map(({ value }) => value)
-            .reduce((prev, curr) => prev + curr, 0);
-        const cachedSeconds = cachedSecondsMetric.values
-            .map(({ value }) => value)
-            .reduce((prev, curr) => prev + curr, 0);
-        const completedSeconds = completedSecondsMetric.values
-            .map(({ value }) => value)
-            .reduce((prev, curr) => prev + curr, 0);
-        const timestamp = timestampMetric.values
-            .map(({ value }) => value)
-            .reduce((prev, curr) => Math.max(prev, curr), 0);
-        const progress = timestamp === 0
-            ? 0
-            : totalSeconds === 0
-                ? 1
-                : (completedSeconds + cachedSeconds) / totalSeconds;
-        return {
-            mode: progress === 1 ? "live" : "backfill",
-            progress: progress,
-            eta: calculateEta(metrics.progressMetadata.general, totalSeconds, cachedSeconds, completedSeconds),
-        };
+                    : Math.min(prev.progress, curr.progress),
+                eta: curr.progress === 1
+                    ? prev.eta
+                    : prev.eta === undefined || curr.eta === undefined
+                        ? undefined
+                        : Math.max(prev.eta, curr.eta),
+            }), {
+                mode: "live",
+                progress: 1,
+                eta: 0,
+            });
+        }
     }
 }
 function calculateEta(progressMetadata, totalSeconds, cachedSeconds, completedSeconds) {