ponder 0.14.13 → 0.15.0

package/dist/esm/bin/isolatedController.js

@@ -0,0 +1,200 @@
+import path from "node:path";
+import url from "node:url";
+import v8 from "node:v8";
+import { Worker } from "node:worker_threads";
+import { createIndexes, createViews } from '../database/actions.js';
+import { getPonderMetaTable } from '../database/index.js';
+import { NonRetryableUserError, nonRetryableUserErrorNames, } from '../internal/errors.js';
+import { AggregateMetricsService, getAppProgress } from '../internal/metrics.js';
+import { runIsolated } from '../runtime/isolated.js';
+import { chunk } from '../utils/chunk.js';
+import { formatEta, formatPercentage } from '../utils/format.js';
+import { startClock } from '../utils/timer.js';
+import { isTable, isView, sql } from "drizzle-orm";
+const __filename = url.fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+export async function isolatedController({ common, preBuild, namespaceBuild, schemaBuild, indexingBuild, crashRecoveryCheckpoint, database, }) {
+    const backfillEndClock = startClock();
+    const perChainState = new Map();
+    const etaInterval = setInterval(async () => {
+        const { eta, progress } = await getAppProgress(common.metrics);
+        if (eta === undefined && progress === undefined) {
+            return;
+        }
+        common.logger.info({
+            msg: "Updated backfill indexing progress",
+            progress: progress === undefined ? undefined : formatPercentage(progress),
+            estimate: eta === undefined ? undefined : formatEta(eta * 1000),
+        });
+    }, 5000);
+    common.shutdown.add(() => {
+        clearInterval(etaInterval);
+    });
+    let isAllReady = false;
+    let isAllComplete = false;
+    const callback = async () => {
+        if (isAllReady === false &&
+            indexingBuild.chains.every((chain) => perChainState.get(chain.id) !== "backfill")) {
+            isAllReady = true;
+            common.logger.info({
+                msg: "Completed backfill indexing across all chains",
+                duration: backfillEndClock(),
+            });
+            clearInterval(etaInterval);
+            let endClock = startClock();
+            await createIndexes(database.adminQB, {
+                statements: schemaBuild.statements,
+            });
+            if (schemaBuild.statements.indexes.sql.length > 0) {
+                common.logger.info({
+                    msg: "Created database indexes",
+                    count: schemaBuild.statements.indexes.sql.length,
+                    duration: endClock(),
+                });
+            }
+            if (namespaceBuild.viewsSchema !== undefined) {
+                endClock = startClock();
+                const tables = Object.values(schemaBuild.schema).filter(isTable);
+                const views = Object.values(schemaBuild.schema).filter(isView);
+                await createViews(database.adminQB, { tables, views, namespaceBuild });
+                common.logger.info({
+                    msg: "Created database views",
+                    schema: namespaceBuild.viewsSchema,
+                    count: tables.length,
+                    duration: endClock(),
+                });
+                endClock = startClock();
+            }
+            await database.adminQB.wrap({ label: "update_ready" }, (db) => db
+                .update(getPonderMetaTable(namespaceBuild.schema))
+                .set({ value: sql `jsonb_set(value, '{is_ready}', to_jsonb(1))` }));
+            common.logger.info({
+                msg: "Started returning 200 responses",
+                endpoint: "/ready",
+            });
+        }
+        if (isAllComplete === false &&
+            indexingBuild.chains.every((chain) => perChainState.get(chain.id) === "complete")) {
+            isAllComplete = true;
+        }
+    };
+    if (common.options.command === "dev" ||
+        indexingBuild.chains.length === 1 ||
+        database.driver.dialect === "pglite" ||
+        common.options.maxThreads === 1) {
+        common.options.indexingCacheMaxBytes = Math.floor(common.options.indexingCacheMaxBytes / indexingBuild.chains.length);
+        common.options.rpcMaxConcurrency = Math.floor(common.options.rpcMaxConcurrency / indexingBuild.chains.length);
+        common.options.syncEventsQuerySize = Math.floor(common.options.syncEventsQuerySize / indexingBuild.chains.length);
+        await Promise.all(indexingBuild.chains.map(async (chain) => {
+            const chainIndex = indexingBuild.chains.findIndex((c) => c.id === chain.id);
+            const _indexingBuild = {
+                ...indexingBuild,
+                chains: [indexingBuild.chains[chainIndex]],
+                rpcs: [indexingBuild.rpcs[chainIndex]],
+                finalizedBlocks: [indexingBuild.finalizedBlocks[chainIndex]],
+                eventCallbacks: [indexingBuild.eventCallbacks[chainIndex]],
+                setupCallbacks: [indexingBuild.setupCallbacks[chainIndex]],
+                contracts: [indexingBuild.contracts[chainIndex]],
+            };
+            perChainState.set(chain.id, "backfill");
+            await runIsolated({
+                common,
+                preBuild,
+                namespaceBuild,
+                schemaBuild,
+                indexingBuild: _indexingBuild,
+                crashRecoveryCheckpoint,
+                database,
+                onReady: () => {
+                    perChainState.set(chain.id, "live");
+                    callback();
+                },
+            });
+            perChainState.set(chain.id, "complete");
+            callback();
+        }));
+    }
+    else {
+        const workerPath = path.join(__dirname, "isolatedWorker.js");
+        const perThreadChains = chunk(indexingBuild.chains, Math.ceil(indexingBuild.chains.length / common.options.maxThreads));
+        const perThreadWorkers = [];
+        for (const chains of perThreadChains) {
+            const chainIds = chains.map((chain) => chain.id);
+            // Note: This is a hack to force color support in the worker threads
+            if (process.stdout.isTTY) {
+                process.env.FORCE_COLOR = "1";
+            }
+            const heapSizeLimit = v8.getHeapStatistics().heap_size_limit;
+            const perThreadHeapSizeLimit = Math.floor(heapSizeLimit / (common.options.maxThreads + 1) / 1024 / 1024);
+            // Note: This sets `--max-old-space-size` for the worker thread.
+            // `resourceLimits` does not work because it gets overridden by
+            // CLI flags or environment variables. It does not change the heap
+            // size limit for the current main thread
+            v8.setFlagsFromString(`--max-old-space-size=${perThreadHeapSizeLimit}`);
+            const worker = new Worker(workerPath, {
+                workerData: {
+                    options: common.options,
+                    chainIds,
+                    namespaceBuild,
+                    crashRecoveryCheckpoint,
+                },
+            });
+            for (const chainId of chainIds) {
+                perChainState.set(chainId, "backfill");
+            }
+            worker.on("message", (message) => {
+                switch (message.type) {
+                    case "ready": {
+                        perChainState.set(message.chainId, "live");
+                        callback();
+                        break;
+                    }
+                    case "done": {
+                        perChainState.set(message.chainId, "complete");
+                        callback();
+                        break;
+                    }
+                    case "error": {
+                        let error;
+                        if (nonRetryableUserErrorNames.includes(message.error.name)) {
+                            error = new NonRetryableUserError(message.error.message);
+                        }
+                        else {
+                            error = new Error(message.error.message);
+                        }
+                        error.name = message.error.name;
+                        error.stack = message.error.stack;
+                        throw error;
+                    }
+                }
+            });
+            worker.on("error", (error) => {
+                if (nonRetryableUserErrorNames.includes(error.name)) {
+                    error = new NonRetryableUserError(error.message);
+                }
+                else {
+                    error = new Error(error.message);
+                }
+                throw error;
+            });
+            worker.on("exit", (code) => {
+                const error = new Error(`Worker thread exited with code ${code}.`);
+                error.stack = undefined;
+                throw error;
+            });
+            perThreadWorkers.push(worker);
+        }
+        common.logger.debug({
+            msg: "Created worker threads",
+            count: perThreadWorkers.length,
+            duration: backfillEndClock(),
+        });
+        common.metrics = new AggregateMetricsService(common.metrics, perThreadWorkers);
+        common.shutdown.add(() => {
+            for (const worker of perThreadWorkers) {
+                worker.postMessage({ type: "kill" });
+            }
+        });
+    }
+}
+//# sourceMappingURL=isolatedController.js.map

package/dist/esm/bin/isolatedController.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"isolatedController.js","sourceRoot":"","sources":["../../../src/bin/isolatedController.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,GAAG,MAAM,UAAU,CAAC;AAC3B,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAC7C,OAAO,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AACnE,OAAO,EAAiB,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AAExE,OAAO,EACL,qBAAqB,EACrB,0BAA0B,GAC3B,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,uBAAuB,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAQhF,OAAO,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AACpD,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAChE,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,aAAa,CAAC;AAGnD,MAAM,UAAU,GAAG,GAAG,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACtD,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;AAI3C,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,EACvC,MAAM,EACN,QAAQ,EACR,cAAc,EACd,WAAW,EACX,aAAa,EACb,uBAAuB,EACvB,QAAQ,GAST;IACC,MAAM,gBAAgB,GAAG,UAAU,EAAE,CAAC;IACtC,MAAM,aAAa,GAAG,IAAI,GAAG,EAAuB,CAAC;IAErD,MAAM,WAAW,GAAG,WAAW,CAAC,KAAK,IAAI,EAAE;QACzC,MAAM,EAAE,GAAG,EAAE,QAAQ,EAAE,GAAG,MAAM,cAAc,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAE/D,IAAI,GAAG,KAAK,SAAS,IAAI,QAAQ,KAAK,SAAS,EAAE;YAC/C,OAAO;SACR;QAED,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;YACjB,GAAG,EAAE,oCAAoC;YACzC,QAAQ,EAAE,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,gBAAgB,CAAC,QAAQ,CAAC;YACzE,QAAQ,EAAE,GAAG,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,GAAG,GAAG,IAAK,CAAC;SACjE,CAAC,CAAC;IACL,CAAC,EAAE,IAAK,CAAC,CAAC;IAEV,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE;QACvB,aAAa,CAAC,WAAW,CAAC,CAAC;IAC7B,CAAC,CAAC,CAAC;IAEH,IAAI,UAAU,GAAG,KAAK,CAAC;IACvB,IAAI,aAAa,GAAG,KAAK,CAAC;IAC1B,MAAM,QAAQ,GAAG,KAAK,IAAI,EAAE;QAC1B,IACE,UAAU,KAAK,KAAK;YACpB,aAAa,CAAC,MAAM,CAAC,KAAK,CACxB,CAAC,KAAK,EAAE,EAAE,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,UAAU,CACtD,EACD;YACA,UAAU,GAAG,IAAI,CAAC;YAElB,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;gBACjB,GAAG,EAAE,+CAA+C;gBACpD,QAAQ,EAAE,gBAAgB,EAAE;aAC7B,CAAC,CAAC;YACH,aAAa,CAAC,WAAW,CAAC,CAAC;YAE3B,IAAI,QAAQ,GAAG,UAAU,EAAE,CAAC;YAC5B,MAAM,aAAa,CAAC,QAAQ,CAAC,OAAO,EAAE;gBACpC,UAAU,EAAE,WAAW,CAAC,UAAU;aACnC,CAAC,CAAC;YAEH,IAAI,WAAW,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE;gBACjD,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;oBACjB,GAAG,EAAE,0BAA0B;oBAC/B,KAAK,EAAE,WAAW,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM;oBAChD,QAAQ,EAAE,QAAQ,EAAE;iBACrB,CAAC,CAAC;aACJ;YAED,IAAI,cAAc,CAAC,WAAW,KAAK,SAAS,EAAE;gBAC5C,QAAQ,GAAG,UAAU,EAAE,CAAC;gBAExB,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;gBACjE,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;gBAC/D,MAAM,WAAW,CAAC,QAAQ,CAAC,OAAO,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC,CAAC;gBAEvE,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;oBACjB,GAAG,EAAE,wBAAwB;oBAC7B,MAAM,EAAE,cAAc,CAAC,WAAW;oBAClC,KAAK,EAAE,MAAM,CAAC,MAAM;oBACpB,QAAQ,EAAE,QAAQ,EAAE;iBACrB,CAAC,CAAC;gBAEH,QAAQ,GAAG,UAAU,EAAE,CAAC;aACzB;YAED,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,cAAc,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAC5D,EAAE;iBACC,MAAM,CAAC,kBAAkB,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;iBACjD,GAAG,CAAC,EAAE,KAAK,EAAE,GAAG,CAAA,6CAA6C,EAAE,CAAC,CACpE,CAAC;YAEF,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;gBACjB,GAAG,EAAE,iCAAiC;gBACtC,QAAQ,EAAE,QAAQ;aACnB,CAAC,CAAC;SACJ;QAED,IACE,aAAa,KAAK,KAAK;YACvB,aAAa,CAAC,MAAM,CAAC,KAAK,CACxB,CAAC,KAAK,EAAE,EAAE,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,UAAU,CACtD,EACD;YACA,aAAa,GAAG,IAAI,CAAC;SACtB;IACH,CAAC,CAAC;IAEF,IACE,MAAM,CAAC,OAAO,CAAC,OAAO,KAAK,KAAK;QAChC,aAAa,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC;QACjC,QAAQ,CAAC,MAAM,CAAC,OAAO,KAAK,QAAQ;QACpC,MAAM,CAAC,OAAO,CAAC,UAAU,KAAK,CAAC,EAC/B;QACA,MAAM,CAAC,OAAO,CAAC,qBAAqB,GAAG,IAAI,CAAC,KAAK,CAC/C,MAAM,CAAC,OAAO,CAAC,qBAAqB,GAAG,aAAa,CAAC,MAAM,CAAC,MAAM,CACnE,CAAC;QACF,MAAM,CAAC,OAAO,CAAC,iBAAiB,GAAG,IAAI,CAAC,KAAK,CAC3C,MAAM,CAAC,OAAO,CAAC,iBAAiB,GAAG,aAAa,CAAC,MAAM,CAAC,MAAM,CAC/D,CAAC;QACF,MAAM,CAAC,OAAO,CAAC,mBAAmB,GAAG,IAAI,CAAC,KAAK,CAC7C,MAAM,CAAC,OAAO,CAAC,mBAAmB,GAAG,aAAa,CAAC,MAAM,CAAC,MAAM,CACjE,CAAC;QAEF,MAAM,OAAO,CAAC,GAAG,CACf,aAAa,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;YACvC,MAAM,UAAU,GAAG,aAAa,CAAC,MAAM,CAAC,SAAS,CAC/C,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,KAAK,CAAC,EAAE,CACzB,CAAC;YACF,MAAM,cAAc,GAAG;gBACrB,GAAG,aAAa;gBAChB,MAAM,EAAE,CAAC,aAAa,CAAC,MAAM,CAAC,UAAU,CAAE,CAAC;gBAC3C,IAAI,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,UAAU,CAAE,CAAC;gBACvC,eAAe,EAAE,CAAC,aAAa,CAAC,eAAe,CAAC,UAAU,CAAE,CAAC;gBAC7D,cAAc,EAAE,CAAC,aAAa,CAAC,cAAc,CAAC,UAAU,CAAE,CAAC;gBAC3D,cAAc,EAAE,CAAC,aAAa,CAAC,cAAc,CAAC,UAAU,CAAE,CAAC;gBAC3D,SAAS,EAAE,CAAC,aAAa,CAAC,SAAS,CAAC,UAAU,CAAE,CAAC;aAClD,CAAC;YAEF,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC;YAExC,MAAM,WAAW,CAAC;gBAChB,MAAM;gBACN,QAAQ;gBACR,cAAc;gBACd,WAAW;gBACX,aAAa,EAAE,cAAc;gBAC7B,uBAAuB;gBACvB,QAAQ;gBACR,OAAO,EAAE,GAAG,EAAE;oBACZ,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;oBACpC,QAAQ,EAAE,CAAC;gBACb,CAAC;aACF,CAAC,CAAC;YAEH,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC;YACxC,QAAQ,EAAE,CAAC;QACb,CAAC,CAAC,CACH,CAAC;KACH;SAAM;QACL,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,mBAAmB,CAAC,CAAC;QAE7D,MAAM,eAAe,GAAG,KAAK,CAC3B,aAAa,CAAC,MAAM,EACpB,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,CACnE,CAAC;QAEF,MAAM,gBAAgB,GAAa,EAAE,CAAC;QACtC,KAAK,MAAM,MAAM,IAAI,eAAe,EAAE;YACpC,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAEjD,oEAAoE;YACpE,IAAI,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE;gBACxB,OAAO,CAAC,GAAG,CAAC,WAAW,GAAG,GAAG,CAAC;aAC/B;YAED,MAAM,aAAa,GAAG,EAAE,CAAC,iBAAiB,EAAE,CAAC,eAAe,CAAC;YAC7D,MAAM,sBAAsB,GAAG,IAAI,CAAC,KAAK,CACvC,aAAa,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,UAAU,GAAG,CAAC,CAAC,GAAG,IAAI,GAAG,IAAI,CAC9D,CAAC;YAEF,gEAAgE;YAChE,+DAA+D;YAC/D,kEAAkE;YAClE,yCAAyC;YACzC,EAAE,CAAC,kBAAkB,CAAC,wBAAwB,sBAAsB,EAAE,CAAC,CAAC;YAExE,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,UAAU,EAAE;gBACpC,UAAU,EAAE;oBACV,OAAO,EAAE,MAAM,CAAC,OAAO;oBACvB,QAAQ;oBACR,cAAc;oBACd,uBAAuB;iBACuB;aACjD,CAAC,CAAC;YAEH,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE;gBAC9B,aAAa,CAAC,GAAG,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;aACxC;YAED,MAAM,CAAC,EAAE,CACP,SAAS,EACT,CACE,OAGmC,EACnC,EAAE;gBACF,QAAQ,OAAO,CAAC,IAAI,EAAE;oBACpB,KAAK,OAAO,CAAC,CAAC;wBACZ,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;wBAC3C,QAAQ,EAAE,CAAC;wBACX,MAAM;qBACP;oBACD,KAAK,MAAM,CAAC,CAAC;wBACX,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;wBAC/C,QAAQ,EAAE,CAAC;wBACX,MAAM;qBACP;oBACD,KAAK,OAAO,CAAC,CAAC;wBACZ,IAAI,KAAY,CAAC;wBACjB,IAAI,0BAA0B,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE;4BAC3D,KAAK,GAAG,IAAI,qBAAqB,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;yBAC1D;6BAAM;4BACL,KAAK,GAAG,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;yBAC1C;wBACD,KAAK,CAAC,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC;wBAChC,KAAK,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC;wBAClC,MAAM,KAAK,CAAC;qBACb;iBACF;YACH,CAAC,CACF,CAAC;YAEF,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAY,EAAE,EAAE;gBAClC,IAAI,0BAA0B,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE;oBACnD,KAAK,GAAG,IAAI,qBAAqB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;iBAClD;qBAAM;oBACL,KAAK,GAAG,IAAI,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;iBAClC;gBACD,MAAM,KAAK,CAAC;YACd,CAAC,CAAC,CAAC;YAEH,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAY,EAAE,EAAE;gBACjC,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC,kCAAkC,IAAI,GAAG,CAAC,CAAC;gBACnE,KAAK,CAAC,KAAK,GAAG,SAAS,CAAC;gBACxB,MAAM,KAAK,CAAC;YACd,CAAC,CAAC,CAAC;YAEH,gBAAgB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;SAC/B;QAED,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC;YAClB,GAAG,EAAE,wBAAwB;YAC7B,KAAK,EAAE,gBAAgB,CAAC,MAAM;YAC9B,QAAQ,EAAE,gBAAgB,EAAE;SAC7B,CAAC,CAAC;QAEH,MAAM,CAAC,OAAO,GAAG,IAAI,uBAAuB,CAC1C,MAAM,CAAC,OAAO,EACd,gBAAgB,CACjB,CAAC;QAEF,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE;YACvB,KAAK,MAAM,MAAM,IAAI,gBAAgB,EAAE;gBACrC,MAAM,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;aACtC;QACH,CAAC,CAAC,CAAC;KACJ;AACH,CAAC"}

package/dist/esm/bin/isolatedWorker.js

@@ -0,0 +1,146 @@
+import { isMainThread, parentPort, workerData } from "node:worker_threads";
+import { createBuild } from '../build/index.js';
+import { createDatabase } from '../database/index.js';
+import { createLogger } from '../internal/logger.js';
+import { IsolatedMetricsService } from '../internal/metrics.js';
+import { createShutdown } from '../internal/shutdown.js';
+import { createTelemetry } from '../internal/telemetry.js';
+import { runIsolated } from '../runtime/isolated.js';
+if (isMainThread) {
+    throw new Error("'isolatedWorker.ts' must be run in a worker thread");
+}
+try {
+    await isolatedWorker(workerData);
+    parentPort.postMessage({ type: "done" });
+}
+catch (err) {
+    const error = err;
+    parentPort.postMessage({
+        type: "error",
+        error: {
+            name: error.name,
+            message: error.message,
+            stack: error.stack,
+        },
+    });
+}
+export async function isolatedWorker({ options, namespaceBuild, crashRecoveryCheckpoint, chainIds, }) {
+    // Note: telemetry is disabled because the main thread will report telemetry
+    options.telemetryDisabled = true;
+    const logger = createLogger({
+        level: options.logLevel,
+        mode: options.logFormat,
+    });
+    const metrics = new IsolatedMetricsService();
+    const shutdown = createShutdown();
+    const telemetry = createTelemetry({ options, logger, shutdown });
+    const common = {
+        options,
+        logger,
+        metrics,
+        telemetry,
+        shutdown,
+        buildShutdown: shutdown,
+        apiShutdown: shutdown,
+    };
+    let isKilled = false;
+    parentPort.on("message", async (msg) => {
+        if (msg.type === "kill") {
+            if (isKilled)
+                return;
+            isKilled = true;
+            await shutdown.kill();
+        }
+    });
+    const build = await createBuild({
+        common,
+        cliOptions: workerData.cliOptions,
+    });
+    // Note: build is guaranteed to be successful because the main thread
+    // has already run the build.
+    // Note: `namespaceCompile`
+    globalThis.PONDER_NAMESPACE_BUILD = namespaceBuild;
+    const configResult = await build.executeConfig();
+    if (configResult.status === "error") {
+        throw configResult.error;
+    }
+    const schemaResult = await build.executeSchema();
+    if (schemaResult.status === "error") {
+        throw schemaResult.error;
+    }
+    const preBuildResult = build.preCompile(configResult.result);
+    if (preBuildResult.status === "error") {
+        throw preBuildResult.error;
+    }
+    const schemaBuildResult = build.compileSchema({
+        ...schemaResult.result,
+        preBuild: preBuildResult.result,
+    });
+    if (schemaBuildResult.status === "error") {
+        throw schemaBuildResult.error;
+    }
+    const configBuildResult = build.compileConfig({
+        configResult: configResult.result,
+    });
+    if (configBuildResult.status === "error") {
+        throw configBuildResult.error;
+    }
+    const indexingResult = await build.executeIndexingFunctions();
+    if (indexingResult.status === "error") {
+        throw indexingResult.error;
+    }
+    const indexingBuildResult = await build.compileIndexing({
+        configResult: configResult.result,
+        schemaResult: schemaResult.result,
+        indexingResult: indexingResult.result,
+        configBuild: configBuildResult.result,
+    });
+    if (indexingBuildResult.status === "error") {
+        throw indexingBuildResult.error;
+    }
+    options.indexingCacheMaxBytes = Math.floor(options.indexingCacheMaxBytes / indexingBuildResult.result.chains.length);
+    options.rpcMaxConcurrency = Math.floor(options.rpcMaxConcurrency / indexingBuildResult.result.chains.length);
+    options.syncEventsQuerySize = Math.floor(options.syncEventsQuerySize / indexingBuildResult.result.chains.length);
+    const database = createDatabase({
+        common,
+        namespace: namespaceBuild,
+        preBuild: preBuildResult.result,
+        schemaBuild: schemaBuildResult.result,
+    });
+    await Promise.all(chainIds.map(async (chainId) => {
+        const chainIndex = indexingBuildResult.result.chains.findIndex((c) => c.id === chainId);
+        const indexingBuild = {
+            ...indexingBuildResult.result,
+            chains: [indexingBuildResult.result.chains[chainIndex]],
+            rpcs: [indexingBuildResult.result.rpcs[chainIndex]],
+            finalizedBlocks: [
+                indexingBuildResult.result.finalizedBlocks[chainIndex],
+            ],
+            eventCallbacks: [
+                indexingBuildResult.result.eventCallbacks[chainIndex],
+            ],
+            setupCallbacks: [
+                indexingBuildResult.result.setupCallbacks[chainIndex],
+            ],
+            contracts: [indexingBuildResult.result.contracts[chainIndex]],
+        };
+        common.metrics.initializeIndexingMetrics({
+            indexingBuild,
+            schemaBuild: schemaBuildResult.result,
+        });
+        await runIsolated({
+            common,
+            preBuild: preBuildResult.result,
+            namespaceBuild,
+            schemaBuild: schemaBuildResult.result,
+            indexingBuild,
+            crashRecoveryCheckpoint,
+            database,
+            onReady: () => {
+                parentPort.postMessage({ type: "ready", chainId });
+            },
+        });
+        parentPort.postMessage({ type: "done", chainId });
+    }));
+}
+//# sourceMappingURL=isolatedWorker.js.map

package/dist/esm/bin/isolatedWorker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"isolatedWorker.js","sourceRoot":"","sources":["../../../src/bin/isolatedWorker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAC3E,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAErD,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACpD,OAAO,EAAE,sBAAsB,EAAE,MAAM,uBAAuB,CAAC;AAC/D,OAAO,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAK1D,OAAO,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AAEpD,IAAI,YAAY,EAAE;IAChB,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;CACvE;AAED,IAAI;IACF,MAAM,cAAc,CAAC,UAAU,CAAC,CAAC;IAEjC,UAAW,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;CAC3C;AAAC,OAAO,GAAG,EAAE;IACZ,MAAM,KAAK,GAAG,GAAY,CAAC;IAE3B,UAAW,CAAC,WAAW,CAAC;QACtB,IAAI,EAAE,OAAO;QACb,KAAK,EAAE;YACL,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,KAAK,EAAE,KAAK,CAAC,KAAK;SACnB;KACF,CAAC,CAAC;CACJ;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,EACnC,OAAO,EACP,cAAc,EACd,uBAAuB,EACvB,QAAQ,GAMT;IACC,4EAA4E;IAC5E,OAAO,CAAC,iBAAiB,GAAG,IAAI,CAAC;IAEjC,MAAM,MAAM,GAAG,YAAY,CAAC;QAC1B,KAAK,EAAE,OAAO,CAAC,QAAQ;QACvB,IAAI,EAAE,OAAO,CAAC,SAAS;KACxB,CAAC,CAAC;IAEH,MAAM,OAAO,GAAG,IAAI,sBAAsB,EAAE,CAAC;IAC7C,MAAM,QAAQ,GAAG,cAAc,EAAE,CAAC;IAClC,MAAM,SAAS,GAAG,eAAe,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC;IACjE,MAAM,MAAM,GAAG;QACb,OAAO;QACP,MAAM;QACN,OAAO;QACP,SAAS;QACT,QAAQ;QACR,aAAa,EAAE,QAAQ;QACvB,WAAW,EAAE,QAAQ;KACtB,CAAC;IAEF,IAAI,QAAQ,GAAG,KAAK,CAAC;IACrB,UAAW,CAAC,EAAE,CAAC,SAAS,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;QACtC,IAAI,GAAG,CAAC,IAAI,KAAK,MAAM,EAAE;YACvB,IAAI,QAAQ;gBAAE,OAAO;YACrB,QAAQ,GAAG,IAAI,CAAC;YAChB,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;SACvB;IACH,CAAC,CAAC,CAAC;IAEH,MAAM,KAAK,GAAG,MAAM,WAAW,CAAC;QAC9B,MAAM;QACN,UAAU,EAAE,UAAU,CAAC,UAAU;KAClC,CAAC,CAAC;IAEH,qEAAqE;IACrE,6BAA6B;IAE7B,2BAA2B;IAC3B,UAAU,CAAC,sBAAsB,GAAG,cAAc,CAAC;IAEnD,MAAM,YAAY,GAAG,MAAM,KAAK,CAAC,aAAa,EAAE,CAAC;IACjD,IAAI,YAAY,CAAC,MAAM,KAAK,OAAO,EAAE;QACnC,MAAM,YAAY,CAAC,KAAK,CAAC;KAC1B;IAED,MAAM,YAAY,GAAG,MAAM,KAAK,CAAC,aAAa,EAAE,CAAC;IACjD,IAAI,YAAY,CAAC,MAAM,KAAK,OAAO,EAAE;QACnC,MAAM,YAAY,CAAC,KAAK,CAAC;KAC1B;IACD,MAAM,cAAc,GAAG,KAAK,CAAC,UAAU,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;IAC7D,IAAI,cAAc,CAAC,MAAM,KAAK,OAAO,EAAE;QACrC,MAAM,cAAc,CAAC,KAAK,CAAC;KAC5B;IAED,MAAM,iBAAiB,GAAG,KAAK,CAAC,aAAa,CAAC;QAC5C,GAAG,YAAY,CAAC,MAAM;QACtB,QAAQ,EAAE,cAAc,CAAC,MAAM;KAChC,CAAC,CAAC;IACH,IAAI,iBAAiB,CAAC,MAAM,KAAK,OAAO,EAAE;QACxC,MAAM,iBAAiB,CAAC,KAAK,CAAC;KAC/B;IAED,MAAM,iBAAiB,GAAG,KAAK,CAAC,aAAa,CAAC;QAC5C,YAAY,EAAE,YAAY,CAAC,MAAM;KAClC,CAAC,CAAC;IACH,IAAI,iBAAiB,CAAC,MAAM,KAAK,OAAO,EAAE;QACxC,MAAM,iBAAiB,CAAC,KAAK,CAAC;KAC/B;IAED,MAAM,cAAc,GAAG,MAAM,KAAK,CAAC,wBAAwB,EAAE,CAAC;IAC9D,IAAI,cAAc,CAAC,MAAM,KAAK,OAAO,EAAE;QACrC,MAAM,cAAc,CAAC,KAAK,CAAC;KAC5B;IAED,MAAM,mBAAmB,GAAG,MAAM,KAAK,CAAC,eAAe,CAAC;QACtD,YAAY,EAAE,YAAY,CAAC,MAAM;QACjC,YAAY,EAAE,YAAY,CAAC,MAAM;QACjC,cAAc,EAAE,cAAc,CAAC,MAAM;QACrC,WAAW,EAAE,iBAAiB,CAAC,MAAM;KACtC,CAAC,CAAC;IACH,IAAI,mBAAmB,CAAC,MAAM,KAAK,OAAO,EAAE;QAC1C,MAAM,mBAAmB,CAAC,KAAK,CAAC;KACjC;IAED,OAAO,CAAC,qBAAqB,GAAG,IAAI,CAAC,KAAK,CACxC,OAAO,CAAC,qBAAqB,GAAG,mBAAmB,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CACzE,CAAC;IACF,OAAO,CAAC,iBAAiB,GAAG,IAAI,CAAC,KAAK,CACpC,OAAO,CAAC,iBAAiB,GAAG,mBAAmB,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CACrE,CAAC;IACF,OAAO,CAAC,mBAAmB,GAAG,IAAI,CAAC,KAAK,CACtC,OAAO,CAAC,mBAAmB,GAAG,mBAAmB,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CACvE,CAAC;IAEF,MAAM,QAAQ,GAAG,cAAc,CAAC;QAC9B,MAAM;QACN,SAAS,EAAE,cAAc;QACzB,QAAQ,EAAE,cAAc,CAAC,MAAM;QAC/B,WAAW,EAAE,iBAAiB,CAAC,MAAM;KACtC,CAAC,CAAC;IAEH,MAAM,OAAO,CAAC,GAAG,CACf,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;QAC7B,MAAM,UAAU,GAAG,mBAAmB,CAAC,MAAM,CAAC,MAAM,CAAC,SAAS,CAC5D,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,OAAO,CACxB,CAAC;QAEF,MAAM,aAAa,GAAG;YACpB,GAAG,mBAAmB,CAAC,MAAM;YAC7B,MAAM,EAAE,CAAC,mBAAmB,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAE,CAAC;YACxD,IAAI,EAAE,CAAC,mBAAmB,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAE,CAAC;YACpD,eAAe,EAAE;gBACf,mBAAmB,CAAC,MAAM,CAAC,eAAe,CAAC,UAAU,CAAE;aACxD;YACD,cAAc,EAAE;gBACd,mBAAmB,CAAC,MAAM,CAAC,cAAc,CAAC,UAAU,CAAE;aACvD;YACD,cAAc,EAAE;gBACd,mBAAmB,CAAC,MAAM,CAAC,cAAc,CAAC,UAAU,CAAE;aACvD;YACD,SAAS,EAAE,CAAC,mBAAmB,CAAC,MAAM,CAAC,SAAS,CAAC,UAAU,CAAE,CAAC;SAC/D,CAAC;QAEF,MAAM,CAAC,OAAO,CAAC,yBAAyB,CAAC;YACvC,aAAa;YACb,WAAW,EAAE,iBAAiB,CAAC,MAAM;SACtC,CAAC,CAAC;QAEH,MAAM,WAAW,CAAC;YAChB,MAAM;YACN,QAAQ,EAAE,cAAc,CAAC,MAAM;YAC/B,cAAc;YACd,WAAW,EAAE,iBAAiB,CAAC,MAAM;YACrC,aAAa;YACb,uBAAuB;YACvB,QAAQ;YACR,OAAO,EAAE,GAAG,EAAE;gBACZ,UAAW,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;YACtD,CAAC;SACF,CAAC,CAAC;QAEH,UAAW,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC;IACrD,CAAC,CAAC,CACH,CAAC;AACJ,CAAC"}