univo 0.2.18 → 0.2.19

package/dist/cli.mjs

@@ -4,8 +4,8 @@ import open from 'open';
 import { config } from 'dotenv';
 import { WebSocket } from 'partysocket';
 import { defineCommand, runMain } from 'citty';
-import { h as http } from './shared/univo.GuW0GiSl.mjs';
-import { c as createLogger, v as version, b as name } from './shared/univo.TYA11b8i.mjs';
+import { h as http } from './shared/univo.D1x0MkrS.mjs';
+import { c as createLogger, v as version, b as name } from './shared/univo.bwpFiVa7.mjs';
 
 const log = createLogger({ quiet: false });
 const dev = defineCommand({

package/dist/index.d.mts

@@ -1,2 +1,2 @@
-export { B as Block, E as Event, a as Filter, H as Head, I as Indexer, M as Metadata, R as Result, i as indexer } from './shared/univo.BNqa7KED.mjs';
 import '@storagesdk/core';
+export { B as Block, E as Event, F as Filter, H as Head, I as Indexer, M as Metadata, R as Result, i as indexer } from './rpc.mjs';

package/dist/index.d.ts

@@ -1,2 +1,2 @@
-export { B as Block, E as Event, a as Filter, H as Head, I as Indexer, M as Metadata, R as Result, i as indexer } from './shared/univo.BNqa7KED.js';
 import '@storagesdk/core';
+export { B as Block, E as Event, F as Filter, H as Head, I as Indexer, M as Metadata, R as Result, i as indexer } from './rpc.js';

package/dist/index.mjs

@@ -1,6 +1,6 @@
 import { StorageError } from '@storagesdk/core';
-import { g as getException, c as createException, a as catchException, l as local } from './shared/univo.GuW0GiSl.mjs';
-import { c as createLogger, d as decompress, e as decoder, r as retry, v as version, h as hexToNumber, a as isHexEqual, f as normalizeHex, g as compress } from './shared/univo.TYA11b8i.mjs';
+import { g as getException, c as createException, a as catchException, l as local } from './shared/univo.D1x0MkrS.mjs';
+import { c as createLogger, d as decompress, e as decoder, r as retry, v as version, h as hexToNumber, a as isHexEqual, n as numberToHex, f as normalizeHex, g as compress } from './shared/univo.bwpFiVa7.mjs';
 import 'ws';
 import 'partysocket';
 
@@ -136,36 +136,31 @@ function indexer(opts) {
   const log = createLogger({ quiet: opts.quiet ?? false, prefix: "[indexer]" });
   const all_events = [];
   const events_grouped_by_storage_map = /* @__PURE__ */ new Map();
+  const all_actions = [];
   const metadata = {
+    // The blocks store essentially operates as our write-ahead-log (WAL) for unfinalized blocks. Before we
+    // commit any events to storage we first record the block input in our metadata layer. This allows our
+    // finalization handler to always have access to the input data in the case we need to remove events
+    // because of chain reorganisations
     blocks: {
-      async getNextUnfinalizedBlock(chain) {
-        const prefix = `blocks/v1/${normalizeHex(chain)}`;
-        const keys = await opts.metadataStorage.list({ prefix, limit: 1 });
-        const [key] = keys.items;
-        if (key === void 0) {
-          return null;
+      async list(chain, number) {
+        let prefix = `blocks/v1/${normalizeHex(chain)}`;
+        if (typeof number === "string") {
+          prefix += `/${normalizeHex(number, 16)}`;
         }
-        const [_, __, ___, number, hash] = key.path.split("/");
-        return { chain, number, hash };
-      },
-      async getBlocksByNumber(chain, number) {
-        const prefix = `blocks/v1/${normalizeHex(chain)}/${normalizeHex(number, 16)}`;
         const keys = await opts.metadataStorage.list({ prefix });
-        if (keys.items.length === 0) {
-          return [];
-        }
-        const blocks = await Promise.all(
-          keys.items.map(async (item) => {
-            const blob = await opts.metadataStorage.download(item.path, { as: "blob" });
-            const block = await decompress(blob);
-            const parsed = JSON.parse(block);
-            return parsed;
-          })
-        );
-        return blocks;
+        const mapped = keys.items.map((key) => {
+          const [_, __, ___, number2, hash, parent_hash] = key.path.split("/");
+          return { chain, number: number2, hash, parent_hash };
+        });
+        return mapped;
       },
-      async getBlockByNumberAndHash(chain, number, hash) {
-        const prefix = `blocks/v1/${normalizeHex(chain)}/${normalizeHex(number, 16)}/${normalizeHex(hash)}`;
+      async get(head) {
+        const chain = normalizeHex(head.chain);
+        const number = normalizeHex(head.number, 16);
+        const hash = normalizeHex(head.hash);
+        const parentHash = normalizeHex(head.parent_hash);
+        const prefix = `blocks/v1/${chain}/${number}/${hash}/${parentHash}`;
         const blob = await opts.metadataStorage.download(prefix, { as: "blob" }).catch((error) => {
           if (error instanceof StorageError) {
             if (error.code === "NotFound") {
@@ -188,9 +183,10 @@ function indexer(opts) {
         await Promise.all(
           blocks.map(async (block) => {
             const chain = normalizeHex(block.eth_chainId);
-            const hash = normalizeHex(block.eth_getBlockByNumber.hash);
             const number = normalizeHex(block.eth_getBlockByNumber.number, 16);
-            const prefix = `blocks/v1/${chain}/${number}/${hash}`;
+            const hash = normalizeHex(block.eth_getBlockByNumber.hash);
+            const parentHash = normalizeHex(block.eth_getBlockByNumber.parentHash);
+            const prefix = `blocks/v1/${chain}/${number}/${hash}/${parentHash}`;
             const compressed = await compress(JSON.stringify(block));
             await opts.metadataStorage.upload(prefix, compressed);
           })
@@ -202,14 +198,75 @@ function indexer(opts) {
         }
         await Promise.all(
           blocks.map(async (block) => {
-            const chain = normalizeHex(block.eth_chainId);
-            const hash = normalizeHex(block.eth_getBlockByNumber.hash);
-            const number = normalizeHex(block.eth_getBlockByNumber.number, 16);
-            const prefix = `blocks/v1/${chain}/${number}/${hash}`;
+            const chain = normalizeHex(block.chain);
+            const number = normalizeHex(block.number, 16);
+            const hash = normalizeHex(block.hash);
+            const parentHash = normalizeHex(block.parent_hash);
+            const prefix = `blocks/v1/${chain}/${number}/${hash}/${parentHash}`;
+            await opts.metadataStorage.delete(prefix);
+          })
+        );
+      }
+    },
+    // When unfinalized events are upserted we commit a given block number and block hash to indicate
+    // that those events were successfully recorded. This enables an optimisation in our finalized
+    // handler that allows it to skip processing blocks that were successfully processed
+    commits: {
+      async list(chain, number) {
+        let prefix = `commits/v1/${normalizeHex(chain)}`;
+        if (typeof number === "string") {
+          prefix += `/${normalizeHex(number, 16)}`;
+        }
+        const keys = await opts.metadataStorage.list({ prefix });
+        const mapped = keys.items.map((key) => {
+          const [_, __, ___, number2, hash, parent_hash] = key.path.split("/");
+          return { chain, number: number2, hash, parent_hash };
+        });
+        return mapped;
+      },
+      async upsert(head) {
+        const chain = normalizeHex(head.chain);
+        const number = normalizeHex(head.number, 16);
+        const hash = normalizeHex(head.hash);
+        const parentHash = normalizeHex(head.parent_hash);
+        const prefix = `commits/v1/${chain}/${number}/${hash}/${parentHash}`;
+        const body = JSON.stringify({ hello: "world" });
+        await opts.metadataStorage.upload(prefix, body);
+      },
+      async delete(commits) {
+        await Promise.all(
+          commits.map(async (commit) => {
+            const chain = normalizeHex(commit.chain);
+            const number = normalizeHex(commit.number, 16);
+            const hash = normalizeHex(commit.hash);
+            const parentHash = normalizeHex(commit.parent_hash);
+            const prefix = `commits/v1/${chain}/${number}/${hash}/${parentHash}`;
             await opts.metadataStorage.delete(prefix);
           })
         );
       }
+    },
+    // Our indexer always lags behind the canonical chain. The heights table is used as a durable record
+    // to track its own finalized height. As the chain finalizes, we process newly finalized blocks and
+    // then update the heights here to reflect to that outside world what height the indexer has currently
+    // finalized up to. This information is stored in a separate table so that we can `list` keys and
+    // immediately know which blocks were finalized correctly without ever having to load the entire block
+    // values themselves reducing memory usage and increasing finalizing throughput
+    heights: {
+      async list(chain) {
+        const prefix = `heights/v1/${normalizeHex(chain)}`;
+        const keys = await opts.metadataStorage.list({ prefix });
+        const mapped = keys.items.map((key) => {
+          const [_, __, ___, number] = key.path.split("/");
+          return { chain, number };
+        });
+        return mapped;
+      },
+      async upsert(chain, number) {
+        const prefix = `heights/v1/${normalizeHex(chain)}/${normalizeHex(number, 16)}`;
+        const body = JSON.stringify({ hello: "world" });
+        await opts.metadataStorage.upload(prefix, body);
+      }
     }
   };
   async function getBlock(head) {
@@ -237,21 +294,18 @@ function indexer(opts) {
   }
   const GetBlockError = createException("Failed to load block from the provided `getBlock` function");
   const public_getFinalizedHeight = async (chain) => {
-    const [stored, finalizedBlock] = await Promise.all([
-      metadata.blocks.getNextUnfinalizedBlock(chain),
-      //
-      getBlock({ chain, number: "finalized" })
-    ]);
-    if (finalizedBlock === null) {
-      log.debug("Failed to fetch finalized block and unable to determine finalized height, aborting...");
-      throw new Error(GetBlockError);
-    }
-    const finalizedHeight = hexToNumber(finalizedBlock.eth_getBlockByNumber.number);
-    if (stored === null) {
-      return finalizedHeight;
+    const heights = await metadata.heights.list(chain);
+    if (heights.length === 0) {
+      const block = await getBlock({ chain, number: "finalized" });
+      if (block === null) {
+        log.debug("Failed to fetch finalized block and unable to determine finalized height, aborting...");
+        throw new Error(GetBlockError);
+      }
+      await metadata.heights.upsert(chain, block.eth_getBlockByNumber.number);
+      return hexToNumber(block.eth_getBlockByNumber.number);
     }
-    const indexerHeight = hexToNumber(stored.number) - 1;
-    return Math.min(indexerHeight, finalizedHeight);
+    const height = Math.max(...heights.map((height2) => hexToNumber(height2.number)));
+    return height;
   };
   const public_writeUnfinalizedHead = async (head) => {
     log.debug("Received unfinalized head...");
@@ -276,7 +330,7 @@ function indexer(opts) {
     await metadata.blocks.upsert([block]);
     log.debug(`Persisted block to metadata in ${Date.now() - metadata_start}ms`);
     const events_start = Date.now();
-    const promises = events_grouped_by_storage_map.entries().map(async ([storage, grouped_events]) => {
+    const events = events_grouped_by_storage_map.entries().map(async ([storage, grouped_events]) => {
       const batch = [];
       for (const event2 of grouped_events) {
         try {
@@ -284,8 +338,8 @@ function indexer(opts) {
             log.debug(`Block matches no filters for event ${event2.id}`);
             continue;
           }
-          const events = event2.handler(block);
-          for (const event3 of events) {
+          const events2 = event2.handler(block);
+          for (const event3 of events2) {
             batch.push(event3);
           }
         } catch (error) {
@@ -306,14 +360,28 @@ function indexer(opts) {
         }
       }
     });
-    await Promise.all(promises);
-    log.debug(`Wrote head in ${Date.now() - events_start}ms`);
+    await Promise.all(events);
+    log.debug(`Wrote events in ${Date.now() - events_start}ms`);
+    await metadata.commits.upsert(head);
+    const actions = all_actions.flatMap((action2) => {
+      if (typeof action2.handler.latest === "undefined") {
+        return [];
+      }
+      return action2.event.handler(block).map(async (event2) => {
+        await retry(() => action2.handler.latest(event2), 2).catch((error) => {
+          log.error(`Failed to execute 'latest' action ${action2.id}`);
+          throw error;
+        });
+      });
+    });
+    await Promise.allSettled(actions);
   };
   const public_deleteReorganisedHead = async (head) => {
     log.debug(`Received reorganised head ${hexToNumber(head.number)}`);
-    const [canonical_block, stored_block] = await Promise.all([
-      getBlock({ chain: head.chain, number: head.number }),
-      metadata.blocks.getBlockByNumberAndHash(head.chain, head.number, head.hash)
+    const [stored_block, canonical_block] = await Promise.all([
+      metadata.blocks.get(head),
+      //
+      getBlock({ chain: head.chain, number: head.number })
     ]);
     if (stored_block === null) {
       return log.debug("Reorganised block never/already processed");
@@ -373,7 +441,7 @@ function indexer(opts) {
   };
   const public_writeFinalizedHeads = async (heads) => {
     if (heads.length === 0) {
-      return log.debug("No heads received, returning...");
+      throw new Error("No heads received, aborting...");
     }
     log.debug(`Received ${heads.length} finalized heads...`);
     let chain = void 0;
@@ -402,8 +470,8 @@ function indexer(opts) {
       }
       previousHead = head;
     }
-    const [unfinalizedMetadataBlock, finalizedBlock] = await Promise.all([
-      metadata.blocks.getNextUnfinalizedBlock(chain),
+    const [heights, finalizedBlock] = await Promise.all([
+      metadata.heights.list(chain),
       //
       getBlock({ chain, number: "finalized" })
     ]);
@@ -411,56 +479,82 @@ function indexer(opts) {
       log.debug("Failed to load chain finalized block. Request should be retried");
       throw new Error(GetBlockError);
     }
-    const finalizedHeight = hexToNumber(finalizedBlock.eth_getBlockByNumber.number);
-    for (const head of heads) {
-      if (hexToNumber(head.number) > finalizedHeight) {
-        throw new Error("Received head greater than chain finalized height");
-      }
+    if (heights.length === 0) {
+      await metadata.heights.upsert(chain, finalizedBlock.eth_getBlockByNumber.number);
+      heights.push({ chain, number: finalizedBlock.eth_getBlockByNumber.number });
     }
-    if (unfinalizedMetadataBlock === null) {
-      return log.debug("No heads to finalize, returning...");
+    const indexerHeight = Math.max(...heights.map((height) => hexToNumber(height.number)));
+    const finalizedHeight = hexToNumber(finalizedBlock.eth_getBlockByNumber.number);
+    const lastHead = heads[heads.length - 1];
+    if (lastHead === void 0 || hexToNumber(lastHead.number) !== finalizedHeight) {
+      throw new Error("Client has stalled. Chain has finalized greater than the heads received");
     }
-    const indexerHeight = hexToNumber(unfinalizedMetadataBlock.number) - 1;
     log.debug(`Indexer height ${indexerHeight}, finalized height ${finalizedHeight}`);
     if (indexerHeight === finalizedHeight) {
-      return log.debug("No heads to finalize, returning...");
+      log.debug("No heads to finalize, returning...");
+      return await cleanupFinalizedHeads(chain, numberToHex(finalizedHeight));
     }
     const newHeads = heads.filter((head) => {
       return hexToNumber(head.number) > indexerHeight;
     });
     const [firstNewHead] = newHeads;
     if (firstNewHead === void 0) {
-      return log.debug("No new heads received, returning...");
+      throw new Error("Internal error. Expected at least one head greater than the indexer height.");
     }
     if (hexToNumber(firstNewHead.number) !== indexerHeight + 1) {
-      throw new Error(
-        `Client advanced too far (${hexToNumber(firstNewHead.number)}). Heads received are greater than the first new head ${indexerHeight + 1}`
-      );
+      throw new Error("Client advanced too far. Heads received are greater than the first new head");
     }
     for (const head of newHeads) {
-      const processed = await metadata.blocks.getBlocksByNumber(head.chain, head.number);
-      if (processed.length === 0) {
-        await writeFinalizedHead(head, null);
-        continue;
+      await Promise.all([
+        processEventsForCanonicalHead(head),
+        processActionsForCanonicalHead(head)
+        //
+      ]);
+      await metadata.heights.upsert(head.chain, head.number);
+    }
+    await cleanupFinalizedHeads(chain, numberToHex(finalizedHeight));
+  };
+  async function processActionsForCanonicalHead(head) {
+    if (all_actions.every((action2) => typeof action2.handler.finalized === "undefined")) {
+      return;
+    }
+    const block = await getBlock({ chain: head.chain, number: head.number, hash: head.hash });
+    const promises = all_actions.flatMap((action2) => {
+      if (typeof action2.handler.finalized === "undefined") {
+        return [];
       }
-      const reorganised = processed.filter((block2) => {
-        return !isHexEqual(head.hash, block2.eth_getBlockByNumber.hash);
+      return action2.event.handler(block).map(async (event2) => {
+        await retry(() => action2.handler.finalized(event2), 2).catch((error) => {
+          log.error(`Failed to execute 'finalized' action ${action2.id}`);
+          throw error;
+        });
       });
-      if (reorganised.length > 0) {
-        await deleteReorganisedBlocks(reorganised);
+    });
+    await Promise.all(promises);
+  }
+  async function processEventsForCanonicalHead(head) {
+    const [processed, commits] = await Promise.all([
+      metadata.blocks.list(head.chain, head.number),
+      //
+      metadata.commits.list(head.chain, head.number)
+    ]);
+    if (processed.length === 1) {
+      if (commits.some((commit) => isHexEqual(head.hash, commit.hash) && isHexEqual(head.parent_hash, commit.parent_hash))) {
+        return;
       }
-      const block = processed.find((block2) => {
-        return isHexEqual(head.hash, block2.eth_getBlockByNumber.hash);
-      });
-      await writeFinalizedHead(head, block ?? null);
-      await metadata.blocks.delete(processed);
-    }
-  };
-  async function writeFinalizedHead(head, block_nullable) {
-    let block = block_nullable;
-    if (block === null) {
-      block = await getBlock({ chain: head.chain, number: head.number, hash: head.hash });
     }
+    const reorganised = processed.filter((block) => {
+      return !isHexEqual(head.hash, block.hash);
+    });
+    await Promise.all(
+      reorganised.map(async (block) => {
+        await deleteReorganisedHead(block);
+      })
+    );
+    await writeFinalizedHead(head);
+  }
+  async function writeFinalizedHead(head) {
+    const block = await getBlock({ chain: head.chain, number: head.number, hash: head.hash });
     if (block === null) {
       throw new Error("Received unknown head that wasn't canonical");
     }
@@ -496,18 +590,17 @@ function indexer(opts) {
     });
     await Promise.all(promises);
   }
-  async function deleteReorganisedBlocks(blocks) {
-    if (blocks.length === 0) {
-      return;
+  async function deleteReorganisedHead(head) {
+    const block = await metadata.blocks.get(head);
+    if (block === null) {
+      throw new Error("Expected reorganised block to exist in metadata");
     }
     const promises = all_events.map(async (event2) => {
       const batch = [];
       try {
-        for (const block of blocks) {
-          const events = event2.handler(block);
-          for (const event3 of events) {
-            batch.push(event3);
-          }
+        const events = event2.handler(block);
+        for (const event3 of events) {
+          batch.push(event3);
         }
       } catch (error) {
         log.error(`Failed to run your 'handler' for event ${event2.id}`);
@@ -523,6 +616,24 @@ function indexer(opts) {
     });
     await Promise.all(promises);
   }
+  async function cleanupFinalizedHeads(chain, finalized) {
+    const [_blocks, _commits] = await Promise.all([
+      metadata.blocks.list(chain),
+      //
+      metadata.commits.list(chain)
+    ]);
+    const blocks = _blocks.filter((block) => {
+      return hexToNumber(block.number) <= hexToNumber(finalized);
+    });
+    const commits = _commits.filter((commit) => {
+      return hexToNumber(commit.number) < hexToNumber(finalized);
+    });
+    await Promise.all([
+      metadata.blocks.delete(blocks),
+      //
+      metadata.commits.delete(commits)
+    ]);
+  }
   const private_getMetadata = async () => {
     return {
       version,
@@ -795,6 +906,10 @@ function indexer(opts) {
     events_grouped_by_storage_map.set(event2.storage, group);
     return event2;
   };
+  const action = (action2) => {
+    all_actions.push(action2);
+    return action2;
+  };
   const rpc = {
     request: {
       public_getFinalizedHeight,
@@ -816,6 +931,7 @@ function indexer(opts) {
   const indexer2 = {
     ...rpc,
     event,
+    action,
     fetch: server.http
   };
   return indexer2;

package/dist/realtime.d.mts

@@ -1,6 +1,5 @@
 import { NodeRpc, IndexerRpc } from './rpc.mjs';
 import { Transport } from './transport.mjs';
-import './shared/univo.BNqa7KED.mjs';
 import '@storagesdk/core';
 
 type RealtimeOptions = {

package/dist/realtime.d.ts

@@ -1,6 +1,5 @@
 import { NodeRpc, IndexerRpc } from './rpc.js';
 import { Transport } from './transport.js';
-import './shared/univo.BNqa7KED.js';
 import '@storagesdk/core';
 
 type RealtimeOptions = {

package/dist/realtime.mjs

@@ -1,4 +1,4 @@
-import { c as createLogger, i as iife, h as hexToNumber, n as numberToHex, m as mutex, r as retry, a as isHexEqual } from './shared/univo.TYA11b8i.mjs';
+import { c as createLogger, i as iife, h as hexToNumber, n as numberToHex, m as mutex, r as retry, a as isHexEqual } from './shared/univo.bwpFiVa7.mjs';
 
 function defineBlockchain(opts) {
   const log = createLogger({ quiet: opts.quiet });
@@ -165,7 +165,7 @@ function realtime(opts) {
     });
     log.debug("Subscribed indexer chain to new heads");
     let indexerHeight = initialIndexerHeight;
-    const poll = async () => {
+    async function poll() {
       try {
         log.debug("Polling for finalized height...");
         const finalizedBlock = await opts.node.request({
@@ -189,7 +189,11 @@ function realtime(opts) {
           return log.debug("No new finalized heads to deliver, ignoring...");
         }
         log.debug(`Delivering ${newHeads.length} finalized head(s)...`);
-        await opts.indexer.request({ method: "public_writeFinalizedHeads", params: [newHeads] });
+        await opts.indexer.request({
+          params: [newHeads],
+          method: "public_writeFinalizedHeads",
+          signal: AbortSignal.timeout(Number.POSITIVE_INFINITY)
+        });
         log.debug(`Delivered ${newHeads.length} finalized head(s)`);
         indexerHeight = hexToNumber(finalizedBlock.number);
         log.debug(`Updated indexer height to ${indexerHeight}`);
@@ -207,11 +211,9 @@ function realtime(opts) {
         if (error instanceof Error) {
           log.error(`Failed to reconcile finalized head: ${error.message}`);
         }
-      } finally {
-        setTimeout(poll, POLLING_INTERVAL_MS);
       }
-    };
-    await poll();
+    }
+    setInterval(mutex(poll), POLLING_INTERVAL_MS);
     log.debug("Started polling finalized height");
   });
   promise.catch((error) => {