univo 0.2.23 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.mjs CHANGED
@@ -1,6 +1,7 @@
1
- import { StorageError } from '@storagesdk/core';
2
- import { g as getException, c as createException, a as catchException, l as local } from './shared/univo.1clH0g-n.mjs';
3
- import { c as createLogger, d as decompress, e as decoder, r as retry, v as version, h as hexToNumber, a as isHexEqual, n as numberToHex, f as normalizeHex, g as compress } from './shared/univo.DvxWs2zB.mjs';
1
+ import * as viem from 'viem';
2
+ import { g as getException, c as createException, a as catchException, l as local } from './shared/univo.TsC-vBTq.mjs';
3
+ import { c as createLogger, d as decompress, b as decoder, r as retry, v as version, e as normalizeHex, a as isHexEqual, f as compress } from './shared/univo.CX4GZB_1.mjs';
4
+ import { AdapterError } from './metadata/adapters.mjs';
4
5
  import 'ws';
5
6
  import 'partysocket';
6
7
 
@@ -102,16 +103,16 @@ function matchFilter(block, filter) {
102
103
  return true;
103
104
  }
104
105
  const chainValid = (block, filter) => {
105
- if (hexToNumber(block.eth_chainId) === filter.chain) return true;
106
+ if (viem.hexToNumber(block.eth_chainId) === filter.chain) return true;
106
107
  return false;
107
108
  };
108
109
  const fromBlockValid = (block, filter) => {
109
- if (hexToNumber(block.eth_getBlockByNumber.number) >= filter.fromBlock) return true;
110
+ if (viem.hexToNumber(block.eth_getBlockByNumber.number) >= filter.fromBlock) return true;
110
111
  return false;
111
112
  };
112
113
  const toBlockValid = (block, filter) => {
113
114
  if (filter.toBlock === void 0) return true;
114
- if (hexToNumber(block.eth_getBlockByNumber.number) <= filter.toBlock) return true;
115
+ if (viem.hexToNumber(block.eth_getBlockByNumber.number) <= filter.toBlock) return true;
115
116
  return false;
116
117
  };
117
118
  const includesLogAddress = (block, filter) => {
@@ -132,213 +133,243 @@ const includesLogEvent = (block, filter) => {
132
133
  }
133
134
  return false;
134
135
  };
135
- function indexer(opts) {
136
+ function defineIndexer(opts) {
136
137
  const log = createLogger({ quiet: opts.quiet ?? false, prefix: "[indexer]" });
137
- const all_events = [];
138
- const events_grouped_by_storage_map = /* @__PURE__ */ new Map();
139
- const all_actions = [];
140
- const metadata = {
141
- // The blocks store essentially operates as our write-ahead-log (WAL) for unfinalized blocks. Before we
142
- // commit any events to storage we first record the block input in our metadata layer. This allows our
143
- // finalization handler to always have access to the input data in the case we need to remove events
144
- // because of chain reorganisations
145
- blocks: {
146
- async list(chain, number) {
147
- let prefix = `blocks/v1/${normalizeHex(chain)}`;
148
- if (typeof number === "string") {
149
- prefix += `/${normalizeHex(number, 16)}`;
138
+ const allEvents = [];
139
+ const eventsGroupedByStorageMap = /* @__PURE__ */ new Map();
140
+ const allActions = [];
141
+ async function getBlockFromChain(head) {
142
+ try {
143
+ const block = await opts.getBlock({ chain: head.chain, number: head.number });
144
+ if (block === null) {
145
+ throw new Error("Provided `getBlock` function returned null");
146
+ }
147
+ if (typeof head.hash === "string") {
148
+ if (!isHexEqual(head.hash, block.eth_getBlockByNumber.hash)) {
149
+ throw new Error("Method `eth_getBlockByNumber` returned unexpected block hash");
150
150
  }
151
- const keys = await opts.metadataStorage.list({ prefix });
152
- const mapped = keys.items.map((key) => {
153
- const [_, __, ___, number2, hash, parent_hash] = key.path.split("/");
154
- return { chain, number: number2, hash, parent_hash };
155
- });
156
- return mapped;
157
- },
158
- async get(head) {
159
- const chain = normalizeHex(head.chain);
160
- const number = normalizeHex(head.number, 16);
161
- const hash = normalizeHex(head.hash);
162
- const parentHash = normalizeHex(head.parent_hash);
163
- const prefix = `blocks/v1/${chain}/${number}/${hash}/${parentHash}`;
164
- const blob = await opts.metadataStorage.download(prefix, { as: "blob" }).catch((error) => {
165
- if (error instanceof StorageError) {
166
- if (error.code === "NotFound") {
167
- return null;
168
- }
169
- }
170
- throw error;
171
- });
172
- if (blob === null) {
173
- return null;
151
+ }
152
+ if (typeof head.parent_hash === "string") {
153
+ if (!isHexEqual(head.parent_hash, block.eth_getBlockByNumber.parentHash)) {
154
+ throw new Error("Method `eth_getBlockByNumber` returned unexpected parent hash");
174
155
  }
175
- const block = await decompress(blob);
176
- const parsed = JSON.parse(block);
177
- return parsed;
178
- },
179
- async upsert(blocks) {
180
- if (blocks.length === 0) {
181
- return;
156
+ }
157
+ verifyBlockHashes(block);
158
+ verifyLogIndicies(block);
159
+ verifyReceiptsRoot(block);
160
+ verifyTransactionsRoot(block);
161
+ verifyTransactionIndicies(block);
162
+ verifyTransactionGasUsage(block);
163
+ return block;
164
+ } catch (error) {
165
+ if (error instanceof Error) {
166
+ log.error(`Failed to load block: ${error.message}`);
167
+ }
168
+ throw error;
169
+ }
170
+ }
171
+ function verifyBlockHashes(block) {
172
+ const blockHash = block.eth_getBlockByNumber.hash;
173
+ const transactions = block.eth_getBlockByNumber.transactions;
174
+ for (const transaction of transactions) {
175
+ if (!isHexEqual(blockHash, transaction.blockHash)) {
176
+ throw new Error("Method `eth_getBlockByNumber` returned transaction with unexpected block hash");
177
+ }
178
+ }
179
+ for (const receipt of block.eth_getBlockReceipts) {
180
+ if (!isHexEqual(block.eth_getBlockByNumber.hash, receipt.blockHash)) {
181
+ throw new Error("Method `eth_getBlockReceipts` returned receipt with unexpected block hash");
182
+ }
183
+ const transactionIndex = viem.hexToNumber(receipt.transactionIndex);
184
+ const transaction = transactions[transactionIndex];
185
+ if (transaction === void 0 || !isHexEqual(transaction.hash, receipt.transactionHash)) {
186
+ throw new Error("Method `eth_getBlockReceipts` returned receipt with unexpected transaction hash");
187
+ }
188
+ for (const entry of receipt.logs) {
189
+ if (!isHexEqual(blockHash, entry.blockHash)) {
190
+ throw new Error("Method `eth_getBlockReceipts` returned log with unexpected block hash");
182
191
  }
183
- await Promise.all(
184
- blocks.map(async (block) => {
185
- const chain = normalizeHex(block.eth_chainId);
186
- const number = normalizeHex(block.eth_getBlockByNumber.number, 16);
187
- const hash = normalizeHex(block.eth_getBlockByNumber.hash);
188
- const parentHash = normalizeHex(block.eth_getBlockByNumber.parentHash);
189
- const prefix = `blocks/v1/${chain}/${number}/${hash}/${parentHash}`;
190
- const compressed = await compress(JSON.stringify(block));
191
- await opts.metadataStorage.upload(prefix, compressed);
192
- })
193
- );
194
- },
195
- async delete(blocks) {
196
- if (blocks.length === 0) {
197
- return;
192
+ if (!isHexEqual(receipt.transactionHash, entry.transactionHash)) {
193
+ throw new Error("Method `eth_getBlockReceipts` returned log with unexpected transaction hash");
198
194
  }
199
- await Promise.all(
200
- blocks.map(async (block) => {
201
- const chain = normalizeHex(block.chain);
202
- const number = normalizeHex(block.number, 16);
203
- const hash = normalizeHex(block.hash);
204
- const parentHash = normalizeHex(block.parent_hash);
205
- const prefix = `blocks/v1/${chain}/${number}/${hash}/${parentHash}`;
206
- await opts.metadataStorage.delete(prefix);
207
- })
208
- );
209
195
  }
210
- },
211
- // When unfinalized events are upserted we commit a given block number and block hash to indicate
212
- // that those events were successfully recorded. This enables an optimisation in our finalized
213
- // handler that allows it to skip processing blocks that were successfully processed
214
- commits: {
215
- async list(chain, number) {
216
- let prefix = `commits/v1/${normalizeHex(chain)}`;
217
- if (typeof number === "string") {
218
- prefix += `/${normalizeHex(number, 16)}`;
196
+ }
197
+ }
198
+ function verifyLogIndicies(block) {
199
+ let expectedLogIndex = 0n;
200
+ for (const receipt of block.eth_getBlockReceipts) {
201
+ for (const entry of receipt.logs) {
202
+ if (viem.hexToBigInt(entry.logIndex) !== expectedLogIndex) {
203
+ throw new Error("Method `eth_getBlockReceipts` returned non-contiguous log indices");
219
204
  }
220
- const keys = await opts.metadataStorage.list({ prefix });
221
- const mapped = keys.items.map((key) => {
222
- const [_, __, ___, number2, hash, parent_hash] = key.path.split("/");
223
- return { chain, number: number2, hash, parent_hash };
224
- });
225
- return mapped;
226
- },
227
- async upsert(head) {
228
- const chain = normalizeHex(head.chain);
229
- const number = normalizeHex(head.number, 16);
230
- const hash = normalizeHex(head.hash);
231
- const parentHash = normalizeHex(head.parent_hash);
232
- const prefix = `commits/v1/${chain}/${number}/${hash}/${parentHash}`;
233
- const body = JSON.stringify({ hello: "world" });
234
- await opts.metadataStorage.upload(prefix, body);
235
- },
236
- async delete(commits) {
237
- await Promise.all(
238
- commits.map(async (commit) => {
239
- const chain = normalizeHex(commit.chain);
240
- const number = normalizeHex(commit.number, 16);
241
- const hash = normalizeHex(commit.hash);
242
- const parentHash = normalizeHex(commit.parent_hash);
243
- const prefix = `commits/v1/${chain}/${number}/${hash}/${parentHash}`;
244
- await opts.metadataStorage.delete(prefix);
245
- })
246
- );
205
+ expectedLogIndex++;
247
206
  }
248
- },
249
- // Our indexer always lags behind the canonical chain. The heights table is used as a durable record
250
- // to track its own finalized height. As the chain finalizes, we process newly finalized blocks and
251
- // then update the heights here to reflect to that outside world what height the indexer has currently
252
- // finalized up to. This information is stored in a separate table so that we can `list` keys and
253
- // immediately know which blocks were finalized correctly without ever having to load the entire block
254
- // values themselves reducing memory usage and increasing finalizing throughput
255
- heights: {
256
- async list(chain) {
257
- const prefix = `heights/v1/${normalizeHex(chain)}`;
258
- const keys = await opts.metadataStorage.list({ prefix });
259
- const mapped = keys.items.map((key) => {
260
- const [_, __, ___, number] = key.path.split("/");
261
- return { chain, number };
262
- });
263
- return mapped;
264
- },
265
- async upsert(chain, number) {
266
- const prefix = `heights/v1/${normalizeHex(chain)}/${normalizeHex(number, 16)}`;
267
- const body = JSON.stringify({ hello: "world" });
268
- await opts.metadataStorage.upload(prefix, body);
269
- },
270
- async delete(heights) {
271
- await Promise.all(
272
- heights.map(async (height) => {
273
- const prefix = `heights/v1/${normalizeHex(height.chain)}/${normalizeHex(height.number, 16)}`;
274
- await opts.metadataStorage.delete(prefix);
275
- })
276
- );
207
+ }
208
+ }
209
+ function verifyReceiptsRoot(block) {
210
+ const receiptsRoot = calculateTrieRoot(block.eth_getBlockReceipts.map(serializeReceipt));
211
+ if (!isHexEqual(block.eth_getBlockByNumber.receiptsRoot, receiptsRoot)) {
212
+ throw new Error("Method `eth_getBlockReceipts` returned receipts that do not match the block receipts root");
213
+ }
214
+ }
215
+ function calculateTrieRoot(values) {
216
+ if (values.length === 0) {
217
+ return viem.keccak256(viem.toRlp(new Uint8Array(), "bytes"));
218
+ }
219
+ const entries = values.map((value, index) => ({
220
+ key: bytesToNibbles(viem.toRlp(quantityToBytes(BigInt(index)), "bytes")),
221
+ value
222
+ }));
223
+ return viem.keccak256(viem.toRlp(encodeTrieNode(entries), "bytes"));
224
+ }
225
+ function trieNodeReference(node) {
226
+ const encoded = viem.toRlp(node, "bytes");
227
+ return encoded.length < 32 ? node : viem.hexToBytes(viem.keccak256(encoded));
228
+ }
229
+ function encodeTrieNode(entries, depth = 0) {
230
+ if (entries.length === 1) {
231
+ const entry = entries[0];
232
+ return [encodePath(entry.key.slice(depth), true), entry.value];
233
+ }
234
+ let shared = 0;
235
+ while (entries.every((entry) => entry.key[depth + shared] === entries[0].key[depth + shared])) {
236
+ shared++;
237
+ }
238
+ if (shared > 0) {
239
+ const child = encodeTrieNode(entries, depth + shared);
240
+ return [encodePath(entries[0].key.slice(depth, depth + shared), false), trieNodeReference(child)];
241
+ }
242
+ const children = Array.from({ length: 17 }, () => new Uint8Array());
243
+ for (let nibble = 0; nibble < 16; nibble++) {
244
+ const matching = entries.filter((entry) => entry.key[depth] === nibble);
245
+ if (matching.length > 0) {
246
+ children[nibble] = trieNodeReference(encodeTrieNode(matching, depth + 1));
277
247
  }
278
248
  }
279
- };
280
- async function getBlock(head) {
281
- return await retry(() => retry_getBlock(head), 2).catch(() => {
282
- log.error("Failed to load block from the provided `getBlock` function after 3 attempts");
283
- return null;
249
+ const value = entries.find((entry) => entry.key.length === depth)?.value;
250
+ if (value !== void 0) {
251
+ children[16] = value;
252
+ }
253
+ return children;
254
+ }
255
+ function encodePath(path, leaf) {
256
+ const odd = path.length % 2 === 1;
257
+ const nibbles = odd ? [leaf ? 3 : 1, ...path] : [leaf ? 2 : 0, 0, ...path];
258
+ const bytes = new Uint8Array(nibbles.length / 2);
259
+ for (let index = 0; index < nibbles.length; index += 2) {
260
+ bytes[index / 2] = nibbles[index] << 4 | nibbles[index + 1];
261
+ }
262
+ return bytes;
263
+ }
264
+ function quantityToBytes(value) {
265
+ const number = typeof value === "bigint" ? value : viem.hexToBigInt(value);
266
+ return number === 0n ? new Uint8Array() : viem.numberToBytes(number);
267
+ }
268
+ function bytesToNibbles(value) {
269
+ return Array.from(value).flatMap((byte) => [byte >> 4, byte & 15]);
270
+ }
271
+ function serializeReceipt(receipt) {
272
+ const outcome = receipt.status === void 0 ? viem.hexToBytes(receipt.root) : quantityToBytes(receipt.status);
273
+ const fields = [
274
+ outcome,
275
+ quantityToBytes(receipt.cumulativeGasUsed),
276
+ viem.hexToBytes(receipt.logsBloom),
277
+ receipt.logs.map((log2) => [viem.hexToBytes(log2.address), log2.topics.map(viem.hexToBytes), viem.hexToBytes(log2.data)])
278
+ ];
279
+ const encoded = viem.toRlp(fields, "bytes");
280
+ if (!viem.isHex(receipt.type)) {
281
+ throw new Error(`Unsupported receipt type ${receipt.type}`);
282
+ }
283
+ const type = viem.hexToBigInt(receipt.type);
284
+ return type === 0n ? encoded : viem.concatBytes([Uint8Array.of(Number(type)), encoded]);
285
+ }
286
+ function verifyTransactionsRoot(block) {
287
+ const transactions = block.eth_getBlockByNumber.transactions.map((transaction) => {
288
+ const { input, ...formatted } = viem.formatTransaction(transaction);
289
+ const signature = formatted.type === "legacy" ? { r: formatted.r, s: formatted.s, v: formatted.v } : { r: formatted.r, s: formatted.s, yParity: formatted.yParity };
290
+ const serialized = viem.serializeTransaction(
291
+ { ...formatted, data: input },
292
+ //
293
+ signature
294
+ );
295
+ if (!isHexEqual(transaction.hash, viem.keccak256(serialized))) {
296
+ throw new Error("Method `eth_getBlockByNumber` returned transaction with unexpected transaction hash");
297
+ }
298
+ return viem.hexToBytes(serialized);
284
299
  });
300
+ const transactionsRoot = calculateTrieRoot(transactions);
301
+ if (!isHexEqual(block.eth_getBlockByNumber.transactionsRoot, transactionsRoot)) {
302
+ throw new Error("Method `eth_getBlockByNumber` returned transactions that do not match the block transactions root");
303
+ }
285
304
  }
286
- async function retry_getBlock(head) {
287
- const block = await opts.getBlock({ chain: head.chain, number: head.number });
288
- if (block === null) {
289
- throw new Error("Provided `getBlock` function returned null");
290
- }
291
- if (typeof head.hash === "string") {
292
- if (!isHexEqual(head.hash, block.eth_getBlockByNumber.hash)) {
293
- throw new Error("Method `eth_getBlockByNumber` returned unexpected block hash");
294
- }
295
- for (const receipt of block.eth_getBlockReceipts) {
296
- if (!isHexEqual(head.hash, receipt.blockHash)) {
297
- throw new Error("Method `eth_getBlockReceipts` returned receipt with unexpected block hash");
305
+ function verifyTransactionIndicies(block) {
306
+ const receipts = block.eth_getBlockReceipts;
307
+ const transactions = block.eth_getBlockByNumber.transactions;
308
+ if (transactions.length !== receipts.length) {
309
+ throw new Error("Methods `eth_getBlockByNumber` and `eth_getBlockReceipts` returned different transaction counts");
310
+ }
311
+ for (let index = 0; index < transactions.length; index++) {
312
+ const expectedIndex = BigInt(index);
313
+ if (viem.hexToBigInt(transactions[index].transactionIndex) !== expectedIndex) {
314
+ throw new Error("Method `eth_getBlockByNumber` returned non-contiguous transaction indices");
315
+ }
316
+ if (viem.hexToBigInt(receipts[index].transactionIndex) !== expectedIndex) {
317
+ throw new Error("Method `eth_getBlockReceipts` returned non-contiguous transaction indices");
318
+ }
319
+ for (const entry of receipts[index].logs) {
320
+ if (viem.hexToBigInt(entry.transactionIndex) !== expectedIndex) {
321
+ throw new Error("Method `eth_getBlockReceipts` returned log with unexpected transaction index");
298
322
  }
299
323
  }
300
324
  }
301
- return block;
302
325
  }
303
- const GetBlockError = createException("Failed to load block from the provided `getBlock` function");
304
- const public_getFinalizedHeight = async (chain) => {
305
- const heights = await metadata.heights.list(chain);
306
- if (heights.length === 0) {
307
- const block = await getBlock({ chain, number: "finalized" });
308
- if (block === null) {
309
- log.debug("Failed to fetch finalized block and unable to determine finalized height, aborting...");
310
- throw new Error(GetBlockError);
311
- }
312
- await metadata.heights.upsert(chain, block.eth_getBlockByNumber.number);
313
- return hexToNumber(block.eth_getBlockByNumber.number);
326
+ function verifyTransactionGasUsage(block) {
327
+ const gasLimit = viem.hexToBigInt(block.eth_getBlockByNumber.gasLimit);
328
+ const blockGasUsed = viem.hexToBigInt(block.eth_getBlockByNumber.gasUsed);
329
+ if (blockGasUsed > gasLimit) {
330
+ throw new Error("Method `eth_getBlockByNumber` returned gas used greater than the block gas limit");
314
331
  }
315
- const height = Math.max(...heights.map((height2) => hexToNumber(height2.number)));
316
- return height;
317
- };
318
- const public_writeUnfinalizedHead = async (head) => {
319
- log.debug("Received unfinalized head...");
320
- const blocks_start = Date.now();
321
- const [finalizedBlock, block] = await Promise.all([
322
- getBlock({ chain: head.chain, number: "finalized" }),
323
- getBlock({ chain: head.chain, number: head.number, hash: head.hash })
324
- ]);
325
- if (block === null) {
326
- return;
332
+ let cumulativeGasUsed = 0n;
333
+ for (const receipt of block.eth_getBlockReceipts) {
334
+ cumulativeGasUsed += viem.hexToBigInt(receipt.gasUsed);
335
+ if (viem.hexToBigInt(receipt.cumulativeGasUsed) !== cumulativeGasUsed) {
336
+ throw new Error("Method `eth_getBlockReceipts` returned inconsistent cumulative gas used");
337
+ }
338
+ if (cumulativeGasUsed > gasLimit) {
339
+ throw new Error("Method `eth_getBlockReceipts` returned cumulative gas used greater than the block gas limit");
340
+ }
327
341
  }
328
- if (finalizedBlock === null) {
329
- log.debug("Failed to determine finalized height when processing unfinalized heads");
330
- throw new Error(GetBlockError);
342
+ if (cumulativeGasUsed !== blockGasUsed) {
343
+ throw new Error("Methods `eth_getBlockByNumber` and `eth_getBlockReceipts` returned inconsistent gas used");
331
344
  }
332
- const finalizedHeight = hexToNumber(finalizedBlock.eth_getBlockByNumber.number);
333
- if (hexToNumber(head.number) <= finalizedHeight) {
334
- return;
345
+ }
346
+ async function getOrInitManifest(chain) {
347
+ const path = `manifest/v1/${normalizeHex(chain)}`;
348
+ const manifestRes = await opts.metadataStorage.adapter.get(path);
349
+ let manifest;
350
+ if (manifestRes === null) {
351
+ const block = await getBlockFromChain({ chain, number: "finalized" });
352
+ const chainFinalizedHeight = viem.hexToNumber(block.eth_getBlockByNumber.number);
353
+ const newManifest = {
354
+ finalized_block_height: chainFinalizedHeight,
355
+ finalized_block_hash: block.eth_getBlockByNumber.hash,
356
+ next_finalized_height: chainFinalizedHeight,
357
+ updated_at: Date.now()
358
+ };
359
+ await opts.metadataStorage.adapter.put(path, JSON.stringify(newManifest), { ifNoneMatch: "*" });
360
+ manifest = newManifest;
361
+ } else {
362
+ manifest = JSON.parse(decoder.decode(manifestRes.body));
335
363
  }
336
- log.debug(`Loaded block in ${Date.now() - blocks_start}ms`);
337
- const metadata_start = Date.now();
338
- await metadata.blocks.upsert([block]);
339
- log.debug(`Persisted block to metadata in ${Date.now() - metadata_start}ms`);
364
+ return manifest;
365
+ }
366
+ const public_getFinalizedHeight = async (chain) => {
367
+ const manifest = await getOrInitManifest(chain);
368
+ return manifest.finalized_block_height;
369
+ };
370
+ async function writeUnfinalizedBlock(block) {
340
371
  const events_start = Date.now();
341
- const events = events_grouped_by_storage_map.entries().map(async ([storage, grouped_events]) => {
372
+ const events = eventsGroupedByStorageMap.entries().map(async ([storage, grouped_events]) => {
342
373
  const batch = [];
343
374
  for (const event2 of grouped_events) {
344
375
  try {
@@ -370,42 +401,59 @@ function indexer(opts) {
370
401
  });
371
402
  await Promise.all(events);
372
403
  log.debug(`Wrote events in ${Date.now() - events_start}ms`);
373
- await metadata.commits.upsert(head);
374
- const actions = all_actions.flatMap((action2) => {
375
- if (typeof action2.handler.latest === "undefined") {
376
- return [];
377
- }
378
- return action2.event.handler(block).map(async (event2) => {
379
- await retry(() => action2.handler.latest(event2), 2).catch((error) => {
380
- log.error(`Failed to execute 'latest' action ${action2.id}`);
381
- throw error;
382
- });
383
- });
384
- });
385
- await Promise.allSettled(actions);
386
- };
387
- const public_deleteReorganisedHead = async (head) => {
388
- log.debug(`Received reorganised head ${hexToNumber(head.number)}`);
389
- const [stored_block, canonical_block] = await Promise.all([
390
- metadata.blocks.get(head),
404
+ const chain = normalizeHex(block.eth_chainId);
405
+ const hash = normalizeHex(block.eth_getBlockByNumber.hash);
406
+ const number = normalizeHex(block.eth_getBlockByNumber.number, 16);
407
+ const parentHash = normalizeHex(block.eth_getBlockByNumber.parentHash);
408
+ const commitsKey = `commits/v1/${chain}/${number}/${hash}/${parentHash}`;
409
+ await opts.metadataStorage.adapter.put(commitsKey, "commitsValue");
410
+ }
411
+ const public_writeUnfinalizedHead = async (head) => {
412
+ log.debug("Received unfinalized head...");
413
+ const blocksStart = Date.now();
414
+ const [block, manifest] = await Promise.all([
415
+ getBlockFromChain(head),
391
416
  //
392
- getBlock({ chain: head.chain, number: head.number })
417
+ getOrInitManifest(head.chain)
393
418
  ]);
394
- if (stored_block === null) {
395
- return log.debug("Reorganised block never/already processed");
396
- }
397
- if (canonical_block === null) {
398
- throw new Error("Attempted to delete unknown block");
419
+ log.debug(`Loaded block in ${Date.now() - blocksStart}ms`);
420
+ const indexerFinalizedHeight = manifest.finalized_block_height;
421
+ if (viem.hexToNumber(head.number) <= indexerFinalizedHeight) {
422
+ return log.debug("Receiving finalized head, ignoring...");
399
423
  }
400
- if (isHexEqual(head.hash, canonical_block.eth_getBlockByNumber.hash)) {
401
- throw new Error("Attempted to delete canonical block");
424
+ const metadata_start = Date.now();
425
+ const chain = normalizeHex(block.eth_chainId);
426
+ const hash = normalizeHex(block.eth_getBlockByNumber.hash);
427
+ const number = normalizeHex(block.eth_getBlockByNumber.number, 16);
428
+ const parentHash = normalizeHex(block.eth_getBlockByNumber.parentHash);
429
+ const blocksKey = `blocks/v1/${chain}/${number}/${hash}/${parentHash}`;
430
+ const blocksValue = await compress(JSON.stringify(block));
431
+ const conditional = {
432
+ ifNoneMatch: "*"
433
+ };
434
+ const result = await opts.metadataStorage.adapter.put(blocksKey, blocksValue, conditional).catch((error) => {
435
+ if (error instanceof AdapterError) {
436
+ if (error.tag === "PreconditionFailed") {
437
+ return null;
438
+ }
439
+ }
440
+ throw error;
441
+ });
442
+ if (result === null) {
443
+ return log.debug("Block already persisted to wal, ignoring...");
402
444
  }
403
- const deletes = all_events.map(async (event2) => {
445
+ log.debug(`Persisted block to wal in ${Date.now() - metadata_start}ms`);
446
+ await writeUnfinalizedBlock(block);
447
+ };
448
+ async function deleteReorganisedBlocksAndWriteCanonicalBlock(reorganised, canonical) {
449
+ const deletes = allEvents.map(async (event2) => {
404
450
  const batch = [];
405
451
  try {
406
- const events = event2.handler(stored_block);
407
- for (const event3 of events) {
408
- batch.push(event3);
452
+ for (const block of reorganised) {
453
+ const events = event2.handler(block);
454
+ for (const event3 of events) {
455
+ batch.push(event3);
456
+ }
409
457
  }
410
458
  } catch (error) {
411
459
  log.error(`Failed to run your 'handler' for event ${event2.id}`);
@@ -420,14 +468,14 @@ function indexer(opts) {
420
468
  });
421
469
  });
422
470
  await Promise.all(deletes);
423
- const upserts = events_grouped_by_storage_map.entries().map(async ([storage, grouped_events]) => {
471
+ const upserts = eventsGroupedByStorageMap.entries().map(async ([storage, grouped_events]) => {
424
472
  const batch = [];
425
473
  for (const event2 of grouped_events) {
426
474
  try {
427
- if (!event2.filters.some((filter) => matchFilter(canonical_block, filter))) {
475
+ if (!event2.filters.some((filter) => matchFilter(canonical, filter))) {
428
476
  continue;
429
477
  }
430
- const events = event2.handler(canonical_block);
478
+ const events = event2.handler(canonical);
431
479
  for (const event3 of events) {
432
480
  batch.push(event3);
433
481
  }
@@ -446,209 +494,338 @@ function indexer(opts) {
446
494
  }
447
495
  });
448
496
  await Promise.all(upserts);
449
- };
450
- const public_writeFinalizedHeads = async (heads) => {
451
- if (heads.length === 0) {
452
- throw new Error("No heads received, aborting...");
453
- }
454
- log.debug(`Received ${heads.length} finalized heads...`);
455
- let chain = void 0;
456
- for (const head of heads) {
457
- if (chain === void 0) {
458
- chain = head.chain;
459
- }
460
- if (!isHexEqual(chain, head.chain)) {
461
- throw new Error("Received heads from separate chains");
462
- }
497
+ }
498
+ const public_deleteReorganisedHead = async (head) => {
499
+ log.debug(`Received reorganised head ${viem.hexToNumber(head.number)}`);
500
+ const chain = normalizeHex(head.chain);
501
+ const number = normalizeHex(head.number, 16);
502
+ const hash = normalizeHex(head.hash);
503
+ const parentHash = normalizeHex(head.parent_hash);
504
+ const blocksKey = `blocks/v1/${chain}/${number}/${hash}/${parentHash}`;
505
+ const [blocksRes, canonicalBlock] = await Promise.all([
506
+ opts.metadataStorage.adapter.get(blocksKey),
507
+ //
508
+ getBlockFromChain({ chain: head.chain, number: head.number })
509
+ ]);
510
+ if (blocksRes === null) {
511
+ return log.debug("Reorganised block never/already processed");
463
512
  }
464
- if (chain === void 0) {
465
- throw new Error("Internal error. Expected chain to be defined after checking heads length");
513
+ const decompressedBlock = await decompress(blocksRes.body);
514
+ const storedBlock = JSON.parse(decompressedBlock);
515
+ if (isHexEqual(head.hash, canonicalBlock.eth_getBlockByNumber.hash)) {
516
+ throw new Error("Attempted to delete canonical block");
466
517
  }
467
- const [firstHead, ...remainingHeads] = heads;
468
- if (firstHead === void 0) {
469
- throw new Error("Internal error. Expected firstHead to be defined after checking heads length");
518
+ await deleteReorganisedBlocksAndWriteCanonicalBlock([storedBlock], canonicalBlock);
519
+ };
520
+ async function writeFinalizedBlock(block, actions) {
521
+ if (actions.length === 0) {
522
+ return;
470
523
  }
471
- let previousHead = firstHead;
472
- for (const head of remainingHeads) {
473
- if (hexToNumber(head.number) !== hexToNumber(previousHead.number) + 1) {
474
- throw new Error("Found invalid block number in received heads, aborting...");
475
- }
476
- if (!isHexEqual(head.parent_hash, previousHead.hash)) {
477
- throw new Error("Found invalid block hash in received heads, aborting...");
524
+ const chain = normalizeHex(block.eth_chainId);
525
+ const number = normalizeHex(block.eth_getBlockByNumber.number, 16);
526
+ const hash = normalizeHex(block.eth_getBlockByNumber.hash);
527
+ const parentHash = normalizeHex(block.eth_getBlockByNumber.parentHash);
528
+ const prefix = `commits/v1/${chain}/${number}/${hash}/${parentHash}`;
529
+ const promises = actions.map(async (action2) => {
530
+ const events = action2.event.filters.some((filter) => matchFilter(block, filter)) ? action2.event.handler(block) : [];
531
+ const promises2 = events.map(async (event2) => {
532
+ await retry(() => action2.handler(event2), 2).catch((error) => {
533
+ log.error(`Failed to execute action ${action2.id}`);
534
+ throw error;
535
+ });
536
+ });
537
+ const results = await Promise.allSettled(promises2);
538
+ const failure = results.find((result) => result.status === "rejected");
539
+ if (failure !== void 0) {
540
+ throw failure.reason;
478
541
  }
479
- previousHead = head;
480
- }
481
- const [heights, finalizedBlock] = await Promise.all([
482
- metadata.heights.list(chain),
483
- //
484
- getBlock({ chain, number: "finalized" })
485
- ]);
486
- if (finalizedBlock === null) {
487
- log.debug("Failed to load chain finalized block. Request should be retried");
488
- throw new Error(GetBlockError);
489
- }
490
- if (heights.length === 0) {
491
- await metadata.heights.upsert(chain, finalizedBlock.eth_getBlockByNumber.number);
492
- heights.push({ chain, number: finalizedBlock.eth_getBlockByNumber.number });
493
- }
494
- const indexerHeight = Math.max(...heights.map((height) => hexToNumber(height.number)));
495
- const finalizedHeight = hexToNumber(finalizedBlock.eth_getBlockByNumber.number);
496
- const lastHead = heads[heads.length - 1];
497
- if (lastHead === void 0 || hexToNumber(lastHead.number) !== finalizedHeight) {
498
- throw new Error("Client has stalled. Chain has finalized greater than the heads received");
499
- }
500
- log.debug(`Indexer height ${indexerHeight}, finalized height ${finalizedHeight}`);
501
- const newHeads = heads.filter((head) => {
502
- return hexToNumber(head.number) > indexerHeight;
542
+ const commitsKey = `${prefix}/action/${action2.id}`;
543
+ await opts.metadataStorage.adapter.put(commitsKey, "");
503
544
  });
504
- const [firstNewHead] = newHeads;
505
- if (firstNewHead === void 0 && indexerHeight === finalizedHeight) {
506
- return log.debug("No heads to finalize, returning...");
507
- }
508
- if (firstNewHead === void 0) {
509
- throw new Error("Internal error. Expected at least one head greater than the indexer height and less than the finalized height.");
545
+ await Promise.all(promises);
546
+ }
547
+ const public_writeFinalizedHead = async (head) => {
548
+ log.debug("Received finalized head...");
549
+ if (allActions.length === 0) {
550
+ return;
510
551
  }
511
- if (hexToNumber(firstNewHead.number) !== indexerHeight + 1) {
512
- throw new Error("Client advanced too far. Heads received are greater than the first new head");
552
+ const blocksStart = Date.now();
553
+ const [block, manifest, chainFinalizedBlock] = await Promise.all([
554
+ getBlockFromChain(head),
555
+ getOrInitManifest(head.chain),
556
+ getBlockFromChain({ chain: head.chain, number: "finalized" })
557
+ ]);
558
+ log.debug(`Loaded block in ${Date.now() - blocksStart}ms`);
559
+ const receivedHeight = viem.hexToNumber(head.number);
560
+ const indexerFinalizedHeight = manifest.finalized_block_height;
561
+ const chainFinalizedHeight = viem.hexToNumber(chainFinalizedBlock.eth_getBlockByNumber.number);
562
+ if (receivedHeight <= indexerFinalizedHeight) {
563
+ return log.debug(`Received finalized head (${receivedHeight}) below indexer height (${indexerFinalizedHeight})`);
513
564
  }
514
- if (heights.length > 1) {
515
- await cleanupFinalizedHeads(chain, numberToHex(indexerHeight));
565
+ if (receivedHeight > chainFinalizedHeight) {
566
+ return log.error(`Received finalized head (${receivedHeight}) that has not finalized (${chainFinalizedHeight})`);
516
567
  }
517
- for (const head of newHeads) {
518
- await Promise.all([
519
- processEventsForCanonicalHead(head),
520
- processActionsForCanonicalHead(head)
521
- //
522
- ]);
523
- await metadata.heights.upsert(head.chain, head.number);
568
+ const chain = normalizeHex(block.eth_chainId);
569
+ const number = normalizeHex(block.eth_getBlockByNumber.number, 16);
570
+ const hash = normalizeHex(block.eth_getBlockByNumber.hash);
571
+ const parentHash = normalizeHex(block.eth_getBlockByNumber.parentHash);
572
+ const finalizedKey = `finalized/v1/${chain}/${number}/${hash}/${parentHash}`;
573
+ const result = await opts.metadataStorage.adapter.put(finalizedKey, "", { ifNoneMatch: "*" }).catch((error) => {
574
+ if (error instanceof AdapterError && error.tag === "PreconditionFailed") {
575
+ return null;
576
+ }
577
+ throw error;
578
+ });
579
+ if (result === null) {
580
+ return log.debug("Finalized block already persisted to wal, ignoring...");
524
581
  }
525
- await cleanupFinalizedHeads(chain, numberToHex(finalizedHeight));
582
+ await writeFinalizedBlock(block, allActions);
526
583
  };
527
- async function processActionsForCanonicalHead(head) {
528
- if (all_actions.every((action2) => typeof action2.handler.finalized === "undefined")) {
529
- return;
584
+ const FINALIZATION_BATCH_SIZE = 32;
585
+ const LEASE_DURATION_MS = 60 * 1e3;
586
+ async function getBlocksProcessedAndGarbageCollect(chain, finalizedHeight) {
587
+ while (true) {
588
+ const blocksKey = `blocks/v1/${normalizeHex(chain)}/`;
589
+ const blocks = await opts.metadataStorage.adapter.list({ prefix: blocksKey, limit: 1e3 });
590
+ if (blocks.keys.length === 0) {
591
+ return [];
592
+ }
593
+ const garbageCollectionKeys = blocks.keys.filter((key) => {
594
+ const [_, __, ___, number] = key.split("/");
595
+ return viem.hexToNumber(number) <= finalizedHeight;
596
+ });
597
+ if (garbageCollectionKeys.length === 0) {
598
+ return blocks.keys.map((key) => {
599
+ const [_, __, ___, number, hash, parent_hash] = key.split("/");
600
+ return { chain, number, hash, parent_hash };
601
+ });
602
+ }
603
+ const garbageCollectionPromises = garbageCollectionKeys.map(async (key) => {
604
+ await opts.metadataStorage.adapter.delete(key);
605
+ });
606
+ await Promise.all(garbageCollectionPromises);
530
607
  }
531
- const block = await getBlock({ chain: head.chain, number: head.number, hash: head.hash });
532
- const promises = all_actions.flatMap((action2) => {
533
- if (typeof action2.handler.finalized === "undefined") {
608
+ }
609
+ async function getCommitsAndGarbageCollect(chain, finalizedHeight) {
610
+ while (true) {
611
+ const commitsKey = `commits/v1/${normalizeHex(chain)}/`;
612
+ const commits = await opts.metadataStorage.adapter.list({ prefix: commitsKey, limit: 1e3 });
613
+ if (commits.keys.length === 0) {
534
614
  return [];
535
615
  }
536
- return action2.event.handler(block).map(async (event2) => {
537
- await retry(() => action2.handler.finalized(event2), 2).catch((error) => {
538
- log.error(`Failed to execute 'finalized' action ${action2.id}`);
539
- throw error;
616
+ const garbageCollectionKeys = commits.keys.filter((key) => {
617
+ const [_, __, ___, number] = key.split("/");
618
+ return viem.hexToNumber(number) <= finalizedHeight;
619
+ });
620
+ if (garbageCollectionKeys.length === 0) {
621
+ return commits.keys.map((key) => {
622
+ const [_, __, ___, number, hash, parent_hash, type, id] = key.split("/");
623
+ return { chain, number, hash, parent_hash, type, id };
540
624
  });
625
+ }
626
+ const garbageCollectionPromises = garbageCollectionKeys.map(async (key) => {
627
+ await opts.metadataStorage.adapter.delete(key);
541
628
  });
542
- });
543
- await Promise.all(promises);
629
+ await Promise.all(garbageCollectionPromises);
630
+ }
631
+ }
632
+ async function getBlockFromMetadataOrChain(head) {
633
+ const chain = normalizeHex(head.chain);
634
+ const number = normalizeHex(head.number, 16);
635
+ const hash = normalizeHex(head.hash);
636
+ const parentHash = normalizeHex(head.parent_hash);
637
+ const prefix = `blocks/v1/${chain}/${number}/${hash}/${parentHash}`;
638
+ const object = await opts.metadataStorage.adapter.get(prefix);
639
+ if (object !== null) {
640
+ const block = await decompress(object.body);
641
+ const parsed = JSON.parse(block);
642
+ return parsed;
643
+ }
644
+ return await getBlockFromChain(head);
544
645
  }
545
- async function processEventsForCanonicalHead(head) {
546
- const [processed, commits] = await Promise.all([
547
- metadata.blocks.list(head.chain, head.number),
646
+ const public_finalize = async (chain) => {
647
+ const manifestKey = `manifest/v1/${normalizeHex(chain)}`;
648
+ const [chainFinalizedBlock, manifestGetRes] = await Promise.all([
649
+ getBlockFromChain({ chain, number: "finalized" }),
650
+ opts.metadataStorage.adapter.get(manifestKey)
548
651
  //
549
- metadata.commits.list(head.chain, head.number)
550
652
  ]);
551
- if (processed.length === 1) {
552
- if (commits.some((commit) => isHexEqual(head.hash, commit.hash) && isHexEqual(head.parent_hash, commit.parent_hash))) {
553
- return;
653
+ const chainFinalizedHeight = viem.hexToNumber(chainFinalizedBlock.eth_getBlockByNumber.number);
654
+ if (manifestGetRes === null) {
655
+ log.debug("No manifest file found");
656
+ const manifest2 = {
657
+ finalized_block_height: chainFinalizedHeight,
658
+ finalized_block_hash: chainFinalizedBlock.eth_getBlockByNumber.hash,
659
+ next_finalized_height: chainFinalizedHeight,
660
+ updated_at: Date.now()
661
+ };
662
+ await opts.metadataStorage.adapter.put(manifestKey, JSON.stringify(manifest2), { ifNoneMatch: "*" });
663
+ return log.debug("Nothing to finalize, returning...");
664
+ }
665
+ const manifest = JSON.parse(decoder.decode(manifestGetRes.body));
666
+ if (manifest.finalized_block_height !== manifest.next_finalized_height) {
667
+ if (Date.now() - manifest.updated_at < LEASE_DURATION_MS) {
668
+ return log.debug("Found valid lease, returning...");
554
669
  }
670
+ log.debug("Found expired lease");
555
671
  }
556
- const reorganised = processed.filter((block) => {
557
- return !isHexEqual(head.hash, block.hash);
672
+ let indexerFinalizedBlockHeight = manifest.finalized_block_height;
673
+ let indexerFinalizedBlockHash = manifest.finalized_block_hash;
674
+ if (indexerFinalizedBlockHeight >= chainFinalizedHeight) {
675
+ return log.debug("Indexer already finalized, returning...");
676
+ }
677
+ log.debug("Acquiring lease...");
678
+ const updatedManifest = {
679
+ finalized_block_height: indexerFinalizedBlockHeight,
680
+ finalized_block_hash: indexerFinalizedBlockHash,
681
+ next_finalized_height: Math.min(chainFinalizedHeight, indexerFinalizedBlockHeight + FINALIZATION_BATCH_SIZE),
682
+ updated_at: Date.now()
683
+ };
684
+ const conditional = { ifMatch: manifestGetRes.etag };
685
+ const manifestValue = JSON.stringify(updatedManifest);
686
+ const manifestPutRes = await opts.metadataStorage.adapter.put(manifestKey, manifestValue, conditional).catch((error) => {
687
+ if (error instanceof AdapterError && error.tag === "PreconditionFailed") {
688
+ return null;
689
+ }
690
+ throw error;
558
691
  });
559
- await Promise.all(
560
- reorganised.map(async (block) => {
561
- await deleteReorganisedHead(block);
562
- })
563
- );
564
- await writeFinalizedHead(head);
565
- }
566
- async function writeFinalizedHead(head) {
567
- const block = await getBlock({ chain: head.chain, number: head.number, hash: head.hash });
568
- if (block === null) {
569
- throw new Error("Received unknown head that wasn't canonical");
692
+ if (manifestPutRes === null) {
693
+ return log.debug("Failed to acquire lease, returning...");
570
694
  }
571
- const promises = events_grouped_by_storage_map.entries().map(async ([storage, grouped_events]) => {
572
- const batch = [];
573
- for (const event2 of grouped_events) {
574
- try {
575
- if (!event2.filters.some((filter) => matchFilter(block, filter))) {
576
- log.debug(`Block matches no filters for event ${event2.id}`);
577
- continue;
578
- }
579
- const events = event2.handler(block);
580
- for (const event3 of events) {
581
- batch.push(event3);
582
- }
583
- } catch (error) {
584
- log.error(`Failed to run your 'handler' for event ${event2.id}`);
585
- throw error;
586
- }
695
+ let latestManifestEtag = manifestPutRes.etag;
696
+ while (indexerFinalizedBlockHeight < chainFinalizedHeight) {
697
+ const nextFinalizedHeight = Math.min(chainFinalizedHeight, indexerFinalizedBlockHeight + FINALIZATION_BATCH_SIZE);
698
+ const finalizationBatchSize = nextFinalizedHeight - indexerFinalizedBlockHeight;
699
+ const [nextFinalizedBlock, blocksProcessed] = await Promise.all([
700
+ getBlockFromChain({ chain, number: viem.numberToHex(nextFinalizedHeight) }),
701
+ getBlocksProcessedAndGarbageCollect(chain, indexerFinalizedBlockHeight)
702
+ ]);
703
+ const nextFinalizedHead = {
704
+ chain,
705
+ hash: nextFinalizedBlock.eth_getBlockByNumber.hash,
706
+ number: nextFinalizedBlock.eth_getBlockByNumber.number,
707
+ parent_hash: nextFinalizedBlock.eth_getBlockByNumber.parentHash
708
+ };
709
+ const nextFinalizedBlockProcessed = blocksProcessed.some((block) => {
710
+ return isHexEqual(block.hash, nextFinalizedHead.hash) && isHexEqual(block.parent_hash, nextFinalizedHead.parent_hash) && viem.hexToNumber(block.number) === viem.hexToNumber(nextFinalizedHead.number);
711
+ });
712
+ if (nextFinalizedBlockProcessed === false) {
713
+ await Promise.all([
714
+ writeUnfinalizedBlock(nextFinalizedBlock),
715
+ //
716
+ writeFinalizedBlock(nextFinalizedBlock, allActions)
717
+ ]);
718
+ blocksProcessed.push(nextFinalizedHead);
587
719
  }
588
- if (batch.length > 0) {
589
- const start = Date.now();
590
- await retry(() => storage.upsert(batch), 2).catch((error) => {
591
- for (const event2 of grouped_events) {
592
- log.error(`Failed to run your 'upsert' handler for event ${event2.id}`);
593
- }
594
- throw error;
720
+ const canonicalHeads = [nextFinalizedHead];
721
+ let parentHash = nextFinalizedBlock.eth_getBlockByNumber.parentHash;
722
+ for (let index = 1; index < finalizationBatchSize; index++) {
723
+ const number = nextFinalizedHeight - index;
724
+ const blocksProcessedForHeight = blocksProcessed.filter((block) => {
725
+ return viem.hexToNumber(block.number) === number;
595
726
  });
596
- for (const event2 of grouped_events) {
597
- log.debug(`Recorded ${batch.length} ${event2.id} in ${Date.now() - start}ms`);
727
+ const canonicalHead = blocksProcessedForHeight.find((block) => {
728
+ return isHexEqual(block.hash, parentHash);
729
+ });
730
+ if (canonicalHead) {
731
+ canonicalHeads.unshift(canonicalHead);
732
+ parentHash = canonicalHead.parent_hash;
733
+ continue;
598
734
  }
735
+ log.debug("Canonical head never processed, loading from chain...");
736
+ const canonicalBlock = await getBlockFromChain({ chain, number: viem.numberToHex(number) });
737
+ const head = {
738
+ chain,
739
+ hash: canonicalBlock.eth_getBlockByNumber.hash,
740
+ number: canonicalBlock.eth_getBlockByNumber.number,
741
+ parent_hash: canonicalBlock.eth_getBlockByNumber.parentHash
742
+ };
743
+ log.debug("Processing canonical head");
744
+ await Promise.all([
745
+ writeUnfinalizedBlock(canonicalBlock),
746
+ //
747
+ writeFinalizedBlock(canonicalBlock, allActions)
748
+ ]);
749
+ canonicalHeads.unshift(head);
750
+ blocksProcessed.push(head);
751
+ parentHash = canonicalBlock.eth_getBlockByNumber.parentHash;
599
752
  }
600
- });
601
- await Promise.all(promises);
602
- }
603
- async function deleteReorganisedHead(head) {
604
- const block = await metadata.blocks.get(head);
605
- if (block === null) {
606
- throw new Error("Expected reorganised block to exist in metadata");
607
- }
608
- const promises = all_events.map(async (event2) => {
609
- const batch = [];
610
- try {
611
- const events = event2.handler(block);
612
- for (const event3 of events) {
613
- batch.push(event3);
614
- }
615
- } catch (error) {
616
- log.error(`Failed to run your 'handler' for event ${event2.id}`);
617
- throw error;
753
+ if (!isHexEqual(parentHash, indexerFinalizedBlockHash)) {
754
+ throw new Error("Expected chain to match last finalized canonical anchor");
618
755
  }
619
- if (batch.length === 0) {
620
- return;
756
+ if (canonicalHeads.length !== finalizationBatchSize) {
757
+ throw new Error(`Expected to have ${finalizationBatchSize} heads, found ${canonicalHeads.length}`);
621
758
  }
622
- await retry(() => event2.storage.delete(batch), 2).catch((error) => {
623
- log.error(`Failed to run your 'delete' handler for event ${event2.id}`);
759
+ log.debug("Determined canonical chain");
760
+ const commits = await getCommitsAndGarbageCollect(chain, indexerFinalizedBlockHeight);
761
+ for (const canonicalHead of canonicalHeads) {
762
+ const blocksProcessedForHeight = blocksProcessed.filter((head) => {
763
+ return viem.hexToNumber(canonicalHead.number) === viem.hexToNumber(head.number);
764
+ });
765
+ const processedOnlyCanonicalBlock = blocksProcessedForHeight.every((head) => {
766
+ return isHexEqual(canonicalHead.hash, head.hash) && isHexEqual(canonicalHead.parent_hash, head.parent_hash);
767
+ });
768
+ const eventsCommittedForHeight = commits.some((commit) => {
769
+ return commit.type === void 0 && isHexEqual(canonicalHead.hash, commit.hash) && isHexEqual(canonicalHead.parent_hash, commit.parent_hash) && viem.hexToNumber(canonicalHead.number) === viem.hexToNumber(commit.number);
770
+ });
771
+ const actionsWithoutCommit = allActions.filter((action2) => {
772
+ const commitExists = commits.some((commit) => {
773
+ return commit.id === action2.id && commit.type === "action" && isHexEqual(canonicalHead.hash, commit.hash) && isHexEqual(canonicalHead.parent_hash, commit.parent_hash) && viem.hexToNumber(canonicalHead.number) === viem.hexToNumber(commit.number);
774
+ });
775
+ return !commitExists;
776
+ });
777
+ if (eventsCommittedForHeight && processedOnlyCanonicalBlock && actionsWithoutCommit.length === 0 && blocksProcessedForHeight.length === 1) {
778
+ log.debug("Executed fast-path");
779
+ continue;
780
+ }
781
+ log.debug("Fast-path missed for head");
782
+ log.debug(`Events commited for height: ${eventsCommittedForHeight}`);
783
+ log.debug(`Processed only canonical block: ${processedOnlyCanonicalBlock}`);
784
+ log.debug(`Actions without commit (${actionsWithoutCommit.length}): ${actionsWithoutCommit.length === 0}`);
785
+ log.debug(`Blocks processed for height (${blocksProcessedForHeight.length}): ${blocksProcessedForHeight.length === 1}`);
786
+ const reorganisedHeads = blocksProcessedForHeight.flatMap((head) => {
787
+ if (isHexEqual(canonicalHead.hash, head.hash)) {
788
+ return [];
789
+ }
790
+ return { chain, number: head.number, hash: head.hash, parent_hash: head.parent_hash };
791
+ });
792
+ const reorganisedPromises = reorganisedHeads.map((head) => getBlockFromMetadataOrChain(head));
793
+ const [canonicalBlock, ...reorganisedBlocks] = await Promise.all([
794
+ getBlockFromMetadataOrChain(canonicalHead),
795
+ //
796
+ ...reorganisedPromises
797
+ ]);
798
+ log.debug("Re-processing heads");
799
+ await Promise.all([
800
+ writeFinalizedBlock(canonicalBlock, actionsWithoutCommit),
801
+ //
802
+ deleteReorganisedBlocksAndWriteCanonicalBlock(reorganisedBlocks, canonicalBlock)
803
+ ]);
804
+ }
805
+ const nextBatchFinalizingHeight = Math.min(chainFinalizedHeight, nextFinalizedHeight + FINALIZATION_BATCH_SIZE);
806
+ const updatedManifest2 = {
807
+ finalized_block_height: nextFinalizedHeight,
808
+ finalized_block_hash: nextFinalizedBlock.eth_getBlockByNumber.hash,
809
+ next_finalized_height: nextBatchFinalizingHeight,
810
+ updated_at: Date.now()
811
+ };
812
+ const manifestPutRes2 = await opts.metadataStorage.adapter.put(manifestKey, JSON.stringify(updatedManifest2), { ifMatch: latestManifestEtag }).catch((error) => {
813
+ if (error instanceof AdapterError) {
814
+ if (error.tag === "PreconditionFailed") {
815
+ return null;
816
+ }
817
+ }
624
818
  throw error;
625
819
  });
626
- });
627
- await Promise.all(promises);
628
- }
629
- async function cleanupFinalizedHeads(chain, finalized) {
630
- const [_blocks, _commits, _heights] = await Promise.all([
631
- metadata.blocks.list(chain),
632
- //
633
- metadata.commits.list(chain),
634
- metadata.heights.list(chain)
635
- ]);
636
- const blocks = _blocks.filter((block) => {
637
- return hexToNumber(block.number) <= hexToNumber(finalized);
638
- });
639
- const commits = _commits.filter((commit) => {
640
- return hexToNumber(commit.number) <= hexToNumber(finalized);
641
- });
642
- const heights = _heights.filter((height) => {
643
- return hexToNumber(height.number) < hexToNumber(finalized);
644
- });
645
- await Promise.all([
646
- metadata.blocks.delete(blocks),
647
- //
648
- metadata.commits.delete(commits),
649
- metadata.heights.delete(heights)
650
- ]);
651
- }
820
+ if (manifestPutRes2 === null) {
821
+ return log.debug("Failed to commit finalized batch, aborting...");
822
+ }
823
+ indexerFinalizedBlockHeight = nextFinalizedHeight;
824
+ indexerFinalizedBlockHash = nextFinalizedBlock.eth_getBlockByNumber.hash;
825
+ latestManifestEtag = manifestPutRes2.etag;
826
+ }
827
+ log.debug("Indexer finalized, returning...");
828
+ };
652
829
  const private_getMetadata = async () => {
653
830
  return {
654
831
  version,
@@ -656,7 +833,7 @@ function indexer(opts) {
656
833
  };
657
834
  };
658
835
  const private_getEvents = async () => {
659
- return all_events.map((event2) => {
836
+ return allEvents.map((event2) => {
660
837
  const filters = event2.filters.map((filter) => {
661
838
  return {
662
839
  chain: filter.chain,
@@ -712,8 +889,8 @@ function indexer(opts) {
712
889
  };
713
890
  const IncompleteBlockError = createException("Received block with missing required property");
714
891
  const private_writeEvents = async (params) => {
715
- if (all_events.length === 0) return { failures: [] };
716
- const relevant_events = all_events.filter((event2) => params.events.includes(event2.id));
892
+ if (allEvents.length === 0) return { failures: [] };
893
+ const relevant_events = allEvents.filter((event2) => params.events.includes(event2.id));
717
894
  if (relevant_events.length === 0) return { failures: [] };
718
895
  if (params.events.length === 0) return { failures: [] };
719
896
  const failures = {};
@@ -722,7 +899,7 @@ function indexer(opts) {
722
899
  accessed_undefined_key = true;
723
900
  throw new Error(IncompleteBlockError);
724
901
  });
725
- const promises = events_grouped_by_storage_map.entries().map(async ([storage, grouped_events]) => {
902
+ const promises = eventsGroupedByStorageMap.entries().map(async ([storage, grouped_events]) => {
726
903
  const batch = [];
727
904
  for (const event2 of grouped_events) {
728
905
  if (!params.events.includes(event2.id)) continue;
@@ -788,33 +965,19 @@ function indexer(opts) {
788
965
  return { failures: failures_array };
789
966
  };
790
967
  const private_writeEventsAndGetKeys = async (params) => {
791
- if (all_events.length === 0) {
968
+ if (allEvents.length === 0) {
792
969
  return { results: [], keys: [] };
793
970
  }
794
971
  if (params.events.length === 0) {
795
972
  return { results: [], keys: [] };
796
973
  }
797
- const relevant_events = all_events.filter((event2) => params.events.includes(event2.id));
974
+ const relevant_events = allEvents.filter((event2) => params.events.includes(event2.id));
798
975
  if (relevant_events.length === 0) {
799
976
  return { results: [], keys: [] };
800
977
  }
801
- const block = await getBlock(params.head);
802
- if (block === null) {
803
- return {
804
- keys: [],
805
- results: relevant_events.map((event2) => {
806
- return {
807
- event_id: event2.id,
808
- status: "block_error",
809
- chain: params.head.chain,
810
- hash: params.head.hash,
811
- number: params.head.number,
812
- parent_hash: params.head.parent_hash,
813
- created_at: Date.now()
814
- };
815
- })
816
- };
817
- }
978
+ const block = await getBlockFromChain(params.head).catch(() => {
979
+ return null;
980
+ });
818
981
  const keys = /* @__PURE__ */ new Set();
819
982
  const results = {};
820
983
  function createProxyHandler(path) {
@@ -915,20 +1078,27 @@ function indexer(opts) {
915
1078
  if (!/^[A-Za-z0-9_-]+$/.test(event2.id)) {
916
1079
  throw new Error(`Invalid event id \`${event2.id}\`. Only characters A-Z, a-z, 0-9, underscores, and hyphens are permitted.`);
917
1080
  }
918
- all_events.push(event2);
919
- const group = events_grouped_by_storage_map.get(event2.storage) ?? [];
1081
+ allEvents.push(event2);
1082
+ const group = eventsGroupedByStorageMap.get(event2.storage) ?? [];
920
1083
  group.push(event2);
921
- events_grouped_by_storage_map.set(event2.storage, group);
1084
+ eventsGroupedByStorageMap.set(event2.storage, group);
922
1085
  return event2;
923
1086
  };
924
1087
  const action = (action2) => {
925
- all_actions.push(action2);
1088
+ if (!/^[A-Za-z0-9_-]+$/.test(action2.id)) {
1089
+ throw new Error(`Invalid action id \`${action2.id}\`. Only characters A-Z, a-z, 0-9, underscores, and hyphens are permitted.`);
1090
+ }
1091
+ if (allActions.some((existing) => existing.id === action2.id)) {
1092
+ throw new Error(`Duplicate action id \`${action2.id}\`.`);
1093
+ }
1094
+ allActions.push(action2);
926
1095
  return action2;
927
1096
  };
928
1097
  const rpc = {
929
1098
  request: {
1099
+ public_finalize,
930
1100
  public_getFinalizedHeight,
931
- public_writeFinalizedHeads,
1101
+ public_writeFinalizedHead,
932
1102
  public_writeUnfinalizedHead,
933
1103
  public_deleteReorganisedHead,
934
1104
  private_getEvents,
@@ -943,13 +1113,13 @@ function indexer(opts) {
943
1113
  quiet: opts.quiet ?? false,
944
1114
  signingKey: opts.signingKey
945
1115
  });
946
- const indexer2 = {
1116
+ const indexer = {
947
1117
  ...rpc,
948
1118
  event,
949
1119
  action,
950
1120
  fetch: server.http
951
1121
  };
952
- return indexer2;
1122
+ return indexer;
953
1123
  }
954
1124
 
955
- export { indexer };
1125
+ export { defineIndexer };