univo 0.2.23 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.mjs +2 -2
- package/dist/index.d.mts +4 -2
- package/dist/index.d.ts +4 -2
- package/dist/index.mjs +599 -429
- package/dist/metadata/adapters/memory.d.mts +5 -0
- package/dist/metadata/adapters/memory.d.ts +5 -0
- package/dist/metadata/adapters/memory.mjs +52 -0
- package/dist/metadata/adapters/r2-binding.d.mts +10 -0
- package/dist/metadata/adapters/r2-binding.d.ts +10 -0
- package/dist/metadata/adapters/r2-binding.mjs +54 -0
- package/dist/metadata/adapters/r2.d.mts +13 -0
- package/dist/metadata/adapters/r2.d.ts +13 -0
- package/dist/metadata/adapters/r2.mjs +17 -0
- package/dist/metadata/adapters/s3.d.mts +13 -0
- package/dist/metadata/adapters/s3.d.ts +13 -0
- package/dist/metadata/adapters/s3.mjs +122 -0
- package/dist/metadata/adapters.d.mts +38 -0
- package/dist/metadata/adapters.d.ts +38 -0
- package/dist/metadata/adapters.mjs +13 -0
- package/dist/metadata.d.mts +9 -0
- package/dist/metadata.d.ts +9 -0
- package/dist/metadata.mjs +5 -0
- package/dist/realtime.d.mts +7 -4
- package/dist/realtime.d.ts +7 -4
- package/dist/realtime.mjs +70 -77
- package/dist/rpc.d.mts +26 -46
- package/dist/rpc.d.ts +26 -46
- package/dist/shared/{univo.DvxWs2zB.mjs → univo.CX4GZB_1.mjs} +2 -8
- package/dist/shared/{univo.1clH0g-n.mjs → univo.TsC-vBTq.mjs} +1 -1
- package/dist/transport.d.mts +3 -1
- package/dist/transport.d.ts +3 -1
- package/dist/transport.mjs +2 -2
- package/package.json +35 -3
package/dist/index.mjs
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
|
-
import
|
|
2
|
-
import { g as getException, c as createException, a as catchException, l as local } from './shared/univo.
|
|
3
|
-
import { c as createLogger, d as decompress,
|
|
1
|
+
import * as viem from 'viem';
|
|
2
|
+
import { g as getException, c as createException, a as catchException, l as local } from './shared/univo.TsC-vBTq.mjs';
|
|
3
|
+
import { c as createLogger, d as decompress, b as decoder, r as retry, v as version, e as normalizeHex, a as isHexEqual, f as compress } from './shared/univo.CX4GZB_1.mjs';
|
|
4
|
+
import { AdapterError } from './metadata/adapters.mjs';
|
|
4
5
|
import 'ws';
|
|
5
6
|
import 'partysocket';
|
|
6
7
|
|
|
@@ -102,16 +103,16 @@ function matchFilter(block, filter) {
|
|
|
102
103
|
return true;
|
|
103
104
|
}
|
|
104
105
|
const chainValid = (block, filter) => {
|
|
105
|
-
if (hexToNumber(block.eth_chainId) === filter.chain) return true;
|
|
106
|
+
if (viem.hexToNumber(block.eth_chainId) === filter.chain) return true;
|
|
106
107
|
return false;
|
|
107
108
|
};
|
|
108
109
|
const fromBlockValid = (block, filter) => {
|
|
109
|
-
if (hexToNumber(block.eth_getBlockByNumber.number) >= filter.fromBlock) return true;
|
|
110
|
+
if (viem.hexToNumber(block.eth_getBlockByNumber.number) >= filter.fromBlock) return true;
|
|
110
111
|
return false;
|
|
111
112
|
};
|
|
112
113
|
const toBlockValid = (block, filter) => {
|
|
113
114
|
if (filter.toBlock === void 0) return true;
|
|
114
|
-
if (hexToNumber(block.eth_getBlockByNumber.number) <= filter.toBlock) return true;
|
|
115
|
+
if (viem.hexToNumber(block.eth_getBlockByNumber.number) <= filter.toBlock) return true;
|
|
115
116
|
return false;
|
|
116
117
|
};
|
|
117
118
|
const includesLogAddress = (block, filter) => {
|
|
@@ -132,213 +133,243 @@ const includesLogEvent = (block, filter) => {
|
|
|
132
133
|
}
|
|
133
134
|
return false;
|
|
134
135
|
};
|
|
135
|
-
function
|
|
136
|
+
function defineIndexer(opts) {
|
|
136
137
|
const log = createLogger({ quiet: opts.quiet ?? false, prefix: "[indexer]" });
|
|
137
|
-
const
|
|
138
|
-
const
|
|
139
|
-
const
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
prefix += `/${normalizeHex(number, 16)}`;
|
|
138
|
+
const allEvents = [];
|
|
139
|
+
const eventsGroupedByStorageMap = /* @__PURE__ */ new Map();
|
|
140
|
+
const allActions = [];
|
|
141
|
+
async function getBlockFromChain(head) {
|
|
142
|
+
try {
|
|
143
|
+
const block = await opts.getBlock({ chain: head.chain, number: head.number });
|
|
144
|
+
if (block === null) {
|
|
145
|
+
throw new Error("Provided `getBlock` function returned null");
|
|
146
|
+
}
|
|
147
|
+
if (typeof head.hash === "string") {
|
|
148
|
+
if (!isHexEqual(head.hash, block.eth_getBlockByNumber.hash)) {
|
|
149
|
+
throw new Error("Method `eth_getBlockByNumber` returned unexpected block hash");
|
|
150
150
|
}
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
});
|
|
156
|
-
return mapped;
|
|
157
|
-
},
|
|
158
|
-
async get(head) {
|
|
159
|
-
const chain = normalizeHex(head.chain);
|
|
160
|
-
const number = normalizeHex(head.number, 16);
|
|
161
|
-
const hash = normalizeHex(head.hash);
|
|
162
|
-
const parentHash = normalizeHex(head.parent_hash);
|
|
163
|
-
const prefix = `blocks/v1/${chain}/${number}/${hash}/${parentHash}`;
|
|
164
|
-
const blob = await opts.metadataStorage.download(prefix, { as: "blob" }).catch((error) => {
|
|
165
|
-
if (error instanceof StorageError) {
|
|
166
|
-
if (error.code === "NotFound") {
|
|
167
|
-
return null;
|
|
168
|
-
}
|
|
169
|
-
}
|
|
170
|
-
throw error;
|
|
171
|
-
});
|
|
172
|
-
if (blob === null) {
|
|
173
|
-
return null;
|
|
151
|
+
}
|
|
152
|
+
if (typeof head.parent_hash === "string") {
|
|
153
|
+
if (!isHexEqual(head.parent_hash, block.eth_getBlockByNumber.parentHash)) {
|
|
154
|
+
throw new Error("Method `eth_getBlockByNumber` returned unexpected parent hash");
|
|
174
155
|
}
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
156
|
+
}
|
|
157
|
+
verifyBlockHashes(block);
|
|
158
|
+
verifyLogIndicies(block);
|
|
159
|
+
verifyReceiptsRoot(block);
|
|
160
|
+
verifyTransactionsRoot(block);
|
|
161
|
+
verifyTransactionIndicies(block);
|
|
162
|
+
verifyTransactionGasUsage(block);
|
|
163
|
+
return block;
|
|
164
|
+
} catch (error) {
|
|
165
|
+
if (error instanceof Error) {
|
|
166
|
+
log.error(`Failed to load block: ${error.message}`);
|
|
167
|
+
}
|
|
168
|
+
throw error;
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
function verifyBlockHashes(block) {
|
|
172
|
+
const blockHash = block.eth_getBlockByNumber.hash;
|
|
173
|
+
const transactions = block.eth_getBlockByNumber.transactions;
|
|
174
|
+
for (const transaction of transactions) {
|
|
175
|
+
if (!isHexEqual(blockHash, transaction.blockHash)) {
|
|
176
|
+
throw new Error("Method `eth_getBlockByNumber` returned transaction with unexpected block hash");
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
for (const receipt of block.eth_getBlockReceipts) {
|
|
180
|
+
if (!isHexEqual(block.eth_getBlockByNumber.hash, receipt.blockHash)) {
|
|
181
|
+
throw new Error("Method `eth_getBlockReceipts` returned receipt with unexpected block hash");
|
|
182
|
+
}
|
|
183
|
+
const transactionIndex = viem.hexToNumber(receipt.transactionIndex);
|
|
184
|
+
const transaction = transactions[transactionIndex];
|
|
185
|
+
if (transaction === void 0 || !isHexEqual(transaction.hash, receipt.transactionHash)) {
|
|
186
|
+
throw new Error("Method `eth_getBlockReceipts` returned receipt with unexpected transaction hash");
|
|
187
|
+
}
|
|
188
|
+
for (const entry of receipt.logs) {
|
|
189
|
+
if (!isHexEqual(blockHash, entry.blockHash)) {
|
|
190
|
+
throw new Error("Method `eth_getBlockReceipts` returned log with unexpected block hash");
|
|
182
191
|
}
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
const chain = normalizeHex(block.eth_chainId);
|
|
186
|
-
const number = normalizeHex(block.eth_getBlockByNumber.number, 16);
|
|
187
|
-
const hash = normalizeHex(block.eth_getBlockByNumber.hash);
|
|
188
|
-
const parentHash = normalizeHex(block.eth_getBlockByNumber.parentHash);
|
|
189
|
-
const prefix = `blocks/v1/${chain}/${number}/${hash}/${parentHash}`;
|
|
190
|
-
const compressed = await compress(JSON.stringify(block));
|
|
191
|
-
await opts.metadataStorage.upload(prefix, compressed);
|
|
192
|
-
})
|
|
193
|
-
);
|
|
194
|
-
},
|
|
195
|
-
async delete(blocks) {
|
|
196
|
-
if (blocks.length === 0) {
|
|
197
|
-
return;
|
|
192
|
+
if (!isHexEqual(receipt.transactionHash, entry.transactionHash)) {
|
|
193
|
+
throw new Error("Method `eth_getBlockReceipts` returned log with unexpected transaction hash");
|
|
198
194
|
}
|
|
199
|
-
await Promise.all(
|
|
200
|
-
blocks.map(async (block) => {
|
|
201
|
-
const chain = normalizeHex(block.chain);
|
|
202
|
-
const number = normalizeHex(block.number, 16);
|
|
203
|
-
const hash = normalizeHex(block.hash);
|
|
204
|
-
const parentHash = normalizeHex(block.parent_hash);
|
|
205
|
-
const prefix = `blocks/v1/${chain}/${number}/${hash}/${parentHash}`;
|
|
206
|
-
await opts.metadataStorage.delete(prefix);
|
|
207
|
-
})
|
|
208
|
-
);
|
|
209
195
|
}
|
|
210
|
-
}
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
prefix += `/${normalizeHex(number, 16)}`;
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
function verifyLogIndicies(block) {
|
|
199
|
+
let expectedLogIndex = 0n;
|
|
200
|
+
for (const receipt of block.eth_getBlockReceipts) {
|
|
201
|
+
for (const entry of receipt.logs) {
|
|
202
|
+
if (viem.hexToBigInt(entry.logIndex) !== expectedLogIndex) {
|
|
203
|
+
throw new Error("Method `eth_getBlockReceipts` returned non-contiguous log indices");
|
|
219
204
|
}
|
|
220
|
-
|
|
221
|
-
const mapped = keys.items.map((key) => {
|
|
222
|
-
const [_, __, ___, number2, hash, parent_hash] = key.path.split("/");
|
|
223
|
-
return { chain, number: number2, hash, parent_hash };
|
|
224
|
-
});
|
|
225
|
-
return mapped;
|
|
226
|
-
},
|
|
227
|
-
async upsert(head) {
|
|
228
|
-
const chain = normalizeHex(head.chain);
|
|
229
|
-
const number = normalizeHex(head.number, 16);
|
|
230
|
-
const hash = normalizeHex(head.hash);
|
|
231
|
-
const parentHash = normalizeHex(head.parent_hash);
|
|
232
|
-
const prefix = `commits/v1/${chain}/${number}/${hash}/${parentHash}`;
|
|
233
|
-
const body = JSON.stringify({ hello: "world" });
|
|
234
|
-
await opts.metadataStorage.upload(prefix, body);
|
|
235
|
-
},
|
|
236
|
-
async delete(commits) {
|
|
237
|
-
await Promise.all(
|
|
238
|
-
commits.map(async (commit) => {
|
|
239
|
-
const chain = normalizeHex(commit.chain);
|
|
240
|
-
const number = normalizeHex(commit.number, 16);
|
|
241
|
-
const hash = normalizeHex(commit.hash);
|
|
242
|
-
const parentHash = normalizeHex(commit.parent_hash);
|
|
243
|
-
const prefix = `commits/v1/${chain}/${number}/${hash}/${parentHash}`;
|
|
244
|
-
await opts.metadataStorage.delete(prefix);
|
|
245
|
-
})
|
|
246
|
-
);
|
|
205
|
+
expectedLogIndex++;
|
|
247
206
|
}
|
|
248
|
-
}
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
function verifyReceiptsRoot(block) {
|
|
210
|
+
const receiptsRoot = calculateTrieRoot(block.eth_getBlockReceipts.map(serializeReceipt));
|
|
211
|
+
if (!isHexEqual(block.eth_getBlockByNumber.receiptsRoot, receiptsRoot)) {
|
|
212
|
+
throw new Error("Method `eth_getBlockReceipts` returned receipts that do not match the block receipts root");
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
function calculateTrieRoot(values) {
|
|
216
|
+
if (values.length === 0) {
|
|
217
|
+
return viem.keccak256(viem.toRlp(new Uint8Array(), "bytes"));
|
|
218
|
+
}
|
|
219
|
+
const entries = values.map((value, index) => ({
|
|
220
|
+
key: bytesToNibbles(viem.toRlp(quantityToBytes(BigInt(index)), "bytes")),
|
|
221
|
+
value
|
|
222
|
+
}));
|
|
223
|
+
return viem.keccak256(viem.toRlp(encodeTrieNode(entries), "bytes"));
|
|
224
|
+
}
|
|
225
|
+
function trieNodeReference(node) {
|
|
226
|
+
const encoded = viem.toRlp(node, "bytes");
|
|
227
|
+
return encoded.length < 32 ? node : viem.hexToBytes(viem.keccak256(encoded));
|
|
228
|
+
}
|
|
229
|
+
function encodeTrieNode(entries, depth = 0) {
|
|
230
|
+
if (entries.length === 1) {
|
|
231
|
+
const entry = entries[0];
|
|
232
|
+
return [encodePath(entry.key.slice(depth), true), entry.value];
|
|
233
|
+
}
|
|
234
|
+
let shared = 0;
|
|
235
|
+
while (entries.every((entry) => entry.key[depth + shared] === entries[0].key[depth + shared])) {
|
|
236
|
+
shared++;
|
|
237
|
+
}
|
|
238
|
+
if (shared > 0) {
|
|
239
|
+
const child = encodeTrieNode(entries, depth + shared);
|
|
240
|
+
return [encodePath(entries[0].key.slice(depth, depth + shared), false), trieNodeReference(child)];
|
|
241
|
+
}
|
|
242
|
+
const children = Array.from({ length: 17 }, () => new Uint8Array());
|
|
243
|
+
for (let nibble = 0; nibble < 16; nibble++) {
|
|
244
|
+
const matching = entries.filter((entry) => entry.key[depth] === nibble);
|
|
245
|
+
if (matching.length > 0) {
|
|
246
|
+
children[nibble] = trieNodeReference(encodeTrieNode(matching, depth + 1));
|
|
277
247
|
}
|
|
278
248
|
}
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
249
|
+
const value = entries.find((entry) => entry.key.length === depth)?.value;
|
|
250
|
+
if (value !== void 0) {
|
|
251
|
+
children[16] = value;
|
|
252
|
+
}
|
|
253
|
+
return children;
|
|
254
|
+
}
|
|
255
|
+
function encodePath(path, leaf) {
|
|
256
|
+
const odd = path.length % 2 === 1;
|
|
257
|
+
const nibbles = odd ? [leaf ? 3 : 1, ...path] : [leaf ? 2 : 0, 0, ...path];
|
|
258
|
+
const bytes = new Uint8Array(nibbles.length / 2);
|
|
259
|
+
for (let index = 0; index < nibbles.length; index += 2) {
|
|
260
|
+
bytes[index / 2] = nibbles[index] << 4 | nibbles[index + 1];
|
|
261
|
+
}
|
|
262
|
+
return bytes;
|
|
263
|
+
}
|
|
264
|
+
function quantityToBytes(value) {
|
|
265
|
+
const number = typeof value === "bigint" ? value : viem.hexToBigInt(value);
|
|
266
|
+
return number === 0n ? new Uint8Array() : viem.numberToBytes(number);
|
|
267
|
+
}
|
|
268
|
+
function bytesToNibbles(value) {
|
|
269
|
+
return Array.from(value).flatMap((byte) => [byte >> 4, byte & 15]);
|
|
270
|
+
}
|
|
271
|
+
function serializeReceipt(receipt) {
|
|
272
|
+
const outcome = receipt.status === void 0 ? viem.hexToBytes(receipt.root) : quantityToBytes(receipt.status);
|
|
273
|
+
const fields = [
|
|
274
|
+
outcome,
|
|
275
|
+
quantityToBytes(receipt.cumulativeGasUsed),
|
|
276
|
+
viem.hexToBytes(receipt.logsBloom),
|
|
277
|
+
receipt.logs.map((log2) => [viem.hexToBytes(log2.address), log2.topics.map(viem.hexToBytes), viem.hexToBytes(log2.data)])
|
|
278
|
+
];
|
|
279
|
+
const encoded = viem.toRlp(fields, "bytes");
|
|
280
|
+
if (!viem.isHex(receipt.type)) {
|
|
281
|
+
throw new Error(`Unsupported receipt type ${receipt.type}`);
|
|
282
|
+
}
|
|
283
|
+
const type = viem.hexToBigInt(receipt.type);
|
|
284
|
+
return type === 0n ? encoded : viem.concatBytes([Uint8Array.of(Number(type)), encoded]);
|
|
285
|
+
}
|
|
286
|
+
function verifyTransactionsRoot(block) {
|
|
287
|
+
const transactions = block.eth_getBlockByNumber.transactions.map((transaction) => {
|
|
288
|
+
const { input, ...formatted } = viem.formatTransaction(transaction);
|
|
289
|
+
const signature = formatted.type === "legacy" ? { r: formatted.r, s: formatted.s, v: formatted.v } : { r: formatted.r, s: formatted.s, yParity: formatted.yParity };
|
|
290
|
+
const serialized = viem.serializeTransaction(
|
|
291
|
+
{ ...formatted, data: input },
|
|
292
|
+
//
|
|
293
|
+
signature
|
|
294
|
+
);
|
|
295
|
+
if (!isHexEqual(transaction.hash, viem.keccak256(serialized))) {
|
|
296
|
+
throw new Error("Method `eth_getBlockByNumber` returned transaction with unexpected transaction hash");
|
|
297
|
+
}
|
|
298
|
+
return viem.hexToBytes(serialized);
|
|
284
299
|
});
|
|
300
|
+
const transactionsRoot = calculateTrieRoot(transactions);
|
|
301
|
+
if (!isHexEqual(block.eth_getBlockByNumber.transactionsRoot, transactionsRoot)) {
|
|
302
|
+
throw new Error("Method `eth_getBlockByNumber` returned transactions that do not match the block transactions root");
|
|
303
|
+
}
|
|
285
304
|
}
|
|
286
|
-
|
|
287
|
-
const
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
305
|
+
function verifyTransactionIndicies(block) {
|
|
306
|
+
const receipts = block.eth_getBlockReceipts;
|
|
307
|
+
const transactions = block.eth_getBlockByNumber.transactions;
|
|
308
|
+
if (transactions.length !== receipts.length) {
|
|
309
|
+
throw new Error("Methods `eth_getBlockByNumber` and `eth_getBlockReceipts` returned different transaction counts");
|
|
310
|
+
}
|
|
311
|
+
for (let index = 0; index < transactions.length; index++) {
|
|
312
|
+
const expectedIndex = BigInt(index);
|
|
313
|
+
if (viem.hexToBigInt(transactions[index].transactionIndex) !== expectedIndex) {
|
|
314
|
+
throw new Error("Method `eth_getBlockByNumber` returned non-contiguous transaction indices");
|
|
315
|
+
}
|
|
316
|
+
if (viem.hexToBigInt(receipts[index].transactionIndex) !== expectedIndex) {
|
|
317
|
+
throw new Error("Method `eth_getBlockReceipts` returned non-contiguous transaction indices");
|
|
318
|
+
}
|
|
319
|
+
for (const entry of receipts[index].logs) {
|
|
320
|
+
if (viem.hexToBigInt(entry.transactionIndex) !== expectedIndex) {
|
|
321
|
+
throw new Error("Method `eth_getBlockReceipts` returned log with unexpected transaction index");
|
|
298
322
|
}
|
|
299
323
|
}
|
|
300
324
|
}
|
|
301
|
-
return block;
|
|
302
325
|
}
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
const
|
|
306
|
-
if (
|
|
307
|
-
|
|
308
|
-
if (block === null) {
|
|
309
|
-
log.debug("Failed to fetch finalized block and unable to determine finalized height, aborting...");
|
|
310
|
-
throw new Error(GetBlockError);
|
|
311
|
-
}
|
|
312
|
-
await metadata.heights.upsert(chain, block.eth_getBlockByNumber.number);
|
|
313
|
-
return hexToNumber(block.eth_getBlockByNumber.number);
|
|
326
|
+
function verifyTransactionGasUsage(block) {
|
|
327
|
+
const gasLimit = viem.hexToBigInt(block.eth_getBlockByNumber.gasLimit);
|
|
328
|
+
const blockGasUsed = viem.hexToBigInt(block.eth_getBlockByNumber.gasUsed);
|
|
329
|
+
if (blockGasUsed > gasLimit) {
|
|
330
|
+
throw new Error("Method `eth_getBlockByNumber` returned gas used greater than the block gas limit");
|
|
314
331
|
}
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
]);
|
|
325
|
-
if (block === null) {
|
|
326
|
-
return;
|
|
332
|
+
let cumulativeGasUsed = 0n;
|
|
333
|
+
for (const receipt of block.eth_getBlockReceipts) {
|
|
334
|
+
cumulativeGasUsed += viem.hexToBigInt(receipt.gasUsed);
|
|
335
|
+
if (viem.hexToBigInt(receipt.cumulativeGasUsed) !== cumulativeGasUsed) {
|
|
336
|
+
throw new Error("Method `eth_getBlockReceipts` returned inconsistent cumulative gas used");
|
|
337
|
+
}
|
|
338
|
+
if (cumulativeGasUsed > gasLimit) {
|
|
339
|
+
throw new Error("Method `eth_getBlockReceipts` returned cumulative gas used greater than the block gas limit");
|
|
340
|
+
}
|
|
327
341
|
}
|
|
328
|
-
if (
|
|
329
|
-
|
|
330
|
-
throw new Error(GetBlockError);
|
|
342
|
+
if (cumulativeGasUsed !== blockGasUsed) {
|
|
343
|
+
throw new Error("Methods `eth_getBlockByNumber` and `eth_getBlockReceipts` returned inconsistent gas used");
|
|
331
344
|
}
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
345
|
+
}
|
|
346
|
+
async function getOrInitManifest(chain) {
|
|
347
|
+
const path = `manifest/v1/${normalizeHex(chain)}`;
|
|
348
|
+
const manifestRes = await opts.metadataStorage.adapter.get(path);
|
|
349
|
+
let manifest;
|
|
350
|
+
if (manifestRes === null) {
|
|
351
|
+
const block = await getBlockFromChain({ chain, number: "finalized" });
|
|
352
|
+
const chainFinalizedHeight = viem.hexToNumber(block.eth_getBlockByNumber.number);
|
|
353
|
+
const newManifest = {
|
|
354
|
+
finalized_block_height: chainFinalizedHeight,
|
|
355
|
+
finalized_block_hash: block.eth_getBlockByNumber.hash,
|
|
356
|
+
next_finalized_height: chainFinalizedHeight,
|
|
357
|
+
updated_at: Date.now()
|
|
358
|
+
};
|
|
359
|
+
await opts.metadataStorage.adapter.put(path, JSON.stringify(newManifest), { ifNoneMatch: "*" });
|
|
360
|
+
manifest = newManifest;
|
|
361
|
+
} else {
|
|
362
|
+
manifest = JSON.parse(decoder.decode(manifestRes.body));
|
|
335
363
|
}
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
364
|
+
return manifest;
|
|
365
|
+
}
|
|
366
|
+
const public_getFinalizedHeight = async (chain) => {
|
|
367
|
+
const manifest = await getOrInitManifest(chain);
|
|
368
|
+
return manifest.finalized_block_height;
|
|
369
|
+
};
|
|
370
|
+
async function writeUnfinalizedBlock(block) {
|
|
340
371
|
const events_start = Date.now();
|
|
341
|
-
const events =
|
|
372
|
+
const events = eventsGroupedByStorageMap.entries().map(async ([storage, grouped_events]) => {
|
|
342
373
|
const batch = [];
|
|
343
374
|
for (const event2 of grouped_events) {
|
|
344
375
|
try {
|
|
@@ -370,42 +401,59 @@ function indexer(opts) {
|
|
|
370
401
|
});
|
|
371
402
|
await Promise.all(events);
|
|
372
403
|
log.debug(`Wrote events in ${Date.now() - events_start}ms`);
|
|
373
|
-
|
|
374
|
-
const
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
await Promise.allSettled(actions);
|
|
386
|
-
};
|
|
387
|
-
const public_deleteReorganisedHead = async (head) => {
|
|
388
|
-
log.debug(`Received reorganised head ${hexToNumber(head.number)}`);
|
|
389
|
-
const [stored_block, canonical_block] = await Promise.all([
|
|
390
|
-
metadata.blocks.get(head),
|
|
404
|
+
const chain = normalizeHex(block.eth_chainId);
|
|
405
|
+
const hash = normalizeHex(block.eth_getBlockByNumber.hash);
|
|
406
|
+
const number = normalizeHex(block.eth_getBlockByNumber.number, 16);
|
|
407
|
+
const parentHash = normalizeHex(block.eth_getBlockByNumber.parentHash);
|
|
408
|
+
const commitsKey = `commits/v1/${chain}/${number}/${hash}/${parentHash}`;
|
|
409
|
+
await opts.metadataStorage.adapter.put(commitsKey, "commitsValue");
|
|
410
|
+
}
|
|
411
|
+
const public_writeUnfinalizedHead = async (head) => {
|
|
412
|
+
log.debug("Received unfinalized head...");
|
|
413
|
+
const blocksStart = Date.now();
|
|
414
|
+
const [block, manifest] = await Promise.all([
|
|
415
|
+
getBlockFromChain(head),
|
|
391
416
|
//
|
|
392
|
-
|
|
417
|
+
getOrInitManifest(head.chain)
|
|
393
418
|
]);
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
throw new Error("Attempted to delete unknown block");
|
|
419
|
+
log.debug(`Loaded block in ${Date.now() - blocksStart}ms`);
|
|
420
|
+
const indexerFinalizedHeight = manifest.finalized_block_height;
|
|
421
|
+
if (viem.hexToNumber(head.number) <= indexerFinalizedHeight) {
|
|
422
|
+
return log.debug("Receiving finalized head, ignoring...");
|
|
399
423
|
}
|
|
400
|
-
|
|
401
|
-
|
|
424
|
+
const metadata_start = Date.now();
|
|
425
|
+
const chain = normalizeHex(block.eth_chainId);
|
|
426
|
+
const hash = normalizeHex(block.eth_getBlockByNumber.hash);
|
|
427
|
+
const number = normalizeHex(block.eth_getBlockByNumber.number, 16);
|
|
428
|
+
const parentHash = normalizeHex(block.eth_getBlockByNumber.parentHash);
|
|
429
|
+
const blocksKey = `blocks/v1/${chain}/${number}/${hash}/${parentHash}`;
|
|
430
|
+
const blocksValue = await compress(JSON.stringify(block));
|
|
431
|
+
const conditional = {
|
|
432
|
+
ifNoneMatch: "*"
|
|
433
|
+
};
|
|
434
|
+
const result = await opts.metadataStorage.adapter.put(blocksKey, blocksValue, conditional).catch((error) => {
|
|
435
|
+
if (error instanceof AdapterError) {
|
|
436
|
+
if (error.tag === "PreconditionFailed") {
|
|
437
|
+
return null;
|
|
438
|
+
}
|
|
439
|
+
}
|
|
440
|
+
throw error;
|
|
441
|
+
});
|
|
442
|
+
if (result === null) {
|
|
443
|
+
return log.debug("Block already persisted to wal, ignoring...");
|
|
402
444
|
}
|
|
403
|
-
|
|
445
|
+
log.debug(`Persisted block to wal in ${Date.now() - metadata_start}ms`);
|
|
446
|
+
await writeUnfinalizedBlock(block);
|
|
447
|
+
};
|
|
448
|
+
async function deleteReorganisedBlocksAndWriteCanonicalBlock(reorganised, canonical) {
|
|
449
|
+
const deletes = allEvents.map(async (event2) => {
|
|
404
450
|
const batch = [];
|
|
405
451
|
try {
|
|
406
|
-
const
|
|
407
|
-
|
|
408
|
-
|
|
452
|
+
for (const block of reorganised) {
|
|
453
|
+
const events = event2.handler(block);
|
|
454
|
+
for (const event3 of events) {
|
|
455
|
+
batch.push(event3);
|
|
456
|
+
}
|
|
409
457
|
}
|
|
410
458
|
} catch (error) {
|
|
411
459
|
log.error(`Failed to run your 'handler' for event ${event2.id}`);
|
|
@@ -420,14 +468,14 @@ function indexer(opts) {
|
|
|
420
468
|
});
|
|
421
469
|
});
|
|
422
470
|
await Promise.all(deletes);
|
|
423
|
-
const upserts =
|
|
471
|
+
const upserts = eventsGroupedByStorageMap.entries().map(async ([storage, grouped_events]) => {
|
|
424
472
|
const batch = [];
|
|
425
473
|
for (const event2 of grouped_events) {
|
|
426
474
|
try {
|
|
427
|
-
if (!event2.filters.some((filter) => matchFilter(
|
|
475
|
+
if (!event2.filters.some((filter) => matchFilter(canonical, filter))) {
|
|
428
476
|
continue;
|
|
429
477
|
}
|
|
430
|
-
const events = event2.handler(
|
|
478
|
+
const events = event2.handler(canonical);
|
|
431
479
|
for (const event3 of events) {
|
|
432
480
|
batch.push(event3);
|
|
433
481
|
}
|
|
@@ -446,209 +494,338 @@ function indexer(opts) {
|
|
|
446
494
|
}
|
|
447
495
|
});
|
|
448
496
|
await Promise.all(upserts);
|
|
449
|
-
}
|
|
450
|
-
const
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
497
|
+
}
|
|
498
|
+
const public_deleteReorganisedHead = async (head) => {
|
|
499
|
+
log.debug(`Received reorganised head ${viem.hexToNumber(head.number)}`);
|
|
500
|
+
const chain = normalizeHex(head.chain);
|
|
501
|
+
const number = normalizeHex(head.number, 16);
|
|
502
|
+
const hash = normalizeHex(head.hash);
|
|
503
|
+
const parentHash = normalizeHex(head.parent_hash);
|
|
504
|
+
const blocksKey = `blocks/v1/${chain}/${number}/${hash}/${parentHash}`;
|
|
505
|
+
const [blocksRes, canonicalBlock] = await Promise.all([
|
|
506
|
+
opts.metadataStorage.adapter.get(blocksKey),
|
|
507
|
+
//
|
|
508
|
+
getBlockFromChain({ chain: head.chain, number: head.number })
|
|
509
|
+
]);
|
|
510
|
+
if (blocksRes === null) {
|
|
511
|
+
return log.debug("Reorganised block never/already processed");
|
|
463
512
|
}
|
|
464
|
-
|
|
465
|
-
|
|
513
|
+
const decompressedBlock = await decompress(blocksRes.body);
|
|
514
|
+
const storedBlock = JSON.parse(decompressedBlock);
|
|
515
|
+
if (isHexEqual(head.hash, canonicalBlock.eth_getBlockByNumber.hash)) {
|
|
516
|
+
throw new Error("Attempted to delete canonical block");
|
|
466
517
|
}
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
518
|
+
await deleteReorganisedBlocksAndWriteCanonicalBlock([storedBlock], canonicalBlock);
|
|
519
|
+
};
|
|
520
|
+
async function writeFinalizedBlock(block, actions) {
|
|
521
|
+
if (actions.length === 0) {
|
|
522
|
+
return;
|
|
470
523
|
}
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
524
|
+
const chain = normalizeHex(block.eth_chainId);
|
|
525
|
+
const number = normalizeHex(block.eth_getBlockByNumber.number, 16);
|
|
526
|
+
const hash = normalizeHex(block.eth_getBlockByNumber.hash);
|
|
527
|
+
const parentHash = normalizeHex(block.eth_getBlockByNumber.parentHash);
|
|
528
|
+
const prefix = `commits/v1/${chain}/${number}/${hash}/${parentHash}`;
|
|
529
|
+
const promises = actions.map(async (action2) => {
|
|
530
|
+
const events = action2.event.filters.some((filter) => matchFilter(block, filter)) ? action2.event.handler(block) : [];
|
|
531
|
+
const promises2 = events.map(async (event2) => {
|
|
532
|
+
await retry(() => action2.handler(event2), 2).catch((error) => {
|
|
533
|
+
log.error(`Failed to execute action ${action2.id}`);
|
|
534
|
+
throw error;
|
|
535
|
+
});
|
|
536
|
+
});
|
|
537
|
+
const results = await Promise.allSettled(promises2);
|
|
538
|
+
const failure = results.find((result) => result.status === "rejected");
|
|
539
|
+
if (failure !== void 0) {
|
|
540
|
+
throw failure.reason;
|
|
478
541
|
}
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
const [heights, finalizedBlock] = await Promise.all([
|
|
482
|
-
metadata.heights.list(chain),
|
|
483
|
-
//
|
|
484
|
-
getBlock({ chain, number: "finalized" })
|
|
485
|
-
]);
|
|
486
|
-
if (finalizedBlock === null) {
|
|
487
|
-
log.debug("Failed to load chain finalized block. Request should be retried");
|
|
488
|
-
throw new Error(GetBlockError);
|
|
489
|
-
}
|
|
490
|
-
if (heights.length === 0) {
|
|
491
|
-
await metadata.heights.upsert(chain, finalizedBlock.eth_getBlockByNumber.number);
|
|
492
|
-
heights.push({ chain, number: finalizedBlock.eth_getBlockByNumber.number });
|
|
493
|
-
}
|
|
494
|
-
const indexerHeight = Math.max(...heights.map((height) => hexToNumber(height.number)));
|
|
495
|
-
const finalizedHeight = hexToNumber(finalizedBlock.eth_getBlockByNumber.number);
|
|
496
|
-
const lastHead = heads[heads.length - 1];
|
|
497
|
-
if (lastHead === void 0 || hexToNumber(lastHead.number) !== finalizedHeight) {
|
|
498
|
-
throw new Error("Client has stalled. Chain has finalized greater than the heads received");
|
|
499
|
-
}
|
|
500
|
-
log.debug(`Indexer height ${indexerHeight}, finalized height ${finalizedHeight}`);
|
|
501
|
-
const newHeads = heads.filter((head) => {
|
|
502
|
-
return hexToNumber(head.number) > indexerHeight;
|
|
542
|
+
const commitsKey = `${prefix}/action/${action2.id}`;
|
|
543
|
+
await opts.metadataStorage.adapter.put(commitsKey, "");
|
|
503
544
|
});
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
if (
|
|
509
|
-
|
|
545
|
+
await Promise.all(promises);
|
|
546
|
+
}
|
|
547
|
+
const public_writeFinalizedHead = async (head) => {
|
|
548
|
+
log.debug("Received finalized head...");
|
|
549
|
+
if (allActions.length === 0) {
|
|
550
|
+
return;
|
|
510
551
|
}
|
|
511
|
-
|
|
512
|
-
|
|
552
|
+
const blocksStart = Date.now();
|
|
553
|
+
const [block, manifest, chainFinalizedBlock] = await Promise.all([
|
|
554
|
+
getBlockFromChain(head),
|
|
555
|
+
getOrInitManifest(head.chain),
|
|
556
|
+
getBlockFromChain({ chain: head.chain, number: "finalized" })
|
|
557
|
+
]);
|
|
558
|
+
log.debug(`Loaded block in ${Date.now() - blocksStart}ms`);
|
|
559
|
+
const receivedHeight = viem.hexToNumber(head.number);
|
|
560
|
+
const indexerFinalizedHeight = manifest.finalized_block_height;
|
|
561
|
+
const chainFinalizedHeight = viem.hexToNumber(chainFinalizedBlock.eth_getBlockByNumber.number);
|
|
562
|
+
if (receivedHeight <= indexerFinalizedHeight) {
|
|
563
|
+
return log.debug(`Received finalized head (${receivedHeight}) below indexer height (${indexerFinalizedHeight})`);
|
|
513
564
|
}
|
|
514
|
-
if (
|
|
515
|
-
|
|
565
|
+
if (receivedHeight > chainFinalizedHeight) {
|
|
566
|
+
return log.error(`Received finalized head (${receivedHeight}) that has not finalized (${chainFinalizedHeight})`);
|
|
516
567
|
}
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
568
|
+
const chain = normalizeHex(block.eth_chainId);
|
|
569
|
+
const number = normalizeHex(block.eth_getBlockByNumber.number, 16);
|
|
570
|
+
const hash = normalizeHex(block.eth_getBlockByNumber.hash);
|
|
571
|
+
const parentHash = normalizeHex(block.eth_getBlockByNumber.parentHash);
|
|
572
|
+
const finalizedKey = `finalized/v1/${chain}/${number}/${hash}/${parentHash}`;
|
|
573
|
+
const result = await opts.metadataStorage.adapter.put(finalizedKey, "", { ifNoneMatch: "*" }).catch((error) => {
|
|
574
|
+
if (error instanceof AdapterError && error.tag === "PreconditionFailed") {
|
|
575
|
+
return null;
|
|
576
|
+
}
|
|
577
|
+
throw error;
|
|
578
|
+
});
|
|
579
|
+
if (result === null) {
|
|
580
|
+
return log.debug("Finalized block already persisted to wal, ignoring...");
|
|
524
581
|
}
|
|
525
|
-
await
|
|
582
|
+
await writeFinalizedBlock(block, allActions);
|
|
526
583
|
};
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
584
|
+
const FINALIZATION_BATCH_SIZE = 32;
|
|
585
|
+
const LEASE_DURATION_MS = 60 * 1e3;
|
|
586
|
+
async function getBlocksProcessedAndGarbageCollect(chain, finalizedHeight) {
|
|
587
|
+
while (true) {
|
|
588
|
+
const blocksKey = `blocks/v1/${normalizeHex(chain)}/`;
|
|
589
|
+
const blocks = await opts.metadataStorage.adapter.list({ prefix: blocksKey, limit: 1e3 });
|
|
590
|
+
if (blocks.keys.length === 0) {
|
|
591
|
+
return [];
|
|
592
|
+
}
|
|
593
|
+
const garbageCollectionKeys = blocks.keys.filter((key) => {
|
|
594
|
+
const [_, __, ___, number] = key.split("/");
|
|
595
|
+
return viem.hexToNumber(number) <= finalizedHeight;
|
|
596
|
+
});
|
|
597
|
+
if (garbageCollectionKeys.length === 0) {
|
|
598
|
+
return blocks.keys.map((key) => {
|
|
599
|
+
const [_, __, ___, number, hash, parent_hash] = key.split("/");
|
|
600
|
+
return { chain, number, hash, parent_hash };
|
|
601
|
+
});
|
|
602
|
+
}
|
|
603
|
+
const garbageCollectionPromises = garbageCollectionKeys.map(async (key) => {
|
|
604
|
+
await opts.metadataStorage.adapter.delete(key);
|
|
605
|
+
});
|
|
606
|
+
await Promise.all(garbageCollectionPromises);
|
|
530
607
|
}
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
608
|
+
}
|
|
609
|
+
async function getCommitsAndGarbageCollect(chain, finalizedHeight) {
|
|
610
|
+
while (true) {
|
|
611
|
+
const commitsKey = `commits/v1/${normalizeHex(chain)}/`;
|
|
612
|
+
const commits = await opts.metadataStorage.adapter.list({ prefix: commitsKey, limit: 1e3 });
|
|
613
|
+
if (commits.keys.length === 0) {
|
|
534
614
|
return [];
|
|
535
615
|
}
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
616
|
+
const garbageCollectionKeys = commits.keys.filter((key) => {
|
|
617
|
+
const [_, __, ___, number] = key.split("/");
|
|
618
|
+
return viem.hexToNumber(number) <= finalizedHeight;
|
|
619
|
+
});
|
|
620
|
+
if (garbageCollectionKeys.length === 0) {
|
|
621
|
+
return commits.keys.map((key) => {
|
|
622
|
+
const [_, __, ___, number, hash, parent_hash, type, id] = key.split("/");
|
|
623
|
+
return { chain, number, hash, parent_hash, type, id };
|
|
540
624
|
});
|
|
625
|
+
}
|
|
626
|
+
const garbageCollectionPromises = garbageCollectionKeys.map(async (key) => {
|
|
627
|
+
await opts.metadataStorage.adapter.delete(key);
|
|
541
628
|
});
|
|
542
|
-
|
|
543
|
-
|
|
629
|
+
await Promise.all(garbageCollectionPromises);
|
|
630
|
+
}
|
|
631
|
+
}
|
|
632
|
+
async function getBlockFromMetadataOrChain(head) {
|
|
633
|
+
const chain = normalizeHex(head.chain);
|
|
634
|
+
const number = normalizeHex(head.number, 16);
|
|
635
|
+
const hash = normalizeHex(head.hash);
|
|
636
|
+
const parentHash = normalizeHex(head.parent_hash);
|
|
637
|
+
const prefix = `blocks/v1/${chain}/${number}/${hash}/${parentHash}`;
|
|
638
|
+
const object = await opts.metadataStorage.adapter.get(prefix);
|
|
639
|
+
if (object !== null) {
|
|
640
|
+
const block = await decompress(object.body);
|
|
641
|
+
const parsed = JSON.parse(block);
|
|
642
|
+
return parsed;
|
|
643
|
+
}
|
|
644
|
+
return await getBlockFromChain(head);
|
|
544
645
|
}
|
|
545
|
-
async
|
|
546
|
-
const
|
|
547
|
-
|
|
646
|
+
const public_finalize = async (chain) => {
|
|
647
|
+
const manifestKey = `manifest/v1/${normalizeHex(chain)}`;
|
|
648
|
+
const [chainFinalizedBlock, manifestGetRes] = await Promise.all([
|
|
649
|
+
getBlockFromChain({ chain, number: "finalized" }),
|
|
650
|
+
opts.metadataStorage.adapter.get(manifestKey)
|
|
548
651
|
//
|
|
549
|
-
metadata.commits.list(head.chain, head.number)
|
|
550
652
|
]);
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
653
|
+
const chainFinalizedHeight = viem.hexToNumber(chainFinalizedBlock.eth_getBlockByNumber.number);
|
|
654
|
+
if (manifestGetRes === null) {
|
|
655
|
+
log.debug("No manifest file found");
|
|
656
|
+
const manifest2 = {
|
|
657
|
+
finalized_block_height: chainFinalizedHeight,
|
|
658
|
+
finalized_block_hash: chainFinalizedBlock.eth_getBlockByNumber.hash,
|
|
659
|
+
next_finalized_height: chainFinalizedHeight,
|
|
660
|
+
updated_at: Date.now()
|
|
661
|
+
};
|
|
662
|
+
await opts.metadataStorage.adapter.put(manifestKey, JSON.stringify(manifest2), { ifNoneMatch: "*" });
|
|
663
|
+
return log.debug("Nothing to finalize, returning...");
|
|
664
|
+
}
|
|
665
|
+
const manifest = JSON.parse(decoder.decode(manifestGetRes.body));
|
|
666
|
+
if (manifest.finalized_block_height !== manifest.next_finalized_height) {
|
|
667
|
+
if (Date.now() - manifest.updated_at < LEASE_DURATION_MS) {
|
|
668
|
+
return log.debug("Found valid lease, returning...");
|
|
554
669
|
}
|
|
670
|
+
log.debug("Found expired lease");
|
|
555
671
|
}
|
|
556
|
-
|
|
557
|
-
|
|
672
|
+
let indexerFinalizedBlockHeight = manifest.finalized_block_height;
|
|
673
|
+
let indexerFinalizedBlockHash = manifest.finalized_block_hash;
|
|
674
|
+
if (indexerFinalizedBlockHeight >= chainFinalizedHeight) {
|
|
675
|
+
return log.debug("Indexer already finalized, returning...");
|
|
676
|
+
}
|
|
677
|
+
log.debug("Acquiring lease...");
|
|
678
|
+
const updatedManifest = {
|
|
679
|
+
finalized_block_height: indexerFinalizedBlockHeight,
|
|
680
|
+
finalized_block_hash: indexerFinalizedBlockHash,
|
|
681
|
+
next_finalized_height: Math.min(chainFinalizedHeight, indexerFinalizedBlockHeight + FINALIZATION_BATCH_SIZE),
|
|
682
|
+
updated_at: Date.now()
|
|
683
|
+
};
|
|
684
|
+
const conditional = { ifMatch: manifestGetRes.etag };
|
|
685
|
+
const manifestValue = JSON.stringify(updatedManifest);
|
|
686
|
+
const manifestPutRes = await opts.metadataStorage.adapter.put(manifestKey, manifestValue, conditional).catch((error) => {
|
|
687
|
+
if (error instanceof AdapterError && error.tag === "PreconditionFailed") {
|
|
688
|
+
return null;
|
|
689
|
+
}
|
|
690
|
+
throw error;
|
|
558
691
|
});
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
await deleteReorganisedHead(block);
|
|
562
|
-
})
|
|
563
|
-
);
|
|
564
|
-
await writeFinalizedHead(head);
|
|
565
|
-
}
|
|
566
|
-
async function writeFinalizedHead(head) {
|
|
567
|
-
const block = await getBlock({ chain: head.chain, number: head.number, hash: head.hash });
|
|
568
|
-
if (block === null) {
|
|
569
|
-
throw new Error("Received unknown head that wasn't canonical");
|
|
692
|
+
if (manifestPutRes === null) {
|
|
693
|
+
return log.debug("Failed to acquire lease, returning...");
|
|
570
694
|
}
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
695
|
+
let latestManifestEtag = manifestPutRes.etag;
|
|
696
|
+
while (indexerFinalizedBlockHeight < chainFinalizedHeight) {
|
|
697
|
+
const nextFinalizedHeight = Math.min(chainFinalizedHeight, indexerFinalizedBlockHeight + FINALIZATION_BATCH_SIZE);
|
|
698
|
+
const finalizationBatchSize = nextFinalizedHeight - indexerFinalizedBlockHeight;
|
|
699
|
+
const [nextFinalizedBlock, blocksProcessed] = await Promise.all([
|
|
700
|
+
getBlockFromChain({ chain, number: viem.numberToHex(nextFinalizedHeight) }),
|
|
701
|
+
getBlocksProcessedAndGarbageCollect(chain, indexerFinalizedBlockHeight)
|
|
702
|
+
]);
|
|
703
|
+
const nextFinalizedHead = {
|
|
704
|
+
chain,
|
|
705
|
+
hash: nextFinalizedBlock.eth_getBlockByNumber.hash,
|
|
706
|
+
number: nextFinalizedBlock.eth_getBlockByNumber.number,
|
|
707
|
+
parent_hash: nextFinalizedBlock.eth_getBlockByNumber.parentHash
|
|
708
|
+
};
|
|
709
|
+
const nextFinalizedBlockProcessed = blocksProcessed.some((block) => {
|
|
710
|
+
return isHexEqual(block.hash, nextFinalizedHead.hash) && isHexEqual(block.parent_hash, nextFinalizedHead.parent_hash) && viem.hexToNumber(block.number) === viem.hexToNumber(nextFinalizedHead.number);
|
|
711
|
+
});
|
|
712
|
+
if (nextFinalizedBlockProcessed === false) {
|
|
713
|
+
await Promise.all([
|
|
714
|
+
writeUnfinalizedBlock(nextFinalizedBlock),
|
|
715
|
+
//
|
|
716
|
+
writeFinalizedBlock(nextFinalizedBlock, allActions)
|
|
717
|
+
]);
|
|
718
|
+
blocksProcessed.push(nextFinalizedHead);
|
|
587
719
|
}
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
throw error;
|
|
720
|
+
const canonicalHeads = [nextFinalizedHead];
|
|
721
|
+
let parentHash = nextFinalizedBlock.eth_getBlockByNumber.parentHash;
|
|
722
|
+
for (let index = 1; index < finalizationBatchSize; index++) {
|
|
723
|
+
const number = nextFinalizedHeight - index;
|
|
724
|
+
const blocksProcessedForHeight = blocksProcessed.filter((block) => {
|
|
725
|
+
return viem.hexToNumber(block.number) === number;
|
|
595
726
|
});
|
|
596
|
-
|
|
597
|
-
|
|
727
|
+
const canonicalHead = blocksProcessedForHeight.find((block) => {
|
|
728
|
+
return isHexEqual(block.hash, parentHash);
|
|
729
|
+
});
|
|
730
|
+
if (canonicalHead) {
|
|
731
|
+
canonicalHeads.unshift(canonicalHead);
|
|
732
|
+
parentHash = canonicalHead.parent_hash;
|
|
733
|
+
continue;
|
|
598
734
|
}
|
|
735
|
+
log.debug("Canonical head never processed, loading from chain...");
|
|
736
|
+
const canonicalBlock = await getBlockFromChain({ chain, number: viem.numberToHex(number) });
|
|
737
|
+
const head = {
|
|
738
|
+
chain,
|
|
739
|
+
hash: canonicalBlock.eth_getBlockByNumber.hash,
|
|
740
|
+
number: canonicalBlock.eth_getBlockByNumber.number,
|
|
741
|
+
parent_hash: canonicalBlock.eth_getBlockByNumber.parentHash
|
|
742
|
+
};
|
|
743
|
+
log.debug("Processing canonical head");
|
|
744
|
+
await Promise.all([
|
|
745
|
+
writeUnfinalizedBlock(canonicalBlock),
|
|
746
|
+
//
|
|
747
|
+
writeFinalizedBlock(canonicalBlock, allActions)
|
|
748
|
+
]);
|
|
749
|
+
canonicalHeads.unshift(head);
|
|
750
|
+
blocksProcessed.push(head);
|
|
751
|
+
parentHash = canonicalBlock.eth_getBlockByNumber.parentHash;
|
|
599
752
|
}
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
}
|
|
603
|
-
async function deleteReorganisedHead(head) {
|
|
604
|
-
const block = await metadata.blocks.get(head);
|
|
605
|
-
if (block === null) {
|
|
606
|
-
throw new Error("Expected reorganised block to exist in metadata");
|
|
607
|
-
}
|
|
608
|
-
const promises = all_events.map(async (event2) => {
|
|
609
|
-
const batch = [];
|
|
610
|
-
try {
|
|
611
|
-
const events = event2.handler(block);
|
|
612
|
-
for (const event3 of events) {
|
|
613
|
-
batch.push(event3);
|
|
614
|
-
}
|
|
615
|
-
} catch (error) {
|
|
616
|
-
log.error(`Failed to run your 'handler' for event ${event2.id}`);
|
|
617
|
-
throw error;
|
|
753
|
+
if (!isHexEqual(parentHash, indexerFinalizedBlockHash)) {
|
|
754
|
+
throw new Error("Expected chain to match last finalized canonical anchor");
|
|
618
755
|
}
|
|
619
|
-
if (
|
|
620
|
-
|
|
756
|
+
if (canonicalHeads.length !== finalizationBatchSize) {
|
|
757
|
+
throw new Error(`Expected to have ${finalizationBatchSize} heads, found ${canonicalHeads.length}`);
|
|
621
758
|
}
|
|
622
|
-
|
|
623
|
-
|
|
759
|
+
log.debug("Determined canonical chain");
|
|
760
|
+
const commits = await getCommitsAndGarbageCollect(chain, indexerFinalizedBlockHeight);
|
|
761
|
+
for (const canonicalHead of canonicalHeads) {
|
|
762
|
+
const blocksProcessedForHeight = blocksProcessed.filter((head) => {
|
|
763
|
+
return viem.hexToNumber(canonicalHead.number) === viem.hexToNumber(head.number);
|
|
764
|
+
});
|
|
765
|
+
const processedOnlyCanonicalBlock = blocksProcessedForHeight.every((head) => {
|
|
766
|
+
return isHexEqual(canonicalHead.hash, head.hash) && isHexEqual(canonicalHead.parent_hash, head.parent_hash);
|
|
767
|
+
});
|
|
768
|
+
const eventsCommittedForHeight = commits.some((commit) => {
|
|
769
|
+
return commit.type === void 0 && isHexEqual(canonicalHead.hash, commit.hash) && isHexEqual(canonicalHead.parent_hash, commit.parent_hash) && viem.hexToNumber(canonicalHead.number) === viem.hexToNumber(commit.number);
|
|
770
|
+
});
|
|
771
|
+
const actionsWithoutCommit = allActions.filter((action2) => {
|
|
772
|
+
const commitExists = commits.some((commit) => {
|
|
773
|
+
return commit.id === action2.id && commit.type === "action" && isHexEqual(canonicalHead.hash, commit.hash) && isHexEqual(canonicalHead.parent_hash, commit.parent_hash) && viem.hexToNumber(canonicalHead.number) === viem.hexToNumber(commit.number);
|
|
774
|
+
});
|
|
775
|
+
return !commitExists;
|
|
776
|
+
});
|
|
777
|
+
if (eventsCommittedForHeight && processedOnlyCanonicalBlock && actionsWithoutCommit.length === 0 && blocksProcessedForHeight.length === 1) {
|
|
778
|
+
log.debug("Executed fast-path");
|
|
779
|
+
continue;
|
|
780
|
+
}
|
|
781
|
+
log.debug("Fast-path missed for head");
|
|
782
|
+
log.debug(`Events commited for height: ${eventsCommittedForHeight}`);
|
|
783
|
+
log.debug(`Processed only canonical block: ${processedOnlyCanonicalBlock}`);
|
|
784
|
+
log.debug(`Actions without commit (${actionsWithoutCommit.length}): ${actionsWithoutCommit.length === 0}`);
|
|
785
|
+
log.debug(`Blocks processed for height (${blocksProcessedForHeight.length}): ${blocksProcessedForHeight.length === 1}`);
|
|
786
|
+
const reorganisedHeads = blocksProcessedForHeight.flatMap((head) => {
|
|
787
|
+
if (isHexEqual(canonicalHead.hash, head.hash)) {
|
|
788
|
+
return [];
|
|
789
|
+
}
|
|
790
|
+
return { chain, number: head.number, hash: head.hash, parent_hash: head.parent_hash };
|
|
791
|
+
});
|
|
792
|
+
const reorganisedPromises = reorganisedHeads.map((head) => getBlockFromMetadataOrChain(head));
|
|
793
|
+
const [canonicalBlock, ...reorganisedBlocks] = await Promise.all([
|
|
794
|
+
getBlockFromMetadataOrChain(canonicalHead),
|
|
795
|
+
//
|
|
796
|
+
...reorganisedPromises
|
|
797
|
+
]);
|
|
798
|
+
log.debug("Re-processing heads");
|
|
799
|
+
await Promise.all([
|
|
800
|
+
writeFinalizedBlock(canonicalBlock, actionsWithoutCommit),
|
|
801
|
+
//
|
|
802
|
+
deleteReorganisedBlocksAndWriteCanonicalBlock(reorganisedBlocks, canonicalBlock)
|
|
803
|
+
]);
|
|
804
|
+
}
|
|
805
|
+
const nextBatchFinalizingHeight = Math.min(chainFinalizedHeight, nextFinalizedHeight + FINALIZATION_BATCH_SIZE);
|
|
806
|
+
const updatedManifest2 = {
|
|
807
|
+
finalized_block_height: nextFinalizedHeight,
|
|
808
|
+
finalized_block_hash: nextFinalizedBlock.eth_getBlockByNumber.hash,
|
|
809
|
+
next_finalized_height: nextBatchFinalizingHeight,
|
|
810
|
+
updated_at: Date.now()
|
|
811
|
+
};
|
|
812
|
+
const manifestPutRes2 = await opts.metadataStorage.adapter.put(manifestKey, JSON.stringify(updatedManifest2), { ifMatch: latestManifestEtag }).catch((error) => {
|
|
813
|
+
if (error instanceof AdapterError) {
|
|
814
|
+
if (error.tag === "PreconditionFailed") {
|
|
815
|
+
return null;
|
|
816
|
+
}
|
|
817
|
+
}
|
|
624
818
|
throw error;
|
|
625
819
|
});
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
]);
|
|
636
|
-
const blocks = _blocks.filter((block) => {
|
|
637
|
-
return hexToNumber(block.number) <= hexToNumber(finalized);
|
|
638
|
-
});
|
|
639
|
-
const commits = _commits.filter((commit) => {
|
|
640
|
-
return hexToNumber(commit.number) <= hexToNumber(finalized);
|
|
641
|
-
});
|
|
642
|
-
const heights = _heights.filter((height) => {
|
|
643
|
-
return hexToNumber(height.number) < hexToNumber(finalized);
|
|
644
|
-
});
|
|
645
|
-
await Promise.all([
|
|
646
|
-
metadata.blocks.delete(blocks),
|
|
647
|
-
//
|
|
648
|
-
metadata.commits.delete(commits),
|
|
649
|
-
metadata.heights.delete(heights)
|
|
650
|
-
]);
|
|
651
|
-
}
|
|
820
|
+
if (manifestPutRes2 === null) {
|
|
821
|
+
return log.debug("Failed to commit finalized batch, aborting...");
|
|
822
|
+
}
|
|
823
|
+
indexerFinalizedBlockHeight = nextFinalizedHeight;
|
|
824
|
+
indexerFinalizedBlockHash = nextFinalizedBlock.eth_getBlockByNumber.hash;
|
|
825
|
+
latestManifestEtag = manifestPutRes2.etag;
|
|
826
|
+
}
|
|
827
|
+
log.debug("Indexer finalized, returning...");
|
|
828
|
+
};
|
|
652
829
|
const private_getMetadata = async () => {
|
|
653
830
|
return {
|
|
654
831
|
version,
|
|
@@ -656,7 +833,7 @@ function indexer(opts) {
|
|
|
656
833
|
};
|
|
657
834
|
};
|
|
658
835
|
const private_getEvents = async () => {
|
|
659
|
-
return
|
|
836
|
+
return allEvents.map((event2) => {
|
|
660
837
|
const filters = event2.filters.map((filter) => {
|
|
661
838
|
return {
|
|
662
839
|
chain: filter.chain,
|
|
@@ -712,8 +889,8 @@ function indexer(opts) {
|
|
|
712
889
|
};
|
|
713
890
|
const IncompleteBlockError = createException("Received block with missing required property");
|
|
714
891
|
const private_writeEvents = async (params) => {
|
|
715
|
-
if (
|
|
716
|
-
const relevant_events =
|
|
892
|
+
if (allEvents.length === 0) return { failures: [] };
|
|
893
|
+
const relevant_events = allEvents.filter((event2) => params.events.includes(event2.id));
|
|
717
894
|
if (relevant_events.length === 0) return { failures: [] };
|
|
718
895
|
if (params.events.length === 0) return { failures: [] };
|
|
719
896
|
const failures = {};
|
|
@@ -722,7 +899,7 @@ function indexer(opts) {
|
|
|
722
899
|
accessed_undefined_key = true;
|
|
723
900
|
throw new Error(IncompleteBlockError);
|
|
724
901
|
});
|
|
725
|
-
const promises =
|
|
902
|
+
const promises = eventsGroupedByStorageMap.entries().map(async ([storage, grouped_events]) => {
|
|
726
903
|
const batch = [];
|
|
727
904
|
for (const event2 of grouped_events) {
|
|
728
905
|
if (!params.events.includes(event2.id)) continue;
|
|
@@ -788,33 +965,19 @@ function indexer(opts) {
|
|
|
788
965
|
return { failures: failures_array };
|
|
789
966
|
};
|
|
790
967
|
const private_writeEventsAndGetKeys = async (params) => {
|
|
791
|
-
if (
|
|
968
|
+
if (allEvents.length === 0) {
|
|
792
969
|
return { results: [], keys: [] };
|
|
793
970
|
}
|
|
794
971
|
if (params.events.length === 0) {
|
|
795
972
|
return { results: [], keys: [] };
|
|
796
973
|
}
|
|
797
|
-
const relevant_events =
|
|
974
|
+
const relevant_events = allEvents.filter((event2) => params.events.includes(event2.id));
|
|
798
975
|
if (relevant_events.length === 0) {
|
|
799
976
|
return { results: [], keys: [] };
|
|
800
977
|
}
|
|
801
|
-
const block = await
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
keys: [],
|
|
805
|
-
results: relevant_events.map((event2) => {
|
|
806
|
-
return {
|
|
807
|
-
event_id: event2.id,
|
|
808
|
-
status: "block_error",
|
|
809
|
-
chain: params.head.chain,
|
|
810
|
-
hash: params.head.hash,
|
|
811
|
-
number: params.head.number,
|
|
812
|
-
parent_hash: params.head.parent_hash,
|
|
813
|
-
created_at: Date.now()
|
|
814
|
-
};
|
|
815
|
-
})
|
|
816
|
-
};
|
|
817
|
-
}
|
|
978
|
+
const block = await getBlockFromChain(params.head).catch(() => {
|
|
979
|
+
return null;
|
|
980
|
+
});
|
|
818
981
|
const keys = /* @__PURE__ */ new Set();
|
|
819
982
|
const results = {};
|
|
820
983
|
function createProxyHandler(path) {
|
|
@@ -915,20 +1078,27 @@ function indexer(opts) {
|
|
|
915
1078
|
if (!/^[A-Za-z0-9_-]+$/.test(event2.id)) {
|
|
916
1079
|
throw new Error(`Invalid event id \`${event2.id}\`. Only characters A-Z, a-z, 0-9, underscores, and hyphens are permitted.`);
|
|
917
1080
|
}
|
|
918
|
-
|
|
919
|
-
const group =
|
|
1081
|
+
allEvents.push(event2);
|
|
1082
|
+
const group = eventsGroupedByStorageMap.get(event2.storage) ?? [];
|
|
920
1083
|
group.push(event2);
|
|
921
|
-
|
|
1084
|
+
eventsGroupedByStorageMap.set(event2.storage, group);
|
|
922
1085
|
return event2;
|
|
923
1086
|
};
|
|
924
1087
|
const action = (action2) => {
|
|
925
|
-
|
|
1088
|
+
if (!/^[A-Za-z0-9_-]+$/.test(action2.id)) {
|
|
1089
|
+
throw new Error(`Invalid action id \`${action2.id}\`. Only characters A-Z, a-z, 0-9, underscores, and hyphens are permitted.`);
|
|
1090
|
+
}
|
|
1091
|
+
if (allActions.some((existing) => existing.id === action2.id)) {
|
|
1092
|
+
throw new Error(`Duplicate action id \`${action2.id}\`.`);
|
|
1093
|
+
}
|
|
1094
|
+
allActions.push(action2);
|
|
926
1095
|
return action2;
|
|
927
1096
|
};
|
|
928
1097
|
const rpc = {
|
|
929
1098
|
request: {
|
|
1099
|
+
public_finalize,
|
|
930
1100
|
public_getFinalizedHeight,
|
|
931
|
-
|
|
1101
|
+
public_writeFinalizedHead,
|
|
932
1102
|
public_writeUnfinalizedHead,
|
|
933
1103
|
public_deleteReorganisedHead,
|
|
934
1104
|
private_getEvents,
|
|
@@ -943,13 +1113,13 @@ function indexer(opts) {
|
|
|
943
1113
|
quiet: opts.quiet ?? false,
|
|
944
1114
|
signingKey: opts.signingKey
|
|
945
1115
|
});
|
|
946
|
-
const
|
|
1116
|
+
const indexer = {
|
|
947
1117
|
...rpc,
|
|
948
1118
|
event,
|
|
949
1119
|
action,
|
|
950
1120
|
fetch: server.http
|
|
951
1121
|
};
|
|
952
|
-
return
|
|
1122
|
+
return indexer;
|
|
953
1123
|
}
|
|
954
1124
|
|
|
955
|
-
export {
|
|
1125
|
+
export { defineIndexer };
|