ponder 0.17.2 → 0.17.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -0
- package/dist/esm/indexing/index.js +2 -2
- package/dist/esm/indexing/index.js.map +1 -1
- package/dist/esm/indexing-store/cache.js +6 -21
- package/dist/esm/indexing-store/cache.js.map +1 -1
- package/dist/esm/indexing-store/index.js +35 -39
- package/dist/esm/indexing-store/index.js.map +1 -1
- package/dist/esm/indexing-store/utils.js +21 -1
- package/dist/esm/indexing-store/utils.js.map +1 -1
- package/dist/esm/internal/metrics.js +68 -0
- package/dist/esm/internal/metrics.js.map +1 -1
- package/dist/esm/runtime/isolated.js +1 -0
- package/dist/esm/runtime/isolated.js.map +1 -1
- package/dist/esm/runtime/multichain.js +1 -0
- package/dist/esm/runtime/multichain.js.map +1 -1
- package/dist/esm/runtime/omnichain.js +1 -0
- package/dist/esm/runtime/omnichain.js.map +1 -1
- package/dist/esm/sync-realtime/index.js +1 -1
- package/dist/esm/sync-realtime/index.js.map +1 -1
- package/dist/esm/utils/copy.js +69 -58
- package/dist/esm/utils/copy.js.map +1 -1
- package/dist/types/indexing/index.d.ts.map +1 -1
- package/dist/types/indexing-store/cache.d.ts.map +1 -1
- package/dist/types/indexing-store/index.d.ts.map +1 -1
- package/dist/types/indexing-store/utils.d.ts +5 -0
- package/dist/types/indexing-store/utils.d.ts.map +1 -1
- package/dist/types/internal/metrics.d.ts +13 -0
- package/dist/types/internal/metrics.d.ts.map +1 -1
- package/dist/types/runtime/isolated.d.ts.map +1 -1
- package/dist/types/runtime/multichain.d.ts.map +1 -1
- package/dist/types/runtime/omnichain.d.ts.map +1 -1
- package/dist/types/utils/copy.d.ts +1 -1
- package/dist/types/utils/copy.d.ts.map +1 -1
- package/package.json +2 -2
- package/src/indexing/index.ts +4 -6
- package/src/indexing-store/cache.ts +20 -22
- package/src/indexing-store/index.ts +76 -38
- package/src/indexing-store/utils.ts +29 -1
- package/src/internal/metrics.ts +81 -0
- package/src/runtime/isolated.ts +1 -0
- package/src/runtime/multichain.ts +1 -0
- package/src/runtime/omnichain.ts +1 -0
- package/src/sync-realtime/index.ts +1 -1
- package/src/utils/copy.ts +76 -62
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"omnichain.d.ts","sourceRoot":"","sources":["../../../src/runtime/omnichain.ts"],"names":[],"mappings":"AAYA,OAAO,EACL,KAAK,QAAQ,EAGd,MAAM,qBAAqB,CAAC;AAU7B,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AAOnD,OAAO,KAAK,EACV,KAAK,EACL,uBAAuB,EAEvB,aAAa,EAEb,cAAc,EACd,QAAQ,EACR,WAAW,EAEZ,MAAM,qBAAqB,CAAC;AAsB7B,OAAO,EAML,KAAK,YAAY,EAClB,MAAM,YAAY,CAAC;AAGpB,wBAAsB,YAAY,CAAC,EACjC,MAAM,EACN,QAAQ,EACR,cAAc,EACd,WAAW,EACX,aAAa,EACb,uBAAuB,EACvB,QAAQ,GACT,EAAE;IACD,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,QAAQ,CAAC;IACnB,cAAc,EAAE,cAAc,CAAC;IAC/B,WAAW,EAAE,WAAW,CAAC;IACzB,aAAa,EAAE,aAAa,CAAC;IAC7B,uBAAuB,EAAE,uBAAuB,CAAC;IACjD,QAAQ,EAAE,QAAQ,CAAC;CACpB,
|
|
1
|
+
{"version":3,"file":"omnichain.d.ts","sourceRoot":"","sources":["../../../src/runtime/omnichain.ts"],"names":[],"mappings":"AAYA,OAAO,EACL,KAAK,QAAQ,EAGd,MAAM,qBAAqB,CAAC;AAU7B,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AAOnD,OAAO,KAAK,EACV,KAAK,EACL,uBAAuB,EAEvB,aAAa,EAEb,cAAc,EACd,QAAQ,EACR,WAAW,EAEZ,MAAM,qBAAqB,CAAC;AAsB7B,OAAO,EAML,KAAK,YAAY,EAClB,MAAM,YAAY,CAAC;AAGpB,wBAAsB,YAAY,CAAC,EACjC,MAAM,EACN,QAAQ,EACR,cAAc,EACd,WAAW,EACX,aAAa,EACb,uBAAuB,EACvB,QAAQ,GACT,EAAE;IACD,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,QAAQ,CAAC;IACnB,cAAc,EAAE,cAAc,CAAC;IAC/B,WAAW,EAAE,WAAW,CAAC;IACzB,aAAa,EAAE,aAAa,CAAC;IAC7B,uBAAuB,EAAE,uBAAuB,CAAC;IACjD,QAAQ,EAAE,QAAQ,CAAC;CACpB,iBAmxBA;AAED;;GAEG;AACH,eAAO,MAAM,sBAAsB;kBAMnB,IAAI,KAAK,EAAE;QAAE,YAAY,EAAE,YAAY,CAAA;KAAE,CAAC;;qDAqCzD,CAAC"}
|
|
@@ -12,5 +12,5 @@ export declare const copyOnWrite: <T extends object>(obj: T) => T;
|
|
|
12
12
|
* @dev This function supports copying objects that
|
|
13
13
|
* have been created with `copyOnWrite`.
|
|
14
14
|
*/
|
|
15
|
-
export declare const copy: <T>(obj: T) => T;
|
|
15
|
+
export declare const copy: <T>(obj: T, fast: boolean) => T;
|
|
16
16
|
//# sourceMappingURL=copy.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"copy.d.ts","sourceRoot":"","sources":["../../../src/utils/copy.ts"],"names":[],"mappings":"AAEA;;GAEG;AACH,eAAO,MAAM,aAAa,eAAmC,CAAC;
|
|
1
|
+
{"version":3,"file":"copy.d.ts","sourceRoot":"","sources":["../../../src/utils/copy.ts"],"names":[],"mappings":"AAEA;;GAEG;AACH,eAAO,MAAM,aAAa,eAAmC,CAAC;AAQ9D;;GAEG;AACH,eAAO,MAAM,WAAW,0BAA2B,CAAC,KAAG,CA4DtD,CAAC;AAEF;;;;;GAKG;AACH,eAAO,MAAM,IAAI,WAAY,CAAC,QAAQ,OAAO,KAAG,CAoC/C,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "ponder",
|
|
3
|
-
"version": "0.17.
|
|
3
|
+
"version": "0.17.4",
|
|
4
4
|
"description": "An open-source TypeScript framework for EVM data indexing",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -102,7 +102,7 @@
|
|
|
102
102
|
"rimraf": "^5.0.5",
|
|
103
103
|
"tsx": "^4.19.2",
|
|
104
104
|
"vitest": "1.6.1",
|
|
105
|
-
"@ponder/client": "0.17.
|
|
105
|
+
"@ponder/client": "0.17.4"
|
|
106
106
|
},
|
|
107
107
|
"engines": {
|
|
108
108
|
"node": ">=22"
|
package/src/indexing/index.ts
CHANGED
|
@@ -549,9 +549,8 @@ export const createIndexing = ({
|
|
|
549
549
|
// @ts-expect-error
|
|
550
550
|
await executeEvent({ ...event, event: proxyEvent });
|
|
551
551
|
|
|
552
|
-
common.metrics.
|
|
553
|
-
|
|
554
|
-
1,
|
|
552
|
+
common.metrics.incrementIndexingCompletedEvents(
|
|
553
|
+
event.eventCallback.name,
|
|
555
554
|
);
|
|
556
555
|
columnAccessPattern.get(event.eventCallback.name)!.count++;
|
|
557
556
|
eventCount[event.eventCallback.name] =
|
|
@@ -716,9 +715,8 @@ export const createIndexing = ({
|
|
|
716
715
|
|
|
717
716
|
await executeEvent(event);
|
|
718
717
|
|
|
719
|
-
common.metrics.
|
|
720
|
-
|
|
721
|
-
1,
|
|
718
|
+
common.metrics.incrementIndexingCompletedEvents(
|
|
719
|
+
event.eventCallback.name,
|
|
722
720
|
);
|
|
723
721
|
eventCount[event.eventCallback.name] =
|
|
724
722
|
eventCount[event.eventCallback.name]! + 1;
|
|
@@ -435,10 +435,10 @@ export const createIndexingCache = ({
|
|
|
435
435
|
updateBuffer.get(table)!.get(ck) ?? insertBuffer.get(table)!.get(ck);
|
|
436
436
|
|
|
437
437
|
if (bufferEntry) {
|
|
438
|
-
common.metrics.
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
438
|
+
common.metrics.incrementIndexingCacheRequests(
|
|
439
|
+
getTableName(table),
|
|
440
|
+
cache.get(table)!.isCacheComplete ? "complete" : "hit",
|
|
441
|
+
);
|
|
442
442
|
return bufferEntry.row;
|
|
443
443
|
}
|
|
444
444
|
|
|
@@ -452,29 +452,29 @@ export const createIndexingCache = ({
|
|
|
452
452
|
cache.get(table)!.spillover.add(ck);
|
|
453
453
|
}
|
|
454
454
|
|
|
455
|
-
common.metrics.
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
455
|
+
common.metrics.incrementIndexingCacheRequests(
|
|
456
|
+
getTableName(table),
|
|
457
|
+
cache.get(table)!.isCacheComplete ? "complete" : "hit",
|
|
458
|
+
);
|
|
459
459
|
return entry;
|
|
460
460
|
}
|
|
461
461
|
|
|
462
462
|
cache.get(table)!.diskReads++;
|
|
463
463
|
|
|
464
464
|
if (cache.get(table)!.isCacheComplete) {
|
|
465
|
-
common.metrics.
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
465
|
+
common.metrics.incrementIndexingCacheRequests(
|
|
466
|
+
getTableName(table),
|
|
467
|
+
"complete",
|
|
468
|
+
);
|
|
469
469
|
return null;
|
|
470
470
|
}
|
|
471
471
|
|
|
472
472
|
cache.get(table)!.spillover.add(ck);
|
|
473
473
|
|
|
474
|
-
common.metrics.
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
474
|
+
common.metrics.incrementIndexingCacheRequests(
|
|
475
|
+
getTableName(table),
|
|
476
|
+
"miss",
|
|
477
|
+
);
|
|
478
478
|
|
|
479
479
|
const endClock = startClock();
|
|
480
480
|
|
|
@@ -502,7 +502,7 @@ export const createIndexingCache = ({
|
|
|
502
502
|
return result;
|
|
503
503
|
},
|
|
504
504
|
set({ table, key, row: _row, isUpdate }) {
|
|
505
|
-
const row = normalizeRow(table, _row,
|
|
505
|
+
const row = isUpdate ? _row : normalizeRow(table, _row, false);
|
|
506
506
|
const ck = getCacheKey(table, key, primaryKeyCache);
|
|
507
507
|
|
|
508
508
|
if (isUpdate) {
|
|
@@ -964,11 +964,9 @@ export const createIndexingCache = ({
|
|
|
964
964
|
}
|
|
965
965
|
|
|
966
966
|
for (const [table, tablePredictions] of prediction) {
|
|
967
|
-
common.metrics.
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
type: "prefetch",
|
|
971
|
-
},
|
|
967
|
+
common.metrics.incrementIndexingCacheRequests(
|
|
968
|
+
getTableName(table),
|
|
969
|
+
"prefetch",
|
|
972
970
|
tablePredictions.size,
|
|
973
971
|
);
|
|
974
972
|
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
type Column,
|
|
3
|
+
getTableColumns,
|
|
3
4
|
getTableName,
|
|
4
5
|
getViewName,
|
|
5
6
|
isTable,
|
|
@@ -35,7 +36,7 @@ import { prettyPrint } from "@/utils/print.js";
|
|
|
35
36
|
import { getSQLQueryRelations, isReadonlySQLQuery } from "@/utils/sql-parse.js";
|
|
36
37
|
import { startClock } from "@/utils/timer.js";
|
|
37
38
|
import type { IndexingCache, Row } from "./cache.js";
|
|
38
|
-
import { getPrimaryKeyCache } from "./utils.js";
|
|
39
|
+
import { getPrimaryKeyCache, normalizeUpdateSet } from "./utils.js";
|
|
39
40
|
|
|
40
41
|
export type IndexingStore = {
|
|
41
42
|
db: Db<Schema>;
|
|
@@ -117,6 +118,14 @@ export const createIndexingStore = ({
|
|
|
117
118
|
const tables = Object.values(schema).filter(isTable);
|
|
118
119
|
const views = Object.values(schema).filter(isView);
|
|
119
120
|
const primaryKeyCache = getPrimaryKeyCache(tables);
|
|
121
|
+
const isCopyFast = new Map(
|
|
122
|
+
tables.map((table) => [
|
|
123
|
+
table,
|
|
124
|
+
Object.values(getTableColumns(table)).every((column) =>
|
|
125
|
+
["string", "number", "boolean", "bigint"].includes(column.dataType),
|
|
126
|
+
),
|
|
127
|
+
]),
|
|
128
|
+
);
|
|
120
129
|
|
|
121
130
|
const lock = createLock();
|
|
122
131
|
|
|
@@ -161,10 +170,10 @@ export const createIndexingStore = ({
|
|
|
161
170
|
return {
|
|
162
171
|
db: {
|
|
163
172
|
find: storeMethodWrapper(async (table: Table, key) => {
|
|
164
|
-
common.metrics.
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
173
|
+
common.metrics.incrementIndexingStoreQueries(
|
|
174
|
+
getTableName(table),
|
|
175
|
+
"find",
|
|
176
|
+
);
|
|
168
177
|
checkOnchainTable(table, "find");
|
|
169
178
|
checkTableAccess(table, "find", key, chainId);
|
|
170
179
|
const ponderRow = await indexingCache.get({ table, key });
|
|
@@ -177,13 +186,17 @@ export const createIndexingStore = ({
|
|
|
177
186
|
values: (userValues: any) => {
|
|
178
187
|
const inner = {
|
|
179
188
|
onConflictDoNothing: storeMethodWrapper(async () => {
|
|
180
|
-
common.metrics.
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
189
|
+
common.metrics.incrementIndexingStoreQueries(
|
|
190
|
+
getTableName(table),
|
|
191
|
+
"insert",
|
|
192
|
+
);
|
|
184
193
|
checkOnchainTable(table, "insert");
|
|
185
194
|
|
|
186
|
-
const ponderValues =
|
|
195
|
+
const ponderValues = Array.isArray(userValues)
|
|
196
|
+
? userValues.map((value) =>
|
|
197
|
+
copy(value, isCopyFast.get(table)!),
|
|
198
|
+
)
|
|
199
|
+
: copy(userValues, isCopyFast.get(table)!);
|
|
187
200
|
|
|
188
201
|
if (Array.isArray(ponderValues)) {
|
|
189
202
|
const ponderRows = [];
|
|
@@ -232,10 +245,10 @@ export const createIndexingStore = ({
|
|
|
232
245
|
}),
|
|
233
246
|
onConflictDoUpdate: storeMethodWrapper(
|
|
234
247
|
async (userUpdateValues: any) => {
|
|
235
|
-
common.metrics.
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
248
|
+
common.metrics.incrementIndexingStoreQueries(
|
|
249
|
+
getTableName(table),
|
|
250
|
+
"insert",
|
|
251
|
+
);
|
|
239
252
|
checkOnchainTable(table, "insert");
|
|
240
253
|
|
|
241
254
|
if (Array.isArray(userValues)) {
|
|
@@ -248,17 +261,24 @@ export const createIndexingStore = ({
|
|
|
248
261
|
});
|
|
249
262
|
|
|
250
263
|
if (ponderRowUpdate) {
|
|
251
|
-
ponderRowUpdate = copy(
|
|
264
|
+
ponderRowUpdate = copy(
|
|
265
|
+
ponderRowUpdate,
|
|
266
|
+
isCopyFast.get(table)!,
|
|
267
|
+
);
|
|
252
268
|
if (typeof userUpdateValues === "function") {
|
|
253
269
|
const userRowUpdate = copyOnWrite(ponderRowUpdate);
|
|
254
270
|
const userSet = userUpdateValues(userRowUpdate);
|
|
255
|
-
const ponderSet = copy(
|
|
271
|
+
const ponderSet = copy(
|
|
272
|
+
userSet,
|
|
273
|
+
isCopyFast.get(table)!,
|
|
274
|
+
);
|
|
256
275
|
validateUpdateSet(
|
|
257
276
|
table,
|
|
258
277
|
ponderSet,
|
|
259
278
|
ponderRowUpdate,
|
|
260
279
|
primaryKeyCache,
|
|
261
280
|
);
|
|
281
|
+
normalizeUpdateSet(table, ponderSet);
|
|
262
282
|
for (const [key, value] of Object.entries(
|
|
263
283
|
ponderSet,
|
|
264
284
|
)) {
|
|
@@ -267,13 +287,17 @@ export const createIndexingStore = ({
|
|
|
267
287
|
}
|
|
268
288
|
} else {
|
|
269
289
|
const userSet = userUpdateValues;
|
|
270
|
-
const ponderSet = copy(
|
|
290
|
+
const ponderSet = copy(
|
|
291
|
+
userSet,
|
|
292
|
+
isCopyFast.get(table)!,
|
|
293
|
+
);
|
|
271
294
|
validateUpdateSet(
|
|
272
295
|
table,
|
|
273
296
|
ponderSet,
|
|
274
297
|
ponderRowUpdate,
|
|
275
298
|
primaryKeyCache,
|
|
276
299
|
);
|
|
300
|
+
normalizeUpdateSet(table, ponderSet);
|
|
277
301
|
for (const [key, value] of Object.entries(
|
|
278
302
|
ponderSet,
|
|
279
303
|
)) {
|
|
@@ -290,7 +314,7 @@ export const createIndexingStore = ({
|
|
|
290
314
|
}),
|
|
291
315
|
);
|
|
292
316
|
} else {
|
|
293
|
-
const ponderValue = copy(value);
|
|
317
|
+
const ponderValue = copy(value, isCopyFast.get(table)!);
|
|
294
318
|
ponderRows.push(
|
|
295
319
|
indexingCache.set({
|
|
296
320
|
table,
|
|
@@ -313,30 +337,35 @@ export const createIndexingStore = ({
|
|
|
313
337
|
});
|
|
314
338
|
|
|
315
339
|
if (ponderRowUpdate) {
|
|
316
|
-
ponderRowUpdate = copy(
|
|
340
|
+
ponderRowUpdate = copy(
|
|
341
|
+
ponderRowUpdate,
|
|
342
|
+
isCopyFast.get(table)!,
|
|
343
|
+
);
|
|
317
344
|
if (typeof userUpdateValues === "function") {
|
|
318
345
|
const userRowUpdate = copyOnWrite(ponderRowUpdate);
|
|
319
346
|
const userSet = userUpdateValues(userRowUpdate);
|
|
320
|
-
const ponderSet = copy(userSet);
|
|
347
|
+
const ponderSet = copy(userSet, isCopyFast.get(table)!);
|
|
321
348
|
validateUpdateSet(
|
|
322
349
|
table,
|
|
323
350
|
ponderSet,
|
|
324
351
|
ponderRowUpdate,
|
|
325
352
|
primaryKeyCache,
|
|
326
353
|
);
|
|
354
|
+
normalizeUpdateSet(table, ponderSet);
|
|
327
355
|
for (const [key, value] of Object.entries(ponderSet)) {
|
|
328
356
|
if (value === undefined) continue;
|
|
329
357
|
ponderRowUpdate[key] = value;
|
|
330
358
|
}
|
|
331
359
|
} else {
|
|
332
360
|
const userSet = userUpdateValues;
|
|
333
|
-
const ponderSet = copy(userSet);
|
|
361
|
+
const ponderSet = copy(userSet, isCopyFast.get(table)!);
|
|
334
362
|
validateUpdateSet(
|
|
335
363
|
table,
|
|
336
364
|
ponderSet,
|
|
337
365
|
ponderRowUpdate,
|
|
338
366
|
primaryKeyCache,
|
|
339
367
|
);
|
|
368
|
+
normalizeUpdateSet(table, ponderSet);
|
|
340
369
|
for (const [key, value] of Object.entries(ponderSet)) {
|
|
341
370
|
if (value === undefined) continue;
|
|
342
371
|
ponderRowUpdate[key] = value;
|
|
@@ -352,7 +381,10 @@ export const createIndexingStore = ({
|
|
|
352
381
|
return userRow;
|
|
353
382
|
}
|
|
354
383
|
|
|
355
|
-
const ponderValues = copy(
|
|
384
|
+
const ponderValues = copy(
|
|
385
|
+
userValues,
|
|
386
|
+
isCopyFast.get(table)!,
|
|
387
|
+
);
|
|
356
388
|
|
|
357
389
|
const ponderRowInsert = indexingCache.set({
|
|
358
390
|
table,
|
|
@@ -369,12 +401,16 @@ export const createIndexingStore = ({
|
|
|
369
401
|
// biome-ignore lint/suspicious/noThenProperty: The returned object is intentionally thenable for the query API.
|
|
370
402
|
then: (onFulfilled, onRejected) =>
|
|
371
403
|
storeMethodWrapper(async () => {
|
|
372
|
-
common.metrics.
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
404
|
+
common.metrics.incrementIndexingStoreQueries(
|
|
405
|
+
getTableName(table),
|
|
406
|
+
"insert",
|
|
407
|
+
);
|
|
376
408
|
checkOnchainTable(table, "insert");
|
|
377
|
-
const ponderValues =
|
|
409
|
+
const ponderValues = Array.isArray(userValues)
|
|
410
|
+
? userValues.map((value) =>
|
|
411
|
+
copy(value, isCopyFast.get(table)!),
|
|
412
|
+
)
|
|
413
|
+
: copy(userValues, isCopyFast.get(table)!);
|
|
378
414
|
|
|
379
415
|
if (Array.isArray(ponderValues)) {
|
|
380
416
|
const ponderRows = [];
|
|
@@ -488,10 +524,10 @@ export const createIndexingStore = ({
|
|
|
488
524
|
update(table: Table, key) {
|
|
489
525
|
return {
|
|
490
526
|
set: storeMethodWrapper(async (userValues: any) => {
|
|
491
|
-
common.metrics.
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
527
|
+
common.metrics.incrementIndexingStoreQueries(
|
|
528
|
+
getTableName(table),
|
|
529
|
+
"update",
|
|
530
|
+
);
|
|
495
531
|
checkOnchainTable(table, "update");
|
|
496
532
|
checkTableAccess(table, "update", key, chainId);
|
|
497
533
|
|
|
@@ -507,31 +543,33 @@ export const createIndexingStore = ({
|
|
|
507
543
|
throw error;
|
|
508
544
|
}
|
|
509
545
|
|
|
510
|
-
ponderRowUpdate = copy(ponderRowUpdate);
|
|
546
|
+
ponderRowUpdate = copy(ponderRowUpdate, isCopyFast.get(table)!);
|
|
511
547
|
|
|
512
548
|
if (typeof userValues === "function") {
|
|
513
549
|
const userRow = copyOnWrite(ponderRowUpdate);
|
|
514
550
|
const userSet = userValues(userRow);
|
|
515
|
-
const ponderSet = copy(userSet);
|
|
551
|
+
const ponderSet = copy(userSet, isCopyFast.get(table)!);
|
|
516
552
|
validateUpdateSet(
|
|
517
553
|
table,
|
|
518
554
|
ponderSet,
|
|
519
555
|
ponderRowUpdate,
|
|
520
556
|
primaryKeyCache,
|
|
521
557
|
);
|
|
558
|
+
normalizeUpdateSet(table, ponderSet);
|
|
522
559
|
for (const [key, value] of Object.entries(ponderSet)) {
|
|
523
560
|
if (value === undefined) continue;
|
|
524
561
|
ponderRowUpdate[key] = value;
|
|
525
562
|
}
|
|
526
563
|
} else {
|
|
527
564
|
const userSet = userValues;
|
|
528
|
-
const ponderSet = copy(userSet);
|
|
565
|
+
const ponderSet = copy(userSet, isCopyFast.get(table)!);
|
|
529
566
|
validateUpdateSet(
|
|
530
567
|
table,
|
|
531
568
|
ponderSet,
|
|
532
569
|
ponderRowUpdate,
|
|
533
570
|
primaryKeyCache,
|
|
534
571
|
);
|
|
572
|
+
normalizeUpdateSet(table, ponderSet);
|
|
535
573
|
for (const [key, value] of Object.entries(ponderSet)) {
|
|
536
574
|
if (value === undefined) continue;
|
|
537
575
|
ponderRowUpdate[key] = value;
|
|
@@ -550,10 +588,10 @@ export const createIndexingStore = ({
|
|
|
550
588
|
};
|
|
551
589
|
},
|
|
552
590
|
delete: storeMethodWrapper(async (table: Table, key) => {
|
|
553
|
-
common.metrics.
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
591
|
+
common.metrics.incrementIndexingStoreQueries(
|
|
592
|
+
getTableName(table),
|
|
593
|
+
"delete",
|
|
594
|
+
);
|
|
557
595
|
checkOnchainTable(table, "delete");
|
|
558
596
|
checkTableAccess(table, "delete", key, chainId);
|
|
559
597
|
return indexingCache.delete({ table, key });
|
|
@@ -16,6 +16,21 @@ import {
|
|
|
16
16
|
} from "@/internal/errors.js";
|
|
17
17
|
import { prettyPrint } from "@/utils/print.js";
|
|
18
18
|
|
|
19
|
+
const tableColumnCache = new WeakMap<
|
|
20
|
+
Table,
|
|
21
|
+
{ columns: Record<string, Column>; entries: [string, Column][] }
|
|
22
|
+
>();
|
|
23
|
+
|
|
24
|
+
const getCachedTableColumns = (table: Table) => {
|
|
25
|
+
let cached = tableColumnCache.get(table);
|
|
26
|
+
if (cached === undefined) {
|
|
27
|
+
const columns = getTableColumns(table);
|
|
28
|
+
cached = { columns, entries: Object.entries(columns) };
|
|
29
|
+
tableColumnCache.set(table, cached);
|
|
30
|
+
}
|
|
31
|
+
return cached;
|
|
32
|
+
};
|
|
33
|
+
|
|
19
34
|
/**
|
|
20
35
|
* Returns true if the column has a "default" value that is used when no value is passed.
|
|
21
36
|
* Handles `.default`, `.$defaultFn()`, `.$onUpdateFn()`.
|
|
@@ -102,7 +117,7 @@ export const normalizeRow = (
|
|
|
102
117
|
row: { [key: string]: unknown },
|
|
103
118
|
isUpdate: boolean,
|
|
104
119
|
) => {
|
|
105
|
-
for (const [columnName, column] of
|
|
120
|
+
for (const [columnName, column] of getCachedTableColumns(table).entries) {
|
|
106
121
|
// not-null constraint
|
|
107
122
|
if (
|
|
108
123
|
isUpdate === false &&
|
|
@@ -127,6 +142,19 @@ export const normalizeRow = (
|
|
|
127
142
|
return row;
|
|
128
143
|
};
|
|
129
144
|
|
|
145
|
+
export const normalizeUpdateSet = (
|
|
146
|
+
table: Table,
|
|
147
|
+
row: { [key: string]: unknown },
|
|
148
|
+
) => {
|
|
149
|
+
const { columns } = getCachedTableColumns(table);
|
|
150
|
+
for (const [columnName, value] of Object.entries(row)) {
|
|
151
|
+
const column = columns[columnName];
|
|
152
|
+
if (value === undefined || column === undefined) continue;
|
|
153
|
+
row[columnName] = normalizeColumn(column, value, true);
|
|
154
|
+
}
|
|
155
|
+
return row;
|
|
156
|
+
};
|
|
157
|
+
|
|
130
158
|
export const getCacheKey = (
|
|
131
159
|
table: Table,
|
|
132
160
|
key: object,
|
package/src/internal/metrics.ts
CHANGED
|
@@ -25,6 +25,9 @@ const httpRequestSizeBytes = [
|
|
|
25
25
|
const GET_METRICS_REQ = "prom-client:getMetricsReq";
|
|
26
26
|
const GET_METRICS_RES = "prom-client:getMetricsRes";
|
|
27
27
|
|
|
28
|
+
type IndexingCacheRequestType = "complete" | "hit" | "miss" | "prefetch";
|
|
29
|
+
type IndexingStoreMethod = "find" | "insert" | "update" | "delete";
|
|
30
|
+
|
|
28
31
|
export class MetricsService {
|
|
29
32
|
registry: prometheus.Registry;
|
|
30
33
|
start_timestamp: number;
|
|
@@ -112,6 +115,16 @@ export class MetricsService {
|
|
|
112
115
|
ponder_postgres_query_queue_size: prometheus.Gauge<"pool"> = null!;
|
|
113
116
|
ponder_postgres_pool_connections: prometheus.Gauge<"pool" | "kind"> = null!;
|
|
114
117
|
|
|
118
|
+
private indexingCompletedEvents = new Map<string, number>();
|
|
119
|
+
private indexingCacheRequests = new Map<
|
|
120
|
+
string,
|
|
121
|
+
Map<IndexingCacheRequestType, number>
|
|
122
|
+
>();
|
|
123
|
+
private indexingStoreQueries = new Map<
|
|
124
|
+
string,
|
|
125
|
+
Map<IndexingStoreMethod, number>
|
|
126
|
+
>();
|
|
127
|
+
|
|
115
128
|
constructor() {
|
|
116
129
|
this.registry = new prometheus.Registry();
|
|
117
130
|
this.start_timestamp = Date.now();
|
|
@@ -206,6 +219,7 @@ export class MetricsService {
|
|
|
206
219
|
labelNames: ["chain", "event"] as const,
|
|
207
220
|
registers: [this.registry],
|
|
208
221
|
aggregator: "sum",
|
|
222
|
+
collect: () => this.flushIndexingCompletedEvents(),
|
|
209
223
|
});
|
|
210
224
|
this.ponder_indexing_timestamp = new prometheus.Gauge({
|
|
211
225
|
name: "ponder_indexing_timestamp",
|
|
@@ -258,6 +272,7 @@ export class MetricsService {
|
|
|
258
272
|
labelNames: ["table", "type"] as const,
|
|
259
273
|
registers: [this.registry],
|
|
260
274
|
aggregator: "sum",
|
|
275
|
+
collect: () => this.flushIndexingCacheRequests(),
|
|
261
276
|
});
|
|
262
277
|
this.ponder_indexing_store_queries_total = new prometheus.Counter({
|
|
263
278
|
name: "ponder_indexing_store_queries_total",
|
|
@@ -265,6 +280,7 @@ export class MetricsService {
|
|
|
265
280
|
labelNames: ["table", "method"] as const,
|
|
266
281
|
registers: [this.registry],
|
|
267
282
|
aggregator: "sum",
|
|
283
|
+
collect: () => this.flushIndexingStoreQueries(),
|
|
268
284
|
});
|
|
269
285
|
this.ponder_indexing_store_raw_sql_duration = new prometheus.Histogram({
|
|
270
286
|
name: "ponder_indexing_store_raw_sql_duration",
|
|
@@ -449,6 +465,70 @@ export class MetricsService {
|
|
|
449
465
|
return this.registry;
|
|
450
466
|
}
|
|
451
467
|
|
|
468
|
+
incrementIndexingCompletedEvents(event: string, value = 1) {
|
|
469
|
+
this.indexingCompletedEvents.set(
|
|
470
|
+
event,
|
|
471
|
+
(this.indexingCompletedEvents.get(event) ?? 0) + value,
|
|
472
|
+
);
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
incrementIndexingCacheRequests(
|
|
476
|
+
table: string,
|
|
477
|
+
type: IndexingCacheRequestType,
|
|
478
|
+
value = 1,
|
|
479
|
+
) {
|
|
480
|
+
let requests = this.indexingCacheRequests.get(table);
|
|
481
|
+
if (requests === undefined) {
|
|
482
|
+
requests = new Map();
|
|
483
|
+
this.indexingCacheRequests.set(table, requests);
|
|
484
|
+
}
|
|
485
|
+
requests.set(type, (requests.get(type) ?? 0) + value);
|
|
486
|
+
}
|
|
487
|
+
|
|
488
|
+
incrementIndexingStoreQueries(
|
|
489
|
+
table: string,
|
|
490
|
+
method: IndexingStoreMethod,
|
|
491
|
+
value = 1,
|
|
492
|
+
) {
|
|
493
|
+
let queries = this.indexingStoreQueries.get(table);
|
|
494
|
+
if (queries === undefined) {
|
|
495
|
+
queries = new Map();
|
|
496
|
+
this.indexingStoreQueries.set(table, queries);
|
|
497
|
+
}
|
|
498
|
+
queries.set(method, (queries.get(method) ?? 0) + value);
|
|
499
|
+
}
|
|
500
|
+
|
|
501
|
+
flushIndexingMetrics() {
|
|
502
|
+
this.flushIndexingCompletedEvents();
|
|
503
|
+
this.flushIndexingCacheRequests();
|
|
504
|
+
this.flushIndexingStoreQueries();
|
|
505
|
+
}
|
|
506
|
+
|
|
507
|
+
private flushIndexingCompletedEvents() {
|
|
508
|
+
for (const [event, value] of this.indexingCompletedEvents) {
|
|
509
|
+
this.ponder_indexing_completed_events.inc({ event }, value);
|
|
510
|
+
}
|
|
511
|
+
this.indexingCompletedEvents.clear();
|
|
512
|
+
}
|
|
513
|
+
|
|
514
|
+
private flushIndexingCacheRequests() {
|
|
515
|
+
for (const [table, requests] of this.indexingCacheRequests) {
|
|
516
|
+
for (const [type, value] of requests) {
|
|
517
|
+
this.ponder_indexing_cache_requests_total.inc({ table, type }, value);
|
|
518
|
+
}
|
|
519
|
+
}
|
|
520
|
+
this.indexingCacheRequests.clear();
|
|
521
|
+
}
|
|
522
|
+
|
|
523
|
+
private flushIndexingStoreQueries() {
|
|
524
|
+
for (const [table, queries] of this.indexingStoreQueries) {
|
|
525
|
+
for (const [method, value] of queries) {
|
|
526
|
+
this.ponder_indexing_store_queries_total.inc({ table, method }, value);
|
|
527
|
+
}
|
|
528
|
+
}
|
|
529
|
+
this.indexingStoreQueries.clear();
|
|
530
|
+
}
|
|
531
|
+
|
|
452
532
|
initializeIndexingMetrics({
|
|
453
533
|
indexingBuild,
|
|
454
534
|
schemaBuild,
|
|
@@ -480,6 +560,7 @@ export class MetricsService {
|
|
|
480
560
|
}
|
|
481
561
|
|
|
482
562
|
resetIndexingMetrics() {
|
|
563
|
+
this.flushIndexingMetrics();
|
|
483
564
|
this.start_timestamp = Date.now();
|
|
484
565
|
this.rps = {};
|
|
485
566
|
this.progressMetadata = {
|
package/src/runtime/isolated.ts
CHANGED
|
@@ -398,6 +398,7 @@ export async function runIsolated({
|
|
|
398
398
|
} catch (error) {
|
|
399
399
|
// Note: This can cause a bug with "dev" command, because there are multiple instances
|
|
400
400
|
// updating the same metric.
|
|
401
|
+
common.metrics.flushIndexingMetrics();
|
|
401
402
|
for (const value of initialCompletedEvents.values) {
|
|
402
403
|
common.metrics.ponder_indexing_completed_events.set(
|
|
403
404
|
value.labels,
|
|
@@ -451,6 +451,7 @@ export async function runMultichain({
|
|
|
451
451
|
);
|
|
452
452
|
endClock = startClock();
|
|
453
453
|
} catch (error) {
|
|
454
|
+
common.metrics.flushIndexingMetrics();
|
|
454
455
|
for (const value of initialCompletedEvents.values) {
|
|
455
456
|
common.metrics.ponder_indexing_completed_events.set(
|
|
456
457
|
value.labels,
|
package/src/runtime/omnichain.ts
CHANGED
|
@@ -470,6 +470,7 @@ export async function runOmnichain({
|
|
|
470
470
|
);
|
|
471
471
|
endClock = startClock();
|
|
472
472
|
} catch (error) {
|
|
473
|
+
common.metrics.flushIndexingMetrics();
|
|
473
474
|
for (const value of initialCompletedEvents.values) {
|
|
474
475
|
common.metrics.ponder_indexing_completed_events.set(
|
|
475
476
|
value.labels,
|
|
@@ -1138,7 +1138,7 @@ export const createRealtimeSync = (
|
|
|
1138
1138
|
|
|
1139
1139
|
// Make sure `transactions` can be garbage collected
|
|
1140
1140
|
blockWithEventData.block.transactions =
|
|
1141
|
-
blockWithFilteredEventData.
|
|
1141
|
+
blockWithFilteredEventData.transactions;
|
|
1142
1142
|
|
|
1143
1143
|
yield {
|
|
1144
1144
|
type: "block",
|