ponder 0.17.2 → 0.17.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/CHANGELOG.md +18 -0
  2. package/dist/esm/indexing/index.js +2 -2
  3. package/dist/esm/indexing/index.js.map +1 -1
  4. package/dist/esm/indexing-store/cache.js +6 -21
  5. package/dist/esm/indexing-store/cache.js.map +1 -1
  6. package/dist/esm/indexing-store/index.js +35 -39
  7. package/dist/esm/indexing-store/index.js.map +1 -1
  8. package/dist/esm/indexing-store/utils.js +21 -1
  9. package/dist/esm/indexing-store/utils.js.map +1 -1
  10. package/dist/esm/internal/metrics.js +68 -0
  11. package/dist/esm/internal/metrics.js.map +1 -1
  12. package/dist/esm/runtime/isolated.js +1 -0
  13. package/dist/esm/runtime/isolated.js.map +1 -1
  14. package/dist/esm/runtime/multichain.js +1 -0
  15. package/dist/esm/runtime/multichain.js.map +1 -1
  16. package/dist/esm/runtime/omnichain.js +1 -0
  17. package/dist/esm/runtime/omnichain.js.map +1 -1
  18. package/dist/esm/sync-realtime/index.js +1 -1
  19. package/dist/esm/sync-realtime/index.js.map +1 -1
  20. package/dist/esm/utils/copy.js +69 -58
  21. package/dist/esm/utils/copy.js.map +1 -1
  22. package/dist/types/indexing/index.d.ts.map +1 -1
  23. package/dist/types/indexing-store/cache.d.ts.map +1 -1
  24. package/dist/types/indexing-store/index.d.ts.map +1 -1
  25. package/dist/types/indexing-store/utils.d.ts +5 -0
  26. package/dist/types/indexing-store/utils.d.ts.map +1 -1
  27. package/dist/types/internal/metrics.d.ts +13 -0
  28. package/dist/types/internal/metrics.d.ts.map +1 -1
  29. package/dist/types/runtime/isolated.d.ts.map +1 -1
  30. package/dist/types/runtime/multichain.d.ts.map +1 -1
  31. package/dist/types/runtime/omnichain.d.ts.map +1 -1
  32. package/dist/types/utils/copy.d.ts +1 -1
  33. package/dist/types/utils/copy.d.ts.map +1 -1
  34. package/package.json +2 -2
  35. package/src/indexing/index.ts +4 -6
  36. package/src/indexing-store/cache.ts +20 -22
  37. package/src/indexing-store/index.ts +76 -38
  38. package/src/indexing-store/utils.ts +29 -1
  39. package/src/internal/metrics.ts +81 -0
  40. package/src/runtime/isolated.ts +1 -0
  41. package/src/runtime/multichain.ts +1 -0
  42. package/src/runtime/omnichain.ts +1 -0
  43. package/src/sync-realtime/index.ts +1 -1
  44. package/src/utils/copy.ts +76 -62
@@ -1 +1 @@
1
- {"version":3,"file":"omnichain.d.ts","sourceRoot":"","sources":["../../../src/runtime/omnichain.ts"],"names":[],"mappings":"AAYA,OAAO,EACL,KAAK,QAAQ,EAGd,MAAM,qBAAqB,CAAC;AAU7B,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AAOnD,OAAO,KAAK,EACV,KAAK,EACL,uBAAuB,EAEvB,aAAa,EAEb,cAAc,EACd,QAAQ,EACR,WAAW,EAEZ,MAAM,qBAAqB,CAAC;AAsB7B,OAAO,EAML,KAAK,YAAY,EAClB,MAAM,YAAY,CAAC;AAGpB,wBAAsB,YAAY,CAAC,EACjC,MAAM,EACN,QAAQ,EACR,cAAc,EACd,WAAW,EACX,aAAa,EACb,uBAAuB,EACvB,QAAQ,GACT,EAAE;IACD,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,QAAQ,CAAC;IACnB,cAAc,EAAE,cAAc,CAAC;IAC/B,WAAW,EAAE,WAAW,CAAC;IACzB,aAAa,EAAE,aAAa,CAAC;IAC7B,uBAAuB,EAAE,uBAAuB,CAAC;IACjD,QAAQ,EAAE,QAAQ,CAAC;CACpB,iBAkxBA;AAED;;GAEG;AACH,eAAO,MAAM,sBAAsB;kBAMnB,IAAI,KAAK,EAAE;QAAE,YAAY,EAAE,YAAY,CAAA;KAAE,CAAC;;qDAqCzD,CAAC"}
1
+ {"version":3,"file":"omnichain.d.ts","sourceRoot":"","sources":["../../../src/runtime/omnichain.ts"],"names":[],"mappings":"AAYA,OAAO,EACL,KAAK,QAAQ,EAGd,MAAM,qBAAqB,CAAC;AAU7B,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AAOnD,OAAO,KAAK,EACV,KAAK,EACL,uBAAuB,EAEvB,aAAa,EAEb,cAAc,EACd,QAAQ,EACR,WAAW,EAEZ,MAAM,qBAAqB,CAAC;AAsB7B,OAAO,EAML,KAAK,YAAY,EAClB,MAAM,YAAY,CAAC;AAGpB,wBAAsB,YAAY,CAAC,EACjC,MAAM,EACN,QAAQ,EACR,cAAc,EACd,WAAW,EACX,aAAa,EACb,uBAAuB,EACvB,QAAQ,GACT,EAAE;IACD,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,QAAQ,CAAC;IACnB,cAAc,EAAE,cAAc,CAAC;IAC/B,WAAW,EAAE,WAAW,CAAC;IACzB,aAAa,EAAE,aAAa,CAAC;IAC7B,uBAAuB,EAAE,uBAAuB,CAAC;IACjD,QAAQ,EAAE,QAAQ,CAAC;CACpB,iBAmxBA;AAED;;GAEG;AACH,eAAO,MAAM,sBAAsB;kBAMnB,IAAI,KAAK,EAAE;QAAE,YAAY,EAAE,YAAY,CAAA;KAAE,CAAC;;qDAqCzD,CAAC"}
@@ -12,5 +12,5 @@ export declare const copyOnWrite: <T extends object>(obj: T) => T;
12
12
  * @dev This function supports copying objects that
13
13
  * have been created with `copyOnWrite`.
14
14
  */
15
- export declare const copy: <T>(obj: T) => T;
15
+ export declare const copy: <T>(obj: T, fast: boolean) => T;
16
16
  //# sourceMappingURL=copy.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"copy.d.ts","sourceRoot":"","sources":["../../../src/utils/copy.ts"],"names":[],"mappings":"AAEA;;GAEG;AACH,eAAO,MAAM,aAAa,eAAmC,CAAC;AAE9D;;GAEG;AACH,eAAO,MAAM,WAAW,0BAA2B,CAAC,KAAG,CAsDtD,CAAC;AAEF;;;;;GAKG;AACH,eAAO,MAAM,IAAI,WAAY,CAAC,KAAG,CA8EhC,CAAC"}
1
+ {"version":3,"file":"copy.d.ts","sourceRoot":"","sources":["../../../src/utils/copy.ts"],"names":[],"mappings":"AAEA;;GAEG;AACH,eAAO,MAAM,aAAa,eAAmC,CAAC;AAQ9D;;GAEG;AACH,eAAO,MAAM,WAAW,0BAA2B,CAAC,KAAG,CA4DtD,CAAC;AAEF;;;;;GAKG;AACH,eAAO,MAAM,IAAI,WAAY,CAAC,QAAQ,OAAO,KAAG,CAoC/C,CAAC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "ponder",
3
- "version": "0.17.2",
3
+ "version": "0.17.4",
4
4
  "description": "An open-source TypeScript framework for EVM data indexing",
5
5
  "license": "MIT",
6
6
  "type": "module",
@@ -102,7 +102,7 @@
102
102
  "rimraf": "^5.0.5",
103
103
  "tsx": "^4.19.2",
104
104
  "vitest": "1.6.1",
105
- "@ponder/client": "0.17.2"
105
+ "@ponder/client": "0.17.4"
106
106
  },
107
107
  "engines": {
108
108
  "node": ">=22"
@@ -549,9 +549,8 @@ export const createIndexing = ({
549
549
  // @ts-expect-error
550
550
  await executeEvent({ ...event, event: proxyEvent });
551
551
 
552
- common.metrics.ponder_indexing_completed_events.inc(
553
- { event: event.eventCallback.name },
554
- 1,
552
+ common.metrics.incrementIndexingCompletedEvents(
553
+ event.eventCallback.name,
555
554
  );
556
555
  columnAccessPattern.get(event.eventCallback.name)!.count++;
557
556
  eventCount[event.eventCallback.name] =
@@ -716,9 +715,8 @@ export const createIndexing = ({
716
715
 
717
716
  await executeEvent(event);
718
717
 
719
- common.metrics.ponder_indexing_completed_events.inc(
720
- { event: event.eventCallback.name },
721
- 1,
718
+ common.metrics.incrementIndexingCompletedEvents(
719
+ event.eventCallback.name,
722
720
  );
723
721
  eventCount[event.eventCallback.name] =
724
722
  eventCount[event.eventCallback.name]! + 1;
@@ -435,10 +435,10 @@ export const createIndexingCache = ({
435
435
  updateBuffer.get(table)!.get(ck) ?? insertBuffer.get(table)!.get(ck);
436
436
 
437
437
  if (bufferEntry) {
438
- common.metrics.ponder_indexing_cache_requests_total.inc({
439
- table: getTableName(table),
440
- type: cache.get(table)!.isCacheComplete ? "complete" : "hit",
441
- });
438
+ common.metrics.incrementIndexingCacheRequests(
439
+ getTableName(table),
440
+ cache.get(table)!.isCacheComplete ? "complete" : "hit",
441
+ );
442
442
  return bufferEntry.row;
443
443
  }
444
444
 
@@ -452,29 +452,29 @@ export const createIndexingCache = ({
452
452
  cache.get(table)!.spillover.add(ck);
453
453
  }
454
454
 
455
- common.metrics.ponder_indexing_cache_requests_total.inc({
456
- table: getTableName(table),
457
- type: cache.get(table)!.isCacheComplete ? "complete" : "hit",
458
- });
455
+ common.metrics.incrementIndexingCacheRequests(
456
+ getTableName(table),
457
+ cache.get(table)!.isCacheComplete ? "complete" : "hit",
458
+ );
459
459
  return entry;
460
460
  }
461
461
 
462
462
  cache.get(table)!.diskReads++;
463
463
 
464
464
  if (cache.get(table)!.isCacheComplete) {
465
- common.metrics.ponder_indexing_cache_requests_total.inc({
466
- table: getTableName(table),
467
- type: "complete",
468
- });
465
+ common.metrics.incrementIndexingCacheRequests(
466
+ getTableName(table),
467
+ "complete",
468
+ );
469
469
  return null;
470
470
  }
471
471
 
472
472
  cache.get(table)!.spillover.add(ck);
473
473
 
474
- common.metrics.ponder_indexing_cache_requests_total.inc({
475
- table: getTableName(table),
476
- type: "miss",
477
- });
474
+ common.metrics.incrementIndexingCacheRequests(
475
+ getTableName(table),
476
+ "miss",
477
+ );
478
478
 
479
479
  const endClock = startClock();
480
480
 
@@ -502,7 +502,7 @@ export const createIndexingCache = ({
502
502
  return result;
503
503
  },
504
504
  set({ table, key, row: _row, isUpdate }) {
505
- const row = normalizeRow(table, _row, isUpdate);
505
+ const row = isUpdate ? _row : normalizeRow(table, _row, false);
506
506
  const ck = getCacheKey(table, key, primaryKeyCache);
507
507
 
508
508
  if (isUpdate) {
@@ -964,11 +964,9 @@ export const createIndexingCache = ({
964
964
  }
965
965
 
966
966
  for (const [table, tablePredictions] of prediction) {
967
- common.metrics.ponder_indexing_cache_requests_total.inc(
968
- {
969
- table: getTableName(table),
970
- type: "prefetch",
971
- },
967
+ common.metrics.incrementIndexingCacheRequests(
968
+ getTableName(table),
969
+ "prefetch",
972
970
  tablePredictions.size,
973
971
  );
974
972
  }
@@ -1,5 +1,6 @@
1
1
  import {
2
2
  type Column,
3
+ getTableColumns,
3
4
  getTableName,
4
5
  getViewName,
5
6
  isTable,
@@ -35,7 +36,7 @@ import { prettyPrint } from "@/utils/print.js";
35
36
  import { getSQLQueryRelations, isReadonlySQLQuery } from "@/utils/sql-parse.js";
36
37
  import { startClock } from "@/utils/timer.js";
37
38
  import type { IndexingCache, Row } from "./cache.js";
38
- import { getPrimaryKeyCache } from "./utils.js";
39
+ import { getPrimaryKeyCache, normalizeUpdateSet } from "./utils.js";
39
40
 
40
41
  export type IndexingStore = {
41
42
  db: Db<Schema>;
@@ -117,6 +118,14 @@ export const createIndexingStore = ({
117
118
  const tables = Object.values(schema).filter(isTable);
118
119
  const views = Object.values(schema).filter(isView);
119
120
  const primaryKeyCache = getPrimaryKeyCache(tables);
121
+ const isCopyFast = new Map(
122
+ tables.map((table) => [
123
+ table,
124
+ Object.values(getTableColumns(table)).every((column) =>
125
+ ["string", "number", "boolean", "bigint"].includes(column.dataType),
126
+ ),
127
+ ]),
128
+ );
120
129
 
121
130
  const lock = createLock();
122
131
 
@@ -161,10 +170,10 @@ export const createIndexingStore = ({
161
170
  return {
162
171
  db: {
163
172
  find: storeMethodWrapper(async (table: Table, key) => {
164
- common.metrics.ponder_indexing_store_queries_total.inc({
165
- table: getTableName(table),
166
- method: "find",
167
- });
173
+ common.metrics.incrementIndexingStoreQueries(
174
+ getTableName(table),
175
+ "find",
176
+ );
168
177
  checkOnchainTable(table, "find");
169
178
  checkTableAccess(table, "find", key, chainId);
170
179
  const ponderRow = await indexingCache.get({ table, key });
@@ -177,13 +186,17 @@ export const createIndexingStore = ({
177
186
  values: (userValues: any) => {
178
187
  const inner = {
179
188
  onConflictDoNothing: storeMethodWrapper(async () => {
180
- common.metrics.ponder_indexing_store_queries_total.inc({
181
- table: getTableName(table),
182
- method: "insert",
183
- });
189
+ common.metrics.incrementIndexingStoreQueries(
190
+ getTableName(table),
191
+ "insert",
192
+ );
184
193
  checkOnchainTable(table, "insert");
185
194
 
186
- const ponderValues = copy(userValues);
195
+ const ponderValues = Array.isArray(userValues)
196
+ ? userValues.map((value) =>
197
+ copy(value, isCopyFast.get(table)!),
198
+ )
199
+ : copy(userValues, isCopyFast.get(table)!);
187
200
 
188
201
  if (Array.isArray(ponderValues)) {
189
202
  const ponderRows = [];
@@ -232,10 +245,10 @@ export const createIndexingStore = ({
232
245
  }),
233
246
  onConflictDoUpdate: storeMethodWrapper(
234
247
  async (userUpdateValues: any) => {
235
- common.metrics.ponder_indexing_store_queries_total.inc({
236
- table: getTableName(table),
237
- method: "insert",
238
- });
248
+ common.metrics.incrementIndexingStoreQueries(
249
+ getTableName(table),
250
+ "insert",
251
+ );
239
252
  checkOnchainTable(table, "insert");
240
253
 
241
254
  if (Array.isArray(userValues)) {
@@ -248,17 +261,24 @@ export const createIndexingStore = ({
248
261
  });
249
262
 
250
263
  if (ponderRowUpdate) {
251
- ponderRowUpdate = copy(ponderRowUpdate);
264
+ ponderRowUpdate = copy(
265
+ ponderRowUpdate,
266
+ isCopyFast.get(table)!,
267
+ );
252
268
  if (typeof userUpdateValues === "function") {
253
269
  const userRowUpdate = copyOnWrite(ponderRowUpdate);
254
270
  const userSet = userUpdateValues(userRowUpdate);
255
- const ponderSet = copy(userSet);
271
+ const ponderSet = copy(
272
+ userSet,
273
+ isCopyFast.get(table)!,
274
+ );
256
275
  validateUpdateSet(
257
276
  table,
258
277
  ponderSet,
259
278
  ponderRowUpdate,
260
279
  primaryKeyCache,
261
280
  );
281
+ normalizeUpdateSet(table, ponderSet);
262
282
  for (const [key, value] of Object.entries(
263
283
  ponderSet,
264
284
  )) {
@@ -267,13 +287,17 @@ export const createIndexingStore = ({
267
287
  }
268
288
  } else {
269
289
  const userSet = userUpdateValues;
270
- const ponderSet = copy(userSet);
290
+ const ponderSet = copy(
291
+ userSet,
292
+ isCopyFast.get(table)!,
293
+ );
271
294
  validateUpdateSet(
272
295
  table,
273
296
  ponderSet,
274
297
  ponderRowUpdate,
275
298
  primaryKeyCache,
276
299
  );
300
+ normalizeUpdateSet(table, ponderSet);
277
301
  for (const [key, value] of Object.entries(
278
302
  ponderSet,
279
303
  )) {
@@ -290,7 +314,7 @@ export const createIndexingStore = ({
290
314
  }),
291
315
  );
292
316
  } else {
293
- const ponderValue = copy(value);
317
+ const ponderValue = copy(value, isCopyFast.get(table)!);
294
318
  ponderRows.push(
295
319
  indexingCache.set({
296
320
  table,
@@ -313,30 +337,35 @@ export const createIndexingStore = ({
313
337
  });
314
338
 
315
339
  if (ponderRowUpdate) {
316
- ponderRowUpdate = copy(ponderRowUpdate);
340
+ ponderRowUpdate = copy(
341
+ ponderRowUpdate,
342
+ isCopyFast.get(table)!,
343
+ );
317
344
  if (typeof userUpdateValues === "function") {
318
345
  const userRowUpdate = copyOnWrite(ponderRowUpdate);
319
346
  const userSet = userUpdateValues(userRowUpdate);
320
- const ponderSet = copy(userSet);
347
+ const ponderSet = copy(userSet, isCopyFast.get(table)!);
321
348
  validateUpdateSet(
322
349
  table,
323
350
  ponderSet,
324
351
  ponderRowUpdate,
325
352
  primaryKeyCache,
326
353
  );
354
+ normalizeUpdateSet(table, ponderSet);
327
355
  for (const [key, value] of Object.entries(ponderSet)) {
328
356
  if (value === undefined) continue;
329
357
  ponderRowUpdate[key] = value;
330
358
  }
331
359
  } else {
332
360
  const userSet = userUpdateValues;
333
- const ponderSet = copy(userSet);
361
+ const ponderSet = copy(userSet, isCopyFast.get(table)!);
334
362
  validateUpdateSet(
335
363
  table,
336
364
  ponderSet,
337
365
  ponderRowUpdate,
338
366
  primaryKeyCache,
339
367
  );
368
+ normalizeUpdateSet(table, ponderSet);
340
369
  for (const [key, value] of Object.entries(ponderSet)) {
341
370
  if (value === undefined) continue;
342
371
  ponderRowUpdate[key] = value;
@@ -352,7 +381,10 @@ export const createIndexingStore = ({
352
381
  return userRow;
353
382
  }
354
383
 
355
- const ponderValues = copy(userValues);
384
+ const ponderValues = copy(
385
+ userValues,
386
+ isCopyFast.get(table)!,
387
+ );
356
388
 
357
389
  const ponderRowInsert = indexingCache.set({
358
390
  table,
@@ -369,12 +401,16 @@ export const createIndexingStore = ({
369
401
  // biome-ignore lint/suspicious/noThenProperty: The returned object is intentionally thenable for the query API.
370
402
  then: (onFulfilled, onRejected) =>
371
403
  storeMethodWrapper(async () => {
372
- common.metrics.ponder_indexing_store_queries_total.inc({
373
- table: getTableName(table),
374
- method: "insert",
375
- });
404
+ common.metrics.incrementIndexingStoreQueries(
405
+ getTableName(table),
406
+ "insert",
407
+ );
376
408
  checkOnchainTable(table, "insert");
377
- const ponderValues = copy(userValues);
409
+ const ponderValues = Array.isArray(userValues)
410
+ ? userValues.map((value) =>
411
+ copy(value, isCopyFast.get(table)!),
412
+ )
413
+ : copy(userValues, isCopyFast.get(table)!);
378
414
 
379
415
  if (Array.isArray(ponderValues)) {
380
416
  const ponderRows = [];
@@ -488,10 +524,10 @@ export const createIndexingStore = ({
488
524
  update(table: Table, key) {
489
525
  return {
490
526
  set: storeMethodWrapper(async (userValues: any) => {
491
- common.metrics.ponder_indexing_store_queries_total.inc({
492
- table: getTableName(table),
493
- method: "update",
494
- });
527
+ common.metrics.incrementIndexingStoreQueries(
528
+ getTableName(table),
529
+ "update",
530
+ );
495
531
  checkOnchainTable(table, "update");
496
532
  checkTableAccess(table, "update", key, chainId);
497
533
 
@@ -507,31 +543,33 @@ export const createIndexingStore = ({
507
543
  throw error;
508
544
  }
509
545
 
510
- ponderRowUpdate = copy(ponderRowUpdate);
546
+ ponderRowUpdate = copy(ponderRowUpdate, isCopyFast.get(table)!);
511
547
 
512
548
  if (typeof userValues === "function") {
513
549
  const userRow = copyOnWrite(ponderRowUpdate);
514
550
  const userSet = userValues(userRow);
515
- const ponderSet = copy(userSet);
551
+ const ponderSet = copy(userSet, isCopyFast.get(table)!);
516
552
  validateUpdateSet(
517
553
  table,
518
554
  ponderSet,
519
555
  ponderRowUpdate,
520
556
  primaryKeyCache,
521
557
  );
558
+ normalizeUpdateSet(table, ponderSet);
522
559
  for (const [key, value] of Object.entries(ponderSet)) {
523
560
  if (value === undefined) continue;
524
561
  ponderRowUpdate[key] = value;
525
562
  }
526
563
  } else {
527
564
  const userSet = userValues;
528
- const ponderSet = copy(userSet);
565
+ const ponderSet = copy(userSet, isCopyFast.get(table)!);
529
566
  validateUpdateSet(
530
567
  table,
531
568
  ponderSet,
532
569
  ponderRowUpdate,
533
570
  primaryKeyCache,
534
571
  );
572
+ normalizeUpdateSet(table, ponderSet);
535
573
  for (const [key, value] of Object.entries(ponderSet)) {
536
574
  if (value === undefined) continue;
537
575
  ponderRowUpdate[key] = value;
@@ -550,10 +588,10 @@ export const createIndexingStore = ({
550
588
  };
551
589
  },
552
590
  delete: storeMethodWrapper(async (table: Table, key) => {
553
- common.metrics.ponder_indexing_store_queries_total.inc({
554
- table: getTableName(table),
555
- method: "delete",
556
- });
591
+ common.metrics.incrementIndexingStoreQueries(
592
+ getTableName(table),
593
+ "delete",
594
+ );
557
595
  checkOnchainTable(table, "delete");
558
596
  checkTableAccess(table, "delete", key, chainId);
559
597
  return indexingCache.delete({ table, key });
@@ -16,6 +16,21 @@ import {
16
16
  } from "@/internal/errors.js";
17
17
  import { prettyPrint } from "@/utils/print.js";
18
18
 
19
+ const tableColumnCache = new WeakMap<
20
+ Table,
21
+ { columns: Record<string, Column>; entries: [string, Column][] }
22
+ >();
23
+
24
+ const getCachedTableColumns = (table: Table) => {
25
+ let cached = tableColumnCache.get(table);
26
+ if (cached === undefined) {
27
+ const columns = getTableColumns(table);
28
+ cached = { columns, entries: Object.entries(columns) };
29
+ tableColumnCache.set(table, cached);
30
+ }
31
+ return cached;
32
+ };
33
+
19
34
  /**
20
35
  * Returns true if the column has a "default" value that is used when no value is passed.
21
36
  * Handles `.default`, `.$defaultFn()`, `.$onUpdateFn()`.
@@ -102,7 +117,7 @@ export const normalizeRow = (
102
117
  row: { [key: string]: unknown },
103
118
  isUpdate: boolean,
104
119
  ) => {
105
- for (const [columnName, column] of Object.entries(getTableColumns(table))) {
120
+ for (const [columnName, column] of getCachedTableColumns(table).entries) {
106
121
  // not-null constraint
107
122
  if (
108
123
  isUpdate === false &&
@@ -127,6 +142,19 @@ export const normalizeRow = (
127
142
  return row;
128
143
  };
129
144
 
145
+ export const normalizeUpdateSet = (
146
+ table: Table,
147
+ row: { [key: string]: unknown },
148
+ ) => {
149
+ const { columns } = getCachedTableColumns(table);
150
+ for (const [columnName, value] of Object.entries(row)) {
151
+ const column = columns[columnName];
152
+ if (value === undefined || column === undefined) continue;
153
+ row[columnName] = normalizeColumn(column, value, true);
154
+ }
155
+ return row;
156
+ };
157
+
130
158
  export const getCacheKey = (
131
159
  table: Table,
132
160
  key: object,
@@ -25,6 +25,9 @@ const httpRequestSizeBytes = [
25
25
  const GET_METRICS_REQ = "prom-client:getMetricsReq";
26
26
  const GET_METRICS_RES = "prom-client:getMetricsRes";
27
27
 
28
+ type IndexingCacheRequestType = "complete" | "hit" | "miss" | "prefetch";
29
+ type IndexingStoreMethod = "find" | "insert" | "update" | "delete";
30
+
28
31
  export class MetricsService {
29
32
  registry: prometheus.Registry;
30
33
  start_timestamp: number;
@@ -112,6 +115,16 @@ export class MetricsService {
112
115
  ponder_postgres_query_queue_size: prometheus.Gauge<"pool"> = null!;
113
116
  ponder_postgres_pool_connections: prometheus.Gauge<"pool" | "kind"> = null!;
114
117
 
118
+ private indexingCompletedEvents = new Map<string, number>();
119
+ private indexingCacheRequests = new Map<
120
+ string,
121
+ Map<IndexingCacheRequestType, number>
122
+ >();
123
+ private indexingStoreQueries = new Map<
124
+ string,
125
+ Map<IndexingStoreMethod, number>
126
+ >();
127
+
115
128
  constructor() {
116
129
  this.registry = new prometheus.Registry();
117
130
  this.start_timestamp = Date.now();
@@ -206,6 +219,7 @@ export class MetricsService {
206
219
  labelNames: ["chain", "event"] as const,
207
220
  registers: [this.registry],
208
221
  aggregator: "sum",
222
+ collect: () => this.flushIndexingCompletedEvents(),
209
223
  });
210
224
  this.ponder_indexing_timestamp = new prometheus.Gauge({
211
225
  name: "ponder_indexing_timestamp",
@@ -258,6 +272,7 @@ export class MetricsService {
258
272
  labelNames: ["table", "type"] as const,
259
273
  registers: [this.registry],
260
274
  aggregator: "sum",
275
+ collect: () => this.flushIndexingCacheRequests(),
261
276
  });
262
277
  this.ponder_indexing_store_queries_total = new prometheus.Counter({
263
278
  name: "ponder_indexing_store_queries_total",
@@ -265,6 +280,7 @@ export class MetricsService {
265
280
  labelNames: ["table", "method"] as const,
266
281
  registers: [this.registry],
267
282
  aggregator: "sum",
283
+ collect: () => this.flushIndexingStoreQueries(),
268
284
  });
269
285
  this.ponder_indexing_store_raw_sql_duration = new prometheus.Histogram({
270
286
  name: "ponder_indexing_store_raw_sql_duration",
@@ -449,6 +465,70 @@ export class MetricsService {
449
465
  return this.registry;
450
466
  }
451
467
 
468
+ incrementIndexingCompletedEvents(event: string, value = 1) {
469
+ this.indexingCompletedEvents.set(
470
+ event,
471
+ (this.indexingCompletedEvents.get(event) ?? 0) + value,
472
+ );
473
+ }
474
+
475
+ incrementIndexingCacheRequests(
476
+ table: string,
477
+ type: IndexingCacheRequestType,
478
+ value = 1,
479
+ ) {
480
+ let requests = this.indexingCacheRequests.get(table);
481
+ if (requests === undefined) {
482
+ requests = new Map();
483
+ this.indexingCacheRequests.set(table, requests);
484
+ }
485
+ requests.set(type, (requests.get(type) ?? 0) + value);
486
+ }
487
+
488
+ incrementIndexingStoreQueries(
489
+ table: string,
490
+ method: IndexingStoreMethod,
491
+ value = 1,
492
+ ) {
493
+ let queries = this.indexingStoreQueries.get(table);
494
+ if (queries === undefined) {
495
+ queries = new Map();
496
+ this.indexingStoreQueries.set(table, queries);
497
+ }
498
+ queries.set(method, (queries.get(method) ?? 0) + value);
499
+ }
500
+
501
+ flushIndexingMetrics() {
502
+ this.flushIndexingCompletedEvents();
503
+ this.flushIndexingCacheRequests();
504
+ this.flushIndexingStoreQueries();
505
+ }
506
+
507
+ private flushIndexingCompletedEvents() {
508
+ for (const [event, value] of this.indexingCompletedEvents) {
509
+ this.ponder_indexing_completed_events.inc({ event }, value);
510
+ }
511
+ this.indexingCompletedEvents.clear();
512
+ }
513
+
514
+ private flushIndexingCacheRequests() {
515
+ for (const [table, requests] of this.indexingCacheRequests) {
516
+ for (const [type, value] of requests) {
517
+ this.ponder_indexing_cache_requests_total.inc({ table, type }, value);
518
+ }
519
+ }
520
+ this.indexingCacheRequests.clear();
521
+ }
522
+
523
+ private flushIndexingStoreQueries() {
524
+ for (const [table, queries] of this.indexingStoreQueries) {
525
+ for (const [method, value] of queries) {
526
+ this.ponder_indexing_store_queries_total.inc({ table, method }, value);
527
+ }
528
+ }
529
+ this.indexingStoreQueries.clear();
530
+ }
531
+
452
532
  initializeIndexingMetrics({
453
533
  indexingBuild,
454
534
  schemaBuild,
@@ -480,6 +560,7 @@ export class MetricsService {
480
560
  }
481
561
 
482
562
  resetIndexingMetrics() {
563
+ this.flushIndexingMetrics();
483
564
  this.start_timestamp = Date.now();
484
565
  this.rps = {};
485
566
  this.progressMetadata = {
@@ -398,6 +398,7 @@ export async function runIsolated({
398
398
  } catch (error) {
399
399
  // Note: This can cause a bug with "dev" command, because there are multiple instances
400
400
  // updating the same metric.
401
+ common.metrics.flushIndexingMetrics();
401
402
  for (const value of initialCompletedEvents.values) {
402
403
  common.metrics.ponder_indexing_completed_events.set(
403
404
  value.labels,
@@ -451,6 +451,7 @@ export async function runMultichain({
451
451
  );
452
452
  endClock = startClock();
453
453
  } catch (error) {
454
+ common.metrics.flushIndexingMetrics();
454
455
  for (const value of initialCompletedEvents.values) {
455
456
  common.metrics.ponder_indexing_completed_events.set(
456
457
  value.labels,
@@ -470,6 +470,7 @@ export async function runOmnichain({
470
470
  );
471
471
  endClock = startClock();
472
472
  } catch (error) {
473
+ common.metrics.flushIndexingMetrics();
473
474
  for (const value of initialCompletedEvents.values) {
474
475
  common.metrics.ponder_indexing_completed_events.set(
475
476
  value.labels,
@@ -1138,7 +1138,7 @@ export const createRealtimeSync = (
1138
1138
 
1139
1139
  // Make sure `transactions` can be garbage collected
1140
1140
  blockWithEventData.block.transactions =
1141
- blockWithFilteredEventData.block.transactions;
1141
+ blockWithFilteredEventData.transactions;
1142
1142
 
1143
1143
  yield {
1144
1144
  type: "block",