ponder-with-flashblocks 0.16.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +3427 -0
- package/README.md +186 -0
- package/dist/esm/bin/commands/codegen.js +46 -0
- package/dist/esm/bin/commands/codegen.js.map +1 -0
- package/dist/esm/bin/commands/createViews.js +196 -0
- package/dist/esm/bin/commands/createViews.js.map +1 -0
- package/dist/esm/bin/commands/dev.js +430 -0
- package/dist/esm/bin/commands/dev.js.map +1 -0
- package/dist/esm/bin/commands/list.js +148 -0
- package/dist/esm/bin/commands/list.js.map +1 -0
- package/dist/esm/bin/commands/prune.js +223 -0
- package/dist/esm/bin/commands/prune.js.map +1 -0
- package/dist/esm/bin/commands/serve.js +198 -0
- package/dist/esm/bin/commands/serve.js.map +1 -0
- package/dist/esm/bin/commands/start.js +253 -0
- package/dist/esm/bin/commands/start.js.map +1 -0
- package/dist/esm/bin/isolatedController.js +200 -0
- package/dist/esm/bin/isolatedController.js.map +1 -0
- package/dist/esm/bin/isolatedWorker.js +146 -0
- package/dist/esm/bin/isolatedWorker.js.map +1 -0
- package/dist/esm/bin/ponder.js +137 -0
- package/dist/esm/bin/ponder.js.map +1 -0
- package/dist/esm/bin/utils/codegen.js +25 -0
- package/dist/esm/bin/utils/codegen.js.map +1 -0
- package/dist/esm/bin/utils/exit.js +100 -0
- package/dist/esm/bin/utils/exit.js.map +1 -0
- package/dist/esm/build/config.js +746 -0
- package/dist/esm/build/config.js.map +1 -0
- package/dist/esm/build/factory.js +76 -0
- package/dist/esm/build/factory.js.map +1 -0
- package/dist/esm/build/index.js +567 -0
- package/dist/esm/build/index.js.map +1 -0
- package/dist/esm/build/plugin.js +53 -0
- package/dist/esm/build/plugin.js.map +1 -0
- package/dist/esm/build/pre.js +83 -0
- package/dist/esm/build/pre.js.map +1 -0
- package/dist/esm/build/schema.js +202 -0
- package/dist/esm/build/schema.js.map +1 -0
- package/dist/esm/build/stacktrace.js +137 -0
- package/dist/esm/build/stacktrace.js.map +1 -0
- package/dist/esm/client/index.js +441 -0
- package/dist/esm/client/index.js.map +1 -0
- package/dist/esm/config/address.js +2 -0
- package/dist/esm/config/address.js.map +1 -0
- package/dist/esm/config/eventFilter.js +2 -0
- package/dist/esm/config/eventFilter.js.map +1 -0
- package/dist/esm/config/index.js +2 -0
- package/dist/esm/config/index.js.map +1 -0
- package/dist/esm/config/utilityTypes.js +2 -0
- package/dist/esm/config/utilityTypes.js.map +1 -0
- package/dist/esm/database/actions.js +445 -0
- package/dist/esm/database/actions.js.map +1 -0
- package/dist/esm/database/index.js +604 -0
- package/dist/esm/database/index.js.map +1 -0
- package/dist/esm/database/queryBuilder.js +314 -0
- package/dist/esm/database/queryBuilder.js.map +1 -0
- package/dist/esm/drizzle/bigint.js +38 -0
- package/dist/esm/drizzle/bigint.js.map +1 -0
- package/dist/esm/drizzle/bytes.js +47 -0
- package/dist/esm/drizzle/bytes.js.map +1 -0
- package/dist/esm/drizzle/hex.js +40 -0
- package/dist/esm/drizzle/hex.js.map +1 -0
- package/dist/esm/drizzle/index.js +28 -0
- package/dist/esm/drizzle/index.js.map +1 -0
- package/dist/esm/drizzle/json.js +123 -0
- package/dist/esm/drizzle/json.js.map +1 -0
- package/dist/esm/drizzle/kit/index.js +927 -0
- package/dist/esm/drizzle/kit/index.js.map +1 -0
- package/dist/esm/drizzle/onchain.js +184 -0
- package/dist/esm/drizzle/onchain.js.map +1 -0
- package/dist/esm/drizzle/text.js +61 -0
- package/dist/esm/drizzle/text.js.map +1 -0
- package/dist/esm/graphql/graphiql.html.js +59 -0
- package/dist/esm/graphql/graphiql.html.js.map +1 -0
- package/dist/esm/graphql/index.js +934 -0
- package/dist/esm/graphql/index.js.map +1 -0
- package/dist/esm/graphql/json.js +42 -0
- package/dist/esm/graphql/json.js.map +1 -0
- package/dist/esm/graphql/middleware.js +83 -0
- package/dist/esm/graphql/middleware.js.map +1 -0
- package/dist/esm/index.js +9 -0
- package/dist/esm/index.js.map +1 -0
- package/dist/esm/indexing/addStackTrace.js +54 -0
- package/dist/esm/indexing/addStackTrace.js.map +1 -0
- package/dist/esm/indexing/client.js +675 -0
- package/dist/esm/indexing/client.js.map +1 -0
- package/dist/esm/indexing/index.js +663 -0
- package/dist/esm/indexing/index.js.map +1 -0
- package/dist/esm/indexing/profile.js +584 -0
- package/dist/esm/indexing/profile.js.map +1 -0
- package/dist/esm/indexing-store/cache.js +666 -0
- package/dist/esm/indexing-store/cache.js.map +1 -0
- package/dist/esm/indexing-store/index.js +461 -0
- package/dist/esm/indexing-store/index.js.map +1 -0
- package/dist/esm/indexing-store/profile.js +428 -0
- package/dist/esm/indexing-store/profile.js.map +1 -0
- package/dist/esm/indexing-store/utils.js +111 -0
- package/dist/esm/indexing-store/utils.js.map +1 -0
- package/dist/esm/internal/common.js +2 -0
- package/dist/esm/internal/common.js.map +1 -0
- package/dist/esm/internal/errors.js +300 -0
- package/dist/esm/internal/errors.js.map +1 -0
- package/dist/esm/internal/logger.js +178 -0
- package/dist/esm/internal/logger.js.map +1 -0
- package/dist/esm/internal/metrics.js +1049 -0
- package/dist/esm/internal/metrics.js.map +1 -0
- package/dist/esm/internal/options.js +73 -0
- package/dist/esm/internal/options.js.map +1 -0
- package/dist/esm/internal/shutdown.js +24 -0
- package/dist/esm/internal/shutdown.js.map +1 -0
- package/dist/esm/internal/telemetry.js +200 -0
- package/dist/esm/internal/telemetry.js.map +1 -0
- package/dist/esm/internal/types.js +2 -0
- package/dist/esm/internal/types.js.map +1 -0
- package/dist/esm/rpc/actions.js +988 -0
- package/dist/esm/rpc/actions.js.map +1 -0
- package/dist/esm/rpc/http.js +130 -0
- package/dist/esm/rpc/http.js.map +1 -0
- package/dist/esm/rpc/index.js +751 -0
- package/dist/esm/rpc/index.js.map +1 -0
- package/dist/esm/runtime/events.js +664 -0
- package/dist/esm/runtime/events.js.map +1 -0
- package/dist/esm/runtime/filter.js +443 -0
- package/dist/esm/runtime/filter.js.map +1 -0
- package/dist/esm/runtime/fragments.js +478 -0
- package/dist/esm/runtime/fragments.js.map +1 -0
- package/dist/esm/runtime/historical.js +950 -0
- package/dist/esm/runtime/historical.js.map +1 -0
- package/dist/esm/runtime/index.js +316 -0
- package/dist/esm/runtime/index.js.map +1 -0
- package/dist/esm/runtime/isolated.js +463 -0
- package/dist/esm/runtime/isolated.js.map +1 -0
- package/dist/esm/runtime/multichain.js +510 -0
- package/dist/esm/runtime/multichain.js.map +1 -0
- package/dist/esm/runtime/omnichain.js +545 -0
- package/dist/esm/runtime/omnichain.js.map +1 -0
- package/dist/esm/runtime/realtime.js +724 -0
- package/dist/esm/runtime/realtime.js.map +1 -0
- package/dist/esm/server/error.js +56 -0
- package/dist/esm/server/error.js.map +1 -0
- package/dist/esm/server/index.js +121 -0
- package/dist/esm/server/index.js.map +1 -0
- package/dist/esm/sync-historical/index.js +711 -0
- package/dist/esm/sync-historical/index.js.map +1 -0
- package/dist/esm/sync-realtime/bloom.js +76 -0
- package/dist/esm/sync-realtime/bloom.js.map +1 -0
- package/dist/esm/sync-realtime/index.js +928 -0
- package/dist/esm/sync-realtime/index.js.map +1 -0
- package/dist/esm/sync-store/encode.js +105 -0
- package/dist/esm/sync-store/encode.js.map +1 -0
- package/dist/esm/sync-store/index.js +885 -0
- package/dist/esm/sync-store/index.js.map +1 -0
- package/dist/esm/sync-store/migrations.js +1595 -0
- package/dist/esm/sync-store/migrations.js.map +1 -0
- package/dist/esm/sync-store/schema.js +181 -0
- package/dist/esm/sync-store/schema.js.map +1 -0
- package/dist/esm/types/db.js +2 -0
- package/dist/esm/types/db.js.map +1 -0
- package/dist/esm/types/eth.js +2 -0
- package/dist/esm/types/eth.js.map +1 -0
- package/dist/esm/types/utils.js +2 -0
- package/dist/esm/types/utils.js.map +1 -0
- package/dist/esm/types/virtual.js +2 -0
- package/dist/esm/types/virtual.js.map +1 -0
- package/dist/esm/ui/app.js +157 -0
- package/dist/esm/ui/app.js.map +1 -0
- package/dist/esm/ui/index.js +29 -0
- package/dist/esm/ui/index.js.map +1 -0
- package/dist/esm/ui/patch.js +140 -0
- package/dist/esm/ui/patch.js.map +1 -0
- package/dist/esm/utils/abi.js +55 -0
- package/dist/esm/utils/abi.js.map +1 -0
- package/dist/esm/utils/bigint.js +37 -0
- package/dist/esm/utils/bigint.js.map +1 -0
- package/dist/esm/utils/chains.js +21 -0
- package/dist/esm/utils/chains.js.map +1 -0
- package/dist/esm/utils/checkpoint.js +139 -0
- package/dist/esm/utils/checkpoint.js.map +1 -0
- package/dist/esm/utils/chunk.js +8 -0
- package/dist/esm/utils/chunk.js.map +1 -0
- package/dist/esm/utils/copy.js +129 -0
- package/dist/esm/utils/copy.js.map +1 -0
- package/dist/esm/utils/date.js +27 -0
- package/dist/esm/utils/date.js.map +1 -0
- package/dist/esm/utils/debug.js +2 -0
- package/dist/esm/utils/debug.js.map +1 -0
- package/dist/esm/utils/decodeAbiParameters.js +290 -0
- package/dist/esm/utils/decodeAbiParameters.js.map +1 -0
- package/dist/esm/utils/decodeEventLog.js +75 -0
- package/dist/esm/utils/decodeEventLog.js.map +1 -0
- package/dist/esm/utils/dedupe.js +29 -0
- package/dist/esm/utils/dedupe.js.map +1 -0
- package/dist/esm/utils/duplicates.js +19 -0
- package/dist/esm/utils/duplicates.js.map +1 -0
- package/dist/esm/utils/estimate.js +6 -0
- package/dist/esm/utils/estimate.js.map +1 -0
- package/dist/esm/utils/finality.js +38 -0
- package/dist/esm/utils/finality.js.map +1 -0
- package/dist/esm/utils/format.js +20 -0
- package/dist/esm/utils/format.js.map +1 -0
- package/dist/esm/utils/generators.js +121 -0
- package/dist/esm/utils/generators.js.map +1 -0
- package/dist/esm/utils/hash.js +11 -0
- package/dist/esm/utils/hash.js.map +1 -0
- package/dist/esm/utils/interval.js +171 -0
- package/dist/esm/utils/interval.js.map +1 -0
- package/dist/esm/utils/lowercase.js +7 -0
- package/dist/esm/utils/lowercase.js.map +1 -0
- package/dist/esm/utils/mutex.js +26 -0
- package/dist/esm/utils/mutex.js.map +1 -0
- package/dist/esm/utils/never.js +4 -0
- package/dist/esm/utils/never.js.map +1 -0
- package/dist/esm/utils/offset.js +101 -0
- package/dist/esm/utils/offset.js.map +1 -0
- package/dist/esm/utils/order.js +18 -0
- package/dist/esm/utils/order.js.map +1 -0
- package/dist/esm/utils/partition.js +46 -0
- package/dist/esm/utils/partition.js.map +1 -0
- package/dist/esm/utils/pg.js +175 -0
- package/dist/esm/utils/pg.js.map +1 -0
- package/dist/esm/utils/pglite.js +80 -0
- package/dist/esm/utils/pglite.js.map +1 -0
- package/dist/esm/utils/port.js +30 -0
- package/dist/esm/utils/port.js.map +1 -0
- package/dist/esm/utils/print.js +23 -0
- package/dist/esm/utils/print.js.map +1 -0
- package/dist/esm/utils/promiseAllSettledWithThrow.js +19 -0
- package/dist/esm/utils/promiseAllSettledWithThrow.js.map +1 -0
- package/dist/esm/utils/promiseWithResolvers.js +13 -0
- package/dist/esm/utils/promiseWithResolvers.js.map +1 -0
- package/dist/esm/utils/queue.js +150 -0
- package/dist/esm/utils/queue.js.map +1 -0
- package/dist/esm/utils/range.js +8 -0
- package/dist/esm/utils/range.js.map +1 -0
- package/dist/esm/utils/result.js +10 -0
- package/dist/esm/utils/result.js.map +1 -0
- package/dist/esm/utils/sql-parse.js +1326 -0
- package/dist/esm/utils/sql-parse.js.map +1 -0
- package/dist/esm/utils/timer.js +9 -0
- package/dist/esm/utils/timer.js.map +1 -0
- package/dist/esm/utils/truncate.js +15 -0
- package/dist/esm/utils/truncate.js.map +1 -0
- package/dist/esm/utils/wait.js +10 -0
- package/dist/esm/utils/wait.js.map +1 -0
- package/dist/esm/utils/zipper.js +67 -0
- package/dist/esm/utils/zipper.js.map +1 -0
- package/dist/types/bin/commands/codegen.d.ts +5 -0
- package/dist/types/bin/commands/codegen.d.ts.map +1 -0
- package/dist/types/bin/commands/createViews.d.ts +8 -0
- package/dist/types/bin/commands/createViews.d.ts.map +1 -0
- package/dist/types/bin/commands/dev.d.ts +5 -0
- package/dist/types/bin/commands/dev.d.ts.map +1 -0
- package/dist/types/bin/commands/list.d.ts +5 -0
- package/dist/types/bin/commands/list.d.ts.map +1 -0
- package/dist/types/bin/commands/prune.d.ts +5 -0
- package/dist/types/bin/commands/prune.d.ts.map +1 -0
- package/dist/types/bin/commands/serve.d.ts +5 -0
- package/dist/types/bin/commands/serve.d.ts.map +1 -0
- package/dist/types/bin/commands/start.d.ts +19 -0
- package/dist/types/bin/commands/start.d.ts.map +1 -0
- package/dist/types/bin/isolatedController.d.ts +13 -0
- package/dist/types/bin/isolatedController.d.ts.map +1 -0
- package/dist/types/bin/isolatedWorker.d.ts +9 -0
- package/dist/types/bin/isolatedWorker.d.ts.map +1 -0
- package/dist/types/bin/ponder.d.ts +37 -0
- package/dist/types/bin/ponder.d.ts.map +1 -0
- package/dist/types/bin/utils/codegen.d.ts +6 -0
- package/dist/types/bin/utils/codegen.d.ts.map +1 -0
- package/dist/types/bin/utils/exit.d.ts +10 -0
- package/dist/types/bin/utils/exit.d.ts.map +1 -0
- package/dist/types/build/config.d.ts +97 -0
- package/dist/types/build/config.d.ts.map +1 -0
- package/dist/types/build/factory.d.ts +13 -0
- package/dist/types/build/factory.d.ts.map +1 -0
- package/dist/types/build/index.d.ts +84 -0
- package/dist/types/build/index.d.ts.map +1 -0
- package/dist/types/build/plugin.d.ts +4 -0
- package/dist/types/build/plugin.d.ts.map +1 -0
- package/dist/types/build/pre.d.ts +29 -0
- package/dist/types/build/pre.d.ts.map +1 -0
- package/dist/types/build/schema.d.ts +20 -0
- package/dist/types/build/schema.d.ts.map +1 -0
- package/dist/types/build/stacktrace.d.ts +13 -0
- package/dist/types/build/stacktrace.d.ts.map +1 -0
- package/dist/types/client/index.d.ts +27 -0
- package/dist/types/client/index.d.ts.map +1 -0
- package/dist/types/config/address.d.ts +24 -0
- package/dist/types/config/address.d.ts.map +1 -0
- package/dist/types/config/eventFilter.d.ts +18 -0
- package/dist/types/config/eventFilter.d.ts.map +1 -0
- package/dist/types/config/index.d.ts +150 -0
- package/dist/types/config/index.d.ts.map +1 -0
- package/dist/types/config/utilityTypes.d.ts +43 -0
- package/dist/types/config/utilityTypes.d.ts.map +1 -0
- package/dist/types/database/actions.d.ts +99 -0
- package/dist/types/database/actions.d.ts.map +1 -0
- package/dist/types/database/index.d.ts +493 -0
- package/dist/types/database/index.d.ts.map +1 -0
- package/dist/types/database/queryBuilder.d.ts +65 -0
- package/dist/types/database/queryBuilder.d.ts.map +1 -0
- package/dist/types/drizzle/bigint.d.ts +25 -0
- package/dist/types/drizzle/bigint.d.ts.map +1 -0
- package/dist/types/drizzle/bytes.d.ts +31 -0
- package/dist/types/drizzle/bytes.d.ts.map +1 -0
- package/dist/types/drizzle/hex.d.ts +25 -0
- package/dist/types/drizzle/hex.d.ts.map +1 -0
- package/dist/types/drizzle/index.d.ts +6 -0
- package/dist/types/drizzle/index.d.ts.map +1 -0
- package/dist/types/drizzle/json.d.ts +51 -0
- package/dist/types/drizzle/json.d.ts.map +1 -0
- package/dist/types/drizzle/kit/index.d.ts +187 -0
- package/dist/types/drizzle/kit/index.d.ts.map +1 -0
- package/dist/types/drizzle/onchain.d.ts +298 -0
- package/dist/types/drizzle/onchain.d.ts.map +1 -0
- package/dist/types/drizzle/text.d.ts +29 -0
- package/dist/types/drizzle/text.d.ts.map +1 -0
- package/dist/types/graphql/graphiql.html.d.ts +2 -0
- package/dist/types/graphql/graphiql.html.d.ts.map +1 -0
- package/dist/types/graphql/index.d.ts +12 -0
- package/dist/types/graphql/index.d.ts.map +1 -0
- package/dist/types/graphql/json.d.ts +3 -0
- package/dist/types/graphql/json.d.ts.map +1 -0
- package/dist/types/graphql/middleware.d.ts +29 -0
- package/dist/types/graphql/middleware.d.ts.map +1 -0
- package/dist/types/index.d.ts +23 -0
- package/dist/types/index.d.ts.map +1 -0
- package/dist/types/indexing/addStackTrace.d.ts +3 -0
- package/dist/types/indexing/addStackTrace.d.ts.map +1 -0
- package/dist/types/indexing/client.d.ts +154 -0
- package/dist/types/indexing/client.d.ts.map +1 -0
- package/dist/types/indexing/index.d.ts +72 -0
- package/dist/types/indexing/index.d.ts.map +1 -0
- package/dist/types/indexing/profile.d.ts +16 -0
- package/dist/types/indexing/profile.d.ts.map +1 -0
- package/dist/types/indexing-store/cache.d.ts +115 -0
- package/dist/types/indexing-store/cache.d.ts.map +1 -0
- package/dist/types/indexing-store/index.d.ts +24 -0
- package/dist/types/indexing-store/index.d.ts.map +1 -0
- package/dist/types/indexing-store/profile.d.ts +7 -0
- package/dist/types/indexing-store/profile.d.ts.map +1 -0
- package/dist/types/indexing-store/utils.d.ts +19 -0
- package/dist/types/indexing-store/utils.d.ts.map +1 -0
- package/dist/types/internal/common.d.ts +15 -0
- package/dist/types/internal/common.d.ts.map +1 -0
- package/dist/types/internal/errors.d.ts +101 -0
- package/dist/types/internal/errors.d.ts.map +1 -0
- package/dist/types/internal/logger.d.ts +37 -0
- package/dist/types/internal/logger.d.ts.map +1 -0
- package/dist/types/internal/metrics.d.ts +120 -0
- package/dist/types/internal/metrics.d.ts.map +1 -0
- package/dist/types/internal/options.d.ts +62 -0
- package/dist/types/internal/options.d.ts.map +1 -0
- package/dist/types/internal/shutdown.d.ts +8 -0
- package/dist/types/internal/shutdown.d.ts.map +1 -0
- package/dist/types/internal/telemetry.d.ts +43 -0
- package/dist/types/internal/telemetry.d.ts.map +1 -0
- package/dist/types/internal/types.d.ts +434 -0
- package/dist/types/internal/types.d.ts.map +1 -0
- package/dist/types/rpc/actions.d.ts +360 -0
- package/dist/types/rpc/actions.d.ts.map +1 -0
- package/dist/types/rpc/http.d.ts +17 -0
- package/dist/types/rpc/http.d.ts.map +1 -0
- package/dist/types/rpc/index.d.ts +43 -0
- package/dist/types/rpc/index.d.ts.map +1 -0
- package/dist/types/runtime/events.d.ts +40 -0
- package/dist/types/runtime/events.d.ts.map +1 -0
- package/dist/types/runtime/filter.d.ts +87 -0
- package/dist/types/runtime/filter.d.ts.map +1 -0
- package/dist/types/runtime/fragments.d.ts +30 -0
- package/dist/types/runtime/fragments.d.ts.map +1 -0
- package/dist/types/runtime/historical.d.ts +123 -0
- package/dist/types/runtime/historical.d.ts.map +1 -0
- package/dist/types/runtime/index.d.ts +89 -0
- package/dist/types/runtime/index.d.ts.map +1 -0
- package/dist/types/runtime/isolated.d.ts +14 -0
- package/dist/types/runtime/isolated.d.ts.map +1 -0
- package/dist/types/runtime/multichain.d.ts +13 -0
- package/dist/types/runtime/multichain.d.ts.map +1 -0
- package/dist/types/runtime/omnichain.d.ts +23 -0
- package/dist/types/runtime/omnichain.d.ts.map +1 -0
- package/dist/types/runtime/realtime.d.ts +93 -0
- package/dist/types/runtime/realtime.d.ts.map +1 -0
- package/dist/types/server/error.d.ts +5 -0
- package/dist/types/server/error.d.ts.map +1 -0
- package/dist/types/server/index.d.ts +13 -0
- package/dist/types/server/index.d.ts.map +1 -0
- package/dist/types/sync-historical/index.d.ts +36 -0
- package/dist/types/sync-historical/index.d.ts.map +1 -0
- package/dist/types/sync-realtime/bloom.d.ts +18 -0
- package/dist/types/sync-realtime/bloom.d.ts.map +1 -0
- package/dist/types/sync-realtime/index.d.ts +47 -0
- package/dist/types/sync-realtime/index.d.ts.map +1 -0
- package/dist/types/sync-store/encode.d.ts +25 -0
- package/dist/types/sync-store/encode.d.ts.map +1 -0
- package/dist/types/sync-store/index.d.ts +135 -0
- package/dist/types/sync-store/index.d.ts.map +1 -0
- package/dist/types/sync-store/migrations.d.ts +8 -0
- package/dist/types/sync-store/migrations.d.ts.map +1 -0
- package/dist/types/sync-store/schema.d.ts +1828 -0
- package/dist/types/sync-store/schema.d.ts.map +1 -0
- package/dist/types/types/db.d.ts +213 -0
- package/dist/types/types/db.d.ts.map +1 -0
- package/dist/types/types/eth.d.ts +196 -0
- package/dist/types/types/eth.d.ts.map +1 -0
- package/dist/types/types/utils.d.ts +38 -0
- package/dist/types/types/utils.d.ts.map +1 -0
- package/dist/types/types/virtual.d.ts +99 -0
- package/dist/types/types/virtual.d.ts.map +1 -0
- package/dist/types/ui/app.d.ts +22 -0
- package/dist/types/ui/app.d.ts.map +1 -0
- package/dist/types/ui/index.d.ts +5 -0
- package/dist/types/ui/index.d.ts.map +1 -0
- package/dist/types/ui/patch.d.ts +7 -0
- package/dist/types/ui/patch.d.ts.map +1 -0
- package/dist/types/utils/abi.d.ts +23 -0
- package/dist/types/utils/abi.d.ts.map +1 -0
- package/dist/types/utils/bigint.d.ts +15 -0
- package/dist/types/utils/bigint.d.ts.map +1 -0
- package/dist/types/utils/chains.d.ts +42 -0
- package/dist/types/utils/chains.d.ts.map +1 -0
- package/dist/types/utils/checkpoint.d.ts +52 -0
- package/dist/types/utils/checkpoint.d.ts.map +1 -0
- package/dist/types/utils/chunk.d.ts +2 -0
- package/dist/types/utils/chunk.d.ts.map +1 -0
- package/dist/types/utils/copy.d.ts +16 -0
- package/dist/types/utils/copy.d.ts.map +1 -0
- package/dist/types/utils/date.d.ts +7 -0
- package/dist/types/utils/date.d.ts.map +1 -0
- package/dist/types/utils/debug.d.ts +105 -0
- package/dist/types/utils/debug.d.ts.map +1 -0
- package/dist/types/utils/decodeAbiParameters.d.ts +28 -0
- package/dist/types/utils/decodeAbiParameters.d.ts.map +1 -0
- package/dist/types/utils/decodeEventLog.d.ts +12 -0
- package/dist/types/utils/decodeEventLog.d.ts.map +1 -0
- package/dist/types/utils/dedupe.d.ts +20 -0
- package/dist/types/utils/dedupe.d.ts.map +1 -0
- package/dist/types/utils/duplicates.d.ts +7 -0
- package/dist/types/utils/duplicates.d.ts.map +1 -0
- package/dist/types/utils/estimate.d.ts +11 -0
- package/dist/types/utils/estimate.d.ts.map +1 -0
- package/dist/types/utils/finality.d.ts +12 -0
- package/dist/types/utils/finality.d.ts.map +1 -0
- package/dist/types/utils/format.d.ts +3 -0
- package/dist/types/utils/format.d.ts.map +1 -0
- package/dist/types/utils/generators.d.ts +42 -0
- package/dist/types/utils/generators.d.ts.map +1 -0
- package/dist/types/utils/hash.d.ts +11 -0
- package/dist/types/utils/hash.d.ts.map +1 -0
- package/dist/types/utils/interval.d.ts +53 -0
- package/dist/types/utils/interval.d.ts.map +1 -0
- package/dist/types/utils/lowercase.d.ts +5 -0
- package/dist/types/utils/lowercase.d.ts.map +1 -0
- package/dist/types/utils/mutex.d.ts +5 -0
- package/dist/types/utils/mutex.d.ts.map +1 -0
- package/dist/types/utils/never.d.ts +2 -0
- package/dist/types/utils/never.d.ts.map +1 -0
- package/dist/types/utils/offset.d.ts +8 -0
- package/dist/types/utils/offset.d.ts.map +1 -0
- package/dist/types/utils/order.d.ts +2 -0
- package/dist/types/utils/order.d.ts.map +1 -0
- package/dist/types/utils/partition.d.ts +22 -0
- package/dist/types/utils/partition.d.ts.map +1 -0
- package/dist/types/utils/pg.d.ts +8 -0
- package/dist/types/utils/pg.d.ts.map +1 -0
- package/dist/types/utils/pglite.d.ts +25 -0
- package/dist/types/utils/pglite.d.ts.map +1 -0
- package/dist/types/utils/port.d.ts +5 -0
- package/dist/types/utils/port.d.ts.map +1 -0
- package/dist/types/utils/print.d.ts +2 -0
- package/dist/types/utils/print.d.ts.map +1 -0
- package/dist/types/utils/promiseAllSettledWithThrow.d.ts +8 -0
- package/dist/types/utils/promiseAllSettledWithThrow.d.ts.map +1 -0
- package/dist/types/utils/promiseWithResolvers.d.ts +10 -0
- package/dist/types/utils/promiseWithResolvers.d.ts.map +1 -0
- package/dist/types/utils/queue.d.ts +33 -0
- package/dist/types/utils/queue.d.ts.map +1 -0
- package/dist/types/utils/range.d.ts +8 -0
- package/dist/types/utils/range.d.ts.map +1 -0
- package/dist/types/utils/result.d.ts +17 -0
- package/dist/types/utils/result.d.ts.map +1 -0
- package/dist/types/utils/sql-parse.d.ts +21 -0
- package/dist/types/utils/sql-parse.d.ts.map +1 -0
- package/dist/types/utils/timer.d.ts +6 -0
- package/dist/types/utils/timer.d.ts.map +1 -0
- package/dist/types/utils/truncate.d.ts +9 -0
- package/dist/types/utils/truncate.d.ts.map +1 -0
- package/dist/types/utils/wait.d.ts +6 -0
- package/dist/types/utils/wait.d.ts.map +1 -0
- package/dist/types/utils/zipper.d.ts +36 -0
- package/dist/types/utils/zipper.d.ts.map +1 -0
- package/package.json +114 -0
- package/src/bin/commands/codegen.ts +56 -0
- package/src/bin/commands/createViews.ts +318 -0
- package/src/bin/commands/dev.ts +490 -0
- package/src/bin/commands/list.ts +208 -0
- package/src/bin/commands/prune.ts +322 -0
- package/src/bin/commands/serve.ts +236 -0
- package/src/bin/commands/start.ts +319 -0
- package/src/bin/isolatedController.ts +300 -0
- package/src/bin/isolatedWorker.ts +192 -0
- package/src/bin/ponder.ts +208 -0
- package/src/bin/utils/codegen.ts +32 -0
- package/src/bin/utils/exit.ts +112 -0
- package/src/build/config.ts +1142 -0
- package/src/build/factory.ts +122 -0
- package/src/build/index.ts +790 -0
- package/src/build/plugin.ts +58 -0
- package/src/build/pre.ts +114 -0
- package/src/build/schema.ts +358 -0
- package/src/build/stacktrace.ts +137 -0
- package/src/client/index.ts +551 -0
- package/src/config/address.ts +32 -0
- package/src/config/eventFilter.ts +33 -0
- package/src/config/index.ts +246 -0
- package/src/config/utilityTypes.ts +152 -0
- package/src/database/actions.ts +873 -0
- package/src/database/index.ts +1029 -0
- package/src/database/queryBuilder.ts +537 -0
- package/src/drizzle/bigint.ts +57 -0
- package/src/drizzle/bytes.ts +68 -0
- package/src/drizzle/hex.ts +58 -0
- package/src/drizzle/index.ts +40 -0
- package/src/drizzle/json.ts +159 -0
- package/src/drizzle/kit/index.ts +1348 -0
- package/src/drizzle/onchain.ts +476 -0
- package/src/drizzle/text.ts +77 -0
- package/src/graphql/graphiql.html.ts +59 -0
- package/src/graphql/index.ts +1351 -0
- package/src/graphql/json.ts +62 -0
- package/src/graphql/middleware.ts +115 -0
- package/src/index.ts +139 -0
- package/src/indexing/addStackTrace.ts +69 -0
- package/src/indexing/client.ts +1184 -0
- package/src/indexing/index.ts +976 -0
- package/src/indexing/profile.ts +771 -0
- package/src/indexing-store/cache.ts +1057 -0
- package/src/indexing-store/index.ts +628 -0
- package/src/indexing-store/profile.ts +557 -0
- package/src/indexing-store/utils.ts +162 -0
- package/src/internal/common.ts +15 -0
- package/src/internal/errors.ts +228 -0
- package/src/internal/logger.ts +252 -0
- package/src/internal/metrics.ts +1030 -0
- package/src/internal/options.ts +130 -0
- package/src/internal/shutdown.ts +32 -0
- package/src/internal/telemetry.ts +303 -0
- package/src/internal/types.ts +597 -0
- package/src/rpc/actions.ts +1344 -0
- package/src/rpc/http.ts +164 -0
- package/src/rpc/index.ts +961 -0
- package/src/runtime/events.ts +875 -0
- package/src/runtime/filter.ts +664 -0
- package/src/runtime/fragments.ts +674 -0
- package/src/runtime/historical.ts +1509 -0
- package/src/runtime/index.ts +569 -0
- package/src/runtime/isolated.ts +778 -0
- package/src/runtime/multichain.ts +860 -0
- package/src/runtime/omnichain.ts +920 -0
- package/src/runtime/realtime.ts +1166 -0
- package/src/server/error.ts +68 -0
- package/src/server/index.ts +173 -0
- package/src/sync-historical/index.ts +1072 -0
- package/src/sync-realtime/bloom.ts +102 -0
- package/src/sync-realtime/index.ts +1312 -0
- package/src/sync-store/encode.ts +153 -0
- package/src/sync-store/index.ts +1633 -0
- package/src/sync-store/migrations.ts +1801 -0
- package/src/sync-store/schema.ts +248 -0
- package/src/types/db.ts +292 -0
- package/src/types/eth.ts +216 -0
- package/src/types/utils.ts +47 -0
- package/src/types/virtual.ts +244 -0
- package/src/types.d.ts +38 -0
- package/src/ui/app.ts +207 -0
- package/src/ui/index.ts +37 -0
- package/src/ui/patch.ts +204 -0
- package/src/utils/abi.ts +103 -0
- package/src/utils/bigint.ts +41 -0
- package/src/utils/chains.ts +22 -0
- package/src/utils/checkpoint.ts +203 -0
- package/src/utils/chunk.ts +7 -0
- package/src/utils/copy.ts +151 -0
- package/src/utils/date.ts +26 -0
- package/src/utils/debug.ts +110 -0
- package/src/utils/decodeAbiParameters.ts +428 -0
- package/src/utils/decodeEventLog.ts +100 -0
- package/src/utils/dedupe.ts +32 -0
- package/src/utils/duplicates.ts +19 -0
- package/src/utils/estimate.ts +27 -0
- package/src/utils/finality.ts +40 -0
- package/src/utils/format.ts +22 -0
- package/src/utils/generators.ts +157 -0
- package/src/utils/hash.ts +22 -0
- package/src/utils/interval.ts +212 -0
- package/src/utils/lowercase.ts +6 -0
- package/src/utils/mutex.ts +33 -0
- package/src/utils/never.ts +3 -0
- package/src/utils/offset.ts +133 -0
- package/src/utils/order.ts +16 -0
- package/src/utils/partition.ts +53 -0
- package/src/utils/pg.ts +230 -0
- package/src/utils/pglite.ts +97 -0
- package/src/utils/port.ts +34 -0
- package/src/utils/print.ts +31 -0
- package/src/utils/promiseAllSettledWithThrow.ts +27 -0
- package/src/utils/promiseWithResolvers.ts +20 -0
- package/src/utils/queue.ts +258 -0
- package/src/utils/range.ts +8 -0
- package/src/utils/result.ts +26 -0
- package/src/utils/sql-parse.ts +1477 -0
- package/src/utils/timer.ts +8 -0
- package/src/utils/truncate.ts +15 -0
- package/src/utils/wait.ts +8 -0
- package/src/utils/zipper.ts +80 -0
|
@@ -0,0 +1,666 @@
|
|
|
1
|
+
import { getPrimaryKeyColumns } from '../drizzle/index.js';
|
|
2
|
+
import { getColumnCasing } from '../drizzle/kit/index.js';
|
|
3
|
+
import { getPartitionName } from '../drizzle/onchain.js';
|
|
4
|
+
import { addErrorMeta, toErrorMeta } from '../indexing/index.js';
|
|
5
|
+
import { CopyFlushError, DelayedInsertError, ShutdownError, } from '../internal/errors.js';
|
|
6
|
+
import { dedupe } from '../utils/dedupe.js';
|
|
7
|
+
import { prettyPrint } from '../utils/print.js';
|
|
8
|
+
import { promiseAllSettledWithThrow } from '../utils/promiseAllSettledWithThrow.js';
|
|
9
|
+
import { startClock } from '../utils/timer.js';
|
|
10
|
+
import { getTableColumns, getTableName, isTable, or, sql, } from "drizzle-orm";
|
|
11
|
+
import { getTableConfig } from "drizzle-orm/pg-core";
|
|
12
|
+
import copy from "pg-copy-streams";
|
|
13
|
+
import { getProfilePatternKey, recordProfilePattern, recoverProfilePattern, } from "./profile.js";
|
|
14
|
+
import { getCacheKey, getPrimaryKeyCache, getWhereCondition, normalizeRow, } from "./utils.js";
|
|
15
|
+
const SAMPLING_RATE = 10;
|
|
16
|
+
const PREDICTION_THRESHOLD = 0.25;
|
|
17
|
+
const LOW_BATCH_THRESHOLD = 20;
|
|
18
|
+
const getBytes = (value) => {
|
|
19
|
+
let size = 0;
|
|
20
|
+
if (typeof value === "number") {
|
|
21
|
+
size += 8;
|
|
22
|
+
}
|
|
23
|
+
else if (typeof value === "string") {
|
|
24
|
+
size += 2 * value.length;
|
|
25
|
+
}
|
|
26
|
+
else if (typeof value === "boolean") {
|
|
27
|
+
size += 4;
|
|
28
|
+
}
|
|
29
|
+
else if (typeof value === "bigint") {
|
|
30
|
+
size += 48;
|
|
31
|
+
}
|
|
32
|
+
else if (value === null || value === undefined) {
|
|
33
|
+
size += 8;
|
|
34
|
+
}
|
|
35
|
+
else if (Array.isArray(value)) {
|
|
36
|
+
size += 24; // NodeJs object overhead (3 * 8 bytes)
|
|
37
|
+
for (const e of value) {
|
|
38
|
+
// value + 8 bytes for the key
|
|
39
|
+
size += getBytes(e) + 8;
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
else {
|
|
43
|
+
size += 24; // NodeJs object overhead (3 * 8 bytes)
|
|
44
|
+
for (const col of Object.values(value)) {
|
|
45
|
+
// value + 8 bytes for the key
|
|
46
|
+
size += getBytes(col) + 8;
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
return size;
|
|
50
|
+
};
|
|
51
|
+
const ESCAPE_REGEX = /([\\\b\f\n\r\t\v])/g;
|
|
52
|
+
export const getCopyText = (table, rows) => {
|
|
53
|
+
let result = "";
|
|
54
|
+
const columns = Object.entries(getTableColumns(table));
|
|
55
|
+
for (let i = 0; i < rows.length; i++) {
|
|
56
|
+
const isLastRow = i === rows.length - 1;
|
|
57
|
+
const row = rows[i];
|
|
58
|
+
for (let j = 0; j < columns.length; j++) {
|
|
59
|
+
const isLastColumn = j === columns.length - 1;
|
|
60
|
+
const [columnName, column] = columns[j];
|
|
61
|
+
let value = row[columnName];
|
|
62
|
+
if (isLastColumn) {
|
|
63
|
+
if (value === null || value === undefined) {
|
|
64
|
+
result += "\\N";
|
|
65
|
+
}
|
|
66
|
+
else {
|
|
67
|
+
if (column.mapToDriverValue !== undefined) {
|
|
68
|
+
value = column.mapToDriverValue(value);
|
|
69
|
+
if (value === null || value === undefined) {
|
|
70
|
+
result += "\\N";
|
|
71
|
+
}
|
|
72
|
+
else {
|
|
73
|
+
result += `${String(value).replace(ESCAPE_REGEX, "\\$1")}`;
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
else {
|
|
79
|
+
if (value === null || value === undefined) {
|
|
80
|
+
result += "\\N\t";
|
|
81
|
+
}
|
|
82
|
+
else {
|
|
83
|
+
if (column.mapToDriverValue !== undefined) {
|
|
84
|
+
value = column.mapToDriverValue(value);
|
|
85
|
+
}
|
|
86
|
+
if (value === null || value === undefined) {
|
|
87
|
+
result += "\\N\t";
|
|
88
|
+
}
|
|
89
|
+
else {
|
|
90
|
+
result += `${String(value).replace(ESCAPE_REGEX, "\\$1")}\t`;
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
if (isLastRow === false) {
|
|
96
|
+
result += "\n";
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
return result;
|
|
100
|
+
};
|
|
101
|
+
export const getCopyHelper = (qb, chainId) => {
|
|
102
|
+
if (qb.$dialect === "pglite") {
|
|
103
|
+
return async (table, text, includeSchema = true) => {
|
|
104
|
+
const target = includeSchema
|
|
105
|
+
? `"${getTableConfig(table).schema ?? "public"}"."${chainId === undefined ? getTableName(table) : getPartitionName(table, chainId)}"`
|
|
106
|
+
: `"${chainId === undefined ? getTableName(table) : getPartitionName(table, chainId)}"`;
|
|
107
|
+
await qb.$client
|
|
108
|
+
.query(`COPY ${target} FROM '/dev/blob'`, [], {
|
|
109
|
+
blob: new Blob([text]),
|
|
110
|
+
})
|
|
111
|
+
// Note: `TransactionError` is applied because the query
|
|
112
|
+
// uses the low-level `$client.query` method.
|
|
113
|
+
.catch((error) => {
|
|
114
|
+
throw new CopyFlushError(error.message);
|
|
115
|
+
});
|
|
116
|
+
};
|
|
117
|
+
}
|
|
118
|
+
else {
|
|
119
|
+
return async (table, text, includeSchema = true) => {
|
|
120
|
+
const target = includeSchema
|
|
121
|
+
? `"${getTableConfig(table).schema ?? "public"}"."${chainId === undefined ? getTableName(table) : getPartitionName(table, chainId)}"`
|
|
122
|
+
: `"${chainId === undefined ? getTableName(table) : getPartitionName(table, chainId)}"`;
|
|
123
|
+
const copyStream = qb.$client.query(copy.from(`COPY ${target} FROM STDIN`));
|
|
124
|
+
await new Promise((resolve, reject) => {
|
|
125
|
+
copyStream.on("finish", resolve);
|
|
126
|
+
copyStream.on("error", reject);
|
|
127
|
+
copyStream.write(text);
|
|
128
|
+
copyStream.end();
|
|
129
|
+
}).catch((error) => {
|
|
130
|
+
throw new CopyFlushError(error.message);
|
|
131
|
+
});
|
|
132
|
+
};
|
|
133
|
+
}
|
|
134
|
+
};
|
|
135
|
+
export const recoverBatchError = async (values, callback) => {
|
|
136
|
+
try {
|
|
137
|
+
await callback(values);
|
|
138
|
+
return { status: "success" };
|
|
139
|
+
}
|
|
140
|
+
catch (error) {
|
|
141
|
+
if (values.length === 1) {
|
|
142
|
+
return { status: "error", error: error, value: values[0] };
|
|
143
|
+
}
|
|
144
|
+
const left = values.slice(0, Math.floor(values.length / 2));
|
|
145
|
+
const right = values.slice(Math.floor(values.length / 2));
|
|
146
|
+
const resultLeft = await recoverBatchError(left, callback);
|
|
147
|
+
if (resultLeft.status === "error") {
|
|
148
|
+
return resultLeft;
|
|
149
|
+
}
|
|
150
|
+
const resultRight = await recoverBatchError(right, callback);
|
|
151
|
+
if (resultRight.status === "error") {
|
|
152
|
+
return resultRight;
|
|
153
|
+
}
|
|
154
|
+
return { status: "success" };
|
|
155
|
+
}
|
|
156
|
+
};
|
|
157
|
+
export const createIndexingCache = ({ common, schemaBuild: { schema }, crashRecoveryCheckpoint, eventCount, chainId, }) => {
|
|
158
|
+
let event;
|
|
159
|
+
let qb = undefined;
|
|
160
|
+
const cache = new Map();
|
|
161
|
+
const insertBuffer = new Map();
|
|
162
|
+
const updateBuffer = new Map();
|
|
163
|
+
/** Profiling data about access patterns for each event. */
|
|
164
|
+
const profile = new Map();
|
|
165
|
+
const tables = Object.values(schema).filter(isTable);
|
|
166
|
+
const primaryKeyCache = getPrimaryKeyCache(tables);
|
|
167
|
+
for (const table of tables) {
|
|
168
|
+
cache.set(table, {
|
|
169
|
+
cache: new Map(),
|
|
170
|
+
prefetched: new Set(),
|
|
171
|
+
spillover: new Set(),
|
|
172
|
+
isCacheComplete: crashRecoveryCheckpoint === undefined,
|
|
173
|
+
bytes: 0,
|
|
174
|
+
diskReads: 0,
|
|
175
|
+
});
|
|
176
|
+
insertBuffer.set(table, new Map());
|
|
177
|
+
updateBuffer.set(table, new Map());
|
|
178
|
+
}
|
|
179
|
+
return {
|
|
180
|
+
has({ table, key }) {
|
|
181
|
+
if (cache.get(table).isCacheComplete)
|
|
182
|
+
return true;
|
|
183
|
+
const ck = getCacheKey(table, key, primaryKeyCache);
|
|
184
|
+
return (cache.get(table).cache.has(ck) ??
|
|
185
|
+
insertBuffer.get(table).has(ck) ??
|
|
186
|
+
updateBuffer.get(table).has(ck));
|
|
187
|
+
},
|
|
188
|
+
async get({ table, key }) {
|
|
189
|
+
if (event &&
|
|
190
|
+
eventCount[event.eventCallback.name] % SAMPLING_RATE === 1) {
|
|
191
|
+
if (profile.has(event.eventCallback.name) === false) {
|
|
192
|
+
profile.set(event.eventCallback.name, new Map());
|
|
193
|
+
for (const table of tables) {
|
|
194
|
+
profile.get(event.eventCallback.name).set(table, new Map());
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
const pattern = recordProfilePattern(event, table, key, Array.from(profile.get(event.eventCallback.name).get(table).values()).map(({ pattern }) => pattern), primaryKeyCache);
|
|
198
|
+
if (pattern) {
|
|
199
|
+
const key = getProfilePatternKey(pattern);
|
|
200
|
+
if (profile.get(event.eventCallback.name).get(table).has(key)) {
|
|
201
|
+
profile.get(event.eventCallback.name).get(table).get(key)
|
|
202
|
+
.count++;
|
|
203
|
+
}
|
|
204
|
+
else {
|
|
205
|
+
profile
|
|
206
|
+
.get(event.eventCallback.name)
|
|
207
|
+
.get(table)
|
|
208
|
+
.set(key, { pattern, count: 1 });
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
const ck = getCacheKey(table, key, primaryKeyCache);
|
|
213
|
+
// Note: order is important, it is an invariant that update entries
|
|
214
|
+
// are prioritized over insert entries
|
|
215
|
+
const bufferEntry = updateBuffer.get(table).get(ck) ?? insertBuffer.get(table).get(ck);
|
|
216
|
+
if (bufferEntry) {
|
|
217
|
+
common.metrics.ponder_indexing_cache_requests_total.inc({
|
|
218
|
+
table: getTableName(table),
|
|
219
|
+
type: cache.get(table).isCacheComplete ? "complete" : "hit",
|
|
220
|
+
});
|
|
221
|
+
return bufferEntry.row;
|
|
222
|
+
}
|
|
223
|
+
const entry = cache.get(table).cache.get(ck);
|
|
224
|
+
if (entry !== undefined) {
|
|
225
|
+
if (cache.get(table).prefetched.has(ck) === false &&
|
|
226
|
+
cache.get(table).isCacheComplete === false) {
|
|
227
|
+
cache.get(table).spillover.add(ck);
|
|
228
|
+
}
|
|
229
|
+
common.metrics.ponder_indexing_cache_requests_total.inc({
|
|
230
|
+
table: getTableName(table),
|
|
231
|
+
type: cache.get(table).isCacheComplete ? "complete" : "hit",
|
|
232
|
+
});
|
|
233
|
+
return entry;
|
|
234
|
+
}
|
|
235
|
+
cache.get(table).diskReads++;
|
|
236
|
+
if (cache.get(table).isCacheComplete) {
|
|
237
|
+
common.metrics.ponder_indexing_cache_requests_total.inc({
|
|
238
|
+
table: getTableName(table),
|
|
239
|
+
type: "complete",
|
|
240
|
+
});
|
|
241
|
+
return null;
|
|
242
|
+
}
|
|
243
|
+
cache.get(table).spillover.add(ck);
|
|
244
|
+
common.metrics.ponder_indexing_cache_requests_total.inc({
|
|
245
|
+
table: getTableName(table),
|
|
246
|
+
type: "miss",
|
|
247
|
+
});
|
|
248
|
+
const endClock = startClock();
|
|
249
|
+
const result = await qb
|
|
250
|
+
.wrap((db) => db.select().from(table).where(getWhereCondition(table, key)))
|
|
251
|
+
.then((res) => (res.length === 0 ? null : res[0]))
|
|
252
|
+
.then((row) => {
|
|
253
|
+
cache.get(table).cache.set(ck, row);
|
|
254
|
+
// Note: the size is not recorded because it is not possible
|
|
255
|
+
// to miss the cache when in the "full in-memory" mode
|
|
256
|
+
return row;
|
|
257
|
+
});
|
|
258
|
+
common.metrics.ponder_indexing_cache_query_duration.observe({
|
|
259
|
+
table: getTableName(table),
|
|
260
|
+
method: "find",
|
|
261
|
+
}, endClock());
|
|
262
|
+
return result;
|
|
263
|
+
},
|
|
264
|
+
set({ table, key, row: _row, isUpdate }) {
|
|
265
|
+
const row = normalizeRow(table, _row, isUpdate);
|
|
266
|
+
const ck = getCacheKey(table, key, primaryKeyCache);
|
|
267
|
+
if (isUpdate) {
|
|
268
|
+
updateBuffer.get(table).set(ck, { row, metadata: { event } });
|
|
269
|
+
}
|
|
270
|
+
else {
|
|
271
|
+
insertBuffer.get(table).set(ck, { row, metadata: { event } });
|
|
272
|
+
}
|
|
273
|
+
return row;
|
|
274
|
+
},
|
|
275
|
+
async delete({ table, key }) {
|
|
276
|
+
const ck = getCacheKey(table, key, primaryKeyCache);
|
|
277
|
+
const inInsertBuffer = insertBuffer.get(table).delete(ck);
|
|
278
|
+
const inUpdateBuffer = updateBuffer.get(table).delete(ck);
|
|
279
|
+
cache.get(table).cache.delete(ck);
|
|
280
|
+
const inDb = await qb
|
|
281
|
+
.wrap((db) => db.delete(table).where(getWhereCondition(table, key)).returning())
|
|
282
|
+
.then((result) => result.length > 0);
|
|
283
|
+
return inInsertBuffer || inUpdateBuffer || inDb;
|
|
284
|
+
},
|
|
285
|
+
async flush({ tableNames } = {}) {
|
|
286
|
+
const context = {
|
|
287
|
+
logger: common.logger.child({ action: "flush_database_rows" }),
|
|
288
|
+
};
|
|
289
|
+
const flushEndClock = startClock();
|
|
290
|
+
const copy = getCopyHelper(qb, chainId);
|
|
291
|
+
const flushTable = async (table) => {
|
|
292
|
+
const shouldRecordBytes = cache.get(table).isCacheComplete;
|
|
293
|
+
if (tableNames !== undefined &&
|
|
294
|
+
tableNames.has(getTableName(table)) === false) {
|
|
295
|
+
return;
|
|
296
|
+
}
|
|
297
|
+
const tableCache = cache.get(table);
|
|
298
|
+
const target = chainId === undefined
|
|
299
|
+
? getTableName(table)
|
|
300
|
+
: getPartitionName(table, chainId);
|
|
301
|
+
const insertValues = Array.from(insertBuffer.get(table).values());
|
|
302
|
+
const updateValues = Array.from(updateBuffer.get(table).values());
|
|
303
|
+
if (insertValues.length > 0) {
|
|
304
|
+
const endClock = startClock();
|
|
305
|
+
if (insertValues.length > LOW_BATCH_THRESHOLD) {
|
|
306
|
+
const text = getCopyText(table, insertValues.map(({ row }) => row));
|
|
307
|
+
await new Promise(setImmediate);
|
|
308
|
+
await copy(table, text);
|
|
309
|
+
}
|
|
310
|
+
else {
|
|
311
|
+
await qb.wrap((db) => db.insert(table).values(insertValues.map(({ row }) => row)), context);
|
|
312
|
+
}
|
|
313
|
+
common.metrics.ponder_indexing_cache_query_duration.observe({
|
|
314
|
+
table: getTableName(table),
|
|
315
|
+
method: "flush",
|
|
316
|
+
}, endClock());
|
|
317
|
+
let bytes = 0;
|
|
318
|
+
for (const [key, entry] of insertBuffer.get(table)) {
|
|
319
|
+
if (shouldRecordBytes && tableCache.cache.has(key) === false) {
|
|
320
|
+
bytes += getBytes(entry.row) + getBytes(key);
|
|
321
|
+
}
|
|
322
|
+
tableCache.cache.set(key, entry.row);
|
|
323
|
+
}
|
|
324
|
+
tableCache.bytes += bytes;
|
|
325
|
+
insertBuffer.get(table).clear();
|
|
326
|
+
await new Promise(setImmediate);
|
|
327
|
+
}
|
|
328
|
+
if (updateValues.length > 0) {
|
|
329
|
+
const primaryKeys = getPrimaryKeyColumns(table);
|
|
330
|
+
const endClock = startClock();
|
|
331
|
+
if (updateValues.length > LOW_BATCH_THRESHOLD) {
|
|
332
|
+
// Steps for flushing "update" entries:
|
|
333
|
+
// 1. Create temp table
|
|
334
|
+
// 2. Copy into temp table
|
|
335
|
+
// 3. Update target table with data from temp
|
|
336
|
+
const createTempTableQuery = `
|
|
337
|
+
CREATE TEMP TABLE IF NOT EXISTS "${target}"
|
|
338
|
+
AS SELECT * FROM "${getTableConfig(table).schema ?? "public"}"."${target}"
|
|
339
|
+
WITH NO DATA;`;
|
|
340
|
+
const updateQuery = `
|
|
341
|
+
UPDATE "${getTableConfig(table).schema ?? "public"}"."${target}" as target
|
|
342
|
+
SET ${Object.values(getTableColumns(table))
|
|
343
|
+
.map((column) => `"${getColumnCasing(column, "snake_case")}" = source."${getColumnCasing(column, "snake_case")}"`)
|
|
344
|
+
.join(",\n")}
|
|
345
|
+
FROM "${target}" source
|
|
346
|
+
WHERE ${primaryKeys
|
|
347
|
+
.map(({ sql }) => `target."${sql}" = source."${sql}"`)
|
|
348
|
+
.join(" AND ")};`;
|
|
349
|
+
await qb.wrap((db) => db.execute(createTempTableQuery), context);
|
|
350
|
+
const text = getCopyText(table, updateValues.map(({ row }) => row));
|
|
351
|
+
await new Promise(setImmediate);
|
|
352
|
+
await copy(table, text, false);
|
|
353
|
+
await qb.wrap((db) => db.execute(updateQuery), context);
|
|
354
|
+
await qb.wrap((db) => db.execute(`TRUNCATE TABLE "${target}"`), context);
|
|
355
|
+
}
|
|
356
|
+
else {
|
|
357
|
+
await qb.wrap((db) => db
|
|
358
|
+
.insert(table)
|
|
359
|
+
.values(updateValues.map(({ row }) => row))
|
|
360
|
+
.onConflictDoUpdate({
|
|
361
|
+
// @ts-ignore
|
|
362
|
+
target: primaryKeys.map(({ js }) => table[js]),
|
|
363
|
+
set: Object.fromEntries(Object.entries(getTableColumns(table)).map(([columnName, column]) => [
|
|
364
|
+
columnName,
|
|
365
|
+
sql.raw(`excluded."${getColumnCasing(column, "snake_case")}"`),
|
|
366
|
+
])),
|
|
367
|
+
}), context);
|
|
368
|
+
}
|
|
369
|
+
common.metrics.ponder_indexing_cache_query_duration.observe({
|
|
370
|
+
table: getTableName(table),
|
|
371
|
+
method: "flush",
|
|
372
|
+
}, endClock());
|
|
373
|
+
let bytes = 0;
|
|
374
|
+
for (const [key, entry] of updateBuffer.get(table)) {
|
|
375
|
+
if (shouldRecordBytes && tableCache.cache.has(key) === false) {
|
|
376
|
+
bytes += getBytes(entry.row) + getBytes(key);
|
|
377
|
+
}
|
|
378
|
+
tableCache.cache.set(key, entry.row);
|
|
379
|
+
}
|
|
380
|
+
tableCache.bytes += bytes;
|
|
381
|
+
updateBuffer.get(table).clear();
|
|
382
|
+
await new Promise(setImmediate);
|
|
383
|
+
}
|
|
384
|
+
if (insertValues.length > 0 || updateValues.length > 0) {
|
|
385
|
+
common.logger.debug({
|
|
386
|
+
msg: "Wrote database rows",
|
|
387
|
+
table: target,
|
|
388
|
+
row_count: insertValues.length + updateValues.length,
|
|
389
|
+
duration: flushEndClock(),
|
|
390
|
+
});
|
|
391
|
+
}
|
|
392
|
+
};
|
|
393
|
+
try {
|
|
394
|
+
if (qb.$dialect === "postgres") {
|
|
395
|
+
await qb.wrap((db) => db.execute("SAVEPOINT flush"), context);
|
|
396
|
+
await promiseAllSettledWithThrow(Array.from(cache.keys()).map(flushTable));
|
|
397
|
+
await qb.wrap((db) => db.execute("RELEASE flush"), context);
|
|
398
|
+
}
|
|
399
|
+
else {
|
|
400
|
+
// Note: pglite must run sequentially
|
|
401
|
+
for (const table of cache.keys()) {
|
|
402
|
+
await flushTable(table);
|
|
403
|
+
}
|
|
404
|
+
}
|
|
405
|
+
}
|
|
406
|
+
catch (_error) {
|
|
407
|
+
let error = _error;
|
|
408
|
+
if (error instanceof ShutdownError || qb.$dialect === "pglite") {
|
|
409
|
+
throw error;
|
|
410
|
+
}
|
|
411
|
+
// Note `isFlushRetry` is true when the previous flush failed. When `isFlushRetry` is false, this
|
|
412
|
+
// function takes an optimized fast path, with support for small batch sizes. PGlite always takes
|
|
413
|
+
// the fast path because it doesn't support delayed insert errors.
|
|
414
|
+
await qb.wrap((db) => db.execute("ROLLBACK to flush"), context);
|
|
415
|
+
for (const table of cache.keys()) {
|
|
416
|
+
if (tableNames !== undefined &&
|
|
417
|
+
tableNames.has(getTableName(table)) === false) {
|
|
418
|
+
continue;
|
|
419
|
+
}
|
|
420
|
+
const target = chainId === undefined
|
|
421
|
+
? getTableName(table)
|
|
422
|
+
: getPartitionName(table, chainId);
|
|
423
|
+
const insertValues = Array.from(insertBuffer.get(table).values());
|
|
424
|
+
const updateValues = Array.from(updateBuffer.get(table).values());
|
|
425
|
+
if (insertValues.length > 0) {
|
|
426
|
+
const endClock = startClock();
|
|
427
|
+
await qb.wrap((db) => db.execute("SAVEPOINT flush"), context);
|
|
428
|
+
const result = await recoverBatchError(insertValues, async (values) => {
|
|
429
|
+
await qb.wrap((db) => db.execute("ROLLBACK to flush"), context);
|
|
430
|
+
const text = getCopyText(table, values.map(({ row }) => row));
|
|
431
|
+
await copy(table, text);
|
|
432
|
+
await qb.wrap((db) => db.execute("SAVEPOINT flush"), context);
|
|
433
|
+
});
|
|
434
|
+
if (result.status === "error") {
|
|
435
|
+
error = new DelayedInsertError(result.error.message);
|
|
436
|
+
error.stack = undefined;
|
|
437
|
+
addErrorMeta(error, `db.insert arguments:\n${prettyPrint(result.value.row)}`);
|
|
438
|
+
if (result.value.metadata.event) {
|
|
439
|
+
addErrorMeta(error, toErrorMeta(result.value.metadata.event));
|
|
440
|
+
common.logger.warn({
|
|
441
|
+
msg: "Failed to write cached database rows",
|
|
442
|
+
event: result.value.metadata.event.eventCallback.name,
|
|
443
|
+
type: "insert",
|
|
444
|
+
table: getTableName(table),
|
|
445
|
+
row_count: insertValues.length,
|
|
446
|
+
duration: endClock(),
|
|
447
|
+
error,
|
|
448
|
+
});
|
|
449
|
+
}
|
|
450
|
+
else {
|
|
451
|
+
common.logger.warn({
|
|
452
|
+
msg: "Failed to write cached database rows",
|
|
453
|
+
type: "insert",
|
|
454
|
+
table: getTableName(table),
|
|
455
|
+
row_count: insertValues.length,
|
|
456
|
+
duration: endClock(),
|
|
457
|
+
error,
|
|
458
|
+
});
|
|
459
|
+
}
|
|
460
|
+
throw error;
|
|
461
|
+
}
|
|
462
|
+
}
|
|
463
|
+
if (updateValues.length > 0) {
|
|
464
|
+
// Steps for flushing "update" entries:
|
|
465
|
+
// 1. Create temp table
|
|
466
|
+
// 2. Copy into temp table
|
|
467
|
+
// 3. Update target table with data from temp
|
|
468
|
+
const createTempTableQuery = `
|
|
469
|
+
CREATE TEMP TABLE IF NOT EXISTS "${target}"
|
|
470
|
+
AS SELECT * FROM "${getTableConfig(table).schema ?? "public"}"."${target}"
|
|
471
|
+
WITH NO DATA;
|
|
472
|
+
`;
|
|
473
|
+
const endClock = startClock();
|
|
474
|
+
await qb.wrap((db) => db.execute(createTempTableQuery), context);
|
|
475
|
+
await qb.wrap((db) => db.execute("SAVEPOINT flush"), context);
|
|
476
|
+
const result = await recoverBatchError(updateValues, async (values) => {
|
|
477
|
+
await qb.wrap((db) => db.execute("ROLLBACK to flush"), context);
|
|
478
|
+
const text = getCopyText(table, values.map(({ row }) => row));
|
|
479
|
+
await copy(table, text, false);
|
|
480
|
+
await qb.wrap((db) => db.execute("SAVEPOINT flush"), context);
|
|
481
|
+
});
|
|
482
|
+
await qb.wrap((db) => db.execute(`TRUNCATE TABLE "${target}"`), context);
|
|
483
|
+
if (result.status === "error") {
|
|
484
|
+
error = new DelayedInsertError(result.error.message);
|
|
485
|
+
error.stack = undefined;
|
|
486
|
+
addErrorMeta(error, `db.update arguments:\n${prettyPrint(result.value.row)}`);
|
|
487
|
+
if (result.value.metadata.event) {
|
|
488
|
+
addErrorMeta(error, toErrorMeta(result.value.metadata.event));
|
|
489
|
+
common.logger.warn({
|
|
490
|
+
msg: "Failed to write cached database rows",
|
|
491
|
+
event: result.value.metadata.event.eventCallback.name,
|
|
492
|
+
type: "update",
|
|
493
|
+
table: getTableName(table),
|
|
494
|
+
row_count: updateValues.length,
|
|
495
|
+
duration: endClock(),
|
|
496
|
+
error,
|
|
497
|
+
});
|
|
498
|
+
}
|
|
499
|
+
else {
|
|
500
|
+
common.logger.warn({
|
|
501
|
+
msg: "Failed to write cached database rows",
|
|
502
|
+
type: "update",
|
|
503
|
+
table: getTableName(table),
|
|
504
|
+
row_count: updateValues.length,
|
|
505
|
+
duration: endClock(),
|
|
506
|
+
error,
|
|
507
|
+
});
|
|
508
|
+
}
|
|
509
|
+
throw error;
|
|
510
|
+
}
|
|
511
|
+
}
|
|
512
|
+
}
|
|
513
|
+
// Note: if we weren't able to find the exact row that caused the error,
|
|
514
|
+
// throw the original error.
|
|
515
|
+
throw error;
|
|
516
|
+
}
|
|
517
|
+
},
|
|
518
|
+
async prefetch({ events }) {
|
|
519
|
+
const context = {
|
|
520
|
+
logger: common.logger.child({ action: "prefetch_database_rows" }),
|
|
521
|
+
};
|
|
522
|
+
let totalBytes = 0;
|
|
523
|
+
for (const table of tables) {
|
|
524
|
+
totalBytes += cache.get(table).bytes;
|
|
525
|
+
}
|
|
526
|
+
// If data from the cache needs to be evicted, start with the
|
|
527
|
+
// table with the least disk reads.
|
|
528
|
+
if (totalBytes > common.options.indexingCacheMaxBytes) {
|
|
529
|
+
for (const table of tables.sort((a, b) => cache.get(a).diskReads - cache.get(b).diskReads)) {
|
|
530
|
+
if (cache.get(table).isCacheComplete === false)
|
|
531
|
+
continue;
|
|
532
|
+
totalBytes -= cache.get(table).bytes;
|
|
533
|
+
cache.get(table).bytes = 0;
|
|
534
|
+
cache.get(table).cache.clear();
|
|
535
|
+
cache.get(table).isCacheComplete = false;
|
|
536
|
+
// Note: spillover is not cleared because it is an invariant
|
|
537
|
+
// it is empty
|
|
538
|
+
common.logger.debug({
|
|
539
|
+
msg: "Evicted cached database rows",
|
|
540
|
+
table: getTableName(table),
|
|
541
|
+
row_count: cache.get(table).cache.size,
|
|
542
|
+
});
|
|
543
|
+
if (totalBytes < common.options.indexingCacheMaxBytes)
|
|
544
|
+
break;
|
|
545
|
+
}
|
|
546
|
+
}
|
|
547
|
+
if (tables.every((table) => cache.get(table).isCacheComplete)) {
|
|
548
|
+
return;
|
|
549
|
+
}
|
|
550
|
+
// Use historical accesses + next event batch to determine which
|
|
551
|
+
// rows are going to be accessed, and preload them into the cache.
|
|
552
|
+
const prediction = new Map();
|
|
553
|
+
for (const table of tables) {
|
|
554
|
+
prediction.set(table, new Map());
|
|
555
|
+
}
|
|
556
|
+
for (const event of events) {
|
|
557
|
+
if (profile.has(event.eventCallback.name)) {
|
|
558
|
+
for (const table of tables) {
|
|
559
|
+
if (cache.get(table).isCacheComplete)
|
|
560
|
+
continue;
|
|
561
|
+
for (const [, { count, pattern }] of profile
|
|
562
|
+
.get(event.eventCallback.name)
|
|
563
|
+
.get(table)) {
|
|
564
|
+
// Expected value of times the prediction will be used.
|
|
565
|
+
const ev = (count * SAMPLING_RATE) / eventCount[event.eventCallback.name];
|
|
566
|
+
if (ev > PREDICTION_THRESHOLD) {
|
|
567
|
+
const row = recoverProfilePattern(pattern, event);
|
|
568
|
+
const key = getCacheKey(table, row, primaryKeyCache);
|
|
569
|
+
prediction.get(table).set(key, row);
|
|
570
|
+
}
|
|
571
|
+
}
|
|
572
|
+
}
|
|
573
|
+
}
|
|
574
|
+
}
|
|
575
|
+
for (const [table, tableCache] of cache) {
|
|
576
|
+
if (cache.get(table).isCacheComplete)
|
|
577
|
+
continue;
|
|
578
|
+
for (const key of tableCache.cache.keys()) {
|
|
579
|
+
if (tableCache.spillover.has(key) ||
|
|
580
|
+
prediction.get(table).has(key)) {
|
|
581
|
+
prediction.get(table).delete(key);
|
|
582
|
+
}
|
|
583
|
+
else {
|
|
584
|
+
tableCache.cache.delete(key);
|
|
585
|
+
}
|
|
586
|
+
}
|
|
587
|
+
}
|
|
588
|
+
for (const table of tables) {
|
|
589
|
+
cache.get(table).spillover.clear();
|
|
590
|
+
cache.get(table).prefetched.clear();
|
|
591
|
+
}
|
|
592
|
+
for (const [table, tablePredictions] of prediction) {
|
|
593
|
+
common.metrics.ponder_indexing_cache_requests_total.inc({
|
|
594
|
+
table: getTableName(table),
|
|
595
|
+
type: "prefetch",
|
|
596
|
+
}, tablePredictions.size);
|
|
597
|
+
}
|
|
598
|
+
await Promise.all(Array.from(prediction.entries())
|
|
599
|
+
.filter(([, tablePredictions]) => tablePredictions.size > 0)
|
|
600
|
+
.map(async ([table, tablePredictions]) => {
|
|
601
|
+
for (const [key] of tablePredictions) {
|
|
602
|
+
cache.get(table).prefetched.add(key);
|
|
603
|
+
}
|
|
604
|
+
const conditions = dedupe(Array.from(tablePredictions), ([key]) => key).map(([, prediction]) => getWhereCondition(table, prediction));
|
|
605
|
+
if (conditions.length === 0)
|
|
606
|
+
return;
|
|
607
|
+
const endClock = startClock();
|
|
608
|
+
await qb
|
|
609
|
+
.wrap((db) => db
|
|
610
|
+
.select()
|
|
611
|
+
.from(table)
|
|
612
|
+
.where(or(...conditions)), context)
|
|
613
|
+
.then((results) => {
|
|
614
|
+
const resultsPerKey = new Map();
|
|
615
|
+
for (const result of results) {
|
|
616
|
+
const ck = getCacheKey(table, result, primaryKeyCache);
|
|
617
|
+
resultsPerKey.set(ck, result);
|
|
618
|
+
}
|
|
619
|
+
const tableCache = cache.get(table);
|
|
620
|
+
for (const key of tablePredictions.keys()) {
|
|
621
|
+
if (resultsPerKey.has(key)) {
|
|
622
|
+
tableCache.cache.set(key, resultsPerKey.get(key));
|
|
623
|
+
}
|
|
624
|
+
else {
|
|
625
|
+
tableCache.cache.set(key, null);
|
|
626
|
+
}
|
|
627
|
+
}
|
|
628
|
+
common.logger.debug({
|
|
629
|
+
msg: "Pre-queried database rows",
|
|
630
|
+
table: getTableName(table),
|
|
631
|
+
row_count: results.length,
|
|
632
|
+
duration: endClock(),
|
|
633
|
+
});
|
|
634
|
+
});
|
|
635
|
+
common.metrics.ponder_indexing_cache_query_duration.observe({
|
|
636
|
+
table: getTableName(table),
|
|
637
|
+
method: "load",
|
|
638
|
+
}, endClock());
|
|
639
|
+
}));
|
|
640
|
+
},
|
|
641
|
+
invalidate() {
|
|
642
|
+
for (const tableCache of cache.values()) {
|
|
643
|
+
tableCache.isCacheComplete = false;
|
|
644
|
+
}
|
|
645
|
+
},
|
|
646
|
+
clear() {
|
|
647
|
+
for (const tableCache of cache.values()) {
|
|
648
|
+
tableCache.cache.clear();
|
|
649
|
+
tableCache.spillover.clear();
|
|
650
|
+
}
|
|
651
|
+
for (const tableBuffer of insertBuffer.values()) {
|
|
652
|
+
tableBuffer.clear();
|
|
653
|
+
}
|
|
654
|
+
for (const tableBuffer of updateBuffer.values()) {
|
|
655
|
+
tableBuffer.clear();
|
|
656
|
+
}
|
|
657
|
+
},
|
|
658
|
+
set event(_event) {
|
|
659
|
+
event = _event;
|
|
660
|
+
},
|
|
661
|
+
set qb(_qb) {
|
|
662
|
+
qb = _qb;
|
|
663
|
+
},
|
|
664
|
+
};
|
|
665
|
+
};
|
|
666
|
+
//# sourceMappingURL=cache.js.map
|