tapimo 0.10.2 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/App.d.ts +3412 -1968
- package/dist/App.d.ts.map +1 -1
- package/dist/App.js.map +1 -1
- package/dist/Webhooks.d.ts +0 -75
- package/dist/Webhooks.d.ts.map +1 -1
- package/dist/Webhooks.js +1 -143
- package/dist/Webhooks.js.map +1 -1
- package/dist/admin/App.d.ts +74 -74
- package/dist/admin/apps/api-keys.d.ts +4 -4
- package/dist/admin/apps/earn-vaults.d.ts +1 -1
- package/dist/admin/apps/organizations.d.ts +3 -3
- package/dist/admin/apps/verified-tokens.d.ts +74 -74
- package/dist/apps/data/App.d.ts +3618 -2954
- package/dist/apps/data/App.d.ts.map +1 -1
- package/dist/apps/data/App.js +5 -2
- package/dist/apps/data/App.js.map +1 -1
- package/dist/apps/data/routes/activities.d.ts +1 -1
- package/dist/apps/data/routes/activities.d.ts.map +1 -1
- package/dist/apps/data/routes/activities.js +55 -27
- package/dist/apps/data/routes/activities.js.map +1 -1
- package/dist/apps/data/routes/addresses.d.ts +343 -343
- package/dist/apps/data/routes/balances.d.ts +11 -11
- package/dist/apps/data/routes/blocks.d.ts +5 -5
- package/dist/apps/data/routes/coingecko.d.ts +176 -176
- package/dist/apps/data/routes/earn.d.ts +4 -4
- package/dist/apps/data/routes/exchanges.d.ts +4 -4
- package/dist/apps/data/routes/fee-amm.d.ts +26 -26
- package/dist/apps/data/routes/indexer.d.ts +34 -34
- package/dist/apps/data/routes/receipts.d.ts +236 -236
- package/dist/apps/data/routes/tokenlist.d.ts +12 -12
- package/dist/apps/data/routes/tokens.d.ts +3 -3
- package/dist/apps/data/routes/transactions.d.ts +1 -1
- package/dist/apps/data/routes/transfers.d.ts +13 -13
- package/dist/apps/data/routes/valuation.d.ts +21 -21
- package/dist/apps/data/routes/verified-tokens.d.ts +5 -5
- package/dist/apps/data/routes/webhooks.d.ts +1011 -151
- package/dist/apps/data/routes/webhooks.d.ts.map +1 -1
- package/dist/apps/data/routes/webhooks.js +124 -737
- package/dist/apps/data/routes/webhooks.js.map +1 -1
- package/dist/apps/data/routes/zones.d.ts +31 -31
- package/dist/apps/funding/routes/deposit-addresses.d.ts +2 -2
- package/dist/apps/funding/routes/providers.d.ts +13 -13
- package/dist/apps/funding/routes/transfers.d.ts +33 -33
- package/dist/apps/management/App.d.ts +793 -16
- package/dist/apps/management/App.d.ts.map +1 -1
- package/dist/apps/management/routes/api-keys.d.ts +50 -50
- package/dist/apps/management/routes/faucet.d.ts +31 -31
- package/dist/apps/management/routes/invite-links.d.ts +42 -42
- package/dist/apps/management/routes/members.d.ts +79 -79
- package/dist/apps/management/routes/projects.d.ts +40 -40
- package/dist/apps/management/routes/usage.d.ts +80 -80
- package/dist/apps/management/routes/webhooks.d.ts +1031 -59
- package/dist/apps/management/routes/webhooks.d.ts.map +1 -1
- package/dist/apps/management/routes/webhooks.js +27 -4
- package/dist/apps/management/routes/webhooks.js.map +1 -1
- package/dist/db/Db.d.ts.map +1 -1
- package/dist/db/Db.js +21 -0
- package/dist/db/Db.js.map +1 -1
- package/dist/db/Schema.d.ts +7 -1
- package/dist/db/Schema.d.ts.map +1 -1
- package/dist/db/Schema.js +9 -0
- package/dist/db/Schema.js.map +1 -1
- package/dist/db/tables/webhookQueueEvents.d.ts +3 -1
- package/dist/db/tables/webhookQueueEvents.d.ts.map +1 -1
- package/dist/db/tables/webhookQueueEvents.js +4 -0
- package/dist/db/tables/webhookQueueEvents.js.map +1 -1
- package/dist/db/tables/webhookSubscriptions.d.ts +33 -69
- package/dist/db/tables/webhookSubscriptions.d.ts.map +1 -1
- package/dist/db/tables/webhookSubscriptions.js +63 -100
- package/dist/db/tables/webhookSubscriptions.js.map +1 -1
- package/dist/internal/MetricSink.d.ts +35 -125
- package/dist/internal/MetricSink.d.ts.map +1 -1
- package/dist/internal/MetricSink.js +50 -121
- package/dist/internal/MetricSink.js.map +1 -1
- package/dist/internal/Schema.d.ts +3 -1
- package/dist/internal/Schema.d.ts.map +1 -1
- package/dist/internal/Schema.js +4 -2
- package/dist/internal/Schema.js.map +1 -1
- package/dist/internal/WebhookTransfer.d.ts +1 -55
- package/dist/internal/WebhookTransfer.d.ts.map +1 -1
- package/dist/internal/WebhookTransfer.js +1 -239
- package/dist/internal/WebhookTransfer.js.map +1 -1
- package/dist/internal/Webhooks.d.ts +83 -44
- package/dist/internal/Webhooks.d.ts.map +1 -1
- package/dist/internal/Webhooks.js +87 -73
- package/dist/internal/Webhooks.js.map +1 -1
- package/dist/internal/funding/DepositWebhook.d.ts +25 -0
- package/dist/internal/funding/DepositWebhook.d.ts.map +1 -0
- package/dist/internal/funding/DepositWebhook.js +74 -0
- package/dist/internal/funding/DepositWebhook.js.map +1 -0
- package/dist/internal/funding/Reconciliation.d.ts +6 -0
- package/dist/internal/funding/Reconciliation.d.ts.map +1 -1
- package/dist/internal/funding/Reconciliation.js +42 -10
- package/dist/internal/funding/Reconciliation.js.map +1 -1
- package/dist/internal/funding/index.d.ts +1 -0
- package/dist/internal/funding/index.d.ts.map +1 -1
- package/dist/internal/funding/index.js +1 -0
- package/dist/internal/funding/index.js.map +1 -1
- package/dist/internal/webhookDestinations/slack.d.ts.map +1 -1
- package/dist/internal/webhookDestinations/slack.js +16 -0
- package/dist/internal/webhookDestinations/slack.js.map +1 -1
- package/package.json +1 -1
- package/src/App.ts +3 -0
- package/src/Client.test-d.ts +22 -1
- package/src/Webhooks.ts +1 -220
- package/src/apps/data/App.ts +8 -2
- package/src/apps/data/routes/activities.test.ts +107 -11
- package/src/apps/data/routes/activities.ts +64 -32
- package/src/apps/data/routes/webhooks.test.ts +179 -1497
- package/src/apps/data/routes/webhooks.ts +150 -974
- package/src/apps/management/routes/webhooks.test.ts +79 -1
- package/src/apps/management/routes/webhooks.ts +40 -4
- package/src/db/Db.test.ts +1 -5
- package/src/db/Db.ts +23 -0
- package/src/db/Schema.ts +13 -0
- package/src/db/tables/organizations.test.ts +5 -2
- package/src/db/tables/webhookQueueEvents.ts +7 -1
- package/src/db/tables/webhookSubscriptions.ts +89 -138
- package/src/internal/MetricSink.test.ts +41 -373
- package/src/internal/MetricSink.ts +92 -273
- package/src/internal/Schema.test.ts +18 -0
- package/src/internal/Schema.ts +13 -2
- package/src/internal/WebhookTransfer.ts +1 -351
- package/src/internal/Webhooks.test.ts +102 -139
- package/src/internal/Webhooks.ts +175 -106
- package/src/internal/funding/DepositWebhook.test.ts +131 -0
- package/src/internal/funding/DepositWebhook.ts +101 -0
- package/src/internal/funding/Reconciliation.test.ts +14 -0
- package/src/internal/funding/Reconciliation.ts +161 -86
- package/src/internal/funding/index.test.ts +1 -0
- package/src/internal/funding/index.ts +1 -0
- package/src/internal/webhookDestinations/slack.ts +15 -0
- package/src/Webhooks.bench.ts +0 -714
- package/src/Webhooks.memory.bench.ts +0 -250
- package/src/Webhooks.test.ts +0 -146
|
@@ -4,16 +4,6 @@ import * as z from 'zod/mini'
|
|
|
4
4
|
import * as Transfers from '../apps/data/routes/transfers.js'
|
|
5
5
|
import * as Schema from './Schema.js'
|
|
6
6
|
import * as Value from './Value.js'
|
|
7
|
-
import type * as Viem from './Viem.js'
|
|
8
|
-
|
|
9
|
-
/** Subscriptions matched per shared transfer scan. */
|
|
10
|
-
export const scanCohortSize = 32
|
|
11
|
-
|
|
12
|
-
/**
|
|
13
|
-
* Blocks per transfer `eth_getLogs` request. Firehose scans stop after a
|
|
14
|
-
* 100-row page, so small windows bound the raw RPC body retained before that cap.
|
|
15
|
-
*/
|
|
16
|
-
const scanChunkBlocks = 8
|
|
17
7
|
|
|
18
8
|
const event = parseAbiItem(Transfers.signature)
|
|
19
9
|
const topic = encodeEventTopics({ abi: [event] })[0]
|
|
@@ -33,271 +23,6 @@ export type EventData = {
|
|
|
33
23
|
transactionHash: string
|
|
34
24
|
}
|
|
35
25
|
|
|
36
|
-
type Candidate = {
|
|
37
|
-
index: number
|
|
38
|
-
key: string
|
|
39
|
-
state: scan.State & { window: scan.Window }
|
|
40
|
-
}
|
|
41
|
-
|
|
42
|
-
type Cohort = {
|
|
43
|
-
key: string
|
|
44
|
-
states: Candidate[]
|
|
45
|
-
window: { from: number; to: number }
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
type Accumulator = {
|
|
49
|
-
full: boolean
|
|
50
|
-
items: scan.Item[]
|
|
51
|
-
state: Candidate['state']
|
|
52
|
-
}
|
|
53
|
-
|
|
54
|
-
/** Returns a stable key for transfer filters with equivalent RPC pushdowns. */
|
|
55
|
-
function scanKey(filters_input: Record<string, unknown>): string {
|
|
56
|
-
const filters = filters_input as Filters
|
|
57
|
-
return JSON.stringify(
|
|
58
|
-
normalizeRpcValue({
|
|
59
|
-
args: logArgs(filters),
|
|
60
|
-
token: filters.token,
|
|
61
|
-
}),
|
|
62
|
-
)
|
|
63
|
-
}
|
|
64
|
-
|
|
65
|
-
/** Scans transfer subscriptions through shared, memory-bounded RPC cohorts. */
|
|
66
|
-
export async function scan(options: scan.Options): Promise<PromiseSettledResult<scan.Page>[]> {
|
|
67
|
-
const results: PromiseSettledResult<scan.Page>[] = []
|
|
68
|
-
const candidates: Candidate[] = []
|
|
69
|
-
for (let index = 0; index < options.states.length; index++) {
|
|
70
|
-
const state = options.states[index]!
|
|
71
|
-
if (!state.window) {
|
|
72
|
-
results[index] = { status: 'fulfilled', value: { items: [] } }
|
|
73
|
-
continue
|
|
74
|
-
}
|
|
75
|
-
candidates.push({
|
|
76
|
-
index,
|
|
77
|
-
key: scanKey(state.filters),
|
|
78
|
-
state: { ...state, window: state.window },
|
|
79
|
-
})
|
|
80
|
-
}
|
|
81
|
-
|
|
82
|
-
candidates.sort(
|
|
83
|
-
(a, b) =>
|
|
84
|
-
a.key.localeCompare(b.key) ||
|
|
85
|
-
a.state.window.from - b.state.window.from ||
|
|
86
|
-
a.state.window.to - b.state.window.to,
|
|
87
|
-
)
|
|
88
|
-
const cohorts: Cohort[] = []
|
|
89
|
-
for (const candidate of candidates) {
|
|
90
|
-
const previous = cohorts.at(-1)
|
|
91
|
-
const combinedBlocks = previous
|
|
92
|
-
? Math.max(previous.window.to, candidate.state.window.to) - previous.window.from + 1
|
|
93
|
-
: 0
|
|
94
|
-
if (
|
|
95
|
-
!previous ||
|
|
96
|
-
candidate.key !== previous.key ||
|
|
97
|
-
candidate.state.window.from > previous.window.to ||
|
|
98
|
-
combinedBlocks > options.maxBlocks ||
|
|
99
|
-
previous.states.length >= scanCohortSize
|
|
100
|
-
) {
|
|
101
|
-
cohorts.push({
|
|
102
|
-
key: candidate.key,
|
|
103
|
-
states: [candidate],
|
|
104
|
-
window: { from: candidate.state.window.from, to: candidate.state.window.to },
|
|
105
|
-
})
|
|
106
|
-
continue
|
|
107
|
-
}
|
|
108
|
-
previous.states.push(candidate)
|
|
109
|
-
previous.window.to = Math.max(previous.window.to, candidate.state.window.to)
|
|
110
|
-
}
|
|
111
|
-
|
|
112
|
-
const settled = await mapWithLimit(cohorts, options.concurrency, (cohort) =>
|
|
113
|
-
options.schedule(async () => ({
|
|
114
|
-
cohort,
|
|
115
|
-
pages: await scanCohort({ client: options.client, cohort }),
|
|
116
|
-
})),
|
|
117
|
-
)
|
|
118
|
-
for (const entry of settled) {
|
|
119
|
-
if (entry.status === 'rejected') continue
|
|
120
|
-
for (let index = 0; index < entry.value.cohort.states.length; index++)
|
|
121
|
-
results[entry.value.cohort.states[index]!.index] = {
|
|
122
|
-
status: 'fulfilled',
|
|
123
|
-
value: entry.value.pages[index]!,
|
|
124
|
-
}
|
|
125
|
-
}
|
|
126
|
-
for (let index = 0; index < settled.length; index++) {
|
|
127
|
-
const entry = settled[index]!
|
|
128
|
-
if (entry.status !== 'rejected') continue
|
|
129
|
-
for (const candidate of cohorts[index]!.states)
|
|
130
|
-
results[candidate.index] = { reason: entry.reason, status: 'rejected' }
|
|
131
|
-
}
|
|
132
|
-
return results
|
|
133
|
-
}
|
|
134
|
-
|
|
135
|
-
export declare namespace scan {
|
|
136
|
-
/** One transfer match paired with its keyset position. */
|
|
137
|
-
type Item = {
|
|
138
|
-
/** Transfer position as `[blockNumber, logIndex]`. */
|
|
139
|
-
cursor: readonly [number, number]
|
|
140
|
-
/** Decoded webhook transfer payload. */
|
|
141
|
-
data: unknown
|
|
142
|
-
}
|
|
143
|
-
|
|
144
|
-
/** One bounded page returned to the poller. */
|
|
145
|
-
type Page = {
|
|
146
|
-
/** Whether another matching page remains. */
|
|
147
|
-
hasMore?: boolean | undefined
|
|
148
|
-
/** Transfer matches in ascending chain order. */
|
|
149
|
-
items: readonly Item[]
|
|
150
|
-
/** Highest block fully scanned when no matched cursor must be retained. */
|
|
151
|
-
scannedThrough?: number | undefined
|
|
152
|
-
}
|
|
153
|
-
|
|
154
|
-
/** One subscription's inputs for a shared transfer scan. */
|
|
155
|
-
type State = {
|
|
156
|
-
/** Validated transfer subscription filters. */
|
|
157
|
-
filters: Record<string, unknown>
|
|
158
|
-
/** Maximum matches retained for this fair scan page. */
|
|
159
|
-
limit: number
|
|
160
|
-
/** Bounded chain window, or undefined when the cursor is caught up. */
|
|
161
|
-
window?: Window | undefined
|
|
162
|
-
}
|
|
163
|
-
|
|
164
|
-
/** One subscription's chain range and already-dispatched position. */
|
|
165
|
-
type Window = {
|
|
166
|
-
/** First block included in the upstream scan. */
|
|
167
|
-
from: number
|
|
168
|
-
/** Last dispatched `[blockNumber, logIndex]` position. */
|
|
169
|
-
skip: readonly [number, number]
|
|
170
|
-
/** Last block included in the upstream scan. */
|
|
171
|
-
to: number
|
|
172
|
-
}
|
|
173
|
-
|
|
174
|
-
/** Inputs for scanning transfer subscriptions. */
|
|
175
|
-
type Options = {
|
|
176
|
-
/** Maximum concurrent cohort RPC scans. */
|
|
177
|
-
concurrency: number
|
|
178
|
-
/** Viem client for raw chain RPC requests. */
|
|
179
|
-
client: Viem.getClient.ReturnType
|
|
180
|
-
/** Maximum combined block range for one cohort. */
|
|
181
|
-
maxBlocks: number
|
|
182
|
-
/** Shared scheduler that bounds upstream scans across event types. */
|
|
183
|
-
schedule: <value>(fn: () => Promise<value>) => Promise<value>
|
|
184
|
-
/** Subscription states aligned with the returned results. */
|
|
185
|
-
states: readonly State[]
|
|
186
|
-
}
|
|
187
|
-
}
|
|
188
|
-
|
|
189
|
-
/** Runs ordered async work with bounded concurrency. */
|
|
190
|
-
async function mapWithLimit<item, value>(
|
|
191
|
-
items: readonly item[],
|
|
192
|
-
limit: number,
|
|
193
|
-
fn: (item: item) => Promise<value>,
|
|
194
|
-
): Promise<PromiseSettledResult<value>[]> {
|
|
195
|
-
const results: PromiseSettledResult<value>[] = []
|
|
196
|
-
let next = 0
|
|
197
|
-
await Promise.all(
|
|
198
|
-
Array.from({ length: Math.min(limit, items.length) }, async () => {
|
|
199
|
-
while (next < items.length) {
|
|
200
|
-
const index = next++
|
|
201
|
-
try {
|
|
202
|
-
results[index] = { status: 'fulfilled', value: await fn(items[index]!) }
|
|
203
|
-
} catch (reason) {
|
|
204
|
-
results[index] = { reason, status: 'rejected' }
|
|
205
|
-
}
|
|
206
|
-
}
|
|
207
|
-
}),
|
|
208
|
-
)
|
|
209
|
-
return results
|
|
210
|
-
}
|
|
211
|
-
|
|
212
|
-
/** Recursively canonicalizes an RPC filter value for query grouping. */
|
|
213
|
-
function normalizeRpcValue(value: unknown): unknown {
|
|
214
|
-
if (typeof value === 'bigint') return `bigint:${value}`
|
|
215
|
-
if (Array.isArray(value)) return value.map(normalizeRpcValue)
|
|
216
|
-
if (value && typeof value === 'object')
|
|
217
|
-
return Object.fromEntries(
|
|
218
|
-
Object.entries(value)
|
|
219
|
-
.filter(([, item]) => item !== undefined)
|
|
220
|
-
.sort(([a], [b]) => a.localeCompare(b))
|
|
221
|
-
.map(([name, item]) => [name, normalizeRpcValue(item)]),
|
|
222
|
-
)
|
|
223
|
-
return value
|
|
224
|
-
}
|
|
225
|
-
|
|
226
|
-
/** Returns indexed argument combinations pushed into Transfer log reads. */
|
|
227
|
-
function logArgs(filters: Filters) {
|
|
228
|
-
const args = {
|
|
229
|
-
...(filters.sender ? { from: filters.sender } : {}),
|
|
230
|
-
...(filters.recipient ? { to: filters.recipient } : {}),
|
|
231
|
-
}
|
|
232
|
-
return filters.address && !filters.sender && !filters.recipient
|
|
233
|
-
? [{ from: filters.address }, { to: filters.address }]
|
|
234
|
-
: [args]
|
|
235
|
-
}
|
|
236
|
-
|
|
237
|
-
/** Encodes one indexed address as its left-padded event topic. */
|
|
238
|
-
function addressTopic(address: string): `0x${string}` {
|
|
239
|
-
return `0x${'0'.repeat(24)}${address.slice(2).toLowerCase()}`
|
|
240
|
-
}
|
|
241
|
-
|
|
242
|
-
/** Builds the exact Transfer topic filter for one indexed-argument combination. */
|
|
243
|
-
function topics(args: { from?: string | undefined; to?: string | undefined }) {
|
|
244
|
-
const values: (`0x${string}` | null)[] = [
|
|
245
|
-
topic,
|
|
246
|
-
args.from ? addressTopic(args.from) : null,
|
|
247
|
-
args.to ? addressTopic(args.to) : null,
|
|
248
|
-
]
|
|
249
|
-
while (values.at(-1) === null) values.pop()
|
|
250
|
-
return values
|
|
251
|
-
}
|
|
252
|
-
|
|
253
|
-
/** Renders a block number as a JSON-RPC block tag. */
|
|
254
|
-
function blockTag(value: number): `0x${string}` {
|
|
255
|
-
return `0x${value.toString(16)}`
|
|
256
|
-
}
|
|
257
|
-
|
|
258
|
-
/** Issues an untyped JSON-RPC request through the Viem client. */
|
|
259
|
-
function rpc<result>(
|
|
260
|
-
client: Viem.getClient.ReturnType,
|
|
261
|
-
method: string,
|
|
262
|
-
params: readonly unknown[],
|
|
263
|
-
): Promise<result> {
|
|
264
|
-
return (client.request as (args: { method: string; params: unknown[] }) => Promise<result>)({
|
|
265
|
-
method,
|
|
266
|
-
params: [...params],
|
|
267
|
-
})
|
|
268
|
-
}
|
|
269
|
-
|
|
270
|
-
/** Reads raw Transfer logs, splitting the either-side shortcut into two RPC filters. */
|
|
271
|
-
async function getLogs(
|
|
272
|
-
client: Viem.getClient.ReturnType,
|
|
273
|
-
filters: Filters,
|
|
274
|
-
window: { from: number; to: number },
|
|
275
|
-
): Promise<RpcLog[]> {
|
|
276
|
-
const pages = await Promise.all(
|
|
277
|
-
logArgs(filters).map((args) =>
|
|
278
|
-
rpc<RpcLog[]>(client, 'eth_getLogs', [
|
|
279
|
-
{
|
|
280
|
-
...(filters.token ? { address: filters.token } : {}),
|
|
281
|
-
fromBlock: blockTag(window.from),
|
|
282
|
-
topics: topics(args),
|
|
283
|
-
toBlock: blockTag(window.to),
|
|
284
|
-
},
|
|
285
|
-
]),
|
|
286
|
-
),
|
|
287
|
-
)
|
|
288
|
-
const logs = new Map<string, RpcLog>()
|
|
289
|
-
for (const page of pages)
|
|
290
|
-
for (const log of page)
|
|
291
|
-
logs.set(`${log.blockNumber}:${log.logIndex}:${log.transactionHash}`, log)
|
|
292
|
-
return [...logs.values()]
|
|
293
|
-
}
|
|
294
|
-
|
|
295
|
-
/** Sorts raw logs into ascending `(blockNumber, logIndex)` order. */
|
|
296
|
-
function byPosition(a: RpcLog, b: RpcLog): number {
|
|
297
|
-
const block = (Value.hexToNumber(a.blockNumber) ?? 0) - (Value.hexToNumber(b.blockNumber) ?? 0)
|
|
298
|
-
return block || (Value.hexToNumber(a.logIndex) ?? 0) - (Value.hexToNumber(b.logIndex) ?? 0)
|
|
299
|
-
}
|
|
300
|
-
|
|
301
26
|
/** True when an address is in Tempo's deterministic TIP-20 contract range. */
|
|
302
27
|
function isTip20Address(address: string): boolean {
|
|
303
28
|
const value = address.toLowerCase()
|
|
@@ -344,7 +69,7 @@ export function decode(log: RpcLog, timestamp: string): EventData | undefined {
|
|
|
344
69
|
}
|
|
345
70
|
}
|
|
346
71
|
|
|
347
|
-
/** Re-checks every transfer filter after
|
|
72
|
+
/** Re-checks every transfer filter after topic-level matching. */
|
|
348
73
|
export function matches(data: EventData, filters: Filters): boolean {
|
|
349
74
|
if (filters.address && data.sender !== filters.address && data.recipient !== filters.address)
|
|
350
75
|
return false
|
|
@@ -353,78 +78,3 @@ export function matches(data: EventData, filters: Filters): boolean {
|
|
|
353
78
|
if (filters.token && data.address !== filters.token) return false
|
|
354
79
|
return true
|
|
355
80
|
}
|
|
356
|
-
|
|
357
|
-
/** Adds a matched row until one subscription reaches its fair-page limit. */
|
|
358
|
-
function addMatch(accumulator: Accumulator, item: scan.Item): void {
|
|
359
|
-
if (accumulator.full) return
|
|
360
|
-
accumulator.items.push(item)
|
|
361
|
-
accumulator.full = accumulator.items.length >= accumulator.state.limit
|
|
362
|
-
}
|
|
363
|
-
|
|
364
|
-
/** Checks whether a canonical row remains eligible for one subscription. */
|
|
365
|
-
function canMatch(accumulator: Accumulator, blockNumber: number, logIndex: number): boolean {
|
|
366
|
-
const { skip, to } = accumulator.state.window
|
|
367
|
-
return (
|
|
368
|
-
!accumulator.full &&
|
|
369
|
-
blockNumber <= to &&
|
|
370
|
-
(blockNumber > skip[0] || (blockNumber === skip[0] && logIndex > skip[1]))
|
|
371
|
-
)
|
|
372
|
-
}
|
|
373
|
-
|
|
374
|
-
/** Converts match buffers into cursor-aware poller pages. */
|
|
375
|
-
function finish(accumulators: readonly Accumulator[]): scan.Page[] {
|
|
376
|
-
return accumulators.map((accumulator) => {
|
|
377
|
-
if (accumulator.full) return { hasMore: true, items: accumulator.items }
|
|
378
|
-
if (accumulator.items.length > 0) return { items: accumulator.items }
|
|
379
|
-
return { items: accumulator.items, scannedThrough: accumulator.state.window.to }
|
|
380
|
-
})
|
|
381
|
-
}
|
|
382
|
-
|
|
383
|
-
/** Scans one canonical transfer stream and fans rows into matching subscriptions. */
|
|
384
|
-
async function scanCohort(options: {
|
|
385
|
-
client: Viem.getClient.ReturnType
|
|
386
|
-
cohort: Cohort
|
|
387
|
-
}): Promise<scan.Page[]> {
|
|
388
|
-
const { client, cohort } = options
|
|
389
|
-
const accumulators = cohort.states.map<Accumulator>(({ state }) => ({
|
|
390
|
-
full: false,
|
|
391
|
-
items: [],
|
|
392
|
-
state,
|
|
393
|
-
}))
|
|
394
|
-
const timestamps = new Map<number, Promise<string | undefined>>()
|
|
395
|
-
const timestamp = (blockNumber: number) => {
|
|
396
|
-
const existing = timestamps.get(blockNumber)
|
|
397
|
-
if (existing) return existing
|
|
398
|
-
const pending = client
|
|
399
|
-
.getBlock({ blockNumber: BigInt(blockNumber), includeTransactions: false })
|
|
400
|
-
.then((block) => Value.blockToIso(block))
|
|
401
|
-
timestamps.set(blockNumber, pending)
|
|
402
|
-
return pending
|
|
403
|
-
}
|
|
404
|
-
const filters_source = cohort.states[0]!.state.filters as Filters
|
|
405
|
-
// RPC pushdowns bound selective responses; only the unfiltered firehose needs small range chunks.
|
|
406
|
-
const chunkBlocks = Object.values(filters_source).some(Boolean)
|
|
407
|
-
? cohort.window.to - cohort.window.from + 1
|
|
408
|
-
: scanChunkBlocks
|
|
409
|
-
|
|
410
|
-
for (let from = cohort.window.from; from <= cohort.window.to; from += chunkBlocks) {
|
|
411
|
-
const to = Math.min(cohort.window.to, from + chunkBlocks - 1)
|
|
412
|
-
const logs = (await getLogs(client, filters_source, { from, to })).sort(byPosition)
|
|
413
|
-
for (const log of logs) {
|
|
414
|
-
const blockNumber = Value.hexToNumber(log.blockNumber)
|
|
415
|
-
const logIndex = Value.hexToNumber(log.logIndex)
|
|
416
|
-
if (blockNumber === undefined || logIndex === undefined) continue
|
|
417
|
-
const blockTimestamp = await timestamp(blockNumber)
|
|
418
|
-
if (blockTimestamp === undefined) continue
|
|
419
|
-
const data = decode(log, blockTimestamp)
|
|
420
|
-
if (!data) continue
|
|
421
|
-
for (const accumulator of accumulators) {
|
|
422
|
-
if (!canMatch(accumulator, blockNumber, logIndex)) continue
|
|
423
|
-
if (matches(data, accumulator.state.filters as Filters))
|
|
424
|
-
addMatch(accumulator, { cursor: [blockNumber, logIndex], data })
|
|
425
|
-
}
|
|
426
|
-
if (accumulators.every((accumulator) => accumulator.full)) return finish(accumulators)
|
|
427
|
-
}
|
|
428
|
-
}
|
|
429
|
-
return finish(accumulators)
|
|
430
|
-
}
|
|
@@ -304,44 +304,6 @@ describe('updateSubscription', () => {
|
|
|
304
304
|
})
|
|
305
305
|
})
|
|
306
306
|
|
|
307
|
-
describe('listActive', () => {
|
|
308
|
-
test('returns only active subscriptions for a stream', async () => {
|
|
309
|
-
const db = TestApp.database()
|
|
310
|
-
const a = await Webhooks.createSubscription(db, createInput())
|
|
311
|
-
const b = await Webhooks.createSubscription(db, createInput({ owner: mppOwner }))
|
|
312
|
-
await Webhooks.createSubscription(db, createInput({ eventType: 'transaction:included' }))
|
|
313
|
-
|
|
314
|
-
let active = await Webhooks.listActive(db, 4217, 'token:transfer')
|
|
315
|
-
expect(active.map((s) => s.id).sort()).toEqual([a.id, b.id].sort())
|
|
316
|
-
|
|
317
|
-
await Webhooks.updateSubscription(db, apiKeyOwner, a.id, { status: 'paused' })
|
|
318
|
-
active = await Webhooks.listActive(db, 4217, 'token:transfer')
|
|
319
|
-
expect(active.map((s) => s.id)).toEqual([b.id])
|
|
320
|
-
})
|
|
321
|
-
|
|
322
|
-
test('drops a deleted subscription from the index', async () => {
|
|
323
|
-
const db = TestApp.database()
|
|
324
|
-
const a = await Webhooks.createSubscription(db, createInput())
|
|
325
|
-
|
|
326
|
-
await Webhooks.deleteSubscription(db, apiKeyOwner, a.id)
|
|
327
|
-
expect(await Webhooks.listActive(db, 4217, 'token:transfer')).toMatchInlineSnapshot(`[]`)
|
|
328
|
-
})
|
|
329
|
-
})
|
|
330
|
-
|
|
331
|
-
describe('listActiveWithCursors', () => {
|
|
332
|
-
test('pairs each subscription with its stored cursor from one read', async () => {
|
|
333
|
-
const db = TestApp.database()
|
|
334
|
-
const anchored = await Webhooks.createSubscription(db, createInput())
|
|
335
|
-
const fresh = await Webhooks.createSubscription(db, createInput({ owner: mppOwner }))
|
|
336
|
-
await Webhooks.setCursor(db, anchored.id, Cursor.encode([100, 5]))
|
|
337
|
-
|
|
338
|
-
const pairs = await Webhooks.listActiveWithCursors(db, 4217, 'token:transfer')
|
|
339
|
-
const byId = new Map(pairs.map((pair) => [pair.subscription.id, pair.cursor]))
|
|
340
|
-
expect(Cursor.decode(byId.get(anchored.id)!, ['int', 'int'])).toEqual([100, 5])
|
|
341
|
-
expect(byId.get(fresh.id)).toBeNull()
|
|
342
|
-
})
|
|
343
|
-
})
|
|
344
|
-
|
|
345
307
|
describe('listActiveForChain', () => {
|
|
346
308
|
test('returns active subscriptions across event types in one read', async () => {
|
|
347
309
|
const db = TestApp.database()
|
|
@@ -365,102 +327,6 @@ describe('listActiveForChain', () => {
|
|
|
365
327
|
})
|
|
366
328
|
})
|
|
367
329
|
|
|
368
|
-
describe('maxCursorBlock', () => {
|
|
369
|
-
test('returns the freshest cursor block across event types', async () => {
|
|
370
|
-
const db = TestApp.database()
|
|
371
|
-
const transfer = await Webhooks.createSubscription(db, createInput())
|
|
372
|
-
const tx = await Webhooks.createSubscription(
|
|
373
|
-
db,
|
|
374
|
-
createInput({ eventType: 'transaction:included' }),
|
|
375
|
-
)
|
|
376
|
-
await Webhooks.setCursor(db, transfer.id, Cursor.encode([100, 5]))
|
|
377
|
-
await Webhooks.setCursor(db, tx.id, Cursor.encode([250, 0]))
|
|
378
|
-
|
|
379
|
-
expect(await Webhooks.maxCursorBlock(db, 4217)).toBe(250)
|
|
380
|
-
})
|
|
381
|
-
|
|
382
|
-
test('counts creation checkpoints at their recorded head', async () => {
|
|
383
|
-
const db = TestApp.database()
|
|
384
|
-
await Webhooks.createSubscription(db, createInput(), { startBlockNumber: 900 })
|
|
385
|
-
|
|
386
|
-
expect(await Webhooks.maxCursorBlock(db, 4217)).toBe(900)
|
|
387
|
-
})
|
|
388
|
-
|
|
389
|
-
test('ignores other chains and returns null without cursors', async () => {
|
|
390
|
-
const db = TestApp.database()
|
|
391
|
-
expect(await Webhooks.maxCursorBlock(db, 4217)).toBeNull()
|
|
392
|
-
|
|
393
|
-
const other = await Webhooks.createSubscription(db, createInput({ chainId: 42431 }))
|
|
394
|
-
await Webhooks.setCursor(db, other.id, Cursor.encode([999, 0]))
|
|
395
|
-
// A cursorless subscription on the queried chain still reads as null.
|
|
396
|
-
await Webhooks.createSubscription(db, createInput())
|
|
397
|
-
|
|
398
|
-
expect(await Webhooks.maxCursorBlock(db, 4217)).toBeNull()
|
|
399
|
-
expect(await Webhooks.maxCursorBlock(db, 42431)).toBe(999)
|
|
400
|
-
})
|
|
401
|
-
})
|
|
402
|
-
|
|
403
|
-
describe('cursors', () => {
|
|
404
|
-
test('round-trips a per-subscription cursor', async () => {
|
|
405
|
-
const db = TestApp.database()
|
|
406
|
-
// The cursor is a column on the subscription row, so it needs a real
|
|
407
|
-
// subscription (the old key space allowed orphan cursors).
|
|
408
|
-
const created = await Webhooks.createSubscription(db, createInput())
|
|
409
|
-
expect(await Webhooks.getCursor(db, created.id)).toMatchInlineSnapshot(`null`)
|
|
410
|
-
|
|
411
|
-
await Webhooks.setCursor(db, created.id, '12345:7')
|
|
412
|
-
expect(await Webhooks.getCursor(db, created.id)).toMatchInlineSnapshot(`"12345:7"`)
|
|
413
|
-
})
|
|
414
|
-
|
|
415
|
-
test('atomically rejects a stale cursor write', async () => {
|
|
416
|
-
const db = TestApp.database()
|
|
417
|
-
const created = await Webhooks.createSubscription(db, createInput())
|
|
418
|
-
const a = Cursor.encode([100, 1])
|
|
419
|
-
const b = Cursor.encode([100, 2])
|
|
420
|
-
const results = await Promise.all([
|
|
421
|
-
Webhooks.compareAndSetCursor(db, {
|
|
422
|
-
cursor: a,
|
|
423
|
-
expectedCursor: null,
|
|
424
|
-
subscriptionId: created.id,
|
|
425
|
-
}),
|
|
426
|
-
Webhooks.compareAndSetCursor(db, {
|
|
427
|
-
cursor: b,
|
|
428
|
-
expectedCursor: null,
|
|
429
|
-
subscriptionId: created.id,
|
|
430
|
-
}),
|
|
431
|
-
])
|
|
432
|
-
|
|
433
|
-
expect(results.toSorted((a, b) => Number(a) - Number(b))).toEqual([false, true])
|
|
434
|
-
expect([a, b]).toContain(await Webhooks.getCursor(db, created.id))
|
|
435
|
-
})
|
|
436
|
-
})
|
|
437
|
-
|
|
438
|
-
describe('compareAndSetCursors', () => {
|
|
439
|
-
test('atomically advances a batch and reports stale rows', async () => {
|
|
440
|
-
const db = TestApp.database()
|
|
441
|
-
const initialized = await Webhooks.createSubscription(db, createInput())
|
|
442
|
-
const stale = await Webhooks.createSubscription(db, createInput())
|
|
443
|
-
const uninitialized = await Webhooks.createSubscription(db, createInput())
|
|
444
|
-
const initial = Cursor.encode([100, 1])
|
|
445
|
-
const next = Cursor.encode([101, 1])
|
|
446
|
-
await Webhooks.setCursor(db, initialized.id, initial)
|
|
447
|
-
await Webhooks.setCursor(db, stale.id, initial)
|
|
448
|
-
|
|
449
|
-
const committed = await Webhooks.compareAndSetCursors(db, {
|
|
450
|
-
cursors: [
|
|
451
|
-
{ cursor: next, expectedCursor: initial, subscriptionId: initialized.id },
|
|
452
|
-
{ cursor: next, expectedCursor: null, subscriptionId: stale.id },
|
|
453
|
-
{ cursor: next, expectedCursor: null, subscriptionId: uninitialized.id },
|
|
454
|
-
],
|
|
455
|
-
})
|
|
456
|
-
|
|
457
|
-
expect([...committed].toSorted()).toEqual([initialized.id, uninitialized.id].toSorted())
|
|
458
|
-
expect(await Webhooks.getCursor(db, initialized.id)).toBe(next)
|
|
459
|
-
expect(await Webhooks.getCursor(db, stale.id)).toBe(initial)
|
|
460
|
-
expect(await Webhooks.getCursor(db, uninitialized.id)).toBe(next)
|
|
461
|
-
})
|
|
462
|
-
})
|
|
463
|
-
|
|
464
330
|
describe('recordSuccess / recordFailure', () => {
|
|
465
331
|
test('recordSuccess resets failures and stamps lastDeliveryAt', async () => {
|
|
466
332
|
const db = TestApp.database()
|
|
@@ -490,11 +356,13 @@ describe('recordSuccess / recordFailure', () => {
|
|
|
490
356
|
subscription = await Webhooks.recordFailure(db, subscription, { maxFailures: 3 })
|
|
491
357
|
subscription = await Webhooks.recordFailure(db, subscription, { maxFailures: 3 })
|
|
492
358
|
expect(subscription.status).toMatchInlineSnapshot(`"active"`)
|
|
493
|
-
expect(
|
|
494
|
-
|
|
495
|
-
|
|
359
|
+
expect(
|
|
360
|
+
(
|
|
361
|
+
await Webhooks.listActiveForChain(db, { chainId: 4217, eventTypes: ['token:transfer'] })
|
|
362
|
+
).map((pair) => pair.subscription.id),
|
|
363
|
+
).toEqual([subscription.id])
|
|
496
364
|
|
|
497
|
-
// Hitting the threshold disables it and removes it from the
|
|
365
|
+
// Hitting the threshold disables it and removes it from the active index.
|
|
498
366
|
subscription = await Webhooks.recordFailure(db, subscription, { maxFailures: 3 })
|
|
499
367
|
expect({ failureCount: subscription.failureCount, status: subscription.status })
|
|
500
368
|
.toMatchInlineSnapshot(`
|
|
@@ -503,7 +371,9 @@ describe('recordSuccess / recordFailure', () => {
|
|
|
503
371
|
"status": "disabled",
|
|
504
372
|
}
|
|
505
373
|
`)
|
|
506
|
-
expect(
|
|
374
|
+
expect(
|
|
375
|
+
await Webhooks.listActiveForChain(db, { chainId: 4217, eventTypes: ['token:transfer'] }),
|
|
376
|
+
).toMatchInlineSnapshot(`[]`)
|
|
507
377
|
})
|
|
508
378
|
})
|
|
509
379
|
|
|
@@ -1014,6 +884,72 @@ describe('ensureQueueEvents', () => {
|
|
|
1014
884
|
expect(histograms.has('webhook_delivery_subscription_read_ms')).toBe(true)
|
|
1015
885
|
})
|
|
1016
886
|
|
|
887
|
+
test('measures observation to attempt from the staged head stamp', async () => {
|
|
888
|
+
const db = TestApp.database()
|
|
889
|
+
const subscription = await Webhooks.createSubscription(
|
|
890
|
+
db,
|
|
891
|
+
createInput({ eventType: 'transaction:included' }),
|
|
892
|
+
)
|
|
893
|
+
const envelope = Webhooks.buildEnvelope({
|
|
894
|
+
blockNumber: 123,
|
|
895
|
+
data: {},
|
|
896
|
+
logIndex: 4,
|
|
897
|
+
subscription,
|
|
898
|
+
})
|
|
899
|
+
const observedAt = Date.now() - 5_000
|
|
900
|
+
const { references: [reference] } = await Webhooks.ensureQueueEvents(db, [{ envelope, subscription }], { observedAt }) // prettier-ignore
|
|
901
|
+
const histograms = new Map<string, number>()
|
|
902
|
+
const metrics = Metrics.from({
|
|
903
|
+
count() {},
|
|
904
|
+
flush() {},
|
|
905
|
+
gauge() {},
|
|
906
|
+
histogram(name, value) {
|
|
907
|
+
histograms.set(name, value)
|
|
908
|
+
},
|
|
909
|
+
})
|
|
910
|
+
const fetch = (async () => new Response(null, { status: 200 })) satisfies typeof globalThis.fetch // prettier-ignore
|
|
911
|
+
|
|
912
|
+
const result = await Webhooks.deliverQueueEventAndRecord(db, reference!, { fetch, metrics })
|
|
913
|
+
|
|
914
|
+
expect(result.status).toBe('delivered')
|
|
915
|
+
// Measured from the head stamp, so it covers the whole controllable span
|
|
916
|
+
// rather than only the time since this worker dequeued the message.
|
|
917
|
+
expect(histograms.get('webhook_observed_to_attempt_ms')).toBeGreaterThanOrEqual(5_000)
|
|
918
|
+
expect(histograms.has('webhook_delivery_preflight_ms')).toBe(true)
|
|
919
|
+
expect(histograms.has('webhook_delivery_settle_ms')).toBe(true)
|
|
920
|
+
})
|
|
921
|
+
|
|
922
|
+
test('omits observation timing for a replay with no staged head stamp', async () => {
|
|
923
|
+
const db = TestApp.database()
|
|
924
|
+
const subscription = await Webhooks.createSubscription(
|
|
925
|
+
db,
|
|
926
|
+
createInput({ eventType: 'transaction:included' }),
|
|
927
|
+
)
|
|
928
|
+
const envelope = Webhooks.buildEnvelope({
|
|
929
|
+
blockNumber: 124,
|
|
930
|
+
data: {},
|
|
931
|
+
logIndex: 4,
|
|
932
|
+
subscription,
|
|
933
|
+
})
|
|
934
|
+
const { references: [reference] } = await Webhooks.ensureQueueEvents(db, [{ envelope, subscription }]) // prettier-ignore
|
|
935
|
+
const histograms = new Map<string, number>()
|
|
936
|
+
const metrics = Metrics.from({
|
|
937
|
+
count() {},
|
|
938
|
+
flush() {},
|
|
939
|
+
gauge() {},
|
|
940
|
+
histogram(name, value) {
|
|
941
|
+
histograms.set(name, value)
|
|
942
|
+
},
|
|
943
|
+
})
|
|
944
|
+
const fetch = (async () => new Response(null, { status: 200 })) satisfies typeof globalThis.fetch // prettier-ignore
|
|
945
|
+
|
|
946
|
+
await Webhooks.deliverQueueEventAndRecord(db, reference!, { fetch, metrics })
|
|
947
|
+
|
|
948
|
+
// Absent rather than zero: a replay is unattributable, not instant.
|
|
949
|
+
expect(histograms.has('webhook_observed_to_attempt_ms')).toBe(false)
|
|
950
|
+
expect(histograms.has('webhook_event_to_attempt_ms')).toBe(true)
|
|
951
|
+
})
|
|
952
|
+
|
|
1017
953
|
test('skips a staged event after its subscription expires', async () => {
|
|
1018
954
|
vi.useFakeTimers({ now: fixedNow(), toFake: ['Date'] })
|
|
1019
955
|
try {
|
|
@@ -1107,6 +1043,33 @@ describe('ensureQueueEvents', () => {
|
|
|
1107
1043
|
vi.useRealTimers()
|
|
1108
1044
|
}
|
|
1109
1045
|
})
|
|
1046
|
+
|
|
1047
|
+
test('refreshes the observation stamp when a replay replaces an expired row', async () => {
|
|
1048
|
+
vi.useFakeTimers({ toFake: ['Date'] })
|
|
1049
|
+
vi.setSystemTime(fixedNow())
|
|
1050
|
+
try {
|
|
1051
|
+
const db = TestApp.database()
|
|
1052
|
+
const { envelope, reference, subscription } = await stagedObligation(db)
|
|
1053
|
+
const claim = await Webhooks.claimQueueEvent(db, reference)
|
|
1054
|
+
if (claim.type !== 'claimed') throw new Error(`Unexpected claim: ${claim.type}`)
|
|
1055
|
+
await Webhooks.completeQueueEvent(db, reference, {
|
|
1056
|
+
claimedAt: claim.record.attemptingAt!,
|
|
1057
|
+
status: 'succeeded',
|
|
1058
|
+
})
|
|
1059
|
+
|
|
1060
|
+
vi.setSystemTime(new Date(fixedNow().getTime() + Webhooks.deliveryRetentionMs + 1_000))
|
|
1061
|
+
const observedAt = Date.now() - 250
|
|
1062
|
+
await Webhooks.ensureQueueEvents(db, [{ envelope, subscription }], { observedAt })
|
|
1063
|
+
|
|
1064
|
+
// Carrying the replaced row's stamp would measure this delivery against
|
|
1065
|
+
// an observation from the retention window before it.
|
|
1066
|
+
const replayed = await Webhooks.claimQueueEvent(db, reference)
|
|
1067
|
+
if (replayed.type !== 'claimed') throw new Error(`Unexpected claim: ${replayed.type}`)
|
|
1068
|
+
expect(replayed.record.observedAt).toBe(new Date(observedAt).toISOString())
|
|
1069
|
+
} finally {
|
|
1070
|
+
vi.useRealTimers()
|
|
1071
|
+
}
|
|
1072
|
+
})
|
|
1110
1073
|
})
|
|
1111
1074
|
|
|
1112
1075
|
describe('claimQueueEvent', () => {
|