tapimo 0.9.0 → 0.10.1

package/src/internal/MetricSink.ts

@@ -1,6 +1,7 @@
 import type { Metrics } from '../Metrics.js'
 import type * as Webhooks from '../Webhooks.js'
 import type * as Log from './Log.js'
+import * as WebhookDestination from './WebhookDestination.js'
 
 /** Webhook event type, including the synthetic `ping` carried on test envelopes. */
 export type EventType = Webhooks.Envelope['type']
@@ -8,6 +9,12 @@ export type EventType = Webhooks.Envelope['type']
 /** Where a delivery attempt originated. */
 export type Trigger = 'manual_retry' | 'ping' | 'queue'
 
+/** Bounded poller stage names used by `webhook:poll`. */
+export type PollStage = 'cursor_write' | 'enrich' | 'rpc_head' | 'scan' | 'subscription_read'
+
+/** Bounded Durable Object or transport failure classes used by `webhook:poll-failure`. */
+export type PollFailure = 'other' | 'overloaded' | 'remote' | 'retryable'
+
 /** Context for one poller tick that the {@link Webhooks.createPoller.Result} omits. */
 export type TickMeta = {
   /** Chain polled this tick; used when the tick errored before a result existed. */
@@ -20,11 +27,127 @@ export type TickMeta = {
 
 /**
  * A webhook metric event, recorded by the {@link webhooks} sink. The poller
- * emits `webhook:tick`/`webhook:enqueue` and the delivery path emits
- * `webhook:delivery`.
+ * emits head recovery, poll, tick, and enqueue measurements; the delivery path
+ * emits `webhook:delivery`.
  */
 export type WebhookEvent =
   | {
+      /** Tempo chain whose poll cycle completed. */
+      chainId: number
+      /** Whether the poll cycle completed without scan or dispatch failures. */
+      outcome: 'error' | 'ok'
+      /** Requested boundary movement since the preceding successful poll, in blocks. */
+      requestedBoundaryAdvanceBlocks?: number | undefined
+      /** Execution path that acquired this poll claim. */
+      trigger: 'alarm' | 'cron' | 'direct' | 'queue'
+      /** Identifies a transport-neutral poll-cycle measurement. */
+      type: 'webhook:poll-cycle'
+    }
+  | {
+      /** Tempo chain whose durable progress was read. */
+      chainId: number
+      /** Blocks between the desired and fully drained boundaries. */
+      lagBlocks: number
+      /** Identifies durable coordinator progress. */
+      type: 'webhook:head-progress'
+    }
+  | {
+      /** Tempo chain whose listener was checked. */
+      chainId: number
+      /** Heads refused in a row by the monotonic guard; zero on a delivering listener. */
+      skippedHeads: number
+      /** Identifies heads received and discarded rather than heads not received. */
+      type: 'webhook:head-skipped'
+    }
+  | {
+      chainId: number
+      /** Blocks the probed chain head leads the listener's newest observation. */
+      lagBlocks: number
+      /** Identifies the out-of-band listener head probe. */
+      type: 'webhook:listener-head-lag'
+    }
+  | {
+      /** Tempo chain whose recovery poke was dequeued. */
+      chainId: number
+      /** When the recovery poke was dequeued, as epoch milliseconds. */
+      dequeuedAt: number
+      /** When the recovery poke was enqueued, as epoch milliseconds. */
+      enqueuedAt: number
+      /** Recovery source carried through the Queue handoff. */
+      trigger: 'alarm' | 'queue'
+      /** Identifies recovery Queue latency, not observed head volume. */
+      type: 'webhook:head-recovery'
+    }
+  | {
+      /** Tempo chain whose poll claim was acquired. */
+      chainId: number
+      /** Delay from observing the desired boundary to acquiring the claim. */
+      delayMs?: number | undefined
+      /** Execution path that acquired the claim. */
+      trigger: 'alarm' | 'cron' | 'direct' | 'queue'
+      /** Identifies a poll beginning, before scan work runs. */
+      type: 'webhook:poll-start'
+    }
+  | {
+      /** Tempo chain whose poll failed. */
+      chainId: number
+      /** Bounded class of the thrown failure. */
+      failure: PollFailure
+      /** Operation boundary that rejected the poll. */
+      stage: 'claim' | 'continuation_enqueue' | 'finish' | 'scan'
+      /** Execution path that failed before recovery. */
+      trigger: 'alarm' | 'cron' | 'direct' | 'queue'
+      /** Identifies a transport-neutral poll failure. */
+      type: 'webhook:poll-failure'
+    }
+  | {
+      chainId: number
+      durationMs: number
+      eventType?: EventType | undefined
+      outcome: 'error' | 'ok'
+      stage: PollStage
+      type: 'webhook:poll'
+    }
+  | {
+      /** Tempo chain whose eligible subscription count was read. */
+      chainId: number
+      /** Active, unexpired subscriptions selected for the poll. */
+      count: number
+      /** Identifies a pre-scan subscription eligibility measurement. */
+      type: 'webhook:poll-subscriptions'
+    }
+  | {
+      /** Chain scanned by this poll batch. */
+      chainId: number
+      /** Shared upper bound for concurrent upstream scans. */
+      concurrency: number
+      /** Scan duration, present only after completion. */
+      durationMs?: number | undefined
+      /** Subscription event type scanned by this batch. */
+      eventType: EventType
+      /** Subscriptions whose scan rejected, present only after completion. */
+      failedSubscriptions?: number | undefined
+      /** Matching rows retained by completed pages, present only after completion. */
+      matchedItems?: number | undefined
+      /** Largest block window requested by one subscription. */
+      maxWindowBlocks: number
+      /** Whether the checkpoint precedes allocation or follows scan completion. */
+      phase: 'complete' | 'start'
+      /** Outcome of the completed RPC-backed scan. */
+      outcome?: 'empty' | 'error' | 'missing' | 'ok' | undefined
+      /** JSON-RPC method used by this event type's scanner. */
+      rpcMethod: 'eth_getBlockByNumber' | 'eth_getLogs'
+      /** Subscriptions retained in this scan batch. */
+      subscriptionCount: number
+      /** Maximum subscriptions retained by this event-type batch. */
+      subscriptionLimit: number
+      /** Sum of subscription block windows in this scan batch. */
+      totalWindowBlocks: number
+      /** Identifies a memory-attribution scan boundary. */
+      type: 'webhook:poll-scan'
+    }
+  | {
+      cursorLagBlocks?: number | undefined
       meta: TickMeta
       result: Webhooks.createPoller.Result | undefined
       type: 'webhook:tick'
@@ -37,9 +160,37 @@ export type WebhookEvent =
       type: 'webhook:enqueue'
     }
   | {
+      /** Atomic ledger claim outcome. */
+      outcome: 'attempting' | 'claimed' | 'missing' | 'scheduled' | 'terminal'
+      /** Identifies a delivery-obligation claim. */
+      type: 'webhook:delivery-job-claim'
+    }
+  | {
+      /** Obligations represented by this outcome. */
+      count: number
+      /** Whether staging created or found the obligation. */
+      outcome: 'created' | 'existing'
+      /** Identifies obligation staging ahead of Queue handoff. */
+      type: 'webhook:delivery-job-ensure'
+    }
+  | {
+      /** Delivery attempt ordinal: the ledger claim count on the queue path. */
+      attempt: number
       envelope: Webhooks.Envelope
+      /** Cloudflare Queue delivery attempt, absent outside the Queue path. */
+      queueAttempt?: number | undefined
       result: Webhooks.Result
       subscription: Webhooks.Subscription
+      timings?: {
+        deliveryLogMs?: number | undefined
+        envelopeToQueueMs?: number | undefined
+        endToEndMs?: number | undefined
+        eventToEnvelopeMs?: number | undefined
+        processingMs?: number | undefined
+        queueWaitMs?: number | undefined
+        stateWriteMs?: number | undefined
+        subscriptionReadMs?: number | undefined
+      }
       trigger: Trigger
       type: 'webhook:delivery'
     }
@@ -195,7 +346,7 @@ type StatusClass = '2xx' | '4xx' | '5xx' | 'network' | 'timeout' | 'unknown' | '
 /**
  * Builds a webhook delivery-reliability sink over `backend`. Tags stay
  * low-cardinality (`chain_id`, `event_type`, `destination_type`, `status_class`,
- * `attempt_bucket`, `outcome`) — never subscription/org/event ids, which belong
+ * `attempt_bucket`, `failure`, `queue_attempt_bucket`, `outcome`, `stage`) — never subscription/org/event ids, which belong
  * in logs or analytics, not StatsD-style series.
  */
 export function webhooks(backend: Metrics): WebhookSink {
@@ -205,6 +356,111 @@ export function webhooks(backend: Metrics): WebhookSink {
     },
     record(event) {
       switch (event.type) {
+        case 'webhook:head-recovery': {
+          const tags = { chain_id: String(event.chainId), trigger: event.trigger }
+          backend.count('webhook_head_recovery_queue_count', 1, tags)
+          backend.histogram(
+            'webhook_head_recovery_queue_wait_ms',
+            Math.max(0, event.dequeuedAt - event.enqueuedAt),
+            tags,
+          )
+          return
+        }
+        case 'webhook:head-progress':
+          backend.gauge('webhook_head_progress_lag_blocks', event.lagBlocks, {
+            chain_id: String(event.chainId),
+          })
+          return
+        case 'webhook:head-skipped':
+          backend.gauge('webhook_head_skipped_count', event.skippedHeads, {
+            chain_id: String(event.chainId),
+          })
+          return
+        case 'webhook:listener-head-lag':
+          backend.gauge('webhook_listener_head_lag_blocks', event.lagBlocks, {
+            chain_id: String(event.chainId),
+          })
+          return
+        case 'webhook:poll-failure':
+          backend.count('webhook_poll_failure_count', 1, {
+            chain_id: String(event.chainId),
+            failure: event.failure,
+            stage: event.stage,
+            trigger: event.trigger,
+          })
+          return
+        case 'webhook:poll': {
+          const tags = {
+            chain_id: String(event.chainId),
+            ...(event.eventType === undefined ? {} : { event_type: event.eventType }),
+            outcome: event.outcome,
+            stage: event.stage,
+          }
+          backend.histogram('webhook_poller_stage_duration_ms', event.durationMs, tags)
+          return
+        }
+        case 'webhook:poll-scan': {
+          const tags = {
+            chain_id: String(event.chainId),
+            event_type: event.eventType,
+            phase: event.phase,
+          }
+          backend.count('webhook_poller_scan_checkpoint_count', 1, tags)
+          backend.histogram('webhook_poller_scan_concurrency', event.concurrency, tags)
+          backend.histogram('webhook_poller_scan_subscription_count', event.subscriptionCount, tags)
+          backend.histogram('webhook_poller_scan_subscription_limit', event.subscriptionLimit, tags)
+          backend.histogram('webhook_poller_scan_max_window_blocks', event.maxWindowBlocks, tags)
+          backend.histogram(
+            'webhook_poller_scan_total_window_blocks',
+            event.totalWindowBlocks,
+            tags,
+          )
+          if (event.durationMs !== undefined)
+            backend.histogram('webhook_poller_scan_duration_ms', event.durationMs, tags)
+          if (event.failedSubscriptions !== undefined)
+            backend.count(
+              'webhook_poller_scan_failed_subscription_count',
+              event.failedSubscriptions,
+              tags,
+            )
+          if (event.matchedItems !== undefined)
+            backend.count('webhook_poller_scan_matched_item_count', event.matchedItems, tags)
+          if (event.outcome !== undefined)
+            backend.count('webhook_poller_rpc_scan_count', 1, {
+              chain_id: String(event.chainId),
+              event_type: event.eventType,
+              outcome: event.outcome,
+              rpc_method: event.rpcMethod,
+            })
+          return
+        }
+        case 'webhook:poll-subscriptions':
+          backend.gauge('webhook_poller_active_subscription_count', event.count, {
+            chain_id: String(event.chainId),
+          })
+          return
+        case 'webhook:poll-cycle': {
+          const tags = {
+            chain_id: String(event.chainId),
+            outcome: event.outcome,
+            trigger: event.trigger,
+          }
+          backend.count('webhook_poll_cycle_count', 1, tags)
+          if (event.requestedBoundaryAdvanceBlocks !== undefined)
+            backend.histogram(
+              'webhook_poll_requested_boundary_advance_blocks',
+              event.requestedBoundaryAdvanceBlocks,
+              tags,
+            )
+          return
+        }
+        case 'webhook:poll-start': {
+          const tags = { chain_id: String(event.chainId), trigger: event.trigger }
+          backend.count('webhook_poll_start_count', 1, tags)
+          if (event.delayMs !== undefined)
+            backend.histogram('webhook_poll_start_delay_ms', event.delayMs, tags)
+          return
+        }
         case 'webhook:tick': {
           const { meta, result } = event
           const chain_id = String(result?.chainId ?? meta.chainId ?? 'unknown')
@@ -213,6 +469,10 @@ export function webhooks(backend: Metrics): WebhookSink {
             chain_id,
             outcome: meta.outcome,
           })
+          if (event.cursorLagBlocks !== undefined)
+            backend.gauge('webhook_subscription_cursor_lag_blocks', event.cursorLagBlocks, {
+              chain_id,
+            })
           if (!result) return
           backend.histogram('webhook_poller_subscription_count', result.subscriptions, { chain_id })
           backend.count('webhook_poller_scanned_count', result.scanned, { chain_id })
@@ -237,8 +497,29 @@ export function webhooks(backend: Metrics): WebhookSink {
           })
           return
         }
+        case 'webhook:delivery-job-claim': {
+          backend.count('webhook_delivery_job_claim_count', 1, { outcome: event.outcome })
+          return
+        }
+        case 'webhook:delivery-job-ensure': {
+          backend.count('webhook_delivery_job_ensure_count', event.count, {
+            outcome: event.outcome,
+          })
+          return
+        }
         case 'webhook:delivery': {
-          const { envelope, result, subscription } = event
+          const { envelope, result, subscription, timings } = event
+          const queueTags =
+            event.queueAttempt === undefined
+              ? {}
+              : {
+                  queue_attempt_bucket: attemptBucket(event.queueAttempt),
+                  // Retryable results ack the message and schedule a ledger
+                  // retry; the Queue transport never redelivers them.
+                  queue_disposition: WebhookDestination.isRetryable(result)
+                    ? 'scheduled_retry'
+                    : 'acked',
+                }
           const tags = {
             chain_id: String(envelope.chainId),
             destination_type: subscription.destination.type satisfies DestinationType,
@@ -247,8 +528,9 @@ export function webhooks(backend: Metrics): WebhookSink {
           const outcome = result.ok ? 'succeeded' : 'failed'
           backend.count('webhook_delivery_attempt_count', 1, {
             ...tags,
-            attempt_bucket: attemptBucket(subscription.failureCount + 1),
+            attempt_bucket: attemptBucket(event.attempt),
             outcome,
+            ...queueTags,
             status_class: statusClass(result),
             trigger: event.trigger,
           })
@@ -257,11 +539,44 @@ export function webhooks(backend: Metrics): WebhookSink {
               ...tags,
               outcome,
             })
-          backend.histogram(
-            'webhook_delivery_queue_latency_ms',
-            Math.max(0, Date.now() - Date.parse(envelope.createdAt)),
-            tags,
-          )
+          if (timings?.deliveryLogMs !== undefined)
+            backend.histogram('webhook_delivery_log_write_ms', timings.deliveryLogMs, tags)
+          if (timings?.envelopeToQueueMs !== undefined)
+            backend.histogram(
+              'webhook_delivery_envelope_to_queue_ms',
+              timings.envelopeToQueueMs,
+              tags,
+            )
+          if (timings?.endToEndMs !== undefined)
+            backend.histogram('webhook_delivery_end_to_end_ms', timings.endToEndMs, {
+              ...tags,
+              // Unlimited-horizon ledger retries emit hour-plus samples;
+              // the bucket keeps first-attempt percentiles interpretable.
+              attempt_bucket: attemptBucket(event.attempt),
+              outcome,
+            })
+          if (timings?.eventToEnvelopeMs !== undefined)
+            backend.histogram(
+              'webhook_delivery_event_to_envelope_ms',
+              timings.eventToEnvelopeMs,
+              tags,
+            )
+          if (timings?.processingMs !== undefined)
+            backend.histogram('webhook_delivery_processing_ms', timings.processingMs, tags)
+          if (timings?.queueWaitMs !== undefined)
+            backend.histogram('webhook_delivery_queue_wait_ms', timings.queueWaitMs, {
+              ...tags,
+              outcome,
+              ...queueTags,
+            })
+          if (timings?.stateWriteMs !== undefined)
+            backend.histogram('webhook_delivery_state_write_ms', timings.stateWriteMs, tags)
+          if (timings?.subscriptionReadMs !== undefined)
+            backend.histogram(
+              'webhook_delivery_subscription_read_ms',
+              timings.subscriptionReadMs,
+              tags,
+            )
           return
         }
       }

package/src/internal/RateLimit.test.ts

@@ -1,13 +1,14 @@
 import { RateLimit, Store } from 'tapimo'
 
 describe('memory', () => {
-  test('tracks fixed-window quota in a store', async () => {
+  test('tracks fixed-minute quota in a store', async () => {
     const store = RateLimit.memory({
       now: () => new Date('2026-05-25T00:00:30.000Z'),
     })
-    const first = await store.consume({ key: 'public:anonymous', limit: { perMinute: 2 } })
-    const second = await store.consume({ key: 'public:anonymous', limit: { perMinute: 2 } })
-    const third = await store.consume({ key: 'public:anonymous', limit: { perMinute: 2 } })
+    const limit = { limit: 2, period: 'minute' } as const
+    const first = await store.consume({ key: 'public:anonymous', limit })
+    const second = await store.consume({ key: 'public:anonymous', limit })
+    const third = await store.consume({ key: 'public:anonymous', limit })
 
     expect(first).toMatchInlineSnapshot(`
       {
@@ -35,25 +36,26 @@ describe('memory', () => {
     `)
   })
 
-  test('restarts the count when the next window opens', async () => {
-    // The window reset rides on counter expiry (the key is stable, not
-    // bucket-suffixed), so the store clock and the consume clock must agree.
+  test('restarts the count when the next second opens', async () => {
+    // The window reset rides on counter expiry, so the store clock and the
+    // consume clock must agree.
     vi.useFakeTimers()
     try {
-      vi.setSystemTime(new Date('2026-05-25T00:00:30.000Z'))
+      vi.setSystemTime(new Date('2026-05-25T00:00:30.250Z'))
       const store = RateLimit.memory()
-      await store.consume({ key: 'public:anonymous', limit: { perMinute: 2 } })
-      await store.consume({ key: 'public:anonymous', limit: { perMinute: 2 } })
+      const limit = { limit: 2, period: 'second' } as const
+      await store.consume({ key: 'public:anonymous', limit })
+      await store.consume({ key: 'public:anonymous', limit })
 
-      vi.setSystemTime(new Date('2026-05-25T00:01:00.001Z'))
-      const next = await store.consume({ key: 'public:anonymous', limit: { perMinute: 2 } })
+      vi.setSystemTime(new Date('2026-05-25T00:00:31.001Z'))
+      const next = await store.consume({ key: 'public:anonymous', limit })
 
       expect(next).toMatchInlineSnapshot(`
         {
           "allowed": true,
           "limit": 2,
           "remaining": 1,
-          "reset": 1779667320,
+          "reset": 1779667232,
         }
       `)
     } finally {
@@ -72,19 +74,22 @@ describe('memory', () => {
       },
     })
     const store = RateLimit.memory({
-      // 30s into the minute, so 30s of the window remains.
-      now: () => new Date('2026-05-25T00:00:30.000Z'),
+      // 250ms into the second, so 750ms of the window remains.
+      now: () => new Date('2026-05-25T00:00:30.250Z'),
       store: recording,
     })
 
-    await store.consume({ key: 'public:anonymous', limit: { perMinute: 2 } })
+    await store.consume({
+      key: 'public:anonymous',
+      limit: { limit: 2, period: 'second' },
+    })
 
     expect(increments).toMatchInlineSnapshot(`
       [
         {
-          "key": "ratelimit:public:anonymous",
+          "key": "ratelimit:second:1779667230:public:anonymous",
           "options": {
-            "ttl": 30000,
+            "ttl": 750,
           },
         },
       ]

package/src/internal/RateLimit.ts

@@ -6,9 +6,15 @@ import * as Store from './Store.js'
 export namespace schema {
   /** A rate-limit policy; the source of truth for {@link Limit}. */
   export const Limit = z.object({
-    perMinute: z
+    limit: z
       .number()
-      .check(z.int(), z.nonnegative(), z.describe('Number of requests allowed per minute.')),
+      .check(z.int(), z.nonnegative(), z.describe('Number of requests allowed per period.')),
+    period: z
+      .union([z.literal('minute'), z.literal('second')])
+      .check(
+        z.describe('Fixed window used to enforce the request limit.'),
+        z.meta({ examples: ['minute'] }),
+      ),
   })
 }
 
@@ -49,11 +55,8 @@ export namespace RateLimitStore {
  * Each consume is a single `Store.increment`, so a backend with a native
  * counter (e.g. a Durable Object) is one atomic round trip; backends without
  * one fall back to get + put, acceptable for dev/in-process stores only.
- * The storage key is stable per quota key — the
- * window lives in the TTL: the counter expires exactly at the end of the
- * current minute, so the next window's first hit restarts at 1 and idle keys
- * hold no storage. The sub-minute TTL rules out Cloudflare KV as a backend,
- * which is unsuitable for enforced counters anyway.
+ * The storage key includes the fixed-window bucket, so backend latency cannot
+ * carry a previous count into a later window.
  */
 export function memory(options: memory.Options = {}): RateLimitStore {
   const store = options.store ?? Store.memory()
@@ -62,19 +65,24 @@ export function memory(options: memory.Options = {}): RateLimitStore {
   return {
     async consume(options) {
       const time = now().getTime()
-      const bucket = Math.floor(time / 60_000)
-      // Every increment within a window recomputes the TTL against the same
-      // window end, so the counter's absolute expiry never moves under load.
-      const count = await Store.increment(store, `ratelimit:${options.key}`, {
-        ttl: (bucket + 1) * 60_000 - time,
-      })
-      const limit = options.limit.perMinute
+      const window = options.limit.period === 'second' ? 1_000 : 60_000
+      const bucket = Math.floor(time / window)
+      // The bucket isolates window counts if the backing store applies the TTL
+      // after the next window starts.
+      const count = await Store.increment(
+        store,
+        `ratelimit:${options.limit.period}:${bucket}:${options.key}`,
+        {
+          ttl: (bucket + 1) * window - time,
+        },
+      )
+      const limit = options.limit.limit
 
       return {
         allowed: count <= limit,
         limit,
         remaining: Math.max(limit - count, 0),
-        reset: Math.ceil(((bucket + 1) * 60_000) / 1_000),
+        reset: Math.ceil(((bucket + 1) * window) / 1_000),
       }
     },
   }

package/src/internal/Schema.ts

@@ -356,7 +356,7 @@ export const Denomination = z
   .check(
     z.describe(
       'Currency to denominate values in (case-insensitive, e.g. `AUD`). ' +
-        'Must be priced by the configured FX oracle. Defaults to `USD`.',
+        'Must be priced by the configured FX oracle.',
     ),
     z.meta({ examples: ['AUD'] }),
   )