tapimo 0.10.2 → 0.12.0

package/src/internal/MetricSink.ts

@@ -9,48 +9,22 @@ export type EventType = Webhooks.Envelope['type']
 /** Where a delivery attempt originated. */
 export type Trigger = 'manual_retry' | 'ping' | 'queue'
 
-/** Bounded poller stage names used by `webhook:poll`. */
-export type PollStage = 'cursor_write' | 'enrich' | 'rpc_head' | 'scan' | 'subscription_read'
-
-/** Bounded Durable Object or transport failure classes used by `webhook:poll-failure`. */
-export type PollFailure = 'other' | 'overloaded' | 'remote' | 'retryable'
-
-/** Context for one poller tick that the {@link Webhooks.createPoller.Result} omits. */
-export type TickMeta = {
-  /** Chain polled this tick; used when the tick errored before a result existed. */
-  chainId?: number | undefined
-  /** Wall-clock duration of the tick in ms. */
-  durationMs: number
-  /** Whether the tick completed. */
-  outcome: 'error' | 'ok'
-}
+/** Bounded step names inside one block-fanout scan, used by `webhook:scan-step`. */
+export type ScanStep =
+  | 'block_rpc'
+  | 'enrich'
+  | 'finalized_rpc'
+  | 'logs_rpc'
+  | 'match'
+  | 'stage'
+  | 'subscription_read'
 
 /**
- * A webhook metric event, recorded by the {@link webhooks} sink. The poller
- * emits head recovery, poll, tick, and enqueue measurements; the delivery path
+ * A webhook metric event, recorded by the {@link webhooks} sink. The
+ * block-fanout scan path emits head and scan measurements; the delivery path
  * emits `webhook:delivery`.
  */
 export type WebhookEvent =
-  | {
-      /** Tempo chain whose poll cycle completed. */
-      chainId: number
-      /** Whether the poll cycle completed without scan or dispatch failures. */
-      outcome: 'error' | 'ok'
-      /** Requested boundary movement since the preceding successful poll, in blocks. */
-      requestedBoundaryAdvanceBlocks?: number | undefined
-      /** Execution path that acquired this poll claim. */
-      trigger: 'alarm' | 'cron' | 'direct' | 'queue'
-      /** Identifies a transport-neutral poll-cycle measurement. */
-      type: 'webhook:poll-cycle'
-    }
-  | {
-      /** Tempo chain whose durable progress was read. */
-      chainId: number
-      /** Blocks between the desired and fully drained boundaries. */
-      lagBlocks: number
-      /** Identifies durable coordinator progress. */
-      type: 'webhook:head-progress'
-    }
   | {
       /** Tempo chain whose listener was checked. */
       chainId: number
@@ -61,114 +35,35 @@ export type WebhookEvent =
     }
   | {
       chainId: number
-      /** Blocks the probed chain head leads the listener's newest observation. */
-      lagBlocks: number
-      /** Identifies the out-of-band listener head probe. */
-      type: 'webhook:listener-head-lag'
-    }
-  | {
-      /** Tempo chain whose recovery poke was dequeued. */
-      chainId: number
-      /** When the recovery poke was dequeued, as epoch milliseconds. */
-      dequeuedAt: number
-      /** When the recovery poke was enqueued, as epoch milliseconds. */
-      enqueuedAt: number
-      /** Recovery source carried through the Queue handoff. */
-      trigger: 'alarm' | 'queue'
-      /** Identifies recovery Queue latency, not observed head volume. */
-      type: 'webhook:head-recovery'
-    }
-  | {
-      /** Tempo chain whose poll claim was acquired. */
-      chainId: number
-      /** Delay from observing the desired boundary to acquiring the claim. */
-      delayMs?: number | undefined
-      /** Execution path that acquired the claim. */
-      trigger: 'alarm' | 'cron' | 'direct' | 'queue'
-      /** Identifies a poll beginning, before scan work runs. */
-      type: 'webhook:poll-start'
-    }
-  | {
-      /** Tempo chain whose poll failed. */
-      chainId: number
-      /** Bounded class of the thrown failure. */
-      failure: PollFailure
-      /** Operation boundary that rejected the poll. */
-      stage: 'claim' | 'continuation_enqueue' | 'finish' | 'scan'
-      /** Execution path that failed before recovery. */
-      trigger: 'alarm' | 'cron' | 'direct' | 'queue'
-      /** Identifies a transport-neutral poll failure. */
-      type: 'webhook:poll-failure'
-    }
-  | {
-      chainId: number
-      durationMs: number
-      eventType?: EventType | undefined
-      outcome: 'error' | 'ok'
-      stage: PollStage
-      type: 'webhook:poll'
-    }
-  | {
-      /** Tempo chain whose eligible subscription count was read. */
-      chainId: number
-      /** Active, unexpired subscriptions selected for the poll. */
-      count: number
-      /** Identifies a pre-scan subscription eligibility measurement. */
-      type: 'webhook:poll-subscriptions'
-    }
-  | {
-      /** Chain scanned by this poll batch. */
-      chainId: number
-      /** Shared upper bound for concurrent upstream scans. */
-      concurrency: number
-      /** Scan duration, present only after completion. */
-      durationMs?: number | undefined
-      /** Subscription event type scanned by this batch. */
-      eventType: EventType
-      /** Subscriptions whose scan rejected, present only after completion. */
-      failedSubscriptions?: number | undefined
-      /** Matching rows retained by completed pages, present only after completion. */
-      matchedItems?: number | undefined
-      /** Largest block window requested by one subscription. */
-      maxWindowBlocks: number
-      /** Whether the checkpoint precedes allocation or follows scan completion. */
-      phase: 'complete' | 'start'
-      /** Outcome of the completed RPC-backed scan. */
-      outcome?: 'empty' | 'error' | 'missing' | 'ok' | undefined
-      /** JSON-RPC method used by this event type's scanner. */
-      rpcMethod: 'eth_getBlockByNumber' | 'eth_getLogs'
-      /** Subscriptions retained in this scan batch. */
-      subscriptionCount: number
-      /** Maximum subscriptions retained by this event-type batch. */
-      subscriptionLimit: number
-      /** Sum of subscription block windows in this scan batch. */
-      totalWindowBlocks: number
-      /** Identifies a memory-attribution scan boundary. */
-      type: 'webhook:poll-scan'
-    }
-  | {
-      cursorLagBlocks?: number | undefined
-      meta: TickMeta
-      result: Webhooks.createPoller.Result | undefined
-      type: 'webhook:tick'
-    }
-  | {
-      chainId: number
-      durationMs: number
-      eventType: EventType
-      ok: boolean
-      type: 'webhook:enqueue'
-    }
-  | {
-      chainId: number
+      /** True once finality has deferred this block at least one round. */
+      deferred: boolean
       /** Rows matched when dispatched; zero otherwise. */
       matches: number
+      /** Block timestamp to scan completion; absent when the block was not read. */
+      scanLagMs?: number | undefined
+      /** Block timestamp to head observation; absent without a fresh stamp. */
+      observationLagMs?: number | undefined
       /** How the head that enqueued this block was observed. */
       origin: string
       /** One scanned block's outcome. */
       outcome: 'deferred' | 'dispatched' | 'empty'
+      /** Head observation to this scan attempt starting; absent without a stamp. */
+      queueWaitMs?: number | undefined
       /** Identifies one block-fanout scan. */
       type: 'webhook:scan-block'
+      /** Wall-clock work inside this scan attempt. */
+      workMs: number
+    }
+  | {
+      chainId: number
+      /** Times this step ran across the batch, so the total can be normalized. */
+      count: number
+      /** Wall-clock duration of this step summed across the batch. */
+      durationMs: number
+      /** Bounded step name. */
+      step: ScanStep
+      /** Identifies per-step scan attribution. */
+      type: 'webhook:scan-step'
     }
   | {
       chainId: number
@@ -178,6 +73,8 @@ export type WebhookEvent =
       pendingCount: number
       /** Identifies the coordinator's tracked-scan backlog. */
       type: 'webhook:scan-backlog'
+      /** Blocks between the chain head and the enqueue watermark. */
+      watermarkLagBlocks?: number | undefined
     }
   | {
       chainId: number
@@ -212,9 +109,17 @@ export type WebhookEvent =
         deliveryLogMs?: number | undefined
         envelopeToQueueMs?: number | undefined
         endToEndMs?: number | undefined
+        /** Block timestamp to the attempt leaving; the total latency SLI. */
+        eventToAttemptMs?: number | undefined
         eventToEnvelopeMs?: number | undefined
+        /** Head observation to the attempt leaving; the controllable SLI. */
+        observedToAttemptMs?: number | undefined
+        /** Dequeue to the attempt leaving: claim, reads, signing. */
+        preflightMs?: number | undefined
         processingMs?: number | undefined
         queueWaitMs?: number | undefined
+        /** Response headers to bookkeeping complete; occupancy, not latency. */
+        settleMs?: number | undefined
         stateWriteMs?: number | undefined
         subscriptionReadMs?: number | undefined
       }
@@ -227,7 +132,7 @@ export type RequestSink = (entry: Log.Entry) => void
 
 /** Records webhook reliability metrics. Built by {@link webhooks}. */
 export type WebhookSink = {
-  /** Ships buffered metrics (call once per poller tick / delivery). */
+  /** Ships buffered metrics (call once per scan batch / delivery). */
   flush(): void
   /** Records one webhook event. */
   record(event: WebhookEvent): void
@@ -386,161 +291,56 @@ export function webhooks(backend: Metrics): WebhookSink {
     },
     record(event) {
       switch (event.type) {
-        case 'webhook:head-recovery': {
-          const tags = { chain_id: String(event.chainId), trigger: event.trigger }
-          backend.count('webhook_head_recovery_queue_count', 1, tags)
-          backend.histogram(
-            'webhook_head_recovery_queue_wait_ms',
-            Math.max(0, event.dequeuedAt - event.enqueuedAt),
-            tags,
-          )
-          return
-        }
-        case 'webhook:head-progress':
-          backend.gauge('webhook_head_progress_lag_blocks', event.lagBlocks, {
-            chain_id: String(event.chainId),
-          })
-          return
         case 'webhook:head-skipped':
           backend.gauge('webhook_head_skipped_count', event.skippedHeads, {
             chain_id: String(event.chainId),
           })
           return
-        case 'webhook:listener-head-lag':
-          backend.gauge('webhook_listener_head_lag_blocks', event.lagBlocks, {
-            chain_id: String(event.chainId),
-          })
-          return
-        case 'webhook:poll-failure':
-          backend.count('webhook_poll_failure_count', 1, {
-            chain_id: String(event.chainId),
-            failure: event.failure,
-            stage: event.stage,
-            trigger: event.trigger,
-          })
-          return
-        case 'webhook:poll': {
+        case 'webhook:scan-block': {
           const tags = {
             chain_id: String(event.chainId),
-            ...(event.eventType === undefined ? {} : { event_type: event.eventType }),
+            origin: event.origin,
             outcome: event.outcome,
-            stage: event.stage,
-          }
-          backend.histogram('webhook_poller_stage_duration_ms', event.durationMs, tags)
-          return
-        }
-        case 'webhook:poll-scan': {
-          const tags = {
-            chain_id: String(event.chainId),
-            event_type: event.eventType,
-            phase: event.phase,
           }
-          backend.count('webhook_poller_scan_checkpoint_count', 1, tags)
-          backend.histogram('webhook_poller_scan_concurrency', event.concurrency, tags)
-          backend.histogram('webhook_poller_scan_subscription_count', event.subscriptionCount, tags)
-          backend.histogram('webhook_poller_scan_subscription_limit', event.subscriptionLimit, tags)
-          backend.histogram('webhook_poller_scan_max_window_blocks', event.maxWindowBlocks, tags)
-          backend.histogram(
-            'webhook_poller_scan_total_window_blocks',
-            event.totalWindowBlocks,
-            tags,
-          )
-          if (event.durationMs !== undefined)
-            backend.histogram('webhook_poller_scan_duration_ms', event.durationMs, tags)
-          if (event.failedSubscriptions !== undefined)
-            backend.count(
-              'webhook_poller_scan_failed_subscription_count',
-              event.failedSubscriptions,
-              tags,
-            )
-          if (event.matchedItems !== undefined)
-            backend.count('webhook_poller_scan_matched_item_count', event.matchedItems, tags)
-          if (event.outcome !== undefined)
-            backend.count('webhook_poller_rpc_scan_count', 1, {
+          backend.count('webhook_scan_block_count', 1, tags)
+          if (event.matches > 0) backend.count('webhook_scan_match_count', event.matches, tags)
+          backend.histogram('webhook_scan_work_ms', event.workMs, tags)
+          // Deferred rounds are chain finality, not queue depth, so the wait
+          // carries the distinction rather than silently averaging the two.
+          if (event.queueWaitMs !== undefined)
+            backend.histogram('webhook_scan_queue_wait_ms', event.queueWaitMs, {
+              ...tags,
+              deferred: String(event.deferred),
+            })
+          if (event.observationLagMs !== undefined)
+            backend.histogram('webhook_scan_observation_lag_ms', event.observationLagMs, {
               chain_id: String(event.chainId),
-              event_type: event.eventType,
-              outcome: event.outcome,
-              rpc_method: event.rpcMethod,
+              origin: event.origin,
             })
-          return
-        }
-        case 'webhook:poll-subscriptions':
-          backend.gauge('webhook_poller_active_subscription_count', event.count, {
-            chain_id: String(event.chainId),
-          })
-          return
-        case 'webhook:poll-cycle': {
-          const tags = {
-            chain_id: String(event.chainId),
-            outcome: event.outcome,
-            trigger: event.trigger,
-          }
-          backend.count('webhook_poll_cycle_count', 1, tags)
-          if (event.requestedBoundaryAdvanceBlocks !== undefined)
-            backend.histogram(
-              'webhook_poll_requested_boundary_advance_blocks',
-              event.requestedBoundaryAdvanceBlocks,
-              tags,
-            )
-          return
-        }
-        case 'webhook:poll-start': {
-          const tags = { chain_id: String(event.chainId), trigger: event.trigger }
-          backend.count('webhook_poll_start_count', 1, tags)
-          if (event.delayMs !== undefined)
-            backend.histogram('webhook_poll_start_delay_ms', event.delayMs, tags)
-          return
-        }
-        case 'webhook:tick': {
-          const { meta, result } = event
-          const chain_id = String(result?.chainId ?? meta.chainId ?? 'unknown')
-          backend.count('webhook_poller_tick_count', 1, { chain_id, outcome: meta.outcome })
-          backend.histogram('webhook_poller_tick_duration_ms', meta.durationMs, {
-            chain_id,
-            outcome: meta.outcome,
-          })
-          if (event.cursorLagBlocks !== undefined)
-            backend.gauge('webhook_subscription_cursor_lag_blocks', event.cursorLagBlocks, {
-              chain_id,
+          // Block timestamp to scan completion: how far behind the chain the
+          // scanner is, reported by every scanned block rather than only by
+          // blocks that produce a delivery.
+          if (event.scanLagMs !== undefined)
+            backend.histogram('webhook_scan_lag_ms', event.scanLagMs, {
+              chain_id: String(event.chainId),
             })
-          if (!result) return
-          backend.histogram('webhook_poller_subscription_count', result.subscriptions, { chain_id })
-          backend.count('webhook_poller_scanned_count', result.scanned, { chain_id })
-          backend.gauge('webhook_poller_safe_block', result.safeBlock, { chain_id })
-          backend.count('webhook_poller_dispatch_count', result.dispatched, {
-            chain_id,
-            outcome: 'dispatched',
-          })
-          backend.count('webhook_poller_dispatch_count', result.failed, {
-            chain_id,
-            outcome: 'failed',
-          })
           return
         }
-        case 'webhook:enqueue': {
-          const tags = { chain_id: String(event.chainId), event_type: event.eventType }
-          const outcome = event.ok ? 'ok' : 'error'
-          backend.count('webhook_dispatch_enqueue_count', 1, { ...tags, outcome })
-          backend.histogram('webhook_dispatch_enqueue_duration_ms', event.durationMs, {
-            ...tags,
-            outcome,
-          })
-          return
-        }
-        case 'webhook:scan-block': {
-          const tags = {
-            chain_id: String(event.chainId),
-            origin: event.origin,
-            outcome: event.outcome,
-          }
-          backend.count('webhook_scan_block_count', 1, tags)
-          if (event.matches > 0) backend.count('webhook_scan_match_count', event.matches, tags)
+        case 'webhook:scan-step': {
+          const tags = { chain_id: String(event.chainId), step: event.step }
+          // Batch totals, so the count is required to read a per-invocation
+          // mean: a ten-block batch otherwise reports ten times a one-block
+          // batch at identical per-call latency.
+          backend.histogram('webhook_scan_step_duration_ms', event.durationMs, tags)
+          backend.count('webhook_scan_step_count', event.count, tags)
           return
         }
         case 'webhook:scan-backlog': {
           const tags = { chain_id: String(event.chainId) }
           backend.gauge('webhook_scan_pending_count', event.pendingCount, tags)
           backend.gauge('webhook_scan_oldest_pending_age_ms', event.oldestPendingAgeMs ?? 0, tags)
+          if (event.watermarkLagBlocks !== undefined)
+            backend.gauge('webhook_scan_watermark_lag_blocks', event.watermarkLagBlocks, tags)
           return
         }
         case 'webhook:scan-finality': {
@@ -593,6 +393,25 @@ export function webhooks(backend: Metrics): WebhookSink {
             })
           if (timings?.deliveryLogMs !== undefined)
             backend.histogram('webhook_delivery_log_write_ms', timings.deliveryLogMs, tags)
+          if (timings?.preflightMs !== undefined)
+            backend.histogram('webhook_delivery_preflight_ms', timings.preflightMs, tags)
+          if (timings?.settleMs !== undefined)
+            backend.histogram('webhook_delivery_settle_ms', timings.settleMs, tags)
+          // The two headline latency SLIs, both ending when the attempt leaves
+          // rather than when bookkeeping finishes. Attempt-bucketed for the
+          // same reason as end-to-end: retries carry designed backoff.
+          if (timings?.eventToAttemptMs !== undefined)
+            backend.histogram('webhook_event_to_attempt_ms', timings.eventToAttemptMs, {
+              ...tags,
+              attempt_bucket: attemptBucket(event.attempt),
+              outcome,
+            })
+          if (timings?.observedToAttemptMs !== undefined)
+            backend.histogram('webhook_observed_to_attempt_ms', timings.observedToAttemptMs, {
+              ...tags,
+              attempt_bucket: attemptBucket(event.attempt),
+              outcome,
+            })
           if (timings?.envelopeToQueueMs !== undefined)
             backend.histogram(
               'webhook_delivery_envelope_to_queue_ms',