tapimo 0.7.2 → 0.9.0

package/src/internal/funding/Reconciliation.ts

@@ -2,6 +2,7 @@ import { Hex } from 'ox'
 
 import * as Db from '../../db/Db.js'
 import * as FundingDepositAddresses from '../../db/tables/fundingDepositAddresses.js'
+import * as FundingDepositRequestObservations from '../../db/tables/fundingDepositRequestObservations.js'
 import * as FundingDeposits from '../../db/tables/fundingDeposits.js'
 import type * as Metrics from '../../Metrics.js'
 import * as Value from '../Value.js'
@@ -12,7 +13,14 @@ import * as Subsidy from './Subsidy.js'
 const defaultBatchSize = 25
 const defaultLeaseMs = 2 * 60_000
 const defaultPollIntervalMs = 60_000
+const chainProviderFollowUpDelaySeconds = 1
+const chainProviderMaximumFollowUpAttempts = 30
+const followUpDelaySeconds = { delivery: 1, provider: 5, settlement: 2 } as const satisfies Record<
+  Provider.Webhook.receive.FollowUp['kind'],
+  number
+>
 const maximumPages = 100
+const maximumFollowUpAttempts = 6
 
 /** Verified token transfer returned by an authoritative chain reader. */
 export type TransferEvidence = {
@@ -52,10 +60,11 @@ export function createTracker(options: createTracker.Options): createTracker.Tra
   const leaseMs = options.leaseMs ?? defaultLeaseMs
   const pollIntervalMs = options.pollIntervalMs ?? defaultPollIntervalMs
 
-  const record = (outcome: string, providerId: string) => {
+  const record = (outcome: string, providerId: string, trigger: Deposit.DetectionTrigger) => {
     options.metrics?.count('funding_deposit_reconciliation_count', 1, {
       outcome,
       provider: providerId,
+      trigger,
     })
   }
 
@@ -63,6 +72,12 @@ export function createTracker(options: createTracker.Options): createTracker.Tra
     async reconcile(message) {
       const database = Db.get(options.db)
       const startedAt = now()
+      // Legacy poll messages carry a lease version; legacy webhook messages do not.
+      const trigger = (() => {
+        if (message.trigger) return message.trigger
+        if (message.pollLeaseVersion !== undefined) return 'poll'
+        return 'webhook'
+      })()
       const claimed = await claim(database, message, {
         leaseUntil: new Date(startedAt.getTime() + leaseMs).toISOString(),
         now: startedAt.toISOString(),
@@ -72,7 +87,7 @@ export function createTracker(options: createTracker.Options): createTracker.Tra
       const provider = options.providers.find((candidate) => candidate.id === claimed.providerId)
       if (!provider || !Provider.canListDepositAddressRequests(provider)) {
         await fail(database, claimed, startedAt, pollIntervalMs)
-        record('provider_unavailable', claimed.providerId)
+        record('provider_unavailable', claimed.providerId, trigger)
         return { type: 'provider_unavailable' }
       }
 
@@ -84,24 +99,70 @@ export function createTracker(options: createTracker.Options): createTracker.Tra
           requestId: message.requestId,
           signal: options.signal,
         })
+        if (trigger === 'poll')
+          await FundingDepositRequestObservations.observePoll(database, {
+            depositAddressId: claimed.id,
+            observedAt: now().toISOString(),
+            providerRequestIds: requests.requests.map((request) => request.id),
+          })
+        const webhookUpdateObserved = (() => {
+          const timing = message.webhookTiming
+          if (!timing) return true
+          if (!message.requestId) return false
+          // Relay can send a webhook before its requests API indexes the same update.
+          return requests.requests.some(
+            (request) =>
+              request.id === message.requestId && request.updatedAt >= timing.providerUpdatedAt,
+          )
+        })()
         heartbeat.assert()
         let created = 0
         let updated = 0
-        const pendingRequestIds = new Set(requests.pendingRequestIds)
+        const pendingRequestIds = new Set([
+          ...requests.pendingRequestIds,
+          ...(message.requestId ? [message.requestId] : []),
+        ])
+        let followUpKind: Provider.Webhook.receive.FollowUp['kind'] | undefined
         for (const request of requests.requests) {
           const result = await reconcileRequest(database, {
             address: claimed,
             metrics: options.metrics,
+            provider,
             request,
+            signal: options.signal,
             subsidies: options.subsidies,
+            trigger,
             verifyTransfers: options.verifyTransfers,
           })
           heartbeat.assert()
           created += result.created
           updated += result.updated
           if (result.terminal) pendingRequestIds.delete(request.id)
-          else pendingRequestIds.add(request.id)
+          else {
+            pendingRequestIds.add(request.id)
+            followUpKind = result.followUp ?? followUpKind ?? 'provider'
+          }
         }
+        const unattributed = await reconcileUnattributed(database, {
+          address: claimed,
+          metrics: options.metrics,
+          provider,
+          signal: options.signal,
+          subsidies: options.subsidies,
+          verifyTransfers: options.verifyTransfers,
+        })
+        heartbeat.assert()
+        updated += unattributed.updatedIds.size
+        if (!unattributed.terminal)
+          followUpKind = unattributed.followUp ?? followUpKind ?? 'provider'
+        const awaitsProvider = await FundingDeposits.hasUnattributed(database, claimed.id)
+        if (message.requestId && message.webhookTiming && !webhookUpdateObserved)
+          pendingRequestIds.add(message.requestId)
+        if (created > 0)
+          options.metrics?.count('funding_deposit_detection_count', created, {
+            provider: claimed.providerId,
+            trigger,
+          })
 
         const leaseFailure = await heartbeat.stop()
         if (leaseFailure !== undefined) throw leaseFailure
@@ -112,22 +173,65 @@ export function createTracker(options: createTracker.Options): createTracker.Tra
           nextPollAt: new Date(completedAt.getTime() + pollIntervalMs).toISOString(),
           pollLeaseVersion: claimed.pollLeaseVersion,
           providerRequestIds: claimed.providerRequestIds,
-          providerState: reconciliationState(claimed.providerState, requests, [
-            ...pendingRequestIds,
-          ]),
+          providerState: reconciliationState(claimed.providerState, {
+            pendingRequestIds: [...pendingRequestIds],
+            requests,
+            ...(message.webhookTiming && message.requestId && webhookUpdateObserved
+              ? {
+                  webhookTiming: {
+                    ...message.webhookTiming,
+                    reconciliationCompletedAt: completedAt.toISOString(),
+                    requestId: message.requestId,
+                  },
+                }
+              : {}),
+          }),
         })
         if (!stored) return { type: 'stale' }
-        record('completed', claimed.providerId)
+        if (pendingRequestIds.size > 0 || awaitsProvider) {
+          const followUp = nextFollowUp(message, followUpKind ?? 'provider')
+          if (followUp)
+            await options
+              .dispatch(
+                {
+                  addressId: claimed.id,
+                  followUp: followUp.state,
+                  ...(message.requestId ? { requestId: message.requestId } : {}),
+                  trigger,
+                  type: 'funding:deposit-address:reconcile',
+                  ...(message.webhookTiming && !webhookUpdateObserved
+                    ? { webhookTiming: message.webhookTiming }
+                    : {}),
+                },
+                { delaySeconds: followUp.delaySeconds },
+              )
+              .catch((cause) => {
+                record('follow_up_enqueue_failed', claimed.providerId, trigger)
+                throw cause
+              })
+          record(
+            followUp ? 'follow_up_enqueued' : 'follow_up_exhausted',
+            claimed.providerId,
+            trigger,
+          )
+        }
+        if (message.webhookTiming && webhookUpdateObserved)
+          recordWebhookLatency(options.metrics, {
+            completedAt,
+            providerId: claimed.providerId,
+            timing: message.webhookTiming,
+          })
+        record('completed', claimed.providerId, trigger)
         options.metrics?.histogram(
           'funding_deposit_reconciliation_duration_ms',
           completedAt.getTime() - startedAt.getTime(),
-          { provider: claimed.providerId },
+          { provider: claimed.providerId, trigger },
         )
         return { created, type: 'completed', updated }
       } catch (cause) {
         const leaseFailure = await heartbeat.stop()
         await fail(database, claimed, now(), pollIntervalMs)
-        record('failed', claimed.providerId)
+        record('failed', claimed.providerId, trigger)
         throw leaseFailure ?? cause
       } finally {
         options.metrics?.flush()
@@ -159,13 +263,14 @@ export function createTracker(options: createTracker.Options): createTracker.Tra
             await options.dispatch({
               addressId: address.id,
               pollLeaseVersion: address.pollLeaseVersion,
+              trigger: 'poll',
               type: 'funding:deposit-address:reconcile',
             })
-            record('enqueued', provider.id)
+            record('enqueued', provider.id, 'poll')
           } catch {
             failed += 1
             await fail(database, address, startedAt, pollIntervalMs)
-            record('enqueue_failed', provider.id)
+            record('enqueue_failed', provider.id, 'poll')
           }
         }
         if (claimed >= batchSize) break
@@ -184,7 +289,10 @@ export declare namespace createTracker {
     /** Authoritative database or per-invocation database factory. */
     db: Db.Source
     /** Durable reconciliation queue transport. */
-    dispatch: (message: Provider.Webhook.receive.Dispatchable) => Promise<void>
+    dispatch: (
+      message: Provider.Webhook.receive.Dispatchable,
+      options?: createTracker.DispatchOptions,
+    ) => Promise<void>
     /** Duration of one reconciliation lease. */
     leaseMs?: number | undefined
     /** Optional bounded operational metrics sink. */
@@ -203,6 +311,12 @@ export declare namespace createTracker {
     verifyTransfers: VerifyTransfers
   }
 
+  /** Delivery options for one reconciliation queue message. */
+  type DispatchOptions = {
+    /** Delay before the queue makes the message available. */
+    delaySeconds: number
+  }
+
   /** Result of reconciling one queue message. */
   type ReconcileResult =
     | { type: 'busy' | 'provider_unavailable' | 'stale' }
@@ -211,7 +325,7 @@ export declare namespace createTracker {
   /** Scheduled and queue-driven reusable-address tracker. */
   type Tracker = {
     /** Reconciles one address from provider discovery and verified chain evidence. */
-    reconcile(message: Provider.Webhook.receive.Dispatchable): Promise<ReconcileResult>
+    reconcile(message: Provider.Webhook.receive.Queued): Promise<ReconcileResult>
     /** Claims and enqueues one bounded batch of due reusable addresses. */
     tick(): Promise<{ claimed: number; failed: number }>
   }
@@ -222,11 +336,7 @@ type ClaimOptions = {
   now: string
 }
 
-async function claim(
-  db: Db.Db,
-  message: Provider.Webhook.receive.Dispatchable,
-  options: ClaimOptions,
-) {
+async function claim(db: Db.Db, message: Provider.Webhook.receive.Queued, options: ClaimOptions) {
   if (message.pollLeaseVersion === undefined)
     return FundingDepositAddresses.claim(db, {
       id: message.addressId,
@@ -421,18 +531,42 @@ function readHighWater(state: FundingDepositAddresses.Record['providerState']):
   }
 }
 
+type ReconciledWebhookTiming = Provider.Webhook.receive.WebhookTiming & {
+  /** When Tempo completed the webhook-triggered reconciliation. */
+  reconciliationCompletedAt: string
+  /** Provider request matched to the observed update. */
+  requestId: string
+}
+
+type ReconciliationStateOptions = {
+  /** Provider requests that still require reconciliation. */
+  pendingRequestIds: readonly string[]
+  /** Provider requests and high-water state observed in this pass. */
+  requests: Requests
+  /** Completed webhook timing retained for diagnostics. */
+  webhookTiming?: ReconciledWebhookTiming | undefined
+}
+
 function reconciliationState(
   state: FundingDepositAddresses.Record['providerState'],
-  requests: Requests,
-  pendingRequestIds: readonly string[],
+  options: ReconciliationStateOptions,
 ) {
   const current = state && typeof state === 'object' ? state : {}
+  const reconciliation = (() => {
+    const value = current['reconciliation']
+    if (value && typeof value === 'object') return value
+    return {}
+  })()
   return {
     ...current,
     reconciliation: {
-      pendingRequestIds,
-      requestIds: requests.highWaterRequestIds,
-      ...(requests.highWaterUpdatedAt ? { updatedAt: requests.highWaterUpdatedAt } : {}),
+      ...reconciliation,
+      pendingRequestIds: options.pendingRequestIds,
+      requestIds: options.requests.highWaterRequestIds,
+      ...(options.requests.highWaterUpdatedAt
+        ? { updatedAt: options.requests.highWaterUpdatedAt }
+        : {}),
+      ...(options.webhookTiming ? { webhookTiming: options.webhookTiming } : {}),
     },
   }
 }
@@ -446,49 +580,185 @@ type ObservedSourceTransaction = {
 type VerifiedDeposit = {
   deposit: FundingDeposits.Record
   required: bigint
+  source: TransferEvidence
   sourceAmount: bigint
 }
 
+type ReconcileDeliveryOptions = {
+  address: FundingDepositAddresses.Record
+  metrics?: Metrics.Metrics | undefined
+  provider?: Provider.Provider | undefined
+  request?: Provider.listDepositAddressRequests.ReturnType['requests'][number] | undefined
+  signal?: AbortSignal | undefined
+  subsidies?: Subsidy.Settler | undefined
+  verified: readonly VerifiedDeposit[]
+  verifyTransfers: VerifyTransfers
+}
+
+async function observeProviderDelivery(options: ReconcileDeliveryOptions) {
+  const { verified } = options
+  const [entry] = verified
+  if (
+    !entry ||
+    verified.length !== 1 ||
+    options.address.deliveryStrategy !== 'provider' ||
+    options.address.providerOutputToken.tokenKey !==
+      options.address.snapshot.destinationToken.tokenKey ||
+    (options.request?.status === 'success' &&
+      options.request.destinationTransactionHashes.length > 0) ||
+    !options.provider ||
+    !Provider.canObserveDepositAddressRequest(options.provider)
+  )
+    return undefined
+
+  const result = await options.provider
+    .observeDepositAddressRequest(
+      {
+        address: options.address.address,
+        destinationChain: options.address.snapshot.destinationChain,
+        destinationToken: options.address.providerOutputToken,
+        providerState: entry.deposit.providerState ?? undefined,
+        recipient: options.address.recipient,
+        refundAddress: options.address.refundAddress,
+        ...(options.request ? { request: options.request } : {}),
+        source: entry.source,
+        sourceChain: options.address.snapshot.sourceChain,
+        sourceToken: options.address.snapshot.sourceToken,
+      },
+      options.signal ?? new AbortController().signal,
+    )
+    .catch((cause) => {
+      if (options.signal?.aborted) throw cause
+      return undefined
+    })
+  options.metrics?.count('funding_deposit_delivery_observation_count', 1, {
+    outcome: result?.type ?? 'failed',
+    provider: options.address.providerId,
+  })
+  return result
+}
+
+async function reconcileUnattributed(db: Db.Db, options: reconcileUnattributed.Options) {
+  const deposits = await FundingDeposits.listUnattributed(db, options.address.id)
+  if (deposits.length === 0) return { terminal: true, updatedIds: new Set<string>() }
+
+  const transfersByTransaction = new Map(
+    await Promise.all(
+      unique(deposits.map((deposit) => deposit.sourceTransactionHash)).map(
+        async (transactionHash) =>
+          [
+            transactionHash,
+            await options.verifyTransfers({
+              chain: options.address.snapshot.sourceChain,
+              recipient: options.address.address,
+              token: options.address.snapshot.sourceToken,
+              transactionHash,
+            }),
+          ] as const,
+      ),
+    ),
+  )
+  let followUp: reconcileRequest.ReturnType['followUp']
+  let terminal = true
+  const updatedIds = new Set<string>()
+  for (const deposit of deposits) {
+    const source = transfersByTransaction
+      .get(deposit.sourceTransactionHash)
+      ?.find((transfer) => transfer.transferIndex === deposit.sourceTransferIndex)
+    if (!source) {
+      terminal = false
+      continue
+    }
+    const delivery = await reconcileDelivery(db, {
+      ...options,
+      verified: [
+        {
+          deposit,
+          required: destinationBaseUnits(
+            BigInt(source.amount),
+            options.address.snapshot.sourceToken.decimals,
+            options.address.snapshot.destinationToken.decimals,
+          ),
+          source,
+          sourceAmount: BigInt(source.amount),
+        },
+      ],
+    })
+    if (delivery.followUp === 'delivery' || !followUp) followUp = delivery.followUp
+    if (!delivery.terminal) terminal = false
+    for (const id of delivery.updatedIds) updatedIds.add(id)
+  }
+  return { ...(followUp ? { followUp } : {}), terminal, updatedIds }
+}
+
+declare namespace reconcileUnattributed {
+  type Options = Omit<ReconcileDeliveryOptions, 'request' | 'verified'>
+}
+
 /** Reconciles one provider request against verified chain evidence. */
 export async function reconcileRequest(
   db: Db.Db,
   options: reconcileRequest.Options,
 ): Promise<reconcileRequest.ReturnType> {
+  const trigger = options.trigger ?? 'poll'
+  const observation = await FundingDepositRequestObservations.get(db, {
+    depositAddressId: options.address.id,
+    providerRequestId: options.request.id,
+  })
+  if (observation)
+    await FundingDeposits.recordRequestObservation(db, {
+      depositAddressId: observation.depositAddressId,
+      ...(observation.pollObservedAt ? { pollObservedAt: observation.pollObservedAt } : {}),
+      providerRequestId: observation.providerRequestId,
+      ...(observation.webhookReceivedAt
+        ? { webhookReceivedAt: observation.webhookReceivedAt }
+        : {}),
+    })
   let created = 0
   const observed: ObservedSourceTransaction[] = []
   // Persist provider observations first so transient chain failures do not hide acknowledged deposits.
   for (const transactionHash of unique(options.request.sourceTransactionHashes)) {
-    let deposits = await FundingDeposits.listByProviderObservation(db, {
+    const deposits = await FundingDeposits.withSourceTransaction(db, {
       depositAddressId: options.address.id,
-      providerRequestId: options.request.id,
-      sourceTransactionHash: transactionHash,
-    })
-    if (deposits.length === 0)
-      deposits = await FundingDeposits.listBySourceTransaction(db, {
-        depositAddressId: options.address.id,
-        sourceTransactionHash: transactionHash,
-      })
-    if (deposits.length === 0) {
-      deposits = [
-        await Deposit.create(db, {
-          now: new Date(options.request.createdAt),
+      fn: async (tx): Promise<FundingDeposits.Record[]> => {
+        let records = await FundingDeposits.listByProviderObservation(tx, {
+          depositAddressId: options.address.id,
           providerRequestId: options.request.id,
-          snapshot: {
-            depositAddressId: options.address.id,
-            destinationChain: options.address.snapshot.destinationChain,
-            destinationToken: options.address.snapshot.destinationToken,
-            provider: options.address.snapshot.provider,
-            recipient: options.address.recipient,
-            refundAddress: options.address.refundAddress,
-            sourceChain: options.address.snapshot.sourceChain,
-            sourceToken: options.address.snapshot.sourceToken,
-            sourceTransactionHashes: [transactionHash],
-          },
           sourceTransactionHash: transactionHash,
-        }),
-      ]
-      created += 1
-    }
+        })
+        if (records.length === 0)
+          records = await FundingDeposits.listBySourceTransaction(tx, {
+            depositAddressId: options.address.id,
+            sourceTransactionHash: transactionHash,
+          })
+        if (records.length > 0) return records
+        created += 1
+        return [
+          await Deposit.create(tx, {
+            detectionTrigger: trigger,
+            now: new Date(options.request.createdAt),
+            ...(observation?.pollObservedAt ? { pollObservedAt: observation.pollObservedAt } : {}),
+            providerRequestId: options.request.id,
+            snapshot: {
+              depositAddressId: options.address.id,
+              destinationChain: options.address.snapshot.destinationChain,
+              destinationToken: options.address.snapshot.destinationToken,
+              provider: options.address.snapshot.provider,
+              recipient: options.address.recipient,
+              refundAddress: options.address.refundAddress,
+              sourceChain: options.address.snapshot.sourceChain,
+              sourceToken: options.address.snapshot.sourceToken,
+              sourceTransactionHashes: [transactionHash],
+            },
+            sourceTransactionHash: transactionHash,
+            ...(observation?.webhookReceivedAt
+              ? { webhookReceivedAt: observation.webhookReceivedAt }
+              : {}),
+          }),
+        ]
+      },
+      sourceTransactionHash: transactionHash,
+    })
     observed.push({ deposits, transactionHash, transfers: [] })
   }
 
@@ -508,7 +778,6 @@ export async function reconcileRequest(
 
   const sourceEvidenceComplete =
     sources.length > 0 && sources.every((source) => source.transfers.length > 0)
-  let terminal = sourceEvidenceComplete
   const updatedIds = new Set<string>()
   const verified: VerifiedDeposit[] = []
   for (const source of sources) {
@@ -539,7 +808,9 @@ export async function reconcileRequest(
       )
       if (!deposit) {
         deposit = await Deposit.create(db, {
+          detectionTrigger: trigger,
           now: new Date(options.request.createdAt),
+          ...(observation?.pollObservedAt ? { pollObservedAt: observation.pollObservedAt } : {}),
           providerRequestId: options.request.id,
           providerTransferIndex,
           snapshot: {
@@ -558,15 +829,24 @@ export async function reconcileRequest(
           },
           sourceTransactionHash: transfer.transactionHash,
           sourceTransferIndex: transfer.transferIndex,
+          ...(observation?.webhookReceivedAt
+            ? { webhookReceivedAt: observation.webhookReceivedAt }
+            : {}),
         })
         source.deposits.push(deposit)
         created += 1
       }
-      if (!Deposit.isTerminal(deposit.status)) {
+      if (
+        !Deposit.isTerminal(deposit.status) ||
+        deposit.providerRequestId === null ||
+        !deposit.providerRequestIds.includes(options.request.id)
+      ) {
         const result = await Deposit.transition(db, {
           expectedVersion: deposit.version,
           id: deposit.id,
+          providerRequestId: options.request.id,
           providerRequestIds: unique([...deposit.providerRequestIds, options.request.id]),
+          providerTransferIndex,
           snapshot: {
             ...deposit.snapshot,
             ...(options.address.snapshot.subsidize ? { destinationAmountRequired } : {}),
@@ -581,14 +861,72 @@ export async function reconcileRequest(
           updatedIds.add(deposit.id)
         }
       }
-      verified.push({ deposit, required, sourceAmount: BigInt(transfer.amount) })
+      verified.push({ deposit, required, source: transfer, sourceAmount: BigInt(transfer.amount) })
     }
   }
   if (verified.length === 0) return { created, terminal: false, updated: updatedIds.size }
   if (!sourceEvidenceComplete) return { created, terminal: false, updated: updatedIds.size }
+  const delivery = await reconcileDelivery(db, { ...options, verified })
+  for (const id of delivery.updatedIds) updatedIds.add(id)
+  return {
+    created,
+    ...(delivery.followUp ? { followUp: delivery.followUp } : {}),
+    terminal: delivery.terminal,
+    updated: updatedIds.size,
+  }
+}
 
+export declare namespace reconcileRequest {
+  /** Provider request, address, and settlement dependencies. */
+  type Options = {
+    /** Reusable deposit address attributed by the provider. */
+    address: FundingDepositAddresses.Record
+    /** Optional bounded operational metrics sink. */
+    metrics?: Metrics.Metrics | undefined
+    /** Provider integration that discovered the request. */
+    provider?: Provider.Provider | undefined
+    /** Provider request containing discovered chain transactions. */
+    request: Provider.listDepositAddressRequests.ReturnType['requests'][number]
+    /** Abort signal forwarded to provider delivery observation. */
+    signal?: AbortSignal | undefined
+    /** Optional pathUSD-funded subsidy settler. */
+    subsidies?: Subsidy.Settler | undefined
+    /** Mechanism that requested reconciliation. */
+    trigger?: Deposit.DetectionTrigger | undefined
+    /** Authoritative source and destination transfer verifier. */
+    verifyTransfers: VerifyTransfers
+  }
+
+  /** Material changes produced by one provider request. */
+  type ReturnType = {
+    /** Number of newly detected source transfers. */
+    created: number
+    /** Work that needs a bounded delayed follow-up. */
+    followUp?: 'delivery' | 'settlement' | undefined
+    /** Whether every deposit from this request reached a terminal state. */
+    terminal: boolean
+    /** Number of existing or new deposits materially changed. */
+    updated: number
+  }
+}
+
+async function reconcileDelivery(db: Db.Db, options: ReconcileDeliveryOptions) {
+  let followUp: reconcileRequest.ReturnType['followUp']
+  let terminal = true
+  const updatedIds = new Set<string>()
+  if (options.verified.every((entry) => Deposit.isTerminal(entry.deposit.status)))
+    return { terminal, updatedIds }
+  const deliveryObservation = await observeProviderDelivery(options)
+  if (deliveryObservation?.type === 'pending') followUp = 'delivery'
+  const providerTransactionHashes = options.request?.destinationTransactionHashes ?? []
+  const destinationTransactionHashes = unique([
+    ...providerTransactionHashes,
+    ...(deliveryObservation?.type === 'verified'
+      ? deliveryObservation.destinationTransactionHashes
+      : []),
+  ])
   const destinationEvidence = await Promise.all(
-    options.request.destinationTransactionHashes.map((transactionHash) =>
+    destinationTransactionHashes.map((transactionHash) =>
       options.verifyTransfers({
         chain: options.address.snapshot.destinationChain,
         recipient: options.address.recipient,
@@ -599,17 +937,18 @@ export async function reconcileRequest(
   )
   const destinationTransfers = destinationEvidence.flat()
   const providerEvidenceComplete =
-    options.request.status === 'success' &&
+    (options.request?.status === 'success' || deliveryObservation?.type === 'verified') &&
+    destinationEvidence.length > 0 &&
     destinationEvidence.every((transfers) => transfers.length > 0)
   const providerOutput = destinationTransfers.reduce(
     (total, transfer) => total + BigInt(transfer.amount),
     0n,
   )
-  const totalRequired = verified.reduce((total, entry) => total + entry.required, 0n)
+  const totalRequired = options.verified.reduce((total, entry) => total + entry.required, 0n)
   let allocated = 0n
   let cumulativeRequired = 0n
 
-  for (const entry of verified) {
+  for (const entry of options.verified) {
     const { required, sourceAmount } = entry
     cumulativeRequired += required
     const nextAllocated =
@@ -635,6 +974,7 @@ export async function reconcileRequest(
         provider: options.address.providerId,
       })
     if (settlement.status === 'bridging' || settlement.status === 'settling') terminal = false
+    if (settlement.status === 'settling') followUp = 'settlement'
     const destinationAmount = Value.tokenAmount({
       baseUnits: delivered + settlement.delivered,
       currency: options.address.snapshot.destinationToken.currency,
@@ -652,10 +992,21 @@ export async function reconcileRequest(
             }),
           }
         : {}),
-      providerRequestIds: unique([...deposit.providerRequestIds, options.request.id]),
+      providerRequestIds: options.request
+        ? unique([...deposit.providerRequestIds, options.request.id])
+        : deposit.providerRequestIds,
+      ...(deliveryObservation?.providerState
+        ? {
+            providerState: {
+              ...deposit.providerState,
+              ...deliveryObservation.providerState,
+            },
+          }
+        : {}),
       providerTransactionHashes: unique([
         ...deposit.providerTransactionHashes,
-        ...options.request.destinationTransactionHashes,
+        ...providerTransactionHashes,
+        ...(deliveryObservation?.providerTransactionHashes ?? []),
       ]),
       ...(settlement.settlementTransaction
         ? { settlementTransaction: settlement.settlementTransaction }
@@ -697,33 +1048,7 @@ export async function reconcileRequest(
       }
     }
   }
-  return { created, terminal, updated: updatedIds.size }
-}
-
-export declare namespace reconcileRequest {
-  /** Provider request, address, and settlement dependencies. */
-  type Options = {
-    /** Reusable deposit address attributed by the provider. */
-    address: FundingDepositAddresses.Record
-    /** Optional bounded operational metrics sink. */
-    metrics?: Metrics.Metrics | undefined
-    /** Provider request containing discovered chain transactions. */
-    request: Provider.listDepositAddressRequests.ReturnType['requests'][number]
-    /** Optional pathUSD-funded subsidy settler. */
-    subsidies?: Subsidy.Settler | undefined
-    /** Authoritative source and destination transfer verifier. */
-    verifyTransfers: VerifyTransfers
-  }
-
-  /** Material changes produced by one provider request. */
-  type ReturnType = {
-    /** Number of newly detected source transfers. */
-    created: number
-    /** Whether every deposit from this request reached a terminal state. */
-    terminal: boolean
-    /** Number of existing or new deposits materially changed. */
-    updated: number
-  }
+  return { ...(followUp ? { followUp } : {}), terminal, updatedIds }
 }
 
 type ReconcileSubsidyOptions = {
@@ -919,6 +1244,60 @@ function unique(values: readonly string[]) {
   return [...new Set(values)]
 }
 
+function nextFollowUp(
+  message: Provider.Webhook.receive.Queued,
+  kind: Provider.Webhook.receive.FollowUp['kind'],
+) {
+  const attempt = message.followUp?.kind === kind ? message.followUp.attempt + 1 : 0
+  const chainProvider = kind === 'provider' && message.trigger === 'chain'
+  const maximumAttempts = chainProvider
+    ? chainProviderMaximumFollowUpAttempts
+    : maximumFollowUpAttempts
+  if (attempt >= maximumAttempts) return undefined
+  return {
+    delaySeconds: chainProvider ? chainProviderFollowUpDelaySeconds : followUpDelaySeconds[kind],
+    state: { attempt, kind },
+  }
+}
+
+type RecordWebhookLatencyOptions = {
+  /** When Tempo completed the webhook-triggered reconciliation. */
+  completedAt: Date
+  /** Funding provider that sent the webhook. */
+  providerId: string
+  /** Authenticated timestamps carried from webhook receipt. */
+  timing: Provider.Webhook.receive.WebhookTiming
+}
+
+function recordWebhookLatency(
+  metrics: Metrics.Metrics | undefined,
+  options: RecordWebhookLatencyOptions,
+) {
+  if (!metrics) return
+  const completedAt = options.completedAt.getTime()
+  const enqueuedAt = Date.parse(options.timing.enqueuedAt)
+  const providerUpdatedAt = Date.parse(options.timing.providerUpdatedAt)
+  const receivedAt = Date.parse(options.timing.receivedAt)
+  const sentAt = Date.parse(options.timing.sentAt)
+  const segments = [
+    { from: providerUpdatedAt, segment: 'provider_to_sent', to: sentAt },
+    { from: sentAt, segment: 'sent_to_received', to: receivedAt },
+    { from: receivedAt, segment: 'received_to_enqueued', to: enqueuedAt },
+    { from: enqueuedAt, segment: 'enqueued_to_reconciled', to: completedAt },
+    { from: providerUpdatedAt, segment: 'provider_to_reconciled', to: completedAt },
+  ] as const
+  for (const segment of segments)
+    metrics.histogram(
+      'funding_deposit_webhook_latency_ms',
+      Math.max(0, segment.to - segment.from),
+      {
+        provider: options.providerId,
+        segment: segment.segment,
+        status: options.timing.providerStatus,
+      },
+    )
+}
+
 /** Provider pagination exceeded the bounded reconciliation limit. */
 export class PaginationLimitError extends Error {
   override name = 'Funding.Reconciliation.PaginationLimitError'