tapimo 0.14.0 → 0.15.0

package/src/internal/routes/Observability.ts

@@ -0,0 +1,668 @@
+import type * as Db from '../../db/Db.js'
+import * as RoutesDepositAddresses from '../../db/tables/routesDepositAddresses.js'
+import * as RoutesDeposits from '../../db/tables/routesDeposits.js'
+import * as RoutesTransfers from '../../db/tables/routesTransfers.js'
+import type * as Metrics from '../../Metrics.js'
+import * as Deposit from './Deposit.js'
+import * as DepositAddress from './DepositAddress.js'
+import * as Economics from './Economics.js'
+import * as Provider from './Provider.js'
+import * as Transfer from './Transfer.js'
+
+const activeDepositStatuses = Deposit.statuses.filter((status) => !Deposit.isTerminal(status))
+const activeDepositAddressStatuses = DepositAddress.statuses.filter(
+  (status) => !DepositAddress.isTerminal(status),
+)
+const activeTransferStatuses = Transfer.statuses.filter((status) => !Transfer.isTerminal(status))
+const environments = ['production', 'sandbox'] as const
+
+/** One completed route lifecycle stage used for retrospective history. */
+export type LifecycleInterval = {
+  /** UTC time when the stage ended. */
+  endedAt: string
+  /** API-key environment that owns the route journey. */
+  routesEnvironment: 'production' | 'sandbox'
+  /** Route method used by the journey. */
+  method: 'deposit_address' | 'transaction'
+  /** Route provider that owns the journey. */
+  provider: string
+  /** Stable route resource id. */
+  resourceId: string
+  /** Completed lifecycle stage. */
+  stage: 'source_routes'
+  /** UTC time when the stage began. */
+  startedAt: string
+}
+
+/** Returns the source-routes interval completed by an address's first verified deposit. */
+export function depositAddressSourceRoutesInterval(
+  options: depositAddressSourceRoutesInterval.Options,
+): LifecycleInterval {
+  return {
+    endedAt: end(options.address.createdAt, options.request.createdAt),
+    routesEnvironment: options.address.environment,
+    method: 'deposit_address',
+    provider: options.address.providerId,
+    resourceId: options.address.id,
+    stage: 'source_routes',
+    startedAt: options.address.createdAt,
+  }
+}
+
+export declare namespace depositAddressSourceRoutesInterval {
+  /** Address and first verified provider request. */
+  type Options = {
+    /** Reusable deposit address whose first route journey advanced. */
+    address: RoutesDepositAddresses.Record
+    /** Provider request that first reported source funds. */
+    request: Provider.listDepositAddressRequests.ReturnType['requests'][number]
+  }
+}
+
+/** Records a committed deposit transition and its completed lifecycle segments. */
+export function recordDeposit(
+  metrics: Metrics.Metrics | undefined,
+  options: recordDeposit.Options,
+) {
+  if (!metrics || options.previous?.status === options.record.status) return
+  metrics.count('routes_resource_transition_count', 1, {
+    from_status: options.previous?.status ?? 'none',
+    routes_environment: options.record.environment,
+    method: 'deposit_address',
+    provider: options.record.snapshot.provider.id,
+    reason: options.record.statusReason?.code ?? 'none',
+    resource: 'deposit',
+    to_status: options.record.status,
+  })
+
+  if (
+    options.providerEvidenceComplete &&
+    options.record.providerDeliveredAt &&
+    (options.previous?.status === 'bridging' || options.previous?.status === 'detected')
+  )
+    lifecycle(metrics, options.record.providerDeliveredAt, options.record.createdAt, {
+      flow: 'deposit_address',
+      routes_environment: options.record.environment,
+      outcome: 'success',
+      provider: options.record.snapshot.provider.id,
+      segment: 'detected_to_provider_delivery',
+    })
+  if (options.record.status !== 'completed') return
+  const tags = routeTags({
+    routesEnvironment: options.record.environment,
+    method: 'deposit_address',
+    snapshot: options.record.snapshot,
+  })
+  recordCustomerValueLoss(metrics, {
+    destination: options.record.snapshot.destinationAmount,
+    source: options.record.snapshot.sourceAmount,
+    tags: { ...tags, stage: 'executed' },
+  })
+  if (options.record.subsidyAmount) {
+    const subsidy = Economics.usd(options.record.subsidyAmount)
+    if (subsidy !== undefined) metrics.histogram('routes_tempo_subsidy_usd', subsidy, tags)
+  }
+  if (options.record.tempoGasPaid) {
+    const gasPaid = Number(options.record.tempoGasPaid)
+    if (Number.isFinite(gasPaid))
+      metrics.histogram('routes_tempo_gas_paid_base_units', gasPaid, tags)
+  }
+  if (options.previous?.status === 'settling' && options.record.providerDeliveredAt)
+    lifecycle(metrics, options.record.updatedAt, options.record.providerDeliveredAt, {
+      flow: 'deposit_address',
+      routes_environment: options.record.environment,
+      outcome: 'success',
+      provider: options.record.snapshot.provider.id,
+      segment: 'provider_delivery_to_completed',
+    })
+  lifecycle(metrics, options.record.updatedAt, options.record.createdAt, {
+    flow: 'deposit_address',
+    routes_environment: options.record.environment,
+    outcome: 'success',
+    provider: options.record.snapshot.provider.id,
+    segment: 'detected_to_completed',
+  })
+}
+
+export declare namespace recordDeposit {
+  /** Committed deposit transition. */
+  type Options = {
+    /** Record before the transition, absent for first detection. */
+    previous?: RoutesDeposits.Record | undefined
+    /** Whether complete provider delivery evidence informed this transition. */
+    providerEvidenceComplete?: boolean | undefined
+    /** Committed deposit record. */
+    record: RoutesDeposits.Record
+  }
+}
+
+/** Records a committed deposit-address transition. */
+export function recordDepositAddress(
+  metrics: Metrics.Metrics | undefined,
+  options: recordDepositAddress.Options,
+) {
+  if (!metrics || options.previous?.status === options.record.status) return
+  metrics.count('routes_resource_transition_count', 1, {
+    from_status: options.previous?.status ?? 'none',
+    routes_environment: options.record.environment,
+    method: 'deposit_address',
+    provider: options.record.providerId,
+    reason: 'none',
+    resource: 'deposit_address',
+    to_status: options.record.status,
+  })
+}
+
+export declare namespace recordDepositAddress {
+  /** Committed deposit-address transition. */
+  type Options = {
+    /** Record before the transition, absent for creation. */
+    previous?: RoutesDepositAddresses.Record | undefined
+    /** Committed deposit-address record. */
+    record: RoutesDepositAddresses.Record
+  }
+}
+
+/** Records a bounded deposit-address cap rejection. */
+export function recordDepositAddressLimitRejection(
+  metrics: Metrics.Metrics | undefined,
+  options: recordDepositAddressLimitRejection.Options,
+) {
+  if (!metrics) return
+  metrics.count('routes_deposit_address_limit_rejection_count', 1, {
+    routes_environment: options.routesEnvironment,
+  })
+}
+
+export declare namespace recordDepositAddressLimitRejection {
+  /** Bounded dimensions for one rejected address creation. */
+  type Options = {
+    /** API-key environment whose request was rejected. */
+    routesEnvironment: 'production' | 'sandbox'
+  }
+}
+
+/** Records a bounded creation-time Routes subsidy rejection. */
+export function recordSubsidyRejection(
+  metrics: Metrics.Metrics | undefined,
+  options: recordSubsidyRejection.Options,
+) {
+  if (!metrics) return
+  metrics.count('routes_subsidy_request_rejection_count', 1, {
+    reason: options.reason,
+    routes_environment: options.routesEnvironment,
+  })
+}
+
+export declare namespace recordSubsidyRejection {
+  /** Bounded dimensions for one rejected subsidy request. */
+  type Options = {
+    /** Public reason returned to the caller. */
+    reason: 'billing_past_due' | 'billing_required' | 'routes_subsidy_not_enabled'
+    /** API-key environment whose request was rejected. */
+    routesEnvironment: 'production' | 'sandbox'
+  }
+}
+
+/** Records a committed transfer transition and its completed lifecycle segments. */
+export function recordTransfer(
+  metrics: Metrics.Metrics | undefined,
+  options: recordTransfer.Options,
+) {
+  if (!metrics) return
+  const flow = method(options.record.method)
+  const provider = options.record.providerId
+  if (options.previous?.status !== options.record.status)
+    metrics.count('routes_resource_transition_count', 1, {
+      from_status: options.previous?.status ?? 'none',
+      routes_environment: options.record.environment,
+      method: method(options.record.method),
+      provider,
+      reason: options.record.statusReason?.code ?? 'none',
+      resource: 'transfer',
+      to_status: options.record.status,
+    })
+
+  if (options.previous === undefined)
+    recordRouteQuote(metrics, {
+      destinationAmount: options.record.snapshot.destinationAmount,
+      destinationChain: options.record.snapshot.destinationChain,
+      destinationToken: options.record.snapshot.destinationToken,
+      feePayer: options.record.snapshot.subsidize ? 'tempo' : 'customer',
+      fees: options.record.snapshot.fees,
+      routesEnvironment: options.record.environment,
+      method: flow,
+      provider: options.record.snapshot.provider,
+      ...(options.selection === undefined ? {} : { selection: options.selection }),
+      sourceAmount: options.record.snapshot.sourceAmount,
+      sourceChain: options.record.snapshot.sourceChain,
+      sourceToken: options.record.snapshot.sourceToken,
+    })
+
+  if (
+    options.previous?.status === 'awaiting-source' &&
+    options.previous.version === 1 &&
+    options.record.status === 'processing'
+  )
+    lifecycle(metrics, options.record.updatedAt, options.record.createdAt, {
+      flow,
+      routes_environment: options.record.environment,
+      outcome: 'success',
+      provider,
+      segment: 'created_to_source_registered',
+    })
+  if (!options.previous?.providerDeliveredAt && options.record.providerDeliveredAt)
+    lifecycle(
+      metrics,
+      options.record.providerDeliveredAt,
+      options.previous?.statusUpdatedAt ?? options.previous?.updatedAt ?? options.record.createdAt,
+      {
+        flow,
+        routes_environment: options.record.environment,
+        outcome: 'success',
+        provider,
+        segment: 'source_registered_to_provider_delivery',
+      },
+    )
+  if (options.previous?.status !== 'processing' || options.record.status !== 'completed') return
+  const tags = routeTags({
+    routesEnvironment: options.record.environment,
+    method: flow,
+    snapshot: options.record.snapshot,
+  })
+  if (options.record.subsidyAmount) {
+    const subsidy = Economics.usd(options.record.subsidyAmount)
+    if (subsidy !== undefined) metrics.histogram('routes_tempo_subsidy_usd', subsidy, tags)
+  }
+  if (options.record.subsidyTransactionHash && options.previous.providerDeliveredAt)
+    lifecycle(metrics, options.record.updatedAt, options.previous.providerDeliveredAt, {
+      flow,
+      routes_environment: options.record.environment,
+      outcome: 'success',
+      provider,
+      segment: 'provider_delivery_to_completed',
+    })
+  lifecycle(metrics, options.record.updatedAt, options.record.createdAt, {
+    flow,
+    routes_environment: options.record.environment,
+    outcome: 'success',
+    provider,
+    segment: 'created_to_completed',
+  })
+}
+
+export declare namespace recordTransfer {
+  /** Committed transfer transition. */
+  type Options = {
+    /** Record before the transition, absent for creation. */
+    previous?: RoutesTransfers.Record | undefined
+    /** Committed transfer record. */
+    record: RoutesTransfers.Record
+    /** Automatic selection comparison retained only while creating the transfer. */
+    selection?: recordRouteQuote.Options['selection'] | undefined
+  }
+}
+
+/** Records customer economics for one selected route quote. */
+export function recordRouteQuote(
+  metrics: Metrics.Metrics | undefined,
+  options: recordRouteQuote.Options,
+) {
+  if (!metrics || !options.destinationAmount) return
+  const tags = routeTags({
+    routesEnvironment: options.routesEnvironment,
+    method: options.method,
+    snapshot: options,
+  })
+  recordCustomerValueLoss(metrics, {
+    destination: options.destinationAmount,
+    source: options.sourceAmount,
+    tags: { ...tags, stage: 'selected' },
+  })
+
+  const providerFeeUsdMetric =
+    options.feePayer === 'tempo'
+      ? 'routes_tempo_provider_fee_usd'
+      : 'routes_customer_provider_fee_usd'
+  const providerFeeTokenMetric =
+    options.feePayer === 'tempo'
+      ? 'routes_tempo_provider_fee_token_units'
+      : 'routes_customer_provider_fee_token_units'
+  const providerFeesUsd = options.fees.map((fee) => Economics.usd(fee.amount))
+  if (providerFeesUsd.every((cost) => cost !== undefined))
+    metrics.histogram(
+      providerFeeUsdMetric,
+      providerFeesUsd.reduce((total, cost) => total + cost, 0),
+      tags,
+    )
+  const providerFees = new Map<string, typeof options.fees>()
+  for (const fee of options.fees)
+    providerFees.set(fee.token.tokenKey, [...(providerFees.get(fee.token.tokenKey) ?? []), fee])
+  for (const [feeToken, fees] of providerFees) {
+    const values = fees.map((fee) => Economics.units(fee.amount))
+    if (values.every((value) => value !== undefined))
+      metrics.histogram(
+        providerFeeTokenMetric,
+        values.reduce((total, value) => total + value, 0),
+        { ...tags, fee_token: feeToken },
+      )
+  }
+
+  if (options.selection) {
+    const delta = Economics.differenceBps({
+      reference: {
+        ...options.destinationAmount,
+        baseUnits: options.selection.bestDestinationAmount,
+      },
+      value: options.destinationAmount,
+    })
+    if (delta !== undefined) metrics.histogram('routes_selection_delta_bps', delta, tags)
+  }
+}
+
+export declare namespace recordRouteQuote {
+  /** Selected quote and its bounded corridor dimensions. */
+  type Options = {
+    /** Selected expected destination amount. */
+    destinationAmount?: Economics.Amount | undefined
+    /** Destination chain selected for the route. */
+    destinationChain: Pick<Provider.ChainRef, 'id'>
+    /** Destination token selected for the route. */
+    destinationToken: Pick<Provider.TokenRef, 'tokenKey'>
+    /** Party responsible for the provider fees. */
+    feePayer: 'customer' | 'tempo'
+    /** Provider fees included in the selected quote. */
+    fees: readonly {
+      /** Fee amount in the charged token. */
+      amount: Economics.Amount
+      /** Stable key for the charged token. */
+      token: Pick<Provider.TokenRef, 'tokenKey'>
+    }[]
+    /** API-key environment that owns the route. */
+    routesEnvironment: 'production' | 'sandbox'
+    /** Route method selected by the caller. */
+    method: 'deposit_address' | 'transaction'
+    /** Provider selected for the route. */
+    provider: Pick<Provider.RoutesQuote['provider'], 'id'>
+    /** Automatic selection comparison, omitted for provider-forced requests. */
+    selection?: Selection | undefined
+    /** Requested source amount. */
+    sourceAmount: Economics.Amount
+    /** Source chain selected for the route. */
+    sourceChain: Pick<Provider.ChainRef, 'id'>
+    /** Source token selected for the route. */
+    sourceToken: Pick<Provider.TokenRef, 'tokenKey'>
+  }
+
+  /** Automatic route-selection comparison. */
+  type Selection = {
+    /** Highest expected destination amount returned by successful candidates. */
+    bestDestinationAmount: string
+  }
+}
+
+/** Returns the source-routes interval completed by initial source registration. */
+export function transferSourceRoutesInterval(
+  options: transferSourceRoutesInterval.Options,
+): LifecycleInterval | undefined {
+  if (options.record.status !== 'processing' || options.record.version !== 2) return undefined
+  return {
+    endedAt: end(
+      options.record.createdAt,
+      options.record.statusUpdatedAt ?? options.record.updatedAt,
+    ),
+    routesEnvironment: options.record.environment,
+    method: method(options.record.method),
+    provider: options.record.providerId,
+    resourceId: options.record.id,
+    stage: 'source_routes',
+    startedAt: options.record.createdAt,
+  }
+}
+
+export declare namespace transferSourceRoutesInterval {
+  /** Transfer whose initial source registration has completed. */
+  type Options = {
+    /** Registered transfer at the start of provider processing. */
+    record: RoutesTransfers.Record
+  }
+}
+
+/** Queries active durable routes state and emits zero-filled gauges. */
+export async function recordState(db: Db.Db, options: recordState.Options) {
+  const now = new Date()
+  const [addresses, deposits, overdue, transfers] = await Promise.all([
+    RoutesDepositAddresses.summarizeStates(db),
+    RoutesDeposits.summarizeStates(db, { statuses: activeDepositStatuses }),
+    RoutesDepositAddresses.summarizeOverdue(db, { now: now.toISOString() }),
+    RoutesTransfers.summarizeStates(db, { statuses: activeTransferStatuses }),
+  ])
+  const states: State[] = [
+    ...addresses.map((state) => ({
+      ...state,
+      method: 'deposit_address' as const,
+      reason: 'none',
+      resource: 'deposit_address' as const,
+    })),
+    ...deposits.map((state) => ({
+      ...state,
+      method: 'deposit_address' as const,
+      reason: state.reason ?? 'none',
+      resource: 'deposit' as const,
+    })),
+    ...transfers.map((state) => ({
+      ...state,
+      method: method(state.method),
+      reason: state.reason ?? 'none',
+      resource: 'transfer' as const,
+    })),
+  ]
+  const dimensions = new Map(
+    [...providerDimensions(options.providers), ...states].map((state) => [key(state, []), state]),
+  )
+  const statesByKey = new Map(
+    states.map((state) => [key(state, [state.status, state.reason]), state]),
+  )
+  for (const state of dimensions.values())
+    for (const status of statuses(state.resource)) {
+      const reasons = statusReasons(state.resource, status)
+      for (const reason of reasons) {
+        const current = statesByKey.get(key(state, [status, reason]))
+        const tags = {
+          routes_environment: state.environment,
+          method: state.method,
+          provider: state.providerId,
+          reason,
+          resource: state.resource,
+          status,
+        }
+        options.metrics.gauge('routes_resource_state_count', Number(current?.count ?? 0), tags)
+        options.metrics.gauge(
+          'routes_resource_oldest_state_age_ms',
+          current ? age(now, current.oldestStatusUpdatedAt) : 0,
+          tags,
+        )
+      }
+    }
+
+  const overdueByKey = new Map(
+    overdue.map((state) => [[state.providerId, state.environment].join(':'), state]),
+  )
+  for (const state of dimensions.values()) {
+    if (state.resource !== 'deposit_address') continue
+    const current = overdueByKey.get([state.providerId, state.environment].join(':'))
+    const tags = { routes_environment: state.environment, provider: state.providerId }
+    options.metrics.gauge('routes_deposit_address_overdue_count', Number(current?.count ?? 0), tags)
+    options.metrics.gauge(
+      'routes_deposit_address_oldest_overdue_age_ms',
+      current ? age(now, current.oldestNextPollAt) : 0,
+      tags,
+    )
+  }
+}
+
+export declare namespace recordState {
+  /** Durable routes state snapshot dependencies. */
+  type Options = {
+    /** Bounded operational metrics backend. */
+    metrics: Metrics.Metrics
+    /** Route providers configured by the host application. */
+    providers: readonly Provider.Provider[]
+  }
+}
+
+type Dimension = Pick<State, 'environment' | 'method' | 'providerId' | 'resource'>
+
+type State = {
+  count: string
+  environment: 'production' | 'sandbox'
+  method: 'deposit_address' | 'transaction'
+  oldestStatusUpdatedAt: string
+  providerId: string
+  reason: string
+  resource: 'deposit' | 'deposit_address' | 'transfer'
+  status: string
+}
+
+type LifecycleTags = {
+  flow: 'deposit_address' | 'transaction'
+  routes_environment: State['environment']
+  outcome: 'success'
+  provider: string
+  segment:
+    | 'created_to_completed'
+    | 'created_to_source_registered'
+    | 'detected_to_completed'
+    | 'detected_to_provider_delivery'
+    | 'provider_delivery_to_completed'
+    | 'source_registered_to_provider_delivery'
+}
+
+type RouteSnapshot = {
+  destinationChain: Pick<Provider.ChainRef, 'id'>
+  destinationToken: Pick<Provider.TokenRef, 'tokenKey'>
+  provider: Pick<Provider.RoutesQuote['provider'], 'id'>
+  sourceChain: Pick<Provider.ChainRef, 'id'>
+  sourceToken: Pick<Provider.TokenRef, 'tokenKey'>
+}
+
+type RouteTagOptions = {
+  routesEnvironment: State['environment']
+  method: State['method']
+  snapshot: RouteSnapshot
+}
+
+type RouteTags = {
+  destination_chain: string
+  destination_token: string
+  routes_environment: State['environment']
+  method: State['method']
+  provider: string
+  source_chain: string
+  source_token: string
+}
+
+type CustomerValueLossTags = RouteTags & {
+  stage: 'executed' | 'selected'
+}
+
+function recordCustomerValueLoss(
+  metrics: Metrics.Metrics,
+  options: recordCustomerValueLoss.Options,
+) {
+  if (!options.source || !options.destination) return
+  const valueLoss = Economics.differenceBps({
+    reference: options.source,
+    value: options.destination,
+  })
+  if (valueLoss !== undefined)
+    metrics.histogram('routes_customer_value_loss_bps', valueLoss, options.tags)
+}
+
+declare namespace recordCustomerValueLoss {
+  type Options = {
+    destination: Economics.Amount | undefined
+    source: Economics.Amount | undefined
+    tags: CustomerValueLossTags
+  }
+}
+
+function routeTags(options: RouteTagOptions): RouteTags {
+  return {
+    destination_chain: options.snapshot.destinationChain.id,
+    destination_token: options.snapshot.destinationToken.tokenKey,
+    routes_environment: options.routesEnvironment,
+    method: options.method,
+    provider: options.snapshot.provider.id,
+    source_chain: options.snapshot.sourceChain.id,
+    source_token: options.snapshot.sourceToken.tokenKey,
+  }
+}
+
+function lifecycle(metrics: Metrics.Metrics, end: string, start: string, tags: LifecycleTags) {
+  metrics.histogram(
+    'routes_lifecycle_duration_ms',
+    Math.max(0, Date.parse(end) - Date.parse(start)),
+    tags,
+  )
+}
+
+function end(start: string, value: string) {
+  return new Date(Math.max(Date.parse(start), Date.parse(value))).toISOString()
+}
+
+function age(now: Date, timestamp: string) {
+  return Math.max(0, now.getTime() - Date.parse(timestamp))
+}
+
+function key(state: Dimension, suffix: readonly string[]) {
+  return [state.resource, state.providerId, state.environment, state.method, ...suffix].join(':')
+}
+
+function providerDimensions(providers: readonly Provider.Provider[]): Dimension[] {
+  return providers.flatMap((provider) =>
+    environments.flatMap((environment) => [
+      ...(Provider.canCreateDepositAddress(provider)
+        ? ([
+            {
+              environment,
+              method: 'deposit_address',
+              providerId: provider.id,
+              resource: 'deposit',
+            },
+            {
+              environment,
+              method: 'deposit_address',
+              providerId: provider.id,
+              resource: 'deposit_address',
+            },
+          ] as const)
+        : []),
+      ...(Provider.canPrepareTransfer(provider)
+        ? Transfer.methods.map((value) => ({
+            environment,
+            method: method(value),
+            providerId: provider.id,
+            resource: 'transfer' as const,
+          }))
+        : []),
+    ]),
+  )
+}
+
+function method(value: Transfer.Method): State['method'] {
+  return value === 'transaction' ? 'transaction' : 'deposit_address'
+}
+
+function statuses(resource: State['resource']): readonly string[] {
+  if (resource === 'transfer') return activeTransferStatuses
+  if (resource === 'deposit') return activeDepositStatuses
+  return activeDepositAddressStatuses
+}
+
+function statusReasons(resource: State['resource'], status: string): readonly string[] {
+  if (status !== 'action-required') return ['none']
+  if (resource === 'transfer') return ['none', ...Transfer.statusReasonCodes]
+  if (resource === 'deposit') return ['none', ...Deposit.statusReasonCodes]
+  return ['none']
+}