tapimo 0.13.0 → 0.15.0

package/src/internal/routes/Observability.test.ts

@@ -0,0 +1,636 @@
+import * as TestApp from '../../../test/App.js'
+import * as TestRoutes from '../../../test/Routes.js'
+import * as RoutesDepositAddresses from '../../db/tables/routesDepositAddresses.js'
+import * as Metrics from '../../Metrics.js'
+import * as Deposit from './Deposit.js'
+import * as DepositAddress from './DepositAddress.js'
+import * as Observability from './Observability.js'
+import * as Provider from './Provider.js'
+import * as Transfer from './Transfer.js'
+
+const db = TestApp.database()
+const now = new Date('2026-01-01T00:00:00.000Z')
+let sequence = 0
+
+type MetricEvent = {
+  name: string
+  tags: Metrics.Tags | undefined
+  type: 'count' | 'gauge' | 'histogram'
+  value: number
+}
+
+beforeEach(() => vi.useFakeTimers({ now, toFake: ['Date'] }))
+afterEach(() => vi.useRealTimers())
+
+function capture() {
+  const events: MetricEvent[] = []
+  return {
+    events,
+    metrics: Metrics.from({
+      count(name, value, tags) {
+        events.push({ name, tags, type: 'count', value })
+      },
+      flush() {},
+      gauge(name, value, tags) {
+        events.push({ name, tags, type: 'gauge', value })
+      },
+      histogram(name, value, tags) {
+        events.push({ name, tags, type: 'histogram', value })
+      },
+    }),
+  }
+}
+
+async function createAddress(provider: string) {
+  const index = sequence++
+  return DepositAddress.create(db, {
+    deliveryStrategy: 'provider',
+    environment: 'production',
+    orgId: `org_${index}`,
+    providerOutputToken: TestRoutes.providerOutputToken(),
+    snapshot: TestRoutes.depositAddressSnapshot({
+      provider: { id: provider, name: provider },
+      recipient: `0x${index.toString(16).padStart(40, '0')}`,
+    }),
+  })
+}
+
+function metricProvider(id: string) {
+  return Provider.from({
+    createDepositAddress: async () => {
+      throw new Error('Not called.')
+    },
+    getQuote: async () => ({ sampledAt: now.toISOString(), status: 'unavailable' }),
+    id,
+    name: id,
+    prepareTransfer: async () => {
+      throw new Error('Not called.')
+    },
+  })
+}
+
+describe('recordDeposit', () => {
+  test('records provider delivery before an action-required recovery', async () => {
+    const { events, metrics } = capture()
+    const address = await createAddress('deposit-lifecycle')
+    const detected = await Deposit.create(db, {
+      providerRequestId: 'request_deposit_lifecycle',
+      snapshot: TestRoutes.depositSnapshot(address.id),
+      sourceTransactionHash: 'source_deposit_lifecycle',
+    })
+    Observability.recordDeposit(metrics, { record: detected })
+
+    vi.setSystemTime(new Date(now.getTime() + 73_000))
+    const actionRequired = await Deposit.transition(db, {
+      expectedVersion: detected.version,
+      id: detected.id,
+      providerDeliveredAt: new Date().toISOString(),
+      status: 'action-required',
+      statusReason: {
+        code: 'subsidy_balance_unavailable',
+        message: 'Tempo lacks enough pathUSD for the required subsidy.',
+      },
+    })
+    if (actionRequired.type !== 'applied') throw new Error('Expected action to be required.')
+    Observability.recordDeposit(metrics, {
+      previous: detected,
+      providerEvidenceComplete: true,
+      record: actionRequired.record,
+    })
+
+    vi.setSystemTime(new Date(now.getTime() + 75_000))
+    const settling = await Deposit.transition(db, {
+      expectedVersion: actionRequired.record.version,
+      id: detected.id,
+      status: 'settling',
+    })
+    if (settling.type !== 'applied') throw new Error('Expected the deposit to start settling.')
+    Observability.recordDeposit(metrics, {
+      previous: actionRequired.record,
+      providerEvidenceComplete: true,
+      record: settling.record,
+    })
+
+    vi.setSystemTime(new Date(now.getTime() + 79_000))
+    const completed = await Deposit.transition(db, {
+      expectedVersion: settling.record.version,
+      id: detected.id,
+      snapshot: {
+        ...settling.record.snapshot,
+        destinationAmount: {
+          baseUnits: '999000',
+          currency: 'USD',
+          decimals: 6,
+          formatted: '0.999',
+        },
+      },
+      status: 'completed',
+      subsidyAmount: {
+        baseUnits: '1000',
+        currency: 'USD',
+        decimals: 6,
+        formatted: '0.001',
+      },
+      tempoGasPaid: '21000',
+    })
+    if (completed.type !== 'applied') throw new Error('Expected the deposit to complete.')
+    Observability.recordDeposit(metrics, { previous: settling.record, record: completed.record })
+
+    expect(
+      events.map(
+        ({ name, tags, value }) =>
+          `${name}:${tags?.['from_status'] ?? tags?.['segment']}:${tags?.['to_status'] ?? value}:${value}`,
+      ),
+    ).toMatchInlineSnapshot(`
+      [
+        "routes_resource_transition_count:none:detected:1",
+        "routes_resource_transition_count:detected:action-required:1",
+        "routes_lifecycle_duration_ms:detected_to_provider_delivery:73000:73000",
+        "routes_resource_transition_count:action-required:settling:1",
+        "routes_resource_transition_count:settling:completed:1",
+        "routes_customer_value_loss_bps:undefined:10:10",
+        "routes_tempo_subsidy_usd:undefined:0.001:0.001",
+        "routes_tempo_gas_paid_base_units:undefined:21000:21000",
+        "routes_lifecycle_duration_ms:provider_delivery_to_completed:6000:6000",
+        "routes_lifecycle_duration_ms:detected_to_completed:79000:79000",
+      ]
+    `)
+  })
+})
+
+describe('recordDepositAddress', () => {
+  test('records creation after the address is durable', async () => {
+    const { events, metrics } = capture()
+    const record = await createAddress('address-lifecycle')
+    Observability.recordDepositAddress(metrics, { record })
+
+    expect(events).toMatchInlineSnapshot(`
+      [
+        {
+          "name": "routes_resource_transition_count",
+          "tags": {
+            "from_status": "none",
+            "method": "deposit_address",
+            "provider": "address-lifecycle",
+            "reason": "none",
+            "resource": "deposit_address",
+            "routes_environment": "production",
+            "to_status": "active",
+          },
+          "type": "count",
+          "value": 1,
+        },
+      ]
+    `)
+  })
+})
+
+describe('recordDepositAddressLimitRejection', () => {
+  test('records only the bounded route environment', () => {
+    const { events, metrics } = capture()
+
+    Observability.recordDepositAddressLimitRejection(metrics, {
+      routesEnvironment: 'production',
+    })
+
+    expect(events).toMatchInlineSnapshot(`
+      [
+        {
+          "name": "routes_deposit_address_limit_rejection_count",
+          "tags": {
+            "routes_environment": "production",
+          },
+          "type": "count",
+          "value": 1,
+        },
+      ]
+    `)
+  })
+})
+
+describe('recordSubsidyRejection', () => {
+  test('records only bounded policy dimensions', () => {
+    const { events, metrics } = capture()
+
+    Observability.recordSubsidyRejection(metrics, {
+      reason: 'routes_subsidy_not_enabled',
+      routesEnvironment: 'production',
+    })
+
+    expect(events).toMatchInlineSnapshot(`
+      [
+        {
+          "name": "routes_subsidy_request_rejection_count",
+          "tags": {
+            "reason": "routes_subsidy_not_enabled",
+            "routes_environment": "production",
+          },
+          "type": "count",
+          "value": 1,
+        },
+      ]
+    `)
+  })
+})
+
+describe('depositAddressSourceRoutesInterval', () => {
+  test('ends an address journey when its first provider request reports source funds', async () => {
+    const address = await createAddress('address-source-routes')
+
+    expect(
+      Observability.depositAddressSourceRoutesInterval({
+        address,
+        request: {
+          createdAt: '2026-01-01T00:00:12.000Z',
+          destinationTransactionHashes: [],
+          id: 'request_source_routes',
+          refundTransactionHashes: [],
+          sourceTransactionHashes: [`0x${'aa'.repeat(32)}`],
+          status: 'pending',
+          updatedAt: '2026-01-01T00:00:13.000Z',
+        },
+      }),
+    ).toEqual({
+      endedAt: '2026-01-01T00:00:12.000Z',
+      routesEnvironment: 'production',
+      method: 'deposit_address',
+      provider: 'address-source-routes',
+      resourceId: address.id,
+      stage: 'source_routes',
+      startedAt: '2026-01-01T00:00:00.000Z',
+    })
+  })
+})
+
+describe('recordRouteQuote', () => {
+  test('separates Tempo-paid provider fees from customer fees', () => {
+    const { events, metrics } = capture()
+    const snapshot = TestRoutes.transferSnapshot({
+      fees: [
+        {
+          amount: { baseUnits: '20705', currency: 'USD', decimals: 6, formatted: '0.020705' },
+          side: 'source',
+          token: TestRoutes.transferSnapshot().sourceToken,
+          type: 'provider',
+        },
+      ],
+    })
+
+    Observability.recordRouteQuote(metrics, {
+      destinationAmount: snapshot.destinationAmount,
+      destinationChain: snapshot.destinationChain,
+      destinationToken: snapshot.destinationToken,
+      feePayer: 'tempo',
+      fees: snapshot.fees,
+      routesEnvironment: 'production',
+      method: 'deposit_address',
+      provider: snapshot.provider,
+      sourceAmount: snapshot.sourceAmount,
+      sourceChain: snapshot.sourceChain,
+      sourceToken: snapshot.sourceToken,
+    })
+
+    expect(events.map(({ name, tags, value }) => ({ feeToken: tags?.['fee_token'], name, value })))
+      .toMatchInlineSnapshot(`
+        [
+          {
+            "feeToken": undefined,
+            "name": "routes_customer_value_loss_bps",
+            "value": 207.05,
+          },
+          {
+            "feeToken": undefined,
+            "name": "routes_tempo_provider_fee_usd",
+            "value": 0.020705,
+          },
+          {
+            "feeToken": "eip155:8453/erc20:0x833589fcd6edb6e08f4c7c32d4f71b54bda02913",
+            "name": "routes_tempo_provider_fee_token_units",
+            "value": 0.020705,
+          },
+        ]
+      `)
+  })
+})
+
+describe('recordTransfer', () => {
+  test('records provider delivery and settlement as non-overlapping intervals once', async () => {
+    const { events, metrics } = capture()
+    const created = await Transfer.create(db, {
+      apiKeyId: 'key_transfer_lifecycle',
+      environment: 'production',
+      orgId: 'org_transfer_lifecycle',
+      snapshot: TestRoutes.transferSnapshot({
+        fees: [
+          {
+            amount: { baseUnits: '20705', currency: 'USD', decimals: 6, formatted: '0.020705' },
+            side: 'source',
+            token: TestRoutes.transferSnapshot().sourceToken,
+            type: 'provider',
+          },
+        ],
+        method: 'depositAddress',
+        provider: { id: 'transfer-lifecycle', name: 'Transfer Lifecycle' },
+      }),
+    })
+    Observability.recordTransfer(metrics, {
+      record: created,
+      selection: { bestDestinationAmount: '989295' },
+    })
+
+    vi.setSystemTime(new Date(now.getTime() + 12_000))
+    const processing = await Transfer.transition(db, {
+      expectedVersion: created.version,
+      id: created.id,
+      status: 'processing',
+    })
+    if (processing.type !== 'applied') throw new Error('Expected source registration.')
+    Observability.recordTransfer(metrics, { previous: created, record: processing.record })
+    expect(Observability.transferSourceRoutesInterval({ record: processing.record })).toMatchObject(
+      {
+        endedAt: '2026-01-01T00:00:12.000Z',
+        method: 'deposit_address',
+        resourceId: created.id,
+        stage: 'source_routes',
+        startedAt: '2026-01-01T00:00:00.000Z',
+      },
+    )
+    expect(Observability.transferSourceRoutesInterval({ record: created })).toBeUndefined()
+
+    vi.setSystemTime(new Date(now.getTime() + 85_000))
+    const providerDelivered = await Transfer.recordProviderDelivery(db, { id: created.id })
+    if (providerDelivered.type !== 'recorded') throw new Error('Expected provider delivery.')
+    Observability.recordTransfer(metrics, {
+      previous: processing.record,
+      record: providerDelivered.record,
+    })
+
+    vi.setSystemTime(new Date(now.getTime() + 90_000))
+    const retried = await Transfer.recordProviderDelivery(db, { id: created.id })
+    if (retried.type !== 'replayed') throw new Error('Expected provider delivery retry.')
+    Observability.recordTransfer(metrics, {
+      previous: providerDelivered.record,
+      record: retried.record,
+    })
+
+    vi.setSystemTime(new Date(now.getTime() + 120_000))
+    const completed = await Transfer.transition(db, {
+      expectedVersion: processing.record.version,
+      id: created.id,
+      status: 'completed',
+      subsidyTransactionHash: `0x${'aa'.repeat(32)}`,
+    })
+    if (completed.type !== 'applied') throw new Error('Expected subsidy settlement.')
+    Observability.recordTransfer(metrics, { previous: retried.record, record: completed.record })
+
+    expect(
+      events
+        .filter(({ name }) =>
+          [
+            'routes_customer_provider_fee_token_units',
+            'routes_customer_provider_fee_usd',
+            'routes_customer_value_loss_bps',
+            'routes_selection_delta_bps',
+          ].includes(name),
+        )
+        .map(({ name, tags, value }) => ({ name, stage: tags?.['stage'], value })),
+    ).toMatchInlineSnapshot(`
+      [
+        {
+          "name": "routes_customer_value_loss_bps",
+          "stage": "selected",
+          "value": 207.05,
+        },
+        {
+          "name": "routes_customer_provider_fee_usd",
+          "stage": undefined,
+          "value": 0.020705,
+        },
+        {
+          "name": "routes_customer_provider_fee_token_units",
+          "stage": undefined,
+          "value": 0.020705,
+        },
+        {
+          "name": "routes_selection_delta_bps",
+          "stage": undefined,
+          "value": 101.082,
+        },
+      ]
+    `)
+
+    expect(
+      events
+        .filter(({ name }) => name === 'routes_lifecycle_duration_ms')
+        .map(({ tags, value }) => ({
+          flow: tags?.['flow'],
+          segment: tags?.['segment'],
+          value,
+        })),
+    ).toMatchInlineSnapshot(`
+      [
+        {
+          "flow": "deposit_address",
+          "segment": "created_to_source_registered",
+          "value": 12000,
+        },
+        {
+          "flow": "deposit_address",
+          "segment": "source_registered_to_provider_delivery",
+          "value": 73000,
+        },
+        {
+          "flow": "deposit_address",
+          "segment": "provider_delivery_to_completed",
+          "value": 35000,
+        },
+        {
+          "flow": "deposit_address",
+          "segment": "created_to_completed",
+          "value": 120000,
+        },
+      ]
+    `)
+  })
+
+  test('does not record a settlement interval for an unsubsidized direct delivery', async () => {
+    const { events, metrics } = capture()
+    const created = await Transfer.create(db, {
+      apiKeyId: 'key_transfer_direct_delivery',
+      environment: 'production',
+      orgId: 'org_transfer_direct_delivery',
+      snapshot: TestRoutes.transferSnapshot({ subsidize: false }),
+    })
+    vi.setSystemTime(new Date(now.getTime() + 12_000))
+    const processing = await Transfer.transition(db, {
+      expectedVersion: created.version,
+      id: created.id,
+      status: 'processing',
+    })
+    if (processing.type !== 'applied') throw new Error('Expected source registration.')
+    Observability.recordTransfer(metrics, { previous: created, record: processing.record })
+
+    vi.setSystemTime(new Date(now.getTime() + 85_000))
+    const providerDelivered = await Transfer.recordProviderDelivery(db, { id: created.id })
+    if (providerDelivered.type !== 'recorded') throw new Error('Expected provider delivery.')
+    Observability.recordTransfer(metrics, {
+      previous: processing.record,
+      record: providerDelivered.record,
+    })
+    const completed = await Transfer.transition(db, {
+      expectedVersion: processing.record.version,
+      id: created.id,
+      status: 'completed',
+    })
+    if (completed.type !== 'applied') throw new Error('Expected direct delivery.')
+    Observability.recordTransfer(metrics, {
+      previous: providerDelivered.record,
+      record: completed.record,
+    })
+
+    expect(
+      events
+        .filter(({ name }) => name === 'routes_lifecycle_duration_ms')
+        .map(({ tags, value }) => ({ segment: tags?.['segment'], value })),
+    ).toMatchInlineSnapshot(`
+      [
+        {
+          "segment": "created_to_source_registered",
+          "value": 12000,
+        },
+        {
+          "segment": "source_registered_to_provider_delivery",
+          "value": 73000,
+        },
+        {
+          "segment": "created_to_completed",
+          "value": 85000,
+        },
+      ]
+    `)
+  })
+})
+
+describe('recordState', () => {
+  test('zero-fills known active states and measures overdue address checks', async () => {
+    const { events, metrics } = capture()
+    const provider = metricProvider('state-snapshot')
+    const address = await createAddress(provider.id)
+    await createAddress('leased-snapshot')
+    const detected = await Deposit.create(db, {
+      providerRequestId: 'request_state_snapshot',
+      snapshot: TestRoutes.depositSnapshot(address.id),
+      sourceTransactionHash: 'source_state_snapshot',
+    })
+    const bridging = await Deposit.transition(db, {
+      expectedVersion: detected.version,
+      id: detected.id,
+      status: 'bridging',
+    })
+    if (bridging.type !== 'applied') throw new Error('Expected a bridging deposit.')
+    vi.setSystemTime(new Date(now.getTime() + 30_000))
+    const refreshed = await Deposit.transition(db, {
+      expectedVersion: bridging.record.version,
+      id: detected.id,
+      providerState: { observed: true },
+      status: 'bridging',
+    })
+    if (refreshed.type !== 'applied') throw new Error('Expected a refreshed deposit.')
+    vi.setSystemTime(now)
+    const transfer = await Transfer.create(db, {
+      apiKeyId: 'key_state_snapshot',
+      environment: 'production',
+      orgId: 'org_state_snapshot',
+      snapshot: TestRoutes.transferSnapshot({
+        provider: { id: 'state-snapshot', name: 'State Snapshot' },
+      }),
+    })
+    const processing = await Transfer.transition(db, {
+      expectedVersion: transfer.version,
+      id: transfer.id,
+      status: 'processing',
+    })
+    if (processing.type !== 'applied') throw new Error('Expected a processing transfer.')
+    const historical = await Transfer.create(db, {
+      apiKeyId: 'key_state_snapshot_historical',
+      environment: 'production',
+      orgId: 'org_state_snapshot_historical',
+      snapshot: TestRoutes.transferSnapshot({
+        provider: { id: 'state-snapshot', name: 'State Snapshot' },
+      }),
+    })
+    const expired = await Transfer.transition(db, {
+      expectedVersion: historical.version,
+      id: historical.id,
+      status: 'expired',
+    })
+    if (expired.type !== 'applied') throw new Error('Expected terminal history.')
+
+    vi.setSystemTime(new Date(now.getTime() + 60_000))
+    await RoutesDepositAddresses.claimDue(db, {
+      leaseUntil: new Date(now.getTime() + 120_000).toISOString(),
+      limit: 1,
+      now: new Date().toISOString(),
+      providerId: 'leased-snapshot',
+    })
+    await Observability.recordState(db, { metrics, providers: [provider] })
+    const selected = events.filter(
+      (event) =>
+        (event.tags?.['routes_environment'] === 'production' &&
+          (event.tags['resource'] !== 'transfer' || event.tags['method'] === 'transaction') &&
+          event.tags?.['provider'] === 'state-snapshot' &&
+          (event.name.includes('overdue') ||
+            ['active', 'awaiting-source', 'bridging', 'detected', 'processing'].includes(
+              String(event.tags?.['status']),
+            ))) ||
+        (event.tags?.['provider'] === 'leased-snapshot' && event.name.includes('overdue')),
+    )
+
+    expect(
+      selected
+        .map(
+          ({ name, tags, value }) =>
+            `${name}:${tags?.['provider']}:${tags?.['resource'] ?? 'deposit_address'}:${tags?.['status'] ?? 'overdue'}:${value}`,
+        )
+        .toSorted(),
+    ).toMatchInlineSnapshot(`
+      [
+        "routes_deposit_address_oldest_overdue_age_ms:leased-snapshot:deposit_address:overdue:0",
+        "routes_deposit_address_oldest_overdue_age_ms:state-snapshot:deposit_address:overdue:60000",
+        "routes_deposit_address_overdue_count:leased-snapshot:deposit_address:overdue:0",
+        "routes_deposit_address_overdue_count:state-snapshot:deposit_address:overdue:1",
+        "routes_resource_oldest_state_age_ms:state-snapshot:deposit:bridging:60000",
+        "routes_resource_oldest_state_age_ms:state-snapshot:deposit:detected:0",
+        "routes_resource_oldest_state_age_ms:state-snapshot:deposit_address:active:60000",
+        "routes_resource_oldest_state_age_ms:state-snapshot:transfer:awaiting-source:0",
+        "routes_resource_oldest_state_age_ms:state-snapshot:transfer:processing:60000",
+        "routes_resource_state_count:state-snapshot:deposit:bridging:1",
+        "routes_resource_state_count:state-snapshot:deposit:detected:0",
+        "routes_resource_state_count:state-snapshot:deposit_address:active:1",
+        "routes_resource_state_count:state-snapshot:transfer:awaiting-source:0",
+        "routes_resource_state_count:state-snapshot:transfer:processing:1",
+      ]
+    `)
+    expect(events.some(({ tags }) => tags?.['status'] === 'expired')).toBe(false)
+
+    const completed = await Transfer.transition(db, {
+      expectedVersion: processing.record.version,
+      id: processing.record.id,
+      status: 'completed',
+    })
+    if (completed.type !== 'applied') throw new Error('Expected a completed transfer.')
+    const drained = capture()
+    await Observability.recordState(db, { metrics: drained.metrics, providers: [provider] })
+    expect(
+      drained.events.find(
+        ({ name, tags }) =>
+          name === 'routes_resource_state_count' &&
+          tags?.['provider'] === provider.id &&
+          tags['routes_environment'] === 'production' &&
+          tags['method'] === 'transaction' &&
+          tags['resource'] === 'transfer' &&
+          tags['status'] === 'processing',
+      ),
+    ).toMatchObject({ value: 0 })
+  })
+})