tapimo 0.15.0 → 0.18.0

package/src/internal/MetricSink.test.ts

@@ -16,6 +16,66 @@ function describeProviderMetric(event: MetricEvent) {
 }
 
 describe('routesProviderOperations', () => {
+  test('records local skips and upstream cooldowns separately', () => {
+    const events: MetricEvent[] = []
+    const metrics = Metrics.from({
+      count(name, value, tags) {
+        events.push({ name, tags, type: 'count', value })
+      },
+      flush() {},
+      gauge() {},
+      histogram(name, value, tags) {
+        events.push({ name, tags, type: 'histogram', value })
+      },
+    })
+    const record = MetricSink.routesProviderOperations(metrics)
+
+    record({
+      durationMs: 1,
+      failure: 'rate_limit',
+      id: 'relay',
+      operation: 'getQuote',
+      outcome: 'failed',
+      rateLimitSource: 'local',
+    })
+    record({
+      cooldownMs: 2_000,
+      durationMs: 10,
+      failure: 'rate_limit',
+      id: 'squid',
+      operation: 'getQuote',
+      outcome: 'failed',
+      rateLimitSource: 'upstream',
+    })
+
+    expect(
+      events.filter(
+        (event) =>
+          event.name === 'routes_provider_rate_limit_count' ||
+          event.name === 'routes_provider_cooldown_ms',
+      ),
+    ).toEqual([
+      {
+        name: 'routes_provider_rate_limit_count',
+        tags: { operation: 'getQuote', provider: 'relay', source: 'local' },
+        type: 'count',
+        value: 1,
+      },
+      {
+        name: 'routes_provider_rate_limit_count',
+        tags: { operation: 'getQuote', provider: 'squid', source: 'upstream' },
+        type: 'count',
+        value: 1,
+      },
+      {
+        name: 'routes_provider_cooldown_ms',
+        tags: { operation: 'getQuote', provider: 'squid', source: 'upstream' },
+        type: 'histogram',
+        value: 2_000,
+      },
+    ])
+  })
+
   test('normalizes every provider operation and outcome', () => {
     const events: MetricEvent[] = []
     let flushes = 0
@@ -78,6 +138,13 @@ describe('routesProviderOperations', () => {
         operation: 'verifySourceTransaction',
         outcome: 'failed',
       },
+      {
+        durationMs: 9,
+        failure: 'network',
+        id: 'legacy-mesh',
+        operation: 'verifyDestinationDelivery',
+        outcome: 'failed',
+      },
     ] as const satisfies readonly RoutesProvider.ProviderAttemptResult[]
 
     for (const attempt of attempts) record(attempt)
@@ -100,6 +167,8 @@ describe('routesProviderOperations', () => {
         "histogram:routes_provider_operation_duration_ms:7:stargate:verifyDestinationDelivery:success:none",
         "count:routes_provider_operation_count:1:stargate:verifySourceTransaction:failed:timeout",
         "histogram:routes_provider_operation_duration_ms:8:stargate:verifySourceTransaction:failed:timeout",
+        "count:routes_provider_operation_count:1:legacy-mesh:verifyDestinationDelivery:failed:network",
+        "histogram:routes_provider_operation_duration_ms:9:legacy-mesh:verifyDestinationDelivery:failed:network",
       ]
     `)
     expect(flushes).toBe(0)
@@ -133,30 +202,76 @@ describe('requests', () => {
 
     expect(events.find((event) => event.name === 'routes_deposit_count_failure_count'))
       .toMatchInlineSnapshot(`
-      {
-        "name": "routes_deposit_count_failure_count",
-        "tags": {
-          "cache": "miss",
-          "method": "GET",
-          "principal_environment": "none",
-          "principal_type": "unknown",
-          "route": "/v1/routes/transfers",
-        },
-        "type": "count",
-        "value": 1,
-      }
-    `)
+        {
+          "name": "routes_deposit_count_failure_count",
+          "tags": {
+            "cache": "miss",
+            "credential": "none",
+            "method": "GET",
+            "principal_environment": "none",
+            "principal_type": "unknown",
+            "route": "/v1/routes/transfers",
+          },
+          "type": "count",
+          "value": 1,
+        }
+      `)
 
     expect(events.find((event) => event.name === 'routes_transfer_count_failure_count'))
       .toMatchInlineSnapshot(`
+        {
+          "name": "routes_transfer_count_failure_count",
+          "tags": {
+            "cache": "miss",
+            "credential": "none",
+            "method": "GET",
+            "principal_environment": "none",
+            "principal_type": "unknown",
+            "route": "/v1/routes/transfers",
+          },
+          "type": "count",
+          "value": 1,
+        }
+      `)
+  })
+
+  test('tags canceled requests with the presented credential', () => {
+    const events: MetricEvent[] = []
+    const metrics = Metrics.from({
+      count(name, value, tags) {
+        events.push({ name, tags, type: 'count', value })
+      },
+      flush() {},
+      gauge() {},
+      histogram() {},
+    })
+    const record = MetricSink.requests(metrics)
+
+    record({
+      credential: 'api_key',
+      duration: 15_000,
+      errorCode: 'client_canceled',
+      level: 'warn',
+      method: 'GET',
+      path: '/v1/verified-tokens',
+      requestId: 'request_private',
+      route: '/v1/verified-tokens',
+      status: 499,
+    })
+
+    expect(events.find((event) => event.name === 'http_response_count')).toMatchInlineSnapshot(`
       {
-        "name": "routes_transfer_count_failure_count",
+        "name": "http_response_count",
         "tags": {
           "cache": "miss",
+          "chain_id": "unknown",
+          "credential": "api_key",
+          "error_code": "client_canceled",
           "method": "GET",
           "principal_environment": "none",
           "principal_type": "unknown",
-          "route": "/v1/routes/transfers",
+          "route": "/v1/verified-tokens",
+          "status": 499,
         },
         "type": "count",
         "value": 1,
@@ -164,6 +279,48 @@ describe('requests', () => {
     `)
   })
 
+  test('tags response and server error counts with the request chain', () => {
+    const events: MetricEvent[] = []
+    const metrics = Metrics.from({
+      count(name, value, tags) {
+        events.push({ name, tags, type: 'count', value })
+      },
+      flush() {},
+      gauge() {},
+      histogram() {},
+    })
+    const record = MetricSink.requests(metrics)
+
+    record({
+      chainId: 42431,
+      duration: 15_076,
+      errorCode: 'upstream_error',
+      level: 'error',
+      method: 'GET',
+      path: '/v1/transactions',
+      requestId: 'request_private',
+      route: '/v1/transactions',
+      status: 502,
+    })
+
+    expect(
+      events
+        .filter(({ name }) => name === 'http_response_count' || name === 'http_server_error_count')
+        .map(({ name, tags }) => ({ chain: tags?.['chain_id'], name })),
+    ).toMatchInlineSnapshot(`
+      [
+        {
+          "chain": "42431",
+          "name": "http_response_count",
+        },
+        {
+          "chain": "42431",
+          "name": "http_server_error_count",
+        },
+      ]
+    `)
+  })
+
   test('records bounded operational failure signals', () => {
     const events: MetricEvent[] = []
     const metrics = Metrics.from({
@@ -218,6 +375,8 @@ describe('requests', () => {
           "name": "http_server_error_count",
           "tags": {
             "cache": "miss",
+            "chain_id": "unknown",
+            "credential": "none",
             "error_code": "upstream_error",
             "method": "POST",
             "principal_environment": "none",
@@ -232,6 +391,8 @@ describe('requests', () => {
           "name": "rpc_response_error_count",
           "tags": {
             "cache": "miss",
+            "chain_id": "unknown",
+            "credential": "none",
             "method": "POST",
             "principal_environment": "none",
             "principal_type": "unknown",
@@ -245,6 +406,8 @@ describe('requests', () => {
           "name": "rpc_response_server_error_count",
           "tags": {
             "cache": "miss",
+            "chain_id": "unknown",
+            "credential": "none",
             "method": "POST",
             "principal_environment": "none",
             "principal_type": "unknown",
@@ -259,6 +422,7 @@ describe('requests', () => {
           "tags": {
             "cache": "miss",
             "chain_id": "4217",
+            "credential": "none",
             "failure": "network",
             "method": "POST",
             "operation": "eth_call",
@@ -276,6 +440,7 @@ describe('requests', () => {
           "tags": {
             "cache": "miss",
             "chain_id": "4217",
+            "credential": "none",
             "failure": "http",
             "method": "POST",
             "operation": "query",
@@ -293,6 +458,7 @@ describe('requests', () => {
           "tags": {
             "cache": "miss",
             "chain_id": "4217",
+            "credential": "none",
             "method": "eth_sendRawTransaction",
             "principal_environment": "none",
             "principal_type": "unknown",
@@ -336,6 +502,8 @@ describe('requests', () => {
           "name": "http_response_count",
           "tags": {
             "cache": "miss",
+            "chain_id": "unknown",
+            "credential": "none",
             "method": "POST",
             "principal_environment": "none",
             "principal_type": "unknown",
@@ -349,6 +517,8 @@ describe('requests', () => {
           "name": "rpc_response_error_count",
           "tags": {
             "cache": "miss",
+            "chain_id": "unknown",
+            "credential": "none",
             "method": "POST",
             "principal_environment": "none",
             "principal_type": "unknown",
@@ -608,6 +778,7 @@ describe('requests', () => {
           "name": "routes_quote_request_count",
           "tags": {
             "cache": "miss",
+            "credential": "none",
             "method": "GET",
             "outcome": "available",
             "principal_environment": "none",
@@ -633,6 +804,7 @@ describe('requests', () => {
           "name": "routes_quote_age_ms",
           "tags": {
             "cache": "miss",
+            "credential": "none",
             "method": "GET",
             "principal_environment": "none",
             "principal_type": "unknown",
@@ -646,6 +818,7 @@ describe('requests', () => {
           "name": "routes_quote_age_ms",
           "tags": {
             "cache": "miss",
+            "credential": "none",
             "method": "GET",
             "principal_environment": "none",
             "principal_type": "unknown",
@@ -659,6 +832,7 @@ describe('requests', () => {
           "name": "routes_quote_future_skew_ms",
           "tags": {
             "cache": "miss",
+            "credential": "none",
             "method": "GET",
             "principal_environment": "none",
             "principal_type": "unknown",
@@ -672,6 +846,7 @@ describe('requests', () => {
           "name": "routes_quote_request_count",
           "tags": {
             "cache": "miss",
+            "credential": "none",
             "method": "GET",
             "outcome": "unsupported",
             "principal_environment": "none",

package/src/internal/MetricSink.ts

@@ -142,6 +142,7 @@ export type WebhookSink = {
 /** Low-cardinality tags shared by every HTTP metric, derived from a request log entry. */
 type RouteTags = {
   cache: 'hit' | 'miss'
+  credential: 'api_key' | 'none' | 'payment'
   method: string
   principal_environment: 'none' | 'production' | 'sandbox'
   principal_type: 'api_key' | 'public' | 'session' | 'super_admin' | 'unknown'
@@ -162,9 +163,22 @@ export function routesProviderOperations(backend: Metrics | undefined) {
       operation: attempt.operation,
       outcome,
       provider: attempt.id,
+      ...(attempt.rateLimitSource === undefined
+        ? {}
+        : { rate_limit_source: attempt.rateLimitSource }),
     }
     backend.count('routes_provider_operation_count', 1, tags)
     backend.histogram('routes_provider_operation_duration_ms', attempt.durationMs, tags)
+    if (attempt.rateLimitSource !== undefined) {
+      const rateLimitTags = {
+        operation: attempt.operation,
+        provider: attempt.id,
+        source: attempt.rateLimitSource,
+      }
+      backend.count('routes_provider_rate_limit_count', 1, rateLimitTags)
+      if (attempt.cooldownMs !== undefined)
+        backend.histogram('routes_provider_cooldown_ms', attempt.cooldownMs, rateLimitTags)
+    }
   }
 }
 
@@ -181,17 +195,26 @@ export function requests(backend: Metrics): RequestSink {
   const recordRoutesProviderOperation = routesProviderOperations(backend)
   return (entry) => {
     const tags = routeTags(entry)
+    // Request chains are validated against the supported set, so this stays
+    // bounded; alert rules use it to keep testnet failures off production paging.
+    const chain = String(entry.chainId ?? 'unknown')
 
     backend.count('http_response_count', 1, {
       ...tags,
+      chain_id: chain,
       ...(entry.errorCode === undefined ? {} : { error_code: entry.errorCode }),
       ...(entry.payment === undefined ? {} : { payment: entry.payment }),
       status: entry.status,
     })
-    backend.histogram('http_response_duration_ms', entry.duration, tags)
+    // Status is bounded per route and separates fast rejections from real work.
+    backend.histogram('http_response_duration_ms', entry.duration, {
+      ...tags,
+      status: entry.status,
+    })
     if (entry.status >= 500)
       backend.count('http_server_error_count', 1, {
         ...tags,
+        chain_id: chain,
         ...(entry.errorCode === undefined ? {} : { error_code: entry.errorCode }),
         status: entry.status,
       })
@@ -199,11 +222,13 @@ export function requests(backend: Metrics): RequestSink {
     if (entry.rpc) {
       backend.count('rpc_response_error_count', entry.rpc.errors, {
         ...tags,
+        chain_id: chain,
         rpc_error_code: rpcErrorCode(entry.rpc.code),
       })
       if ((entry.rpc.serverErrors ?? 0) > 0)
         backend.count('rpc_response_server_error_count', entry.rpc.serverErrors ?? 0, {
           ...tags,
+          chain_id: chain,
           rpc_error_code: rpcErrorCode(entry.rpc.code),
         })
     }
@@ -327,6 +352,7 @@ function rpcErrorCode(code: number | undefined) {
 function routeTags(entry: Log.Entry): RouteTags {
   return {
     cache: entry.cache ?? 'miss',
+    credential: entry.credential ?? 'none',
     method: entry.method,
     principal_environment: entry.principal?.environment ?? 'none',
     principal_type: entry.principal?.type ?? 'unknown',

package/src/internal/RateLimit.test.ts

@@ -97,9 +97,36 @@ describe('memory', () => {
   })
 })
 
+describe('window', () => {
+  test('derives the bucket, expiry, and reset from the period', () => {
+    const time = Date.parse('2026-05-25T00:00:30.250Z')
+    expect(RateLimit.window({ limit: 1, period: 'second' }, time)).toMatchInlineSnapshot(`
+      {
+        "bucket": 1779667230,
+        "expiresAt": 1779667231000,
+        "reset": 1779667231,
+      }
+    `)
+    expect(RateLimit.window({ limit: 1, period: 'minute' }, time)).toMatchInlineSnapshot(`
+      {
+        "bucket": 29661120,
+        "expiresAt": 1779667260000,
+        "reset": 1779667260,
+      }
+    `)
+  })
+})
+
 describe('edge', () => {
   function binding(success = true) {
-    return { limit: vi.fn(async (_: { key: string }) => ({ success })) }
+    const calls: string[] = []
+    return {
+      calls,
+      async limit({ key }: { key: string }) {
+        calls.push(key)
+        return { success }
+      },
+    }
   }
 
   function request(
@@ -132,9 +159,102 @@ describe('edge', () => {
     })
 
     expect(response).toBeUndefined()
-    expect(ip.limit).toHaveBeenCalledWith({ key: '203.0.113.7' })
-    expect(asn.limit).toHaveBeenCalledWith({ key: 'asn:64512' })
-    expect(global.limit).toHaveBeenCalledWith({ key: 'global' })
+    expect(ip.calls).toContain('203.0.113.7')
+    expect(asn.calls).toContain('asn:64512')
+    expect(global.calls).toContain('global')
+  })
+
+  test('does not resolve a credential partition when the global limit rejects', async () => {
+    const global = binding(false)
+    const ip = binding(false)
+    let partitionAttempts = 0
+
+    const response = await RateLimit.edge(request({ ip: '203.0.113.7' }), {
+      global,
+      ip,
+      resolvePartition: async () => {
+        partitionAttempts++
+        return 'api-key:confirmed'
+      },
+    })
+
+    expect(response?.status).toBe(429)
+    expect(global.calls).toContain('global')
+    expect(partitionAttempts).toBe(0)
+    expect(ip.calls).toContain('203.0.113.7')
+  })
+
+  test('does not resolve a credential partition when the shared network limit allows', async () => {
+    let partitionAttempts = 0
+
+    const response = await RateLimit.edge(request({ ip: '203.0.113.7' }), {
+      global: binding(),
+      ip: binding(),
+      resolvePartition: async () => {
+        partitionAttempts++
+        return 'api-key:confirmed'
+      },
+    })
+
+    expect(response).toBeUndefined()
+    expect(partitionAttempts).toBe(0)
+  })
+
+  test('retries a rejected network limit with a confirmed partition', async () => {
+    const asnCalls: string[] = []
+    const asn = {
+      async limit({ key }: { key: string }) {
+        asnCalls.push(key)
+        return { success: key !== 'asn:64512' }
+      },
+    }
+    const global = binding()
+    const ipCalls: string[] = []
+    const ip = {
+      async limit({ key }: { key: string }) {
+        ipCalls.push(key)
+        return { success: key !== '203.0.113.7' }
+      },
+    }
+
+    const response = await RateLimit.edge(request({ asn: 64512, ip: '203.0.113.7' }), {
+      asn,
+      global,
+      ip,
+      resolvePartition: async () => 'api-key:confirmed',
+    })
+
+    expect(response).toBeUndefined()
+    expect(global.calls).toContain('global')
+    expect(ipCalls).toContain('203.0.113.7')
+    const ip_partition = ipCalls[2]
+    const asn_partition = asnCalls[2]
+    expect(ip_partition).toMatch(/^[a-f0-9]{64}$/)
+    expect(asn_partition).toMatch(/^[a-f0-9]{64}$/)
+    expect(ip_partition).not.toBe(asn_partition)
+  })
+
+  test('bounds credential probes after the shared network limit rejects', async () => {
+    let partitionAttempts = 0
+    let attempts = 0
+    const ip = {
+      async limit() {
+        attempts++
+        return { success: false }
+      },
+    } satisfies RateLimit.Binding
+
+    const response = await RateLimit.edge(request({ ip: '203.0.113.7' }), {
+      ip,
+      resolvePartition: async () => {
+        partitionAttempts++
+        return 'api-key:confirmed'
+      },
+    })
+
+    expect(response?.status).toBe(429)
+    expect(attempts).toBe(2)
+    expect(partitionAttempts).toBe(0)
   })
 
   test('buckets IPv6 clients by /64 prefix', async () => {
@@ -142,7 +262,7 @@ describe('edge', () => {
 
     await RateLimit.edge(request({ ip: '2001:db8:abcd:12::1' }), { ip })
 
-    expect(ip.limit).toHaveBeenCalledWith({ key: '2001:db8:abcd:12' })
+    expect(ip.calls).toContain('2001:db8:abcd:12')
   })
 
   test('normalizes equivalent IPv6 addresses to the same /64 prefix', () => {
@@ -174,7 +294,7 @@ describe('edge', () => {
     const response = await RateLimit.edge(request({ ip: '203.0.113.7' }), { asn })
 
     expect(response).toBeUndefined()
-    expect(asn.limit).not.toHaveBeenCalled()
+    expect(asn.calls).toHaveLength(0)
   })
 
   test('uses an ASN forwarded by a trusted service binding', async () => {
@@ -182,7 +302,7 @@ describe('edge', () => {
 
     await RateLimit.edge(request({ forwardedAsn: 64512 }), { asn })
 
-    expect(asn.limit).toHaveBeenCalledWith({ key: 'asn:64512' })
+    expect(asn.calls).toContain('asn:64512')
   })
 
   test('keys unknown clients together when the ip header is missing', async () => {
@@ -190,7 +310,7 @@ describe('edge', () => {
 
     await RateLimit.edge(request(), { ip })
 
-    expect(ip.limit).toHaveBeenCalledWith({ key: 'unknown' })
+    expect(ip.calls).toContain('unknown')
   })
 
   test('fails open when a binding errors', async () => {
@@ -210,3 +330,19 @@ describe('edge', () => {
     }
   })
 })
+
+describe('defers', () => {
+  test('always waits for small limits', () => {
+    expect(RateLimit.defers({ limit: 10, period: 'minute' }, 1)).toBe(false)
+    expect(RateLimit.defers({ limit: 1, period: 'second' }, 1)).toBe(false)
+  })
+
+  test('waits within one percent of the limit, floor one request', () => {
+    expect(RateLimit.defers({ limit: 100, period: 'minute' }, 98)).toBe(true)
+    expect(RateLimit.defers({ limit: 100, period: 'minute' }, 99)).toBe(false)
+    expect(RateLimit.defers({ limit: 20, period: 'minute' }, 18)).toBe(true)
+    expect(RateLimit.defers({ limit: 20, period: 'minute' }, 19)).toBe(false)
+    expect(RateLimit.defers({ limit: 10_000, period: 'minute' }, 9_899)).toBe(true)
+    expect(RateLimit.defers({ limit: 10_000, period: 'minute' }, 9_900)).toBe(false)
+  })
+})