tapimo 0.15.0 → 0.18.0

package/src/internal/routes/Quota.ts

@@ -0,0 +1,226 @@
+import * as Store from '../Store.js'
+import { ProviderRateLimitError, ProviderResponseError } from './Provider.js'
+
+type TenantQuota = {
+  /** Requests accepted for this tenant in the active window. */
+  count: number
+  /** Stable non-secret tenant identity. */
+  key: string
+}
+
+type State = {
+  /** Epoch milliseconds before which new requests are rejected. */
+  blockedUntil?: number | undefined
+  /** Fixed-window number. */
+  bucket: number
+  /** Requests accepted globally in the active window. */
+  count: number
+  /** Per-tenant counters for accepted requests. */
+  tenants: readonly TenantQuota[]
+}
+
+type Native = {
+  block(options: block.Options): Promise<void>
+  check(options: check.Options): Promise<void>
+  consume(options: consume.Options): Promise<void>
+}
+
+const nativeStores = new WeakMap<Store.Store, Native>()
+const checks = new WeakMap<Store.Store, Map<string, { blockedUntil?: number; refreshAt: number }>>()
+const updates = new WeakMap<Store.Store, Map<string, Promise<string>>>()
+// Adaptive-only providers refresh shared cooldown state at most once per second per isolate.
+const checkRefreshMs = 1_000
+
+/** Creates a quota store whose state transitions execute in one Durable Object RPC. */
+export function durableObject(
+  namespace: Store.durableObject.Namespace,
+  options: durableObject.Options,
+) {
+  const store = Store.durableObject(namespace, options)
+  // Durable Object stubs are request-scoped I/O objects in Workers.
+  const stub = () => namespace.getByName(options.name) as NativeStub
+  nativeStores.set(store, {
+    block: (parameters) => stub().routesQuotaBlock(parameters),
+    async check(parameters) {
+      if (!(await stub().routesQuotaCheck(parameters))) throw new ProviderRateLimitError()
+    },
+    async consume(parameters) {
+      if (!(await stub().routesQuotaConsume(parameters))) throw new ProviderRateLimitError()
+    },
+  })
+  return store
+}
+
+export declare namespace durableObject {
+  /** One Durable Object instance dedicated to a provider quota. */
+  type Options = { name: string }
+}
+
+type NativeStub = Store.durableObject.Stub & {
+  routesQuotaBlock(options: block.Options): Promise<void>
+  routesQuotaCheck(options: check.Options): Promise<boolean>
+  routesQuotaConsume(options: consume.Options): Promise<boolean>
+}
+
+/** Rejects while an upstream provider quota is in a reported cooldown. */
+export async function check(store: Store.Store, options: check.Options) {
+  const time = options.now.getTime()
+  const cache = checks.get(store) ?? new Map()
+  if (!checks.has(store)) checks.set(store, cache)
+  const cached = cache.get(options.key)
+  if (cached?.blockedUntil !== undefined && cached.blockedUntil > time)
+    throw new ProviderRateLimitError()
+  if (cached !== undefined && cached.refreshAt > time) return
+
+  const native = nativeStores.get(store)
+  if (native) await native.check(options)
+  else await checkStored(store, options)
+  // A cooldown installed during the check takes precedence over its stale success.
+  if (cache.get(options.key) === cached)
+    cache.set(options.key, { refreshAt: time + checkRefreshMs })
+}
+
+async function checkStored(store: Store.Store, options: check.Options) {
+  const current = await store.get(options.key)
+  if (current === null) return
+  const state = JSON.parse(current) as State
+  if (state.blockedUntil !== undefined && state.blockedUntil > options.now.getTime())
+    throw new ProviderRateLimitError()
+}
+
+export declare namespace check {
+  /** One provider cooldown check. */
+  type Options = Pick<block.Options, 'key' | 'now'>
+}
+
+/** Atomically reserves one request from an upstream provider quota. */
+export async function consume(store: Store.Store, options: consume.Options) {
+  const native = nativeStores.get(store)
+  if (native) return native.consume(options)
+  const bucket = Math.floor(options.now.getTime() / options.windowMs)
+  await update(store, options.key, (current) => {
+    const stored = current === null ? undefined : (JSON.parse(current) as State)
+    const activeBucket = Math.max(bucket, stored?.bucket ?? bucket)
+    const quota: State =
+      stored?.bucket === activeBucket
+        ? stored
+        : {
+            ...(stored?.blockedUntil === undefined ? {} : { blockedUntil: stored.blockedUntil }),
+            bucket: activeBucket,
+            count: 0,
+            tenants: [],
+          }
+    if (quota.blockedUntil !== undefined && quota.blockedUntil > options.now.getTime())
+      throw new ProviderRateLimitError()
+    const tenant = options.tenant
+      ? quota.tenants.find((entry) => entry.key === options.tenant?.key)
+      : undefined
+    if (
+      quota.count >= options.limit ||
+      (options.tenant && (tenant?.count ?? 0) >= options.tenant.limit)
+    )
+      throw new ProviderRateLimitError()
+    const tenants = !options.tenant
+      ? quota.tenants
+      : tenant
+        ? quota.tenants.map((entry) =>
+            entry.key === options.tenant?.key ? { ...entry, count: entry.count + 1 } : entry,
+          )
+        : [...quota.tenants, { count: 1, key: options.tenant.key }]
+    return JSON.stringify({ bucket: activeBucket, count: quota.count + 1, tenants })
+  })
+}
+
+/** Blocks an upstream quota after a provider reports rate-limit exhaustion. */
+export async function block(store: Store.Store, options: block.Options) {
+  const time = options.now.getTime()
+  const blockedUntil = time + options.durationMs
+  const cache = checks.get(store) ?? new Map()
+  if (!checks.has(store)) checks.set(store, cache)
+  cache.set(options.key, { blockedUntil, refreshAt: blockedUntil })
+  const native = nativeStores.get(store)
+  if (native) return native.block(options)
+  const bucket = Math.floor(time / options.windowMs)
+  await update(store, options.key, (current) => {
+    const stored = current === null ? undefined : (JSON.parse(current) as State)
+    const activeBucket = Math.max(bucket, stored?.bucket ?? bucket)
+    const quota: State =
+      stored?.bucket === activeBucket
+        ? stored
+        : {
+            ...(stored?.blockedUntil === undefined ? {} : { blockedUntil: stored.blockedUntil }),
+            bucket: activeBucket,
+            count: 0,
+            tenants: [],
+          }
+    return JSON.stringify({
+      ...quota,
+      blockedUntil: Math.max(quota.blockedUntil ?? 0, blockedUntil),
+    })
+  })
+}
+
+export declare namespace block {
+  /** One provider-reported quota cooldown. */
+  type Options = {
+    /** Cooldown duration derived from `Retry-After` or the local window. */
+    durationMs: number
+    /** Durable key shared by every call that consumes this upstream bucket. */
+    key: string
+    /** Current wall clock. */
+    now: Date
+    /** Fixed-window duration in milliseconds. */
+    windowMs: number
+  }
+}
+
+/** Applies a shared cooldown when an upstream provider responds with HTTP 429. */
+export async function blockOnRateLimit(
+  store: Store.Store,
+  cause: unknown,
+  options: blockOnRateLimit.Options,
+) {
+  if (!(cause instanceof ProviderResponseError) || cause.status !== 429) return
+  const durationMs = cause.retryAfterMs ?? options.windowMs
+  await block(store, {
+    ...options,
+    durationMs,
+  })
+  cause.cooldownMs = durationMs
+}
+
+export declare namespace blockOnRateLimit {
+  /** Provider quota bucket receiving an upstream cooldown. */
+  type Options = Omit<block.Options, 'durationMs'>
+}
+
+export declare namespace consume {
+  /** One upstream quota reservation. */
+  type Options = {
+    /** Durable key shared by every call that consumes this upstream bucket. */
+    key: string
+    /** Maximum global count accepted for this operation. */
+    limit: number
+    /** Current wall clock. */
+    now: Date
+    /** Optional per-tenant fairness limit within the same window. */
+    tenant?: { key: string; limit: number } | undefined
+    /** Fixed-window duration in milliseconds. */
+    windowMs: number
+  }
+}
+
+// Serialize local transitions so bursts cannot exhaust the bounded CAS retries.
+// Production runs this queue inside the provider's Durable Object.
+async function update(store: Store.Store, key: string, fn: (current: null | string) => string) {
+  const pending = updates.get(store) ?? new Map<string, Promise<string>>()
+  if (!updates.has(store)) updates.set(store, pending)
+  const previous = pending.get(key) ?? Promise.resolve()
+  const next = previous.catch(() => {}).then(() => Store.update(store, key, fn))
+  pending.set(key, next)
+  try {
+    await next
+  } finally {
+    if (pending.get(key) === next) pending.delete(key)
+  }
+}

package/src/internal/routes/Reconciliation.local.test.ts

@@ -8,9 +8,12 @@ import { Abis, Actions, Addresses } from 'viem/tempo'
 
 import * as Organizations from '../../db/tables/organizations.js'
 import * as RoutesCatalog from '../../db/tables/routesCatalog.js'
+import * as RoutesDepositAddresses from '../../db/tables/routesDepositAddresses.js'
 import * as RoutesDeposits from '../../db/tables/routesDeposits.js'
+import * as RoutesPolicies from '../../db/tables/routesPolicies.js'
 import * as RoutesSubsidies from '../../db/tables/routesSubsidies.js'
 import * as StripeCustomers from '../../db/tables/stripeCustomers.js'
+import * as Metrics from '../../Metrics.js'
 import * as TestApp from '../../../test/App.js'
 import * as Runtime from '../../../test/runtime.js'
 import * as TestRelay from '../../../test/Relay.js'
@@ -19,6 +22,7 @@ import * as Viem from '../Viem.js'
 import * as Chain from './Chain.js'
 import * as DepositAddress from './DepositAddress.js'
 import * as Evidence from './Evidence.js'
+import * as Policy from './Policy.js'
 import * as Reconciliation from './Reconciliation.js'
 import * as Subsidy from './Subsidy.js'
 
@@ -31,6 +35,287 @@ const chain = {
 } as const
 
 describe.skipIf(runtime.mode !== 'localnet')('reconcileRequest', () => {
+  test.each([
+    { subsidyPayer: 'tempo', delivered: 950_000n, lateAt: 'completed' },
+    { subsidyPayer: 'enterprise', delivered: 950_000n, lateAt: 'completed' },
+    { subsidyPayer: 'enterprise', delivered: 950_000n, lateAt: 'settling' },
+    { subsidyPayer: 'enterprise', delivered: 960_000n, lateAt: 'completed' },
+  ] as const)(
+    'settles pinned $subsidyPayer terms with $delivered delivery and late expense at $lateAt',
+    async ({ subsidyPayer, delivered, lateAt }) => {
+      const db = TestApp.database()
+      const client = TestTempo.getClient()
+      const source = TestTempo.accounts[4]!
+      const depositAccount = TestTempo.accounts[5]!
+      const subsidyAccount = TestTempo.accounts[6]!
+      const recipient = TestTempo.accounts[7]!
+      const provider = TestTempo.accounts[0]!
+      const org = await Organizations.create(db, { name: `Pinned ${subsidyPayer}` })
+      const scope = { environment: 'production', orgId: org.id } as const
+      const token = {
+        address: Addresses.pathUsd,
+        currency: 'USD',
+        decimals: 6,
+        name: 'pathUSD',
+        standard: 'TIP-20',
+        symbol: 'pathUSD',
+        tokenKey: `${chain.id}/erc20:${Addresses.pathUsd}`,
+        verified: true,
+      } as const
+      const commitment: Policy.Commitment = {
+        maxAmount: '0.05',
+        scope,
+        subsidyPayer,
+        versions: [1],
+      }
+      const grandfathered =
+        subsidyPayer === 'enterprise' && lateAt === 'completed' && delivered === 950_000n
+      await RoutesPolicies.save(db, {
+        policy: { maxAmount: '0.05', subsidyEnabled: true, subsidyPayer },
+        scope,
+        version: 0,
+      })
+      await RoutesSubsidies.upsertOrganization(db, {
+        frequency: 'tx',
+        maxAmount: '0.05',
+        orgId: org.id,
+      })
+      await StripeCustomers.create(db, { orgId: org.id, stripeCustomerId: `cus_${subsidyPayer}` })
+      await StripeCustomers.setStatus(db, org.id, 'active')
+      await RoutesCatalog.publish(db, {
+        chainTokens: [],
+        chains: [
+          {
+            aliases: [],
+            id: chain.id as Chain.Id,
+            name: chain.name,
+            parentChainId: null,
+            rpcUrls: [runtime.rpcUrl!],
+            slug: 'tempo-localnet',
+          },
+        ],
+        routes: [],
+        tokens: [],
+      })
+      const address = await DepositAddress.create(db, {
+        ...scope,
+        deliveryStrategy: 'provider',
+        providerOutputToken: token,
+        ...(grandfathered
+          ? {}
+          : {
+              providerState: {
+                routePolicy: commitment,
+                routePolicyKey: Policy.commitmentKey(commitment),
+              },
+            }),
+        snapshot: {
+          address: depositAccount.address,
+          destinationChain: chain,
+          destinationToken: token,
+          provider: { id: 'local', name: 'Local' },
+          recipient: recipient.address,
+          refundAddress: source.address,
+          sourceChain: chain,
+          sourceToken: token,
+          subsidize: true,
+        },
+      })
+      if (grandfathered)
+        await RoutesDepositAddresses.grandfatherLegacySponsored(db, {
+          addressIds: [address.id],
+          apply: true,
+          commitment,
+        })
+
+      const replacementCommitment: Policy.Commitment = {
+        ...commitment,
+        maxAmount: '0.15',
+        subsidyPayer: subsidyPayer === 'tempo' ? 'enterprise' : 'tempo',
+        versions: [2],
+      }
+      await RoutesPolicies.save(db, {
+        policy: {
+          maxAmount: '0.15',
+          subsidyEnabled: true,
+          subsidyPayer: replacementCommitment.subsidyPayer,
+        },
+        scope,
+        version: 1,
+      })
+      const replacement = await DepositAddress.create(db, {
+        ...scope,
+        deliveryStrategy: 'provider',
+        providerOutputToken: token,
+        providerState: {
+          routePolicy: replacementCommitment,
+          routePolicyKey: Policy.commitmentKey(replacementCommitment),
+        },
+        snapshot: { ...address.snapshot, address: TestTempo.accounts[8]!.address },
+      })
+
+      // Revoke before a deposit arrives, not just after its settlement has been prepared.
+      await RoutesPolicies.save(db, {
+        policy: {
+          access: [],
+          maxAmount: '0.01',
+          subsidyEnabled: false,
+          subsidyPayer: subsidyPayer === 'tempo' ? 'enterprise' : 'tempo',
+          suspended: true,
+        },
+        scope,
+        version: 2,
+      })
+      await RoutesSubsidies.removeOrganization(db, org.id)
+      await StripeCustomers.setStatus(db, org.id, 'past_due')
+      await Actions.faucet.fundSync(client, { account: source, timeout: 60_000 })
+      await Actions.faucet.fundSync(client, { account: subsidyAccount, timeout: 60_000 })
+      const before = await Actions.token.getBalance(client, {
+        account: recipient.address,
+        decimals: 6,
+        token: token.address,
+      })
+      const sourceReceipt = await Actions.token.transferSync(
+        TestTempo.getClient({ account: source }),
+        {
+          amount: 1_000_000n,
+          to: depositAccount.address,
+          token: token.address,
+        },
+      )
+      const providerReceipt = await Actions.token.transferSync(
+        TestTempo.getClient({ account: provider }),
+        {
+          amount: delivered,
+          to: recipient.address,
+          token: token.address,
+        },
+      )
+      const request = {
+        createdAt: new Date().toISOString(),
+        destinationTransactionHashes: [providerReceipt.receipt.transactionHash],
+        id: `pinned-${subsidyPayer}`,
+        refundTransactionHashes: [],
+        sourceTransactionHashes: [sourceReceipt.receipt.transactionHash],
+        status: 'success',
+        updatedAt: new Date().toISOString(),
+      }
+      const subsidy = {
+        amount: { baseUnits: '10000', currency: 'USD', decimals: 6, formatted: '0.01' },
+        provider: 'relay',
+        token,
+      }
+      const counts: Parameters<Metrics.Metrics['count']>[] = []
+      const options = {
+        address: (await RoutesDepositAddresses.getByProviderAddress(db, {
+          address: depositAccount.address,
+          providerId: 'local',
+        }))!,
+        metrics: Metrics.from({
+          count(...parameters) {
+            counts.push(parameters)
+          },
+          flush() {},
+          gauge() {},
+          histogram() {},
+        }),
+        request,
+        subsidies: Subsidy.create({
+          account: subsidyAccount,
+          getClient: TestTempo.readClient(),
+          policy: { maxAmount: '0.000001' },
+        }),
+        verifyTransfers: Evidence.createVerifier({ db, confirmations: { [chain.id]: 1 } }),
+      }
+      await Reconciliation.reconcileRequest(db, options)
+      const [prepared] = await RoutesDeposits.listByAddress(db, scope, address.id, { limit: 1 })
+      const localExpense = 1_000_000n - delivered
+      expect(prepared).toMatchObject({
+        depositAddressId: address.id,
+        status: 'settling',
+        subsidyAmount: { baseUnits: String(localExpense) },
+      })
+      if (!prepared?.settlementTransactionHash) throw new Error('Expected a prepared settlement.')
+      await waitForTransactionReceipt(client, {
+        hash: prepared.settlementTransactionHash as Hex.Hex,
+      })
+      await expect
+        .poll(async () => {
+          await Reconciliation.reconcileRequest(db, {
+            ...options,
+            request: { ...request, ...(lateAt === 'settling' ? { subsidy } : {}) },
+          })
+          return (await RoutesDeposits.get(db, prepared.id))?.status
+        })
+        .toBe('completed')
+      const completed = await RoutesDeposits.get(db, prepared.id)
+      expect(completed).toMatchObject({
+        settlementTransactionHash: prepared.settlementTransactionHash,
+        snapshot: { destinationAmount: { baseUnits: '1000000' } },
+        subsidyAmount: { baseUnits: String(localExpense) },
+      })
+      expect((await RoutesDeposits.listUnreportedSubsidies(db)).map(({ id }) => id)).toEqual(
+        subsidyPayer === 'enterprise' ? [prepared.id] : [],
+      )
+      expect(
+        (await RoutesDepositAddresses.get(db, address.id))?.providerState?.['routePolicy'],
+      ).toEqual(commitment)
+
+      await Reconciliation.reconcileRequest(db, options)
+      expect(await RoutesDeposits.get(db, prepared.id)).toEqual(completed)
+      const after = await Actions.token.getBalance(client, {
+        account: recipient.address,
+        decimals: 6,
+        token: token.address,
+      })
+      expect(after.amount - before.amount).toBe(1_000_000n)
+
+      const lateOptions = {
+        ...options,
+        request: { ...request, subsidy },
+      }
+      await Reconciliation.reconcileRequest(db, lateOptions)
+      const billed = await RoutesDeposits.listUnreportedSubsidies(db)
+      const expense = billed
+        .flatMap((row) => RoutesDeposits.subsidies(row))
+        .reduce((sum, subsidy) => sum + BigInt(subsidy.amount.baseUnits), 0n)
+      expect(expense).toBeLessThanOrEqual(50_000n)
+      const overCap = localExpense + 10_000n > 50_000n
+      const recorded = await RoutesDeposits.get(db, prepared.id)
+      expect(recorded).toMatchObject({
+        status: 'completed',
+        providerSubsidy: overCap ? null : subsidy,
+        ...(overCap ? { providerState: { unbilledProviderSubsidy: subsidy } } : {}),
+      })
+      const rejection = counts.filter(
+        ([, , tags]) => tags?.['outcome'] === 'provider_subsidy_unbilled',
+      )
+      expect(rejection).toHaveLength(overCap ? 1 : 0)
+      await Reconciliation.reconcileRequest(db, lateOptions)
+      expect(await RoutesDeposits.get(db, prepared.id)).toEqual(recorded)
+      expect(
+        counts.filter(([, , tags]) => tags?.['outcome'] === 'provider_subsidy_unbilled'),
+      ).toHaveLength(overCap ? 1 : 0)
+      expect(await RoutesDepositAddresses.get(db, replacement.id)).toEqual(replacement)
+      expect(
+        await RoutesDeposits.listByDepositAddress(db, {
+          depositAddress: replacement.address,
+          limit: 10,
+          owner: scope,
+        }),
+      ).toEqual([])
+      expect(
+        (
+          await RoutesDeposits.listByDepositAddress(db, {
+            depositAddress: address.address,
+            limit: 10,
+            owner: scope,
+          })
+        ).map(({ id }) => id),
+      ).toEqual([prepared.id])
+    },
+  )
+
   test('settles an underfilled deposit through the native DEX', async ({ onTestFinished }) => {
     // Vitest retries reuse module state, so each attempt needs a fresh database.
     const db = TestApp.database()