tapimo 0.15.1 → 0.18.0

package/src/apps/management/Billing.ts

@@ -13,7 +13,6 @@ import type * as Scope from '../../Scope.js'
 import * as SponsoredTransactions from '../../db/tables/sponsoredTransactions.js'
 import type * as Store from '../../internal/Store.js'
 import * as StripeCustomers from '../../db/tables/stripeCustomers.js'
-import * as Viem from '../../internal/Viem.js'
 
 /**
  * Whether the organization has any active billing source — the production
@@ -449,12 +448,18 @@ export class DuplicateSubscriptionError extends Error {
   }
 }
 
+/** Safe Stripe meter-event ingestion window below its documented 35-day limit. */
+const meterEventIngestionWindowMs = 34 * 24 * 60 * 60 * 1_000
+
+/** Safe retry window below Stripe's documented 24-hour identifier uniqueness period. */
+const meterEventRetryWindowMs = 23 * 60 * 60 * 1_000
+
 /**
  * Creates the sponsorship and Route Subsidy metering job. Each tick reports
  * finalized usage, then marks each row after Stripe acknowledges it.
  */
 export function createReporter(options: createReporter.Options): createReporter.Reporter {
-  const { batchSize = 100, chainIds = [Viem.chainId.mainnet], db, stripe } = options
+  const { batchSize = 100, db, stripe } = options
   return {
     async tick() {
       // Resolve per tick so Workers factories build a fresh Hyperdrive-pooled
@@ -462,7 +467,10 @@ export function createReporter(options: createReporter.Options): createReporter.
       const database = Db.get(db)
       const [depositSubsidies, sponsorships, transferSubsidies] = await Promise.all([
         RoutesDeposits.listUnreportedSubsidies(database, { limit: batchSize }),
-        SponsoredTransactions.listUnreported(database, { chainIds, limit: batchSize }),
+        SponsoredTransactions.claimUnreported(database, {
+          attemptedAt: new Date().toISOString(),
+          limit: batchSize,
+        }),
         RoutesTransfers.listUnreportedSubsidies(database, { limit: batchSize }),
       ])
       const rows: UsageRow[] = [
@@ -532,7 +540,7 @@ export function createReporter(options: createReporter.Options): createReporter.
             // just-created meter item bills nothing before its start, so
             // its backlog stamps at now too.
             const finalizedAt = new Date(row.finalizedAt!).getTime()
-            const recent = Date.now() - finalizedAt < 34 * 24 * 60 * 60 * 1000
+            const recent = Date.now() - finalizedAt < meterEventIngestionWindowMs
             const item = subscription.items[kind]
             const stamp =
               recent && item !== undefined && !item.created && finalizedAt >= item.createdAt * 1_000
@@ -540,6 +548,19 @@ export function createReporter(options: createReporter.Options): createReporter.
             meteredAt = new Date(
               (timestamp ?? Math.floor(Date.now() / 1_000)) * 1_000,
             ).toISOString()
+            if (row.source === 'sponsorship') {
+              const firstAttemptedAt = await SponsoredTransactions.beginMeterEvent(
+                database,
+                row.id,
+                new Date().toISOString(),
+              )
+              // Stripe only deduplicates identifiers for 24 hours. An older
+              // uncertain acknowledgment requires reconciliation before resend.
+              if (Date.now() - new Date(firstAttemptedAt).getTime() >= meterEventRetryWindowMs)
+                throw new Error(
+                  `Stripe acknowledgement is ambiguous after the safe meter-event retry window for ${row.id}`,
+                )
+            }
             // The row id makes short-window retries idempotent on Stripe's
             // side; `meter_reported_at` is the durable exactly-once guard.
             try {
@@ -556,7 +577,20 @@ export function createReporter(options: createReporter.Options): createReporter.
             } catch (error) {
               // Stripe may persist the event before the Worker records its ack.
               // Its duplicate response proves this exact row was already metered.
-              if (!isDuplicateMeterEvent(error, row.id)) throw error
+              if (!isDuplicateMeterEvent(error, row.id)) {
+                // An explicit client error rejects this submission. Only
+                // uncertain outcomes retain the first-send retry clock.
+                if (
+                  row.source === 'sponsorship' &&
+                  error instanceof Stripe.errors.StripeError &&
+                  error.statusCode !== undefined &&
+                  error.statusCode >= 400 &&
+                  error.statusCode < 500 &&
+                  error.statusCode !== 409
+                )
+                  await SponsoredTransactions.clearRejectedMeterEventAttempt(database, row.id)
+                throw error
+              }
             }
             reported++
           } else if (row.source === 'routes-deposit' && row.orgId.startsWith('key_')) {
@@ -564,16 +598,11 @@ export function createReporter(options: createReporter.Options): createReporter.
             // Stripe customer. Provider subsidies on those routes are non-billable.
             meteredAt = new Date().toISOString()
             skipped++
-          } else if (row.source !== 'sponsorship') {
-            // Production Routes subsidies were authorized against active billing.
-            // Keep the liability durable if that customer disappears before reporting.
-            throw new Error(`Stripe customer unavailable for Routes subsidy ${row.id}`)
           } else {
-            // Deleted orgs and Stripe-deleted (canceled) customers keep their
-            // ledger rows; nobody remains to invoice.
-            skipped++
+            // Missing billing identity cannot discharge a committed liability; preserve it for retry and alert through the scheduled-task failure counter.
+            throw new Error(`Stripe customer unavailable for ${row.source} ${row.id}`)
           }
-          // Mark only after the Stripe ack (or a definitive no-customer skip).
+          // Mark only after the Stripe ack or the non-billable one-off-key exemption.
           const reportedAt = new Date().toISOString()
           if (row.source !== 'sponsorship') {
             if (!meteredAt) throw new Error(`Routes subsidy ${row.id} has no meter timestamp`)
@@ -635,10 +664,8 @@ function isDuplicateMeterEvent(error: unknown, identifier: string): boolean {
 export declare namespace createReporter {
   /** Options for {@link createReporter}. */
   type Options = {
-    /** Maximum rows reported per source per tick. @default 100 */
+    /** Maximum rows attempted per source per tick. @default 100 */
     batchSize?: number | undefined
-    /** Chains whose spend is metered. @default mainnet */
-    chainIds?: readonly number[] | undefined
     /** Database holding sponsorship rows, or a factory resolved per tick (Workers/Hyperdrive). */
     db: Db.Source
     /** Stripe client used to send meter events. */
@@ -647,7 +674,7 @@ export declare namespace createReporter {
 
   /** A scheduled metering job. */
   type Reporter = {
-    /** Runs one reporting pass. Failed rows stay unmarked and retry next tick. */
+    /** Runs one reporting pass. Failed rows stay unmarked and retry on later ticks. */
     tick(): Promise<{ failed: number; reported: number; skipped: number }>
   }
 }
@@ -729,14 +756,36 @@ export function createRequestUsageReporter(
         const blocked = new Set<string>()
         for (const event of await RequestUsage.listPending(database, environment)) {
           try {
-            await client.billing.meterEvents.create({
-              event_name: meters.apiRequests.eventName,
-              identifier: event.identifier,
-              payload: {
-                stripe_customer_id: event.stripeCustomerId,
-                value: String(event.deltaCount),
-              },
-            })
+            const eventTimestamp = event.eventTimestamp
+            if (
+              nowMs - new Date(event.firstAttemptedAt).getTime() >= meterEventRetryWindowMs ||
+              (eventTimestamp !== undefined &&
+                nowMs - new Date(eventTimestamp).getTime() >= meterEventIngestionWindowMs)
+            ) {
+              failed++
+              blocked.add(bucketKey(event.orgId, event.bucketStart))
+              await RequestUsage.markError(
+                database,
+                event.identifier,
+                'Stripe acknowledgement is ambiguous after the safe meter-event retry window.',
+              )
+              continue
+            }
+            try {
+              await client.billing.meterEvents.create({
+                event_name: meters.apiRequests.eventName,
+                identifier: event.identifier,
+                payload: {
+                  stripe_customer_id: event.stripeCustomerId,
+                  value: String(event.deltaCount),
+                },
+                ...(eventTimestamp === undefined
+                  ? {}
+                  : { timestamp: Math.floor(new Date(eventTimestamp).getTime() / 1_000) }),
+              })
+            } catch (error) {
+              if (!isDuplicateMeterEvent(error, event.identifier)) throw error
+            }
             if (await RequestUsage.settle(database, event.identifier, new Date().toISOString()))
               reported++
           } catch (error) {
@@ -781,9 +830,17 @@ export function createRequestUsageReporter(
             // The observed watermark is stable for every tick claiming this delta.
             const sequence = watermark?.reportedCount ?? 0
             const identifier = usageIdentifier(environment, orgId, bucketStart, sequence)
+            const item = subscription.items.apiRequests
+            if (!item) throw new Error(`API request meter item unavailable for ${orgId}`)
+            // Meter events before the subscription item's creation are accepted
+            // but not invoiced, so clamp backlog to its billing eligibility window.
+            const eventTimestamp = new Date(
+              Math.max(new Date(bucketStart).getTime(), item.createdAt * 1_000),
+            ).toISOString()
             const inserted = await RequestUsage.insertPending(database, {
               ...key,
               deltaCount: delta,
+              eventTimestamp,
               identifier,
               sequence,
               stripeCustomerId: customer.stripeCustomerId,
@@ -791,14 +848,11 @@ export function createRequestUsageReporter(
             // A concurrent tick already claimed this delta; leave it to that run.
             if (!inserted) continue
 
-            // Backdate to the bucket hour unless the item was just created (it
-            // bills nothing before its start, so its backlog stamps at now).
-            const stamp = !subscription.items.apiRequests?.created
             await client.billing.meterEvents.create({
               event_name: meters.apiRequests.eventName,
               identifier,
               payload: { stripe_customer_id: customer.stripeCustomerId, value: String(delta) },
-              ...(stamp ? { timestamp: Math.floor(new Date(bucketStart).getTime() / 1000) } : {}),
+              timestamp: Math.floor(new Date(eventTimestamp).getTime() / 1_000),
             })
             if (await RequestUsage.settle(database, identifier, new Date().toISOString()))
               reported++

package/src/apps/management/routes/api-keys.test.ts

@@ -1010,7 +1010,7 @@ describe('keys', () => {
       {
         "error": {
           "code": "api_key_forbidden",
-          "message": "API key missing required scope",
+          "message": "API key is missing required permissions: data:read. Use an API key with these scopes.",
         },
       }
     `)

package/src/apps/management/routes/api-keys.ts

@@ -7,6 +7,7 @@ import * as ApiKey from '../../../ApiKey.js'
 import * as ApiKeys from '../../../ApiKeys.js'
 import type * as App from '../../../App.js'
 import * as Db from '../../../db/Db.js'
+import * as RoutesPolicies from '../../../db/tables/routesPolicies.js'
 import * as Auth from '../../../internal/Auth.js'
 import * as Billing from '../Billing.js'
 import * as OpenApi from '../../../internal/OpenApi.js'
@@ -84,7 +85,7 @@ export namespace schema {
           .array(z.string().check(z.minLength(1)))
           .check(
             z.describe(
-              "Granted scopes. '*' and non-self-serve scopes appear only on keys minted by the super admin.",
+              "Granted scopes. '*' requires the super admin; a CLI can derive Routes writes from a provisioned project key.",
             ),
             z.meta({ examples: [['data:read']] }),
           ),
@@ -205,6 +206,14 @@ export namespace schema {
             z.describe('Unique scopes to grant.'),
             z.meta({ examples: [['data:read']] }),
           ),
+        sourceKeyId: z.optional(
+          z
+            .string()
+            .check(
+              z.describe('Approved project key to derive a scoped CLI credential from.'),
+              z.meta({ examples: ['key_1a2b3c4d5e6f7g8h9j0k1m2n'] }),
+            ),
+        ),
       })
       .check(z.describe('Fields for minting an API key.')),
     'CreateApiKeyRequest',
@@ -384,6 +393,7 @@ export function apiKeys(options: apiKeys.Options = {}) {
         if (!kv) return notEnabled(c)
         const createdBy = mintedBy(c)
         if (!createdBy) return ManagementError.sessionRequired(c)
+        const principal = Auth.getPrincipal(c)
 
         const body = c.req.valid('json')
 
@@ -395,7 +405,23 @@ export function apiKeys(options: apiKeys.Options = {}) {
           })
 
         // Session issuers are limited to the finite self-serve catalog, so uniqueness bounds the vector without a new policy cap.
-        if (!canIssue(c, body.scopes))
+        const source =
+          body.sourceKeyId && principal?.type === 'session' && principal.cli
+            ? await ApiKeys.get(kv.store, body.sourceKeyId, { scopeCatalog: c.get('scopeCatalog') })
+            : undefined
+        if (
+          body.sourceKeyId &&
+          (!source ||
+            source.orgId !== Auth.org(c).id ||
+            source.projectId !== Auth.project(c).id ||
+            source.environment !== body.environment)
+        )
+          return Response.error(c, {
+            code: 'scope_not_issuable',
+            message: 'Select an approved API key from this project and environment.',
+            status: 403,
+          })
+        if (!canIssue(c, body.scopes, source ?? undefined))
           return Response.error(c, {
             code: 'scope_not_issuable',
             message: 'The requested scopes are not issuable.',
@@ -416,14 +442,27 @@ export function apiKeys(options: apiKeys.Options = {}) {
             {
               createdBy,
               environment: body.environment,
-              ...(body.allowedIps === undefined ? {} : { allowedIps: body.allowedIps }),
+              ...(principal?.type === 'session' && principal.cli
+                ? { expiresAt: principal.cli.expiresAt }
+                : {}),
+              ...(!source && body.allowedIps !== undefined ? { allowedIps: body.allowedIps } : {}),
               ...(billingActive === undefined ? {} : { billingActive }),
               ...(body.name === undefined ? {} : { name: body.name }),
               orgId,
               projectId: Auth.project(c).id,
               scopes: body.scopes,
             },
-            { scopeCatalog: c.get('scopeCatalog') },
+            {
+              ...(source
+                ? {
+                    policySourceApiKeyId: source.id,
+                    sourceScopes: body.scopes.filter(
+                      (scope) => !Scope.selfServeFrom(c.get('scopeCatalog')).includes(scope),
+                    ),
+                  }
+                : {}),
+              scopeCatalog: c.get('scopeCatalog'),
+            },
           )
           return c.json(
             Response.validated(schema.createApiKey.Response, { ...serializeKey(record), token }),
@@ -432,6 +471,12 @@ export function apiKeys(options: apiKeys.Options = {}) {
         } catch (cause) {
           if (cause instanceof ApiKeys.OwnerDeletedError) return Auth.ensureProjectError(c)
           if (cause instanceof ApiKeys.LiveKeyLimitError) return keyLimitReached(c)
+          if (cause instanceof RoutesPolicies.InvalidScopeError)
+            return Response.error(c, {
+              code: 'scope_not_issuable',
+              message: 'The approved source key is no longer available.',
+              status: 403,
+            })
           return Response.upstream(c, cause)
         }
       },
@@ -637,6 +682,7 @@ export function apiKeys(options: apiKeys.Options = {}) {
             })
           return await rotate(c, kv.store, source, createdBy)
         } catch (cause) {
+          if (cause instanceof RoutesPolicies.InvalidScopeError) return keyNotFound(c)
           if (cause instanceof ApiKeys.OwnerDeletedError) return Auth.ensureOrgError(c)
           if (cause instanceof ApiKeys.LiveKeyLimitError) return keyLimitReached(c)
           return Response.upstream(c, cause)
@@ -848,6 +894,7 @@ export function apiKeys(options: apiKeys.Options = {}) {
             })
           return await rotate(c, kv.store, source, createdBy)
         } catch (cause) {
+          if (cause instanceof RoutesPolicies.InvalidScopeError) return keyNotFound(c)
           if (cause instanceof ApiKeys.OwnerDeletedError) return Auth.ensureProjectError(c)
           if (cause instanceof ApiKeys.LiveKeyLimitError) return keyLimitReached(c)
           return Response.upstream(c, cause)
@@ -967,14 +1014,27 @@ function mintedBy(c: Context<App.Environment>) {
 }
 
 /** Checks whether the caller may grant every requested scope. */
-function canIssue(c: Context<App.Environment>, scopes: readonly Scope.Id[]) {
+function canIssue(
+  c: Context<App.Environment>,
+  scopes: readonly Scope.Id[],
+  source?: ApiKeys.Record,
+) {
   const principal = Auth.getPrincipal(c)
   if (principal?.type === 'super_admin') return true
   if (principal?.type === 'session') {
     const selfServe = Scope.selfServeFrom(c.get('scopeCatalog'))
     return (
-      scopes.every((scope) => selfServe.includes(scope)) &&
-      (!scopes.includes('management:write') || Auth.membership(c)?.role === 'owner')
+      scopes.every(
+        (scope) =>
+          selfServe.includes(scope) ||
+          (principal.cli &&
+            source &&
+            scope === 'routes:write' &&
+            (source.scopes.includes(scope) || source.scopes.includes(Scope.wildcard))),
+      ) &&
+      (!principal.cli || scopes.every((scope) => principal.cli!.scopes.includes(scope))) &&
+      ((!scopes.includes('management:write') && !scopes.includes('routes:write')) ||
+        Auth.membership(c)?.role === 'owner')
     )
   }
   return false
@@ -986,8 +1046,11 @@ function canRotate(c: Context<App.Environment>, scopes: readonly Scope.Id[]) {
   if (principal?.type === 'super_admin') return true
   if (principal?.type !== 'session') return false
   return (
-    (!scopes.includes('management:write') && !scopes.includes(Scope.wildcard)) ||
-    Auth.membership(c)?.role === 'owner'
+    (!principal.cli || scopes.every((scope) => principal.cli!.scopes.includes(scope))) &&
+    ((!scopes.includes('management:write') &&
+      !scopes.includes('routes:write') &&
+      !scopes.includes(Scope.wildcard)) ||
+      Auth.membership(c)?.role === 'owner')
   )
 }
 
@@ -998,6 +1061,8 @@ async function rotate(
   source: ApiKeys.Record,
   createdBy: string,
 ) {
+  const principal = Auth.getPrincipal(c)
+  const expiresAt = principal?.type === 'session' ? principal.cli?.expiresAt : undefined
   const billingActive =
     source.environment === 'sandbox'
       ? await Billing.active(Db.get(c.get('db')), source.orgId, 'sandbox')
@@ -1013,11 +1078,20 @@ async function rotate(
       ...(source.projectId === undefined ? {} : { projectId: source.projectId }),
       scopes: [...new Set(source.scopes)],
       ...(billingActive === undefined ? {} : { billingActive }),
-      ...(source.expiresAt === undefined ? {} : { expiresAt: source.expiresAt }),
+      ...(expiresAt || source.expiresAt
+        ? {
+            expiresAt: [expiresAt, source.expiresAt]
+              .filter(Boolean)
+              .sort((a, b) => Date.parse(a!) - Date.parse(b!))[0]!,
+          }
+        : {}),
       ...(source.name === undefined ? {} : { name: source.name }),
       ...(source.rateLimits === undefined ? {} : { rateLimits: source.rateLimits }),
     },
-    { scopeCatalog: c.get('scopeCatalog') },
+    {
+      policySourceApiKeyId: source.id,
+      scopeCatalog: c.get('scopeCatalog'),
+    },
   )
   return c.json(
     Response.validated(schema.rotateApiKey.Response, { ...serializeKey(record), token }),

package/src/apps/management/routes/billing.test.ts

@@ -1636,16 +1636,18 @@ describe.runIf(TestStripe.secretKey)('stripe sandbox (metering)', () => {
     const finalize = async (input: { feeAmount: string; orgId: string; seed: string }) => {
       const row = await SponsoredTransactions.upsert(db, { ...meterable(input.orgId), signPayload: `0x${input.seed.repeat(32)}` }) // prettier-ignore
       await SponsoredTransactions.finalize(db, row.id, { feeAmount: input.feeAmount, finalizedAt: new Date().toISOString() }) // prettier-ignore
+      return row
     }
     await finalize({ feeAmount: '100000', orgId: 'org_1', seed: '41' })
     await finalize({ feeAmount: '250000', orgId: 'org_1', seed: '42' })
-    // A deleted org's row marks without a meter event; a pending row waits.
-    await finalize({ feeAmount: '900000', orgId: 'org_ghost', seed: '43' })
+    // A missing customer's liability stays unreported for retry; a pending row waits.
+    const ghost = await finalize({ feeAmount: '900000', orgId: 'org_ghost', seed: '43' })
     await SponsoredTransactions.upsert(db, { ...meterable('org_1'), signPayload: `0x${'44'.repeat(32)}` }) // prettier-ignore
 
     const reporter = core_Billing.createReporter({ db, stripe })
-    expect(await reporter.tick()).toStrictEqual({ failed: 0, reported: 2, skipped: 1 })
-    expect(await reporter.tick()).toStrictEqual({ failed: 0, reported: 0, skipped: 0 })
+    expect(await reporter.tick()).toStrictEqual({ failed: 1, reported: 2, skipped: 0 })
+    expect(await reporter.tick()).toStrictEqual({ failed: 1, reported: 0, skipped: 0 })
+    expect((await SponsoredTransactions.get(db, ghost.id))?.meterReportedAt).toBeNull()
   })
 
   test('the metered invoice preview matches reported spend', { timeout: 240_000 }, async () => {
@@ -1689,6 +1691,68 @@ describe.runIf(TestStripe.secretKey)('stripe sandbox (metering)', () => {
     expect(line?.amount).toBe(35)
   })
 
+  test.each(['request-first', 'sponsorship-first'] as const)(
+    'request backlog reaches the invoice when the %s reporter provisions meters',
+    { timeout: 240_000 },
+    async (ordering) => {
+      const db = TestApp.database()
+      const stripe = TestStripe.client()
+      const orgId = `org_${nanoid(10)}`
+      const customer = await stripe.customers.create({
+        metadata: { orgId },
+        name: `Test Org ${nanoid(10)}`,
+      })
+      TestStripe.track(customer.id)
+      await StripeCustomers.create(db, { orgId, stripeCustomerId: customer.id })
+      const bucket = new Date(Date.now() - 2 * 60 * 60 * 1_000)
+      bucket.setUTCMinutes(0, 0, 0)
+      const bucketStart = bucket.toISOString()
+      const reportRequests = () =>
+        core_Billing
+          .createRequestUsageReporter({
+            db,
+            readCounts: async () => [{ bucketStart, count: 100, orgId }],
+            stripe,
+          })
+          .tick()
+      const reportSponsorship = async () => {
+        const seed = ordering === 'request-first' ? '61' : '62'
+        const row = await SponsoredTransactions.upsert(db, {
+          ...meterable(orgId),
+          signPayload: `0x${seed.repeat(32)}`,
+        })
+        await SponsoredTransactions.finalize(db, row.id, {
+          feeAmount: '100000',
+          finalizedAt: new Date().toISOString(),
+        })
+        return core_Billing.createReporter({ db, stripe }).tick()
+      }
+
+      if (ordering === 'request-first') {
+        expect(await reportRequests()).toStrictEqual({ failed: 0, reported: 1, skipped: 0 })
+        expect(await reportSponsorship()).toStrictEqual({ failed: 0, reported: 1, skipped: 0 })
+      } else {
+        expect(await reportSponsorship()).toStrictEqual({ failed: 0, reported: 1, skipped: 0 })
+        expect(await reportRequests()).toStrictEqual({ failed: 0, reported: 1, skipped: 0 })
+      }
+
+      const { priceId } = await core_Billing.ensureFixtures(stripe, 'apiRequests')
+      const [subscription] = (
+        await stripe.subscriptions.list({ customer: customer.id, limit: 1, price: priceId })
+      ).data
+      let line: { amount: number; quantity: number | null } | undefined
+      await until(async () => {
+        const preview = await stripe.invoices.createPreview({
+          customer: customer.id,
+          subscription: subscription!.id,
+        })
+        line = preview.lines.data.find((item) => item.pricing?.price_details?.price === priceId)
+        return line?.quantity === 100
+      }, 180_000)
+      expect(line).toMatchObject({ amount: 1, quantity: 100 })
+    },
+  )
+
   test.each(['past_due', 'canceled'] as const)(
     'a failed metered invoice blocks a %s subscription until payment',
     { timeout: 600_000 },