payment-kit 1.27.2 → 1.29.0

package/api/src/libs/timing.ts

@@ -0,0 +1,35 @@
+/**
+ * Cross-environment phase timing helper.
+ *
+ * In CF Workers, writes phase durations to `globalThis.__d1Timing__.phases`
+ * which the worker middleware reads and emits as Server-Timing headers.
+ *
+ * In Node.js (Blocklet Server), this is a no-op.
+ */
+
+/**
+ * Measure the duration of an async operation and record it under a named phase.
+ * Usage:
+ *   const result = await measurePhase('chain', () => checkTokenBalance(...));
+ *
+ * Multiple calls to the same phase name accumulate.
+ */
+export async function measurePhase<T>(name: string, fn: () => Promise<T>): Promise<T> {
+  const t0 = typeof performance !== 'undefined' ? performance.now() : Date.now();
+  try {
+    return await fn();
+  } finally {
+    const t = (globalThis as any).__d1Timing__;
+    if (t && t.phases) {
+      const dur = (typeof performance !== 'undefined' ? performance.now() : Date.now()) - t0;
+      t.phases[name] = (t.phases[name] || 0) + dur;
+    }
+  }
+}
+
+/** Record a phase duration directly (for non-Promise paths or manual timing). */
+export function recordPhase(name: string, durationMs: number): void {
+  const t = (globalThis as any).__d1Timing__;
+  if (!t || !t.phases) return;
+  t.phases[name] = (t.phases[name] || 0) + durationMs;
+}

package/api/src/libs/util.ts

@@ -4,7 +4,7 @@ import { buffer } from 'node:stream/consumers';
 import { getUrl } from '@blocklet/sdk/lib/component';
 import { env } from '@blocklet/sdk/lib/env';
 import { getWalletDid } from '@blocklet/sdk/lib/did';
-import { toStakeAddress } from '@arcblock/did-util';
+import { toStakeAddress } from '@arcblock/did-util/cbor';
 import { customAlphabet } from 'nanoid';
 import type { LiteralUnion } from 'type-fest';
 import { joinURL, withQuery, withTrailingSlash } from 'ufo';
@@ -13,8 +13,7 @@ import axios from 'axios';
 import { ethers } from 'ethers';
 import { fromUnitToToken } from '@ocap/util';
 import get from 'lodash/get';
-import numbro from 'numbro';
-import { trimEnd } from 'lodash';
+import trimEnd from 'lodash/trimEnd';
 import dayjs from './dayjs';
 import { blocklet, wallet } from './auth';
 import type { PaymentCurrency, PaymentMethod, Subscription } from '../store/models';
@@ -31,6 +30,19 @@ export const CHECKOUT_SESSION_TTL = 6 * 60 * 60; // expires in 6 hours, then rem
 
 export const STRIPE_API_VERSION = '2023-08-16';
 export const STRIPE_ENDPOINT: string = getUrl('/api/integrations/stripe/webhook');
+// Pub/Sub OIDC tokens carry the push-subscription endpoint as `aud` claim. When
+// the dev server is reached via a custom domain (e.g. a Cloudflare tunnel), the
+// BLOCKLET_APP_URL-derived default won't match the URL Google signs against.
+// Set GOOGLE_PLAY_WEBHOOK_URL to the externally-visible URL in that case.
+// Lazy-eval (function not constant) because dotenv loads env AFTER this module
+// is imported — a constant captured at module-load would only see BLOCKLET_APP_URL.
+export const googlePlayEndpoint = (): string =>
+  process.env.GOOGLE_PLAY_WEBHOOK_URL || getUrl('/api/integrations/google-play/webhook');
+
+// Back-compat constant for any caller that captures it at module-load.
+// Prefer googlePlayEndpoint() going forward.
+export const GOOGLE_PLAY_ENDPOINT: string = googlePlayEndpoint();
+export const APP_STORE_ENDPOINT: string = getUrl('/api/integrations/app-store/webhook');
 export const STRIPE_EVENTS: any[] = [
   'checkout.session.async_payment_failed',
   'checkout.session.async_payment_succeeded',
@@ -259,7 +271,11 @@ export async function getBlockletJson(url?: string) {
       return cached.data;
     }
   }
-  const scriptUrl = new URL('__blocklet__.js?type=json', withTrailingSlash(url || process.env.BLOCKLET_APP_URL));
+  const baseUrl = url || process.env.BLOCKLET_APP_URL;
+  if (!baseUrl) {
+    return null;
+  }
+  const scriptUrl = new URL('__blocklet__.js?type=json', withTrailingSlash(baseUrl));
   try {
     const { data: blockletMeta } = await api.get(scriptUrl.href);
     cachedBlockletJsonResult.set(blockletKey, { data: blockletMeta, expiry: now + CACHE_TTL });
@@ -686,17 +702,15 @@ export function formatNumber(
   if (!n || n === '0') {
     return '0';
   }
-  const num = numbro(n);
-  const options = {
-    thousandSeparated,
-    ...((precision || precision === 0) && { mantissa: precision }),
-  };
-  const result = num.format(options);
-  if (!trim) {
-    return result;
-  }
-  const [left, right] = result.split('.');
-  return right ? [left, trimEnd(right, '0')].filter(Boolean).join('.') : left;
+  const value = typeof n === 'string' ? parseFloat(n) : n;
+  if (Number.isNaN(value)) return '0';
+  const fixed = precision || precision === 0 ? value.toFixed(precision) : String(value);
+  const [intPart = '0', decPart] = fixed.split('.');
+  const left = thousandSeparated ? intPart.replace(/\B(?=(\d{3})+(?!\d))/g, ',') : intPart;
+  if (!decPart) return left;
+  if (!trim) return `${left}.${decPart}`;
+  const trimmed = trimEnd(decPart, '0');
+  return trimmed ? `${left}.${trimmed}` : left;
 }
 
 const CURRENCY_SYMBOLS: Record<string, string> = {

package/api/src/libs/wallet-migration.ts

@@ -8,7 +8,7 @@
  * This module provides fallback query mechanisms using migratedFrom list.
  */
 
-import { toDelegateAddress, toStakeAddress } from '@arcblock/did-util';
+import { toDelegateAddress, toStakeAddress } from '@arcblock/did-util/cbor';
 import type OcapClient from '@ocap/client';
 
 import { wallet } from './auth';
@@ -16,23 +16,26 @@ import logger from './logger';
 import { Subscription } from '../store/models';
 
 // Cache for migratedFrom list (keyed by wallet.address + chain host)
+// Empty results are cached too (most common case) with TTL to allow refresh if app migrates
+const MIGRATED_FROM_CACHE_TTL_MS = 5 * 60 * 1000; // 5 minutes
 let cachedMigratedFrom: string[] | null = null;
 let cachedWalletAddress: string | null = null;
 let cachedChainHost: string | null = null;
+let cachedMigratedFromAt = 0;
 
 /**
  * Get the migratedFrom list for the current app wallet (with caching)
- * The cache is invalidated when wallet.address or chain host changes
+ * The cache is invalidated when wallet.address or chain host changes, or after TTL expires
  */
 export async function getMigratedFromList(client: OcapClient): Promise<string[]> {
   // @ts-ignore - OcapClient has host property
   const chainHost = client?.config?.httpEndpoint || 'unknown';
-  // If wallet address and chain host haven't changed, use cached value
+  // If wallet address and chain host haven't changed, and cache is fresh, use cached value
   if (
     wallet.address === cachedWalletAddress &&
     chainHost === cachedChainHost &&
-    cachedMigratedFrom &&
-    cachedMigratedFrom.length > 0
+    cachedMigratedFrom !== null &&
+    Date.now() - cachedMigratedFromAt < MIGRATED_FROM_CACHE_TTL_MS
   ) {
     return cachedMigratedFrom;
   }
@@ -43,6 +46,7 @@ export async function getMigratedFromList(client: OcapClient): Promise<string[]>
     cachedMigratedFrom = state?.migratedFrom || [];
     cachedWalletAddress = wallet.address;
     cachedChainHost = chainHost;
+    cachedMigratedFromAt = Date.now();
 
     if (cachedMigratedFrom.length > 0) {
       logger.info('wallet-migration: loaded migratedFrom list', {
@@ -126,31 +130,34 @@ export async function getDelegationAddressWithFallback({
     // Continue to fallback instead of returning early
   }
 
-  // 2. Try current wallet.address
+  // 2 & 3. Query current wallet delegation and migratedFrom list in parallel
   const currentAddress = toDelegateAddress(delegator, wallet.address);
   let currentStateResult: { hasState: boolean; opsCount: number; error?: string } = {
     hasState: false,
     opsCount: 0,
   };
-  try {
-    const { state: currentState } = await client.getDelegateState({ address: currentAddress });
-    currentStateResult = {
-      hasState: !!currentState,
-      opsCount: currentState?.ops?.length || 0,
-    };
-    if (currentState?.ops?.length > 0) {
-      return { address: currentAddress, needsBackfill: true, source: 'current' };
-    }
-  } catch (err) {
-    currentStateResult.error = String(err);
-    logger.warn('wallet-migration: failed to query current delegation state', {
-      address: currentAddress,
-      error: err,
-    });
-  }
 
-  // 3. Fallback to migratedFrom addresses
-  const migratedFrom = await getMigratedFromList(client);
+  const [currentDelegateResult, migratedFrom] = await Promise.all([
+    client.getDelegateState({ address: currentAddress }).catch((err: any) => {
+      currentStateResult.error = String(err);
+      logger.warn('wallet-migration: failed to query current delegation state', {
+        address: currentAddress,
+        error: err,
+      });
+      return { state: null };
+    }),
+    getMigratedFromList(client),
+  ]);
+
+  const currentState = currentDelegateResult.state;
+  currentStateResult = {
+    hasState: !!currentState,
+    opsCount: currentState?.ops?.length || 0,
+    error: currentStateResult.error, // preserve error from catch
+  };
+  if ((currentState?.ops?.length ?? 0) > 0) {
+    return { address: currentAddress, needsBackfill: true, source: 'current' };
+  }
   const migratedResults: Array<{
     appDid: string;
     address: string;
@@ -158,38 +165,50 @@ export async function getDelegationAddressWithFallback({
     opsCount: number;
     error?: string;
   }> = [];
-  for (const oldAppDid of migratedFrom) {
-    const oldAddress = toDelegateAddress(delegator, oldAppDid);
-    try {
-      // eslint-disable-next-line no-await-in-loop
-      const { state: oldState } = await client.getDelegateState({ address: oldAddress });
-      migratedResults.push({
-        appDid: oldAppDid,
-        address: oldAddress,
-        hasState: !!oldState,
-        opsCount: oldState?.ops?.length || 0,
-      });
-      if (oldState?.ops?.length > 0) {
-        logger.info('wallet-migration: found delegation in migratedFrom', {
-          delegator,
+
+  if (migratedFrom.length > 0) {
+    const results = await Promise.allSettled(
+      migratedFrom.map(async (oldAppDid) => {
+        const oldAddress = toDelegateAddress(delegator, oldAppDid);
+        const { state: oldState } = await client.getDelegateState({ address: oldAddress });
+        return { appDid: oldAppDid, address: oldAddress, state: oldState };
+      })
+    );
+
+    for (let i = 0; i < results.length; i++) {
+      const result = results[i]!;
+      const oldAppDid = migratedFrom[i]!;
+      if (result.status === 'fulfilled') {
+        const { address, state: oldState } = result.value;
+        migratedResults.push({
+          appDid: oldAppDid,
+          address,
+          hasState: !!oldState,
+          opsCount: oldState?.ops?.length || 0,
+        });
+        if ((oldState?.ops?.length ?? 0) > 0) {
+          logger.info('wallet-migration: found delegation in migratedFrom', {
+            delegator,
+            oldAppDid,
+            oldAddress: address,
+          });
+          return { address, needsBackfill: true, source: 'migrated' };
+        }
+      } else {
+        const oldAddress = toDelegateAddress(delegator, oldAppDid);
+        migratedResults.push({
+          appDid: oldAppDid,
+          address: oldAddress,
+          hasState: false,
+          opsCount: 0,
+          error: String(result.reason),
+        });
+        logger.warn('wallet-migration: failed to query migrated delegation state', {
+          address: oldAddress,
           oldAppDid,
-          oldAddress,
+          error: result.reason,
         });
-        return { address: oldAddress, needsBackfill: true, source: 'migrated' };
       }
-    } catch (err) {
-      migratedResults.push({
-        appDid: oldAppDid,
-        address: oldAddress,
-        hasState: false,
-        opsCount: 0,
-        error: String(err),
-      });
-      logger.warn('wallet-migration: failed to query migrated delegation state', {
-        address: oldAddress,
-        oldAppDid,
-        error: err,
-      });
     }
   }
 

package/api/src/queues/auto-recharge.ts

@@ -707,7 +707,7 @@ export async function checkAndTriggerAutoRecharge(
 
     await autoRechargeQueue.push({
       job: jobData,
-      id: `auto-recharge-${customer.id}-${currencyId}}`,
+      id: `auto-recharge-${customer.id}-${currencyId}`,
       persist: true,
     });
 

package/api/src/queues/credit-consume.ts

@@ -16,6 +16,12 @@ import { handlePastDueSubscriptionRecovery } from './payment';
 import { checkAndTriggerAutoRecharge } from './auto-recharge';
 import { addTokenTransferJob } from './token-transfer';
 
+// CF Workers: lower retry limit to conserve Queue ops (10K/day free plan).
+// In Blocklet Server (no Queue ops limit), use the full MAX_RETRY_COUNT.
+// After max retries, mark as requires_action — retryFailedEventsForCustomer()
+// picks them up when credit is granted.
+const CREDIT_MAX_RETRY = (globalThis as any).__CF_ENV__ ? 5 : MAX_RETRY_COUNT;
+
 type CreditConsumptionJob = {
   meterEventId: string;
 };
@@ -25,6 +31,27 @@ type BatchCreditConsumptionJob = {
   batchKey?: string;
 };
 
+/**
+ * Returns true if `error` is one of the failure modes where retrying is
+ * guaranteed to fail again — the underlying condition does not change with
+ * time. These should short-circuit straight to `markAsRequiresAction` so the
+ * event leaves the retry chain immediately, rather than producing N more
+ * scheduled queue messages on the indefinite exponential-backoff schedule.
+ *
+ * History: by 2026-04-25 a single staging customer had accumulated ~7,000
+ * `requires_capture` events stuck looping on these two messages, draining
+ * the daily Queue cap before any real business traffic could land. Once
+ * cleared, this guard prevents the same accumulation from re-forming.
+ */
+function isNonRetryableCreditError(error: any): boolean {
+  const message = typeof error?.message === 'string' ? error.message : '';
+  if (!message) return false;
+  // Customer has no credit left — only resolved by an external top-up, never
+  // by retry. retryFailedEventsForCustomer() picks them up on credit grant.
+  if (message.startsWith('Insufficient credit balance')) return true;
+  return false;
+}
+
 type CreditConsumptionContext = {
   meterEvent: MeterEvent;
   meter: TMeterExpanded;
@@ -667,13 +694,14 @@ export async function handleCreditConsumption(job: CreditConsumptionJob) {
       const meterEvent = await MeterEvent.findByPk(meterEventId);
       if (meterEvent && !['completed', 'canceled', 'requires_action'].includes(meterEvent.status)) {
         const attemptCount = meterEvent.attempt_count + 1;
+        const nonRetryable = isNonRetryableCreditError(error);
 
-        if (attemptCount >= MAX_RETRY_COUNT) {
+        if (attemptCount >= CREDIT_MAX_RETRY || nonRetryable) {
           await meterEvent.markAsRequiresAction(error.message);
           logger.warn('MeterEvent marked as requires_action', {
             meterEventId,
             attemptCount,
-            reason: attemptCount >= MAX_RETRY_COUNT ? 'max_retries_exceeded' : 'non_retryable_error',
+            reason: nonRetryable ? 'non_retryable_error' : 'max_retries_exceeded',
           });
         } else {
           const nextAttemptTime = getNextRetry(attemptCount);
@@ -1136,7 +1164,7 @@ async function handleBatchCreditConsumptionInner(meterEventIds: string[], batchS
       failedEvents.map(({ event, error: failError }) => async () => {
         try {
           const attemptCount = event.attempt_count + 1;
-          if (attemptCount >= MAX_RETRY_COUNT) {
+          if (attemptCount >= CREDIT_MAX_RETRY || isNonRetryableCreditError(failError)) {
             await event.markAsRequiresAction(failError.message);
           } else {
             const nextAttemptTime = getNextRetry(attemptCount);
@@ -1360,16 +1388,70 @@ export async function startCreditConsumeQueue(): Promise<void> {
         });
       }
 
-      // eslint-disable-next-line no-await-in-loop
-      const results = await Promise.allSettled(
-        batchEvents.map(async (event) => {
-          const jobId = `meter-event-${event.id}`;
-          const existingJob = await creditQueue.get(jobId);
-          if (!existingJob) {
-            addCreditConsumptionJob(event.id, true);
+      // Group pending events by (customerId, eventName, subscriptionId) and issue ONE
+      // batched creditQueue push per group. Previously every event produced its own
+      // queue message, so a backlog (e.g., after CF Queue throttling resets at UTC
+      // midnight) burned N queue writes where N could reach hundreds on an active
+      // tenant. A single recovery run catching 50 pending events from the same
+      // customer now drops from 50 writes to 1. Singletons (one event per group, or
+      // events lacking customer_id) keep the legacy per-event path so failure
+      // isolation is unchanged.
+      const groups = new Map<
+        string,
+        { customerId: string; eventName: string; subscriptionId?: string; eventIds: string[] }
+      >();
+      const standalone: string[] = [];
+
+      for (const event of batchEvents) {
+        const customerId = event.payload?.customer_id;
+        if (!customerId) {
+          standalone.push(event.id);
+        } else {
+          const subscriptionId = event.payload?.subscription_id;
+          const key = getBatchKey(customerId, event.event_name, subscriptionId);
+          const group = groups.get(key);
+          if (group) {
+            group.eventIds.push(event.id);
+          } else {
+            groups.set(key, {
+              customerId,
+              eventName: event.event_name,
+              subscriptionId,
+              eventIds: [event.id],
+            });
           }
-        })
-      );
+        }
+      }
+
+      const dispatches: Promise<any>[] = [];
+      for (const group of groups.values()) {
+        if (group.eventIds.length === 1) {
+          // Single event — use the original per-event path so replace/dedupe/status
+          // checks stay identical to pre-change behavior.
+          dispatches.push(addCreditConsumptionJob(group.eventIds[0]!, true));
+        } else {
+          // Multi-event group: one batched push replaces N single pushes.
+          // Stable jobId (no Date.now) — if a previous recovery batch for this
+          // key is still in flight, D1's unique constraint rejects the second
+          // push silently, preventing duplicate queue writes every cron cycle.
+          // Worst case: events arriving while a batch is in flight wait one cron
+          // interval (10 min) for the next recovery push.
+          const jobId = `batch-credit-recovery-${getBatchKey(group.customerId, group.eventName, group.subscriptionId)}`;
+          creditQueue.push({
+            id: jobId,
+            job: {
+              meterEventIds: group.eventIds,
+              batchKey: getBatchKey(group.customerId, group.eventName, group.subscriptionId),
+            } as any,
+          });
+        }
+      }
+      for (const eventId of standalone) {
+        dispatches.push(addCreditConsumptionJob(eventId, true));
+      }
+
+      // eslint-disable-next-line no-await-in-loop
+      const results = await Promise.allSettled(dispatches);
 
       totalFailed += results.filter((r) => r.status === 'rejected').length;
       totalProcessed += batchEvents.length;

package/api/src/queues/credit-grant.ts

@@ -796,6 +796,10 @@ async function handleInvoiceCredit(invoiceId: string) {
           creditGrantId: creditGrant.id,
         });
 
+        // Credit grant activation is handled by the customer.credit_grant.created event
+        // listener via createEvent() in the afterCreate hook. createEvent() self-registers
+        // in __cfPendingJobs__ so it completes in CF Workers before the request ends.
+
         return creditGrant;
       });
       await Promise.all(createPromises);

package/api/src/queues/event.ts

@@ -34,29 +34,35 @@ export const handleEvent = async (job: EventJob) => {
     return;
   }
 
-  await event.update({ pending_webhooks: eventWebhooks.length });
-  logger.info(`Updated event ${event.id} with ${eventWebhooks.length} pending webhooks`);
-
-  eventWebhooks.forEach(async (webhook) => {
-    const attemptCount = await WebhookAttempt.count({
-      where: {
-        event_id: event.id,
-        webhook_endpoint_id: webhook.id,
-        response_status: {
-          [Op.gte]: 200,
-          [Op.lt]: 300,
-        },
-      },
+  // Decide which endpoints still need a first attempt. The previous logic
+  // counted only SUCCESS attempts (2xx), so any permanent-failure endpoint
+  // (e.g. wrong URL → 404) would forever match attemptCount===0 and get
+  // re-pushed by the retry cron every minute, producing thousands of bogus
+  // attempts. We now count ANY attempt: webhookQueue has its own retry
+  // ladder (MAX_RETRY_COUNT=20) that owns recovery for transient failures,
+  // so re-pushing from this handler after the first attempt is double-work.
+  let stillPending = 0;
+  for (const webhook of eventWebhooks) {
+    // eslint-disable-next-line no-await-in-loop -- sequential per-webhook scheduling keeps D1 writes ordered
+    const anyAttempt = await WebhookAttempt.count({
+      where: { event_id: event.id, webhook_endpoint_id: webhook.id },
     });
-
-    // we should only push webhook if it's not successfully attempted before
-    if (attemptCount === 0) {
+    if (anyAttempt === 0) {
+      stillPending += 1;
       logger.info(`Scheduling attempt for event ${event.id} and webhook ${webhook.id}`, job);
-      await addWebhookJob(event.id, webhook.id, { persist: false });
+      // persist=true: write a D1 jobs row so CF Queue transport failures
+      // (momentary unavailability, consumer skip) don't silently lose the
+      // delivery — cron picks orphaned rows back up.
+      // eslint-disable-next-line no-await-in-loop -- same reason as above
+      await addWebhookJob(event.id, webhook.id, { persist: true });
     }
-  });
+  }
 
-  logger.info(`Finished handling event ${job.eventId}`);
+  // pending_webhooks reflects "endpoints that have not yet been attempted at
+  // all" — once all matched endpoints have any attempt row, the cron's
+  // pending>0 scan stops finding this event and it falls out of rotation.
+  await event.update({ pending_webhooks: stillPending });
+  logger.info(`Finished handling event ${job.eventId} (stillPending=${stillPending})`);
 };
 
 export const eventQueue = createQueue<EventJob>({
@@ -93,6 +99,18 @@ eventQueue.on('failed', ({ id, job, error }) => {
   logger.error('event job failed', { id, job, error });
 });
 
-events.on('event.created', (event) => {
-  eventQueue.push({ id: event.id, job: { eventId: event.id }, persist: false });
+events.on('event.created', async (event) => {
+  if ((globalThis as any).__CF_ENV__) {
+    // CF Workers: execute inline to save 2 CF Queue ops per event.
+    // eventQueue only dispatches webhooks — lightweight DB lookup + webhookQueue.push.
+    // Webhook delivery still goes through webhookQueue with full retry guarantees.
+    try {
+      await handleEvent({ eventId: event.id });
+    } catch (err: any) {
+      logger.error('event inline handler failed', { eventId: event.id, error: err?.message });
+    }
+  } else {
+    // Blocklet Server: use queue as before
+    eventQueue.push({ id: event.id, job: { eventId: event.id }, persist: false });
+  }
 });

package/api/src/queues/invoice.ts

@@ -10,6 +10,7 @@ import { CheckoutSession } from '../store/models/checkout-session';
 import { PaymentIntent } from '../store/models/payment-intent';
 import { Subscription } from '../store/models/subscription';
 import { PriceQuote } from '../store/models/price-quote';
+// eslint-disable-next-line import/no-cycle
 import { paymentQueue } from './payment';
 import { getQuoteService } from '../libs/quote-service';