payment-kit 1.24.3 → 1.25.0

package/api/src/queues/invoice.ts

@@ -5,11 +5,13 @@ import { createEvent } from '../libs/audit';
 import dayjs from '../libs/dayjs';
 import logger from '../libs/logger';
 import createQueue from '../libs/queue';
-import { PaymentMethod, Invoice } from '../store/models';
+import { PaymentMethod, Invoice, InvoiceItem } from '../store/models';
 import { CheckoutSession } from '../store/models/checkout-session';
 import { PaymentIntent } from '../store/models/payment-intent';
 import { Subscription } from '../store/models/subscription';
+import { PriceQuote } from '../store/models/price-quote';
 import { paymentQueue } from './payment';
+import { getQuoteService } from '../libs/quote-service';
 
 import { getLock } from '../libs/lock';
 import { events } from '../libs/event';
@@ -280,6 +282,51 @@ events.on('invoice.paid', async ({ id: invoiceId }) => {
   await handleOverdraftProtectionInvoiceAfterPayment(invoice);
 });
 
+// Separate listener to mark quotes as paid - independent of other invoice.paid handlers
+events.on('invoice.paid', async ({ id: invoiceId }) => {
+  try {
+    const invoiceItems = await InvoiceItem.findAll({
+      where: { invoice_id: invoiceId },
+    });
+
+    const quoteIds = invoiceItems
+      .map((item) => item.metadata?.quote_id)
+      .filter((id): id is string => typeof id === 'string' && id.length > 0);
+
+    if (quoteIds.length === 0) {
+      return;
+    }
+
+    const quoteService = getQuoteService();
+    const markedQuoteIds: string[] = [];
+
+    await Promise.all(
+      quoteIds.map(async (quoteId) => {
+        try {
+          const quote = await PriceQuote.findByPk(quoteId);
+          if (quote && quote.status !== 'paid') {
+            await quote.update({ invoice_id: invoiceId });
+            await quoteService.markAsPaid(quoteId);
+            markedQuoteIds.push(quoteId);
+          }
+        } catch (err: any) {
+          logger.warn('Failed to mark quote as paid', { quoteId, invoiceId, error: err.message });
+        }
+      })
+    );
+
+    if (markedQuoteIds.length > 0) {
+      logger.info('Marked quotes as paid for invoice', {
+        invoiceId,
+        quoteIds: markedQuoteIds,
+        totalQuotes: quoteIds.length,
+      });
+    }
+  } catch (err: any) {
+    logger.error('Failed to process quotes for paid invoice', { invoiceId, error: err.message });
+  }
+});
+
 events.on('invoice.queued', async (id, job, args = {}) => {
   const { sync, ...extraArgs } = args;
   if (sync) {

package/api/src/queues/notification.ts

@@ -1,4 +1,5 @@
 import { Op } from 'sequelize';
+import debounce from 'lodash/debounce';
 /* eslint-disable @typescript-eslint/indent */
 import { events } from '../libs/event';
 import logger from '../libs/logger';
@@ -118,6 +119,18 @@ import {
   WebhookEndpointFailedEmailTemplate,
   WebhookEndpointFailedEmailTemplateOptions,
 } from '../libs/notification/template/webhook-endpoint-failed';
+import {
+  SubscriptionSlippageWarningEmailTemplate,
+  SubscriptionSlippageWarningEmailTemplateOptions,
+} from '../libs/notification/template/subscription-slippage-warning';
+import {
+  SubscriptionSlippageExceededEmailTemplate,
+  SubscriptionSlippageExceededEmailTemplateOptions,
+} from '../libs/notification/template/subscription-slippage-exceeded';
+import {
+  ExchangeRateAlertEmailTemplate,
+  ExchangeRateAlertEmailTemplateOptions,
+} from '../libs/notification/template/exchange-rate-alert';
 import type { TJob } from '../store/models/job';
 
 export type NotificationQueueJobOptions = any;
@@ -144,7 +157,11 @@ export type NotificationQueueJobType =
   | 'customer.credit.low_balance'
   | 'customer.auto_recharge.failed'
   | 'customer.auto_recharge.daily_limit_exceeded'
-  | 'webhook.endpoint.failed';
+  | 'webhook.endpoint.failed'
+  | 'exchange_rate.providers_unavailable'
+  | 'exchange_rate.spread_exceeded'
+  | 'subscription.slippage_warning'
+  | 'subscription.slippage_exceeded';
 
 export type NotificationQueueJob = {
   type: NotificationQueueJobType;
@@ -300,6 +317,21 @@ async function getNotificationTemplate(job: NotificationQueueJob): Promise<BaseE
     return new WebhookEndpointFailedEmailTemplate(job.options as WebhookEndpointFailedEmailTemplateOptions);
   }
 
+  if (job.type === 'subscription.slippage_warning') {
+    return new SubscriptionSlippageWarningEmailTemplate(job.options as SubscriptionSlippageWarningEmailTemplateOptions);
+  }
+
+  if (job.type === 'subscription.slippage_exceeded') {
+    return new SubscriptionSlippageExceededEmailTemplate(
+      job.options as SubscriptionSlippageExceededEmailTemplateOptions
+    );
+  }
+
+  // Exchange rate alerts (admin notifications)
+  if (job.type === 'exchange_rate.providers_unavailable' || job.type === 'exchange_rate.spread_exceeded') {
+    return new ExchangeRateAlertEmailTemplate(job.options as ExchangeRateAlertEmailTemplateOptions);
+  }
+
   throw new Error(`Unknown job type: ${job.type}`);
 }
 
@@ -337,6 +369,8 @@ interface NotificationItem<T = Record<string, any>> {
 
 // 内存缓存，记录最近发送的通知
 const notificationCache = new Map<string, number>();
+// 去抖函数（按 key 去重）
+const debounceHandlers = new Map<string, ReturnType<typeof debounce>>();
 
 // 清理过期缓存的定时器
 setInterval(
@@ -738,11 +772,164 @@ export async function startNotificationQueue() {
     }
   );
 
+  // System events - Exchange rate alerts (admin notifications)
+  events.on('exchange_rate.providers_unavailable', (alertData: Record<string, any>) => {
+    logger.info('Exchange rate providers unavailable event received', { alertData });
+    addNotificationJob(
+      'exchange_rate.providers_unavailable',
+      alertData,
+      ['exchange_rate.providers_unavailable', alertData.symbol],
+      true, // Prevent duplicate
+      30 * 60 // 30 minutes window
+    );
+  });
+
+  events.on('exchange_rate.spread_exceeded', (alertData: Record<string, any>) => {
+    logger.info('Exchange rate spread exceeded event received', { alertData });
+    addNotificationJob(
+      'exchange_rate.spread_exceeded',
+      alertData,
+      ['exchange_rate.spread_exceeded', alertData.symbol],
+      true, // Prevent duplicate
+      30 * 60 // 30 minutes window (matches service rate limit)
+    );
+  });
+
+  // Auto-recharge skipped due to dynamic pricing constraints
+  events.on(
+    'auto_recharge.skipped',
+    async (data: {
+      config_id: string;
+      customer_id: string;
+      reason: 'slippage_exceeded' | 'exchange_rate_not_supported' | 'exchange_rate_fetch_failed';
+      current_rate?: string;
+      min_acceptable_rate?: string;
+      payment_currency_symbol?: string;
+      payment_currency_name?: string;
+      credit_currency_name?: string;
+    }) => {
+      logger.info('Auto recharge skipped event received', {
+        configId: data.config_id,
+        customerId: data.customer_id,
+        reason: data.reason,
+        currentRate: data.current_rate,
+        minAcceptableRate: data.min_acceptable_rate,
+        paymentCurrencySymbol: data.payment_currency_symbol,
+      });
+
+      // Reuse customer.auto_recharge.failed notification with skipped reason
+      // This ensures users are notified when auto-recharge cannot proceed
+      const customer = await Customer.findByPk(data.customer_id);
+      if (customer) {
+        addNotificationJob(
+          'customer.auto_recharge.failed',
+          {
+            customerId: data.customer_id,
+            autoRechargeConfigId: data.config_id,
+            result: {
+              sufficient: false,
+              reason: data.reason,
+              currentRate: data.current_rate,
+              minAcceptableRate: data.min_acceptable_rate,
+              paymentCurrencySymbol: data.payment_currency_symbol,
+              paymentCurrencyName: data.payment_currency_name,
+              creditCurrencyName: data.credit_currency_name,
+            },
+          },
+          [data.customer_id, data.config_id, data.reason],
+          true, // Prevent duplicate notifications
+          6 * 3600 // 6 hours window - avoid spamming if rate stays bad
+        );
+      }
+    }
+  );
+
+  // Slippage exceeded notification
+  events.on(
+    'subscription.slippage_exceeded',
+    (
+      subscription: Subscription,
+      {
+        invoiceId,
+        paymentIntentId,
+        currentRate,
+        minAcceptableRate,
+      }: {
+        invoiceId: string;
+        paymentIntentId: string;
+        currentRate: string;
+        minAcceptableRate: string;
+      }
+    ) => {
+      addNotificationJob(
+        'subscription.slippage_exceeded',
+        {
+          subscriptionId: subscription.id,
+          invoiceId,
+          paymentIntentId,
+          currentRate,
+          minAcceptableRate,
+        },
+        [subscription.id, invoiceId],
+        true, // Prevent duplicate
+        24 * 3600 // 24 hours window
+      );
+    }
+  );
+
+  // Slippage warning notification (1 hour before renewal)
+  events.on(
+    'subscription.slippage_warning',
+    (
+      subscription: Subscription,
+      {
+        currentRate,
+        minAcceptableRate,
+        renewalTime,
+      }: {
+        currentRate: string;
+        minAcceptableRate: string;
+        renewalTime: number;
+      }
+    ) => {
+      addNotificationJob(
+        'subscription.slippage_warning',
+        {
+          subscriptionId: subscription.id,
+          currentRate,
+          minAcceptableRate,
+          renewalTime,
+        },
+        [subscription.id],
+        true, // Prevent duplicate
+        3600 // 1 hour window - avoid duplicate warnings
+      );
+    }
+  );
+
   // Clear credit notification cache when customer recharges
   // This allows customer to receive insufficient/low_balance notifications again
   events.on('customer.credit_grant.granted', (creditGrant: CreditGrant) => {
-    clearNotificationCache(`customer.credit.insufficient.${creditGrant.customer_id}.${creditGrant.currency_id}`);
-    clearNotificationCache(`customer.credit.low_balance.${creditGrant.customer_id}.${creditGrant.currency_id}`);
+    const cacheKeyBase = `${creditGrant.customer_id}.${creditGrant.currency_id}`;
+    const debounceKey = `credit_grant_clear.${cacheKeyBase}`;
+    const delayMs = 5 * 60 * 1000;
+
+    let handler = debounceHandlers.get(debounceKey);
+    if (!handler) {
+      handler = debounce(() => {
+        clearNotificationCache(`customer.credit.insufficient.${cacheKeyBase}`);
+        clearNotificationCache(`customer.credit.low_balance.${cacheKeyBase}`);
+        debounceHandlers.delete(debounceKey);
+        logger.info('Notification cache cleared after delay', {
+          customerId: creditGrant.customer_id,
+          currencyId: creditGrant.currency_id,
+          delayMs,
+        });
+      }, delayMs);
+      debounceHandlers.set(debounceKey, handler);
+    }
+
+    handler();
   });
 }
 

package/api/src/queues/payment.ts

@@ -11,6 +11,8 @@ import CustomError from '../libs/error';
 import { events } from '../libs/event';
 import logger from '../libs/logger';
 import { getGasPayerExtra, isDelegationSufficientForPayment, checkDepositVaultAmount } from '../libs/payment';
+import { validateQuoteForPayment, handleExpiredQuotePayment } from '../libs/quote-validation';
+import { getQuoteService } from '../libs/quote-service';
 import {
   checkRemainingStake,
   getDaysUntilCancel,
@@ -42,7 +44,7 @@ import { AutoRechargeConfig, Lock, MeterEvent } from '../store/models';
 import { ensureOverdraftProtectionPrice } from '../libs/overdraft-protection';
 import createQueue from '../libs/queue';
 import { CHARGE_SUPPORTED_CHAIN_TYPES, EVM_CHAIN_TYPES } from '../libs/constants';
-import { getCheckoutSessionSubscriptionIds, getSubscriptionCreateSetup } from '../libs/session';
+import { getCheckoutSessionSubscriptionIds, getSubscriptionCreateSetup, SlippageOptions } from '../libs/session';
 import { syncStripeSubscriptionAfterRecovery } from '../integrations/stripe/handlers/subscription';
 import { getLock } from '../libs/lock';
 
@@ -214,7 +216,15 @@ export async function handlePastDueSubscriptionRecovery(
     // Reset billing cycle
     const subscriptionItems = await SubscriptionItem.findAll({ where: { subscription_id: subscription.id } });
     const lineItems = await Price.expand(subscriptionItems.map((x) => x.toJSON()));
-    const setup = getSubscriptionCreateSetup(lineItems, subscription.currency_id, 0);
+    // Use the subscription's slippage config if available, otherwise use default 0.5%
+    const slippageConfig = subscription.slippage_config;
+    const currency = await PaymentCurrency.findByPk(subscription.currency_id);
+    const slippageOptions: SlippageOptions = {
+      percent: slippageConfig?.percent ?? 0.5,
+      minAcceptableRate: slippageConfig?.min_acceptable_rate,
+      currencyDecimal: currency?.decimal,
+    };
+    const setup = getSubscriptionCreateSetup(lineItems, subscription.currency_id, 0, 0, slippageOptions);
     await subscription.update({
       status: 'active',
       pending_invoice_item_interval: setup.recurring,
@@ -403,6 +413,18 @@ export const handlePaymentSucceed = async (
     logger.info(`Invoice ${invoice.id} updated on payment done: ${paymentIntent.id}`);
   }
 
+  // Mark quotes as paid after successful payment
+  // Query quotes by invoiceId - more reliable than depending on metadata
+  if (invoice?.id) {
+    const quoteService = getQuoteService();
+    const markedIds = await quoteService.markQuotesAsPaidByInvoice(invoice.id);
+    if (markedIds.length > 0) {
+      logger.info('Marked quotes as paid after successful payment', {
+        invoiceId: invoice.id,
+        quoteIds: markedIds,
+      });
+    }
+  }
   // Update subscription status
   if (invoice && invoice.subscription_id && !slashStake) {
     const subscription = await Subscription.findByPk(invoice.subscription_id);
@@ -919,6 +941,55 @@ export const handlePayment = async (job: PaymentJob) => {
       return;
     }
 
+    // Queue delay monitoring - critical for quote lock window observability
+    const queueDelayMs = Date.now() - new Date(paymentIntent.created_at).getTime();
+    const QUOTE_LOCK_WINDOW_MS = 180000; // 180 seconds
+
+    logger.info('Queue processing payment intent', {
+      paymentIntentId: paymentIntent.id,
+      queueDelayMs,
+      delaySeconds: Math.round(queueDelayMs / 1000),
+      lockWindowSeconds: QUOTE_LOCK_WINDOW_MS / 1000,
+    });
+
+    if (queueDelayMs > QUOTE_LOCK_WINDOW_MS) {
+      logger.warn('CRITICAL: Queue delay exceeds quote lock window', {
+        paymentIntentId: paymentIntent.id,
+        invoiceId: paymentIntent.invoice_id,
+        queueDelayMs,
+        delaySeconds: Math.round(queueDelayMs / 1000),
+        lockWindowSeconds: QUOTE_LOCK_WINDOW_MS / 1000,
+        riskLevel: 'HIGH - Quote lock expired, slippage protection critical',
+      });
+    }
+
+    const preQuoteValidation = await validateQuoteForPayment({
+      checkoutSessionId: invoice?.checkout_session_id,
+      invoiceId: paymentIntent.invoice_id,
+      amount: paymentIntent.amount,
+    });
+
+    if (!preQuoteValidation.valid) {
+      // Enhanced error logging with queue delay context
+      let failureReason = 'QUOTE_VALIDATION_FAILED';
+      if (preQuoteValidation.reason?.includes('QUOTE_MAX_PAYABLE_EXCEEDED')) {
+        failureReason = 'SLIPPAGE_EXCEEDED';
+      } else if (preQuoteValidation.reason?.includes('expired')) {
+        failureReason = 'QUOTE_EXPIRED';
+      }
+
+      logger.error('Payment aborted due to quote validation before capture', {
+        paymentIntentId: paymentIntent.id,
+        invoiceId: paymentIntent.invoice_id,
+        quoteId: preQuoteValidation.quoteId,
+        reason: preQuoteValidation.reason,
+        failureReason,
+        queueDelayMs,
+        delaySeconds: Math.round(queueDelayMs / 1000),
+      });
+      throw new CustomError(failureReason, `Payment validation failed: ${preQuoteValidation.reason || 'unknown'}`);
+    }
+
     await paymentIntent.update({ status: 'processing', last_payment_error: null });
 
     if (paymentMethod.type === 'arcblock') {
@@ -938,11 +1009,15 @@ export const handlePayment = async (job: PaymentJob) => {
       });
 
       if (result.sufficient === false) {
-        logger.error('PaymentIntent capture aborted on preCheck', {
-          id: paymentIntent.id,
+        logger.error('PaymentIntent capture aborted - insufficient balance/delegation', {
+          paymentIntentId: paymentIntent.id,
+          invoiceId: paymentIntent.invoice_id,
+          failureReason: 'INSUFFICIENT_BALANCE',
           result,
+          queueDelayMs,
+          delaySeconds: Math.round(queueDelayMs / 1000),
         });
-        throw new CustomError(result.reason, 'payer balance or delegation not sufficient for this payment');
+        throw new CustomError('INSUFFICIENT_BALANCE', 'Payer balance or delegation not sufficient for this payment');
       }
 
       const signed = await client.signTransferV2Tx({
@@ -978,6 +1053,50 @@ export const handlePayment = async (job: PaymentJob) => {
       logger.info('PaymentIntent txHash', { txHash });
       logger.info('PaymentIntent capture done', { id: paymentIntent.id, txHash });
 
+      // CRITICAL: Validate quote before marking payment as succeeded
+      const quoteValidation = await validateQuoteForPayment({
+        checkoutSessionId: undefined, // Will be found through invoice if exists
+        invoiceId: paymentIntent.invoice_id,
+        amount: paymentIntent.amount,
+      });
+
+      if (!quoteValidation.valid) {
+        if (quoteValidation.action === 'reject') {
+          // Reject the payment - mark as failed
+          logger.error('Automated payment rejected due to quote validation', {
+            paymentIntentId: paymentIntent.id,
+            invoiceId: paymentIntent.invoice_id,
+            quoteId: quoteValidation.quoteId,
+            reason: quoteValidation.reason,
+            txHash,
+          });
+          throw new CustomError('quote_validation_failed', `Payment validation failed: ${quoteValidation.reason}`);
+        }
+
+        if (quoteValidation.action === 'require_manual_review') {
+          // Hold for manual review
+          logger.error('Automated payment requires manual review due to expired quote', {
+            paymentIntentId: paymentIntent.id,
+            invoiceId: paymentIntent.invoice_id,
+            quoteId: quoteValidation.quoteId,
+            reason: quoteValidation.reason,
+            txHash,
+          });
+
+          await handleExpiredQuotePayment({
+            quoteId: quoteValidation.quoteId!,
+            checkoutSessionId: undefined,
+            invoiceId: paymentIntent.invoice_id,
+            paymentIntentId: paymentIntent.id,
+            amount: paymentIntent.amount,
+            currencyId: paymentIntent.currency_id,
+            txHash,
+          });
+
+          throw new CustomError('quote_expired', `Payment held for manual review: ${quoteValidation.reason}`);
+        }
+      }
+
       await paymentIntent.update({
         status: 'succeeded',
         last_payment_error: null,
@@ -1011,14 +1130,65 @@ export const handlePayment = async (job: PaymentJob) => {
         amount: paymentIntent.amount,
       });
       if (result.sufficient === false) {
-        logger.error('PaymentIntent capture aborted on preCheck', { id: paymentIntent.id, result });
-        throw new CustomError(result.reason, 'payer balance or delegation not sufficient for this payment');
+        logger.error('PaymentIntent capture aborted - insufficient balance/delegation', {
+          paymentIntentId: paymentIntent.id,
+          invoiceId: paymentIntent.invoice_id,
+          failureReason: 'INSUFFICIENT_BALANCE',
+          result,
+          queueDelayMs,
+          delaySeconds: Math.round(queueDelayMs / 1000),
+        });
+        throw new CustomError('INSUFFICIENT_BALANCE', 'Payer balance or delegation not sufficient for this payment');
       }
 
       // do the capture
       const receipt = await transferErc20FromUser(client, paymentCurrency.contract, payer, paymentIntent.amount);
       logger.info('PaymentIntent capture done', { id: paymentIntent.id, txHash: receipt.hash });
 
+      // CRITICAL: Validate quote before marking payment as succeeded
+      const quoteValidation = await validateQuoteForPayment({
+        checkoutSessionId: undefined, // Will be found through invoice if exists
+        invoiceId: paymentIntent.invoice_id,
+        amount: paymentIntent.amount,
+      });
+
+      if (!quoteValidation.valid) {
+        if (quoteValidation.action === 'reject') {
+          // Reject the payment - mark as failed
+          logger.error('Automated EVM payment rejected due to quote validation', {
+            paymentIntentId: paymentIntent.id,
+            invoiceId: paymentIntent.invoice_id,
+            quoteId: quoteValidation.quoteId,
+            reason: quoteValidation.reason,
+            txHash: receipt.hash,
+          });
+          throw new CustomError('quote_validation_failed', `Payment validation failed: ${quoteValidation.reason}`);
+        }
+
+        if (quoteValidation.action === 'require_manual_review') {
+          // Hold for manual review
+          logger.error('Automated EVM payment requires manual review due to expired quote', {
+            paymentIntentId: paymentIntent.id,
+            invoiceId: paymentIntent.invoice_id,
+            quoteId: quoteValidation.quoteId,
+            reason: quoteValidation.reason,
+            txHash: receipt.hash,
+          });
+
+          await handleExpiredQuotePayment({
+            quoteId: quoteValidation.quoteId!,
+            checkoutSessionId: undefined,
+            invoiceId: paymentIntent.invoice_id,
+            paymentIntentId: paymentIntent.id,
+            amount: paymentIntent.amount,
+            currencyId: paymentIntent.currency_id,
+            txHash: receipt.hash,
+          });
+
+          throw new CustomError('quote_expired', `Payment held for manual review: ${quoteValidation.reason}`);
+        }
+      }
+
       await paymentIntent.update({
         status: 'succeeded',
         last_payment_error: null,