payment-kit 1.19.0 → 1.19.2

package/api/src/queues/credit-consume.ts

@@ -0,0 +1,728 @@
+import { BN, fromUnitToToken } from '@ocap/util';
+
+import { getLock } from '../libs/lock';
+import logger from '../libs/logger';
+import createQueue from '../libs/queue';
+import { createEvent } from '../libs/audit';
+import {
+  MeterEvent,
+  CreditGrant,
+  CreditTransaction,
+  Meter,
+  Customer,
+  Subscription,
+  PaymentCurrency,
+  TMeterExpanded,
+} from '../store/models';
+
+import { MAX_RETRY_COUNT, getNextRetry } from '../libs/util';
+import { getDaysUntilCancel, getDueUnit, getMeterPriceIdsFromSubscription } from '../libs/subscription';
+import { events } from '../libs/event';
+import { handlePastDueSubscriptionRecovery } from './payment';
+
+type CreditConsumptionJob = {
+  meterEventId: string;
+};
+
+type CreditConsumptionContext = {
+  meterEvent: MeterEvent;
+  meter: TMeterExpanded;
+  customer: Customer;
+  subscription?: Subscription;
+  priceIds?: string[];
+  transactions: string[];
+  warnings: string[];
+};
+
+type CreditConsumptionResult = {
+  consumed: string;
+  pending: string;
+  available_balance: string;
+  fully_consumed: boolean;
+};
+
+async function validateAndLoadData(meterEventId: string): Promise<CreditConsumptionContext | null> {
+  const meterEvent = await MeterEvent.findByPk(meterEventId);
+  if (!meterEvent) {
+    logger.warn('Skipping credit consumption job: MeterEvent not found', { meterEventId });
+    return null;
+  }
+
+  if (meterEvent.status === 'completed' || meterEvent.status === 'canceled') {
+    logger.info('Skipping credit consumption job: MeterEvent already processed', {
+      meterEventId,
+      status: meterEvent.status,
+    });
+    return null;
+  }
+
+  if (meterEvent.status === 'requires_action') {
+    logger.warn('Skipping credit consumption job: MeterEvent requires manual intervention', {
+      meterEventId,
+      attemptCount: meterEvent.attempt_count,
+    });
+    return null;
+  }
+
+  const meter = (await Meter.findOne({
+    where: { event_name: meterEvent.event_name },
+    include: [{ model: PaymentCurrency, as: 'paymentCurrency' }],
+  })) as TMeterExpanded | null;
+
+  if (!meter) {
+    logger.warn('Skipping credit consumption job: Meter not found', {
+      meterEventId,
+      eventName: meterEvent.event_name,
+    });
+    return null;
+  }
+
+  if (!meter.currency_id) {
+    logger.warn('Skipping credit consumption job: Meter missing currency_id', {
+      meterEventId,
+      meterId: meter.id,
+    });
+    return null;
+  }
+
+  const customerId = meterEvent.getCustomerId();
+  if (!customerId) {
+    logger.warn('Skipping credit consumption job: MeterEvent missing customer_id', { meterEventId });
+    return null;
+  }
+
+  const customer = await Customer.findByPk(customerId);
+  if (!customer) {
+    logger.warn('Skipping credit consumption job: Customer not found', {
+      meterEventId,
+      customerId,
+    });
+    return null;
+  }
+
+  const subscriptionId = meterEvent.getSubscriptionId();
+  if (subscriptionId) {
+    const subscription = await Subscription.findByPk(subscriptionId);
+    if (!subscription) {
+      logger.warn('Skipping credit consumption job: Subscription not found', {
+        meterEventId,
+        subscriptionId,
+      });
+      return null;
+    }
+
+    const priceIds = await getMeterPriceIdsFromSubscription(subscription);
+
+    return {
+      meterEvent,
+      meter,
+      customer,
+      subscription: subscription!,
+      transactions: [],
+      warnings: [],
+      priceIds,
+    };
+  }
+
+  return {
+    meterEvent,
+    meter,
+    customer,
+    transactions: [],
+    warnings: [],
+  };
+}
+
+async function consumeAvailableCredits(
+  context: CreditConsumptionContext,
+  totalRequiredAmount: string
+): Promise<CreditConsumptionResult> {
+  const customerId = context.meterEvent.getCustomerId();
+  const currencyId = context.meter.currency_id!;
+  const meterEventId = context.meterEvent.id;
+
+  const existingTransactions = await CreditTransaction.findAll({
+    where: {
+      source: meterEventId,
+    },
+  });
+
+  const alreadyConsumed = existingTransactions.reduce((sum, tx) => sum.add(new BN(tx.credit_amount)), new BN(0));
+  const remainingToConsume = new BN(totalRequiredAmount).sub(alreadyConsumed);
+
+  // 如果已经完全消费，直接返回结果
+  if (remainingToConsume.lte(new BN(0))) {
+    logger.info('Credit consumption already completed', {
+      meterEventId,
+      totalRequiredAmount,
+      alreadyConsumed: alreadyConsumed.toString(),
+    });
+
+    context.transactions.push(...existingTransactions.map((tx) => tx.id));
+
+    return {
+      consumed: alreadyConsumed.toString(),
+      pending: '0',
+      available_balance: '0',
+      fully_consumed: true,
+    };
+  }
+
+  // Get all available grants sorted by priority
+  const availableGrants = await CreditGrant.getAvailableCreditsForCustomer(customerId, currencyId, context.priceIds);
+
+  // Calculate total available balance
+  const totalAvailable = availableGrants.reduce((sum, grant) => sum.add(new BN(grant.remaining_amount)), new BN(0));
+  logger.debug('Total available credits calculated', { totalAvailable: totalAvailable.toString() });
+
+  let remaining = remainingToConsume;
+  let consumed = new BN(0);
+
+  // Consume credits from grants in priority order
+  for (const grant of availableGrants) {
+    if (remaining.lte(new BN(0))) break;
+
+    const availableInGrant = new BN(grant.remaining_amount);
+
+    // if remaining is less than availableInGrant, consume remaining
+    // otherwise, consume availableInGrant
+    const consumeAmount = remaining.lte(availableInGrant) ? remaining : availableInGrant;
+
+    // if consumeAmount is greater than 0, process grant consumption
+    if (consumeAmount.gt(new BN(0))) {
+      await processGrantConsumption(context, grant, consumeAmount.toString()); // eslint-disable-line no-await-in-loop
+      consumed = consumed.add(consumeAmount);
+      remaining = remaining.sub(consumeAmount);
+    }
+  }
+
+  const totalConsumed = alreadyConsumed.add(consumed);
+  const finalPending = new BN(totalRequiredAmount).sub(totalConsumed);
+
+  logger.info('Consumed credits', {
+    customerId,
+    currencyId,
+    totalRequiredAmount,
+    alreadyConsumed: alreadyConsumed.toString(),
+    newlyConsumed: consumed.toString(),
+    totalConsumed: totalConsumed.toString(),
+    finalPending: finalPending.toString(),
+  });
+
+  const pendingAmount = Math.max(0, finalPending.toNumber()).toString();
+  const remainingBalance = totalAvailable.sub(consumed).toString();
+
+  // 如果无法完全消费所需额度，记录不足事件
+  if (finalPending.gt(new BN(0))) {
+    logger.warn('Insufficient credit balance', {
+      customerId,
+      currencyId,
+      totalRequiredAmount,
+      availableAmount: totalAvailable.toString(),
+      totalConsumed: totalConsumed.toString(),
+      pendingAmount,
+    });
+    if ((context.subscription && context.subscription.isActive()) || !context.subscription) {
+      await createEvent('Customer', 'customer.credit.insufficient', context.customer, {
+        metadata: {
+          meter_event_id: context.meterEvent.id,
+          meter_event_name: context.meterEvent.event_name,
+          required_amount: remainingToConsume.toString(),
+          available_amount: totalAvailable.toString(),
+          consumed_amount: consumed.toString(),
+          pending_amount: pendingAmount,
+          currency_id: currencyId,
+          subscription_id: context.subscription?.id,
+        },
+      }).catch(console.error);
+    }
+
+    // 如果有关联订阅且订阅活跃，将其标记为逾期
+    if (context.subscription && context.subscription.isActive()) {
+      const cancelUpdates: { [key: string]: any } = {};
+      const { interval } = context.subscription.pending_invoice_item_interval;
+      const dueUnit = getDueUnit(interval);
+      const daysUntilCancel = getDaysUntilCancel(context.subscription);
+      if (daysUntilCancel > 0) {
+        cancelUpdates.cancel_at = context.subscription.current_period_start + daysUntilCancel * dueUnit;
+      } else {
+        cancelUpdates.cancel_at_period_end = true;
+      }
+      await context.subscription.update({
+        status: 'past_due',
+        ...cancelUpdates,
+        cancelation_details: {
+          comment: 'past_due',
+          feedback: 'other',
+          reason: 'insufficient_credit',
+        },
+      });
+    }
+  } else if (remainingBalance === '0') {
+    await createEvent('Customer', 'customer.credit.insufficient', context.customer, {
+      metadata: {
+        meter_event_id: context.meterEvent.id,
+        meter_event_name: context.meterEvent.event_name,
+        required_amount: remainingToConsume.toString(),
+        available_amount: '0',
+        consumed_amount: consumed.toString(),
+        pending_amount: pendingAmount,
+        currency_id: currencyId,
+        subscription_id: context.subscription?.id,
+      },
+    }).catch(console.error);
+  }
+
+  return {
+    consumed: totalConsumed.toString(),
+    pending: pendingAmount,
+    available_balance: remainingBalance,
+    fully_consumed: finalPending.lte(new BN(0)),
+  };
+}
+
+async function processGrantConsumption(
+  context: CreditConsumptionContext,
+  creditGrant: CreditGrant,
+  consumeAmount: string
+): Promise<void> {
+  logger.info('Processing grant consumption', {
+    creditGrantId: creditGrant.id,
+    consumeAmount,
+  });
+  const result = await creditGrant.consumeCredit(consumeAmount, {
+    subscription_id: context.subscription?.id,
+    meter_event_id: context.meterEvent.id,
+  });
+  await createCreditTransaction(context, creditGrant, consumeAmount, result);
+}
+
+async function createCreditTransaction(
+  context: CreditConsumptionContext,
+  creditGrant: CreditGrant,
+  consumeAmount: string,
+  consumeResult: any
+): Promise<void> {
+  const meterEventId = context.meterEvent.id;
+  const creditGrantId = creditGrant.id;
+
+  // 检查是否已经为这个 MeterEvent 和 CreditGrant 创建过 transaction
+  const existingTransaction = await CreditTransaction.findOne({
+    where: {
+      source: meterEventId,
+      credit_grant_id: creditGrantId,
+    },
+  });
+
+  if (existingTransaction) {
+    logger.warn('CreditTransaction already exists for this MeterEvent and CreditGrant', {
+      meterEventId,
+      creditGrantId,
+      existingTransactionId: existingTransaction.id,
+    });
+    context.transactions.push(existingTransaction.id);
+    return;
+  }
+
+  logger.debug('start to create credit transaction', {
+    customerId: context.meterEvent.getCustomerId(),
+    creditGrantId: creditGrant.id,
+    meterId: context.meter.id,
+    subscriptionId: context.meterEvent.getSubscriptionId(),
+    meterEventId: context.meterEvent.id,
+  });
+
+  let description = `Consume ${fromUnitToToken(consumeAmount, context.meter.paymentCurrency.decimal)}${context.meter.paymentCurrency.symbol}`;
+  if (context.meterEvent.getSubscriptionId()) {
+    description += 'for Subscription';
+  }
+
+  try {
+    const transaction = await CreditTransaction.create({
+      customer_id: context.meterEvent.getCustomerId(),
+      credit_grant_id: creditGrant.id,
+      meter_id: context.meter.id,
+      subscription_id: context.meterEvent.getSubscriptionId(),
+      meter_event_name: context.meterEvent.event_name,
+      meter_unit: context.meter.unit,
+      quantity: consumeAmount,
+      credit_amount: consumeResult.consumed,
+      remaining_balance: consumeResult.remaining,
+      source: context.meterEvent.id,
+      description,
+      metadata: {
+        ...(context.meterEvent.metadata || {}),
+        meter_event_id: context.meterEvent.id,
+        grant_depleted: consumeResult.depleted,
+      },
+    });
+
+    logger.info('Created credit transaction', {
+      transactionId: transaction.id,
+      creditGrantId: creditGrant.id,
+      meterId: context.meter.id,
+      subscriptionId: context.meterEvent.getSubscriptionId(),
+      meterEventId: context.meterEvent.id,
+    });
+    context.transactions.push(transaction.id);
+  } catch (error: any) {
+    // 处理唯一约束违反错误
+    if (error.name === 'SequelizeUniqueConstraintError' || error.message.includes('UNIQUE constraint failed')) {
+      logger.warn('CreditTransaction creation failed due to unique constraint - transaction already exists', {
+        meterEventId,
+        creditGrantId,
+        error: error.message,
+      });
+
+      // 重新查询已存在的 transaction
+      const duplicateTransaction = await CreditTransaction.findOne({
+        where: {
+          source: meterEventId,
+          credit_grant_id: creditGrantId,
+        },
+      });
+
+      if (duplicateTransaction) {
+        context.transactions.push(duplicateTransaction.id);
+      }
+      return;
+    }
+
+    // 其他错误重新抛出
+    throw error;
+  }
+}
+
+export async function handleCreditConsumption(job: CreditConsumptionJob) {
+  const { meterEventId } = job;
+
+  logger.info('Starting credit consumption job', { meterEventId });
+
+  // Load and validate data first to get customer_id
+  const context: CreditConsumptionContext | null = await validateAndLoadData(meterEventId);
+  if (!context) {
+    logger.warn('Credit consumption context not found, skipping job', { meterEventId });
+    return;
+  }
+
+  const customerId = context.customer.id;
+  const lock = getLock(`credit-consumption-customer-${customerId}`);
+
+  try {
+    await lock.acquire();
+
+    // 重新加载MeterEvent以获取最新状态
+    await context.meterEvent.reload();
+    if (context.meterEvent.status === 'completed' || context.meterEvent.status === 'canceled') {
+      logger.info('MeterEvent already processed, skipping', {
+        meterEventId,
+        status: context.meterEvent.status,
+      });
+      return;
+    }
+
+    // 检查是否正在处理中，避免重复处理
+    if (context.meterEvent.status === 'processing') {
+      logger.warn('MeterEvent is already being processed, skipping duplicate job', {
+        meterEventId,
+        status: context.meterEvent.status,
+      });
+      return;
+    }
+
+    // Mark as processing to prevent duplicate processing
+    await context.meterEvent.markAsProcessing();
+
+    const totalRequiredAmount = context.meterEvent.getValue();
+
+    logger.info('Attempting to consume credits', {
+      meterEventId,
+      customerId,
+      totalRequiredAmount,
+      currencyId: context.meter.currency_id,
+    });
+
+    // Consume available credits (handles existing transactions internally)
+    const consumptionResult = await consumeAvailableCredits(context, totalRequiredAmount);
+
+    // Update MeterEvent with consumption details
+    await context.meterEvent.update({
+      credit_consumed: consumptionResult.consumed,
+      credit_pending: consumptionResult.pending,
+      metadata: {
+        ...context.meterEvent.metadata,
+        consumption_result: consumptionResult,
+        processed_at: new Date().toISOString(),
+      },
+    });
+
+    if (consumptionResult.fully_consumed) {
+      logger.info('Credit consumption completed successfully', {
+        meterEventId,
+        customerId,
+        totalRequired: totalRequiredAmount,
+        consumed: consumptionResult.consumed,
+        pending: consumptionResult.pending,
+        transactionCount: context.transactions.length,
+      });
+      await context.meterEvent.markAsCompleted();
+      if (context.subscription && context.subscription.status === 'past_due') {
+        handlePastDueSubscriptionRecovery(context.subscription, null);
+      }
+    } else {
+      logger.warn('Credit consumption partially completed - insufficient balance', {
+        meterEventId,
+        customerId,
+        totalRequired: totalRequiredAmount,
+        finalConsumed: consumptionResult.consumed,
+        finalPending: consumptionResult.pending,
+        availableBalance: consumptionResult.available_balance,
+      });
+      throw new Error(
+        `Insufficient credit balance: required ${totalRequiredAmount}, consumed ${consumptionResult.consumed}, pending ${consumptionResult.pending}`
+      );
+    }
+  } catch (error: any) {
+    logger.error('Credit consumption failed', {
+      meterEventId,
+      customerId,
+      error: error.message,
+    });
+
+    // Handle retry logic with more robust error handling
+    try {
+      const meterEvent = await MeterEvent.findByPk(meterEventId);
+      if (meterEvent && !['completed', 'canceled', 'requires_action'].includes(meterEvent.status)) {
+        const attemptCount = meterEvent.attempt_count + 1;
+
+        if (attemptCount >= MAX_RETRY_COUNT) {
+          await meterEvent.markAsRequiresAction(error.message);
+          logger.warn('MeterEvent marked as requires_action', {
+            meterEventId,
+            attemptCount,
+            reason: attemptCount >= MAX_RETRY_COUNT ? 'max_retries_exceeded' : 'non_retryable_error',
+          });
+        } else {
+          const nextAttemptTime = getNextRetry(attemptCount);
+          await meterEvent.markAsRequiresCapture(error.message, nextAttemptTime);
+
+          // Reschedule the job with exponential backoff
+          addCreditConsumptionJob(meterEventId, true, { runAt: nextAttemptTime });
+
+          logger.warn('Credit consumption retry scheduled', {
+            meterEventId,
+            attemptCount,
+            nextAttemptTime,
+            error: error.message,
+          });
+        }
+      }
+    } catch (updateError: any) {
+      logger.error('Failed to update meter event failure status', {
+        meterEventId,
+        originalError: error.message,
+        updateError: updateError.message,
+      });
+    }
+
+    // 重新抛出错误，让队列知道任务失败
+    throw error;
+  } finally {
+    lock.release();
+  }
+}
+
+export const creditQueue = createQueue<CreditConsumptionJob>({
+  name: 'credit-consumption',
+  onJob: handleCreditConsumption,
+  options: {
+    concurrency: 1,
+    maxRetries: 0,
+  },
+});
+
+creditQueue.on('finished', ({ id, job }) => {
+  logger.info('Credit consumption job finished', { id, job });
+});
+
+creditQueue.on('failed', ({ id, job, error }) => {
+  logger.error('Credit consumption job failed', { id, job, error });
+
+  // Error handling is now done in the main handler function
+  // following payment.ts pattern with proper retry logic
+});
+
+const addCreditConsumptionJob = async (
+  meterEventId: string,
+  replace: boolean = false,
+  options: { delay?: number; runAt?: number } = {}
+) => {
+  const jobId = `meter-event-${meterEventId}`;
+
+  try {
+    const existingJob = await creditQueue.get(jobId);
+    if (existingJob && !replace) {
+      logger.debug('Credit consumption job already exists, skipping duplicate', {
+        meterEventId,
+        jobId,
+        existingJobRunAt: existingJob.runAt,
+      });
+      return;
+    }
+
+    if (existingJob && replace) {
+      await creditQueue.delete(jobId);
+      logger.info('Replaced existing credit consumption job', {
+        meterEventId,
+        jobId,
+        previousRunAt: existingJob.runAt,
+      });
+    }
+
+    // 检查 MeterEvent 状态，避免为已完成的事件添加任务
+    const meterEvent = await MeterEvent.findByPk(meterEventId);
+    if (!meterEvent) {
+      logger.warn('Cannot add credit consumption job: MeterEvent not found', { meterEventId });
+      return;
+    }
+
+    if (meterEvent.status === 'completed' || meterEvent.status === 'canceled') {
+      logger.debug('Skipping credit consumption job: MeterEvent already processed', {
+        meterEventId,
+        status: meterEvent.status,
+      });
+      return;
+    }
+
+    creditQueue.push({ id: jobId, job: { meterEventId }, ...options });
+    logger.info('Credit consumption job added to queue', {
+      meterEventId,
+      jobId,
+      meterEventStatus: meterEvent.status,
+    });
+  } catch (error: any) {
+    logger.error('Failed to add credit consumption job', {
+      meterEventId,
+      jobId,
+      error: error.message,
+    });
+  }
+};
+
+creditQueue.on('retry', ({ id, job }) => {
+  logger.info('Retrying credit consumption job', { id, job });
+});
+
+export async function startCreditConsumeQueue(): Promise<void> {
+  const lock = getLock('startCreditConsumeQueue');
+  if (lock.locked) {
+    return;
+  }
+
+  try {
+    await lock.acquire();
+    logger.info('Starting credit queue and processing pending meter events');
+
+    // Find pending and requires_capture events that need processing
+    const pendingEvents = await MeterEvent.findAll({
+      where: {
+        status: ['pending', 'requires_capture'],
+      },
+      limit: 50,
+    });
+
+    const results = await Promise.allSettled(
+      pendingEvents.map(async (event) => {
+        const jobId = `meter-event-${event.id}`;
+        const existingJob = await creditQueue.get(jobId);
+        if (!existingJob) {
+          addCreditConsumptionJob(event.id, true);
+        }
+      })
+    );
+
+    const failed = results.filter((r) => r.status === 'rejected').length;
+    if (failed > 0) {
+      logger.warn(`Failed to queue ${failed} pending meter events`);
+    }
+
+    logger.info(`Credit queue started, processed ${pendingEvents.length} pending events`);
+  } catch (error: any) {
+    logger.error('Failed to start credit queue', { error: error.message });
+  } finally {
+    lock.release();
+  }
+}
+
+events.on('billing.meter_event.created', (meterEvent) => {
+  addCreditConsumptionJob(meterEvent.id, true);
+});
+
+events.on('customer.credit_grant.granted', async (creditGrant: CreditGrant) => {
+  try {
+    await retryFailedEventsForCustomer(creditGrant);
+  } catch (error: any) {
+    logger.error('Failed to retry failed events for customer', {
+      creditGrantId: creditGrant.id,
+      customerId: creditGrant.customer_id,
+      error: error.message,
+    });
+  }
+});
+
+async function retryFailedEventsForCustomer(creditGrant: CreditGrant): Promise<void> {
+  try {
+    logger.info('Retrying failed events for customer', {
+      customerId: creditGrant.customer_id,
+      currencyId: creditGrant.currency_id,
+      livemode: creditGrant.livemode,
+    });
+    const [, , failedEvents] = await MeterEvent.getPendingAmounts({
+      customerId: creditGrant.customer_id,
+      currencyId: creditGrant.currency_id,
+      status: ['requires_action', 'requires_capture'],
+      livemode: creditGrant.livemode,
+    });
+
+    if (failedEvents.length === 0) {
+      logger.debug('No failed events with pending credit found', {
+        customerId: creditGrant.customer_id,
+        currencyId: creditGrant.currency_id,
+      });
+      return;
+    }
+
+    logger.info('Updating failed events status after credit grant', {
+      customerId: creditGrant.customer_id,
+      currencyId: creditGrant.currency_id,
+      eventCount: failedEvents.length,
+    });
+
+    await Promise.all(
+      failedEvents.map((event) =>
+        event.update({
+          status: 'pending',
+          attempt_count: 0,
+          next_attempt: undefined,
+        })
+      )
+    );
+
+    failedEvents.forEach((event, index) => {
+      const delay = index * 1000;
+      addCreditConsumptionJob(event.id, true, { delay });
+    });
+
+    logger.info('Successfully updated failed events status', {
+      customerId: creditGrant.customer_id,
+      currencyId: creditGrant.currency_id,
+      eventCount: failedEvents.length,
+    });
+  } catch (error: any) {
+    logger.error('Failed to update failed events for customer', {
+      customerId: creditGrant.customer_id,
+      currencyId: creditGrant.currency_id,
+      error: error.message,
+    });
+  }
+}