payment-kit 1.26.4 → 1.27.0

package/api/src/queues/credit-consume.ts

@@ -1,20 +1,13 @@
 import { BN, fromUnitToToken } from '@ocap/util';
+import { Op } from 'sequelize';
+import pAll from 'p-all';
 
 import { getLock } from '../libs/lock';
 import logger from '../libs/logger';
-import dayjs from '../libs/dayjs';
 import createQueue from '../libs/queue';
 import { createEvent } from '../libs/audit';
-import {
-  MeterEvent,
-  CreditGrant,
-  CreditTransaction,
-  Meter,
-  Customer,
-  Subscription,
-  PaymentCurrency,
-  TMeterExpanded,
-} from '../store/models';
+import { MeterEvent, CreditGrant, CreditTransaction, Customer, Subscription, TMeterExpanded } from '../store/models';
+import { getCachedMeterExpanded } from '../libs/reference-cache';
 
 import { MAX_RETRY_COUNT, getNextRetry, formatCreditAmount } from '../libs/util';
 import { getDaysUntilCancel, getDueUnit, getMeterPriceIdsFromSubscription } from '../libs/subscription';
@@ -27,6 +20,11 @@ type CreditConsumptionJob = {
   meterEventId: string;
 };
 
+type BatchCreditConsumptionJob = {
+  meterEventIds: string[];
+  batchKey?: string;
+};
+
 type CreditConsumptionContext = {
   meterEvent: MeterEvent;
   meter: TMeterExpanded;
@@ -35,6 +33,7 @@ type CreditConsumptionContext = {
   priceIds?: string[];
   transactions: string[];
   warnings: string[];
+  _subscriptionId?: string;
 };
 
 type CreditConsumptionResult = {
@@ -78,33 +77,22 @@ async function checkLowBalance(
     });
   }
 }
-async function validateAndLoadData(meterEventId: string): Promise<CreditConsumptionContext | null> {
-  const meterEvent = await MeterEvent.findByPk(meterEventId);
-  if (!meterEvent) {
-    logger.warn('Skipping credit consumption job: MeterEvent not found', { meterEventId });
-    return null;
-  }
-
-  if (meterEvent.status === 'completed' || meterEvent.status === 'canceled') {
-    logger.info('Skipping credit consumption job: MeterEvent already processed', {
-      meterEventId,
-      status: meterEvent.status,
-    });
+async function loadReferenceData(
+  meterEventId: string,
+  meterEvent: MeterEvent
+): Promise<CreditConsumptionContext | null> {
+  const customerId = meterEvent.getCustomerId();
+  if (!customerId) {
+    logger.warn('Skipping credit consumption job: MeterEvent missing customer_id', { meterEventId });
     return null;
   }
 
-  if (meterEvent.status === 'requires_action') {
-    logger.warn('Skipping credit consumption job: MeterEvent requires manual intervention', {
-      meterEventId,
-      attemptCount: meterEvent.attempt_count,
-    });
-    return null;
-  }
+  const subscriptionId = meterEvent.getSubscriptionId();
 
-  const meter = (await Meter.findOne({
-    where: { event_name: meterEvent.event_name },
-    include: [{ model: PaymentCurrency, as: 'paymentCurrency' }],
-  })) as TMeterExpanded | null;
+  const [meter, customer] = await Promise.all([
+    getCachedMeterExpanded(meterEvent.event_name) as Promise<TMeterExpanded | null>,
+    Customer.findByPk(customerId),
+  ]);
 
   if (!meter) {
     logger.warn('Skipping credit consumption job: Meter not found', {
@@ -122,13 +110,6 @@ async function validateAndLoadData(meterEventId: string): Promise<CreditConsumpt
     return null;
   }
 
-  const customerId = meterEvent.getCustomerId();
-  if (!customerId) {
-    logger.warn('Skipping credit consumption job: MeterEvent missing customer_id', { meterEventId });
-    return null;
-  }
-
-  const customer = await Customer.findByPk(customerId);
   if (!customer) {
     logger.warn('Skipping credit consumption job: Customer not found', {
       meterEventId,
@@ -137,36 +118,13 @@ async function validateAndLoadData(meterEventId: string): Promise<CreditConsumpt
     return null;
   }
 
-  const subscriptionId = meterEvent.getSubscriptionId();
-  if (subscriptionId) {
-    const subscription = await Subscription.findByPk(subscriptionId);
-    if (!subscription) {
-      logger.warn('Skipping credit consumption job: Subscription not found', {
-        meterEventId,
-        subscriptionId,
-      });
-      return null;
-    }
-
-    const priceIds = await getMeterPriceIdsFromSubscription(subscription);
-
-    return {
-      meterEvent,
-      meter,
-      customer,
-      subscription: subscription!,
-      transactions: [],
-      warnings: [],
-      priceIds,
-    };
-  }
-
   return {
     meterEvent,
     meter,
     customer,
     transactions: [],
     warnings: [],
+    _subscriptionId: subscriptionId,
   };
 }
 
@@ -184,6 +142,8 @@ async function consumeAvailableCredits(
     },
   });
 
+  const existingTxByGrantId = new Map(existingTransactions.map((tx) => [tx.credit_grant_id, tx]));
+
   const alreadyConsumed = existingTransactions.reduce((sum, tx) => sum.add(new BN(tx.credit_amount)), new BN(0));
   const remainingToConsume = new BN(totalRequiredAmount).sub(alreadyConsumed);
 
@@ -205,7 +165,6 @@ async function consumeAvailableCredits(
     };
   }
 
-  // Get all available grants sorted by priority
   const availableGrants = await CreditGrant.getAvailableCreditsForCustomer(customerId, currencyId, context.priceIds);
 
   const totalCreditAmountBN = availableGrants.reduce((sum, grant) => sum.add(new BN(grant.amount)), new BN(0));
@@ -229,7 +188,7 @@ async function consumeAvailableCredits(
 
     // if consumeAmount is greater than 0, process grant consumption
     if (consumeAmount.gt(new BN(0))) {
-      await processGrantConsumption(context, grant, consumeAmount.toString()); // eslint-disable-line no-await-in-loop
+      await processGrantConsumption(context, grant, consumeAmount.toString(), existingTxByGrantId); // eslint-disable-line no-await-in-loop
       consumed = consumed.add(consumeAmount);
       remaining = remaining.sub(consumeAmount);
     }
@@ -246,12 +205,48 @@ async function consumeAvailableCredits(
     newlyConsumed: consumed.toString(),
     totalConsumed: totalConsumed.toString(),
     finalPending: finalPending.toString(),
+    grantCount: availableGrants.length,
   });
 
-  const pendingAmount = Math.max(0, finalPending.toNumber()).toString();
+  const pendingAmount = finalPending.lte(new BN(0)) ? '0' : finalPending.toString();
   const remainingBalance = totalAvailable.sub(consumed).toString();
 
-  // Track whether insufficient event was triggered to skip low_balance notification
+  // Handle insufficient/low balance events
+  await handlePostConsumptionEvents(context, {
+    consumed: consumed.toString(),
+    remainingToConsume,
+    totalAvailable,
+    pendingAmount,
+    remainingBalance,
+    totalCreditAmountBN,
+    availableGrants,
+  });
+
+  return {
+    consumed: totalConsumed.toString(),
+    pending: pendingAmount,
+    available_balance: remainingBalance,
+    fully_consumed: finalPending.lte(new BN(0)),
+  };
+}
+
+async function handlePostConsumptionEvents(
+  context: CreditConsumptionContext,
+  params: {
+    consumed: string;
+    remainingToConsume: BN;
+    totalAvailable: BN;
+    pendingAmount: string;
+    remainingBalance: string;
+    totalCreditAmountBN: BN;
+    availableGrants: CreditGrant[];
+  }
+): Promise<void> {
+  const customerId = context.meterEvent.getCustomerId();
+  const currencyId = context.meter.currency_id!;
+  const { consumed, remainingToConsume, totalAvailable, pendingAmount, remainingBalance, totalCreditAmountBN } = params;
+  const finalPending = new BN(pendingAmount);
+
   let insufficientTriggered = false;
 
   // 如果无法完全消费所需额度，记录不足事件
@@ -259,9 +254,9 @@ async function consumeAvailableCredits(
     logger.warn('Insufficient credit balance', {
       customerId,
       currencyId,
-      totalRequiredAmount,
+      requiredAmount: remainingToConsume.toString(),
       availableAmount: totalAvailable.toString(),
-      totalConsumed: totalConsumed.toString(),
+      consumedAmount: consumed,
       pendingAmount,
     });
     if ((context.subscription && context.subscription.isActive()) || !context.subscription) {
@@ -273,7 +268,7 @@ async function consumeAvailableCredits(
           meter_event_name: context.meterEvent.event_name,
           required_amount: remainingToConsume.toString(),
           available_amount: totalAvailable.toString(),
-          consumed_amount: consumed.toString(),
+          consumed_amount: consumed,
           pending_amount: pendingAmount,
           currency_id: currencyId,
           subscription_id: context.subscription?.id,
@@ -302,7 +297,7 @@ async function consumeAvailableCredits(
         },
       });
     }
-  } else if (remainingBalance === '0') {
+  } else if (new BN(remainingBalance).lte(new BN(0))) {
     insufficientTriggered = true;
     await createEvent('Customer', 'customer.credit.insufficient', context.customer, {
       metadata: {
@@ -311,7 +306,7 @@ async function consumeAvailableCredits(
         meter_event_name: context.meterEvent.event_name,
         required_amount: remainingToConsume.toString(),
         available_amount: '0',
-        consumed_amount: consumed.toString(),
+        consumed_amount: consumed,
         pending_amount: pendingAmount,
         currency_id: currencyId,
         subscription_id: context.subscription?.id,
@@ -319,23 +314,16 @@ async function consumeAvailableCredits(
     }).catch(console.error);
   }
 
-  // Skip low_balance check if insufficient was already triggered (insufficient takes priority)
   if (!insufficientTriggered) {
     await checkLowBalance(customerId, currencyId, totalCreditAmountBN.toString(), remainingBalance, context);
   }
-
-  return {
-    consumed: totalConsumed.toString(),
-    pending: pendingAmount,
-    available_balance: remainingBalance,
-    fully_consumed: finalPending.lte(new BN(0)),
-  };
 }
 
 async function processGrantConsumption(
   context: CreditConsumptionContext,
   creditGrant: CreditGrant,
-  consumeAmount: string
+  consumeAmount: string,
+  existingTxByGrantId?: Map<string, CreditTransaction>
 ): Promise<void> {
   logger.info('Processing grant consumption', {
     creditGrantId: creditGrant.id,
@@ -347,7 +335,7 @@ async function processGrantConsumption(
     meter_event_id: context.meterEvent.id,
   });
 
-  const transactionId = await createCreditTransaction(context, creditGrant, consumeAmount, result);
+  const transactionId = await createCreditTransaction(context, creditGrant, consumeAmount, result, existingTxByGrantId);
 
   if (CreditGrant.hasOnchainToken(creditGrant)) {
     addTokenTransferJob({
@@ -371,18 +359,13 @@ async function createCreditTransaction(
   context: CreditConsumptionContext,
   creditGrant: CreditGrant,
   consumeAmount: string,
-  consumeResult: any
+  consumeResult: any,
+  existingTxByGrantId?: Map<string, CreditTransaction>
 ): Promise<string> {
   const meterEventId = context.meterEvent.id;
   const creditGrantId = creditGrant.id;
 
-  // 检查是否已经为这个 MeterEvent 和 CreditGrant 创建过 transaction
-  const existingTransaction = await CreditTransaction.findOne({
-    where: {
-      source: meterEventId,
-      credit_grant_id: creditGrantId,
-    },
-  });
+  const existingTransaction = existingTxByGrantId?.get(creditGrantId) ?? null;
 
   if (existingTransaction) {
     logger.warn('CreditTransaction already exists for this MeterEvent and CreditGrant', {
@@ -487,13 +470,39 @@ async function createCreditTransaction(
   }
 }
 
+// ============================================================================
+// Single-event handler
+// ============================================================================
+
 export async function handleCreditConsumption(job: CreditConsumptionJob) {
   const { meterEventId } = job;
+  const jobStart = performance.now();
 
   logger.info('Starting credit consumption job', { meterEventId });
 
-  // Load and validate data first to get customer_id
-  const context: CreditConsumptionContext | null = await validateAndLoadData(meterEventId);
+  // Pre-check before acquiring lock
+  const preCheckEvent = await MeterEvent.findByPk(meterEventId);
+  if (!preCheckEvent) {
+    logger.warn('Skipping credit consumption job: MeterEvent not found', { meterEventId });
+    return;
+  }
+  if (preCheckEvent.status === 'completed' || preCheckEvent.status === 'canceled') {
+    logger.info('Skipping credit consumption job: MeterEvent already processed', {
+      meterEventId,
+      status: preCheckEvent.status,
+    });
+    return;
+  }
+  if (preCheckEvent.status === 'requires_action') {
+    logger.warn('Skipping credit consumption job: MeterEvent requires manual intervention', { meterEventId });
+    return;
+  }
+
+  // Load reference data
+  const validateStart = performance.now();
+  const context: CreditConsumptionContext | null = await loadReferenceData(meterEventId, preCheckEvent);
+  const validateTime = performance.now() - validateStart;
+
   if (!context) {
     logger.warn('Credit consumption context not found, skipping job', { meterEventId });
     return;
@@ -502,32 +511,62 @@ export async function handleCreditConsumption(job: CreditConsumptionJob) {
   const customerId = context.customer.id;
   const lock = getLock(`credit-consumption-customer-${customerId}`);
 
+  let rechargeParams: {
+    customer: Customer;
+    currencyId: string;
+    availableBalance: string;
+    currency: any;
+    meter: TMeterExpanded;
+  } | null = null;
+  let rechargeTime = 0;
+
   try {
+    const lockStart = performance.now();
     await lock.acquire();
+    const lockTime = performance.now() - lockStart;
+
+    // Fresh loads inside lock for consistency
+    const [freshEvent, freshSubscription] = await Promise.all([
+      MeterEvent.findByPk(meterEventId),
+      context._subscriptionId ? Subscription.findByPk(context._subscriptionId) : Promise.resolve(null),
+    ]);
+    if (!freshEvent) {
+      logger.warn('MeterEvent disappeared after lock acquired', { meterEventId });
+      return;
+    }
+    context.meterEvent = freshEvent;
+
+    if (context._subscriptionId && !freshSubscription) {
+      logger.warn('Skipping credit consumption: Subscription not found inside lock', {
+        meterEventId,
+        subscriptionId: context._subscriptionId,
+      });
+      return;
+    }
+    if (freshSubscription) {
+      context.subscription = freshSubscription;
+      context.priceIds = await getMeterPriceIdsFromSubscription(freshSubscription);
+    }
 
-    // 重新加载MeterEvent以获取最新状态
-    await context.meterEvent.reload();
-    if (context.meterEvent.status === 'completed' || context.meterEvent.status === 'canceled') {
+    if (freshEvent.status === 'completed' || freshEvent.status === 'canceled') {
       logger.info('MeterEvent already processed, skipping', {
         meterEventId,
-        status: context.meterEvent.status,
+        status: freshEvent.status,
       });
       return;
     }
 
-    // 检查是否正在处理中，避免重复处理
-    if (context.meterEvent.status === 'processing') {
+    if (freshEvent.status === 'processing') {
       logger.warn('MeterEvent is already being processed, skipping duplicate job', {
         meterEventId,
-        status: context.meterEvent.status,
+        status: freshEvent.status,
       });
       return;
     }
 
     // Mark as processing to prevent duplicate processing
-    await context.meterEvent.markAsProcessing();
-
-    const totalRequiredAmount = context.meterEvent.getValue();
+    await freshEvent.markAsProcessing();
+    const totalRequiredAmount = freshEvent.getValue();
 
     logger.info('Attempting to consume credits', {
       meterEventId,
@@ -536,35 +575,34 @@ export async function handleCreditConsumption(job: CreditConsumptionJob) {
       currencyId: context.meter.currency_id,
     });
 
-    // Consume available credits (handles existing transactions internally)
+    const consumeStart = performance.now();
     const consumptionResult = await consumeAvailableCredits(context, totalRequiredAmount);
+    const consumeTime = performance.now() - consumeStart;
 
-    // Check for auto recharge after successful consumption
-    try {
-      await checkAndTriggerAutoRecharge(
-        context.customer,
-        context.meter.currency_id!,
-        consumptionResult.available_balance
-      );
-    } catch (error: any) {
-      logger.warn('Auto recharge check failed after credit consumption', {
-        meterEventId,
-        customerId,
-        error: error.message,
+    if (consumptionResult.fully_consumed) {
+      await freshEvent.update({
+        status: 'completed',
+        processed_at: Math.floor(Date.now() / 1000),
+        next_attempt: undefined,
+        credit_consumed: consumptionResult.consumed,
+        credit_pending: consumptionResult.pending,
+        metadata: {
+          ...freshEvent.metadata,
+          consumption_result: consumptionResult,
+          last_error: undefined,
+          failed_at: undefined,
+        },
       });
-    }
-    // Update MeterEvent with consumption details
-    await context.meterEvent.update({
-      credit_consumed: consumptionResult.consumed,
-      credit_pending: consumptionResult.pending,
-      metadata: {
-        ...context.meterEvent.metadata,
-        consumption_result: consumptionResult,
-        processed_at: new Date().toISOString(),
-      },
-    });
 
-    if (consumptionResult.fully_consumed) {
+      rechargeParams = {
+        customer: context.customer,
+        currencyId: context.meter.currency_id!,
+        availableBalance: consumptionResult.available_balance,
+        currency: context.meter.paymentCurrency,
+        meter: context.meter,
+      };
+
+      const totalTime = performance.now() - jobStart;
       logger.info('Credit consumption completed successfully', {
         meterEventId,
         customerId,
@@ -572,12 +610,29 @@ export async function handleCreditConsumption(job: CreditConsumptionJob) {
         consumed: consumptionResult.consumed,
         pending: consumptionResult.pending,
         transactionCount: context.transactions.length,
+        timing: {
+          validateMs: Math.round(validateTime),
+          lockMs: Math.round(lockTime),
+          consumeMs: Math.round(consumeTime),
+          totalMs: Math.round(totalTime),
+        },
       });
-      await context.meterEvent.markAsCompleted();
+
       if (context.subscription && context.subscription.status === 'past_due') {
         await handlePastDueSubscriptionRecovery(context.subscription, null);
       }
     } else {
+      // Save partial consumption progress before throwing
+      await freshEvent.update({
+        credit_consumed: consumptionResult.consumed,
+        credit_pending: consumptionResult.pending,
+        metadata: {
+          ...freshEvent.metadata,
+          consumption_result: consumptionResult,
+          processed_at: new Date().toISOString(),
+        },
+      });
+
       logger.warn('Credit consumption partially completed - insufficient balance', {
         meterEventId,
         customerId,
@@ -586,6 +641,16 @@ export async function handleCreditConsumption(job: CreditConsumptionJob) {
         finalPending: consumptionResult.pending,
         availableBalance: consumptionResult.available_balance,
       });
+
+      // Trigger auto-recharge even for partial consumption
+      rechargeParams = {
+        customer: context.customer,
+        currencyId: context.meter.currency_id!,
+        availableBalance: consumptionResult.available_balance,
+        currency: context.meter.paymentCurrency,
+        meter: context.meter,
+      };
+
       throw new Error(
         `Insufficient credit balance: required ${totalRequiredAmount}, consumed ${consumptionResult.consumed}, pending ${consumptionResult.pending}`
       );
@@ -614,8 +679,7 @@ export async function handleCreditConsumption(job: CreditConsumptionJob) {
           const nextAttemptTime = getNextRetry(attemptCount);
           await meterEvent.markAsRequiresCapture(error.message, nextAttemptTime);
 
-          // Reschedule the job with exponential backoff
-          addCreditConsumptionJob(meterEventId, true, { runAt: nextAttemptTime });
+          addCreditConsumptionJob(meterEventId, true, { runAt: nextAttemptTime, skipStatusCheck: true });
 
           logger.warn('Credit consumption retry scheduled', {
             meterEventId,
@@ -637,14 +701,541 @@ export async function handleCreditConsumption(job: CreditConsumptionJob) {
     throw error;
   } finally {
     lock.release();
+
+    // Auto-recharge check outside lock to reduce lock hold time
+    // Runs for both successful and partial consumption (insufficient balance)
+    if (rechargeParams) {
+      const rechargeStart = performance.now();
+      try {
+        await checkAndTriggerAutoRecharge(
+          rechargeParams.customer,
+          rechargeParams.currencyId,
+          rechargeParams.availableBalance,
+          {
+            currency: rechargeParams.currency,
+            meter: rechargeParams.meter,
+          }
+        );
+      } catch (error: any) {
+        logger.warn('Auto recharge check failed after credit consumption', {
+          meterEventId,
+          customerId,
+          error: error.message,
+        });
+      }
+      rechargeTime = performance.now() - rechargeStart;
+      if (rechargeTime > 50) {
+        logger.info('Auto recharge check timing', { meterEventId, rechargeMs: Math.round(rechargeTime) });
+      }
+    }
+  }
+}
+
+// ============================================================================
+// Batch credit consumption
+// ============================================================================
+const CREDIT_BATCH_SIZE = Math.max(1, parseInt(process.env.CREDIT_BATCH_SIZE || '50', 10));
+const CREDIT_BATCH_WINDOW_MS = Math.max(10, parseInt(process.env.CREDIT_BATCH_WINDOW_MS || '3000', 10));
+
+type PendingBatch = { eventIds: string[]; timer: NodeJS.Timeout | null };
+const pendingBatches = new Map<string, PendingBatch>();
+// Track in-flight batch jobs per key to avoid head-of-line blocking.
+// Only one batch job per key runs at a time; new events accumulate until it finishes.
+const inflightBatches = new Set<string>();
+
+function getBatchKey(customerId: string, eventName: string, subscriptionId?: string): string {
+  return `${customerId}::${eventName}::${subscriptionId || ''}`;
+}
+
+function addToBatch(customerId: string, eventName: string, meterEventId: string, subscriptionId?: string) {
+  const key = getBatchKey(customerId, eventName, subscriptionId);
+  let batch = pendingBatches.get(key);
+  if (!batch) {
+    batch = { eventIds: [], timer: null };
+    pendingBatches.set(key, batch);
+  }
+
+  batch.eventIds.push(meterEventId);
+
+  if (batch.eventIds.length >= CREDIT_BATCH_SIZE) {
+    flushBatch(key);
+    return;
+  }
+
+  if (!batch.timer) {
+    batch.timer = setTimeout(() => flushBatch(key), CREDIT_BATCH_WINDOW_MS);
+  }
+}
+
+function flushBatch(key: string) {
+  const batch = pendingBatches.get(key);
+  if (!batch || batch.eventIds.length === 0) {
+    pendingBatches.delete(key);
+    return;
+  }
+
+  // If a batch for this key is already processing, keep events pending.
+  // They will be flushed when the in-flight batch completes (see onBatchComplete).
+  if (inflightBatches.has(key)) {
+    return;
+  }
+
+  if (batch.timer) {
+    clearTimeout(batch.timer);
+  }
+  const eventIds = batch.eventIds.splice(0);
+  pendingBatches.delete(key);
+
+  inflightBatches.add(key);
+  const jobId = `batch-credit-${key}-${Date.now()}`;
+  creditQueue.push({
+    id: jobId,
+    job: { meterEventIds: eventIds, batchKey: key } as any,
+    skipDuplicateCheck: true,
+  });
+}
+
+function onBatchComplete(key: string) {
+  inflightBatches.delete(key);
+  // Flush any events that accumulated while this batch was processing
+  if (pendingBatches.has(key)) {
+    flushBatch(key);
+  }
+}
+
+// Graceful shutdown: flush pending batches. If push doesn't persist before exit,
+// events remain 'pending' and are recovered by startCreditConsumeQueue on restart.
+function flushAllBatches() {
+  const keys = Array.from(pendingBatches.keys());
+  keys.forEach((key) => flushBatch(key));
+}
+process.on('SIGTERM', flushAllBatches);
+process.on('SIGINT', flushAllBatches);
+
+export async function handleBatchCreditConsumption(job: BatchCreditConsumptionJob) {
+  const { meterEventIds, batchKey } = job;
+  const batchStart = performance.now();
+
+  try {
+    await handleBatchCreditConsumptionInner(meterEventIds, batchStart);
+  } finally {
+    if (batchKey) {
+      onBatchComplete(batchKey);
+    }
   }
 }
 
-export const creditQueue = createQueue<CreditConsumptionJob>({
+async function handleBatchCreditConsumptionInner(meterEventIds: string[], batchStart: number) {
+  if (!meterEventIds || meterEventIds.length === 0) return;
+
+  if (meterEventIds.length === 1) {
+    await handleCreditConsumption({ meterEventId: meterEventIds[0]! });
+    return;
+  }
+
+  // ==========================================
+  // Pre-check: filter already-processed events
+  // ==========================================
+  const preCheckEvents = await MeterEvent.findAll({
+    where: { id: { [Op.in]: meterEventIds } },
+  });
+
+  const processableEvents = preCheckEvents.filter(
+    (e) => !['completed', 'canceled', 'requires_action'].includes(e.status)
+  );
+  if (processableEvents.length === 0) return;
+
+  const customerId = processableEvents[0]!.getCustomerId();
+  const eventName = processableEvents[0]!.event_name;
+
+  const mismatchedEvents = processableEvents.filter((e) => e.event_name !== eventName);
+  if (mismatchedEvents.length > 0) {
+    logger.error('Batch contains mixed event_names, falling back to single processing', {
+      customerId,
+      expectedEventName: eventName,
+      mismatchedCount: mismatchedEvents.length,
+    });
+    for (const event of processableEvents) {
+      addCreditConsumptionJob(event.id, true, { skipStatusCheck: true });
+    }
+    return;
+  }
+
+  // ==========================================
+  // Load reference data
+  // ==========================================
+  const [meter, customer] = await Promise.all([
+    getCachedMeterExpanded(eventName) as Promise<TMeterExpanded | null>,
+    Customer.findByPk(customerId),
+  ]);
+
+  if (!meter || !meter.currency_id || !customer) {
+    logger.warn('Batch credit consumption: reference data not found', {
+      batchSize: meterEventIds.length,
+      customerId,
+    });
+    return;
+  }
+
+  const currencyId = meter.currency_id;
+  const lock = getLock(`credit-consumption-customer-${customerId}`);
+
+  let rechargeParams: {
+    customer: Customer;
+    currencyId: string;
+    availableBalance: string;
+    currency: any;
+    meter: TMeterExpanded;
+  } | null = null;
+
+  try {
+    const lockStart = performance.now();
+    await lock.acquire();
+    const lockTime = performance.now() - lockStart;
+
+    // ==========================================
+    // Fresh read inside lock
+    // ==========================================
+    const processableIds = processableEvents.map((e) => e.id);
+    const freshEvents = await MeterEvent.findAll({
+      where: { id: { [Op.in]: processableIds } },
+    });
+
+    const freshProcessable = freshEvents.filter(
+      (e) => !['completed', 'canceled', 'processing', 'requires_action'].includes(e.status)
+    );
+    if (freshProcessable.length === 0) return;
+
+    // Subscription fresh read (once for entire batch)
+    const subscriptionId = freshProcessable[0]!.getSubscriptionId();
+    let subscription: Subscription | null = null;
+    let priceIds: string[] | undefined;
+
+    if (subscriptionId) {
+      subscription = await Subscription.findByPk(subscriptionId);
+      if (!subscription) {
+        logger.warn('Batch: Subscription not found inside lock, skipping', {
+          customerId,
+          subscriptionId,
+        });
+        return;
+      }
+      priceIds = await getMeterPriceIdsFromSubscription(subscription);
+    }
+
+    // Mark all processable events as 'processing' in one SQL
+    await MeterEvent.update(
+      { status: 'processing' },
+      { where: { id: { [Op.in]: freshProcessable.map((e) => e.id) } } }
+    );
+    // Sync in-memory status for subsequent logic
+    freshProcessable.forEach((e) => {
+      e.status = 'processing';
+    });
+
+    // ==========================================
+    // Batch idempotency check
+    // ==========================================
+    const allEventIds = freshProcessable.map((e) => e.id);
+    const existingTransactions = await CreditTransaction.findAll({
+      where: { source: { [Op.in]: allEventIds } },
+    });
+    const txBySource = new Map<string, CreditTransaction[]>();
+    for (const tx of existingTransactions) {
+      const source = tx.source!;
+      const arr = txBySource.get(source) || [];
+      arr.push(tx);
+      txBySource.set(source, arr);
+    }
+
+    // ==========================================
+    // Query available grants once for entire batch
+    // ==========================================
+    const availableGrants = await CreditGrant.getAvailableCreditsForCustomer(customerId, currencyId, priceIds);
+
+    const totalCreditAmountBN = availableGrants.reduce((sum, grant) => sum.add(new BN(grant.amount)), new BN(0));
+
+    // ==========================================
+    // Consume credits per event — MUST be sequential because events share
+    // the same grant objects and consumeCredit() mutates remaining_amount.
+    // ==========================================
+    /* eslint-disable no-await-in-loop */
+    const successEvents: Array<MeterEvent & { _consumptionResult?: CreditConsumptionResult }> = [];
+    const failedEvents: { event: MeterEvent; error: Error }[] = [];
+
+    // FIFO order by creation time
+    freshProcessable.sort((a, b) => {
+      const aTime = a.created_at instanceof Date ? a.created_at.getTime() : Number(a.created_at);
+      const bTime = b.created_at instanceof Date ? b.created_at.getTime() : Number(b.created_at);
+      return aTime - bTime;
+    });
+
+    for (const freshEvent of freshProcessable) {
+      try {
+        const context: CreditConsumptionContext = {
+          meterEvent: freshEvent,
+          meter,
+          customer,
+          subscription: subscription || undefined,
+          priceIds,
+          transactions: [],
+          warnings: [],
+        };
+
+        const totalRequiredAmount = freshEvent.getValue();
+
+        // Use pre-fetched existingTx data (from batch query)
+        const eventExistingTx = txBySource.get(freshEvent.id) || [];
+        const existingTxByGrantId = new Map(eventExistingTx.map((tx) => [tx.credit_grant_id, tx]));
+        const alreadyConsumed = eventExistingTx.reduce((sum, tx) => sum.add(new BN(tx.credit_amount)), new BN(0));
+        const remainingToConsume = new BN(totalRequiredAmount).sub(alreadyConsumed);
+
+        if (remainingToConsume.lte(new BN(0))) {
+          // Already fully consumed
+          (freshEvent as any)._consumptionResult = {
+            consumed: alreadyConsumed.toString(),
+            pending: '0',
+            available_balance: '0',
+            fully_consumed: true,
+          };
+          successEvents.push(freshEvent);
+          continue; // eslint-disable-line no-continue
+        }
+
+        // Consume from grants (grant.remaining_amount is updated in-memory by consumeCredit)
+        let remaining = remainingToConsume;
+        let consumed = new BN(0);
+
+        for (const grant of availableGrants) {
+          if (remaining.lte(new BN(0))) break;
+
+          // Read directly from grant object — consumeCredit() updates this.remaining_amount after each call
+          const grantBalance = new BN(grant.remaining_amount);
+          if (grantBalance.lte(new BN(0))) continue; // eslint-disable-line no-continue
+
+          const consumeAmount = remaining.lte(grantBalance) ? remaining : grantBalance;
+
+          if (consumeAmount.gt(new BN(0))) {
+            await processGrantConsumption(context, grant, consumeAmount.toString(), existingTxByGrantId);
+            consumed = consumed.add(consumeAmount);
+            remaining = remaining.sub(consumeAmount);
+          }
+        }
+
+        const totalConsumed = alreadyConsumed.add(consumed);
+        const finalPending = new BN(totalRequiredAmount).sub(totalConsumed);
+        const totalAvailable = availableGrants.reduce((sum, g) => sum.add(new BN(g.remaining_amount)), new BN(0));
+        const remainingBalance = totalAvailable.toString();
+
+        if (finalPending.lte(new BN(0))) {
+          (freshEvent as any)._consumptionResult = {
+            consumed: totalConsumed.toString(),
+            pending: '0',
+            available_balance: remainingBalance,
+            fully_consumed: true,
+          };
+          successEvents.push(freshEvent);
+        } else {
+          // Insufficient balance — handle events + save partial progress
+          const pendingAmount = finalPending.toString();
+          await handlePostConsumptionEvents(context, {
+            consumed: consumed.toString(),
+            remainingToConsume,
+            totalAvailable,
+            pendingAmount,
+            remainingBalance,
+            totalCreditAmountBN,
+            availableGrants,
+          });
+
+          // Save partial consumption progress
+          await freshEvent.update({
+            credit_consumed: totalConsumed.toString(),
+            credit_pending: pendingAmount,
+            metadata: {
+              ...freshEvent.metadata,
+              consumption_result: {
+                consumed: totalConsumed.toString(),
+                pending: pendingAmount,
+                available_balance: remainingBalance,
+                fully_consumed: false,
+              },
+              processed_at: new Date().toISOString(),
+            },
+          });
+
+          failedEvents.push({
+            event: freshEvent,
+            error: new Error(
+              `Insufficient credit balance: required ${totalRequiredAmount}, consumed ${totalConsumed.toString()}, pending ${pendingAmount}`
+            ),
+          });
+        }
+      } catch (error: any) {
+        failedEvents.push({ event: freshEvent, error });
+      }
+    }
+    /* eslint-enable no-await-in-loop */
+    // ==========================================
+    // Update successful events
+    // ==========================================
+    if (successEvents.length > 0) {
+      const now = Math.floor(Date.now() / 1000);
+      await pAll(
+        successEvents.map(
+          (event) => () =>
+            event.update({
+              status: 'completed',
+              processed_at: now,
+              next_attempt: undefined,
+              credit_consumed: (event as any)._consumptionResult?.consumed || '0',
+              credit_pending: '0',
+              metadata: {
+                ...event.metadata,
+                consumption_result: (event as any)._consumptionResult,
+                last_error: undefined,
+                failed_at: undefined,
+                batch_processed: true,
+              },
+            })
+        ),
+        { concurrency: 5 }
+      );
+    }
+
+    // Calculate actual available balance for auto-recharge
+    const finalAvailableBalance = availableGrants
+      .reduce((sum, g) => sum.add(new BN(g.remaining_amount)), new BN(0))
+      .toString();
+
+    // Check low balance after batch consumption (only if all events succeeded, no insufficient already triggered)
+    if (successEvents.length > 0 && failedEvents.length === 0) {
+      await checkLowBalance(customerId, currencyId, totalCreditAmountBN.toString(), finalAvailableBalance, {
+        meterEvent: successEvents[successEvents.length - 1]!,
+        meter,
+        customer,
+        subscription: subscription || undefined,
+        priceIds,
+        transactions: [],
+        warnings: [],
+      });
+    }
+
+    rechargeParams = {
+      customer,
+      currencyId,
+      availableBalance: finalAvailableBalance,
+      currency: meter.paymentCurrency,
+      meter,
+    };
+
+    // ==========================================
+    // Retry failed events — each is independent, safe to parallelize
+    // ==========================================
+    await pAll(
+      failedEvents.map(({ event, error: failError }) => async () => {
+        try {
+          const attemptCount = event.attempt_count + 1;
+          if (attemptCount >= MAX_RETRY_COUNT) {
+            await event.markAsRequiresAction(failError.message);
+          } else {
+            const nextAttemptTime = getNextRetry(attemptCount);
+            await event.markAsRequiresCapture(failError.message, nextAttemptTime);
+            addCreditConsumptionJob(event.id, true, { runAt: nextAttemptTime, skipStatusCheck: true });
+          }
+        } catch (updateError: any) {
+          logger.error('Batch: failed to update meter event failure status', {
+            meterEventId: event.id,
+            error: updateError.message,
+          });
+        }
+      }),
+      { concurrency: 5 }
+    );
+
+    // Past due recovery
+    if (subscription && subscription.status === 'past_due' && successEvents.length > 0) {
+      await handlePastDueSubscriptionRecovery(subscription, null);
+    }
+
+    const totalTime = performance.now() - batchStart;
+    logger.info('Batch credit consumption completed', {
+      customerId,
+      batchSize: meterEventIds.length,
+      processable: freshProcessable.length,
+      succeeded: successEvents.length,
+      failed: failedEvents.length,
+      grantCount: availableGrants.length,
+      timing: {
+        lockMs: Math.round(lockTime),
+        totalMs: Math.round(totalTime),
+        avgPerEvent: Math.round(totalTime / freshProcessable.length),
+      },
+    });
+  } catch (error: any) {
+    logger.error('Batch credit consumption failed, falling back to single processing', {
+      customerId,
+      batchSize: meterEventIds.length,
+      error: error.message,
+    });
+    // Batch-level error: reset only events still in 'processing' to 'requires_capture'.
+    // Some events may have already been completed before the error occurred.
+    // Use requires_capture (not pending) so attempt_count increments, preventing infinite retry loops.
+    try {
+      await MeterEvent.update(
+        { status: 'requires_capture' },
+        { where: { id: { [Op.in]: meterEventIds }, status: 'processing' } }
+      );
+    } catch (resetError: any) {
+      logger.error('Failed to reset event status before single fallback', { error: resetError.message });
+    }
+    for (const id of meterEventIds) {
+      addCreditConsumptionJob(id, true, { skipStatusCheck: false });
+    }
+  } finally {
+    lock.release();
+  }
+
+  // Auto-recharge check outside lock
+  if (rechargeParams) {
+    try {
+      await checkAndTriggerAutoRecharge(
+        rechargeParams.customer,
+        rechargeParams.currencyId,
+        rechargeParams.availableBalance,
+        {
+          currency: rechargeParams.currency,
+          meter: rechargeParams.meter,
+        }
+      );
+    } catch (error: any) {
+      logger.warn('Auto recharge check failed after batch consumption', {
+        customerId,
+        error: error.message,
+      });
+    }
+  }
+}
+
+// ============================================================================
+// Queue setup
+// ============================================================================
+
+const creditQueueConcurrency = Math.max(
+  1,
+  Math.min(20, parseInt(process.env.CREDIT_QUEUE_CONCURRENCY || '5', 10) || 5)
+);
+
+export const creditQueue = createQueue<CreditConsumptionJob | BatchCreditConsumptionJob>({
   name: 'credit-consumption',
-  onJob: handleCreditConsumption,
+  onJob: (job) => {
+    if ('meterEventIds' in job && Array.isArray((job as BatchCreditConsumptionJob).meterEventIds)) {
+      return handleBatchCreditConsumption(job as BatchCreditConsumptionJob);
+    }
+    return handleCreditConsumption(job as CreditConsumptionJob);
+  },
   options: {
-    concurrency: 1,
+    concurrency: creditQueueConcurrency,
     maxRetries: 0,
     enableScheduledJob: true,
   },
@@ -664,7 +1255,7 @@ creditQueue.on('failed', ({ id, job, error }) => {
 const addCreditConsumptionJob = async (
   meterEventId: string,
   replace: boolean = false,
-  options: { delay?: number; runAt?: number } = {}
+  options: { delay?: number; runAt?: number; skipStatusCheck?: boolean } = {}
 ) => {
   const jobId = `meter-event-${meterEventId}`;
 
@@ -680,7 +1271,7 @@ const addCreditConsumptionJob = async (
     }
 
     if (existingJob && replace) {
-      await creditQueue.delete(jobId);
+      await creditQueue.delete(jobId, true);
       logger.info('Replaced existing credit consumption job', {
         meterEventId,
         jobId,
@@ -688,26 +1279,26 @@ const addCreditConsumptionJob = async (
       });
     }
 
-    // 检查 MeterEvent 状态，避免为已完成的事件添加任务
-    const meterEvent = await MeterEvent.findByPk(meterEventId);
-    if (!meterEvent) {
-      logger.warn('Cannot add credit consumption job: MeterEvent not found', { meterEventId });
-      return;
-    }
+    if (!options.skipStatusCheck) {
+      const meterEvent = await MeterEvent.findByPk(meterEventId);
+      if (!meterEvent) {
+        logger.warn('Cannot add credit consumption job: MeterEvent not found', { meterEventId });
+        return;
+      }
 
-    if (meterEvent.status === 'completed' || meterEvent.status === 'canceled') {
-      logger.debug('Skipping credit consumption job: MeterEvent already processed', {
-        meterEventId,
-        status: meterEvent.status,
-      });
-      return;
+      if (meterEvent.status === 'completed' || meterEvent.status === 'canceled') {
+        logger.debug('Skipping credit consumption job: MeterEvent already processed', {
+          meterEventId,
+          status: meterEvent.status,
+        });
+        return;
+      }
     }
 
-    creditQueue.push({ id: jobId, job: { meterEventId }, ...options });
+    creditQueue.push({ id: jobId, job: { meterEventId }, skipDuplicateCheck: true, ...options });
     logger.info('Credit consumption job added to queue', {
       meterEventId,
       jobId,
-      meterEventStatus: meterEvent.status,
     });
   } catch (error: any) {
     logger.error('Failed to add credit consumption job', {
@@ -732,30 +1323,63 @@ export async function startCreditConsumeQueue(): Promise<void> {
     await lock.acquire();
     logger.info('Starting credit queue and processing pending meter events');
 
-    // Find pending and requires_capture events that need processing
-    const pendingEvents = await MeterEvent.findAll({
-      where: {
-        status: ['pending', 'requires_capture'],
-      },
-      limit: 50,
-    });
+    // Find pending, requires_capture, and processing (stale from crash/restart) events
+    // Process in batches of 50 to avoid loading too many records at once
+    let totalProcessed = 0;
+    let totalFailed = 0;
+    let batchEvents: MeterEvent[];
 
-    const results = await Promise.allSettled(
-      pendingEvents.map(async (event) => {
-        const jobId = `meter-event-${event.id}`;
-        const existingJob = await creditQueue.get(jobId);
-        if (!existingJob) {
-          addCreditConsumptionJob(event.id, true);
-        }
-      })
-    );
+    do {
+      // eslint-disable-next-line no-await-in-loop
+      batchEvents = await MeterEvent.findAll({
+        where: {
+          status: ['pending', 'requires_capture', 'processing'],
+        },
+        limit: 50,
+        offset: totalProcessed,
+      });
+
+      // Reset stale 'processing' events back to 'pending' — these were mid-consumption
+      // when the process crashed/restarted. Without reset, both single and batch handlers
+      // skip 'processing' events (line ~559, ~905), causing them to be stuck forever.
+      // Only reset events that have been in 'processing' for > 5 minutes to avoid
+      // interfering with events actively being processed by a concurrent handler.
+      const STALE_PROCESSING_THRESHOLD = 5 * 60 * 1000; // 5 minutes
+      const staleProcessing = batchEvents.filter(
+        (e) => e.status === 'processing' && Date.now() - new Date(e.updated_at).getTime() > STALE_PROCESSING_THRESHOLD
+      );
+      if (staleProcessing.length > 0) {
+        // eslint-disable-next-line no-await-in-loop
+        await MeterEvent.update(
+          { status: 'pending' },
+          { where: { id: { [Op.in]: staleProcessing.map((e) => e.id) } } }
+        );
+        logger.info('Reset stale processing events', {
+          count: staleProcessing.length,
+          eventIds: staleProcessing.map((e) => e.id),
+        });
+      }
+
+      // eslint-disable-next-line no-await-in-loop
+      const results = await Promise.allSettled(
+        batchEvents.map(async (event) => {
+          const jobId = `meter-event-${event.id}`;
+          const existingJob = await creditQueue.get(jobId);
+          if (!existingJob) {
+            addCreditConsumptionJob(event.id, true);
+          }
+        })
+      );
+
+      totalFailed += results.filter((r) => r.status === 'rejected').length;
+      totalProcessed += batchEvents.length;
+    } while (batchEvents.length === 50);
 
-    const failed = results.filter((r) => r.status === 'rejected').length;
-    if (failed > 0) {
-      logger.warn(`Failed to queue ${failed} pending meter events`);
+    if (totalFailed > 0) {
+      logger.warn(`Failed to queue ${totalFailed} pending meter events`);
     }
 
-    logger.info(`Credit queue started, processed ${pendingEvents.length} pending events`);
+    logger.info(`Credit queue started, processed ${totalProcessed} pending events`);
   } catch (error: any) {
     logger.error('Failed to start credit queue', { error: error.message });
   } finally {
@@ -764,7 +1388,12 @@ export async function startCreditConsumeQueue(): Promise<void> {
 }
 
 events.on('billing.meter_event.created', (meterEvent) => {
-  addCreditConsumptionJob(meterEvent.id, true);
+  const customerId = meterEvent.payload?.customer_id;
+  if (customerId) {
+    addToBatch(customerId, meterEvent.event_name, meterEvent.id, meterEvent.payload?.subscription_id);
+  } else {
+    addCreditConsumptionJob(meterEvent.id, true, { skipStatusCheck: true });
+  }
 });
 
 events.on('customer.credit_grant.granted', async (creditGrant: CreditGrant) => {
@@ -838,49 +1467,49 @@ async function retryFailedEventsForCustomer(creditGrant: CreditGrant): Promise<v
 
     let cumulativePending = new BN(0);
     const affordableEvents: MeterEvent[] = [];
-    const now = dayjs().unix();
-    const BATCH_SIZE = 10;
-    const BATCH_DELAY_SECONDS = 0.1;
 
-    for (let i = 0; i < sortedEvents.length; i++) {
-      const event = sortedEvents[i];
-      if (!event) {
+    for (const event of sortedEvents) {
+      const eventPending = new BN(event.credit_pending);
+
+      affordableEvents.push(event);
+      cumulativePending = cumulativePending.add(eventPending);
+      if (cumulativePending.gt(totalAvailableCredit)) {
         break;
       }
+    }
 
-      const eventPending = new BN(event.credit_pending);
-      const newCumulative = cumulativePending.add(eventPending);
+    if (affordableEvents.length > 0) {
+      const eventIds = affordableEvents.map((e) => e.id);
+      await MeterEvent.update(
+        { status: 'pending', attempt_count: 0, next_attempt: undefined },
+        { where: { id: { [Op.in]: eventIds } } }
+      );
 
-      if (newCumulative.gt(totalAvailableCredit)) {
-        if (eventPending.gt(totalAvailableCredit)) {
-          break;
+      // Group by subscription_id before pushing batch jobs, because the batch handler
+      // reads subscription from the first event only (line ~910) and applies its priceIds
+      // to all events in the batch.
+      const bySubscription = new Map<string, string[]>();
+      for (const event of affordableEvents) {
+        const subKey = event.getSubscriptionId() || 'default';
+        const group = bySubscription.get(subKey);
+        if (group) {
+          group.push(event.id);
+        } else {
+          bySubscription.set(subKey, [event.id]);
         }
       }
 
-      // eslint-disable-next-line no-await-in-loop
-      await event.update({
-        status: 'pending',
-        attempt_count: 0,
-        next_attempt: undefined,
-      });
-
-      const batchIndex = Math.floor(i / BATCH_SIZE);
-      const runAt = now + batchIndex * BATCH_DELAY_SECONDS;
-
-      try {
-        // eslint-disable-next-line no-await-in-loop
-        await addCreditConsumptionJob(event.id, true, { runAt });
-      } catch (jobError: any) {
-        logger.error('Failed to add credit consumption job after status update', {
-          eventId: event.id,
-          customerId,
-          currencyId,
-          error: jobError.message,
-        });
+      let chunkIndex = 0;
+      for (const [, subEventIds] of bySubscription) {
+        for (let i = 0; i < subEventIds.length; i += CREDIT_BATCH_SIZE) {
+          const chunk = subEventIds.slice(i, i + CREDIT_BATCH_SIZE);
+          creditQueue.push({
+            id: `retry-batch-${customerId}-${Date.now()}-${chunkIndex++}`,
+            job: { meterEventIds: chunk } as any,
+            skipDuplicateCheck: true,
+          });
+        }
       }
-
-      affordableEvents.push(event);
-      cumulativePending = newCumulative;
     }
 
     const skippedEvents = failedEvents.length - affordableEvents.length;