payment-kit 1.23.10 → 1.24.0

package/api/src/libs/invoice.ts

@@ -1121,20 +1121,27 @@ export async function retryUncollectibleInvoices(options: {
       failed: [] as Array<{ id: string; reason: string }>,
     };
 
+    const now = dayjs().unix();
+    const BATCH_SIZE = 10;
+    const BATCH_DELAY_SECONDS = 0.1;
+
     const settledResults = await Promise.allSettled(
       overdueInvoices.map(async (invoice, index) => {
         const { paymentIntent } = invoice;
-        const delay = index * 2;
         if (!paymentIntent) {
           throw new Error('No payment intent found');
         }
 
         await paymentIntent.update({ status: 'requires_capture' });
+
+        const batchIndex = Math.floor(index / BATCH_SIZE);
+        const runAt = now + batchIndex * BATCH_DELAY_SECONDS;
+
         await emitAsync(
           'payment.queued',
           paymentIntent.id,
           { paymentIntentId: paymentIntent.id, retryOnError: true, ignoreMaxRetryCheck: true },
-          { sync: false, delay }
+          { sync: false, runAt }
         );
 
         return invoice;

package/api/src/queues/credit-consume.ts

@@ -2,6 +2,7 @@ import { BN, fromUnitToToken } from '@ocap/util';
 
 import { getLock } from '../libs/lock';
 import logger from '../libs/logger';
+import dayjs from '../libs/dayjs';
 import createQueue from '../libs/queue';
 import { createEvent } from '../libs/audit';
 import {
@@ -396,7 +397,9 @@ async function createCreditTransaction(
     context.meter.paymentCurrency.symbol
   );
   let description = `Consume ${formattedAmount}`;
-  if (context.meterEvent.getSubscriptionId()) {
+  const resolvedSubscriptionId =
+    context.meterEvent.getSubscriptionId() || creditGrant.metadata?.subscription_id || context.subscription?.id;
+  if (resolvedSubscriptionId) {
     description += ' for Subscription';
   }
   if (context.meterEvent.metadata?.description) {
@@ -408,7 +411,7 @@ async function createCreditTransaction(
       customer_id: context.meterEvent.getCustomerId(),
       credit_grant_id: creditGrant.id,
       meter_id: context.meter.id,
-      subscription_id: context.meterEvent.getSubscriptionId(),
+      subscription_id: resolvedSubscriptionId,
       meter_event_name: context.meterEvent.event_name,
       meter_unit: context.meter.unit,
       quantity: consumeAmount,
@@ -551,7 +554,7 @@ export async function handleCreditConsumption(job: CreditConsumptionJob) {
       });
       await context.meterEvent.markAsCompleted();
       if (context.subscription && context.subscription.status === 'past_due') {
-        handlePastDueSubscriptionRecovery(context.subscription, null);
+        await handlePastDueSubscriptionRecovery(context.subscription, null);
       }
     } else {
       logger.warn('Credit consumption partially completed - insufficient balance', {
@@ -622,6 +625,7 @@ export const creditQueue = createQueue<CreditConsumptionJob>({
   options: {
     concurrency: 1,
     maxRetries: 0,
+    enableScheduledJob: true,
   },
 });
 
@@ -755,58 +759,128 @@ events.on('customer.credit_grant.granted', async (creditGrant: CreditGrant) => {
 });
 
 async function retryFailedEventsForCustomer(creditGrant: CreditGrant): Promise<void> {
+  const grant = await CreditGrant.findByPk(creditGrant.id);
+  if (!grant) {
+    logger.error('Credit grant not found', {
+      creditGrantId: creditGrant.id,
+    });
+    return;
+  }
+
+  const customerId = grant.customer_id;
+  const currencyId = grant.currency_id;
+  const lock = getLock(`retry-failed-events-${customerId}-${currencyId}`);
+
   try {
+    await lock.acquire();
+
     logger.info('Retrying failed events for customer', {
-      customerId: creditGrant.customer_id,
-      currencyId: creditGrant.currency_id,
-      livemode: creditGrant.livemode,
+      customerId,
+      currencyId,
+      livemode: grant.livemode,
+      grantAmount: grant.amount,
+      grantRemaining: grant.remaining_amount,
     });
+
+    const availableGrants = await CreditGrant.getAvailableCreditsForCustomer(customerId, currencyId);
+    const totalAvailableCredit = availableGrants.reduce((sum, g) => sum.add(new BN(g.remaining_amount)), new BN(0));
+
     const [, , failedEvents] = await MeterEvent.getPendingAmounts({
-      customerId: creditGrant.customer_id,
-      currencyId: creditGrant.currency_id,
+      customerId,
+      currencyId,
       status: ['requires_action', 'requires_capture'],
-      livemode: creditGrant.livemode,
+      livemode: grant.livemode,
     });
 
     if (failedEvents.length === 0) {
       logger.debug('No failed events with pending credit found', {
-        customerId: creditGrant.customer_id,
-        currencyId: creditGrant.currency_id,
+        customerId,
+        currencyId,
       });
       return;
     }
 
-    logger.info('Updating failed events status after credit grant', {
-      customerId: creditGrant.customer_id,
-      currencyId: creditGrant.currency_id,
-      eventCount: failedEvents.length,
+    if (totalAvailableCredit.lte(new BN(0))) {
+      logger.debug('No available credit to retry failed events', {
+        customerId,
+        currencyId,
+        totalAvailableCredit: totalAvailableCredit.toString(),
+      });
+      return;
+    }
+
+    const sortedEvents = failedEvents.sort((a, b) => {
+      const aPending = new BN(a.credit_pending);
+      const bPending = new BN(b.credit_pending);
+      return aPending.cmp(bPending);
     });
 
-    await Promise.all(
-      failedEvents.map((event) =>
-        event.update({
-          status: 'pending',
-          attempt_count: 0,
-          next_attempt: undefined,
-        })
-      )
-    );
+    let cumulativePending = new BN(0);
+    const affordableEvents: MeterEvent[] = [];
+    const now = dayjs().unix();
+    const BATCH_SIZE = 10;
+    const BATCH_DELAY_SECONDS = 0.1;
 
-    failedEvents.forEach((event, index) => {
-      const delay = index * 1000;
-      addCreditConsumptionJob(event.id, true, { delay });
-    });
+    for (let i = 0; i < sortedEvents.length; i++) {
+      const event = sortedEvents[i];
+      if (!event) {
+        break;
+      }
 
-    logger.info('Successfully updated failed events status', {
-      customerId: creditGrant.customer_id,
-      currencyId: creditGrant.currency_id,
-      eventCount: failedEvents.length,
+      const eventPending = new BN(event.credit_pending);
+      const newCumulative = cumulativePending.add(eventPending);
+
+      if (newCumulative.gt(totalAvailableCredit)) {
+        if (eventPending.gt(totalAvailableCredit)) {
+          break;
+        }
+      }
+
+      // eslint-disable-next-line no-await-in-loop
+      await event.update({
+        status: 'pending',
+        attempt_count: 0,
+        next_attempt: undefined,
+      });
+
+      const batchIndex = Math.floor(i / BATCH_SIZE);
+      const runAt = now + batchIndex * BATCH_DELAY_SECONDS;
+
+      try {
+        // eslint-disable-next-line no-await-in-loop
+        await addCreditConsumptionJob(event.id, true, { runAt });
+      } catch (jobError: any) {
+        logger.error('Failed to add credit consumption job after status update', {
+          eventId: event.id,
+          customerId,
+          currencyId,
+          error: jobError.message,
+        });
+      }
+
+      affordableEvents.push(event);
+      cumulativePending = newCumulative;
+    }
+
+    const skippedEvents = failedEvents.length - affordableEvents.length;
+
+    logger.info('Completed retrying failed events', {
+      customerId,
+      currencyId,
+      totalEvents: failedEvents.length,
+      affordableEvents: affordableEvents.length,
+      skippedEvents,
+      totalAvailableCredit: totalAvailableCredit.toString(),
+      totalAffordablePending: cumulativePending.toString(),
+      grantCount: availableGrants.length,
     });
   } catch (error: any) {
-    logger.error('Failed to update failed events for customer', {
-      customerId: creditGrant.customer_id,
-      currencyId: creditGrant.currency_id,
+    logger.error('Failed to retry failed events for customer', {
+      customerId: grant.customer_id,
+      currencyId: grant.currency_id,
       error: error.message,
     });
+  } finally {
+    lock.release();
   }
 }

package/api/src/queues/credit-grant.ts

@@ -2,6 +2,7 @@
 
 import { Op } from 'sequelize';
 import { BN, fromTokenToUnit, fromUnitToToken } from '@ocap/util';
+import pAll from 'p-all';
 import dayjs from '../libs/dayjs';
 import createQueue from '../libs/queue';
 import {
@@ -19,6 +20,15 @@ import {
 } from '../store/models';
 import { events } from '../libs/event';
 import { calculateExpiresAt, createCreditGrant } from '../libs/credit-grant';
+import {
+  CreditScheduleJob,
+  activateAfterFirstPaymentSchedules,
+  handleScheduledCredit,
+  initializeCreditSchedule,
+  stopCreditSchedule,
+  getScheduleJobId,
+} from '../libs/credit-schedule';
+import { Subscription } from '../store/models/subscription';
 import { getAccountState, mintToken, transferTokenFromCustomer } from '../integrations/arcblock/token';
 import logger from '../libs/logger';
 
@@ -30,7 +40,8 @@ type CreditGrantJob =
   | {
       invoiceId: string;
       action: 'create_from_invoice';
-    };
+    }
+  | CreditScheduleJob;
 
 /**
  * Activate credit grants and handle on-chain minting when required.
@@ -242,15 +253,31 @@ export async function expireGrant(creditGrant: CreditGrant) {
 }
 
 const handleCreditGrantJob = async (job: CreditGrantJob) => {
-  logger.info('Handling credit grant job', {
-    action: job.action,
-    ...(job.action === 'create_from_invoice' ? { invoiceId: job.invoiceId } : { creditGrantId: job.creditGrantId }),
-  });
+  logger.info('Handling credit grant job', job);
+
   if (job.action === 'create_from_invoice') {
     await handleInvoiceCredit(job.invoiceId);
     return;
   }
 
+  if (job.action === 'create_from_schedule') {
+    const scheduleJob = job as CreditScheduleJob;
+    const nextJob = await handleScheduledCredit(scheduleJob);
+
+    logger.info('Scheduled credit grant handled', {
+      scheduleJob,
+      nextJob,
+    });
+
+    // Schedule next grant if returned
+    // Use replace=true because the current job (same id) is still in the DB
+    // until onJobComplete deletes it, which would cause JOB_DUPLICATE error
+    if (nextJob) {
+      await addScheduledCreditJob(nextJob.jobId, nextJob.job, nextJob.runAt, true);
+    }
+    return;
+  }
+
   const { creditGrantId, action } = job as { creditGrantId: string; action: 'activate' | 'expire' };
 
   const creditGrant = await CreditGrant.findByPk(creditGrantId);
@@ -435,6 +462,9 @@ export const startCreditGrantQueue = async () => {
     logger.warn(`Failed to schedule jobs for ${failed} credit grants`);
   }
 
+  // Recover credit schedule jobs for subscriptions with active schedules
+  await recoverCreditScheduleJobs();
+
   logger.info('Credit grant queue started', {
     totalGrants: grantsToSchedule.length,
     succeeded,
@@ -442,6 +472,113 @@ export const startCreditGrantQueue = async () => {
   });
 };
 
+/**
+ * Recover credit schedule jobs after system restart
+ * Finds subscriptions with active credit schedules and ensures their jobs are scheduled
+ */
+async function recoverCreditScheduleJobs(): Promise<void> {
+  logger.info('Recovering credit schedule jobs...');
+
+  try {
+    // Find subscriptions with active credit schedules
+    // We filter for non-null credit_schedule_state in application code
+    const allActiveSubscriptions = await Subscription.findAll({
+      where: {
+        status: ['active', 'trialing'],
+      },
+    });
+
+    // Filter for subscriptions with credit_schedule_state
+    const subscriptions = allActiveSubscriptions.filter(
+      (sub) => sub.credit_schedule_state && Object.keys(sub.credit_schedule_state).length > 0
+    );
+
+    if (subscriptions.length === 0) {
+      logger.info('No subscriptions with credit schedules to recover');
+      return;
+    }
+
+    const now = dayjs().unix();
+    let recoveredCount = 0;
+    let scheduledCount = 0;
+
+    const tasks: Array<() => Promise<void>> = [];
+    const recoveryConcurrency = 5;
+
+    // Process each subscription's schedule state with bounded concurrency
+    subscriptions.forEach((subscription) => {
+      const state = subscription.credit_schedule_state;
+      if (!state) return;
+
+      Object.keys(state).forEach((priceId) => {
+        tasks.push(async () => {
+          const priceState = state[priceId];
+          if (!priceState?.enabled) return;
+
+          // Next job seq is last_grant_seq + 1
+          const nextSeq = priceState.last_grant_seq + 1;
+          const jobId = getScheduleJobId(subscription.id, priceId, nextSeq);
+          const nextGrantAt = priceState.next_grant_at;
+
+          // Skip if no next grant time scheduled
+          if (!nextGrantAt || nextGrantAt === 0) {
+            return;
+          }
+
+          // Check if job already exists
+          const existingJob = await creditGrantQueue.get(jobId);
+
+          if (nextGrantAt <= now) {
+            // Job is overdue - schedule immediately with replace=true
+            const job: CreditScheduleJob = {
+              subscriptionId: subscription.id,
+              priceId,
+              scheduledAt: nextGrantAt,
+              action: 'create_from_schedule',
+            };
+            await addScheduledCreditJob(jobId, job, now, true);
+            recoveredCount++;
+            logger.info('Recovered overdue credit schedule job', {
+              subscriptionId: subscription.id,
+              priceId,
+              originalScheduledAt: nextGrantAt,
+              overdueDays: Math.floor((now - nextGrantAt) / 86400),
+            });
+          } else if (!existingJob) {
+            // Job is in future but not scheduled - create it
+            const job: CreditScheduleJob = {
+              subscriptionId: subscription.id,
+              priceId,
+              scheduledAt: nextGrantAt,
+              action: 'create_from_schedule',
+            };
+            await addScheduledCreditJob(jobId, job, nextGrantAt, false);
+            scheduledCount++;
+            logger.debug('Scheduled missing credit schedule job', {
+              subscriptionId: subscription.id,
+              priceId,
+              scheduledAt: dayjs.unix(nextGrantAt).format(),
+            });
+          }
+        });
+      });
+    });
+
+    await pAll(tasks, { concurrency: recoveryConcurrency, stopOnError: false });
+
+    logger.info('Credit schedule jobs recovery completed', {
+      subscriptionsChecked: subscriptions.length,
+      overdueJobsRecovered: recoveredCount,
+      missingJobsScheduled: scheduledCount,
+    });
+  } catch (error: any) {
+    logger.error('Failed to recover credit schedule jobs', {
+      error: error.message,
+      stack: error.stack,
+    });
+  }
+}
+
 creditGrantQueue.on('failed', ({ id, job, error }) => {
   logger.error('Credit grant job failed', { id, job, error });
 });
@@ -513,6 +650,19 @@ async function handleInvoiceCredit(invoiceId: string) {
 
         const metadata = price.metadata || {};
         const creditConfig = metadata.credit_config || {};
+
+        // Check if this price uses schedule-based delivery
+        // If delivery_mode is 'schedule', skip invoice-based credit grant creation
+        const scheduleConfig = creditConfig.schedule;
+        if (scheduleConfig?.enabled && scheduleConfig?.delivery_mode === 'schedule') {
+          logger.info('Skipping invoice credit grant for schedule-based price', {
+            invoiceId,
+            priceId: price.id,
+            deliveryMode: scheduleConfig.delivery_mode,
+          });
+          return null;
+        }
+
         const quantity = lineItem.quantity || 1;
         const currency = await PaymentCurrency.findByPk(creditConfig.currency_id);
 
@@ -632,6 +782,7 @@ async function handleInvoiceCredit(invoiceId: string) {
             price_id: item.price.id,
             invoice_id: invoice.id,
             purchased_quantity: item.quantity,
+            subscription_id: invoice.subscription_id,
           },
         });
 
@@ -716,9 +867,113 @@ export async function addInvoiceCreditJob(invoiceId: string) {
   });
 }
 
+/**
+ * Add a scheduled credit job to the queue
+ * @param jobId Unique job ID (typically schedule-{subscriptionId}-{priceId})
+ * @param job The credit schedule job data
+ * @param runAt Unix timestamp when the job should run
+ * @param replace If true, replace existing job; if false, skip if job exists
+ */
+export async function addScheduledCreditJob(
+  jobId: string,
+  job: CreditScheduleJob,
+  runAt: number,
+  replace: boolean = false
+): Promise<boolean> {
+  // Always try to delete existing job first to avoid JOB_DUPLICATE error
+  // This is necessary because when a job is executing and creates the next job,
+  // the current job hasn't been deleted yet, causing ID conflict
+  const existingJob = await creditGrantQueue.get(jobId);
+
+  if (existingJob) {
+    if (replace) {
+      await creditGrantQueue.delete(jobId);
+      logger.info('Scheduled credit job replaced', { jobId, runAt });
+    } else {
+      logger.debug('Scheduled credit job already exists, skipping', {
+        jobId,
+        existingRunAt: existingJob.runAt,
+        newRunAt: runAt,
+      });
+      return false;
+    }
+  }
+
+  // Use store.addJob directly to ensure proper persistence and error handling
+  // instead of relying on push() which has async issues for scheduled jobs
+  try {
+    const now = dayjs().unix();
+    const attrs: { delay?: number; will_run_at?: number } = {};
+
+    // If runAt is in the future, create as scheduled job
+    if (runAt > now) {
+      attrs.delay = 1;
+      attrs.will_run_at = runAt * 1000; // Convert to milliseconds
+    }
+
+    await creditGrantQueue.store.addJob(jobId, job, attrs);
+
+    logger.info('Scheduled credit job added', {
+      jobId,
+      subscriptionId: job.subscriptionId,
+      priceId: job.priceId,
+      runAt,
+      scheduledAt: dayjs.unix(runAt).format(),
+      isImmediate: runAt <= now,
+    });
+
+    // If job should run immediately (runAt <= now), push to queue for execution
+    if (runAt <= now) {
+      creditGrantQueue.push({
+        id: jobId,
+        job,
+        persist: false, // Already persisted above
+      });
+      logger.info('Scheduled credit job pushed for immediate execution', { jobId });
+    }
+
+    return true;
+  } catch (err: any) {
+    // Handle duplicate job error gracefully
+    if (err.code === 'JOB_DUPLICATE') {
+      logger.warn('Scheduled credit job already exists (concurrent creation)', {
+        jobId,
+        error: err.message,
+      });
+      return false;
+    }
+    logger.error('Failed to add scheduled credit job', {
+      jobId,
+      error: err.message,
+      stack: err.stack,
+    });
+    throw err;
+  }
+}
+
+/**
+ * Delete a scheduled credit job from the queue
+ */
+export async function deleteScheduledCreditJob(jobId: string): Promise<boolean> {
+  const existingJob = await creditGrantQueue.get(jobId);
+  if (existingJob) {
+    await creditGrantQueue.delete(jobId);
+    logger.info('Scheduled credit job deleted', { jobId });
+    return true;
+  }
+  return false;
+}
+
 events.on('invoice.paid', async (invoice: Invoice) => {
   try {
     await addInvoiceCreditJob(invoice.id);
+    if (invoice.subscription_id) {
+      const subscription = await Subscription.findByPk(invoice.subscription_id);
+      if (subscription) {
+        const jobs = await activateAfterFirstPaymentSchedules(subscription, invoice.status_transitions?.paid_at);
+        await Promise.all(jobs.map((job) => addScheduledCreditJob(job.jobId, job.job, job.runAt, true)));
+      }
+    }
   } catch (error) {
     logger.error('Failed to schedule credit grant job for invoice', {
       invoiceId: invoice.id,
@@ -784,3 +1039,128 @@ events.on('credit-grant.queued', async (id, job, args = {}) => {
     events.emit('credit-grant.queued.error', { id, job, error });
   }
 });
+
+// ========================================
+// Credit Schedule Event Handlers
+// ========================================
+
+/**
+ * Helper function to initialize credit schedule and create jobs
+ * Used by both subscription.created and subscription.started events
+ */
+async function initializeAndScheduleCreditJobs(subscriptionId: string, eventName: string): Promise<void> {
+  const sub = await Subscription.findByPk(subscriptionId);
+  if (!sub) {
+    logger.warn('Subscription not found for credit schedule initialization', {
+      subscriptionId,
+      event: eventName,
+    });
+    return;
+  }
+
+  // initializeCreditSchedule only works for active/trialing subscriptions
+  // and skips if already initialized
+  const state = await initializeCreditSchedule(sub);
+  if (!state) {
+    logger.debug('No credit schedules to initialize for subscription', {
+      subscriptionId,
+      event: eventName,
+      status: sub.status,
+    });
+    return;
+  }
+
+  // Schedule jobs for prices with valid next_grant_at
+  const pricesToSchedule = Object.keys(state).filter((priceId) => {
+    const priceState = state[priceId];
+    return priceState?.enabled && priceState?.next_grant_at > 0;
+  });
+
+  await Promise.all(
+    pricesToSchedule.map((priceId) => {
+      const priceState = state[priceId]!;
+      // First job has seq = 1 (since last_grant_seq starts at 0)
+      const nextSeq = priceState.last_grant_seq + 1;
+      const jobId = getScheduleJobId(subscriptionId, priceId, nextSeq);
+      const job: CreditScheduleJob = {
+        subscriptionId,
+        priceId,
+        scheduledAt: priceState.next_grant_at,
+        action: 'create_from_schedule',
+      };
+      return addScheduledCreditJob(jobId, job, priceState.next_grant_at, true);
+    })
+  );
+
+  logger.info('Credit schedule initialized and jobs scheduled', {
+    subscriptionId,
+    event: eventName,
+    priceCount: pricesToSchedule.length,
+  });
+}
+
+/**
+ * Handle subscription started event (when subscription becomes active)
+ *
+ * This event fires when:
+ * 1. Subscription payment succeeds (checkout flow completes)
+ * 2. Trial ends and subscription becomes active
+ * 3. Stripe SetupIntent succeeds
+ *
+ * For subscriptions created via checkout (status was incomplete),
+ * this is when the credit schedule gets initialized.
+ */
+events.on('customer.subscription.started', async (subscription: Subscription) => {
+  try {
+    await initializeAndScheduleCreditJobs(subscription.id, 'customer.subscription.started');
+  } catch (error: any) {
+    logger.error('Failed to handle credit schedule on subscription started', {
+      subscriptionId: subscription.id,
+      error: error.message,
+    });
+  }
+});
+
+/**
+ * Handle subscription trial start event (when subscription enters trialing state)
+ *
+ * This event fires when subscription has a trial period and payment succeeds.
+ * For immediate timing, credit schedule should start during trial period.
+ */
+events.on('customer.subscription.trial_start', async (subscription: Subscription) => {
+  try {
+    await initializeAndScheduleCreditJobs(subscription.id, 'customer.subscription.trial_start');
+  } catch (error: any) {
+    logger.error('Failed to handle credit schedule on subscription trial start', {
+      subscriptionId: subscription.id,
+      error: error.message,
+    });
+  }
+});
+
+/**
+ * Handle subscription deleted event - stop all credit schedules
+ */
+events.on('customer.subscription.deleted', async (subscription: Subscription) => {
+  try {
+    const sub = await Subscription.findByPk(subscription.id);
+    if (!sub) {
+      return;
+    }
+
+    const jobIds = await stopCreditSchedule(sub);
+
+    // Delete all scheduled jobs
+    await Promise.all(jobIds.map((id) => deleteScheduledCreditJob(id)));
+
+    logger.info('Credit schedules stopped on subscription deletion', {
+      subscriptionId: subscription.id,
+      deletedJobs: jobIds.length,
+    });
+  } catch (error: any) {
+    logger.error('Failed to stop credit schedules on subscription deletion', {
+      subscriptionId: subscription.id,
+      error: error.message,
+    });
+  }
+});