payment-kit 1.25.8 → 1.25.10

package/api/src/libs/archive/index.ts

@@ -0,0 +1,7 @@
+export * from './config';
+export * from './executor';
+export * from './lock';
+export * from './policy';
+export * from './query';
+export * from './snapshot';
+export * from './store';

package/api/src/libs/archive/lock.ts

@@ -0,0 +1,50 @@
+import { ArchiveLock } from '../../store/models/archive-lock';
+
+const LOCK_ID = 'archive_job';
+const LOCK_TTL_MS = 2 * 60 * 60 * 1000;
+
+export function acquireArchiveLock(instanceId: string): Promise<boolean> {
+  const { sequelize } = ArchiveLock;
+  if (!sequelize) {
+    return Promise.resolve(false);
+  }
+  const now = Date.now();
+  const expiresAt = now + LOCK_TTL_MS;
+
+  return sequelize.transaction(async (transaction) => {
+    const existing = await ArchiveLock.findByPk(LOCK_ID, { transaction });
+    if (!existing) {
+      await ArchiveLock.create(
+        {
+          id: LOCK_ID,
+          locked_by: instanceId,
+          locked_at: now,
+          expires_at: expiresAt,
+        },
+        { transaction }
+      );
+      return true;
+    }
+
+    if (!existing.expires_at || existing.expires_at <= now) {
+      await existing.update(
+        {
+          locked_by: instanceId,
+          locked_at: now,
+          expires_at: expiresAt,
+        },
+        { transaction }
+      );
+      return true;
+    }
+
+    return false;
+  });
+}
+
+export async function releaseArchiveLock(instanceId: string): Promise<void> {
+  await ArchiveLock.update(
+    { locked_by: null, locked_at: null, expires_at: null },
+    { where: { id: LOCK_ID, locked_by: instanceId } }
+  );
+}

package/api/src/libs/archive/policy.ts

@@ -0,0 +1,55 @@
+import { Op, Sequelize, ModelStatic, Model } from 'sequelize';
+
+import { TableRetentionPolicy } from './config';
+
+export type ArchiveQueryPlan = {
+  where: Record<string, any>;
+  dateField: string;
+};
+
+export function getArchiveDateField(model: ModelStatic<Model>): string {
+  const attributes = model.rawAttributes;
+  if (attributes.updated_at) {
+    return 'updated_at';
+  }
+  if (attributes.created_at) {
+    return 'created_at';
+  }
+  return 'created_at';
+}
+
+export function buildArchiveQueryPlan(
+  model: ModelStatic<Model>,
+  policy: TableRetentionPolicy,
+  cutoffDate: Date
+): ArchiveQueryPlan {
+  const dateField = getArchiveDateField(model);
+  const where: any = {
+    [dateField]: { [Op.lte]: cutoffDate },
+  };
+
+  const conditions: any[] = [];
+
+  if (model.rawAttributes.status) {
+    if (policy.archivableStatuses && policy.archivableStatuses.length > 0) {
+      conditions.push({ status: { [Op.in]: policy.archivableStatuses } });
+    }
+
+    if (policy.excludeConditions?.statuses && policy.excludeConditions.statuses.length > 0) {
+      conditions.push({ status: { [Op.notIn]: policy.excludeConditions.statuses } });
+    }
+  }
+
+  if (policy.excludeConditions?.customCondition) {
+    // customCondition is an EXCLUSION condition: records matching it are KEPT (not archived).
+    // E.g. customCondition='pending_webhooks > 0' means "exclude records with pending webhooks".
+    // We wrap with NOT() so the query selects records that do NOT match the exclusion.
+    conditions.push(Sequelize.literal(`NOT (${policy.excludeConditions.customCondition})`));
+  }
+
+  if (conditions.length > 0) {
+    where[Op.and] = conditions;
+  }
+
+  return { where, dateField };
+}

package/api/src/libs/archive/query.ts

@@ -0,0 +1,136 @@
+/* eslint-disable no-continue */
+/* eslint-disable no-await-in-loop */
+import path from 'path';
+
+import { QueryTypes, Op } from 'sequelize';
+
+import logger from '../logger';
+import { ArchiveMetadata } from '../../store/models/archive-metadata';
+import { listArchiveFiles, openArchiveSequelize } from './store';
+
+type ArchiveQueryParams = {
+  table: string;
+  id?: string;
+  customer_id?: string;
+  from: number;
+  to?: number;
+  page: number;
+  limit: number;
+};
+
+type ArchiveQueryResult = {
+  data: any[];
+  total: number;
+  archiveFiles: string[];
+};
+
+function buildWhereClause(params: ArchiveQueryParams) {
+  const conditions: string[] = [];
+  const replacements: Record<string, any> = {};
+
+  // Filter by data's original created_at, not archived_at
+  // Users query for "2024 invoices", not "invoices archived at some time"
+  const fromDate = new Date(params.from * 1000);
+  const toDate = new Date((params.to ?? Math.floor(Date.now() / 1000)) * 1000);
+
+  conditions.push('created_at >= :from');
+  conditions.push('created_at <= :to');
+  replacements.from = fromDate;
+  replacements.to = toDate;
+
+  if (params.id) {
+    conditions.push('id = :id');
+    replacements.id = params.id;
+  }
+
+  if (params.customer_id) {
+    if (params.table === 'meter_events') {
+      conditions.push("json_extract(payload, '$.customer_id') = :customerId");
+    } else {
+      conditions.push('customer_id = :customerId');
+    }
+    replacements.customerId = params.customer_id;
+  }
+
+  return { clause: conditions.join(' AND '), replacements };
+}
+
+async function recordArchiveQuery(fileNames: string[], actorId?: string) {
+  const uniqueFiles = Array.from(new Set(fileNames));
+  // Find all metadata records that contain any of the queried files
+  // archive_file may contain comma-separated file names like "archive-2024.db,archive-2025.db"
+  const updatedIds = new Set<string>();
+
+  for (const fileName of uniqueFiles) {
+    const metadataList = await ArchiveMetadata.findAll({
+      where: { archive_file: { [Op.like]: `%${fileName}%` } },
+    });
+    for (const metadata of metadataList) {
+      // Avoid updating the same metadata multiple times in one query
+      if (updatedIds.has(metadata.id)) {
+        continue;
+      }
+      updatedIds.add(metadata.id);
+      const currentActors = metadata.query_actor_ids || [];
+      const actorIds = actorId ? Array.from(new Set([...currentActors, actorId])) : currentActors;
+      await metadata.update({
+        query_count: (metadata.query_count || 0) + 1,
+        query_actor_ids: actorIds,
+        last_queried_at: Math.floor(Date.now() / 1000),
+      });
+    }
+  }
+}
+
+const VALID_TABLE_NAME = /^[a-z][a-z0-9_]*$/;
+
+export async function queryArchive(params: ArchiveQueryParams, actorId?: string): Promise<ArchiveQueryResult> {
+  if (!VALID_TABLE_NAME.test(params.table)) {
+    throw new Error(`Invalid table name: ${params.table}`);
+  }
+  const archiveFiles = listArchiveFiles();
+  const { clause, replacements } = buildWhereClause(params);
+  const results: any[] = [];
+  const touchedFiles: string[] = [];
+
+  for (const filePath of archiveFiles) {
+    const archiveSequelize = openArchiveSequelize(filePath);
+    try {
+      // Check if table exists using PRAGMA instead of describeTable
+      // (describeTable has a Sequelize SQLite bug: "Cannot set properties of undefined (setting 'unique')")
+      const [tables] = await archiveSequelize.query(
+        "SELECT name FROM sqlite_master WHERE type='table' AND name=:tableName",
+        { replacements: { tableName: params.table } }
+      );
+      if (!Array.isArray(tables) || tables.length === 0) {
+        // Table doesn't exist in this archive file, skip to next file
+        // Note: don't close here, finally block will handle it
+        continue;
+      }
+
+      const sql = `SELECT * FROM "${params.table}" WHERE ${clause} ORDER BY created_at DESC`;
+      const rows = (await archiveSequelize.query(sql, {
+        replacements,
+        type: QueryTypes.SELECT,
+      })) as any[];
+      if (Array.isArray(rows) && rows.length > 0) {
+        results.push(...rows);
+        touchedFiles.push(path.basename(filePath));
+      }
+    } catch (error) {
+      logger.warn('archive query failed', { filePath, error });
+    } finally {
+      await archiveSequelize.close();
+    }
+  }
+
+  results.sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime());
+  const total = results.length;
+  const start = (params.page - 1) * params.limit;
+  const end = start + params.limit;
+  const data = results.slice(start, end);
+
+  await recordArchiveQuery(touchedFiles, actorId);
+
+  return { data, total, archiveFiles: touchedFiles };
+}

package/api/src/libs/archive/snapshot.ts

@@ -0,0 +1,291 @@
+/* eslint-disable no-await-in-loop */
+import { BN } from '@ocap/util';
+import { Op } from 'sequelize';
+
+import dayjs from '../dayjs';
+import logger from '../logger';
+import {
+  CreditGrant,
+  Invoice,
+  InvoiceItem,
+  Payout,
+  PaymentCurrency,
+  Refund,
+  RevenueSnapshot,
+} from '../../store/models';
+
+const BILLING_REASON_EXCLUSIONS = ['stake', 'stake_overdraft_protection', 'recharge'];
+
+type MonthlyRevenueData = {
+  currencyId: string;
+  livemode: boolean;
+  totalRevenue: string;
+  refundAmount: string;
+  promotionCost: string;
+  creditGrantCost: string;
+  vendorCost: string;
+  taxedRevenue: string;
+  netRevenue: string;
+};
+
+async function getTaxedInvoiceIds(
+  livemode: boolean,
+  startDate: Date,
+  endDate: Date,
+  currencyIds: string[]
+): Promise<Set<string>> {
+  const invoiceItems = await InvoiceItem.findAll({
+    where: {
+      tax_rate_id: { [Op.ne]: null },
+    } as any,
+    include: [
+      {
+        model: Invoice,
+        as: 'invoice',
+        where: {
+          livemode,
+          status: 'paid',
+          billing_reason: { [Op.notIn]: BILLING_REASON_EXCLUSIONS },
+          currency_id: { [Op.in]: currencyIds },
+          created_at: { [Op.gte]: startDate, [Op.lt]: endDate },
+        },
+        attributes: ['id'],
+      },
+    ],
+    attributes: ['invoice_id'],
+  });
+
+  return new Set(invoiceItems.map((item: any) => item.invoice_id).filter(Boolean));
+}
+
+async function calculateMonthlyRevenue(
+  livemode: boolean,
+  startDate: Date,
+  endDate: Date
+): Promise<MonthlyRevenueData[]> {
+  const currencies = await PaymentCurrency.findAll({ where: { livemode } });
+  if (currencies.length === 0) {
+    return [];
+  }
+
+  const currencyIds = currencies.map((c) => c.id);
+
+  const [invoices, taxedInvoiceIds, payouts, refunds, creditGrants] = await Promise.all([
+    Invoice.findAll({
+      where: {
+        livemode,
+        status: 'paid',
+        billing_reason: { [Op.notIn]: BILLING_REASON_EXCLUSIONS },
+        currency_id: { [Op.in]: currencyIds },
+        created_at: { [Op.gte]: startDate, [Op.lt]: endDate },
+      },
+      attributes: ['id', 'total', 'total_discount_amounts', 'currency_id'],
+    }),
+    getTaxedInvoiceIds(livemode, startDate, endDate, currencyIds),
+    Payout.findAll({
+      where: {
+        livemode,
+        status: { [Op.in]: ['paid', 'deferred'] },
+        currency_id: { [Op.in]: currencyIds },
+        created_at: { [Op.gte]: startDate, [Op.lt]: endDate },
+      } as any,
+      attributes: ['amount', 'currency_id', 'vendor_info'],
+    }).then((result) => result.filter((p: any) => p.vendor_info != null)),
+    Refund.findAll({
+      where: {
+        livemode,
+        status: 'succeeded',
+        currency_id: { [Op.in]: currencyIds },
+        created_at: { [Op.gte]: startDate, [Op.lt]: endDate },
+      },
+      attributes: ['amount', 'currency_id'],
+    }),
+    // Only count CONSUMED credits as cost (depleted or expired)
+    // granted credits are still active, voided credits were revoked
+    // Cost = amount - remaining_amount (the actually consumed portion)
+    CreditGrant.findAll({
+      where: {
+        livemode,
+        status: { [Op.in]: ['depleted', 'expired'] },
+        currency_id: { [Op.in]: currencyIds },
+        created_at: { [Op.gte]: startDate, [Op.lt]: endDate },
+      },
+      attributes: ['amount', 'remaining_amount', 'currency_id'],
+    }),
+  ]);
+
+  const byCurrency: Record<
+    string,
+    {
+      totalRevenue: BN;
+      refundAmount: BN;
+      promotionCost: BN;
+      creditGrantCost: BN;
+      vendorCost: BN;
+      taxedRevenue: BN;
+      livemode: boolean;
+    }
+  > = {};
+
+  for (const currency of currencies) {
+    byCurrency[currency.id] = {
+      totalRevenue: new BN('0'),
+      refundAmount: new BN('0'),
+      promotionCost: new BN('0'),
+      creditGrantCost: new BN('0'),
+      vendorCost: new BN('0'),
+      taxedRevenue: new BN('0'),
+      livemode: currency.livemode,
+    };
+  }
+
+  for (const inv of invoices) {
+    const entry = byCurrency[inv.currency_id];
+    if (entry) {
+      entry.totalRevenue = entry.totalRevenue.add(new BN(inv.total || '0'));
+
+      const discounts = inv.total_discount_amounts || [];
+      for (const discount of discounts) {
+        entry.promotionCost = entry.promotionCost.add(new BN(discount.amount || '0'));
+      }
+
+      if (taxedInvoiceIds.has(inv.id)) {
+        entry.taxedRevenue = entry.taxedRevenue.add(new BN(inv.total || '0'));
+      }
+    }
+  }
+
+  for (const payout of payouts) {
+    const entry = byCurrency[payout.currency_id];
+    if (entry) {
+      entry.vendorCost = entry.vendorCost.add(new BN(payout.amount || '0'));
+    }
+  }
+
+  for (const refund of refunds) {
+    const entry = byCurrency[refund.currency_id];
+    if (entry) {
+      entry.refundAmount = entry.refundAmount.add(new BN(refund.amount || '0'));
+    }
+  }
+
+  for (const grant of creditGrants) {
+    const entry = byCurrency[grant.currency_id];
+    if (entry) {
+      // Cost = consumed amount = total amount - remaining amount
+      const consumed = new BN(grant.amount || '0').sub(new BN(grant.remaining_amount || '0'));
+      entry.creditGrantCost = entry.creditGrantCost.add(consumed);
+    }
+  }
+
+  const results: MonthlyRevenueData[] = [];
+  for (const [currencyId, data] of Object.entries(byCurrency)) {
+    // Net Revenue = Total Revenue - Refunds
+    const netRevenue = data.totalRevenue.sub(data.refundAmount);
+    results.push({
+      currencyId,
+      livemode: data.livemode,
+      totalRevenue: data.totalRevenue.toString(),
+      refundAmount: data.refundAmount.toString(),
+      promotionCost: data.promotionCost.toString(),
+      creditGrantCost: data.creditGrantCost.toString(),
+      vendorCost: data.vendorCost.toString(),
+      taxedRevenue: data.taxedRevenue.toString(),
+      netRevenue: netRevenue.toString(),
+    });
+  }
+
+  return results;
+}
+
+export async function createRevenueSnapshotsForArchive(
+  cutoffTimestamp: number,
+  archiveMetadataId: string
+): Promise<number> {
+  const cutoffDate = dayjs.unix(cutoffTimestamp);
+  const monthsToSnapshot: { start: dayjs.Dayjs; end: dayjs.Dayjs }[] = [];
+
+  const existingSnapshots = await RevenueSnapshot.findAll({
+    attributes: ['timestamp', 'livemode', 'currency_id'],
+    order: [['timestamp', 'DESC']],
+    limit: 1,
+  });
+
+  let startMonth: dayjs.Dayjs;
+  if (existingSnapshots.length > 0) {
+    startMonth = dayjs.unix(existingSnapshots[0]!.timestamp).add(1, 'month').startOf('month');
+  } else {
+    const oldestInvoice = await Invoice.findOne({
+      where: { status: 'paid' },
+      order: [['created_at', 'ASC']],
+      attributes: ['created_at'],
+    });
+    if (!oldestInvoice) {
+      logger.info('No paid invoices found, skipping revenue snapshot');
+      return 0;
+    }
+    startMonth = dayjs(oldestInvoice.created_at).startOf('month');
+  }
+
+  let current = startMonth;
+  while (current.isBefore(cutoffDate)) {
+    const monthEnd = current.endOf('month');
+    if (monthEnd.isBefore(cutoffDate)) {
+      monthsToSnapshot.push({
+        start: current,
+        end: current.add(1, 'month').startOf('month'),
+      });
+    }
+    current = current.add(1, 'month').startOf('month');
+  }
+
+  if (monthsToSnapshot.length === 0) {
+    logger.info('No complete months to snapshot before cutoff', { cutoffDate: cutoffDate.toISOString() });
+    return 0;
+  }
+
+  let totalCreated = 0;
+
+  for (const { start, end } of monthsToSnapshot) {
+    const timestamp = start.unix();
+
+    for (const livemode of [true, false]) {
+      const revenueData = await calculateMonthlyRevenue(livemode, start.toDate(), end.toDate());
+
+      const snapshots = revenueData.map((data) => ({
+        livemode: data.livemode,
+        currency_id: data.currencyId,
+        timestamp,
+        period_type: 'monthly' as const,
+        total_revenue: data.totalRevenue,
+        refund_amount: data.refundAmount,
+        promotion_cost: data.promotionCost,
+        credit_grant_cost: data.creditGrantCost,
+        vendor_cost: data.vendorCost,
+        taxed_revenue: data.taxedRevenue,
+        net_revenue: data.netRevenue,
+        archive_metadata_id: archiveMetadataId,
+      }));
+
+      if (snapshots.length > 0) {
+        await RevenueSnapshot.bulkCreate(snapshots, {
+          updateOnDuplicate: [
+            'total_revenue',
+            'refund_amount',
+            'promotion_cost',
+            'credit_grant_cost',
+            'vendor_cost',
+            'taxed_revenue',
+            'net_revenue',
+            'archive_metadata_id',
+          ],
+        });
+        totalCreated += snapshots.length;
+      }
+    }
+
+    logger.info('Revenue snapshot created', { month: start.format('YYYY-MM'), records: totalCreated });
+  }
+
+  return totalCreated;
+}