payment-kit 1.25.7 → 1.25.10

package/api/src/libs/archive/store.ts

@@ -0,0 +1,200 @@
+import fs from 'fs';
+import path from 'path';
+
+import config from '@blocklet/sdk/lib/config';
+import { Sequelize, DataTypes } from 'sequelize';
+
+import logger from '../logger';
+
+export function getArchiveDir(): string {
+  const dataDir = config.env?.dataDir || '/tmp';
+  const archiveDir = path.join(dataDir, 'archive');
+  if (!fs.existsSync(archiveDir)) {
+    fs.mkdirSync(archiveDir, { recursive: true });
+  }
+  return archiveDir;
+}
+
+export function getArchiveFilePath(fileName: string): string {
+  return path.join(getArchiveDir(), fileName);
+}
+
+/**
+ * Get archive file path for a specific year based on record's created_at.
+ * Archive files are organized by data year (e.g., archive-2024.db contains 2024 data).
+ */
+export function getArchiveFilePathForYear(year: number): string {
+  return path.join(getArchiveDir(), `archive-${year}.db`);
+}
+
+/**
+ * Extract year from a record's created_at (fallback to updated_at, then current year).
+ */
+export function getRecordYear(record: any): number {
+  const createdAt = record?.created_at;
+  const updatedAt = record?.updated_at;
+  const dateValue = createdAt || updatedAt;
+
+  if (dateValue) {
+    const date = dateValue instanceof Date ? dateValue : new Date(dateValue);
+    if (!Number.isNaN(date.getTime())) {
+      return date.getFullYear();
+    }
+  }
+
+  return new Date().getFullYear();
+}
+
+export function openArchiveSequelize(filePath: string): Sequelize {
+  return new Sequelize({
+    dialect: 'sqlite',
+    storage: filePath,
+    logging: false,
+  });
+}
+
+function normalizeCreateTableSql(sql: string): string {
+  return sql.replace(/^CREATE TABLE\s+/i, 'CREATE TABLE IF NOT EXISTS ');
+}
+
+function normalizeCreateIndexSql(sql: string): string {
+  return sql
+    .replace(/^CREATE INDEX\s+/i, 'CREATE INDEX IF NOT EXISTS ')
+    .replace(/^CREATE UNIQUE INDEX\s+/i, 'CREATE UNIQUE INDEX IF NOT EXISTS ');
+}
+
+type ColumnInfo = { name: string; type: string; notnull: number; dflt_value: any; pk: number };
+
+export async function ensureArchiveTable(
+  tableName: string,
+  mainSequelize: Sequelize,
+  archiveSequelize: Sequelize
+): Promise<void> {
+  // 1. Copy table schema (CREATE TABLE IF NOT EXISTS)
+  const [tableRows] = await mainSequelize.query('SELECT sql FROM sqlite_master WHERE type = :type AND name = :name', {
+    replacements: { type: 'table', name: tableName },
+  });
+  const tableRow = (Array.isArray(tableRows) ? tableRows[0] : null) as { sql?: string } | null;
+  const createSql = tableRow?.sql;
+  if (!createSql) {
+    throw new Error(`table schema not found: ${tableName}`);
+  }
+
+  const normalizedSql = normalizeCreateTableSql(createSql);
+  await archiveSequelize.query(normalizedSql);
+
+  // 2. Copy indexes (PRIMARY KEY auto-indexes have sql=NULL, skip them)
+  const [indexRows] = await mainSequelize.query(
+    'SELECT sql FROM sqlite_master WHERE type = :type AND tbl_name = :tableName AND sql IS NOT NULL',
+    { replacements: { type: 'index', tableName } }
+  );
+  for (const indexRow of indexRows as { sql?: string }[]) {
+    if (indexRow?.sql) {
+      try {
+        const indexSql = normalizeCreateIndexSql(indexRow.sql);
+        // eslint-disable-next-line no-await-in-loop
+        await archiveSequelize.query(indexSql);
+      } catch (err: any) {
+        // Ignore "index already exists" errors
+        if (!err.message?.includes('already exists')) {
+          logger.warn('failed to create archive index', { tableName, error: err.message });
+        }
+      }
+    }
+  }
+
+  // 3. Sync missing columns from main DB to archive DB
+  // This handles schema migrations: if main DB has new columns, add them to archive DB
+  const [mainColumns] = await mainSequelize.query(`PRAGMA table_info("${tableName}")`);
+  const [archiveColumns] = await archiveSequelize.query(`PRAGMA table_info("${tableName}")`);
+
+  const mainColumnNames = new Set((mainColumns as ColumnInfo[]).map((c) => c.name));
+  const archiveColumnNames = new Set((archiveColumns as ColumnInfo[]).map((c) => c.name));
+
+  // Find columns in main DB but not in archive DB
+  const missingColumns = (mainColumns as ColumnInfo[]).filter((c) => !archiveColumnNames.has(c.name));
+
+  for (const col of missingColumns) {
+    try {
+      // SQLite ALTER TABLE ADD COLUMN only supports columns with default values or NULL
+      // For NOT NULL columns without defaults, we need to add them as nullable
+      const nullable = col.notnull === 0 || col.dflt_value !== null;
+      const defaultClause = col.dflt_value !== null ? ` DEFAULT ${col.dflt_value}` : '';
+
+      // eslint-disable-next-line no-await-in-loop
+      await archiveSequelize.query(
+        `ALTER TABLE "${tableName}" ADD COLUMN "${col.name}" ${col.type}${nullable ? '' : ' NULL'}${defaultClause}`
+      );
+      logger.info('added missing column to archive table', { tableName, column: col.name });
+    } catch (err: any) {
+      // Ignore "duplicate column" errors (race condition protection)
+      if (!err.message?.includes('duplicate column')) {
+        logger.warn('failed to add missing column to archive table', {
+          tableName,
+          column: col.name,
+          error: err.message,
+        });
+      }
+    }
+  }
+
+  // 4. Add archived_at column if not exists
+  if (!archiveColumnNames.has('archived_at') && !mainColumnNames.has('archived_at')) {
+    const queryInterface = archiveSequelize.getQueryInterface();
+    await queryInterface.addColumn(tableName, 'archived_at', {
+      type: DataTypes.DATE,
+      allowNull: false,
+    });
+  }
+}
+
+export function listArchiveFiles(): string[] {
+  const archiveDir = getArchiveDir();
+  const files = fs.readdirSync(archiveDir).filter((name) => name.endsWith('.db'));
+  files.sort();
+  return files.map((name) => path.join(archiveDir, name));
+}
+
+export function getFileSize(filePath: string): number {
+  try {
+    return fs.statSync(filePath).size;
+  } catch (error) {
+    logger.warn('stat archive file failed', { filePath, error });
+    return 0;
+  }
+}
+
+/**
+ * Remove oldest archive files when exceeding maxFiles limit.
+ * Archive files are organized by year (archive-YYYY.db), so maxFiles effectively means
+ * "keep N years of archived data" (default 10 years).
+ *
+ * This is part of the data retention policy: archive files older than maxFiles years
+ * are considered disposable history. The archived data has already served its purpose
+ * (query window, compliance retention period). If longer retention is needed,
+ * external backup should be configured before enabling archive cleanup.
+ *
+ * Compliance note: this deletion is a deliberate product decision — archived data beyond
+ * the retention window (default 10 years) is treated as expired. Operators requiring
+ * longer audit trails should configure external backup before enabling data retention.
+ */
+export function cleanupOldArchiveFiles(maxFiles: number): string[] {
+  const files = listArchiveFiles();
+  if (files.length <= maxFiles) {
+    return [];
+  }
+
+  // Files are sorted ascending by name (oldest first), remove the oldest ones
+  const toRemove = files.slice(0, files.length - maxFiles);
+  const removed: string[] = [];
+  for (const filePath of toRemove) {
+    try {
+      fs.unlinkSync(filePath);
+      removed.push(path.basename(filePath));
+      logger.info('removed old archive file', { filePath });
+    } catch (error) {
+      logger.warn('failed to remove old archive file', { filePath, error });
+    }
+  }
+  return removed;
+}

package/api/src/libs/credit-grant.ts

@@ -1,4 +1,5 @@
 import { BN } from '@ocap/util';
+import { Op } from 'sequelize';
 
 import { CreditGrant, Customer, PaymentCurrency, Subscription } from '../store/models';
 import { formatMetadata } from './util';
@@ -145,3 +146,135 @@ export function calculateExpiresAt(validDurationValue: number, validDurationUnit
 
   return expiresAt.unix();
 }
+
+/**
+ * Get credit grant statistics with flexible filtering
+ */
+export async function getCreditGrantStats(params: {
+  grantedBy?: string;
+  category?: 'paid' | 'promotional';
+  currencyId: string;
+  startDate: number;
+  endDate: number;
+  timezoneOffset?: number;
+}) {
+  const { grantedBy, category, currencyId, startDate, endDate, timezoneOffset } = params;
+  const offset = typeof timezoneOffset === 'number' ? timezoneOffset : 0;
+
+  // Fetch currency once at the start (since currencyId is required, results will only contain this currency)
+  const currency = await PaymentCurrency.findByPk(currencyId, {
+    attributes: ['id', 'name', 'symbol', 'decimal'],
+  });
+
+  if (!currency) {
+    return {
+      stats: {
+        currency_id: currencyId,
+        currency: null,
+        grant_count: 0,
+        total_granted: '0',
+        total_remaining: '0',
+        total_consumed: '0',
+      },
+      daily_stats: [],
+    };
+  }
+
+  const currencyJson = currency.toJSON();
+
+  // Build where clause
+  const where: any = {
+    currency_id: currencyId,
+    created_at: {
+      [Op.gte]: new Date(startDate * 1000),
+      [Op.lte]: new Date(endDate * 1000),
+    },
+  };
+
+  if (grantedBy) {
+    where['metadata.granted_by'] = grantedBy;
+  }
+
+  if (category) {
+    where.category = category;
+  }
+
+  const grants = (await CreditGrant.findAll({
+    where,
+    attributes: ['amount', 'remaining_amount', 'created_at'],
+    raw: true,
+  })) as any[];
+
+  const dailyMap = new Map<
+    string,
+    {
+      date: string;
+      grant_count: number;
+      total_granted: BN;
+      total_remaining: BN;
+    }
+  >();
+
+  const aggregate = {
+    grant_count: 0,
+    total_granted: new BN(0),
+    total_remaining: new BN(0),
+  };
+
+  grants.forEach((grant) => {
+    const date = dayjs.utc(grant.created_at).utcOffset(offset).format('YYYY-MM-DD');
+    const amount = grant.amount || '0';
+    const remainingAmount = grant.remaining_amount || '0';
+    if (!dailyMap.has(date)) {
+      dailyMap.set(date, {
+        date,
+        grant_count: 0,
+        total_granted: new BN(0),
+        total_remaining: new BN(0),
+      });
+    }
+
+    const daily = dailyMap.get(date)!;
+    daily.grant_count += 1;
+    daily.total_granted = daily.total_granted.add(new BN(amount));
+    daily.total_remaining = daily.total_remaining.add(new BN(remainingAmount));
+
+    aggregate.grant_count += 1;
+    aggregate.total_granted = aggregate.total_granted.add(new BN(amount));
+    aggregate.total_remaining = aggregate.total_remaining.add(new BN(remainingAmount));
+  });
+
+  const dailyStats = Array.from(dailyMap.values())
+    .sort((a, b) => a.date.localeCompare(b.date))
+    .map((day) => {
+      const totalGranted = day.total_granted.toString();
+      const totalRemaining = day.total_remaining.toString();
+      const totalConsumed = day.total_granted.sub(day.total_remaining).toString();
+
+      return {
+        date: day.date,
+        currency_id: currencyId,
+        grant_count: day.grant_count,
+        total_granted: totalGranted,
+        total_remaining: totalRemaining,
+        total_consumed: totalConsumed,
+      };
+    });
+
+  const totalGranted = aggregate.total_granted.toString();
+  const totalRemaining = aggregate.total_remaining.toString();
+  const totalConsumed = aggregate.total_granted.sub(aggregate.total_remaining).toString();
+  const statsWithConsumed = {
+    currency_id: currencyId,
+    currency: currencyJson,
+    grant_count: aggregate.grant_count,
+    total_granted: totalGranted,
+    total_remaining: totalRemaining,
+    total_consumed: totalConsumed,
+  };
+
+  return {
+    stats: statsWithConsumed,
+    daily_stats: dailyStats,
+  };
+}

package/api/src/queues/archive.ts

@@ -0,0 +1,32 @@
+import { nanoid } from 'nanoid';
+
+import logger from '../libs/logger';
+import createQueue from '../libs/queue';
+import { runArchiveJob } from '../libs/archive/executor';
+
+export type ArchiveQueueJob = {
+  tables?: string[];
+  dryRun?: boolean;
+  triggeredBy: 'cron' | 'manual';
+  triggeredByUserId?: string;
+};
+
+export const archiveQueue = createQueue<ArchiveQueueJob>({
+  name: 'archive',
+  onJob: (job) => runArchiveJob(job),
+  options: {
+    concurrency: 1,
+    maxRetries: 1,
+  },
+});
+
+export const enqueueArchiveJob = (job: ArchiveQueueJob) => {
+  const id = nanoid();
+  archiveQueue.push({ id, job, persist: false });
+  logger.info('archive job queued', { id, job });
+  return id;
+};
+
+archiveQueue.on('failed', ({ id, job, error }) => {
+  logger.error('archive job failed', { id, job, error });
+});

package/api/src/routes/archive.ts

@@ -0,0 +1,176 @@
+import { Router } from 'express';
+import Joi from 'joi';
+
+import dayjs from '../libs/dayjs';
+import { authenticate } from '../libs/security';
+import logger from '../libs/logger';
+import { createFlexibleEvent } from '../libs/audit';
+import { getRetentionConfig } from '../libs/archive/config';
+import { previewArchive } from '../libs/archive/executor';
+import { queryArchive } from '../libs/archive/query';
+import { enqueueArchiveJob } from '../queues/archive';
+import { ArchiveMetadata } from '../store/models/archive-metadata';
+import { listArchiveFiles, getFileSize } from '../libs/archive/store';
+
+const router = Router();
+const authAdmin = authenticate({ component: true, roles: ['owner', 'admin'] });
+
+const querySchema = Joi.object({
+  id: Joi.string().optional(),
+  customer_id: Joi.string().optional(),
+  from: Joi.number().integer().required(),
+  to: Joi.number().integer().optional(),
+  page: Joi.number().integer().min(1).default(1),
+  limit: Joi.number().integer().min(1).max(100).default(20),
+});
+
+router.get('/status', authAdmin, async (_req, res) => {
+  try {
+    const config = getRetentionConfig();
+    const lastRun = await ArchiveMetadata.findOne({ order: [['created_at', 'DESC']] });
+    const oldest = await ArchiveMetadata.findOne({ order: [['created_at', 'ASC']] });
+    const archives = await listArchiveFiles();
+    const totalSize = archives.reduce((sum, file) => sum + getFileSize(file), 0);
+
+    // Calculate next Wednesday at scheduled hour (cron: 0 0 ${hour} * * 3)
+    const scheduleHour = config.schedule.hour;
+    let nextRun = dayjs().day(3).hour(scheduleHour).minute(0).second(0).millisecond(0);
+    if (nextRun.isBefore(dayjs())) {
+      nextRun = nextRun.add(1, 'week');
+    }
+
+    res.json({
+      enabled: config.enabled,
+      config,
+      lastRun: lastRun
+        ? {
+            id: lastRun.id,
+            timestamp: lastRun.created_at,
+            duration_ms: lastRun.duration_ms,
+            status: lastRun.status,
+            tables_processed: Object.keys(lastRun.tables || {}).length,
+            total_archived: lastRun.total_records,
+            total_failed: Object.values(lastRun.tables || {}).reduce(
+              (sum: number, x: any) => sum + (x.failed_count || 0),
+              0
+            ),
+          }
+        : undefined,
+      nextScheduledRun: config.schedule.enabled ? nextRun.unix() : undefined,
+      archives: {
+        count: archives.length,
+        totalSize,
+        oldestArchive: oldest?.archive_file,
+        newestArchive: lastRun?.archive_file,
+      },
+    });
+  } catch (error: any) {
+    logger.error('archive status error', error);
+    res.status(500).json({ error: error.message });
+  }
+});
+
+router.get('/:table', authAdmin, async (req, res) => {
+  try {
+    const config = getRetentionConfig();
+    if (!config.enabled) {
+      return res.status(400).json({ error: 'Archive is disabled' });
+    }
+
+    const { value, error } = querySchema.validate(req.query, { stripUnknown: true, convert: true });
+    if (error) {
+      return res.status(400).json({ error: error.message });
+    }
+
+    const { table } = req.params;
+    if (!table) {
+      return res.status(400).json({ error: 'table is required' });
+    }
+    if (!Object.prototype.hasOwnProperty.call(config.tables, table)) {
+      return res.status(400).json({ error: `Unsupported table: ${table}` });
+    }
+
+    if (value.from === undefined) {
+      return res.status(400).json({ error: 'from is required' });
+    }
+
+    const result = await queryArchive(
+      {
+        table,
+        id: value.id,
+        customer_id: value.customer_id,
+        from: value.from,
+        to: value.to,
+        page: value.page,
+        limit: value.limit,
+      },
+      req.user?.did
+    );
+
+    await createFlexibleEvent(
+      'archive.query',
+      'Archive',
+      table,
+      {
+        table,
+        time_range: { from: value.from, to: value.to },
+        result_count: result.data.length,
+        archive_files: result.archiveFiles,
+      },
+      { requestedBy: req.user?.did }
+    );
+
+    return res.json({
+      data: result.data,
+      pagination: {
+        page: value.page,
+        limit: value.limit,
+        total: result.total,
+        hasMore: value.page * value.limit < result.total,
+      },
+      source: {
+        archive_files: result.archiveFiles,
+      },
+    });
+  } catch (error: any) {
+    logger.error('archive query error', error);
+    return res.status(500).json({ error: error.message });
+  }
+});
+
+const runSchema = Joi.object({
+  tables: Joi.array().items(Joi.string()).optional(),
+  dryRun: Joi.boolean().optional(),
+});
+
+router.post('/run', authAdmin, async (req, res) => {
+  try {
+    const config = getRetentionConfig();
+    if (!config.enabled) {
+      return res.status(400).json({ error: 'Archive is disabled' });
+    }
+
+    const { value, error } = runSchema.validate(req.body, { stripUnknown: true });
+    if (error) {
+      return res.status(400).json({ error: error.message });
+    }
+
+    if (value?.dryRun) {
+      const preview = await previewArchive({ tables: value.tables });
+      return res.json({ status: 'dry_run_complete', preview });
+    }
+
+    const jobId = enqueueArchiveJob({
+      tables: value.tables,
+      triggeredBy: 'manual',
+      triggeredByUserId: req.user?.did,
+    });
+
+    return res.json({ status: 'started', jobId });
+  } catch (error: any) {
+    logger.error('archive run error', error);
+    return res.status(500).json({ error: error.message });
+  }
+});
+
+export default router;

package/api/src/routes/credit-grants.ts

@@ -20,7 +20,7 @@ import {
   Product,
   Subscription,
 } from '../store/models';
-import { createCreditGrant } from '../libs/credit-grant';
+import { createCreditGrant, getCreditGrantStats } from '../libs/credit-grant';
 import { expireGrant } from '../queues/credit-grant';
 import { getMeterPriceIdsFromSubscription } from '../libs/subscription';
 import { blocklet } from '../libs/auth';
@@ -48,7 +48,7 @@ const authPortal = authenticate<CreditGrant>({
 const creditGrantSchema = Joi.object({
   amount: Joi.number().required(),
   currency_id: Joi.string().max(15).optional(),
-  customer_id: Joi.string().max(18).required(),
+  customer_id: Joi.string().max(45).required(),
   name: Joi.string().max(255).optional(),
   category: Joi.string().valid('paid', 'promotional').required(),
   priority: Joi.number().integer().min(0).max(100).default(50),
@@ -661,6 +661,59 @@ router.get('/verify-availability', authMine, async (req, res) => {
   }
 });
 
+// Schema for stats endpoint
+const statsSchema = Joi.object({
+  // Credit granted by did
+  // The did that grants the credit is not necessarily the component that send the request.
+  granted_by: Joi.string().optional(),
+  category: Joi.string().valid('paid', 'promotional').optional(),
+  currency_id: Joi.string().required(),
+  start_date: Joi.number().integer().required(),
+  end_date: Joi.number().integer().required(),
+  timezone_offset: Joi.number()
+    .integer()
+    .min(-12 * 60)
+    .max(14 * 60)
+    .optional(),
+});
+
+// Get credit grant statistics with flexible filtering
+router.get('/stats', auth, async (req, res) => {
+  try {
+    const { error, value } = statsSchema.validate(req.query, { stripUnknown: true });
+    if (error) {
+      return res.status(400).json({ error: error.message });
+    }
+
+    const {
+      granted_by: grantedBy,
+      category,
+      currency_id: currencyId,
+      start_date: startDate,
+      end_date: endDate,
+      timezone_offset: timezoneOffset,
+    } = value;
+
+    if (startDate > endDate) {
+      return res.status(400).json({ error: 'start_date must be less than or equal to end_date' });
+    }
+
+    const result = await getCreditGrantStats({
+      grantedBy,
+      category,
+      currencyId,
+      startDate,
+      endDate,
+      timezoneOffset,
+    });
+
+    return res.json(result);
+  } catch (err: any) {
+    logger.error('Error getting credit grant stats', { error: err.message, query: req.query });
+    return res.status(400).json({ error: err.message });
+  }
+});
+
 router.get('/:id', authPortal, async (req, res) => {
   const creditGrant = (await CreditGrant.findByPk(req.params.id, {
     include: [
@@ -745,7 +798,7 @@ router.post('/', auth, async (req, res) => {
     const creditGrant = await createCreditGrant({
       amount: unitAmount,
       currency_id: currencyId,
-      customer_id: req.body.customer_id,
+      customer_id: customer.id,
       name: req.body.name,
       category: req.body.category,
       priority: req.body.priority,
@@ -764,7 +817,7 @@ router.post('/', auth, async (req, res) => {
       paymentCurrency,
     });
   } catch (err: any) {
-    logger.error('create credit grant failed', { error: err.message, request: req.body });
+    logger.error('create credit grant failed', { error: err, request: req.body });
     return res.status(400).json({ error: err.message });
   }
 });

package/api/src/routes/index.ts

@@ -38,6 +38,7 @@ import webhookAttempts from './webhook-attempts';
 import webhookEndpoints from './webhook-endpoints';
 import vendor from './vendor';
 import tool from './tool';
+import archive from './archive';
 
 const router = Router();
 
@@ -97,5 +98,6 @@ router.use('/webhook-attempts', webhookAttempts);
 router.use('/webhook-endpoints', webhookEndpoints);
 router.use('/vendors', vendor);
 router.use('/tool', tool);
+router.use('/archive', archive);
 
 export default router;