payment-kit 1.25.8 → 1.25.10

package/api/src/libs/archive/store.ts

@@ -0,0 +1,200 @@
+import fs from 'fs';
+import path from 'path';
+
+import config from '@blocklet/sdk/lib/config';
+import { Sequelize, DataTypes } from 'sequelize';
+
+import logger from '../logger';
+
+export function getArchiveDir(): string {
+  const dataDir = config.env?.dataDir || '/tmp';
+  const archiveDir = path.join(dataDir, 'archive');
+  if (!fs.existsSync(archiveDir)) {
+    fs.mkdirSync(archiveDir, { recursive: true });
+  }
+  return archiveDir;
+}
+
+export function getArchiveFilePath(fileName: string): string {
+  return path.join(getArchiveDir(), fileName);
+}
+
+/**
+ * Get archive file path for a specific year based on record's created_at.
+ * Archive files are organized by data year (e.g., archive-2024.db contains 2024 data).
+ */
+export function getArchiveFilePathForYear(year: number): string {
+  return path.join(getArchiveDir(), `archive-${year}.db`);
+}
+
+/**
+ * Extract year from a record's created_at (fallback to updated_at, then current year).
+ */
+export function getRecordYear(record: any): number {
+  const createdAt = record?.created_at;
+  const updatedAt = record?.updated_at;
+  const dateValue = createdAt || updatedAt;
+
+  if (dateValue) {
+    const date = dateValue instanceof Date ? dateValue : new Date(dateValue);
+    if (!Number.isNaN(date.getTime())) {
+      return date.getFullYear();
+    }
+  }
+
+  return new Date().getFullYear();
+}
+
+export function openArchiveSequelize(filePath: string): Sequelize {
+  return new Sequelize({
+    dialect: 'sqlite',
+    storage: filePath,
+    logging: false,
+  });
+}
+
+function normalizeCreateTableSql(sql: string): string {
+  return sql.replace(/^CREATE TABLE\s+/i, 'CREATE TABLE IF NOT EXISTS ');
+}
+
+function normalizeCreateIndexSql(sql: string): string {
+  return sql
+    .replace(/^CREATE INDEX\s+/i, 'CREATE INDEX IF NOT EXISTS ')
+    .replace(/^CREATE UNIQUE INDEX\s+/i, 'CREATE UNIQUE INDEX IF NOT EXISTS ');
+}
+
+type ColumnInfo = { name: string; type: string; notnull: number; dflt_value: any; pk: number };
+
+export async function ensureArchiveTable(
+  tableName: string,
+  mainSequelize: Sequelize,
+  archiveSequelize: Sequelize
+): Promise<void> {
+  // 1. Copy table schema (CREATE TABLE IF NOT EXISTS)
+  const [tableRows] = await mainSequelize.query('SELECT sql FROM sqlite_master WHERE type = :type AND name = :name', {
+    replacements: { type: 'table', name: tableName },
+  });
+  const tableRow = (Array.isArray(tableRows) ? tableRows[0] : null) as { sql?: string } | null;
+  const createSql = tableRow?.sql;
+  if (!createSql) {
+    throw new Error(`table schema not found: ${tableName}`);
+  }
+
+  const normalizedSql = normalizeCreateTableSql(createSql);
+  await archiveSequelize.query(normalizedSql);
+
+  // 2. Copy indexes (PRIMARY KEY auto-indexes have sql=NULL, skip them)
+  const [indexRows] = await mainSequelize.query(
+    'SELECT sql FROM sqlite_master WHERE type = :type AND tbl_name = :tableName AND sql IS NOT NULL',
+    { replacements: { type: 'index', tableName } }
+  );
+  for (const indexRow of indexRows as { sql?: string }[]) {
+    if (indexRow?.sql) {
+      try {
+        const indexSql = normalizeCreateIndexSql(indexRow.sql);
+        // eslint-disable-next-line no-await-in-loop
+        await archiveSequelize.query(indexSql);
+      } catch (err: any) {
+        // Ignore "index already exists" errors
+        if (!err.message?.includes('already exists')) {
+          logger.warn('failed to create archive index', { tableName, error: err.message });
+        }
+      }
+    }
+  }
+
+  // 3. Sync missing columns from main DB to archive DB
+  // This handles schema migrations: if main DB has new columns, add them to archive DB
+  const [mainColumns] = await mainSequelize.query(`PRAGMA table_info("${tableName}")`);
+  const [archiveColumns] = await archiveSequelize.query(`PRAGMA table_info("${tableName}")`);
+
+  const mainColumnNames = new Set((mainColumns as ColumnInfo[]).map((c) => c.name));
+  const archiveColumnNames = new Set((archiveColumns as ColumnInfo[]).map((c) => c.name));
+
+  // Find columns in main DB but not in archive DB
+  const missingColumns = (mainColumns as ColumnInfo[]).filter((c) => !archiveColumnNames.has(c.name));
+
+  for (const col of missingColumns) {
+    try {
+      // SQLite ALTER TABLE ADD COLUMN only supports columns with default values or NULL
+      // For NOT NULL columns without defaults, we need to add them as nullable
+      const nullable = col.notnull === 0 || col.dflt_value !== null;
+      const defaultClause = col.dflt_value !== null ? ` DEFAULT ${col.dflt_value}` : '';
+
+      // eslint-disable-next-line no-await-in-loop
+      await archiveSequelize.query(
+        `ALTER TABLE "${tableName}" ADD COLUMN "${col.name}" ${col.type}${nullable ? '' : ' NULL'}${defaultClause}`
+      );
+      logger.info('added missing column to archive table', { tableName, column: col.name });
+    } catch (err: any) {
+      // Ignore "duplicate column" errors (race condition protection)
+      if (!err.message?.includes('duplicate column')) {
+        logger.warn('failed to add missing column to archive table', {
+          tableName,
+          column: col.name,
+          error: err.message,
+        });
+      }
+    }
+  }
+
+  // 4. Add archived_at column if not exists
+  if (!archiveColumnNames.has('archived_at') && !mainColumnNames.has('archived_at')) {
+    const queryInterface = archiveSequelize.getQueryInterface();
+    await queryInterface.addColumn(tableName, 'archived_at', {
+      type: DataTypes.DATE,
+      allowNull: false,
+    });
+  }
+}
+
+export function listArchiveFiles(): string[] {
+  const archiveDir = getArchiveDir();
+  const files = fs.readdirSync(archiveDir).filter((name) => name.endsWith('.db'));
+  files.sort();
+  return files.map((name) => path.join(archiveDir, name));
+}
+
+export function getFileSize(filePath: string): number {
+  try {
+    return fs.statSync(filePath).size;
+  } catch (error) {
+    logger.warn('stat archive file failed', { filePath, error });
+    return 0;
+  }
+}
+
+/**
+ * Remove oldest archive files when exceeding maxFiles limit.
+ * Archive files are organized by year (archive-YYYY.db), so maxFiles effectively means
+ * "keep N years of archived data" (default 10 years).
+ *
+ * This is part of the data retention policy: archive files older than maxFiles years
+ * are considered disposable history. The archived data has already served its purpose
+ * (query window, compliance retention period). If longer retention is needed,
+ * external backup should be configured before enabling archive cleanup.
+ *
+ * Compliance note: this deletion is a deliberate product decision — archived data beyond
+ * the retention window (default 10 years) is treated as expired. Operators requiring
+ * longer audit trails should configure external backup before enabling data retention.
+ */
+export function cleanupOldArchiveFiles(maxFiles: number): string[] {
+  const files = listArchiveFiles();
+  if (files.length <= maxFiles) {
+    return [];
+  }
+
+  // Files are sorted ascending by name (oldest first), remove the oldest ones
+  const toRemove = files.slice(0, files.length - maxFiles);
+  const removed: string[] = [];
+  for (const filePath of toRemove) {
+    try {
+      fs.unlinkSync(filePath);
+      removed.push(path.basename(filePath));
+      logger.info('removed old archive file', { filePath });
+    } catch (error) {
+      logger.warn('failed to remove old archive file', { filePath, error });
+    }
+  }
+  return removed;
+}

package/api/src/queues/archive.ts

@@ -0,0 +1,32 @@
+import { nanoid } from 'nanoid';
+
+import logger from '../libs/logger';
+import createQueue from '../libs/queue';
+import { runArchiveJob } from '../libs/archive/executor';
+
+export type ArchiveQueueJob = {
+  tables?: string[];
+  dryRun?: boolean;
+  triggeredBy: 'cron' | 'manual';
+  triggeredByUserId?: string;
+};
+
+export const archiveQueue = createQueue<ArchiveQueueJob>({
+  name: 'archive',
+  onJob: (job) => runArchiveJob(job),
+  options: {
+    concurrency: 1,
+    maxRetries: 1,
+  },
+});
+
+export const enqueueArchiveJob = (job: ArchiveQueueJob) => {
+  const id = nanoid();
+  archiveQueue.push({ id, job, persist: false });
+  logger.info('archive job queued', { id, job });
+  return id;
+};
+
+archiveQueue.on('failed', ({ id, job, error }) => {
+  logger.error('archive job failed', { id, job, error });
+});

package/api/src/routes/archive.ts

@@ -0,0 +1,176 @@
+import { Router } from 'express';
+import Joi from 'joi';
+
+import dayjs from '../libs/dayjs';
+import { authenticate } from '../libs/security';
+import logger from '../libs/logger';
+import { createFlexibleEvent } from '../libs/audit';
+import { getRetentionConfig } from '../libs/archive/config';
+import { previewArchive } from '../libs/archive/executor';
+import { queryArchive } from '../libs/archive/query';
+import { enqueueArchiveJob } from '../queues/archive';
+import { ArchiveMetadata } from '../store/models/archive-metadata';
+import { listArchiveFiles, getFileSize } from '../libs/archive/store';
+
+const router = Router();
+const authAdmin = authenticate({ component: true, roles: ['owner', 'admin'] });
+
+const querySchema = Joi.object({
+  id: Joi.string().optional(),
+  customer_id: Joi.string().optional(),
+  from: Joi.number().integer().required(),
+  to: Joi.number().integer().optional(),
+  page: Joi.number().integer().min(1).default(1),
+  limit: Joi.number().integer().min(1).max(100).default(20),
+});
+
+router.get('/status', authAdmin, async (_req, res) => {
+  try {
+    const config = getRetentionConfig();
+    const lastRun = await ArchiveMetadata.findOne({ order: [['created_at', 'DESC']] });
+    const oldest = await ArchiveMetadata.findOne({ order: [['created_at', 'ASC']] });
+    const archives = await listArchiveFiles();
+    const totalSize = archives.reduce((sum, file) => sum + getFileSize(file), 0);
+
+    // Calculate next Wednesday at scheduled hour (cron: 0 0 ${hour} * * 3)
+    const scheduleHour = config.schedule.hour;
+    let nextRun = dayjs().day(3).hour(scheduleHour).minute(0).second(0).millisecond(0);
+    if (nextRun.isBefore(dayjs())) {
+      nextRun = nextRun.add(1, 'week');
+    }
+
+    res.json({
+      enabled: config.enabled,
+      config,
+      lastRun: lastRun
+        ? {
+            id: lastRun.id,
+            timestamp: lastRun.created_at,
+            duration_ms: lastRun.duration_ms,
+            status: lastRun.status,
+            tables_processed: Object.keys(lastRun.tables || {}).length,
+            total_archived: lastRun.total_records,
+            total_failed: Object.values(lastRun.tables || {}).reduce(
+              (sum: number, x: any) => sum + (x.failed_count || 0),
+              0
+            ),
+          }
+        : undefined,
+      nextScheduledRun: config.schedule.enabled ? nextRun.unix() : undefined,
+      archives: {
+        count: archives.length,
+        totalSize,
+        oldestArchive: oldest?.archive_file,
+        newestArchive: lastRun?.archive_file,
+      },
+    });
+  } catch (error: any) {
+    logger.error('archive status error', error);
+    res.status(500).json({ error: error.message });
+  }
+});
+
+router.get('/:table', authAdmin, async (req, res) => {
+  try {
+    const config = getRetentionConfig();
+    if (!config.enabled) {
+      return res.status(400).json({ error: 'Archive is disabled' });
+    }
+
+    const { value, error } = querySchema.validate(req.query, { stripUnknown: true, convert: true });
+    if (error) {
+      return res.status(400).json({ error: error.message });
+    }
+
+    const { table } = req.params;
+    if (!table) {
+      return res.status(400).json({ error: 'table is required' });
+    }
+    if (!Object.prototype.hasOwnProperty.call(config.tables, table)) {
+      return res.status(400).json({ error: `Unsupported table: ${table}` });
+    }
+
+    if (value.from === undefined) {
+      return res.status(400).json({ error: 'from is required' });
+    }
+
+    const result = await queryArchive(
+      {
+        table,
+        id: value.id,
+        customer_id: value.customer_id,
+        from: value.from,
+        to: value.to,
+        page: value.page,
+        limit: value.limit,
+      },
+      req.user?.did
+    );
+
+    await createFlexibleEvent(
+      'archive.query',
+      'Archive',
+      table,
+      {
+        table,
+        time_range: { from: value.from, to: value.to },
+        result_count: result.data.length,
+        archive_files: result.archiveFiles,
+      },
+      { requestedBy: req.user?.did }
+    );
+
+    return res.json({
+      data: result.data,
+      pagination: {
+        page: value.page,
+        limit: value.limit,
+        total: result.total,
+        hasMore: value.page * value.limit < result.total,
+      },
+      source: {
+        archive_files: result.archiveFiles,
+      },
+    });
+  } catch (error: any) {
+    logger.error('archive query error', error);
+    return res.status(500).json({ error: error.message });
+  }
+});
+
+const runSchema = Joi.object({
+  tables: Joi.array().items(Joi.string()).optional(),
+  dryRun: Joi.boolean().optional(),
+});
+
+router.post('/run', authAdmin, async (req, res) => {
+  try {
+    const config = getRetentionConfig();
+    if (!config.enabled) {
+      return res.status(400).json({ error: 'Archive is disabled' });
+    }
+
+    const { value, error } = runSchema.validate(req.body, { stripUnknown: true });
+    if (error) {
+      return res.status(400).json({ error: error.message });
+    }
+
+    if (value?.dryRun) {
+      const preview = await previewArchive({ tables: value.tables });
+      return res.json({ status: 'dry_run_complete', preview });
+    }
+
+    const jobId = enqueueArchiveJob({
+      tables: value.tables,
+      triggeredBy: 'manual',
+      triggeredByUserId: req.user?.did,
+    });
+
+    return res.json({ status: 'started', jobId });
+  } catch (error: any) {
+    logger.error('archive run error', error);
+    return res.status(500).json({ error: error.message });
+  }
+});
+
+export default router;

package/api/src/routes/index.ts

@@ -38,6 +38,7 @@ import webhookAttempts from './webhook-attempts';
 import webhookEndpoints from './webhook-endpoints';
 import vendor from './vendor';
 import tool from './tool';
+import archive from './archive';
 
 const router = Router();
 
@@ -97,5 +98,6 @@ router.use('/webhook-attempts', webhookAttempts);
 router.use('/webhook-endpoints', webhookEndpoints);
 router.use('/vendors', vendor);
 router.use('/tool', tool);
+router.use('/archive', archive);
 
 export default router;