payment-kit 1.18.46 → 1.18.47

package/api/src/libs/pagination.ts

@@ -0,0 +1,403 @@
+import logger from './logger';
+
+// In-memory cache with TTL support for simulated data
+const cache = new Map<string, { data: any; expireTime: number }>();
+const DEFAULT_TTL = 60 * 60 * 6 * 1000; // 6 hours
+
+/**
+ * Fetches data from cache if available and not expired, otherwise fetches fresh data
+ * Implements lazy cache cleanup to prevent memory leaks
+ */
+export async function getCachedOrFetch<T>(key: string, fetcher: () => Promise<T>, ttl?: number): Promise<T> {
+  const now = Date.now();
+  const cached = cache.get(key);
+
+  // Return cached data if valid
+  if (cached && now < cached.expireTime) {
+    return cached.data;
+  }
+
+  // Fetch and cache new data
+  const result = await fetcher();
+  cache.set(key, {
+    data: result,
+    expireTime: now + (ttl || DEFAULT_TTL),
+  });
+
+  // Lazy cleanup: Randomly clean expired entries to prevent memory leaks
+  if (Math.random() < 0.1) {
+    // 10% chance to trigger cleanup
+    for (const [k, v] of cache.entries()) {
+      if (now > v.expireTime) {
+        cache.delete(k);
+      }
+    }
+  }
+
+  return result;
+}
+
+export interface PaginationOptions {
+  page: number;
+  pageSize?: number; // Make optional to support pageSize = 0
+}
+
+export interface PaginatedResult<T> {
+  total: number;
+  data: T[];
+  paging: {
+    page: number;
+    pageSize: number;
+    totalPages: number;
+  };
+}
+
+// Generic data source interface
+export interface DataSource<T> {
+  count: () => Promise<number>;
+  fetch: (limit: number, offset?: number) => Promise<T[]>;
+  // Optional metadata for optimization hints
+  meta?: {
+    type?: 'database' | 'cached' | 'computed';
+    estimatedSize?: number;
+    cacheable?: boolean;
+  };
+}
+
+/**
+ * Merges multiple sorted arrays efficiently
+ * Handles undefined values and provides type safety
+ */
+function mergeSortedArrays<T>(arrays: T[][], orderBy: (a: T, b: T) => number, offset: number, limit: number): T[] {
+  // Initialize pointers for each array
+  const pointers = new Array(arrays.length).fill(0);
+  const result: T[] = [];
+  let skipped = 0;
+
+  while (result.length < limit) {
+    // Find the next item to process
+    let bestArrayIndex = -1;
+    let bestItem: T | undefined;
+
+    for (let i = 0; i < arrays.length; i++) {
+      const array = arrays[i];
+      const pointer = pointers[i];
+
+      if (!array || pointer >= array.length) {
+        // Array is undefined or exhausted
+        // eslint-disable-next-line no-continue
+        continue;
+      }
+
+      const currentItem = array[pointer];
+      if (currentItem === undefined) {
+        // Safety check for undefined items
+        // eslint-disable-next-line no-continue
+        continue;
+      }
+
+      if (bestItem === undefined || orderBy(currentItem, bestItem) < 0) {
+        bestItem = currentItem;
+        bestArrayIndex = i;
+      }
+    }
+
+    // No more items available
+    if (bestArrayIndex === -1 || bestItem === undefined) break;
+
+    // Advance the pointer for the selected array
+    pointers[bestArrayIndex]++;
+
+    // Apply pagination logic
+    if (skipped < offset) {
+      skipped++;
+    } else {
+      result.push(bestItem);
+    }
+  }
+
+  return result;
+}
+
+/**
+ * Estimates the optimal fetch strategy for data sources
+ */
+function calculateFetchStrategy<T>(
+  sources: DataSource<T>[],
+  options: PaginationOptions,
+  totalCounts: number[]
+): { fetchLimit: number; fetchOffset: number }[] {
+  const { page, pageSize = 0 } = options;
+  const offset = (page - 1) * pageSize;
+  const total = totalCounts.reduce((sum, count) => sum + count, 0);
+
+  return sources.map((source, index) => {
+    const sourceCount = totalCounts[index] ?? 0;
+    const sourceMeta = source.meta;
+
+    // Handle pageSize = 0: fetch all data
+    if (!pageSize) {
+      return { fetchLimit: Math.max(sourceCount, 1000), fetchOffset: 0 };
+    }
+
+    // For database sources with multiple sources, use conservative strategy
+    if (sourceMeta?.type === 'database') {
+      if (sources.length > 1) {
+        // For multi-source scenarios, we need more data to ensure correct merging
+        // Especially for later pages, estimation can be inaccurate
+        const bufferMultiplier = Math.max(2, Math.ceil(page / 2)); // More buffer for later pages
+        const fetchLimit = Math.min(
+          sourceCount,
+          Math.max(pageSize * bufferMultiplier, Math.ceil(sourceCount * 0.5)) // At least 50% of source data
+        );
+        return { fetchLimit, fetchOffset: 0 };
+      }
+      // Single database source can use precise offset
+      const estimatedRatio = total > 0 ? sourceCount / total : 0;
+      const estimatedOffset = Math.max(0, Math.floor(offset * estimatedRatio) - pageSize);
+      const fetchLimit = Math.min(pageSize * 3, Math.max(pageSize, sourceCount - estimatedOffset));
+      return { fetchLimit, fetchOffset: estimatedOffset };
+    }
+
+    // For cached/computed sources, always fetch more data to ensure accuracy
+    if (sourceMeta?.type === 'cached' || sourceMeta?.type === 'computed') {
+      // For multi-source, be more conservative
+      if (sources.length > 1) {
+        const fetchLimit = Math.min(sourceCount, Math.max(pageSize * 3, Math.ceil(sourceCount * 0.8))); // Get 80% of data
+        return { fetchLimit, fetchOffset: 0 };
+      }
+      const bufferSize = Math.min(sourceMeta.estimatedSize ?? sourceCount, pageSize * 2);
+      return { fetchLimit: Math.max(pageSize, bufferSize), fetchOffset: 0 };
+    }
+
+    // Default strategy: more conservative for multi-source
+    if (sources.length > 1) {
+      const fetchLimit = Math.min(sourceCount, Math.max(pageSize * 2, Math.ceil(sourceCount * 0.6))); // Get 60% of data
+      return { fetchLimit, fetchOffset: 0 };
+    }
+    const fetchLimit = Math.min(pageSize * 2, sourceCount);
+    return { fetchLimit: Math.max(pageSize, fetchLimit), fetchOffset: 0 };
+  });
+}
+
+/**
+ * Enhanced merge pagination with intelligent fetching strategy
+ * Supports multiple data sources with different characteristics
+ */
+export async function mergePaginate<T>(
+  sources: DataSource<T>[],
+  options: PaginationOptions,
+  orderBy: (a: T, b: T) => number
+): Promise<PaginatedResult<T>> {
+  const page = Math.max(1, options.page || 1);
+  const pageSize = options.pageSize ?? 0;
+  const offset = (page - 1) * pageSize;
+
+  try {
+    // Get total counts from all sources with error handling
+    const totalCounts = await Promise.all(
+      sources.map(async (source, index) => {
+        try {
+          return await source.count();
+        } catch (error) {
+          logger.error('Failed to get count from data source', { error, sourceIndex: index });
+          return 0;
+        }
+      })
+    );
+
+    const total = totalCounts.reduce((sum, count) => sum + count, 0);
+
+    // Handle pageSize = 0: return all data
+    if (!pageSize) {
+      const allData = await Promise.all(
+        sources.map(async (source, index) => {
+          try {
+            const sourceTotal = totalCounts[index] ?? 0;
+            const fetchLimit = Math.max(sourceTotal, 1000);
+            return await source.fetch(fetchLimit, 0);
+          } catch (error) {
+            logger.error('Failed to fetch all data from source', { error, sourceIndex: index });
+            return [];
+          }
+        })
+      ).then((arrays) => arrays.flat());
+
+      return {
+        total,
+        data: allData.sort(orderBy),
+        paging: {
+          page,
+          pageSize: 0,
+          totalPages: 1,
+        },
+      };
+    }
+
+    // Fast path: Single data source optimization
+    if (sources.length === 1) {
+      const source = sources[0];
+      if (!source) {
+        return {
+          total: 0,
+          data: [],
+          paging: {
+            page,
+            pageSize,
+            totalPages: 0,
+          },
+        };
+      }
+
+      try {
+        // For single source, we need to get enough data to sort correctly
+        // We can't just fetch the page directly because sorting might change the order
+        const sourceTotal = totalCounts[0] ?? 0;
+
+        // For database sources, we can try to be smart about fetching
+        if (source.meta?.type === 'database') {
+          // For database sources, assume they can handle sorting at the database level
+          // and fetch the exact page we need
+          const data = await source.fetch(pageSize, offset);
+          return {
+            total,
+            data,
+            paging: {
+              page,
+              pageSize,
+              totalPages: Math.ceil(total / pageSize),
+            },
+          };
+        }
+        // For other sources (cached, computed), fetch all data and sort
+        const allData = await source.fetch(sourceTotal, 0);
+        const sortedData = allData.sort(orderBy);
+        const pageData = sortedData.slice(offset, offset + pageSize);
+
+        return {
+          total,
+          data: pageData,
+          paging: {
+            page,
+            pageSize,
+            totalPages: Math.ceil(total / pageSize),
+          },
+        };
+      } catch (error) {
+        logger.error('Failed to fetch data from single source', { error });
+        return {
+          total: 0,
+          data: [],
+          paging: {
+            page,
+            pageSize,
+            totalPages: 0,
+          },
+        };
+      }
+    }
+
+    // Calculate optimal fetch strategy for each source
+    const fetchStrategies = calculateFetchStrategy(sources, { page, pageSize }, totalCounts);
+
+    // Fetch data from all sources
+    const dataArrays = await Promise.all(
+      sources.map(async (source, index) => {
+        try {
+          const strategy = fetchStrategies[index];
+          if (!strategy) {
+            logger.warn('No fetch strategy found for source', { sourceIndex: index });
+            return [];
+          }
+
+          const data = await source.fetch(strategy.fetchLimit, strategy.fetchOffset);
+
+          // Only sort non-database sources (database sources are assumed to be pre-sorted)
+          if (source.meta?.type === 'database') {
+            // Database sources are assumed to handle sorting at the database level
+            return data;
+          }
+          // Non-database sources need to be sorted in memory
+          return data.sort(orderBy);
+        } catch (error) {
+          logger.error('Failed to fetch data from source', { error, sourceIndex: index });
+          return [];
+        }
+      })
+    );
+
+    // Merge all sorted arrays efficiently
+    const paged = mergeSortedArrays(dataArrays, orderBy, offset, pageSize);
+
+    return {
+      total,
+      data: paged,
+      paging: {
+        page,
+        pageSize,
+        totalPages: Math.ceil(total / pageSize),
+      },
+    };
+  } catch (error) {
+    logger.error('Failed to merge paginate data', { error, options });
+    return {
+      total: 0,
+      data: [],
+      paging: {
+        page,
+        pageSize,
+        totalPages: 0,
+      },
+    };
+  }
+}
+
+/**
+ * Creates a time-based sorting function with optional secondary sort
+ * Extended to support custom secondary field and maintains backward compatibility
+ *
+ * @param order - Sort direction ('asc' or 'desc')
+ * @param secondaryField - Secondary sort field (defaults to 'id')
+ * @returns Sorting function that compares timestamps and secondary field
+ */
+export function defaultTimeOrderBy(
+  order: 'asc' | 'desc' = 'desc', // Keep original default
+  secondaryField: string = 'id'
+): (a: any, b: any) => number {
+  return (a, b) => {
+    const dateA = new Date(a.created_at).getTime();
+    const dateB = new Date(b.created_at).getTime();
+
+    // First compare by timestamp
+    if (dateA !== dateB) {
+      return order === 'asc' ? dateA - dateB : dateB - dateA;
+    }
+
+    // If timestamps are equal, compare by secondary field
+    const aValue = a[secondaryField];
+    const bValue = b[secondaryField];
+
+    // If secondary field is undefined or not comparable, use id as fallback
+    if (aValue === undefined || bValue === undefined || typeof aValue !== typeof bValue) {
+      // Use ID as secondary sort key when timestamps are equal - original behavior
+      if (a.id && b.id) {
+        return order === 'asc' ? a.id.localeCompare(b.id) : b.id.localeCompare(a.id);
+      }
+      return 0;
+    }
+
+    // Compare based on field type
+    if (typeof aValue === 'string') {
+      return order === 'asc' ? aValue.localeCompare(bValue) : bValue.localeCompare(aValue);
+    }
+    if (typeof aValue === 'number') {
+      return order === 'asc' ? aValue - bValue : bValue - aValue;
+    }
+
+    // For unsupported types, fallback to id comparison
+    if (a.id && b.id) {
+      return order === 'asc' ? a.id.localeCompare(b.id) : b.id.localeCompare(a.id);
+    }
+    return 0;
+  };
+}