webspresso 0.0.102 → 0.0.103

package/plugins/site-analytics/tracking.js

@@ -11,6 +11,66 @@ const { detectCountry } = require('./geo');
 const STATIC_EXT = /\.(js|css|png|jpg|jpeg|gif|svg|ico|woff2?|ttf|eot|map|webp|avif|mp4|webm)$/i;
 const DEFAULT_EXCLUDE = ['/_admin', '/api/', '/health', '/_next', '/__'];
 
+/**
+ * Permanent / Fatal DB error codes that cannot be recovered by immediate retries.
+ */
+const PERMANENT_ERROR_CODES = new Set([
+  'SQLITE_FULL',
+  'SQLITE_IOERR',
+  'SQLITE_CORRUPT',
+  'SQLITE_CANTOPEN',
+  'SQLITE_NOTADB',
+  'ER_DISK_FULL',
+  'ENOSPC',
+]);
+
+/**
+ * Patterns matching fatal disk full or corrupted database errors.
+ */
+const PERMANENT_ERROR_PATTERNS = [
+  /database or disk is full/i,
+  /disk full/i,
+  /no space left on device/i,
+  /disk i\/o error/i,
+  /database disk image is malformed/i,
+  /file is not a database/i,
+];
+
+/**
+ * Sanitize and format database error messages to prevent raw SQL query leakage in logs.
+ * Knex appends executed queries ("insert into ... - <actual error>") which can be 50-100 KB.
+ * @param {Error|Object|string} e
+ * @returns {string}
+ */
+function formatDbError(e) {
+  if (!e) return 'Unknown error';
+
+  let msg;
+  if (typeof e.message === 'string' && e.message.includes(' - ')) {
+    msg = e.message.split(' - ').pop().trim();
+  } else if (typeof e.message === 'string') {
+    msg = e.message.trim();
+  } else if (typeof e === 'string') {
+    msg = e.trim();
+  } else {
+    msg = String(e.code || 'Database error');
+  }
+
+  return msg.replace(/[\r\n]+/g, ' ').trim().slice(0, 150);
+}
+
+/**
+ * Detect whether a database error is permanent/fatal (e.g. disk full, corruption).
+ * @param {Error|Object|string} e
+ * @returns {boolean}
+ */
+function isPermanentDbError(e) {
+  if (!e) return false;
+  if (e.code && PERMANENT_ERROR_CODES.has(e.code)) return true;
+  const msg = typeof e.message === 'string' ? e.message : (typeof e === 'string' ? e : '');
+  return PERMANENT_ERROR_PATTERNS.some((pattern) => pattern.test(msg));
+}
+
 /**
  * Hash a string with SHA-256 (truncated to 16 hex chars for compactness)
  */
@@ -38,6 +98,10 @@ function getClientIp(req) {
  * @param {string} [options.tableName='analytics_page_views'] - DB table name
  * @param {number} [options.batchSize=20] - Flush when queue reaches this size
  * @param {number} [options.flushIntervalMs=3000] - Flush interval for low traffic
+ * @param {number} [options.maxRetries=3] - Maximum retry attempts per batch item before dropping
+ * @param {number} [options.maxBackoffMs=60000] - Max exponential backoff cooldown
+ * @param {number} [options.maxQueueSize=5000] - Queue memory cap to prevent OOM
+ * @param {number} [options.errorLogIntervalMs=60000] - Rate limit window for batch failure logs
  */
 function createTrackingMiddleware(options) {
   const {
@@ -47,12 +111,26 @@ function createTrackingMiddleware(options) {
     tableName = 'analytics_page_views',
     batchSize = 20,
     flushIntervalMs = 3000,
+    maxRetries = 3,
+    maxBackoffMs = 60000,
+    maxQueueSize = 5000,
+    errorLogIntervalMs = 60000,
   } = options;
 
   const allExcludes = [...DEFAULT_EXCLUDE, ...excludePaths];
   let tableReady = false;
   const queue = [];
-  let flushScheduled = false;
+
+  // Circuit breaker & backoff state
+  let failureCount = 0;
+  let nextAllowedFlushTime = 0;
+  let isFlushing = false;
+  let flushPromise = null;
+  let flushTimeout = null;
+
+  // Log throttling state
+  let lastErrorLogTime = 0;
+  let suppressedErrorCount = 0;
 
   async function ensureTable() {
     if (tableReady) return;
@@ -75,7 +153,26 @@ function createTrackingMiddleware(options) {
       }
       tableReady = true;
     } catch (e) {
-      console.error('[site-analytics] Failed to ensure table:', e.message);
+      logError(e, 'Failed to ensure table');
+      throw e;
+    }
+  }
+
+  function logError(e, prefix = 'Batch insert failed') {
+    const now = Date.now();
+    const formatted = formatDbError(e);
+
+    if (lastErrorLogTime === 0 || now - lastErrorLogTime >= errorLogIntervalMs) {
+      const elapsedSec = lastErrorLogTime > 0 ? Math.round((now - lastErrorLogTime) / 1000) : 0;
+      if (suppressedErrorCount > 0) {
+        console.error(`[site-analytics] ${prefix} (suppressed ${suppressedErrorCount} occurrences in last ${elapsedSec}s): ${formatted}`);
+      } else {
+        console.error(`[site-analytics] ${prefix}: ${formatted}`);
+      }
+      lastErrorLogTime = now;
+      suppressedErrorCount = 0;
+    } else {
+      suppressedErrorCount++;
     }
   }
 
@@ -83,17 +180,17 @@ function createTrackingMiddleware(options) {
   const sessionMap = new Map();
   const SESSION_TIMEOUT = 30 * 60 * 1000; // 30 minutes
   const MAX_SESSIONS = 10000;
-  const MAX_QUEUE_SIZE = 5000;
 
   // Periodic session cleanup
-  setInterval(() => {
+  const sessionCleanupInterval = setInterval(() => {
     const now = Date.now();
     for (const [vid, data] of sessionMap) {
       if (now - data.lastSeen > SESSION_TIMEOUT * 2) {
         sessionMap.delete(vid);
       }
     }
-  }, 5 * 60 * 1000).unref();
+  }, 5 * 60 * 1000);
+  if (sessionCleanupInterval.unref) sessionCleanupInterval.unref();
 
   function getSessionId(visitorId) {
     const now = Date.now();
@@ -115,44 +212,93 @@ function createTrackingMiddleware(options) {
   }
 
   async function flushQueue() {
+    if (flushPromise) return flushPromise;
     if (queue.length === 0) return;
-    const batch = queue.splice(0, queue.length);
 
-    try {
-      await ensureTable();
-      for (const row of batch) {
-        row.created_at = knex.fn.now();
-      }
-      await knex(tableName).insert(batch);
-    } catch (e) {
-      console.error('[site-analytics] Batch insert failed:', e.message);
-      // Re-queue failed items with max size limit to prevent memory leak
-      if (queue.length + batch.length <= MAX_QUEUE_SIZE) {
-        queue.unshift(...batch);
-      } else {
-        const available = Math.max(0, MAX_QUEUE_SIZE - queue.length);
-        if (available > 0) {
-          queue.unshift(...batch.slice(batch.length - available));
+    const now = Date.now();
+    if (now < nextAllowedFlushTime) {
+      scheduleFlush(nextAllowedFlushTime - now);
+      return;
+    }
+
+    isFlushing = true;
+    flushPromise = (async () => {
+      const batch = queue.splice(0, queue.length);
+
+      try {
+        await ensureTable();
+        const dbBatch = batch.map(({ _retries, ...row }) => ({
+          ...row,
+          created_at: knex.fn.now(),
+        }));
+        await knex(tableName).insert(dbBatch);
+
+        // Successfully inserted: reset circuit breaker & failure counters
+        failureCount = 0;
+        nextAllowedFlushTime = 0;
+        lastErrorLogTime = 0;
+        suppressedErrorCount = 0;
+      } catch (e) {
+        failureCount++;
+        const backoffDelay = Math.min(flushIntervalMs * Math.pow(2, failureCount), maxBackoffMs);
+        nextAllowedFlushTime = Date.now() + backoffDelay;
+
+        // Rate-limited sanitized logging (no raw SQL query leakage)
+        logError(e, 'Batch insert failed');
+
+        // Schedule next attempt respecting backoff
+        scheduleFlush(backoffDelay);
+
+        // Drop policy: If fatal/permanent error (disk full, corrupted DB, etc.), drop entire batch
+        if (!isPermanentDbError(e)) {
+          const retryable = [];
+          for (const item of batch) {
+            item._retries = (item._retries || 0) + 1;
+            if (item._retries < maxRetries) {
+              retryable.push(item);
+            }
+          }
+
+          // Re-queue items that haven't exceeded retry limits, respecting maxQueueSize
+          if (retryable.length > 0) {
+            const available = Math.max(0, maxQueueSize - queue.length);
+            if (available > 0) {
+              queue.unshift(...retryable.slice(0, available));
+            }
+          }
         }
+      } finally {
+        isFlushing = false;
+        flushPromise = null;
       }
-    }
+    })();
+
+    return flushPromise;
   }
 
-  function scheduleFlush() {
-    if (flushScheduled || queue.length === 0) return;
-    flushScheduled = true;
-    setTimeout(() => {
-      flushScheduled = false;
+  function scheduleFlush(customDelay) {
+    if (flushTimeout || queue.length === 0) return;
+    const now = Date.now();
+    const delay = customDelay !== undefined
+      ? customDelay
+      : Math.max(flushIntervalMs, nextAllowedFlushTime - now, 0);
+
+    flushTimeout = setTimeout(() => {
+      flushTimeout = null;
       flushQueue().catch(() => {});
-    }, flushIntervalMs);
+    }, delay);
+    if (flushTimeout.unref) flushTimeout.unref();
   }
 
   // Periodic flush (catches low-traffic case)
-  setInterval(() => {
-    if (queue.length > 0) flushQueue().catch(() => {});
-  }, flushIntervalMs).unref();
+  const periodicFlushInterval = setInterval(() => {
+    if (queue.length > 0 && Date.now() >= nextAllowedFlushTime && !isFlushing) {
+      flushQueue().catch(() => {});
+    }
+  }, flushIntervalMs);
+  if (periodicFlushInterval.unref) periodicFlushInterval.unref();
 
-  return async function trackingMiddleware(req, res, next) {
+  async function trackingMiddleware(req, res, next) {
     next();
 
     if (req.method !== 'GET') return;
@@ -168,33 +314,86 @@ function createTrackingMiddleware(options) {
 
       if (isBot && !trackBots) return;
 
+      // Protect queue under pressure or during active database failures:
+      // Drop bots immediately to preserve queue capacity and avoid wasting DB I/O
+      if (isBot && (failureCount > 0 || queue.length >= maxQueueSize * 0.7)) {
+        return;
+      }
+
       const ipHash = quickHash(ip);
       const visitorId = quickHash(ip + '|' + userAgent);
-      const sessionId = getSessionId(visitorId);
+
+      // Do not thrash human sessionMap with bot/crawler visits
+      const sessionId = isBot
+        ? quickHash('bot-' + (botName || 'crawler') + '-' + visitorId)
+        : getSessionId(visitorId);
+
       const country = detectCountry(req);
       const referrer = req.headers['referer'] || req.headers['referrer'] || null;
 
-      queue.push({
-        session_id: sessionId,
-        visitor_id: visitorId,
-        path,
-        referrer: referrer ? referrer.slice(0, 500) : null,
-        user_agent: userAgent.slice(0, 1000),
-        ip_hash: ipHash,
-        country,
-        is_bot: isBot,
-        bot_name: botName,
-      });
+      if (queue.length < maxQueueSize) {
+        queue.push({
+          session_id: sessionId,
+          visitor_id: visitorId,
+          path,
+          referrer: referrer ? referrer.slice(0, 500) : null,
+          user_agent: userAgent.slice(0, 1000),
+          ip_hash: ipHash,
+          country,
+          is_bot: isBot,
+          bot_name: botName,
+        });
+      }
 
       if (queue.length >= batchSize) {
-        flushQueue().catch(() => {});
+        if (Date.now() >= nextAllowedFlushTime) {
+          flushQueue().catch(() => {});
+        } else {
+          scheduleFlush();
+        }
       } else {
         scheduleFlush();
       }
     } catch (e) {
       // Non-blocking: don't let tracking errors affect the app
     }
+  }
+
+  // Diagnostic / testing accessors
+  trackingMiddleware.flush = () => flushQueue();
+  trackingMiddleware.getQueue = () => queue;
+  trackingMiddleware.getFailureCount = () => failureCount;
+  trackingMiddleware.getNextAllowedFlushTime = () => nextAllowedFlushTime;
+  trackingMiddleware.resetState = (opts = {}) => {
+    failureCount = 0;
+    nextAllowedFlushTime = 0;
+    const resetLogs = typeof opts === 'boolean' ? opts : !!opts.resetLogs;
+    if (resetLogs) {
+      lastErrorLogTime = 0;
+      suppressedErrorCount = 0;
+    }
+    queue.length = 0;
+    if (flushTimeout) {
+      clearTimeout(flushTimeout);
+      flushTimeout = null;
+    }
   };
+  trackingMiddleware.stop = () => {
+    if (flushTimeout) {
+      clearTimeout(flushTimeout);
+      flushTimeout = null;
+    }
+    clearInterval(sessionCleanupInterval);
+    clearInterval(periodicFlushInterval);
+  };
+
+  return trackingMiddleware;
 }
 
-module.exports = { createTrackingMiddleware };
+module.exports = {
+  createTrackingMiddleware,
+  formatDbError,
+  isPermanentDbError,
+  PERMANENT_ERROR_CODES,
+  PERMANENT_ERROR_PATTERNS,
+};

package/src/api/api-loader.js

@@ -95,7 +95,7 @@ function createApiHandler(descriptor, context) {
       // 1. Reload module in development
       if (isDev) {
         try {
-          const resolvedPath = require.resolve(descriptor.file);
+          const resolvedPath = descriptor._resolvedPath || (descriptor._resolvedPath = require.resolve(descriptor.file));
           if (require.cache[resolvedPath]) {
             delete require.cache[resolvedPath];
             cachedMiddlewares = null;

package/src/discovery/scan-routes.js

@@ -94,6 +94,19 @@ function scanDirSafely(dirPath, baseDir = dirPath, checkWarnings = false) {
   return results;
 }
 
+let isWebspressoPackageCache = null;
+function checkIsWebspressoPackage() {
+  if (isWebspressoPackageCache === null) {
+    try {
+      const pkg = JSON.parse(fs.readFileSync(path.join(process.cwd(), 'package.json'), 'utf8'));
+      isWebspressoPackageCache = Boolean(pkg && pkg.name === 'webspresso');
+    } catch (_) {
+      isWebspressoPackageCache = false;
+    }
+  }
+  return isWebspressoPackageCache;
+}
+
 /**
  * Scans configured pages, api, and module directories
  * @param {Object} options
@@ -106,10 +119,7 @@ function scanDirSafely(dirPath, baseDir = dirPath, checkWarnings = false) {
 function scanRoutes(options = {}) {
   const rootDir = options.rootDir || (() => {
     if (process.env.NODE_ENV === 'test') {
-      try {
-        const pkg = JSON.parse(fs.readFileSync(path.join(process.cwd(), 'package.json'), 'utf8'));
-        if (pkg.name === 'webspresso') return process.cwd();
-      } catch (_) {}
+      if (checkIsWebspressoPackage()) return process.cwd();
       return path.resolve(__dirname, '../..');
     }
     return process.cwd();

package/src/file-router.js

@@ -190,6 +190,10 @@ const translatorCache = new Map();
 
 function clearTranslatorCache() {
   translatorCache.clear();
+  intlNumberFormatCache.clear();
+  intlDateTimeFormatCache.clear();
+  intlRelativeTimeFormatCache.clear();
+  filePathToRouteCache.clear();
 }
 
 // Cache for route configs in production
@@ -215,6 +219,48 @@ const methodFromFilenameCache = new Map();
 // Cache for translation param replacement regexes
 const paramRegexCache = new Map();
 
+// Caches for file-to-route resolution and Intl formatters
+const filePathToRouteCache = new Map();
+const intlNumberFormatCache = new Map();
+const intlDateTimeFormatCache = new Map();
+const intlRelativeTimeFormatCache = new Map();
+
+function getCachedNumberFormat(locale, opts) {
+  const key = opts ? `${locale}::${JSON.stringify(opts)}` : locale;
+  let fmt = intlNumberFormatCache.get(key);
+  if (!fmt) {
+    fmt = new Intl.NumberFormat(locale, opts);
+    if (intlNumberFormatCache.size < 500) {
+      intlNumberFormatCache.set(key, fmt);
+    }
+  }
+  return fmt;
+}
+
+function getCachedDateTimeFormat(locale, opts) {
+  const key = opts ? `${locale}::${JSON.stringify(opts)}` : locale;
+  let fmt = intlDateTimeFormatCache.get(key);
+  if (!fmt) {
+    fmt = new Intl.DateTimeFormat(locale, opts);
+    if (intlDateTimeFormatCache.size < 500) {
+      intlDateTimeFormatCache.set(key, fmt);
+    }
+  }
+  return fmt;
+}
+
+function getCachedRelativeTimeFormat(locale, opts) {
+  const key = opts ? `${locale}::${JSON.stringify(opts)}` : locale;
+  let fmt = intlRelativeTimeFormatCache.get(key);
+  if (!fmt) {
+    fmt = new Intl.RelativeTimeFormat(locale, opts);
+    if (intlRelativeTimeFormatCache.size < 500) {
+      intlRelativeTimeFormatCache.set(key, fmt);
+    }
+  }
+  return fmt;
+}
+
 const MAX_LOCALE_LEN = 16;
 
 let cachedSupportedLocalesRaw = null;
@@ -404,6 +450,12 @@ function isApiRouteFile(file) {
  * @returns {string} Express route pattern
  */
 function filePathToRoute(filePath, ext) {
+  const cacheKey = filePath + '::' + ext;
+  const cached = filePathToRouteCache.get(cacheKey);
+  if (cached !== undefined) {
+    return cached;
+  }
+
   // Remove extension
   let route = filePath.replace(ext, '');
   
@@ -429,6 +481,10 @@ function filePathToRoute(filePath, ext) {
     route = '/' + route;
   }
   
+  if (filePathToRouteCache.size < 5000) {
+    filePathToRouteCache.set(cacheKey, route);
+  }
+
   return route;
 }
 
@@ -886,8 +942,8 @@ function createTranslator(translations = {}, options = EMPTY_OBJECT) {
       return value;
     }
 
-    // Replace params like {{name}} in the translation
-    if (interpolationParams && interpolationParams !== EMPTY_OBJECT && Object.keys(interpolationParams).length > 0) {
+    // Replace params like {{name}} in the translation (skip if no placeholders in template)
+    if (value.includes('{{') && interpolationParams && interpolationParams !== EMPTY_OBJECT && Object.keys(interpolationParams).length > 0) {
       for (const [paramKey, paramValue] of Object.entries(interpolationParams)) {
         let regex = paramRegexCache.get(paramKey);
         if (regex === undefined) {
@@ -911,7 +967,7 @@ function createTranslator(translations = {}, options = EMPTY_OBJECT) {
   t.number = function formatNumber(num, numOpts) {
     if (num === null || num === undefined || Number.isNaN(Number(num))) return '';
     try {
-      return new Intl.NumberFormat(locale, numOpts).format(Number(num));
+      return getCachedNumberFormat(locale, numOpts).format(Number(num));
     } catch {
       return String(num);
     }
@@ -921,7 +977,7 @@ function createTranslator(translations = {}, options = EMPTY_OBJECT) {
   t.currency = function formatCurrency(amount, currency = 'USD', currOpts = {}) {
     if (amount === null || amount === undefined || Number.isNaN(Number(amount))) return '';
     try {
-      return new Intl.NumberFormat(locale, { style: 'currency', currency, ...currOpts }).format(Number(amount));
+      return getCachedNumberFormat(locale, { style: 'currency', currency, ...currOpts }).format(Number(amount));
     } catch {
       return `${amount} ${currency}`;
     }
@@ -933,7 +989,7 @@ function createTranslator(translations = {}, options = EMPTY_OBJECT) {
     try {
       const d = date instanceof Date ? date : new Date(date);
       if (Number.isNaN(d.getTime())) return '';
-      return new Intl.DateTimeFormat(locale, dateOpts).format(d);
+      return getCachedDateTimeFormat(locale, dateOpts).format(d);
     } catch {
       return String(date);
     }
@@ -943,7 +999,7 @@ function createTranslator(translations = {}, options = EMPTY_OBJECT) {
   t.relativeTime = function formatRelativeTime(val, unit = 'day', relOpts = { numeric: 'auto' }) {
     if (val === null || val === undefined || Number.isNaN(Number(val))) return '';
     try {
-      return new Intl.RelativeTimeFormat(locale, relOpts).format(Number(val), unit);
+      return getCachedRelativeTimeFormat(locale, relOpts).format(Number(val), unit);
     } catch {
       return `${val} ${unit}`;
     }